; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=knl | FileCheck %s -check-prefixes=CHECK,AVX,AVX512,AVX512F ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s -check-prefixes=CHECK,AVX,AVX512,AVX512VL ; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 -mattr=+avx512vbmi2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX512,AVX512VBMI define i128 @bext_i128(i128 %a0, i128 %idx, i128 %len) nounwind { ; SSE-LABEL: bext_i128: ; SSE: # %bb.0: ; SSE-NEXT: movq %r8, %rcx ; SSE-NEXT: movq %rdx, %r8 ; SSE-NEXT: movl $1, %eax ; SSE-NEXT: xorl %r9d, %r9d ; SSE-NEXT: shldq %cl, %rax, %r9 ; SSE-NEXT: xorl %edx, %edx ; SSE-NEXT: shlq %cl, %rax ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rax, %r9 ; SSE-NEXT: cmovneq %rdx, %rax ; SSE-NEXT: addq $-1, %rax ; SSE-NEXT: adcq $-1, %r9 ; SSE-NEXT: movl %r8d, %ecx ; SSE-NEXT: shrdq %cl, %rsi, %rdi ; SSE-NEXT: shrq %cl, %rsi ; SSE-NEXT: testb $64, %r8b ; SSE-NEXT: cmovneq %rsi, %rdi ; SSE-NEXT: cmoveq %rsi, %rdx ; SSE-NEXT: andq %rdi, %rax ; SSE-NEXT: andq %r9, %rdx ; SSE-NEXT: retq ; ; AVX2-LABEL: bext_i128: ; AVX2: # %bb.0: ; AVX2-NEXT: movq %r8, %rcx ; AVX2-NEXT: movl $1, %eax ; AVX2-NEXT: xorl %r8d, %r8d ; AVX2-NEXT: shldq %cl, %rax, %r8 ; AVX2-NEXT: xorl %r9d, %r9d ; AVX2-NEXT: shlxq %rcx, %rax, %rax ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rax, %r8 ; AVX2-NEXT: cmovneq %r9, %rax ; AVX2-NEXT: movq %rdx, %rcx ; AVX2-NEXT: addq $-1, %rax ; AVX2-NEXT: adcq $-1, %r8 ; AVX2-NEXT: shrdq %cl, %rsi, %rdi ; AVX2-NEXT: shrxq %rdx, %rsi, %rdx ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rdx, %rdi ; AVX2-NEXT: cmovneq %r9, %rdx ; AVX2-NEXT: andq %rdi, %rax ; AVX2-NEXT: andq %r8, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bext_i128: ; AVX512F: # %bb.0: ; AVX512F-NEXT: movq %r8, %rcx ; AVX512F-NEXT: movl $1, %eax ; AVX512F-NEXT: xorl %r8d, %r8d ; AVX512F-NEXT: shldq %cl, %rax, %r8 ; AVX512F-NEXT: xorl %r9d, %r9d ; AVX512F-NEXT: shlxq %rcx, %rax, %rax ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rax, %r8 ; AVX512F-NEXT: cmovneq %r9, %rax ; AVX512F-NEXT: movq %rdx, %rcx ; AVX512F-NEXT: addq $-1, %rax ; AVX512F-NEXT: adcq $-1, %r8 ; AVX512F-NEXT: shrdq %cl, %rsi, %rdi ; AVX512F-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rdx, %rdi ; AVX512F-NEXT: cmovneq %r9, %rdx ; AVX512F-NEXT: andq %rdi, %rax ; AVX512F-NEXT: andq %r8, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bext_i128: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: movq %r8, %rcx ; AVX512VL-NEXT: movl $1, %eax ; AVX512VL-NEXT: xorl %r9d, %r9d ; AVX512VL-NEXT: shldq %cl, %rax, %r9 ; AVX512VL-NEXT: movq %rdx, %r8 ; AVX512VL-NEXT: xorl %r10d, %r10d ; AVX512VL-NEXT: shlxq %rcx, %rax, %rax ; AVX512VL-NEXT: testb $64, %cl ; AVX512VL-NEXT: cmovneq %rax, %r9 ; AVX512VL-NEXT: cmovneq %r10, %rax ; AVX512VL-NEXT: addq $-1, %rax ; AVX512VL-NEXT: adcq $-1, %r9 ; AVX512VL-NEXT: movl %r8d, %ecx ; AVX512VL-NEXT: shrdq %cl, %rsi, %rdi ; AVX512VL-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512VL-NEXT: testb $64, %r8b ; AVX512VL-NEXT: cmovneq %rdx, %rdi ; AVX512VL-NEXT: cmovneq %r10, %rdx ; AVX512VL-NEXT: andq %rdi, %rax ; AVX512VL-NEXT: andq %r9, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bext_i128: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: movq %r8, %rcx ; AVX512VBMI-NEXT: movl $1, %eax ; AVX512VBMI-NEXT: xorl %r9d, %r9d ; AVX512VBMI-NEXT: shldq %cl, %rax, %r9 ; AVX512VBMI-NEXT: movq %rdx, %r8 ; AVX512VBMI-NEXT: xorl %r10d, %r10d ; AVX512VBMI-NEXT: shlxq %rcx, %rax, %rax ; AVX512VBMI-NEXT: testb $64, %cl ; AVX512VBMI-NEXT: cmovneq %rax, %r9 ; AVX512VBMI-NEXT: cmovneq %r10, %rax ; AVX512VBMI-NEXT: addq $-1, %rax ; AVX512VBMI-NEXT: adcq $-1, %r9 ; AVX512VBMI-NEXT: movl %r8d, %ecx ; AVX512VBMI-NEXT: shrdq %cl, %rsi, %rdi ; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512VBMI-NEXT: testb $64, %r8b ; AVX512VBMI-NEXT: cmovneq %rdx, %rdi ; AVX512VBMI-NEXT: cmovneq %r10, %rdx ; AVX512VBMI-NEXT: andq %rdi, %rax ; AVX512VBMI-NEXT: andq %r9, %rdx ; AVX512VBMI-NEXT: retq %bit = shl i128 1, %len %msk = sub i128 %bit, 1 %shift = lshr i128 %a0, %idx %res = and i128 %shift, %msk ret i128 %res } define i128 @bext_i128_vector(<2 x i64> %v0, i128 %idx, i128 %len) nounwind { ; SSE2-LABEL: bext_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: movq %rdx, %rcx ; SSE2-NEXT: movq %xmm0, %rax ; SSE2-NEXT: movl $1, %r8d ; SSE2-NEXT: xorl %esi, %esi ; SSE2-NEXT: shldq %cl, %r8, %rsi ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm0, %rdx ; SSE2-NEXT: xorl %r9d, %r9d ; SSE2-NEXT: shlq %cl, %r8 ; SSE2-NEXT: testb $64, %cl ; SSE2-NEXT: cmovneq %r8, %rsi ; SSE2-NEXT: cmovneq %r9, %r8 ; SSE2-NEXT: addq $-1, %r8 ; SSE2-NEXT: adcq $-1, %rsi ; SSE2-NEXT: movl %edi, %ecx ; SSE2-NEXT: shrdq %cl, %rdx, %rax ; SSE2-NEXT: shrq %cl, %rdx ; SSE2-NEXT: testb $64, %dil ; SSE2-NEXT: cmovneq %rdx, %rax ; SSE2-NEXT: cmovneq %r9, %rdx ; SSE2-NEXT: andq %r8, %rax ; SSE2-NEXT: andq %rsi, %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: bext_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: pextrq $1, %xmm0, %rsi ; SSE42-NEXT: movq %rdx, %rcx ; SSE42-NEXT: movl $1, %r8d ; SSE42-NEXT: xorl %edx, %edx ; SSE42-NEXT: shldq %cl, %r8, %rdx ; SSE42-NEXT: movq %xmm0, %rax ; SSE42-NEXT: xorl %r9d, %r9d ; SSE42-NEXT: shlq %cl, %r8 ; SSE42-NEXT: testb $64, %cl ; SSE42-NEXT: cmovneq %r8, %rdx ; SSE42-NEXT: cmovneq %r9, %r8 ; SSE42-NEXT: addq $-1, %r8 ; SSE42-NEXT: adcq $-1, %rdx ; SSE42-NEXT: movl %edi, %ecx ; SSE42-NEXT: shrdq %cl, %rsi, %rax ; SSE42-NEXT: shrq %cl, %rsi ; SSE42-NEXT: testb $64, %dil ; SSE42-NEXT: cmovneq %rsi, %rax ; SSE42-NEXT: cmovneq %r9, %rsi ; SSE42-NEXT: andq %r8, %rax ; SSE42-NEXT: andq %rdx, %rsi ; SSE42-NEXT: movq %rsi, %rdx ; SSE42-NEXT: retq ; ; AVX2-LABEL: bext_i128_vector: ; AVX2: # %bb.0: ; AVX2-NEXT: movq %rdx, %rcx ; AVX2-NEXT: vpextrq $1, %xmm0, %rdx ; AVX2-NEXT: movl $1, %eax ; AVX2-NEXT: xorl %esi, %esi ; AVX2-NEXT: shldq %cl, %rax, %rsi ; AVX2-NEXT: xorl %r8d, %r8d ; AVX2-NEXT: shlxq %rcx, %rax, %r9 ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %r9, %rsi ; AVX2-NEXT: cmovneq %r8, %r9 ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: addq $-1, %r9 ; AVX2-NEXT: adcq $-1, %rsi ; AVX2-NEXT: movl %edi, %ecx ; AVX2-NEXT: shrdq %cl, %rdx, %rax ; AVX2-NEXT: shrxq %rdi, %rdx, %rdx ; AVX2-NEXT: testb $64, %dil ; AVX2-NEXT: cmovneq %rdx, %rax ; AVX2-NEXT: cmovneq %r8, %rdx ; AVX2-NEXT: andq %r9, %rax ; AVX2-NEXT: andq %rsi, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bext_i128_vector: ; AVX512F: # %bb.0: ; AVX512F-NEXT: movq %rdx, %rcx ; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512F-NEXT: movl $1, %eax ; AVX512F-NEXT: xorl %esi, %esi ; AVX512F-NEXT: shldq %cl, %rax, %rsi ; AVX512F-NEXT: xorl %r8d, %r8d ; AVX512F-NEXT: shlxq %rcx, %rax, %r9 ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %r9, %rsi ; AVX512F-NEXT: cmovneq %r8, %r9 ; AVX512F-NEXT: vmovq %xmm0, %rax ; AVX512F-NEXT: addq $-1, %r9 ; AVX512F-NEXT: adcq $-1, %rsi ; AVX512F-NEXT: movl %edi, %ecx ; AVX512F-NEXT: shrdq %cl, %rdx, %rax ; AVX512F-NEXT: shrxq %rdi, %rdx, %rdx ; AVX512F-NEXT: testb $64, %dil ; AVX512F-NEXT: cmovneq %rdx, %rax ; AVX512F-NEXT: cmovneq %r8, %rdx ; AVX512F-NEXT: andq %r9, %rax ; AVX512F-NEXT: andq %rsi, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bext_i128_vector: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: movq %rdx, %rcx ; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VL-NEXT: movl $1, %esi ; AVX512VL-NEXT: xorl %r8d, %r8d ; AVX512VL-NEXT: shldq %cl, %rsi, %r8 ; AVX512VL-NEXT: vmovq %xmm0, %rax ; AVX512VL-NEXT: xorl %r9d, %r9d ; AVX512VL-NEXT: shlxq %rcx, %rsi, %rsi ; AVX512VL-NEXT: testb $64, %cl ; AVX512VL-NEXT: cmovneq %rsi, %r8 ; AVX512VL-NEXT: cmovneq %r9, %rsi ; AVX512VL-NEXT: addq $-1, %rsi ; AVX512VL-NEXT: adcq $-1, %r8 ; AVX512VL-NEXT: movl %edi, %ecx ; AVX512VL-NEXT: shrdq %cl, %rdx, %rax ; AVX512VL-NEXT: shrxq %rdi, %rdx, %rdx ; AVX512VL-NEXT: testb $64, %dil ; AVX512VL-NEXT: cmovneq %rdx, %rax ; AVX512VL-NEXT: cmovneq %r9, %rdx ; AVX512VL-NEXT: andq %rsi, %rax ; AVX512VL-NEXT: andq %r8, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bext_i128_vector: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: movq %rdx, %rcx ; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VBMI-NEXT: movl $1, %esi ; AVX512VBMI-NEXT: xorl %r8d, %r8d ; AVX512VBMI-NEXT: shldq %cl, %rsi, %r8 ; AVX512VBMI-NEXT: vmovq %xmm0, %rax ; AVX512VBMI-NEXT: xorl %r9d, %r9d ; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rsi ; AVX512VBMI-NEXT: testb $64, %cl ; AVX512VBMI-NEXT: cmovneq %rsi, %r8 ; AVX512VBMI-NEXT: cmovneq %r9, %rsi ; AVX512VBMI-NEXT: addq $-1, %rsi ; AVX512VBMI-NEXT: adcq $-1, %r8 ; AVX512VBMI-NEXT: movl %edi, %ecx ; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rax ; AVX512VBMI-NEXT: shrxq %rdi, %rdx, %rdx ; AVX512VBMI-NEXT: testb $64, %dil ; AVX512VBMI-NEXT: cmovneq %rdx, %rax ; AVX512VBMI-NEXT: cmovneq %r9, %rdx ; AVX512VBMI-NEXT: andq %rsi, %rax ; AVX512VBMI-NEXT: andq %r8, %rdx ; AVX512VBMI-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %bit = shl i128 1, %len %msk = sub i128 %bit, 1 %shift = lshr i128 %a0, %idx %res = and i128 %shift, %msk ret i128 %res } define i128 @bext_i128_load(i128 %a0, i128 %idx, i128 %len) nounwind { ; SSE-LABEL: bext_i128_load: ; SSE: # %bb.0: ; SSE-NEXT: movq %r8, %rcx ; SSE-NEXT: movq %rdx, %r8 ; SSE-NEXT: movl $1, %eax ; SSE-NEXT: xorl %r9d, %r9d ; SSE-NEXT: shldq %cl, %rax, %r9 ; SSE-NEXT: xorl %edx, %edx ; SSE-NEXT: shlq %cl, %rax ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rax, %r9 ; SSE-NEXT: cmovneq %rdx, %rax ; SSE-NEXT: addq $-1, %rax ; SSE-NEXT: adcq $-1, %r9 ; SSE-NEXT: movl %r8d, %ecx ; SSE-NEXT: shrdq %cl, %rsi, %rdi ; SSE-NEXT: shrq %cl, %rsi ; SSE-NEXT: testb $64, %r8b ; SSE-NEXT: cmovneq %rsi, %rdi ; SSE-NEXT: cmoveq %rsi, %rdx ; SSE-NEXT: andq %rdi, %rax ; SSE-NEXT: andq %r9, %rdx ; SSE-NEXT: retq ; ; AVX2-LABEL: bext_i128_load: ; AVX2: # %bb.0: ; AVX2-NEXT: movq %r8, %rcx ; AVX2-NEXT: movl $1, %eax ; AVX2-NEXT: xorl %r8d, %r8d ; AVX2-NEXT: shldq %cl, %rax, %r8 ; AVX2-NEXT: xorl %r9d, %r9d ; AVX2-NEXT: shlxq %rcx, %rax, %rax ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rax, %r8 ; AVX2-NEXT: cmovneq %r9, %rax ; AVX2-NEXT: movq %rdx, %rcx ; AVX2-NEXT: addq $-1, %rax ; AVX2-NEXT: adcq $-1, %r8 ; AVX2-NEXT: shrdq %cl, %rsi, %rdi ; AVX2-NEXT: shrxq %rdx, %rsi, %rdx ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rdx, %rdi ; AVX2-NEXT: cmovneq %r9, %rdx ; AVX2-NEXT: andq %rdi, %rax ; AVX2-NEXT: andq %r8, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bext_i128_load: ; AVX512F: # %bb.0: ; AVX512F-NEXT: movq %r8, %rcx ; AVX512F-NEXT: movl $1, %eax ; AVX512F-NEXT: xorl %r8d, %r8d ; AVX512F-NEXT: shldq %cl, %rax, %r8 ; AVX512F-NEXT: xorl %r9d, %r9d ; AVX512F-NEXT: shlxq %rcx, %rax, %rax ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rax, %r8 ; AVX512F-NEXT: cmovneq %r9, %rax ; AVX512F-NEXT: movq %rdx, %rcx ; AVX512F-NEXT: addq $-1, %rax ; AVX512F-NEXT: adcq $-1, %r8 ; AVX512F-NEXT: shrdq %cl, %rsi, %rdi ; AVX512F-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rdx, %rdi ; AVX512F-NEXT: cmovneq %r9, %rdx ; AVX512F-NEXT: andq %rdi, %rax ; AVX512F-NEXT: andq %r8, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bext_i128_load: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: movq %r8, %rcx ; AVX512VL-NEXT: movl $1, %eax ; AVX512VL-NEXT: xorl %r9d, %r9d ; AVX512VL-NEXT: shldq %cl, %rax, %r9 ; AVX512VL-NEXT: movq %rdx, %r8 ; AVX512VL-NEXT: xorl %r10d, %r10d ; AVX512VL-NEXT: shlxq %rcx, %rax, %rax ; AVX512VL-NEXT: testb $64, %cl ; AVX512VL-NEXT: cmovneq %rax, %r9 ; AVX512VL-NEXT: cmovneq %r10, %rax ; AVX512VL-NEXT: addq $-1, %rax ; AVX512VL-NEXT: adcq $-1, %r9 ; AVX512VL-NEXT: movl %r8d, %ecx ; AVX512VL-NEXT: shrdq %cl, %rsi, %rdi ; AVX512VL-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512VL-NEXT: testb $64, %r8b ; AVX512VL-NEXT: cmovneq %rdx, %rdi ; AVX512VL-NEXT: cmovneq %r10, %rdx ; AVX512VL-NEXT: andq %rdi, %rax ; AVX512VL-NEXT: andq %r9, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bext_i128_load: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: movq %r8, %rcx ; AVX512VBMI-NEXT: movl $1, %eax ; AVX512VBMI-NEXT: xorl %r9d, %r9d ; AVX512VBMI-NEXT: shldq %cl, %rax, %r9 ; AVX512VBMI-NEXT: movq %rdx, %r8 ; AVX512VBMI-NEXT: xorl %r10d, %r10d ; AVX512VBMI-NEXT: shlxq %rcx, %rax, %rax ; AVX512VBMI-NEXT: testb $64, %cl ; AVX512VBMI-NEXT: cmovneq %rax, %r9 ; AVX512VBMI-NEXT: cmovneq %r10, %rax ; AVX512VBMI-NEXT: addq $-1, %rax ; AVX512VBMI-NEXT: adcq $-1, %r9 ; AVX512VBMI-NEXT: movl %r8d, %ecx ; AVX512VBMI-NEXT: shrdq %cl, %rsi, %rdi ; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rdx ; AVX512VBMI-NEXT: testb $64, %r8b ; AVX512VBMI-NEXT: cmovneq %rdx, %rdi ; AVX512VBMI-NEXT: cmovneq %r10, %rdx ; AVX512VBMI-NEXT: andq %rdi, %rax ; AVX512VBMI-NEXT: andq %r9, %rdx ; AVX512VBMI-NEXT: retq %bit = shl i128 1, %len %msk = sub i128 %bit, 1 %shift = lshr i128 %a0, %idx %res = and i128 %shift, %msk ret i128 %res } define i128 @blsi_i128(i128 %a0) nounwind { ; CHECK-LABEL: blsi_i128: ; CHECK: # %bb.0: ; CHECK-NEXT: xorl %edx, %edx ; CHECK-NEXT: movq %rdi, %rax ; CHECK-NEXT: negq %rax ; CHECK-NEXT: sbbq %rsi, %rdx ; CHECK-NEXT: andq %rsi, %rdx ; CHECK-NEXT: andq %rdi, %rax ; CHECK-NEXT: retq %neg = sub i128 0, %a0 %res = and i128 %a0, %neg ret i128 %res } define i128 @blsi_i128_vector(<2 x i64> %v0) nounwind { ; SSE2-LABEL: blsi_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm1, %rcx ; SSE2-NEXT: movq %xmm0, %rsi ; SSE2-NEXT: xorl %edx, %edx ; SSE2-NEXT: movq %rsi, %rax ; SSE2-NEXT: negq %rax ; SSE2-NEXT: sbbq %rcx, %rdx ; SSE2-NEXT: andq %rcx, %rdx ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: retq ; ; SSE42-LABEL: blsi_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: pextrq $1, %xmm0, %rcx ; SSE42-NEXT: movq %xmm0, %rsi ; SSE42-NEXT: xorl %edx, %edx ; SSE42-NEXT: movq %rsi, %rax ; SSE42-NEXT: negq %rax ; SSE42-NEXT: sbbq %rcx, %rdx ; SSE42-NEXT: andq %rcx, %rdx ; SSE42-NEXT: andq %rsi, %rax ; SSE42-NEXT: retq ; ; AVX-LABEL: blsi_i128_vector: ; AVX: # %bb.0: ; AVX-NEXT: vpextrq $1, %xmm0, %rcx ; AVX-NEXT: vmovq %xmm0, %rsi ; AVX-NEXT: xorl %edx, %edx ; AVX-NEXT: movq %rsi, %rax ; AVX-NEXT: negq %rax ; AVX-NEXT: sbbq %rcx, %rdx ; AVX-NEXT: andq %rcx, %rdx ; AVX-NEXT: andq %rsi, %rax ; AVX-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %neg = sub i128 0, %a0 %res = and i128 %a0, %neg ret i128 %res } define i128 @blsi_i128_load(ptr %p0) nounwind { ; CHECK-LABEL: blsi_i128_load: ; CHECK: # %bb.0: ; CHECK-NEXT: movq (%rdi), %rcx ; CHECK-NEXT: movq 8(%rdi), %rsi ; CHECK-NEXT: xorl %edx, %edx ; CHECK-NEXT: movq %rcx, %rax ; CHECK-NEXT: negq %rax ; CHECK-NEXT: sbbq %rsi, %rdx ; CHECK-NEXT: andq %rsi, %rdx ; CHECK-NEXT: andq %rcx, %rax ; CHECK-NEXT: retq %a0 = load i128, ptr %p0 %neg = sub i128 0, %a0 %res = and i128 %a0, %neg ret i128 %res } define i128 @blsmsk_i128(i128 %a0) nounwind { ; CHECK-LABEL: blsmsk_i128: ; CHECK: # %bb.0: ; CHECK-NEXT: movq %rdi, %rax ; CHECK-NEXT: addq $-1, %rax ; CHECK-NEXT: movq %rsi, %rdx ; CHECK-NEXT: adcq $-1, %rdx ; CHECK-NEXT: xorq %rdi, %rax ; CHECK-NEXT: xorq %rsi, %rdx ; CHECK-NEXT: retq %dec = sub i128 %a0, 1 %res = xor i128 %a0, %dec ret i128 %res } define i128 @blsmsk_i128_vector(<2 x i64> %v0) nounwind { ; SSE2-LABEL: blsmsk_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm1, %rcx ; SSE2-NEXT: movq %xmm0, %rsi ; SSE2-NEXT: movq %rsi, %rax ; SSE2-NEXT: addq $-1, %rax ; SSE2-NEXT: movq %rcx, %rdx ; SSE2-NEXT: adcq $-1, %rdx ; SSE2-NEXT: xorq %rsi, %rax ; SSE2-NEXT: xorq %rcx, %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: blsmsk_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: pextrq $1, %xmm0, %rcx ; SSE42-NEXT: movq %xmm0, %rsi ; SSE42-NEXT: movq %rsi, %rax ; SSE42-NEXT: addq $-1, %rax ; SSE42-NEXT: movq %rcx, %rdx ; SSE42-NEXT: adcq $-1, %rdx ; SSE42-NEXT: xorq %rsi, %rax ; SSE42-NEXT: xorq %rcx, %rdx ; SSE42-NEXT: retq ; ; AVX-LABEL: blsmsk_i128_vector: ; AVX: # %bb.0: ; AVX-NEXT: vpextrq $1, %xmm0, %rcx ; AVX-NEXT: vmovq %xmm0, %rsi ; AVX-NEXT: movq %rsi, %rax ; AVX-NEXT: addq $-1, %rax ; AVX-NEXT: movq %rcx, %rdx ; AVX-NEXT: adcq $-1, %rdx ; AVX-NEXT: xorq %rsi, %rax ; AVX-NEXT: xorq %rcx, %rdx ; AVX-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %dec = sub i128 %a0, 1 %res = xor i128 %a0, %dec ret i128 %res } define i128 @blsmsk_i128_load(ptr %p0) nounwind { ; CHECK-LABEL: blsmsk_i128_load: ; CHECK: # %bb.0: ; CHECK-NEXT: movq (%rdi), %rcx ; CHECK-NEXT: movq 8(%rdi), %rsi ; CHECK-NEXT: movq %rcx, %rax ; CHECK-NEXT: addq $-1, %rax ; CHECK-NEXT: movq %rsi, %rdx ; CHECK-NEXT: adcq $-1, %rdx ; CHECK-NEXT: xorq %rcx, %rax ; CHECK-NEXT: xorq %rsi, %rdx ; CHECK-NEXT: retq %a0 = load i128, ptr %p0 %dec = sub i128 %a0, 1 %res = xor i128 %a0, %dec ret i128 %res } define i128 @blsr_i128(i128 %a0) nounwind { ; CHECK-LABEL: blsr_i128: ; CHECK: # %bb.0: ; CHECK-NEXT: movq %rdi, %rax ; CHECK-NEXT: addq $-1, %rax ; CHECK-NEXT: movq %rsi, %rdx ; CHECK-NEXT: adcq $-1, %rdx ; CHECK-NEXT: andq %rdi, %rax ; CHECK-NEXT: andq %rsi, %rdx ; CHECK-NEXT: retq %dec = sub i128 %a0, 1 %res = and i128 %a0, %dec ret i128 %res } define i128 @blsr_i128_vector(<2 x i64> %v0) nounwind { ; SSE2-LABEL: blsr_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm1, %rcx ; SSE2-NEXT: movq %xmm0, %rsi ; SSE2-NEXT: movq %rsi, %rax ; SSE2-NEXT: addq $-1, %rax ; SSE2-NEXT: movq %rcx, %rdx ; SSE2-NEXT: adcq $-1, %rdx ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: andq %rcx, %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: blsr_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: pextrq $1, %xmm0, %rcx ; SSE42-NEXT: movq %xmm0, %rsi ; SSE42-NEXT: movq %rsi, %rax ; SSE42-NEXT: addq $-1, %rax ; SSE42-NEXT: movq %rcx, %rdx ; SSE42-NEXT: adcq $-1, %rdx ; SSE42-NEXT: andq %rsi, %rax ; SSE42-NEXT: andq %rcx, %rdx ; SSE42-NEXT: retq ; ; AVX-LABEL: blsr_i128_vector: ; AVX: # %bb.0: ; AVX-NEXT: vpextrq $1, %xmm0, %rcx ; AVX-NEXT: vmovq %xmm0, %rsi ; AVX-NEXT: movq %rsi, %rax ; AVX-NEXT: addq $-1, %rax ; AVX-NEXT: movq %rcx, %rdx ; AVX-NEXT: adcq $-1, %rdx ; AVX-NEXT: andq %rsi, %rax ; AVX-NEXT: andq %rcx, %rdx ; AVX-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %dec = sub i128 %a0, 1 %res = and i128 %a0, %dec ret i128 %res } define i128 @blsr_i128_load(ptr %p0) nounwind { ; CHECK-LABEL: blsr_i128_load: ; CHECK: # %bb.0: ; CHECK-NEXT: movq (%rdi), %rcx ; CHECK-NEXT: movq 8(%rdi), %rsi ; CHECK-NEXT: movq %rcx, %rax ; CHECK-NEXT: addq $-1, %rax ; CHECK-NEXT: movq %rsi, %rdx ; CHECK-NEXT: adcq $-1, %rdx ; CHECK-NEXT: andq %rcx, %rax ; CHECK-NEXT: andq %rsi, %rdx ; CHECK-NEXT: retq %a0 = load i128, ptr %p0 %dec = sub i128 %a0, 1 %res = and i128 %a0, %dec ret i128 %res } define i128 @bzhi_i128(i128 %a0, i128 %idx) nounwind { ; SSE-LABEL: bzhi_i128: ; SSE: # %bb.0: ; SSE-NEXT: movq %rdx, %rcx ; SSE-NEXT: movl $1, %eax ; SSE-NEXT: xorl %edx, %edx ; SSE-NEXT: shldq %cl, %rax, %rdx ; SSE-NEXT: xorl %r8d, %r8d ; SSE-NEXT: shlq %cl, %rax ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rax, %rdx ; SSE-NEXT: cmovneq %r8, %rax ; SSE-NEXT: addq $-1, %rax ; SSE-NEXT: adcq $-1, %rdx ; SSE-NEXT: andq %rsi, %rdx ; SSE-NEXT: andq %rdi, %rax ; SSE-NEXT: retq ; ; AVX2-LABEL: bzhi_i128: ; AVX2: # %bb.0: ; AVX2-NEXT: movq %rdx, %rcx ; AVX2-NEXT: movl $1, %eax ; AVX2-NEXT: xorl %edx, %edx ; AVX2-NEXT: shldq %cl, %rax, %rdx ; AVX2-NEXT: xorl %r8d, %r8d ; AVX2-NEXT: shlxq %rcx, %rax, %rax ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rax, %rdx ; AVX2-NEXT: cmovneq %r8, %rax ; AVX2-NEXT: addq $-1, %rax ; AVX2-NEXT: adcq $-1, %rdx ; AVX2-NEXT: andq %rsi, %rdx ; AVX2-NEXT: andq %rdi, %rax ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bzhi_i128: ; AVX512F: # %bb.0: ; AVX512F-NEXT: movq %rdx, %rcx ; AVX512F-NEXT: movl $1, %eax ; AVX512F-NEXT: xorl %edx, %edx ; AVX512F-NEXT: shldq %cl, %rax, %rdx ; AVX512F-NEXT: xorl %r8d, %r8d ; AVX512F-NEXT: shlxq %rcx, %rax, %rax ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rax, %rdx ; AVX512F-NEXT: cmovneq %r8, %rax ; AVX512F-NEXT: addq $-1, %rax ; AVX512F-NEXT: adcq $-1, %rdx ; AVX512F-NEXT: andq %rsi, %rdx ; AVX512F-NEXT: andq %rdi, %rax ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bzhi_i128: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: movq %rdx, %rcx ; AVX512VL-NEXT: xorl %r8d, %r8d ; AVX512VL-NEXT: movl $1, %eax ; AVX512VL-NEXT: xorl %edx, %edx ; AVX512VL-NEXT: shldq %cl, %rax, %rdx ; AVX512VL-NEXT: shlxq %rcx, %rax, %rax ; AVX512VL-NEXT: testb $64, %cl ; AVX512VL-NEXT: cmovneq %rax, %rdx ; AVX512VL-NEXT: cmovneq %r8, %rax ; AVX512VL-NEXT: addq $-1, %rax ; AVX512VL-NEXT: adcq $-1, %rdx ; AVX512VL-NEXT: andq %rsi, %rdx ; AVX512VL-NEXT: andq %rdi, %rax ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bzhi_i128: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: movq %rdx, %rcx ; AVX512VBMI-NEXT: xorl %r8d, %r8d ; AVX512VBMI-NEXT: movl $1, %eax ; AVX512VBMI-NEXT: xorl %edx, %edx ; AVX512VBMI-NEXT: shldq %cl, %rax, %rdx ; AVX512VBMI-NEXT: shlxq %rcx, %rax, %rax ; AVX512VBMI-NEXT: testb $64, %cl ; AVX512VBMI-NEXT: cmovneq %rax, %rdx ; AVX512VBMI-NEXT: cmovneq %r8, %rax ; AVX512VBMI-NEXT: addq $-1, %rax ; AVX512VBMI-NEXT: adcq $-1, %rdx ; AVX512VBMI-NEXT: andq %rsi, %rdx ; AVX512VBMI-NEXT: andq %rdi, %rax ; AVX512VBMI-NEXT: retq %bit = shl i128 1, %idx %msk = sub i128 %bit, 1 %res = and i128 %a0, %msk ret i128 %res } define i128 @bzhi_i128_vector(<2 x i64> %v0, i128 %idx) nounwind { ; SSE2-LABEL: bzhi_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: movq %rdi, %rcx ; SSE2-NEXT: movl $1, %eax ; SSE2-NEXT: xorl %edx, %edx ; SSE2-NEXT: shldq %cl, %rax, %rdx ; SSE2-NEXT: movq %xmm0, %rsi ; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] ; SSE2-NEXT: xorl %edi, %edi ; SSE2-NEXT: shlq %cl, %rax ; SSE2-NEXT: testb $64, %cl ; SSE2-NEXT: cmovneq %rax, %rdx ; SSE2-NEXT: cmovneq %rdi, %rax ; SSE2-NEXT: movq %xmm0, %rcx ; SSE2-NEXT: addq $-1, %rax ; SSE2-NEXT: adcq $-1, %rdx ; SSE2-NEXT: andq %rcx, %rdx ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: retq ; ; SSE42-LABEL: bzhi_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: movq %rdi, %rcx ; SSE42-NEXT: movl $1, %eax ; SSE42-NEXT: xorl %edx, %edx ; SSE42-NEXT: shldq %cl, %rax, %rdx ; SSE42-NEXT: pextrq $1, %xmm0, %rsi ; SSE42-NEXT: xorl %edi, %edi ; SSE42-NEXT: shlq %cl, %rax ; SSE42-NEXT: testb $64, %cl ; SSE42-NEXT: cmovneq %rax, %rdx ; SSE42-NEXT: cmovneq %rdi, %rax ; SSE42-NEXT: movq %xmm0, %rcx ; SSE42-NEXT: addq $-1, %rax ; SSE42-NEXT: adcq $-1, %rdx ; SSE42-NEXT: andq %rsi, %rdx ; SSE42-NEXT: andq %rcx, %rax ; SSE42-NEXT: retq ; ; AVX2-LABEL: bzhi_i128_vector: ; AVX2: # %bb.0: ; AVX2-NEXT: vpextrq $1, %xmm0, %rsi ; AVX2-NEXT: movq %rdi, %rcx ; AVX2-NEXT: xorl %edi, %edi ; AVX2-NEXT: movl $1, %eax ; AVX2-NEXT: xorl %edx, %edx ; AVX2-NEXT: shldq %cl, %rax, %rdx ; AVX2-NEXT: shlxq %rcx, %rax, %rax ; AVX2-NEXT: testb $64, %cl ; AVX2-NEXT: cmovneq %rax, %rdx ; AVX2-NEXT: vmovq %xmm0, %rcx ; AVX2-NEXT: cmovneq %rdi, %rax ; AVX2-NEXT: addq $-1, %rax ; AVX2-NEXT: adcq $-1, %rdx ; AVX2-NEXT: andq %rsi, %rdx ; AVX2-NEXT: andq %rcx, %rax ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bzhi_i128_vector: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpextrq $1, %xmm0, %rsi ; AVX512F-NEXT: movq %rdi, %rcx ; AVX512F-NEXT: xorl %edi, %edi ; AVX512F-NEXT: movl $1, %eax ; AVX512F-NEXT: xorl %edx, %edx ; AVX512F-NEXT: shldq %cl, %rax, %rdx ; AVX512F-NEXT: shlxq %rcx, %rax, %rax ; AVX512F-NEXT: testb $64, %cl ; AVX512F-NEXT: cmovneq %rax, %rdx ; AVX512F-NEXT: vmovq %xmm0, %rcx ; AVX512F-NEXT: cmovneq %rdi, %rax ; AVX512F-NEXT: addq $-1, %rax ; AVX512F-NEXT: adcq $-1, %rdx ; AVX512F-NEXT: andq %rsi, %rdx ; AVX512F-NEXT: andq %rcx, %rax ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bzhi_i128_vector: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: movq %rdi, %rcx ; AVX512VL-NEXT: vmovq %xmm0, %rsi ; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdi ; AVX512VL-NEXT: xorl %r8d, %r8d ; AVX512VL-NEXT: movl $1, %eax ; AVX512VL-NEXT: xorl %edx, %edx ; AVX512VL-NEXT: shldq %cl, %rax, %rdx ; AVX512VL-NEXT: shlxq %rcx, %rax, %rax ; AVX512VL-NEXT: testb $64, %cl ; AVX512VL-NEXT: cmovneq %rax, %rdx ; AVX512VL-NEXT: cmovneq %r8, %rax ; AVX512VL-NEXT: addq $-1, %rax ; AVX512VL-NEXT: adcq $-1, %rdx ; AVX512VL-NEXT: andq %rdi, %rdx ; AVX512VL-NEXT: andq %rsi, %rax ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bzhi_i128_vector: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: movq %rdi, %rcx ; AVX512VBMI-NEXT: vmovq %xmm0, %rsi ; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdi ; AVX512VBMI-NEXT: xorl %r8d, %r8d ; AVX512VBMI-NEXT: movl $1, %eax ; AVX512VBMI-NEXT: xorl %edx, %edx ; AVX512VBMI-NEXT: shldq %cl, %rax, %rdx ; AVX512VBMI-NEXT: shlxq %rcx, %rax, %rax ; AVX512VBMI-NEXT: testb $64, %cl ; AVX512VBMI-NEXT: cmovneq %rax, %rdx ; AVX512VBMI-NEXT: cmovneq %r8, %rax ; AVX512VBMI-NEXT: addq $-1, %rax ; AVX512VBMI-NEXT: adcq $-1, %rdx ; AVX512VBMI-NEXT: andq %rdi, %rdx ; AVX512VBMI-NEXT: andq %rsi, %rax ; AVX512VBMI-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %bit = shl i128 1, %idx %msk = sub i128 %bit, 1 %res = and i128 %a0, %msk ret i128 %res } define i128 @bzhi_i128_load(ptr %p0, i128 %idx) nounwind { ; SSE-LABEL: bzhi_i128_load: ; SSE: # %bb.0: ; SSE-NEXT: movq %rsi, %rcx ; SSE-NEXT: movl $1, %eax ; SSE-NEXT: xorl %edx, %edx ; SSE-NEXT: shldq %cl, %rax, %rdx ; SSE-NEXT: shlq %cl, %rax ; SSE-NEXT: xorl %esi, %esi ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rax, %rdx ; SSE-NEXT: cmovneq %rsi, %rax ; SSE-NEXT: addq $-1, %rax ; SSE-NEXT: adcq $-1, %rdx ; SSE-NEXT: andq 8(%rdi), %rdx ; SSE-NEXT: andq (%rdi), %rax ; SSE-NEXT: retq ; ; AVX-LABEL: bzhi_i128_load: ; AVX: # %bb.0: ; AVX-NEXT: movq %rsi, %rcx ; AVX-NEXT: movl $1, %eax ; AVX-NEXT: xorl %edx, %edx ; AVX-NEXT: shldq %cl, %rax, %rdx ; AVX-NEXT: xorl %esi, %esi ; AVX-NEXT: shlxq %rcx, %rax, %rax ; AVX-NEXT: testb $64, %cl ; AVX-NEXT: cmovneq %rax, %rdx ; AVX-NEXT: cmovneq %rsi, %rax ; AVX-NEXT: addq $-1, %rax ; AVX-NEXT: adcq $-1, %rdx ; AVX-NEXT: andq 8(%rdi), %rdx ; AVX-NEXT: andq (%rdi), %rax ; AVX-NEXT: retq %a0 = load i128, ptr %p0 %bit = shl i128 1, %idx %msk = sub i128 %bit, 1 %res = and i128 %a0, %msk ret i128 %res } define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind { ; SSE-LABEL: isolate_msb_i128: ; SSE: # %bb.0: ; SSE-NEXT: bsrq %rsi, %rax ; SSE-NEXT: xorl $63, %eax ; SSE-NEXT: bsrq %rdi, %rcx ; SSE-NEXT: xorl $63, %ecx ; SSE-NEXT: orb $64, %cl ; SSE-NEXT: testq %rsi, %rsi ; SSE-NEXT: movzbl %cl, %ecx ; SSE-NEXT: cmovnel %eax, %ecx ; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; SSE-NEXT: xorl %eax, %eax ; SSE-NEXT: shrdq %cl, %rdx, %rax ; SSE-NEXT: xorl %r8d, %r8d ; SSE-NEXT: shrq %cl, %rdx ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rdx, %rax ; SSE-NEXT: cmovneq %r8, %rdx ; SSE-NEXT: orq %rsi, %rdi ; SSE-NEXT: cmoveq %r8, %rax ; SSE-NEXT: cmoveq %r8, %rdx ; SSE-NEXT: retq ; ; AVX-LABEL: isolate_msb_i128: ; AVX: # %bb.0: ; AVX-NEXT: lzcntq %rdi, %rax ; AVX-NEXT: addb $64, %al ; AVX-NEXT: lzcntq %rsi, %rdx ; AVX-NEXT: movzbl %al, %ecx ; AVX-NEXT: cmovael %edx, %ecx ; AVX-NEXT: xorl %r8d, %r8d ; AVX-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; AVX-NEXT: xorl %eax, %eax ; AVX-NEXT: shrdq %cl, %rdx, %rax ; AVX-NEXT: shrxq %rcx, %rdx, %rdx ; AVX-NEXT: testb $64, %cl ; AVX-NEXT: cmovneq %rdx, %rax ; AVX-NEXT: cmovneq %r8, %rdx ; AVX-NEXT: orq %rsi, %rdi ; AVX-NEXT: cmoveq %r8, %rax ; AVX-NEXT: cmoveq %r8, %rdx ; AVX-NEXT: retq %eqz = icmp eq i128 %a0, 0 %clz = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1) %bit = shl i128 1, 127 %msk = lshr i128 %bit, %clz %res = select i1 %eqz, i128 0, i128 %msk ret i128 %res } define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind { ; SSE2-LABEL: isolate_msb_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: movq %xmm0, %rax ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm1, %rcx ; SSE2-NEXT: pxor %xmm1, %xmm1 ; SSE2-NEXT: pcmpeqd %xmm0, %xmm1 ; SSE2-NEXT: movmskps %xmm1, %esi ; SSE2-NEXT: xorl $15, %esi ; SSE2-NEXT: bsrq %rcx, %rdx ; SSE2-NEXT: xorl $63, %edx ; SSE2-NEXT: bsrq %rax, %rax ; SSE2-NEXT: xorl $63, %eax ; SSE2-NEXT: orb $64, %al ; SSE2-NEXT: testq %rcx, %rcx ; SSE2-NEXT: movzbl %al, %ecx ; SSE2-NEXT: cmovnel %edx, %ecx ; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; SSE2-NEXT: xorl %eax, %eax ; SSE2-NEXT: shrdq %cl, %rdx, %rax ; SSE2-NEXT: xorl %edi, %edi ; SSE2-NEXT: shrq %cl, %rdx ; SSE2-NEXT: testb $64, %cl ; SSE2-NEXT: cmovneq %rdx, %rax ; SSE2-NEXT: cmovneq %rdi, %rdx ; SSE2-NEXT: testl %esi, %esi ; SSE2-NEXT: cmoveq %rdi, %rax ; SSE2-NEXT: cmoveq %rdi, %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: isolate_msb_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: movq %xmm0, %rax ; SSE42-NEXT: pextrq $1, %xmm0, %rcx ; SSE42-NEXT: bsrq %rcx, %rdx ; SSE42-NEXT: xorl $63, %edx ; SSE42-NEXT: bsrq %rax, %rax ; SSE42-NEXT: xorl $63, %eax ; SSE42-NEXT: orb $64, %al ; SSE42-NEXT: testq %rcx, %rcx ; SSE42-NEXT: movzbl %al, %ecx ; SSE42-NEXT: cmovnel %edx, %ecx ; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; SSE42-NEXT: xorl %eax, %eax ; SSE42-NEXT: shrdq %cl, %rdx, %rax ; SSE42-NEXT: xorl %esi, %esi ; SSE42-NEXT: shrq %cl, %rdx ; SSE42-NEXT: testb $64, %cl ; SSE42-NEXT: cmovneq %rdx, %rax ; SSE42-NEXT: cmovneq %rsi, %rdx ; SSE42-NEXT: ptest %xmm0, %xmm0 ; SSE42-NEXT: cmoveq %rsi, %rax ; SSE42-NEXT: cmoveq %rsi, %rdx ; SSE42-NEXT: retq ; ; AVX-LABEL: isolate_msb_i128_vector: ; AVX: # %bb.0: ; AVX-NEXT: vpextrq $1, %xmm0, %rax ; AVX-NEXT: vmovq %xmm0, %rcx ; AVX-NEXT: lzcntq %rcx, %rcx ; AVX-NEXT: addb $64, %cl ; AVX-NEXT: lzcntq %rax, %rax ; AVX-NEXT: movzbl %cl, %ecx ; AVX-NEXT: cmovael %eax, %ecx ; AVX-NEXT: xorl %esi, %esi ; AVX-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; AVX-NEXT: xorl %eax, %eax ; AVX-NEXT: shrdq %cl, %rdx, %rax ; AVX-NEXT: shrxq %rcx, %rdx, %rdx ; AVX-NEXT: testb $64, %cl ; AVX-NEXT: cmovneq %rdx, %rax ; AVX-NEXT: cmovneq %rsi, %rdx ; AVX-NEXT: vptest %xmm0, %xmm0 ; AVX-NEXT: cmoveq %rsi, %rax ; AVX-NEXT: cmoveq %rsi, %rdx ; AVX-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %eqz = icmp eq i128 %a0, 0 %clz = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1) %bit = shl i128 1, 127 %msk = lshr i128 %bit, %clz %res = select i1 %eqz, i128 0, i128 %msk ret i128 %res } define i128 @isolate_msb_i128_load(ptr %p0, i128 %idx) nounwind { ; SSE-LABEL: isolate_msb_i128_load: ; SSE: # %bb.0: ; SSE-NEXT: movq (%rdi), %rsi ; SSE-NEXT: movq 8(%rdi), %rdi ; SSE-NEXT: bsrq %rdi, %rax ; SSE-NEXT: xorl $63, %eax ; SSE-NEXT: bsrq %rsi, %rcx ; SSE-NEXT: xorl $63, %ecx ; SSE-NEXT: orb $64, %cl ; SSE-NEXT: testq %rdi, %rdi ; SSE-NEXT: movzbl %cl, %ecx ; SSE-NEXT: cmovnel %eax, %ecx ; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; SSE-NEXT: xorl %eax, %eax ; SSE-NEXT: shrdq %cl, %rdx, %rax ; SSE-NEXT: xorl %r8d, %r8d ; SSE-NEXT: shrq %cl, %rdx ; SSE-NEXT: testb $64, %cl ; SSE-NEXT: cmovneq %rdx, %rax ; SSE-NEXT: cmovneq %r8, %rdx ; SSE-NEXT: orq %rdi, %rsi ; SSE-NEXT: cmoveq %r8, %rax ; SSE-NEXT: cmoveq %r8, %rdx ; SSE-NEXT: retq ; ; AVX-LABEL: isolate_msb_i128_load: ; AVX: # %bb.0: ; AVX-NEXT: movq (%rdi), %rsi ; AVX-NEXT: movq 8(%rdi), %rdi ; AVX-NEXT: lzcntq %rsi, %rax ; AVX-NEXT: addb $64, %al ; AVX-NEXT: lzcntq %rdi, %rdx ; AVX-NEXT: movzbl %al, %ecx ; AVX-NEXT: cmovael %edx, %ecx ; AVX-NEXT: xorl %r8d, %r8d ; AVX-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000 ; AVX-NEXT: xorl %eax, %eax ; AVX-NEXT: shrdq %cl, %rdx, %rax ; AVX-NEXT: shrxq %rcx, %rdx, %rdx ; AVX-NEXT: testb $64, %cl ; AVX-NEXT: cmovneq %rdx, %rax ; AVX-NEXT: cmovneq %r8, %rdx ; AVX-NEXT: orq %rdi, %rsi ; AVX-NEXT: cmoveq %r8, %rax ; AVX-NEXT: cmoveq %r8, %rdx ; AVX-NEXT: retq %a0 = load i128, ptr %p0 %eqz = icmp eq i128 %a0, 0 %clz = call i128 @llvm.ctlz.i128(i128 %a0, i1 -1) %bit = shl i128 1, 127 %msk = lshr i128 %bit, %clz %res = select i1 %eqz, i128 0, i128 %msk ret i128 %res } define i128 @bswap_i128(i128 %a0) nounwind { ; CHECK-LABEL: bswap_i128: ; CHECK: # %bb.0: ; CHECK-NEXT: movq %rsi, %rax ; CHECK-NEXT: bswapq %rax ; CHECK-NEXT: bswapq %rdi ; CHECK-NEXT: movq %rdi, %rdx ; CHECK-NEXT: retq %res = call i128 @llvm.bswap.i128(i128 %a0) ret i128 %res } define i128 @bswap_i128_vector(<2 x i64> %v0) nounwind { ; SSE2-LABEL: bswap_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm1, %rax ; SSE2-NEXT: movq %xmm0, %rdx ; SSE2-NEXT: bswapq %rdx ; SSE2-NEXT: bswapq %rax ; SSE2-NEXT: retq ; ; SSE42-LABEL: bswap_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: movq %xmm0, %rdx ; SSE42-NEXT: pextrq $1, %xmm0, %rax ; SSE42-NEXT: bswapq %rax ; SSE42-NEXT: bswapq %rdx ; SSE42-NEXT: retq ; ; AVX-LABEL: bswap_i128_vector: ; AVX: # %bb.0: ; AVX-NEXT: vmovq %xmm0, %rdx ; AVX-NEXT: vpextrq $1, %xmm0, %rax ; AVX-NEXT: bswapq %rax ; AVX-NEXT: bswapq %rdx ; AVX-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %res = call i128 @llvm.bswap.i128(i128 %a0) ret i128 %res } define i128 @bswap_i128_load(ptr %p0) nounwind { ; SSE-LABEL: bswap_i128_load: ; SSE: # %bb.0: ; SSE-NEXT: movq (%rdi), %rdx ; SSE-NEXT: movq 8(%rdi), %rax ; SSE-NEXT: bswapq %rax ; SSE-NEXT: bswapq %rdx ; SSE-NEXT: retq ; ; AVX-LABEL: bswap_i128_load: ; AVX: # %bb.0: ; AVX-NEXT: movbeq 8(%rdi), %rax ; AVX-NEXT: movbeq (%rdi), %rdx ; AVX-NEXT: retq %a0 = load i128, ptr %p0 %res = call i128 @llvm.bswap.i128(i128 %a0) ret i128 %res } define i128 @bitreverse_i128(i128 %a0) nounwind { ; SSE2-LABEL: bitreverse_i128: ; SSE2: # %bb.0: ; SSE2-NEXT: bswapq %rsi ; SSE2-NEXT: movq %rsi, %rax ; SSE2-NEXT: shrq $4, %rax ; SSE2-NEXT: movabsq $1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F ; SSE2-NEXT: andq %rdx, %rax ; SSE2-NEXT: andq %rdx, %rsi ; SSE2-NEXT: shlq $4, %rsi ; SSE2-NEXT: orq %rax, %rsi ; SSE2-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333 ; SSE2-NEXT: movq %rsi, %rax ; SSE2-NEXT: andq %rcx, %rax ; SSE2-NEXT: shrq $2, %rsi ; SSE2-NEXT: andq %rcx, %rsi ; SSE2-NEXT: leaq (%rsi,%rax,4), %rax ; SSE2-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555 ; SSE2-NEXT: movq %rax, %r8 ; SSE2-NEXT: andq %rsi, %r8 ; SSE2-NEXT: shrq %rax ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: leaq (%rax,%r8,2), %rax ; SSE2-NEXT: bswapq %rdi ; SSE2-NEXT: movq %rdi, %r8 ; SSE2-NEXT: shrq $4, %r8 ; SSE2-NEXT: andq %rdx, %r8 ; SSE2-NEXT: andq %rdx, %rdi ; SSE2-NEXT: shlq $4, %rdi ; SSE2-NEXT: orq %r8, %rdi ; SSE2-NEXT: movq %rdi, %rdx ; SSE2-NEXT: andq %rcx, %rdx ; SSE2-NEXT: shrq $2, %rdi ; SSE2-NEXT: andq %rcx, %rdi ; SSE2-NEXT: leaq (%rdi,%rdx,4), %rcx ; SSE2-NEXT: movq %rcx, %rdx ; SSE2-NEXT: andq %rsi, %rdx ; SSE2-NEXT: shrq %rcx ; SSE2-NEXT: andq %rsi, %rcx ; SSE2-NEXT: leaq (%rcx,%rdx,2), %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: bitreverse_i128: ; SSE42: # %bb.0: ; SSE42-NEXT: movq %rdi, %xmm0 ; SSE42-NEXT: movq %rsi, %xmm1 ; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] ; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8] ; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; SSE42-NEXT: movdqa %xmm1, %xmm2 ; SSE42-NEXT: pand %xmm0, %xmm2 ; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; SSE42-NEXT: pshufb %xmm2, %xmm3 ; SSE42-NEXT: psrlw $4, %xmm1 ; SSE42-NEXT: pand %xmm0, %xmm1 ; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; SSE42-NEXT: pshufb %xmm1, %xmm0 ; SSE42-NEXT: por %xmm3, %xmm0 ; SSE42-NEXT: movq %xmm0, %rax ; SSE42-NEXT: pextrq $1, %xmm0, %rdx ; SSE42-NEXT: retq ; ; AVX2-LABEL: bitreverse_i128: ; AVX2: # %bb.0: ; AVX2-NEXT: vmovq %rdi, %xmm0 ; AVX2-NEXT: vmovq %rsi, %xmm1 ; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8] ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: vpextrq $1, %xmm0, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bitreverse_i128: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovq %rdi, %xmm0 ; AVX512F-NEXT: vmovq %rsi, %xmm1 ; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8] ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512F-NEXT: vmovq %xmm0, %rax ; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bitreverse_i128: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vmovq %rdi, %xmm0 ; AVX512VL-NEXT: vmovq %rsi, %xmm1 ; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8] ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VL-NEXT: vmovq %xmm0, %rax ; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bitreverse_i128: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: vmovq %rdi, %xmm0 ; AVX512VBMI-NEXT: vmovq %rsi, %xmm1 ; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] ; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8] ; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VBMI-NEXT: vmovq %xmm0, %rax ; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VBMI-NEXT: retq %res = call i128 @llvm.bitreverse.i128(i128 %a0) ret i128 %res } define i128 @bitreverse_i128_vector(<2 x i64> %v0) nounwind { ; SSE2-LABEL: bitreverse_i128_vector: ; SSE2: # %bb.0: ; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] ; SSE2-NEXT: movq %xmm0, %rdx ; SSE2-NEXT: bswapq %rdx ; SSE2-NEXT: movq %xmm1, %rax ; SSE2-NEXT: movq %rdx, %rcx ; SSE2-NEXT: shrq $4, %rcx ; SSE2-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F ; SSE2-NEXT: andq %rsi, %rcx ; SSE2-NEXT: andq %rsi, %rdx ; SSE2-NEXT: shlq $4, %rdx ; SSE2-NEXT: orq %rcx, %rdx ; SSE2-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333 ; SSE2-NEXT: movq %rdx, %rdi ; SSE2-NEXT: andq %rcx, %rdi ; SSE2-NEXT: shrq $2, %rdx ; SSE2-NEXT: andq %rcx, %rdx ; SSE2-NEXT: leaq (%rdx,%rdi,4), %rdx ; SSE2-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555 ; SSE2-NEXT: movq %rdx, %r8 ; SSE2-NEXT: andq %rdi, %r8 ; SSE2-NEXT: shrq %rdx ; SSE2-NEXT: andq %rdi, %rdx ; SSE2-NEXT: leaq (%rdx,%r8,2), %rdx ; SSE2-NEXT: bswapq %rax ; SSE2-NEXT: movq %rax, %r8 ; SSE2-NEXT: shrq $4, %r8 ; SSE2-NEXT: andq %rsi, %r8 ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: shlq $4, %rax ; SSE2-NEXT: orq %r8, %rax ; SSE2-NEXT: movq %rax, %rsi ; SSE2-NEXT: andq %rcx, %rsi ; SSE2-NEXT: shrq $2, %rax ; SSE2-NEXT: andq %rcx, %rax ; SSE2-NEXT: leaq (%rax,%rsi,4), %rax ; SSE2-NEXT: movq %rax, %rcx ; SSE2-NEXT: andq %rdi, %rcx ; SSE2-NEXT: shrq %rax ; SSE2-NEXT: andq %rdi, %rax ; SSE2-NEXT: leaq (%rax,%rcx,2), %rax ; SSE2-NEXT: retq ; ; SSE42-LABEL: bitreverse_i128_vector: ; SSE42: # %bb.0: ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; SSE42-NEXT: movdqa %xmm0, %xmm2 ; SSE42-NEXT: pand %xmm1, %xmm2 ; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; SSE42-NEXT: pshufb %xmm2, %xmm3 ; SSE42-NEXT: psrlw $4, %xmm0 ; SSE42-NEXT: pand %xmm1, %xmm0 ; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; SSE42-NEXT: pshufb %xmm0, %xmm1 ; SSE42-NEXT: por %xmm3, %xmm1 ; SSE42-NEXT: movq %xmm1, %rax ; SSE42-NEXT: pextrq $1, %xmm1, %rdx ; SSE42-NEXT: retq ; ; AVX2-LABEL: bitreverse_i128_vector: ; AVX2: # %bb.0: ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: vpextrq $1, %xmm0, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bitreverse_i128_vector: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512F-NEXT: vmovq %xmm0, %rax ; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bitreverse_i128_vector: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VL-NEXT: vmovq %xmm0, %rax ; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bitreverse_i128_vector: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VBMI-NEXT: vmovq %xmm0, %rax ; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VBMI-NEXT: retq %a0 = bitcast <2 x i64> %v0 to i128 %res = call i128 @llvm.bitreverse.i128(i128 %a0) ret i128 %res } define i128 @bitreverse_i128_load(ptr %p0) nounwind { ; SSE2-LABEL: bitreverse_i128_load: ; SSE2: # %bb.0: ; SSE2-NEXT: movq 8(%rdi), %rax ; SSE2-NEXT: bswapq %rax ; SSE2-NEXT: movq (%rdi), %rcx ; SSE2-NEXT: movq %rax, %rdx ; SSE2-NEXT: shrq $4, %rdx ; SSE2-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F ; SSE2-NEXT: andq %rsi, %rdx ; SSE2-NEXT: andq %rsi, %rax ; SSE2-NEXT: shlq $4, %rax ; SSE2-NEXT: orq %rdx, %rax ; SSE2-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333 ; SSE2-NEXT: movq %rax, %rdi ; SSE2-NEXT: andq %rdx, %rdi ; SSE2-NEXT: shrq $2, %rax ; SSE2-NEXT: andq %rdx, %rax ; SSE2-NEXT: leaq (%rax,%rdi,4), %rax ; SSE2-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555 ; SSE2-NEXT: movq %rax, %r8 ; SSE2-NEXT: andq %rdi, %r8 ; SSE2-NEXT: shrq %rax ; SSE2-NEXT: andq %rdi, %rax ; SSE2-NEXT: leaq (%rax,%r8,2), %rax ; SSE2-NEXT: bswapq %rcx ; SSE2-NEXT: movq %rcx, %r8 ; SSE2-NEXT: shrq $4, %r8 ; SSE2-NEXT: andq %rsi, %r8 ; SSE2-NEXT: andq %rsi, %rcx ; SSE2-NEXT: shlq $4, %rcx ; SSE2-NEXT: orq %r8, %rcx ; SSE2-NEXT: movq %rcx, %rsi ; SSE2-NEXT: andq %rdx, %rsi ; SSE2-NEXT: shrq $2, %rcx ; SSE2-NEXT: andq %rdx, %rcx ; SSE2-NEXT: leaq (%rcx,%rsi,4), %rcx ; SSE2-NEXT: movq %rcx, %rdx ; SSE2-NEXT: andq %rdi, %rdx ; SSE2-NEXT: shrq %rcx ; SSE2-NEXT: andq %rdi, %rcx ; SSE2-NEXT: leaq (%rcx,%rdx,2), %rdx ; SSE2-NEXT: retq ; ; SSE42-LABEL: bitreverse_i128_load: ; SSE42: # %bb.0: ; SSE42-NEXT: movdqa (%rdi), %xmm0 ; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; SSE42-NEXT: movdqa %xmm0, %xmm2 ; SSE42-NEXT: pand %xmm1, %xmm2 ; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; SSE42-NEXT: pshufb %xmm2, %xmm3 ; SSE42-NEXT: psrlw $4, %xmm0 ; SSE42-NEXT: pand %xmm1, %xmm0 ; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; SSE42-NEXT: pshufb %xmm0, %xmm1 ; SSE42-NEXT: por %xmm3, %xmm1 ; SSE42-NEXT: movq %xmm1, %rax ; SSE42-NEXT: pextrq $1, %xmm1, %rdx ; SSE42-NEXT: retq ; ; AVX2-LABEL: bitreverse_i128_load: ; AVX2: # %bb.0: ; AVX2-NEXT: vmovdqa (%rdi), %xmm0 ; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX2-NEXT: vmovq %xmm0, %rax ; AVX2-NEXT: vpextrq $1, %xmm0, %rdx ; AVX2-NEXT: retq ; ; AVX512F-LABEL: bitreverse_i128_load: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovdqa (%rdi), %xmm0 ; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0 ; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512F-NEXT: vmovq %xmm0, %rax ; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: bitreverse_i128_load: ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vmovdqa (%rdi), %xmm0 ; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VL-NEXT: vmovq %xmm0, %rax ; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VL-NEXT: retq ; ; AVX512VBMI-LABEL: bitreverse_i128_load: ; AVX512VBMI: # %bb.0: ; AVX512VBMI-NEXT: vmovdqa (%rdi), %xmm0 ; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0] ; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240] ; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2 ; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15] ; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0 ; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0 ; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512VBMI-NEXT: vmovq %xmm0, %rax ; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx ; AVX512VBMI-NEXT: retq %a0 = load i128, ptr %p0 %res = call i128 @llvm.bitreverse.i128(i128 %a0) ret i128 %res } ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; AVX512: {{.*}}