; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-SSE-O3,CHECK-SSE4-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX2-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O3,CHECK-AVX-O3,CHECK-AVX512-O3
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-SSE-O0,CHECK-SSE4-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX2-O0
; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -O0 -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,CHECK-O0,CHECK-AVX-O0,CHECK-AVX512-O0

define void @test1(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test1:
; CHECK:       # %bb.0:
; CHECK-NEXT:    xchgl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic i32 %val1, ptr %ptr seq_cst, align 4
  ret void
}

define void @test2(ptr %ptr, i32 %val1) {
; CHECK-LABEL: test2:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic i32 %val1, ptr %ptr release, align 4
  ret void
}

define i32 @test3(ptr %ptr) {
; CHECK-LABEL: test3:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %val = load atomic i32, ptr %ptr seq_cst, align 4
  ret i32 %val
}

define <1 x i32> @atomic_vec1_i32(ptr %x) {
; CHECK-LABEL: atomic_vec1_i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x i32>, ptr %x acquire, align 4
  ret <1 x i32> %ret
}

define <1 x i8> @atomic_vec1_i8(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzbl (%rdi), %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i8:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb (%rdi), %al
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i8>, ptr %x acquire, align 1
  ret <1 x i8> %ret
}

define <1 x i16> @atomic_vec1_i16(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i16:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw (%rdi), %ax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i16>, ptr %x acquire, align 2
  ret <1 x i16> %ret
}

define <1 x i32> @atomic_vec1_i8_zext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i8_zext:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzbl (%rdi), %eax
; CHECK-O3-NEXT:    movzbl %al, %eax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i8_zext:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb (%rdi), %al
; CHECK-O0-NEXT:    movzbl %al, %eax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i8>, ptr %x acquire, align 1
  %zret = zext <1 x i8> %ret to <1 x i32>
  ret <1 x i32> %zret
}

define <1 x i64> @atomic_vec1_i16_sext(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_i16_sext:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-O3-NEXT:    movswq %ax, %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i16_sext:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw (%rdi), %ax
; CHECK-O0-NEXT:    movswq %ax, %rax
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i16>, ptr %x acquire, align 2
  %sret = sext <1 x i16> %ret to <1 x i64>
  ret <1 x i64> %sret
}

define <1 x ptr addrspace(270)> @atomic_vec1_ptr270(ptr %x) {
; CHECK-LABEL: atomic_vec1_ptr270:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl (%rdi), %eax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x ptr addrspace(270)>, ptr %x acquire, align 4
  ret <1 x ptr addrspace(270)> %ret
}

define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x bfloat>, ptr %x acquire, align 2
  ret <1 x bfloat> %ret
}

define <1 x ptr> @atomic_vec1_ptr_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_ptr_align:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq (%rdi), %rax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x ptr>, ptr %x acquire, align 8
  ret <1 x ptr> %ret
}

define <1 x i64> @atomic_vec1_i64_align(ptr %x) nounwind {
; CHECK-LABEL: atomic_vec1_i64_align:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq (%rdi), %rax
; CHECK-NEXT:    retq
  %ret = load atomic <1 x i64>, ptr %x acquire, align 8
  ret <1 x i64> %ret
}

define void @store_atomic_vec1_i32(ptr %x, <1 x i32> %v) {
; CHECK-LABEL: store_atomic_vec1_i32:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl %esi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x i32> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_i8(ptr %x, <1 x i8> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i8:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movb %sil, (%rdi)
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i8:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movb %sil, %al
; CHECK-O0-NEXT:    movb %al, (%rdi)
; CHECK-O0-NEXT:    retq
  store atomic <1 x i8> %v, ptr %x release, align 1
  ret void
}

define void @store_atomic_vec1_i16(ptr %x, <1 x i16> %v) {
; CHECK-O3-LABEL: store_atomic_vec1_i16:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    movw %si, (%rdi)
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i16:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    movw %si, %ax
; CHECK-O0-NEXT:    movw %ax, (%rdi)
; CHECK-O0-NEXT:    retq
  store atomic <1 x i16> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_i64(ptr %x, <1 x i64> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_i64:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq %rsi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x i64> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec1_ptr(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-LABEL: store_atomic_vec1_ptr:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movq %rsi, (%rdi)
; CHECK-NEXT:    retq
  store atomic <1 x ptr> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec1_bfloat(ptr %x, <1 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    movw %ax, %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, (%rsp) # 8-byte Spill
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    movw %ax, %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x bfloat> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_half(ptr %x, <1 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O3-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movw %ax, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x half> %v, ptr %x release, align 2
  ret void
}

define void @store_atomic_vec1_float(ptr %x, <1 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x float> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movsd %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovsd %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movsd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovsd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x double> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) {
; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    movd %xmm0, %eax
; CHECK-SSE2-O3-NEXT:    movw %ax, (%rdi)
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, (%rdi)
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i8> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i16> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i32> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x float> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x i8> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x i16> %v, ptr %x release, align 8
  ret void
}

define <2 x i8> @atomic_vec2_i8(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i8>, ptr %x acquire, align 4
  ret <2 x i8> %ret
}

define <2 x i16> @atomic_vec2_i16(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i16>, ptr %x acquire, align 4
  ret <2 x i16> %ret
}

define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_ptr270:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8
  ret <2 x ptr addrspace(270)> %ret
}
define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_ptr_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x ptr>, ptr %x acquire, align 16
  ret <2 x ptr> %ret
}
define <4 x ptr addrspace(270)> @atomic_vec4_ptr270(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movaps (%rdi), %xmm0
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_ptr270:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movapd (%rdi), %xmm0
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_ptr270:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovapd (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
  ret <4 x ptr addrspace(270)> %ret
}

define <2 x i32> @atomic_vec2_i32_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i32>, ptr %x acquire, align 8
  ret <2 x i32> %ret
}

define <2 x float> @atomic_vec2_float_align(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_float_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x float>, ptr %x acquire, align 8
  ret <2 x float> %ret
}

define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    psrld $16, %xmm0
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE-O3-NEXT:    shll $16, %ecx
; CHECK-SSE-O3-NEXT:    movzwl %ax, %eax
; CHECK-SSE-O3-NEXT:    orl %ecx, %eax
; CHECK-SSE-O3-NEXT:    movl %eax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movaps %xmm1, %xmm0
; CHECK-SSE-O0-NEXT:    psrld $16, %xmm1
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    # implicit-def: $ecx
; CHECK-SSE-O0-NEXT:    movw %ax, %cx
; CHECK-SSE-O0-NEXT:    shll $16, %ecx
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movzwl %ax, %eax
; CHECK-SSE-O0-NEXT:    orl %ecx, %eax
; CHECK-SSE-O0-NEXT:    movl %eax, (%rdi)
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovd %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x half> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O3-NEXT:    psrld $16, %xmm0
; CHECK-SSE-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE-O3-NEXT:    shll $16, %ecx
; CHECK-SSE-O3-NEXT:    movzwl %ax, %eax
; CHECK-SSE-O3-NEXT:    orl %ecx, %eax
; CHECK-SSE-O3-NEXT:    movl %eax, (%rdi)
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $24, %rsp
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    movd %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    movw %ax, %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-SSE-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movzwl %ax, %eax
; CHECK-SSE-O0-NEXT:    orl %ecx, %eax
; CHECK-SSE-O0-NEXT:    movl %eax, (%rdi)
; CHECK-SSE-O0-NEXT:    addq $24, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $24, %rsp
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX-O0-NEXT:    vpextrw $1, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    vmovd %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    movw %ax, %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-AVX-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movzwl %ax, %eax
; CHECK-AVX-O0-NEXT:    orl %ecx, %eax
; CHECK-AVX-O0-NEXT:    movl %eax, (%rdi)
; CHECK-AVX-O0-NEXT:    addq $24, %rsp
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x bfloat> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_half:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE2-O3-NEXT:    psrld $16, %xmm1
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE2-O3-NEXT:    shll $16, %eax
; CHECK-SSE2-O3-NEXT:    movzwl %cx, %ecx
; CHECK-SSE2-O3-NEXT:    orl %eax, %ecx
; CHECK-SSE2-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE2-O3-NEXT:    psrlq $48, %xmm1
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O3-NEXT:    shll $16, %eax
; CHECK-SSE2-O3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm0, %edx
; CHECK-SSE2-O3-NEXT:    movzwl %dx, %edx
; CHECK-SSE2-O3-NEXT:    orl %eax, %edx
; CHECK-SSE2-O3-NEXT:    shlq $32, %rdx
; CHECK-SSE2-O3-NEXT:    orq %rcx, %rdx
; CHECK-SSE2-O3-NEXT:    movq %rdx, (%rdi)
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_half:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE4-O3-NEXT:    psrld $16, %xmm1
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE4-O3-NEXT:    shll $16, %eax
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE4-O3-NEXT:    movzwl %cx, %ecx
; CHECK-SSE4-O3-NEXT:    orl %eax, %ecx
; CHECK-SSE4-O3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; CHECK-SSE4-O3-NEXT:    psrlq $48, %xmm0
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE4-O3-NEXT:    shll $16, %eax
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm1, %edx
; CHECK-SSE4-O3-NEXT:    movzwl %dx, %edx
; CHECK-SSE4-O3-NEXT:    orl %eax, %edx
; CHECK-SSE4-O3-NEXT:    shlq $32, %rdx
; CHECK-SSE4-O3-NEXT:    orq %rcx, %rdx
; CHECK-SSE4-O3-NEXT:    movq %rdx, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm3
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm2
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm1
; CHECK-SSE2-O0-NEXT:    psrlq $48, %xmm1
; CHECK-SSE2-O0-NEXT:    movaps %xmm3, %xmm0
; CHECK-SSE2-O0-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-SSE2-O0-NEXT:    psrld $16, %xmm3
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm3, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    # implicit-def: $ecx
; CHECK-SSE2-O0-NEXT:    movw %ax, %cx
; CHECK-SSE2-O0-NEXT:    shll $16, %ecx
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm2, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movzwl %ax, %eax
; CHECK-SSE2-O0-NEXT:    orl %ecx, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $rax killed $eax
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %ecx
; CHECK-SSE2-O0-NEXT:    movw %cx, %dx
; CHECK-SSE2-O0-NEXT:    # implicit-def: $ecx
; CHECK-SSE2-O0-NEXT:    movw %dx, %cx
; CHECK-SSE2-O0-NEXT:    shll $16, %ecx
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %edx
; CHECK-SSE2-O0-NEXT:    # kill: def $dx killed $dx killed $edx
; CHECK-SSE2-O0-NEXT:    movzwl %dx, %edx
; CHECK-SSE2-O0-NEXT:    orl %ecx, %edx
; CHECK-SSE2-O0-NEXT:    # implicit-def: $rcx
; CHECK-SSE2-O0-NEXT:    movl %edx, %ecx
; CHECK-SSE2-O0-NEXT:    shlq $32, %rcx
; CHECK-SSE2-O0-NEXT:    orq %rcx, %rax
; CHECK-SSE2-O0-NEXT:    movq %rax, (%rdi)
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec4_half:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm3
; CHECK-SSE4-O0-NEXT:    movaps %xmm3, %xmm2
; CHECK-SSE4-O0-NEXT:    movaps %xmm3, %xmm1
; CHECK-SSE4-O0-NEXT:    psrlq $48, %xmm1
; CHECK-SSE4-O0-NEXT:    movshdup {{.*#+}} xmm0 = xmm3[1,1,3,3]
; CHECK-SSE4-O0-NEXT:    psrld $16, %xmm3
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm3, %eax
; CHECK-SSE4-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT:    # implicit-def: $ecx
; CHECK-SSE4-O0-NEXT:    movw %ax, %cx
; CHECK-SSE4-O0-NEXT:    shll $16, %ecx
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm2, %eax
; CHECK-SSE4-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE4-O0-NEXT:    movzwl %ax, %eax
; CHECK-SSE4-O0-NEXT:    orl %ecx, %eax
; CHECK-SSE4-O0-NEXT:    # kill: def $rax killed $eax
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, %ecx
; CHECK-SSE4-O0-NEXT:    movw %cx, %dx
; CHECK-SSE4-O0-NEXT:    # implicit-def: $ecx
; CHECK-SSE4-O0-NEXT:    movw %dx, %cx
; CHECK-SSE4-O0-NEXT:    shll $16, %ecx
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, %edx
; CHECK-SSE4-O0-NEXT:    # kill: def $dx killed $dx killed $edx
; CHECK-SSE4-O0-NEXT:    movzwl %dx, %edx
; CHECK-SSE4-O0-NEXT:    orl %ecx, %edx
; CHECK-SSE4-O0-NEXT:    # implicit-def: $rcx
; CHECK-SSE4-O0-NEXT:    movl %edx, %ecx
; CHECK-SSE4-O0-NEXT:    shlq $32, %rcx
; CHECK-SSE4-O0-NEXT:    orq %rcx, %rax
; CHECK-SSE4-O0-NEXT:    movq %rax, (%rdi)
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rdi)
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x half> %v, ptr %x release, align 8
  ret void
}

define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind {
; CHECK-SSE2-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE2-O3-NEXT:    psrld $16, %xmm1
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE2-O3-NEXT:    shll $16, %eax
; CHECK-SSE2-O3-NEXT:    movzwl %cx, %ecx
; CHECK-SSE2-O3-NEXT:    orl %eax, %ecx
; CHECK-SSE2-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE2-O3-NEXT:    psrlq $48, %xmm1
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O3-NEXT:    shll $16, %eax
; CHECK-SSE2-O3-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-SSE2-O3-NEXT:    pextrw $0, %xmm0, %edx
; CHECK-SSE2-O3-NEXT:    movzwl %dx, %edx
; CHECK-SSE2-O3-NEXT:    orl %eax, %edx
; CHECK-SSE2-O3-NEXT:    shlq $32, %rdx
; CHECK-SSE2-O3-NEXT:    orq %rcx, %rdx
; CHECK-SSE2-O3-NEXT:    movq %rdx, (%rdi)
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    movdqa %xmm0, %xmm1
; CHECK-SSE4-O3-NEXT:    psrld $16, %xmm1
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE4-O3-NEXT:    shll $16, %eax
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, %ecx
; CHECK-SSE4-O3-NEXT:    movzwl %cx, %ecx
; CHECK-SSE4-O3-NEXT:    orl %eax, %ecx
; CHECK-SSE4-O3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; CHECK-SSE4-O3-NEXT:    psrlq $48, %xmm0
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE4-O3-NEXT:    shll $16, %eax
; CHECK-SSE4-O3-NEXT:    pextrw $0, %xmm1, %edx
; CHECK-SSE4-O3-NEXT:    movzwl %dx, %edx
; CHECK-SSE4-O3-NEXT:    orl %eax, %edx
; CHECK-SSE4-O3-NEXT:    shlq $32, %rdx
; CHECK-SSE4-O3-NEXT:    orq %rcx, %rdx
; CHECK-SSE4-O3-NEXT:    movq %rdx, (%rdi)
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rdi)
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    movd %xmm1, %eax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    movw %ax, %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE-O0-NEXT:    movzwl %ax, %eax
; CHECK-SSE-O0-NEXT:    orl %ecx, %eax
; CHECK-SSE-O0-NEXT:    # kill: def $rax killed $eax
; CHECK-SSE-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE-O0-NEXT:    movw %ax, %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    shll $16, %eax
; CHECK-SSE-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-SSE-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; CHECK-SSE-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-SSE-O0-NEXT:    pextrw $0, %xmm0, %edx
; CHECK-SSE-O0-NEXT:    # kill: def $dx killed $dx killed $edx
; CHECK-SSE-O0-NEXT:    movzwl %dx, %edx
; CHECK-SSE-O0-NEXT:    orl %ecx, %edx
; CHECK-SSE-O0-NEXT:    # implicit-def: $rcx
; CHECK-SSE-O0-NEXT:    movl %edx, %ecx
; CHECK-SSE-O0-NEXT:    shlq $32, %rcx
; CHECK-SSE-O0-NEXT:    orq %rcx, %rax
; CHECK-SSE-O0-NEXT:    movq %rax, (%rdi)
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX-O0-NEXT:    vpextrw $3, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    vpextrw $2, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    vpextrw $1, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    vmovd %xmm1, %eax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    movw %ax, %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-AVX-O0-NEXT:    movzwl %ax, %eax
; CHECK-AVX-O0-NEXT:    orl %ecx, %eax
; CHECK-AVX-O0-NEXT:    # kill: def $rax killed $eax
; CHECK-AVX-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm1, %eax
; CHECK-AVX-O0-NEXT:    movw %ax, %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    shll $16, %eax
; CHECK-AVX-O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX-O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Reload
; CHECK-AVX-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; CHECK-AVX-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
; CHECK-AVX-O0-NEXT:    vpextrw $0, %xmm0, %edx
; CHECK-AVX-O0-NEXT:    # kill: def $dx killed $dx killed $edx
; CHECK-AVX-O0-NEXT:    movzwl %dx, %edx
; CHECK-AVX-O0-NEXT:    orl %ecx, %edx
; CHECK-AVX-O0-NEXT:    # implicit-def: $rcx
; CHECK-AVX-O0-NEXT:    movl %edx, %ecx
; CHECK-AVX-O0-NEXT:    shlq $32, %rcx
; CHECK-AVX-O0-NEXT:    orq %rcx, %rax
; CHECK-AVX-O0-NEXT:    movq %rax, (%rdi)
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x bfloat> %v, ptr %x release, align 8
  ret void
}

define <2 x half> @atomic_vec2_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x half>, ptr %x acquire, align 4
  ret <2 x half> %ret
}
define <2 x bfloat> @atomic_vec2_bfloat(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x bfloat>, ptr %x acquire, align 4
  ret <2 x bfloat> %ret
}
define <1 x ptr> @atomic_vec1_ptr(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_ptr:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rsi
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movl $2, %ecx
; CHECK-O3-NEXT:    callq __atomic_load@PLT
; CHECK-O3-NEXT:    movq (%rsp), %rax
; CHECK-O3-NEXT:    popq %rcx
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_ptr:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $2, %ecx
; CHECK-O0-NEXT:    callq __atomic_load@PLT
; CHECK-O0-NEXT:    movq (%rsp), %rax
; CHECK-O0-NEXT:    popq %rcx
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x ptr>, ptr %x acquire, align 4
  ret <1 x ptr> %ret
}

define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-SSE-O3-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    movzwl (%rdi), %eax
; CHECK-AVX-O3-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movw (%rdi), %cx
; CHECK-SSE-O0-NEXT:    # implicit-def: $eax
; CHECK-SSE-O0-NEXT:    movw %cx, %ax
; CHECK-SSE-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    movw (%rdi), %cx
; CHECK-AVX-O0-NEXT:    # implicit-def: $eax
; CHECK-AVX-O0-NEXT:    movw %cx, %ax
; CHECK-AVX-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x half>, ptr %x acquire, align 2
  ret <1 x half> %ret
}

define <1 x float> @atomic_vec1_float(ptr %x) {
; CHECK-SSE-O3-LABEL: atomic_vec1_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x float>, ptr %x acquire, align 4
  ret <1 x float> %ret
}

define <1 x double> @atomic_vec1_double_align(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double_align:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x double>, ptr %x acquire, align 8
  ret <1 x double> %ret
}

define <1 x i64> @atomic_vec1_i64(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec1_i64:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rsi
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movl $2, %ecx
; CHECK-O3-NEXT:    callq __atomic_load@PLT
; CHECK-O3-NEXT:    movq (%rsp), %rax
; CHECK-O3-NEXT:    popq %rcx
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: atomic_vec1_i64:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $2, %ecx
; CHECK-O0-NEXT:    callq __atomic_load@PLT
; CHECK-O0-NEXT:    movq (%rsp), %rax
; CHECK-O0-NEXT:    popq %rcx
; CHECK-O0-NEXT:    retq
  %ret = load atomic <1 x i64>, ptr %x acquire, align 4
  ret <1 x i64> %ret
}

define <1 x double> @atomic_vec1_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec1_double:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_double:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_double:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_double:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <1 x double>, ptr %x acquire, align 4
  ret <1 x double> %ret
}

define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec2_i32:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i32:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i32:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i32:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovq {{.*#+}} xmm0 = mem[0],zero
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <2 x i32>, ptr %x acquire, align 4
  ret <2 x i32> %ret
}

define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i8:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i8:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i8:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i8:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x i8>, ptr %x acquire, align 4
  ret <4 x i8> %ret
}

define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_i16:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i16:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i16:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i16:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x i16>, ptr %x acquire, align 8
  ret <4 x i16> %ret
}

define <4 x half> @atomic_vec4_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_half:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_half:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x half>, ptr %x acquire, align 8
  ret <4 x half> %ret
}
define <4 x bfloat> @atomic_vec4_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    movq (%rdi), %xmm0
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovq (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x bfloat>, ptr %x acquire, align 8
  ret <4 x bfloat> %ret
}

define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec4_float:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $24, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $16, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    addq $24, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $24, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $16, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovaps (%rsp), %xmm0
; CHECK-AVX-O3-NEXT:    addq $24, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_float:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $24, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $16, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    addq $24, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $24, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $16, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovaps (%rsp), %xmm0
; CHECK-AVX-O0-NEXT:    addq $24, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x float>, ptr %x acquire, align 4
  ret <4 x float> %ret
}

define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind {
; CHECK-SSE2-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O3:       # %bb.0:
; CHECK-SSE2-O3-NEXT:    pushq %rax
; CHECK-SSE2-O3-NEXT:    movl $2, %esi
; CHECK-SSE2-O3-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O3-NEXT:    popq %rax
; CHECK-SSE2-O3-NEXT:    retq
;
; CHECK-SSE4-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O3:       # %bb.0:
; CHECK-SSE4-O3-NEXT:    pushq %rbx
; CHECK-SSE4-O3-NEXT:    xorl %eax, %eax
; CHECK-SSE4-O3-NEXT:    xorl %edx, %edx
; CHECK-SSE4-O3-NEXT:    xorl %ecx, %ecx
; CHECK-SSE4-O3-NEXT:    xorl %ebx, %ebx
; CHECK-SSE4-O3-NEXT:    lock cmpxchg16b (%rdi)
; CHECK-SSE4-O3-NEXT:    movq %rdx, %xmm1
; CHECK-SSE4-O3-NEXT:    movq %rax, %xmm0
; CHECK-SSE4-O3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O3-NEXT:    popq %rbx
; CHECK-SSE4-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    pushq %rax
; CHECK-SSE2-O0-NEXT:    movl $2, %esi
; CHECK-SSE2-O0-NEXT:    callq __atomic_load_16@PLT
; CHECK-SSE2-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE2-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE2-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE2-O0-NEXT:    popq %rax
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    pushq %rbx
; CHECK-SSE4-O0-NEXT:    xorl %eax, %eax
; CHECK-SSE4-O0-NEXT:    movl %eax, %ebx
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rax
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rdx
; CHECK-SSE4-O0-NEXT:    movq %rbx, %rcx
; CHECK-SSE4-O0-NEXT:    lock cmpxchg16b (%rdi)
; CHECK-SSE4-O0-NEXT:    movq %rdx, %xmm1
; CHECK-SSE4-O0-NEXT:    movq %rax, %xmm0
; CHECK-SSE4-O0-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; CHECK-SSE4-O0-NEXT:    popq %rbx
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    vmovaps (%rdi), %xmm0
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <4 x float>, ptr %x acquire, align 16
  ret <4 x float> %ret
}

define <8 x double> @atomic_vec8_double(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec8_double:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec8_double:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec8_double:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec8_double:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movapd (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movapd {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec8_double:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovupd (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovupd {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec8_double:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovupd (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <8 x double>, ptr %x acquire, align 4
  ret <8 x double> %ret
}

define <16 x bfloat> @atomic_vec16_bfloat(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $2, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec16_bfloat:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $40, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $32, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    addq $40, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: atomic_vec16_bfloat:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $40, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    movl $32, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $2, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX-O0-NEXT:    addq $40, %rsp
; CHECK-AVX-O0-NEXT:    retq
  %ret = load atomic <16 x bfloat>, ptr %x acquire, align 4
  ret <16 x bfloat> %ret
}

define <32 x half> @atomic_vec32_half(ptr %x) nounwind {
; CHECK-SSE-O3-LABEL: atomic_vec32_half:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $2, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O3-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: atomic_vec32_half:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $2, %ecx
; CHECK-AVX2-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O3-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O3-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: atomic_vec32_half:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $2, %ecx
; CHECK-AVX512-O3-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O3-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: atomic_vec32_half:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $2, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_load@PLT
; CHECK-SSE-O0-NEXT:    movaps (%rsp), %xmm0
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm1
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm2
; CHECK-SSE-O0-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm3
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: atomic_vec32_half:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $2, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX2-O0-NEXT:    vmovups (%rsp), %ymm0
; CHECK-AVX2-O0-NEXT:    vmovups {{[0-9]+}}(%rsp), %ymm1
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: atomic_vec32_half:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $2, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_load@PLT
; CHECK-AVX512-O0-NEXT:    vmovups (%rsp), %zmm0
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  %ret = load atomic <32 x half>, ptr %x acquire, align 4
  ret <32 x half> %ret
}

define void @store_atomic_vec1_ptr_unaligned(ptr %x, <1 x ptr> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rax
; CHECK-O3-NEXT:    movq %rsi, (%rsp)
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movq %rax, %rsi
; CHECK-O3-NEXT:    movl $3, %ecx
; CHECK-O3-NEXT:    callq __atomic_store@PLT
; CHECK-O3-NEXT:    popq %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_ptr_unaligned:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rsi, %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movq %rax, (%rsp)
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $3, %ecx
; CHECK-O0-NEXT:    callq __atomic_store@PLT
; CHECK-O0-NEXT:    popq %rax
; CHECK-O0-NEXT:    retq
  store atomic <1 x ptr> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_i64_unaligned(ptr %x, <1 x i64> %v) nounwind {
; CHECK-O3-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O3:       # %bb.0:
; CHECK-O3-NEXT:    pushq %rax
; CHECK-O3-NEXT:    movq %rdi, %rax
; CHECK-O3-NEXT:    movq %rsi, (%rsp)
; CHECK-O3-NEXT:    movq %rsp, %rdx
; CHECK-O3-NEXT:    movl $8, %edi
; CHECK-O3-NEXT:    movq %rax, %rsi
; CHECK-O3-NEXT:    movl $3, %ecx
; CHECK-O3-NEXT:    callq __atomic_store@PLT
; CHECK-O3-NEXT:    popq %rax
; CHECK-O3-NEXT:    retq
;
; CHECK-O0-LABEL: store_atomic_vec1_i64_unaligned:
; CHECK-O0:       # %bb.0:
; CHECK-O0-NEXT:    pushq %rax
; CHECK-O0-NEXT:    movq %rsi, %rax
; CHECK-O0-NEXT:    movq %rdi, %rsi
; CHECK-O0-NEXT:    movq %rax, (%rsp)
; CHECK-O0-NEXT:    movl $8, %edi
; CHECK-O0-NEXT:    movq %rsp, %rdx
; CHECK-O0-NEXT:    movl $3, %ecx
; CHECK-O0-NEXT:    callq __atomic_store@PLT
; CHECK-O0-NEXT:    popq %rax
; CHECK-O0-NEXT:    retq
  store atomic <1 x i64> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec1_double_unaligned(ptr %x, <1 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movsd %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovsd %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movsd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec1_double_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovsd %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <1 x double> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec2_i32_unaligned(ptr %x, <2 x i32> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    pushq %rax
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movlps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $8, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    popq %rax
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    pushq %rax
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovlps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $8, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    popq %rax
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    pushq %rax
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movq %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $8, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    popq %rax
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    pushq %rax
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovq %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $8, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    popq %rax
; CHECK-AVX-O0-NEXT:    retq
  store atomic <2 x i32> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec4_float_unaligned(ptr %x, <4 x float> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $24, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $16, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $24, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $24, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovaps %xmm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $16, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $24, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $24, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movl $16, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $24, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_unaligned:
; CHECK-AVX-O0:       # %bb.0:
; CHECK-AVX-O0-NEXT:    subq $24, %rsp
; CHECK-AVX-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O0-NEXT:    vmovaps %xmm0, (%rsp)
; CHECK-AVX-O0-NEXT:    movl $16, %edi
; CHECK-AVX-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O0-NEXT:    movl $3, %ecx
; CHECK-AVX-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O0-NEXT:    addq $24, %rsp
; CHECK-AVX-O0-NEXT:    retq
  store atomic <4 x float> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec8_double_unaligned(ptr %x, <8 x double> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movapd %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movapd %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovupd %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec8_double_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovupd %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <8 x double> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec16_bfloat_unaligned(ptr %x, <16 x bfloat> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $40, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $32, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $40, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX-O3-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX-O3:       # %bb.0:
; CHECK-AVX-O3-NEXT:    subq $40, %rsp
; CHECK-AVX-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX-O3-NEXT:    movl $32, %edi
; CHECK-AVX-O3-NEXT:    movl $3, %ecx
; CHECK-AVX-O3-NEXT:    vzeroupper
; CHECK-AVX-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX-O3-NEXT:    addq $40, %rsp
; CHECK-AVX-O3-NEXT:    retq
;
; CHECK-SSE2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE2-O0:       # %bb.0:
; CHECK-SSE2-O0-NEXT:    subq $120, %rsp
; CHECK-SSE2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $4, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE2-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movd %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $7, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $6, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $5, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $4, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $3, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $2, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    pextrw $1, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE2-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE2-O0-NEXT:    movd %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    shll $16, %eax
; CHECK-SSE2-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE2-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm1, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE2-O0-NEXT:    pextrw $0, %xmm0, %eax
; CHECK-SSE2-O0-NEXT:    # kill: def $ax killed $ax killed $eax
; CHECK-SSE2-O0-NEXT:    movw %ax, {{[0-9]+}}(%rsp)
; CHECK-SSE2-O0-NEXT:    movl $32, %edi
; CHECK-SSE2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE2-O0-NEXT:    movl $3, %ecx
; CHECK-SSE2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE2-O0-NEXT:    addq $120, %rsp
; CHECK-SSE2-O0-NEXT:    retq
;
; CHECK-SSE4-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-SSE4-O0:       # %bb.0:
; CHECK-SSE4-O0-NEXT:    subq $120, %rsp
; CHECK-SSE4-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $4, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $1, %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm2
; CHECK-SSE4-O0-NEXT:    movss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movd %xmm1, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $7, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $6, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $5, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $4, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $3, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $2, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    pextrw $1, %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm1
; CHECK-SSE4-O0-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-SSE4-O0-NEXT:    movd %xmm0, %eax
; CHECK-SSE4-O0-NEXT:    shll $16, %eax
; CHECK-SSE4-O0-NEXT:    movd %eax, %xmm0
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movaps %xmm0, %xmm1
; CHECK-SSE4-O0-NEXT:    movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-SSE4-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-SSE4-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-SSE4-O0-NEXT:    pextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-SSE4-O0-NEXT:    movl $32, %edi
; CHECK-SSE4-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-SSE4-O0-NEXT:    movl $3, %ecx
; CHECK-SSE4-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE4-O0-NEXT:    addq $120, %rsp
; CHECK-SSE4-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $120, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX2-O0-NEXT:    vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vpextrw $7, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $6, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $5, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $4, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $3, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $2, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $1, %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX2-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovd %xmm1, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-AVX2-O0-NEXT:    vpextrw $7, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $6, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $5, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $4, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $3, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $2, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vpextrw $1, %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX2-O0-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX2-O0-NEXT:    shll $16, %eax
; CHECK-AVX2-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX2-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX2-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX2-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-AVX2-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $32, %edi
; CHECK-AVX2-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $120, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec16_bfloat_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $120, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-AVX512-O0-NEXT:    vextractf128 $1, %ymm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vpextrw $7, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $6, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $5, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $4, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $3, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $2, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $1, %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm2
; CHECK-AVX512-O0-NEXT:    vmovss %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vmovd %xmm1, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
; CHECK-AVX512-O0-NEXT:    vpextrw $7, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $6, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $5, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $4, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $3, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $2, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vpextrw $1, %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; CHECK-AVX512-O0-NEXT:    vmovd %xmm0, %eax
; CHECK-AVX512-O0-NEXT:    shll $16, %eax
; CHECK-AVX512-O0-NEXT:    vmovd %eax, %xmm0
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    vmovaps %xmm0, %xmm1
; CHECK-AVX512-O0-NEXT:    vmovss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-AVX512-O0-NEXT:    # xmm0 = mem[0],zero,zero,zero
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm1, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    callq __truncsfbf2@PLT
; CHECK-AVX512-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; CHECK-AVX512-O0-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
; CHECK-AVX512-O0-NEXT:    movl $32, %edi
; CHECK-AVX512-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $120, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <16 x bfloat> %v, ptr %x release, align 4
  ret void
}

define void @store_atomic_vec32_half_unaligned(ptr %x, <32 x half> %v) nounwind {
; CHECK-SSE-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O3:       # %bb.0:
; CHECK-SSE-O3-NEXT:    subq $72, %rsp
; CHECK-SSE-O3-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O3-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O3-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O3-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O3-NEXT:    movl $64, %edi
; CHECK-SSE-O3-NEXT:    movl $3, %ecx
; CHECK-SSE-O3-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O3-NEXT:    addq $72, %rsp
; CHECK-SSE-O3-NEXT:    retq
;
; CHECK-AVX2-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O3:       # %bb.0:
; CHECK-AVX2-O3-NEXT:    subq $72, %rsp
; CHECK-AVX2-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O3-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O3-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O3-NEXT:    movl $64, %edi
; CHECK-AVX2-O3-NEXT:    movl $3, %ecx
; CHECK-AVX2-O3-NEXT:    vzeroupper
; CHECK-AVX2-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O3-NEXT:    addq $72, %rsp
; CHECK-AVX2-O3-NEXT:    retq
;
; CHECK-AVX512-O3-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O3:       # %bb.0:
; CHECK-AVX512-O3-NEXT:    subq $72, %rsp
; CHECK-AVX512-O3-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O3-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O3-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O3-NEXT:    movl $64, %edi
; CHECK-AVX512-O3-NEXT:    movl $3, %ecx
; CHECK-AVX512-O3-NEXT:    vzeroupper
; CHECK-AVX512-O3-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O3-NEXT:    addq $72, %rsp
; CHECK-AVX512-O3-NEXT:    retq
;
; CHECK-SSE-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-SSE-O0:       # %bb.0:
; CHECK-SSE-O0-NEXT:    subq $72, %rsp
; CHECK-SSE-O0-NEXT:    movq %rdi, %rsi
; CHECK-SSE-O0-NEXT:    movaps %xmm0, (%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm1, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm2, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movaps %xmm3, {{[0-9]+}}(%rsp)
; CHECK-SSE-O0-NEXT:    movl $64, %edi
; CHECK-SSE-O0-NEXT:    movq %rsp, %rdx
; CHECK-SSE-O0-NEXT:    movl $3, %ecx
; CHECK-SSE-O0-NEXT:    callq __atomic_store@PLT
; CHECK-SSE-O0-NEXT:    addq $72, %rsp
; CHECK-SSE-O0-NEXT:    retq
;
; CHECK-AVX2-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX2-O0:       # %bb.0:
; CHECK-AVX2-O0-NEXT:    subq $72, %rsp
; CHECK-AVX2-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX2-O0-NEXT:    vmovups %ymm0, (%rsp)
; CHECK-AVX2-O0-NEXT:    vmovups %ymm1, {{[0-9]+}}(%rsp)
; CHECK-AVX2-O0-NEXT:    movl $64, %edi
; CHECK-AVX2-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX2-O0-NEXT:    movl $3, %ecx
; CHECK-AVX2-O0-NEXT:    vzeroupper
; CHECK-AVX2-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX2-O0-NEXT:    addq $72, %rsp
; CHECK-AVX2-O0-NEXT:    retq
;
; CHECK-AVX512-O0-LABEL: store_atomic_vec32_half_unaligned:
; CHECK-AVX512-O0:       # %bb.0:
; CHECK-AVX512-O0-NEXT:    subq $72, %rsp
; CHECK-AVX512-O0-NEXT:    movq %rdi, %rsi
; CHECK-AVX512-O0-NEXT:    vmovups %zmm0, (%rsp)
; CHECK-AVX512-O0-NEXT:    movl $64, %edi
; CHECK-AVX512-O0-NEXT:    movq %rsp, %rdx
; CHECK-AVX512-O0-NEXT:    movl $3, %ecx
; CHECK-AVX512-O0-NEXT:    vzeroupper
; CHECK-AVX512-O0-NEXT:    callq __atomic_store@PLT
; CHECK-AVX512-O0-NEXT:    addq $72, %rsp
; CHECK-AVX512-O0-NEXT:    retq
  store atomic <32 x half> %v, ptr %x release, align 4
  ret void
}