; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK64
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=i686-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK32
; RUN: opt -S %s -passes='require<libcall-lowering-info>,atomic-expand' -mtriple=x86_64-linux-gnu -mattr=+cx16,-avx | FileCheck %s --check-prefixes=CX16

; This file tests AtomicExpand's non-integer atomic type conversions and
; X86-specific expansion choices. If X86 stops using this functionality, please
; move this test to a target which still is.

define float @float_load_expand(ptr %ptr) {
; CHECK-LABEL: define float @float_load_expand(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] unordered, align 4
; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CHECK-NEXT:    ret float [[TMP2]]
;
  %res = load atomic float, ptr %ptr unordered, align 4
  ret float %res
}

define float @float_load_expand_seq_cst(ptr %ptr) {
; CHECK-LABEL: define float @float_load_expand_seq_cst(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[PTR]] seq_cst, align 4
; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CHECK-NEXT:    ret float [[TMP2]]
;
  %res = load atomic float, ptr %ptr seq_cst, align 4
  ret float %res
}

define float @float_load_expand_vol(ptr %ptr) {
; CHECK-LABEL: define float @float_load_expand_vol(
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = load atomic volatile i32, ptr [[PTR]] unordered, align 4
; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CHECK-NEXT:    ret float [[TMP2]]
;
  %res = load atomic volatile float, ptr %ptr unordered, align 4
  ret float %res
}

define float @float_load_expand_addr1(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define float @float_load_expand_addr1(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4
; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[TMP1]] to float
; CHECK-NEXT:    ret float [[TMP2]]
;
  %res = load atomic float, ptr addrspace(1) %ptr unordered, align 4
  ret float %res
}

define void @float_store_expand(ptr %ptr, float %v) {
; CHECK-LABEL: define void @float_store_expand(
; CHECK-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] unordered, align 4
; CHECK-NEXT:    ret void
;
  store atomic float %v, ptr %ptr unordered, align 4
  ret void
}

define void @float_store_expand_seq_cst(ptr %ptr, float %v) {
; CHECK-LABEL: define void @float_store_expand_seq_cst(
; CHECK-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr [[PTR]] seq_cst, align 4
; CHECK-NEXT:    ret void
;
  store atomic float %v, ptr %ptr seq_cst, align 4
  ret void
}

define void @float_store_expand_vol(ptr %ptr, float %v) {
; CHECK-LABEL: define void @float_store_expand_vol(
; CHECK-SAME: ptr [[PTR:%.*]], float [[V:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
; CHECK-NEXT:    store atomic volatile i32 [[TMP1]], ptr [[PTR]] unordered, align 4
; CHECK-NEXT:    ret void
;
  store atomic volatile float %v, ptr %ptr unordered, align 4
  ret void
}

define void @float_store_expand_addr1(ptr addrspace(1) %ptr, float %v) {
; CHECK-LABEL: define void @float_store_expand_addr1(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]], float [[V:%.*]]) {
; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[V]] to i32
; CHECK-NEXT:    store atomic i32 [[TMP1]], ptr addrspace(1) [[PTR]] unordered, align 4
; CHECK-NEXT:    ret void
;
  store atomic float %v, ptr addrspace(1) %ptr unordered, align 4
  ret void
}

define void @pointer_cmpxchg_expand(ptr %ptr, ptr %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand(
; CHECK64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst monotonic, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand(
; CHECK32-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i32 0, i32 [[TMP1]] seq_cst monotonic, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg ptr %ptr, ptr null, ptr %v seq_cst monotonic
  ret void
}

define void @pointer_cmpxchg_expand2(ptr %ptr, ptr %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand2(
; CHECK64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] release monotonic, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand2(
; CHECK32-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i32 0, i32 [[TMP1]] release monotonic, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg ptr %ptr, ptr null, ptr %v release monotonic
  ret void
}

define void @pointer_cmpxchg_expand3(ptr %ptr, ptr %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand3(
; CHECK64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand3(
; CHECK32-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i32 0, i32 [[TMP1]] seq_cst seq_cst, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg ptr %ptr, ptr null, ptr %v seq_cst seq_cst
  ret void
}

define void @pointer_cmpxchg_expand4(ptr %ptr, ptr %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand4(
; CHECK64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg weak ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand4(
; CHECK32-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg weak ptr [[PTR]], i32 0, i32 [[TMP1]] seq_cst seq_cst, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg weak ptr %ptr, ptr null, ptr %v seq_cst seq_cst
  ret void
}

define void @pointer_cmpxchg_expand5(ptr %ptr, ptr %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand5(
; CHECK64-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg volatile ptr [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand5(
; CHECK32-SAME: ptr [[PTR:%.*]], ptr [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg volatile ptr [[PTR]], i32 0, i32 [[TMP1]] seq_cst seq_cst, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr, i1 } poison, ptr [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr, i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg volatile ptr %ptr, ptr null, ptr %v seq_cst seq_cst
  ret void
}

define void @pointer_cmpxchg_expand6(ptr addrspace(1) %ptr, ptr addrspace(2) %v) {
; CHECK64-LABEL: define void @pointer_cmpxchg_expand6(
; CHECK64-SAME: ptr addrspace(1) [[PTR:%.*]], ptr addrspace(2) [[V:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(2) [[V]] to i64
; CHECK64-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 0, i64 [[TMP1]] seq_cst seq_cst, align 8
; CHECK64-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP2]], 0
; CHECK64-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
; CHECK64-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP3]] to ptr addrspace(2)
; CHECK64-NEXT:    [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
; CHECK64-NEXT:    [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @pointer_cmpxchg_expand6(
; CHECK32-SAME: ptr addrspace(1) [[PTR:%.*]], ptr addrspace(2) [[V:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(2) [[V]] to i32
; CHECK32-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 0, i32 [[TMP1]] seq_cst seq_cst, align 4
; CHECK32-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP2]], 0
; CHECK32-NEXT:    [[TMP4:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
; CHECK32-NEXT:    [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(2)
; CHECK32-NEXT:    [[TMP6:%.*]] = insertvalue { ptr addrspace(2), i1 } poison, ptr addrspace(2) [[TMP5]], 0
; CHECK32-NEXT:    [[TMP7:%.*]] = insertvalue { ptr addrspace(2), i1 } [[TMP6]], i1 [[TMP4]], 1
; CHECK32-NEXT:    ret void
;
  cmpxchg ptr addrspace(1) %ptr, ptr addrspace(2) null, ptr addrspace(2) %v seq_cst seq_cst
  ret void
}

define <2 x ptr> @atomic_vec2_ptr_align(ptr %x) nounwind {
; CHECK64-LABEL: define <2 x ptr> @atomic_vec2_ptr_align(
; CHECK64-SAME: ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
; CHECK64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <2 x i64>
; CHECK64-NEXT:    [[TMP3:%.*]] = inttoptr <2 x i64> [[TMP2]] to <2 x ptr>
; CHECK64-NEXT:    ret <2 x ptr> [[TMP3]]
;
; CHECK32-LABEL: define <2 x ptr> @atomic_vec2_ptr_align(
; CHECK32-SAME: ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK32-NEXT:    [[RET:%.*]] = load atomic <2 x ptr>, ptr [[X]] acquire, align 16
; CHECK32-NEXT:    ret <2 x ptr> [[RET]]
;
  %ret = load atomic <2 x ptr>, ptr %x acquire, align 16
  ret <2 x ptr> %ret
}

define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(ptr %x) nounwind {
; CHECK64-LABEL: define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(
; CHECK64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
; CHECK64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
; CHECK64-NEXT:    [[TMP3:%.*]] = inttoptr <4 x i32> [[TMP2]] to <4 x ptr addrspace(270)>
; CHECK64-NEXT:    ret <4 x ptr addrspace(270)> [[TMP3]]
;
; CHECK32-LABEL: define <4 x ptr addrspace(270)> @atomic_vec4_ptr_align(
; CHECK32-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK32-NEXT:    [[TMP1:%.*]] = alloca <4 x ptr addrspace(270)>, align 16
; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
; CHECK32-NEXT:    call void @__atomic_load(i32 16, ptr [[X]], ptr [[TMP1]], i32 2)
; CHECK32-NEXT:    [[TMP2:%.*]] = load <4 x ptr addrspace(270)>, ptr [[TMP1]], align 16
; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT:    ret <4 x ptr addrspace(270)> [[TMP2]]
;
  %ret = load atomic <4 x ptr addrspace(270)>, ptr %x acquire, align 16
  ret <4 x ptr addrspace(270)> %ret
}

define <2 x i16> @atomic_vec2_i16(ptr %x) nounwind {
; CHECK-LABEL: define <2 x i16> @atomic_vec2_i16(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:    [[RET:%.*]] = load atomic <2 x i16>, ptr [[X]] acquire, align 8
; CHECK-NEXT:    ret <2 x i16> [[RET]]
;
  %ret = load atomic <2 x i16>, ptr %x acquire, align 8
  ret <2 x i16> %ret
}

define <2 x half> @atomic_vec2_half(ptr %x) nounwind {
; CHECK-LABEL: define <2 x half> @atomic_vec2_half(
; CHECK-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i32, ptr [[X]] acquire, align 8
; CHECK-NEXT:    [[RET:%.*]] = bitcast i32 [[TMP1]] to <2 x half>
; CHECK-NEXT:    ret <2 x half> [[RET]]
;
  %ret = load atomic <2 x half>, ptr %x acquire, align 8
  ret <2 x half> %ret
}

define <4 x i32> @atomic_vec4_i32(ptr %x) nounwind {
; CHECK64-LABEL: define <4 x i32> @atomic_vec4_i32(
; CHECK64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
; CHECK64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x i32>
; CHECK64-NEXT:    ret <4 x i32> [[TMP2]]
;
; CHECK32-LABEL: define <4 x i32> @atomic_vec4_i32(
; CHECK32-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK32-NEXT:    [[TMP1:%.*]] = alloca <4 x i32>, align 16
; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
; CHECK32-NEXT:    call void @__atomic_load(i32 16, ptr [[X]], ptr [[TMP1]], i32 2)
; CHECK32-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 16
; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT:    ret <4 x i32> [[TMP2]]
;
  %ret = load atomic <4 x i32>, ptr %x acquire, align 16
  ret <4 x i32> %ret
}

define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
; CHECK64-LABEL: define <4 x float> @atomic_vec4_float(
; CHECK64-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[X]], i32 2)
; CHECK64-NEXT:    [[TMP2:%.*]] = bitcast i128 [[TMP1]] to <4 x float>
; CHECK64-NEXT:    ret <4 x float> [[TMP2]]
;
; CHECK32-LABEL: define <4 x float> @atomic_vec4_float(
; CHECK32-SAME: ptr [[X:%.*]]) #[[ATTR0]] {
; CHECK32-NEXT:    [[TMP1:%.*]] = alloca <4 x float>, align 16
; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
; CHECK32-NEXT:    call void @__atomic_load(i32 16, ptr [[X]], ptr [[TMP1]], i32 2)
; CHECK32-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[TMP1]], align 16
; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT:    ret <4 x float> [[TMP2]]
;
  %ret = load atomic <4 x float>, ptr %x acquire, align 16
  ret <4 x float> %ret
}

; Vector atomic loads should expand to a same-sized integer cmpxchg followed by
; a bitcast back to the original type when X86 chooses the cmpxchg expansion.

define <2 x i64> @load_v2i64_cmpxchg(ptr %p) {
; CX16-LABEL: define <2 x i64> @load_v2i64_cmpxchg(
; CX16-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CX16-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i128 0, i128 0 seq_cst seq_cst, align 16
; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <2 x i64>
; CX16-NEXT:    ret <2 x i64> [[TMP2]]
;
  %r = load atomic <2 x i64>, ptr %p seq_cst, align 16
  ret <2 x i64> %r
}

define <4 x i32> @load_v4i32_cmpxchg(ptr %p) {
; CX16-LABEL: define <4 x i32> @load_v4i32_cmpxchg(
; CX16-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CX16-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i128 0, i128 0 seq_cst seq_cst, align 16
; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <4 x i32>
; CX16-NEXT:    ret <4 x i32> [[TMP2]]
;
  %r = load atomic <4 x i32>, ptr %p seq_cst, align 16
  ret <4 x i32> %r
}

define <16 x i8> @load_v16i8_cmpxchg(ptr %p) {
; CX16-LABEL: define <16 x i8> @load_v16i8_cmpxchg(
; CX16-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CX16-NEXT:    [[TMP1:%.*]] = cmpxchg ptr [[P]], i128 0, i128 0 seq_cst seq_cst, align 16
; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT:    [[TMP2:%.*]] = bitcast i128 [[LOADED]] to <16 x i8>
; CX16-NEXT:    ret <16 x i8> [[TMP2]]
;
  %r = load atomic <16 x i8>, ptr %p seq_cst, align 16
  ret <16 x i8> %r
}

; The cmpxchg/atomicrmw synthesized when expanding an atomic load/store should
; carry over the original instruction's volatile flag and syncscope.

define i128 @load_i128_volatile_syncscope(ptr %p) {
; CHECK64-LABEL: define i128 @load_i128_volatile_syncscope(
; CHECK64-SAME: ptr [[P:%.*]]) {
; CHECK64-NEXT:    [[TMP1:%.*]] = call i128 @__atomic_load_16(ptr [[P]], i32 5)
; CHECK64-NEXT:    ret i128 [[TMP1]]
;
; CHECK32-LABEL: define i128 @load_i128_volatile_syncscope(
; CHECK32-SAME: ptr [[P:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = alloca i128, align 16
; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
; CHECK32-NEXT:    call void @__atomic_load(i32 16, ptr [[P]], ptr [[TMP1]], i32 5)
; CHECK32-NEXT:    [[TMP2:%.*]] = load i128, ptr [[TMP1]], align 16
; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT:    ret i128 [[TMP2]]
;
; CX16-LABEL: define i128 @load_i128_volatile_syncscope(
; CX16-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CX16-NEXT:    [[TMP1:%.*]] = cmpxchg volatile ptr [[P]], i128 0, i128 0 syncscope("singlethread") seq_cst seq_cst, align 16
; CX16-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP1]], 0
; CX16-NEXT:    ret i128 [[LOADED]]
;
  %v = load atomic volatile i128, ptr %p syncscope("singlethread") seq_cst, align 16
  ret i128 %v
}

define void @store_i128_volatile_syncscope(ptr %p, i128 %x) {
; CHECK64-LABEL: define void @store_i128_volatile_syncscope(
; CHECK64-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
; CHECK64-NEXT:    call void @__atomic_store_16(ptr [[P]], i128 [[X]], i32 5)
; CHECK64-NEXT:    ret void
;
; CHECK32-LABEL: define void @store_i128_volatile_syncscope(
; CHECK32-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) {
; CHECK32-NEXT:    [[TMP1:%.*]] = alloca i128, align 16
; CHECK32-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
; CHECK32-NEXT:    store i128 [[X]], ptr [[TMP1]], align 16
; CHECK32-NEXT:    call void @__atomic_store(i32 16, ptr [[P]], ptr [[TMP1]], i32 5)
; CHECK32-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
; CHECK32-NEXT:    ret void
;
; CX16-LABEL: define void @store_i128_volatile_syncscope(
; CX16-SAME: ptr [[P:%.*]], i128 [[X:%.*]]) #[[ATTR0]] {
; CX16-NEXT:    [[TMP1:%.*]] = load volatile i128, ptr [[P]], align 16
; CX16-NEXT:    br label %[[ATOMICRMW_START:.*]]
; CX16:       [[ATOMICRMW_START]]:
; CX16-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
; CX16-NEXT:    [[TMP2:%.*]] = cmpxchg volatile ptr [[P]], i128 [[LOADED]], i128 [[X]] syncscope("singlethread") seq_cst seq_cst, align 16
; CX16-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1
; CX16-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0
; CX16-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
; CX16:       [[ATOMICRMW_END]]:
; CX16-NEXT:    ret void
;
  store atomic volatile i128 %x, ptr %p syncscope("singlethread") seq_cst, align 16
  ret void
}