; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s --mtriple=wasm32 | FileCheck %s --check-prefixes=NO-ATOMICS
; RUN: llc < %s --mtriple=wasm32 -mattr=+atomics | FileCheck %s --check-prefixes=ATOMICS
; RUN: llc < %s --mtriple=wasm32 -mattr=+atomics,+relaxed-atomics | FileCheck %s --check-prefixes=RELAXED32
; RUN: llc < %s --mtriple=wasm64 -mattr=+atomics,+relaxed-atomics | FileCheck %s --check-prefixes=RELAXED64

; Currently Wasm supports a constrained set of atomic memory orderings.
; Originally it supported only sequential consistency, but now it also
; supports relaxed atomics. Weaker orderings in LLVM IR are "upgraded" to
; the next supported ordering.

target triple = "wasm32-unknown-unknown"

;===----------------------------------------------------------------------------
; Atomic loads
;===----------------------------------------------------------------------------

; The 'release' and 'acq_rel' orderings are not valid on load instructions.

define i32 @load_i32_unordered(ptr %p) {
; NO-ATOMICS-LABEL: load_i32_unordered:
; NO-ATOMICS:         .functype load_i32_unordered (i32) -> (i32)
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: load_i32_unordered:
; ATOMICS:         .functype load_i32_unordered (i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    i32.atomic.load 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: load_i32_unordered:
; RELAXED32:         .functype load_i32_unordered (i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    i32.atomic.load acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: load_i32_unordered:
; RELAXED64:         .functype load_i32_unordered (i64) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    i32.atomic.load acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %v = load atomic i32, ptr %p unordered, align 4
  ret i32 %v
}

define i32 @load_i32_monotonic(ptr %p) {
; NO-ATOMICS-LABEL: load_i32_monotonic:
; NO-ATOMICS:         .functype load_i32_monotonic (i32) -> (i32)
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: load_i32_monotonic:
; ATOMICS:         .functype load_i32_monotonic (i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    i32.atomic.load 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: load_i32_monotonic:
; RELAXED32:         .functype load_i32_monotonic (i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    i32.atomic.load acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: load_i32_monotonic:
; RELAXED64:         .functype load_i32_monotonic (i64) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    i32.atomic.load acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %v = load atomic i32, ptr %p monotonic, align 4
  ret i32 %v
}

define i32 @load_i32_acquire(ptr %p) {
; NO-ATOMICS-LABEL: load_i32_acquire:
; NO-ATOMICS:         .functype load_i32_acquire (i32) -> (i32)
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: load_i32_acquire:
; ATOMICS:         .functype load_i32_acquire (i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    i32.atomic.load 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: load_i32_acquire:
; RELAXED32:         .functype load_i32_acquire (i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    i32.atomic.load acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: load_i32_acquire:
; RELAXED64:         .functype load_i32_acquire (i64) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    i32.atomic.load acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %v = load atomic i32, ptr %p acquire, align 4
  ret i32 %v
}

define i32 @load_i32_seq_cst(ptr %p) {
; NO-ATOMICS-LABEL: load_i32_seq_cst:
; NO-ATOMICS:         .functype load_i32_seq_cst (i32) -> (i32)
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: load_i32_seq_cst:
; ATOMICS:         .functype load_i32_seq_cst (i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    i32.atomic.load 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: load_i32_seq_cst:
; RELAXED32:         .functype load_i32_seq_cst (i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    i32.atomic.load seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: load_i32_seq_cst:
; RELAXED64:         .functype load_i32_seq_cst (i64) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    i32.atomic.load seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  %v = load atomic i32, ptr %p seq_cst, align 4
  ret i32 %v
}

;===----------------------------------------------------------------------------
; Atomic stores
;===----------------------------------------------------------------------------

; The 'acquire' and 'acq_rel' orderings aren’t valid on store instructions.

define void @store_i32_unordered(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: store_i32_unordered:
; NO-ATOMICS:         .functype store_i32_unordered (i32, i32) -> ()
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: store_i32_unordered:
; ATOMICS:         .functype store_i32_unordered (i32, i32) -> ()
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.store 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: store_i32_unordered:
; RELAXED32:         .functype store_i32_unordered (i32, i32) -> ()
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.store acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: store_i32_unordered:
; RELAXED64:         .functype store_i32_unordered (i64, i32) -> ()
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.store acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  store atomic i32 %v, ptr %p unordered, align 4
  ret void
}

define void @store_i32_monotonic(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: store_i32_monotonic:
; NO-ATOMICS:         .functype store_i32_monotonic (i32, i32) -> ()
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: store_i32_monotonic:
; ATOMICS:         .functype store_i32_monotonic (i32, i32) -> ()
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.store 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: store_i32_monotonic:
; RELAXED32:         .functype store_i32_monotonic (i32, i32) -> ()
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.store acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: store_i32_monotonic:
; RELAXED64:         .functype store_i32_monotonic (i64, i32) -> ()
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.store acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  store atomic i32 %v, ptr %p monotonic, align 4
  ret void
}

define void @store_i32_release(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: store_i32_release:
; NO-ATOMICS:         .functype store_i32_release (i32, i32) -> ()
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: store_i32_release:
; ATOMICS:         .functype store_i32_release (i32, i32) -> ()
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.store 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: store_i32_release:
; RELAXED32:         .functype store_i32_release (i32, i32) -> ()
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.store acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: store_i32_release:
; RELAXED64:         .functype store_i32_release (i64, i32) -> ()
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.store acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  store atomic i32 %v, ptr %p release, align 4
  ret void
}

define void @store_i32_seq_cst(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: store_i32_seq_cst:
; NO-ATOMICS:         .functype store_i32_seq_cst (i32, i32) -> ()
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: store_i32_seq_cst:
; ATOMICS:         .functype store_i32_seq_cst (i32, i32) -> ()
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.store 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: store_i32_seq_cst:
; RELAXED32:         .functype store_i32_seq_cst (i32, i32) -> ()
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.store seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: store_i32_seq_cst:
; RELAXED64:         .functype store_i32_seq_cst (i64, i32) -> ()
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.store seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  store atomic i32 %v, ptr %p seq_cst, align 4
  ret void
}

;===----------------------------------------------------------------------------
; Atomic read-modify-writes
;===----------------------------------------------------------------------------

; Out of several binary RMW instructions, here we test 'add' as an example.
; The 'unordered' ordering is not valid on atomicrmw instructions.

define i32 @add_i32_monotonic(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: add_i32_monotonic:
; NO-ATOMICS:         .functype add_i32_monotonic (i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 2
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.add
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: add_i32_monotonic:
; ATOMICS:         .functype add_i32_monotonic (i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.rmw.add 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: add_i32_monotonic:
; RELAXED32:         .functype add_i32_monotonic (i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: add_i32_monotonic:
; RELAXED64:         .functype add_i32_monotonic (i64, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %old = atomicrmw add ptr %p, i32 %v monotonic
  ret i32 %old
}

define i32 @add_i32_acquire(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: add_i32_acquire:
; NO-ATOMICS:         .functype add_i32_acquire (i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 2
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.add
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: add_i32_acquire:
; ATOMICS:         .functype add_i32_acquire (i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.rmw.add 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: add_i32_acquire:
; RELAXED32:         .functype add_i32_acquire (i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: add_i32_acquire:
; RELAXED64:         .functype add_i32_acquire (i64, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %old = atomicrmw add ptr %p, i32 %v acquire
  ret i32 %old
}

define i32 @add_i32_release(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: add_i32_release:
; NO-ATOMICS:         .functype add_i32_release (i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 2
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.add
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: add_i32_release:
; ATOMICS:         .functype add_i32_release (i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.rmw.add 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: add_i32_release:
; RELAXED32:         .functype add_i32_release (i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: add_i32_release:
; RELAXED64:         .functype add_i32_release (i64, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %old = atomicrmw add ptr %p, i32 %v release
  ret i32 %old
}

define i32 @add_i32_acq_rel(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: add_i32_acq_rel:
; NO-ATOMICS:         .functype add_i32_acq_rel (i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 2
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.add
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: add_i32_acq_rel:
; ATOMICS:         .functype add_i32_acq_rel (i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.rmw.add 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: add_i32_acq_rel:
; RELAXED32:         .functype add_i32_acq_rel (i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: add_i32_acq_rel:
; RELAXED64:         .functype add_i32_acq_rel (i64, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.rmw.add acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %old = atomicrmw add ptr %p, i32 %v acq_rel
  ret i32 %old
}

define i32 @add_i32_seq_cst(ptr %p, i32 %v) {
; NO-ATOMICS-LABEL: add_i32_seq_cst:
; NO-ATOMICS:         .functype add_i32_seq_cst (i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 2
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.add
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: add_i32_seq_cst:
; ATOMICS:         .functype add_i32_seq_cst (i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    i32.atomic.rmw.add 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: add_i32_seq_cst:
; RELAXED32:         .functype add_i32_seq_cst (i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    i32.atomic.rmw.add seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: add_i32_seq_cst:
; RELAXED64:         .functype add_i32_seq_cst (i64, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    i32.atomic.rmw.add seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  %old = atomicrmw add ptr %p, i32 %v seq_cst
  ret i32 %old
}

; Ternary RMW instruction: cmpxchg
; The success and failure ordering arguments specify how this cmpxchg
; synchronizes with other atomic operations. Both ordering parameters must be at
; least monotonic, the ordering constraint on failure must be no stronger than
; that on success, and the failure ordering cannot be either release or acq_rel.

define i32 @cmpxchg_i32_monotonic_monotonic(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_monotonic_monotonic:
; NO-ATOMICS:         .functype cmpxchg_i32_monotonic_monotonic (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_monotonic_monotonic:
; ATOMICS:         .functype cmpxchg_i32_monotonic_monotonic (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_monotonic_monotonic:
; RELAXED32:         .functype cmpxchg_i32_monotonic_monotonic (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_monotonic_monotonic:
; RELAXED64:         .functype cmpxchg_i32_monotonic_monotonic (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new monotonic monotonic
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_acquire_monotonic(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_acquire_monotonic:
; NO-ATOMICS:         .functype cmpxchg_i32_acquire_monotonic (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_acquire_monotonic:
; ATOMICS:         .functype cmpxchg_i32_acquire_monotonic (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_acquire_monotonic:
; RELAXED32:         .functype cmpxchg_i32_acquire_monotonic (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_acquire_monotonic:
; RELAXED64:         .functype cmpxchg_i32_acquire_monotonic (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new acquire monotonic
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_release_monotonic(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_release_monotonic:
; NO-ATOMICS:         .functype cmpxchg_i32_release_monotonic (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_release_monotonic:
; ATOMICS:         .functype cmpxchg_i32_release_monotonic (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_release_monotonic:
; RELAXED32:         .functype cmpxchg_i32_release_monotonic (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_release_monotonic:
; RELAXED64:         .functype cmpxchg_i32_release_monotonic (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new release monotonic
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_acq_rel_monotonic(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_acq_rel_monotonic:
; NO-ATOMICS:         .functype cmpxchg_i32_acq_rel_monotonic (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_acq_rel_monotonic:
; ATOMICS:         .functype cmpxchg_i32_acq_rel_monotonic (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_acq_rel_monotonic:
; RELAXED32:         .functype cmpxchg_i32_acq_rel_monotonic (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_acq_rel_monotonic:
; RELAXED64:         .functype cmpxchg_i32_acq_rel_monotonic (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new acq_rel monotonic
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_seq_cst_monotonic(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_seq_cst_monotonic:
; NO-ATOMICS:         .functype cmpxchg_i32_seq_cst_monotonic (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_seq_cst_monotonic:
; ATOMICS:         .functype cmpxchg_i32_seq_cst_monotonic (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_seq_cst_monotonic:
; RELAXED32:         .functype cmpxchg_i32_seq_cst_monotonic (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_seq_cst_monotonic:
; RELAXED64:         .functype cmpxchg_i32_seq_cst_monotonic (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new seq_cst monotonic
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_acquire_acquire(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_acquire_acquire:
; NO-ATOMICS:         .functype cmpxchg_i32_acquire_acquire (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_acquire_acquire:
; ATOMICS:         .functype cmpxchg_i32_acquire_acquire (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_acquire_acquire:
; RELAXED32:         .functype cmpxchg_i32_acquire_acquire (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_acquire_acquire:
; RELAXED64:         .functype cmpxchg_i32_acquire_acquire (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new acquire acquire
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_release_acquire(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_release_acquire:
; NO-ATOMICS:         .functype cmpxchg_i32_release_acquire (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_release_acquire:
; ATOMICS:         .functype cmpxchg_i32_release_acquire (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_release_acquire:
; RELAXED32:         .functype cmpxchg_i32_release_acquire (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_release_acquire:
; RELAXED64:         .functype cmpxchg_i32_release_acquire (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new release acquire
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_acq_rel_acquire(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_acq_rel_acquire:
; NO-ATOMICS:         .functype cmpxchg_i32_acq_rel_acquire (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_acq_rel_acquire:
; ATOMICS:         .functype cmpxchg_i32_acq_rel_acquire (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_acq_rel_acquire:
; RELAXED32:         .functype cmpxchg_i32_acq_rel_acquire (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_acq_rel_acquire:
; RELAXED64:         .functype cmpxchg_i32_acq_rel_acquire (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg acqrel 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new acq_rel acquire
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

; CHECK-LABEL: cmpxchg_i32_seq_cst_acquire:
; CHECK-NEXT: .functype cmpxchg_i32_seq_cst_acquire (i32, i32, i32) -> (i32){{$}}
; CHECK: i32.atomic.rmw.cmpxchg $push0=, 0($0), $1, $2{{$}}
; CHECK-NEXT: return $pop0{{$}}
define i32 @cmpxchg_i32_seq_cst_acquire(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_seq_cst_acquire:
; NO-ATOMICS:         .functype cmpxchg_i32_seq_cst_acquire (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_seq_cst_acquire:
; ATOMICS:         .functype cmpxchg_i32_seq_cst_acquire (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_seq_cst_acquire:
; RELAXED32:         .functype cmpxchg_i32_seq_cst_acquire (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_seq_cst_acquire:
; RELAXED64:         .functype cmpxchg_i32_seq_cst_acquire (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new seq_cst acquire
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}

define i32 @cmpxchg_i32_seq_cst_seq_cst(ptr %p, i32 %exp, i32 %new) {
; NO-ATOMICS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
; NO-ATOMICS:         .functype cmpxchg_i32_seq_cst_seq_cst (i32, i32, i32) -> (i32)
; NO-ATOMICS-NEXT:    .local i32
; NO-ATOMICS-NEXT:  # %bb.0:
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    local.get 2
; NO-ATOMICS-NEXT:    local.get 0
; NO-ATOMICS-NEXT:    i32.load 0
; NO-ATOMICS-NEXT:    local.tee 3
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    local.get 1
; NO-ATOMICS-NEXT:    i32.eq
; NO-ATOMICS-NEXT:    i32.select
; NO-ATOMICS-NEXT:    i32.store 0
; NO-ATOMICS-NEXT:    local.get 3
; NO-ATOMICS-NEXT:    # fallthrough-return
;
; ATOMICS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
; ATOMICS:         .functype cmpxchg_i32_seq_cst_seq_cst (i32, i32, i32) -> (i32)
; ATOMICS-NEXT:  # %bb.0:
; ATOMICS-NEXT:    local.get 0
; ATOMICS-NEXT:    local.get 1
; ATOMICS-NEXT:    local.get 2
; ATOMICS-NEXT:    i32.atomic.rmw.cmpxchg 0
; ATOMICS-NEXT:    # fallthrough-return
;
; RELAXED32-LABEL: cmpxchg_i32_seq_cst_seq_cst:
; RELAXED32:         .functype cmpxchg_i32_seq_cst_seq_cst (i32, i32, i32) -> (i32)
; RELAXED32-NEXT:  # %bb.0:
; RELAXED32-NEXT:    local.get 0
; RELAXED32-NEXT:    local.get 1
; RELAXED32-NEXT:    local.get 2
; RELAXED32-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED32-NEXT:    # fallthrough-return
;
; RELAXED64-LABEL: cmpxchg_i32_seq_cst_seq_cst:
; RELAXED64:         .functype cmpxchg_i32_seq_cst_seq_cst (i64, i32, i32) -> (i32)
; RELAXED64-NEXT:  # %bb.0:
; RELAXED64-NEXT:    local.get 0
; RELAXED64-NEXT:    local.get 1
; RELAXED64-NEXT:    local.get 2
; RELAXED64-NEXT:    i32.atomic.rmw.cmpxchg seqcst 0
; RELAXED64-NEXT:    # fallthrough-return
  %pair = cmpxchg ptr %p, i32 %exp, i32 %new seq_cst seq_cst
  %old = extractvalue { i32, i1 } %pair, 0
  ret i32 %old
}