; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ ; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %} ; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %} define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: xchg_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<14>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r5, [xchg_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r6, %rd2; ; SM90-NEXT: and.b32 %r7, %r6, 3; ; SM90-NEXT: shl.b32 %r1, %r7, 3; ; SM90-NEXT: mov.b32 %r8, 255; ; SM90-NEXT: shl.b32 %r9, %r8, %r1; ; SM90-NEXT: not.b32 %r2, %r9; ; SM90-NEXT: shl.b32 %r3, %r5, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1]; ; SM90-NEXT: $L__BB0_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r13, %r2; ; SM90-NEXT: or.b32 %r11, %r10, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11; ; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13; ; SM90-NEXT: mov.b32 %r13, %r4; ; SM90-NEXT: @%p1 bra $L__BB0_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r12, %r4, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r12; ; SM90-NEXT: ret; %retval = atomicrmw xchg ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @xchg_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: xchg_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<14>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r5, [xchg_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r6, %rd2; ; SM90-NEXT: and.b32 %r7, %r6, 3; ; SM90-NEXT: shl.b32 %r1, %r7, 3; ; SM90-NEXT: mov.b32 %r8, 65535; ; SM90-NEXT: shl.b32 %r9, %r8, %r1; ; SM90-NEXT: not.b32 %r2, %r9; ; SM90-NEXT: shl.b32 %r3, %r5, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r13, [%rd1]; ; SM90-NEXT: $L__BB1_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r13, %r2; ; SM90-NEXT: or.b32 %r11, %r10, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r4, [%rd1], %r13, %r11; ; SM90-NEXT: setp.ne.b32 %p1, %r4, %r13; ; SM90-NEXT: mov.b32 %r13, %r4; ; SM90-NEXT: @%p1 bra $L__BB1_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r12, %r4, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r12; ; SM90-NEXT: ret; %retval = atomicrmw xchg ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @xchg_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: xchg_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [xchg_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.exch.b32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw xchg ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @xchg_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: xchg_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xchg_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [xchg_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.exch.b64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw xchg ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @add_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: add_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r6, [add_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB4_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB4_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @add_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: add_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r6, [add_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 65535; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB5_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB5_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @add_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @sub_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: sub_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r6, [sub_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB8_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: sub.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB8_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw sub ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @sub_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: sub_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r6, [sub_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 65535; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB9_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: sub.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB9_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw sub ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @sub_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: sub_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<4>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [sub_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: neg.s32 %r2, %r1; ; SM90-NEXT: atom.acq_rel.cta.global.add.u32 %r3, [%rd1], %r2; ; SM90-NEXT: st.param.b32 [func_retval0], %r3; ; SM90-NEXT: ret; %retval = atomicrmw sub ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @sub_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: sub_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<5>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [sub_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [sub_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: neg.s64 %rd3, %rd2; ; SM90-NEXT: atom.acq_rel.cta.global.add.u64 %rd4, [%rd1], %rd3; ; SM90-NEXT: st.param.b64 [func_retval0], %rd4; ; SM90-NEXT: ret; %retval = atomicrmw sub ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @and_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: and_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<12>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r1, [and_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: mov.b32 %r5, 255; ; SM90-NEXT: shl.b32 %r6, %r5, %r4; ; SM90-NEXT: not.b32 %r7, %r6; ; SM90-NEXT: shl.b32 %r8, %r1, %r4; ; SM90-NEXT: or.b32 %r9, %r8, %r7; ; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9; ; SM90-NEXT: shr.u32 %r11, %r10, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r11; ; SM90-NEXT: ret; %retval = atomicrmw and ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @and_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: and_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<12>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r1, [and_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: mov.b32 %r5, 65535; ; SM90-NEXT: shl.b32 %r6, %r5, %r4; ; SM90-NEXT: not.b32 %r7, %r6; ; SM90-NEXT: shl.b32 %r8, %r1, %r4; ; SM90-NEXT: or.b32 %r9, %r8, %r7; ; SM90-NEXT: atom.relaxed.cta.global.and.b32 %r10, [%rd2], %r9; ; SM90-NEXT: shr.u32 %r11, %r10, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r11; ; SM90-NEXT: ret; %retval = atomicrmw and ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @and_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: and_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [and_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.and.b32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw and ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @and_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: and_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [and_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [and_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.and.b64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw and ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @nand_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: nand_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r6, [nand_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB16_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB16_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @nand_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: nand_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r6, [nand_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 65535; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB17_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB17_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @nand_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB18_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB18_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB19_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB19_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @or_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: or_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<8>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r1, [or_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: shl.b32 %r5, %r1, %r4; ; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5; ; SM90-NEXT: shr.u32 %r7, %r6, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r7; ; SM90-NEXT: ret; %retval = atomicrmw or ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @or_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: or_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<8>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r1, [or_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: shl.b32 %r5, %r1, %r4; ; SM90-NEXT: atom.relaxed.cta.global.or.b32 %r6, [%rd2], %r5; ; SM90-NEXT: shr.u32 %r7, %r6, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r7; ; SM90-NEXT: ret; %retval = atomicrmw or ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @or_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: or_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [or_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.or.b32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw or ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @or_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: or_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [or_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [or_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.or.b64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw or ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @xor_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: xor_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<8>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r1, [xor_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: shl.b32 %r5, %r1, %r4; ; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5; ; SM90-NEXT: shr.u32 %r7, %r6, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r7; ; SM90-NEXT: ret; %retval = atomicrmw xor ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @xor_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: xor_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<8>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b16 %r1, [xor_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd2, %rd1, -4; ; SM90-NEXT: cvt.u32.u64 %r2, %rd1; ; SM90-NEXT: and.b32 %r3, %r2, 3; ; SM90-NEXT: shl.b32 %r4, %r3, 3; ; SM90-NEXT: shl.b32 %r5, %r1, %r4; ; SM90-NEXT: atom.relaxed.cta.global.xor.b32 %r6, [%rd2], %r5; ; SM90-NEXT: shr.u32 %r7, %r6, %r4; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r7; ; SM90-NEXT: ret; %retval = atomicrmw xor ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @xor_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: xor_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [xor_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.xor.b32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw xor ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @xor_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: xor_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [xor_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [xor_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.xor.b64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw xor ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @max_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: max_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [max_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: cvt.s16.s8 %rs3, %rs1; ; SM90-NEXT: $L__BB28_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r15, %r1; ; SM90-NEXT: cvt.s8.s32 %rs2, %r8; ; SM90-NEXT: max.s16 %rs4, %rs2, %rs3; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: and.b32 %r10, %r9, 255; ; SM90-NEXT: shl.b32 %r11, %r10, %r1; ; SM90-NEXT: and.b32 %r12, %r15, %r2; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15; ; SM90-NEXT: mov.b32 %r15, %r3; ; SM90-NEXT: @%p1 bra $L__BB28_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw max ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @max_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: max_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [max_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB29_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: max.s16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB29_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw max ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @max_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: max_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [max_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.max.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw max ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @max_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: max_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [max_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [max_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.max.s64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw max ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @min_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: min_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [min_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: cvt.s16.s8 %rs3, %rs1; ; SM90-NEXT: $L__BB32_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r15, %r1; ; SM90-NEXT: cvt.s8.s32 %rs2, %r8; ; SM90-NEXT: min.s16 %rs4, %rs2, %rs3; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: and.b32 %r10, %r9, 255; ; SM90-NEXT: shl.b32 %r11, %r10, %r1; ; SM90-NEXT: and.b32 %r12, %r15, %r2; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r15; ; SM90-NEXT: mov.b32 %r15, %r3; ; SM90-NEXT: @%p1 bra $L__BB32_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @min_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: min_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [min_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB33_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: min.s16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB33_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @min_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @min_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: min_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [min_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.min.s64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @umax_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: umax_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [umax_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB36_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: max.u16 %rs4, %rs3, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB36_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @umax_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: umax_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [umax_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB37_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: max.u16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB37_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @umax_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @umax_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: umax_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [umax_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.max.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @umin_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: umin_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [umin_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB40_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: min.u16 %rs4, %rs3, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB40_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw umin ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @umin_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: umin_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [umin_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB41_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: min.u16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB41_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw umin ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @umin_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: umin_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umin_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.min.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umin ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @umin_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: umin_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umin_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [umin_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.min.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw umin ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @uinc_wrap_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: uinc_wrap_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b16 %rs<6>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [uinc_wrap_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB44_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r15, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: add.s16 %rs4, %rs2, 1; ; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1; ; SM90-NEXT: selp.b16 %rs5, 0, %rs4, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs5; ; SM90-NEXT: and.b32 %r10, %r9, 255; ; SM90-NEXT: shl.b32 %r11, %r10, %r1; ; SM90-NEXT: and.b32 %r12, %r15, %r2; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15; ; SM90-NEXT: mov.b32 %r15, %r3; ; SM90-NEXT: @%p2 bra $L__BB44_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw uinc_wrap ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @uinc_wrap_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: uinc_wrap_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [uinc_wrap_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB45_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: add.s16 %rs3, %rs2, 1; ; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1; ; SM90-NEXT: selp.b16 %rs4, 0, %rs3, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p2 bra $L__BB45_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw uinc_wrap ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @uinc_wrap_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: uinc_wrap_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [uinc_wrap_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [uinc_wrap_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.inc.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw uinc_wrap ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @uinc_wrap_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: uinc_wrap_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [uinc_wrap_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [uinc_wrap_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB47_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s64 %rd4, %rd6, 1; ; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd3; ; SM90-NEXT: selp.b64 %rd5, 0, %rd4, %p1; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p2, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p2 bra $L__BB47_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw uinc_wrap ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @udec_wrap_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: udec_wrap_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<4>; ; SM90-NEXT: .reg .b16 %rs<7>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [udec_wrap_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB48_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r15, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: add.s16 %rs4, %rs2, -1; ; SM90-NEXT: setp.eq.b16 %p1, %rs3, 0; ; SM90-NEXT: setp.gt.u16 %p2, %rs3, %rs1; ; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p2; ; SM90-NEXT: selp.b16 %rs6, %rs1, %rs5, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs6; ; SM90-NEXT: and.b32 %r10, %r9, 255; ; SM90-NEXT: shl.b32 %r11, %r10, %r1; ; SM90-NEXT: and.b32 %r12, %r15, %r2; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p3, %r3, %r15; ; SM90-NEXT: mov.b32 %r15, %r3; ; SM90-NEXT: @%p3 bra $L__BB48_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw udec_wrap ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @udec_wrap_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: udec_wrap_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<4>; ; SM90-NEXT: .reg .b16 %rs<6>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [udec_wrap_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB49_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: add.s16 %rs3, %rs2, -1; ; SM90-NEXT: setp.eq.b16 %p1, %rs2, 0; ; SM90-NEXT: setp.gt.u16 %p2, %rs2, %rs1; ; SM90-NEXT: selp.b16 %rs4, %rs1, %rs3, %p2; ; SM90-NEXT: selp.b16 %rs5, %rs1, %rs4, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs5; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p3, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p3 bra $L__BB49_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw udec_wrap ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @udec_wrap_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: udec_wrap_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [udec_wrap_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [udec_wrap_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.dec.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw udec_wrap ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @udec_wrap_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: udec_wrap_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<4>; ; SM90-NEXT: .reg .b64 %rd<8>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [udec_wrap_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [udec_wrap_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd7, [%rd2]; ; SM90-NEXT: $L__BB51_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s64 %rd4, %rd7, -1; ; SM90-NEXT: setp.eq.b64 %p1, %rd7, 0; ; SM90-NEXT: setp.gt.u64 %p2, %rd7, %rd3; ; SM90-NEXT: selp.b64 %rd5, %rd3, %rd4, %p2; ; SM90-NEXT: selp.b64 %rd6, %rd3, %rd5, %p1; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd7, %rd6; ; SM90-NEXT: setp.ne.b64 %p3, %rd1, %rd7; ; SM90-NEXT: mov.b64 %rd7, %rd1; ; SM90-NEXT: @%p3 bra $L__BB51_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw udec_wrap ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @usub_cond_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: usub_cond_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b16 %rs<6>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [usub_cond_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB52_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r15, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: setp.ge.u16 %p1, %rs3, %rs1; ; SM90-NEXT: sub.s16 %rs4, %rs2, %rs1; ; SM90-NEXT: selp.b16 %rs5, %rs4, %rs2, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs5; ; SM90-NEXT: and.b32 %r10, %r9, 255; ; SM90-NEXT: shl.b32 %r11, %r10, %r1; ; SM90-NEXT: and.b32 %r12, %r15, %r2; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p2, %r3, %r15; ; SM90-NEXT: mov.b32 %r15, %r3; ; SM90-NEXT: @%p2 bra $L__BB52_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw usub_cond ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @usub_cond_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: usub_cond_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [usub_cond_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB53_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: setp.ge.u16 %p1, %rs2, %rs1; ; SM90-NEXT: sub.s16 %rs3, %rs2, %rs1; ; SM90-NEXT: selp.b16 %rs4, %rs3, %rs2, %p1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p2, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p2 bra $L__BB53_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw usub_cond ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @usub_cond_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: usub_cond_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [usub_cond_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [usub_cond_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB54_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: setp.ge.u32 %p1, %r5, %r2; ; SM90-NEXT: sub.s32 %r3, %r5, %r2; ; SM90-NEXT: selp.b32 %r4, %r3, %r5, %p1; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p2, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p2 bra $L__BB54_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw usub_cond ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @usub_cond_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: usub_cond_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<3>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [usub_cond_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_cond_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB55_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: setp.ge.u64 %p1, %rd6, %rd3; ; SM90-NEXT: sub.s64 %rd4, %rd6, %rd3; ; SM90-NEXT: selp.b64 %rd5, %rd4, %rd6, %p1; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p2, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p2 bra $L__BB55_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw usub_cond ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i8 @usub_sat_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: usub_sat_acq_rel_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<6>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b8 %rs1, [usub_sat_acq_rel_i8_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 255; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB56_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: and.b16 %rs3, %rs2, 255; ; SM90-NEXT: max.u16 %rs4, %rs3, %rs1; ; SM90-NEXT: sub.s16 %rs5, %rs4, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs5; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB56_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw usub_sat ptr addrspace(1) %addr, i8 %val syncscope("block") acq_rel ret i8 %retval } define i16 @usub_sat_acq_rel_i16_global_cta(ptr addrspace(1) %addr, i16 %val) { ; SM90-LABEL: usub_sat_acq_rel_i16_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<5>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [usub_sat_acq_rel_i16_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_i16_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB57_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: max.u16 %rs3, %rs2, %rs1; ; SM90-NEXT: sub.s16 %rs4, %rs3, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs4; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB57_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw usub_sat ptr addrspace(1) %addr, i16 %val syncscope("block") acq_rel ret i16 %retval } define i32 @usub_sat_acq_rel_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: usub_sat_acq_rel_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [usub_sat_acq_rel_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [usub_sat_acq_rel_i32_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB58_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: max.u32 %r3, %r5, %r2; ; SM90-NEXT: sub.s32 %r4, %r3, %r2; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB58_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw usub_sat ptr addrspace(1) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: usub_sat_acq_rel_i64_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [usub_sat_acq_rel_i64_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [usub_sat_acq_rel_i64_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB59_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: max.u64 %rd4, %rd6, %rd3; ; SM90-NEXT: sub.s64 %rd5, %rd4, %rd3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB59_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw usub_sat ptr addrspace(1) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB60_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .b32 %r<3>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fsub_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB61_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fsub_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB61_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fmin_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: min.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB62_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmin_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB62_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fmax_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: max.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB63_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmax_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB63_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: min.NaN.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB64_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fminimum_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB64_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: max.NaN.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB65_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmaximum_acq_rel_float_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB65_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define double @fadd_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define double @fsub_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fsub_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<6>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [fsub_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2]; ; SM90-NEXT: $L__BB67_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: sub.rn.f64 %rd4, %rd5, %rd3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd5; ; SM90-NEXT: mov.b64 %rd5, %rd1; ; SM90-NEXT: @%p1 bra $L__BB67_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define double @fmin_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fmin_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<6>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [fmin_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2]; ; SM90-NEXT: $L__BB68_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: min.f64 %rd4, %rd5, %rd3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd5; ; SM90-NEXT: mov.b64 %rd5, %rd1; ; SM90-NEXT: @%p1 bra $L__BB68_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define double @fmax_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fmax_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<6>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [fmax_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd5, [%rd2]; ; SM90-NEXT: $L__BB69_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: max.f64 %rd4, %rd5, %rd3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd5, %rd4; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd5; ; SM90-NEXT: mov.b64 %rd5, %rd1; ; SM90-NEXT: @%p1 bra $L__BB69_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define double @fminimum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fminimum_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<6>; ; SM90-NEXT: .reg .b64 %rd<10>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [fminimum_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd2]; ; SM90-NEXT: setp.eq.b64 %p3, %rd3, -9223372036854775808; ; SM90-NEXT: $L__BB70_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd3; ; SM90-NEXT: min.f64 %rd4, %rd9, %rd3; ; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1; ; SM90-NEXT: setp.eq.b64 %p2, %rd9, -9223372036854775808; ; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2; ; SM90-NEXT: selp.f64 %rd7, %rd3, %rd6, %p3; ; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000; ; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd9, %rd8; ; SM90-NEXT: setp.ne.b64 %p5, %rd1, %rd9; ; SM90-NEXT: mov.b64 %rd9, %rd1; ; SM90-NEXT: @%p5 bra $L__BB70_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fmaximum_acq_rel_double_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<6>; ; SM90-NEXT: .reg .b64 %rd<10>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [fmaximum_acq_rel_double_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_double_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b64 %rd9, [%rd2]; ; SM90-NEXT: setp.eq.b64 %p3, %rd3, 0; ; SM90-NEXT: $L__BB71_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: setp.nan.f64 %p1, %rd9, %rd3; ; SM90-NEXT: max.f64 %rd4, %rd9, %rd3; ; SM90-NEXT: selp.f64 %rd5, 0d7FF8000000000000, %rd4, %p1; ; SM90-NEXT: setp.eq.b64 %p2, %rd9, 0; ; SM90-NEXT: selp.f64 %rd6, %rd9, %rd5, %p2; ; SM90-NEXT: selp.f64 %rd7, %rd3, %rd6, %p3; ; SM90-NEXT: setp.eq.f64 %p4, %rd5, 0d0000000000000000; ; SM90-NEXT: selp.f64 %rd8, %rd7, %rd5, %p4; ; SM90-NEXT: atom.relaxed.cta.global.cas.b64 %rd1, [%rd2], %rd9, %rd8; ; SM90-NEXT: setp.ne.b64 %p5, %rd1, %rd9; ; SM90-NEXT: mov.b64 %rd9, %rd1; ; SM90-NEXT: @%p5 bra $L__BB71_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, double %val syncscope("block") acq_rel ret double %retval } define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .b16 %rs<3>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB72_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fsub_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; ; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; ; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB73_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fsub_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB73_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fmin_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; ; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NOFTZ-NEXT: min.f16 %rs3, %rs2, %rs1; ; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB74_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmin_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: min.ftz.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB74_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fmax_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; ; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NOFTZ-NEXT: max.f16 %rs3, %rs2, %rs1; ; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB75_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmax_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: max.ftz.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB75_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; ; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NOFTZ-NEXT: min.NaN.f16 %rs3, %rs2, %rs1; ; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB76_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fminimum_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB76_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { ; SM90-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; ; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NOFTZ-NEXT: max.NaN.f16 %rs3, %rs2, %rs1; ; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB77_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fmaximum_acq_rel_half_global_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b16 %rs<4>; ; SM90-FTZ-NEXT: .reg .b32 %r<15>; ; SM90-FTZ-NEXT: .reg .b64 %rd<3>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; ; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; ; SM90-FTZ-NEXT: mov.b32 %r6, 65535; ; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-FTZ-NEXT: not.b32 %r2, %r7; ; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-FTZ-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1; ; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; ; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; ; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-FTZ-NEXT: mov.b32 %r14, %r3; ; SM90-FTZ-NEXT: @%p1 bra $L__BB77_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fadd_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .b16 %rs<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.bf16 %rs2, [%rd1], %rs1; ; SM90-NEXT: st.param.b16 [func_retval0], %rs2; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fsub_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB79_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: sub.rn.bf16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB79_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fmin_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB80_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: min.bf16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB80_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fmax_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB81_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: max.bf16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB81_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fminimum_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB82_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: min.NaN.bf16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB82_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { ; SM90-LABEL: fmaximum_acq_rel_bfloat_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b16 %rs<4>; ; SM90-NEXT: .reg .b32 %r<15>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r4, %rd2; ; SM90-NEXT: and.b32 %r5, %r4, 3; ; SM90-NEXT: shl.b32 %r1, %r5, 3; ; SM90-NEXT: mov.b32 %r6, 65535; ; SM90-NEXT: shl.b32 %r7, %r6, %r1; ; SM90-NEXT: not.b32 %r2, %r7; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; ; SM90-NEXT: $L__BB83_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: shr.u32 %r8, %r14, %r1; ; SM90-NEXT: cvt.u16.u32 %rs2, %r8; ; SM90-NEXT: max.NaN.bf16 %rs3, %rs2, %rs1; ; SM90-NEXT: cvt.u32.u16 %r9, %rs3; ; SM90-NEXT: shl.b32 %r10, %r9, %r1; ; SM90-NEXT: and.b32 %r11, %r14, %r2; ; SM90-NEXT: or.b32 %r12, %r11, %r10; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; ; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; ; SM90-NEXT: mov.b32 %r14, %r3; ; SM90-NEXT: @%p1 bra $L__BB83_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r13, %r3, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b16 [func_retval0], %r13; ; SM90-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define i8 @add_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: add_monotonic_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_monotonic_i8_global_cta_param_0]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: ld.param.b8 %r6, [add_monotonic_i8_global_cta_param_1]; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB84_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB84_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i8 %val syncscope("block") monotonic ret i8 %retval } define i8 @add_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: add_acquire_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_acquire_i8_global_cta_param_0]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: ld.param.b8 %r6, [add_acquire_i8_global_cta_param_1]; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB85_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB85_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i8 %val syncscope("block") acquire ret i8 %retval } define i8 @add_release_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: add_release_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_release_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r6, [add_release_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB86_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB86_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i8 %val syncscope("block") release ret i8 %retval } define i8 @add_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: add_seq_cst_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<16>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [add_seq_cst_i8_global_cta_param_0]; ; SM90-NEXT: fence.sc.cta; ; SM90-NEXT: ld.param.b8 %r6, [add_seq_cst_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r15, [%rd1]; ; SM90-NEXT: $L__BB87_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: add.s32 %r10, %r15, %r4; ; SM90-NEXT: and.b32 %r11, %r10, %r2; ; SM90-NEXT: and.b32 %r12, %r15, %r3; ; SM90-NEXT: or.b32 %r13, %r12, %r11; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r13; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r15; ; SM90-NEXT: mov.b32 %r15, %r5; ; SM90-NEXT: @%p1 bra $L__BB87_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r14, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r14; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i8 %val syncscope("block") seq_cst ret i8 %retval } define i32 @add_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_monotonic_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_monotonic_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_monotonic_i32_global_cta_param_1]; ; SM90-NEXT: atom.relaxed.cta.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") monotonic ret i32 %retval } define i32 @add_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_acquire_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acquire_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acquire_i32_global_cta_param_1]; ; SM90-NEXT: atom.acquire.cta.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") acquire ret i32 %retval } define i32 @add_release_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_release_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_release_i32_global_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_release_i32_global_cta_param_1]; ; SM90-NEXT: atom.release.cta.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") release ret i32 %retval } define i32 @add_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_seq_cst_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_seq_cst_i32_global_cta_param_0]; ; SM90-NEXT: fence.sc.cta; ; SM90-NEXT: ld.param.b32 %r1, [add_seq_cst_i32_global_cta_param_1]; ; SM90-NEXT: atom.acquire.cta.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("block") seq_cst ret i32 %retval } define i8 @nand_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: nand_monotonic_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_monotonic_i8_global_cta_param_0]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: ld.param.b8 %r6, [nand_monotonic_i8_global_cta_param_1]; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB92_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB92_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i8 %val syncscope("block") monotonic ret i8 %retval } define i8 @nand_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: nand_acquire_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_acquire_i8_global_cta_param_0]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: ld.param.b8 %r6, [nand_acquire_i8_global_cta_param_1]; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB93_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB93_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i8 %val syncscope("block") acquire ret i8 %retval } define i8 @nand_release_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: nand_release_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_release_i8_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.param.b8 %r6, [nand_release_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB94_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB94_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i8 %val syncscope("block") release ret i8 %retval } define i8 @nand_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: nand_seq_cst_i8_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<17>; ; SM90-NEXT: .reg .b64 %rd<3>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd2, [nand_seq_cst_i8_global_cta_param_0]; ; SM90-NEXT: fence.sc.cta; ; SM90-NEXT: ld.param.b8 %r6, [nand_seq_cst_i8_global_cta_param_1]; ; SM90-NEXT: and.b64 %rd1, %rd2, -4; ; SM90-NEXT: cvt.u32.u64 %r7, %rd2; ; SM90-NEXT: and.b32 %r8, %r7, 3; ; SM90-NEXT: shl.b32 %r1, %r8, 3; ; SM90-NEXT: mov.b32 %r9, 255; ; SM90-NEXT: shl.b32 %r2, %r9, %r1; ; SM90-NEXT: not.b32 %r3, %r2; ; SM90-NEXT: shl.b32 %r4, %r6, %r1; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r16, [%rd1]; ; SM90-NEXT: $L__BB95_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r10, %r16, %r4; ; SM90-NEXT: not.b32 %r11, %r10; ; SM90-NEXT: and.b32 %r12, %r11, %r2; ; SM90-NEXT: and.b32 %r13, %r16, %r3; ; SM90-NEXT: or.b32 %r14, %r13, %r12; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r14; ; SM90-NEXT: setp.ne.b32 %p1, %r5, %r16; ; SM90-NEXT: mov.b32 %r16, %r5; ; SM90-NEXT: @%p1 bra $L__BB95_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: shr.u32 %r15, %r5, %r1; ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r15; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i8 %val syncscope("block") seq_cst ret i8 %retval } define i32 @nand_monotonic_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_monotonic_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_monotonic_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_monotonic_i32_global_cta_param_0]; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB96_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB96_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("block") monotonic ret i32 %retval } define i32 @nand_acquire_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_acquire_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acquire_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acquire_i32_global_cta_param_0]; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB97_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB97_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("block") acquire ret i32 %retval } define i32 @nand_release_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_release_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_release_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_release_i32_global_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB98_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB98_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("block") release ret i32 %retval } define i32 @nand_seq_cst_i32_global_cta(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_seq_cst_i32_global_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_seq_cst_i32_global_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_seq_cst_i32_global_cta_param_0]; ; SM90-NEXT: fence.sc.cta; ; SM90-NEXT: ld.relaxed.cta.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB99_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB99_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("block") seq_cst ret i32 %retval } define i32 @add_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_global_sys( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_sys_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_sys_param_1]; ; SM90-NEXT: atom.acq_rel.sys.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_global_sys( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_sys_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_sys_param_1]; ; SM90-NEXT: atom.acq_rel.sys.global.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("") acq_rel ret i64 %retval } define i32 @min_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_global_sys( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_sys_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_sys_param_1]; ; SM90-NEXT: atom.acq_rel.sys.global.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel ret i32 %retval } define i32 @umax_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_global_sys( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_sys_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_sys_param_1]; ; SM90-NEXT: atom.acq_rel.sys.global.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel ret i32 %retval } define i32 @nand_acq_rel_i32_global_sys(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_global_sys( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_global_sys_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_global_sys_param_0]; ; SM90-NEXT: fence.release.sys; ; SM90-NEXT: ld.relaxed.sys.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB104_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB104_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.sys; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_global_sys( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_global_sys_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_global_sys_param_0]; ; SM90-NEXT: fence.release.sys; ; SM90-NEXT: ld.relaxed.sys.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB105_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.sys.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB105_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.sys; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i64 %val syncscope("") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_sys( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; ; SM90-NOFTZ-NEXT: fence.release.sys; ; SM90-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB106_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.sys; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_global_sys( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .b32 %r<3>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; ; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; ; SM90-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel ret float %retval } define double @fadd_acq_rel_double_global_sys(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_global_sys( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_sys_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_sys_param_1]; ; SM90-NEXT: atom.acq_rel.sys.global.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("") acq_rel ret double %retval } define i32 @add_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_global_cluster( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_cluster_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_cluster_param_1]; ; SM90-NEXT: atom.acq_rel.cluster.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_global_cluster( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_cluster_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_cluster_param_1]; ; SM90-NEXT: atom.acq_rel.cluster.global.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("cluster") acq_rel ret i64 %retval } define i32 @min_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_global_cluster( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_cluster_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_cluster_param_1]; ; SM90-NEXT: atom.acq_rel.cluster.global.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel ret i32 %retval } define i32 @umax_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_global_cluster( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_cluster_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_cluster_param_1]; ; SM90-NEXT: atom.acq_rel.cluster.global.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel ret i32 %retval } define i32 @nand_acq_rel_i32_global_cluster(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_global_cluster( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_global_cluster_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_global_cluster_param_0]; ; SM90-NEXT: fence.release.cluster; ; SM90-NEXT: ld.relaxed.cluster.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB112_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB112_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cluster; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("cluster") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_global_cluster( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_global_cluster_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_global_cluster_param_0]; ; SM90-NEXT: fence.release.cluster; ; SM90-NEXT: ld.relaxed.cluster.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB113_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.cluster.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB113_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cluster; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i64 %val syncscope("cluster") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cluster; ; SM90-NOFTZ-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB114_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cluster; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cluster( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .b32 %r<3>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; ; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; ; SM90-FTZ-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel ret float %retval } define double @fadd_acq_rel_double_global_cluster(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_global_cluster( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_cluster_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_cluster_param_1]; ; SM90-NEXT: atom.acq_rel.cluster.global.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("cluster") acq_rel ret double %retval } define i32 @add_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_global_gpu( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_global_gpu_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_global_gpu_param_1]; ; SM90-NEXT: atom.acq_rel.gpu.global.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_global_gpu( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_global_gpu_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_global_gpu_param_1]; ; SM90-NEXT: atom.acq_rel.gpu.global.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(1) %addr, i64 %val syncscope("device") acq_rel ret i64 %retval } define i32 @min_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_global_gpu( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_global_gpu_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_global_gpu_param_1]; ; SM90-NEXT: atom.acq_rel.gpu.global.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel ret i32 %retval } define i32 @umax_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_global_gpu( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_global_gpu_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_global_gpu_param_1]; ; SM90-NEXT: atom.acq_rel.gpu.global.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel ret i32 %retval } define i32 @nand_acq_rel_i32_global_gpu(ptr addrspace(1) %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_global_gpu( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_global_gpu_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_global_gpu_param_0]; ; SM90-NEXT: fence.release.gpu; ; SM90-NEXT: ld.relaxed.gpu.global.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB120_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB120_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.gpu; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i32 %val syncscope("device") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_global_gpu( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_global_gpu_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_global_gpu_param_0]; ; SM90-NEXT: fence.release.gpu; ; SM90-NEXT: ld.relaxed.gpu.global.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB121_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.gpu.global.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB121_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.gpu; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(1) %addr, i64 %val syncscope("device") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; ; SM90-NOFTZ-NEXT: fence.release.gpu; ; SM90-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB122_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.gpu; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_global_gpu( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .b32 %r<3>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; ; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; ; SM90-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel ret float %retval } define double @fadd_acq_rel_double_global_gpu(ptr addrspace(1) %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_global_gpu( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_global_gpu_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_global_gpu_param_1]; ; SM90-NEXT: atom.acq_rel.gpu.global.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, double %val syncscope("device") acq_rel ret double %retval } define i32 @add_acq_rel_i32_generic_cta(ptr %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_generic_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_generic_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_generic_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_generic_cta(ptr %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_generic_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_generic_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_generic_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i32 @min_acq_rel_i32_generic_cta(ptr %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_generic_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_generic_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_generic_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i32 @umax_acq_rel_i32_generic_cta(ptr %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_generic_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_generic_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_generic_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i32 @nand_acq_rel_i32_generic_cta(ptr %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_generic_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_generic_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_generic_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB128_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB128_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_generic_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_generic_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_generic_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB129_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.cta.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB129_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .pred %p<2>; ; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; ; SM90-NOFTZ-NEXT: fence.release.cta; ; SM90-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; ; SM90-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start ; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; ; SM90-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; ; SM90-NOFTZ-NEXT: @%p1 bra $L__BB130_1; ; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NOFTZ-NEXT: fence.acquire.cta; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_generic_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB130_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel ret float %retval } define double @fadd_acq_rel_double_generic_cta(ptr %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_generic_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_generic_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_generic_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr %addr, double %val syncscope("block") acq_rel ret double %retval } define i32 @add_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) { ; SM90-LABEL: add_acq_rel_i32_shared_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i32_shared_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [add_acq_rel_i32_shared_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.shared.add.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @add_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) { ; SM90-LABEL: add_acq_rel_i64_shared_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [add_acq_rel_i64_shared_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [add_acq_rel_i64_shared_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.shared.add.u64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw add ptr addrspace(3) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define i32 @min_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) { ; SM90-LABEL: min_acq_rel_i32_shared_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [min_acq_rel_i32_shared_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [min_acq_rel_i32_shared_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.shared.min.s32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw min ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i32 @umax_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) { ; SM90-LABEL: umax_acq_rel_i32_shared_cta( ; SM90: { ; SM90-NEXT: .reg .b32 %r<3>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [umax_acq_rel_i32_shared_cta_param_0]; ; SM90-NEXT: ld.param.b32 %r1, [umax_acq_rel_i32_shared_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.shared.max.u32 %r2, [%rd1], %r1; ; SM90-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NEXT: ret; %retval = atomicrmw umax ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i32 @nand_acq_rel_i32_shared_cta(ptr addrspace(3) %addr, i32 %val) { ; SM90-LABEL: nand_acq_rel_i32_shared_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b32 %r<6>; ; SM90-NEXT: .reg .b64 %rd<2>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b32 %r2, [nand_acq_rel_i32_shared_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd1, [nand_acq_rel_i32_shared_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.shared.b32 %r5, [%rd1]; ; SM90-NEXT: $L__BB136_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b32 %r3, %r5, %r2; ; SM90-NEXT: not.b32 %r4, %r3; ; SM90-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r5, %r4; ; SM90-NEXT: setp.ne.b32 %p1, %r1, %r5; ; SM90-NEXT: mov.b32 %r5, %r1; ; SM90-NEXT: @%p1 bra $L__BB136_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(3) %addr, i32 %val syncscope("block") acq_rel ret i32 %retval } define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) { ; SM90-LABEL: nand_acq_rel_i64_shared_cta( ; SM90: { ; SM90-NEXT: .reg .pred %p<2>; ; SM90-NEXT: .reg .b64 %rd<7>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd3, [nand_acq_rel_i64_shared_cta_param_1]; ; SM90-NEXT: ld.param.b64 %rd2, [nand_acq_rel_i64_shared_cta_param_0]; ; SM90-NEXT: fence.release.cta; ; SM90-NEXT: ld.relaxed.cta.shared.b64 %rd6, [%rd2]; ; SM90-NEXT: $L__BB137_1: // %atomicrmw.start ; SM90-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-NEXT: and.b64 %rd4, %rd6, %rd3; ; SM90-NEXT: not.b64 %rd5, %rd4; ; SM90-NEXT: atom.relaxed.cta.shared.cas.b64 %rd1, [%rd2], %rd6, %rd5; ; SM90-NEXT: setp.ne.b64 %p1, %rd1, %rd6; ; SM90-NEXT: mov.b64 %rd6, %rd1; ; SM90-NEXT: @%p1 bra $L__BB137_1; ; SM90-NEXT: // %bb.2: // %atomicrmw.end ; SM90-NEXT: fence.acquire.cta; ; SM90-NEXT: st.param.b64 [func_retval0], %rd1; ; SM90-NEXT: ret; %retval = atomicrmw nand ptr addrspace(3) %addr, i64 %val syncscope("block") acq_rel ret i64 %retval } define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) { ; SM90-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta( ; SM90-NOFTZ: { ; SM90-NOFTZ-NEXT: .reg .b32 %r<3>; ; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; ; SM90-NOFTZ-EMPTY: ; SM90-NOFTZ-NEXT: // %bb.0: ; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; ; SM90-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; ; SM90-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1; ; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2; ; SM90-NOFTZ-NEXT: ret; ; ; SM90-FTZ-LABEL: fadd_acq_rel_float_shared_cta( ; SM90-FTZ: { ; SM90-FTZ-NEXT: .reg .pred %p<2>; ; SM90-FTZ-NEXT: .reg .b32 %r<5>; ; SM90-FTZ-NEXT: .reg .b64 %rd<2>; ; SM90-FTZ-EMPTY: ; SM90-FTZ-NEXT: // %bb.0: ; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1]; ; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; ; SM90-FTZ-NEXT: fence.release.cta; ; SM90-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1]; ; SM90-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start ; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 ; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; ; SM90-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3; ; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; ; SM90-FTZ-NEXT: mov.b32 %r4, %r1; ; SM90-FTZ-NEXT: @%p1 bra $L__BB138_1; ; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end ; SM90-FTZ-NEXT: fence.acquire.cta; ; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; ; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel ret float %retval } define double @fadd_acq_rel_double_shared_cta(ptr addrspace(3) %addr, double %val) { ; SM90-LABEL: fadd_acq_rel_double_shared_cta( ; SM90: { ; SM90-NEXT: .reg .b64 %rd<4>; ; SM90-EMPTY: ; SM90-NEXT: // %bb.0: ; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_double_shared_cta_param_0]; ; SM90-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_double_shared_cta_param_1]; ; SM90-NEXT: atom.acq_rel.cta.shared.add.f64 %rd3, [%rd1], %rd2; ; SM90-NEXT: st.param.b64 [func_retval0], %rd3; ; SM90-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(3) %addr, double %val syncscope("block") acq_rel ret double %retval }