; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s
; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s
; RUN: %if ptxas-sm_80 && ptxas-isa-7.1 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
; RUN: %if ptxas-sm_90 && ptxas-isa-7.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}

target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"

define <2 x bfloat> @test_ret_const() #0 {
; CHECK-LABEL: test_ret_const(
; CHECK:       {
; CHECK-EMPTY:
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    st.param.b32 [func_retval0], 1073758080;
; CHECK-NEXT:    ret;
  ret <2 x bfloat> <bfloat 1.0, bfloat 2.0>
}

; Check that we can lower fadd with immediate arguments.
define <2 x bfloat> @test_fadd_imm_0(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_fadd_imm_0(
; SM80:       {
; SM80-NEXT:    .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.b32 %r1, [test_fadd_imm_0_param_0];
; SM80-NEXT:    mov.b32 %r2, 1065369472;
; SM80-NEXT:    mov.b32 %r3, 1073758080;
; SM80-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_fadd_imm_0(
; SM90:       {
; SM90-NEXT:    .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.b32 %r1, [test_fadd_imm_0_param_0];
; SM90-NEXT:    mov.b32 %r2, 1073758080;
; SM90-NEXT:    add.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
; SM90-NEXT:    ret;
  %r = fadd <2 x bfloat> <bfloat 1.0, bfloat 2.0>, %a
  ret <2 x bfloat> %r
}

define bfloat @test_fadd_imm_1(bfloat %a) #0 {
; SM80-LABEL: test_fadd_imm_1(
; SM80:       {
; SM80-NEXT:    .reg .b16 %rs<4>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
; SM80-NEXT:    mov.b16 %rs2, 0x3F80;
; SM80-NEXT:    fma.rn.bf16 %rs3, %rs1, %rs2, %rs2;
; SM80-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_fadd_imm_1(
; SM90:       {
; SM90-NEXT:    .reg .b16 %rs<4>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
; SM90-NEXT:    mov.b16 %rs2, 0x3F80;
; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
; SM90-NEXT:    st.param.b16 [func_retval0], %rs3;
; SM90-NEXT:    ret;
  %r = fadd bfloat %a, 1.0
  ret bfloat %r
}

define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fsubx2(
; SM80:       {
; SM80-NEXT:    .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
; SM80-NEXT:    mov.b32 %r2, -1082081408;
; SM80-NEXT:    ld.param.b32 %r3, [test_fsubx2_param_1];
; SM80-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_fsubx2(
; SM90:       {
; SM90-NEXT:    .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
; SM90-NEXT:    sub.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
; SM90-NEXT:    ret;
  %r = fsub <2 x bfloat> %a, %b
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fmulx2(
; SM80:       {
; SM80-NEXT:    .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
; SM80-NEXT:    mov.b32 %r2, -2147450880;
; SM80-NEXT:    ld.param.b32 %r3, [test_fmulx2_param_1];
; SM80-NEXT:    fma.rn.bf16x2 %r4, %r1, %r3, %r2;
; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_fmulx2(
; SM90:       {
; SM90-NEXT:    .reg .b32 %r<4>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
; SM90-NEXT:    mul.rn.bf16x2 %r3, %r1, %r2;
; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
; SM90-NEXT:    ret;
  %r = fmul <2 x bfloat> %a, %b
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_fdiv(
; CHECK:       {
; CHECK-NEXT:    .reg .b16 %rs<5>;
; CHECK-NEXT:    .reg .b32 %r<8>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
; CHECK-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs3;
; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
; CHECK-NEXT:    div.rn.f32 %r3, %r2, %r1;
; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs4;
; CHECK-NEXT:    cvt.f32.bf16 %r5, %rs2;
; CHECK-NEXT:    div.rn.f32 %r6, %r5, %r4;
; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
; CHECK-NEXT:    ret;
  %r = fdiv <2 x bfloat> %a, %b
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fneg(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_fneg(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_fneg_param_0];
; CHECK-NEXT:    neg.bf16x2 %r2, %r1;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
; CHECK-NEXT:    ret;
  %r = fneg <2 x bfloat> %a
  ret <2 x bfloat> %r
}

define void @test_ldst_v2bf16(ptr %a, ptr %b) {
; CHECK-LABEL: test_ldst_v2bf16(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-NEXT:    .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v2bf16_param_0];
; CHECK-NEXT:    ld.b32 %r1, [%rd1];
; CHECK-NEXT:    ld.param.b64 %rd2, [test_ldst_v2bf16_param_1];
; CHECK-NEXT:    st.b32 [%rd2], %r1;
; CHECK-NEXT:    ret;
  %t1 = load <2 x bfloat>, ptr %a
  store <2 x bfloat> %t1, ptr %b, align 16
  ret void
}

define void @test_ldst_v3bf16(ptr %a, ptr %b) {
; CHECK-LABEL: test_ldst_v3bf16(
; CHECK:       {
; CHECK-NEXT:    .reg .b16 %rs<2>;
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-NEXT:    .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v3bf16_param_0];
; CHECK-NEXT:    ld.b64 %rd2, [%rd1];
; CHECK-NEXT:    mov.b64 {_, %r1}, %rd2;
; CHECK-NEXT:    ld.param.b64 %rd3, [test_ldst_v3bf16_param_1];
; CHECK-NEXT:    st.b32 [%rd3], %rd2;
; CHECK-NEXT:    mov.b32 {%rs1, _}, %r1;
; CHECK-NEXT:    st.b16 [%rd3+4], %rs1;
; CHECK-NEXT:    ret;
  %t1 = load <3 x bfloat>, ptr %a
  store <3 x bfloat> %t1, ptr %b, align 16
  ret void
}

declare <2 x bfloat> @test_callee(<2 x bfloat> %a, <2 x bfloat> %b) #0

define <2 x bfloat> @test_call(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_call(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_call_param_0];
; CHECK-NEXT:    { // callseq 0, 0
; CHECK-NEXT:    .param .align 4 .b8 param0[4];
; CHECK-NEXT:    .param .align 4 .b8 param1[4];
; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
; CHECK-NEXT:    ld.param.b32 %r2, [test_call_param_1];
; CHECK-NEXT:    st.param.b32 [param1], %r2;
; CHECK-NEXT:    st.param.b32 [param0], %r1;
; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
; CHECK-NEXT:    } // callseq 0
; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @test_callee(<2 x bfloat> %a, <2 x bfloat> %b)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_select(<2 x bfloat> %a, <2 x bfloat> %b, i1 zeroext %c) #0 {
; CHECK-LABEL: test_select(
; CHECK:       {
; CHECK-NEXT:    .reg .pred %p<2>;
; CHECK-NEXT:    .reg .b16 %rs<3>;
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-NEXT:    .reg .b64 %rd<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b8 %rs1, [test_select_param_2];
; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
; CHECK-NEXT:    mov.b64 %rd1, test_select_param_1;
; CHECK-NEXT:    mov.b64 %rd2, test_select_param_0;
; CHECK-NEXT:    selp.b64 %rd3, %rd2, %rd1, %p1;
; CHECK-NEXT:    ld.param.b32 %r1, [%rd3];
; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
; CHECK-NEXT:    ret;
  %r = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_select_cc(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c, <2 x bfloat> %d) #0 {
; SM80-LABEL: test_select_cc(
; SM80:       {
; SM80-NEXT:    .reg .pred %p<3>;
; SM80-NEXT:    .reg .b16 %rs<11>;
; SM80-NEXT:    .reg .b32 %r<5>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
; SM80-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_3];
; SM80-NEXT:    cvt.f32.bf16 %r1, %rs3;
; SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [test_select_cc_param_2];
; SM80-NEXT:    cvt.f32.bf16 %r2, %rs5;
; SM80-NEXT:    setp.neu.f32 %p1, %r2, %r1;
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs4;
; SM80-NEXT:    cvt.f32.bf16 %r4, %rs6;
; SM80-NEXT:    setp.neu.f32 %p2, %r4, %r3;
; SM80-NEXT:    ld.param.v2.b16 {%rs7, %rs8}, [test_select_cc_param_1];
; SM80-NEXT:    selp.b16 %rs9, %rs2, %rs8, %p2;
; SM80-NEXT:    selp.b16 %rs10, %rs1, %rs7, %p1;
; SM80-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_select_cc(
; SM90:       {
; SM90-NEXT:    .reg .pred %p<3>;
; SM90-NEXT:    .reg .b16 %rs<7>;
; SM90-NEXT:    .reg .b32 %r<3>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
; SM90-NEXT:    ld.param.b32 %r1, [test_select_cc_param_2];
; SM90-NEXT:    ld.param.b32 %r2, [test_select_cc_param_3];
; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r1, %r2;
; SM90-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_1];
; SM90-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p2;
; SM90-NEXT:    selp.b16 %rs6, %rs1, %rs3, %p1;
; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; SM90-NEXT:    ret;
  %cc = fcmp une <2 x bfloat> %c, %d
  %r = select <2 x i1> %cc, <2 x bfloat> %a, <2 x bfloat> %b
  ret <2 x bfloat> %r
}

define <2 x float> @test_select_cc_f32_bf16(<2 x float> %a, <2 x float> %b,
; SM80-LABEL: test_select_cc_f32_bf16(
; SM80:       {
; SM80-NEXT:    .reg .pred %p<3>;
; SM80-NEXT:    .reg .b16 %rs<5>;
; SM80-NEXT:    .reg .b32 %r<11>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_f32_bf16_param_3];
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs1;
; SM80-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_f32_bf16_param_2];
; SM80-NEXT:    cvt.f32.bf16 %r4, %rs3;
; SM80-NEXT:    setp.neu.f32 %p1, %r4, %r3;
; SM80-NEXT:    cvt.f32.bf16 %r5, %rs2;
; SM80-NEXT:    cvt.f32.bf16 %r6, %rs4;
; SM80-NEXT:    setp.neu.f32 %p2, %r6, %r5;
; SM80-NEXT:    ld.param.v2.b32 {%r7, %r8}, [test_select_cc_f32_bf16_param_1];
; SM80-NEXT:    selp.f32 %r9, %r2, %r8, %p2;
; SM80-NEXT:    selp.f32 %r10, %r1, %r7, %p1;
; SM80-NEXT:    st.param.v2.b32 [func_retval0], {%r10, %r9};
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_select_cc_f32_bf16(
; SM90:       {
; SM90-NEXT:    .reg .pred %p<3>;
; SM90-NEXT:    .reg .b32 %r<9>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
; SM90-NEXT:    ld.param.b32 %r3, [test_select_cc_f32_bf16_param_2];
; SM90-NEXT:    ld.param.b32 %r4, [test_select_cc_f32_bf16_param_3];
; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r3, %r4;
; SM90-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f32_bf16_param_1];
; SM90-NEXT:    selp.f32 %r7, %r2, %r6, %p2;
; SM90-NEXT:    selp.f32 %r8, %r1, %r5, %p1;
; SM90-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
; SM90-NEXT:    ret;
                                           <2 x bfloat> %c, <2 x bfloat> %d) #0 {
  %cc = fcmp une <2 x bfloat> %c, %d
  %r = select <2 x i1> %cc, <2 x float> %a, <2 x float> %b
  ret <2 x float> %r
}

define <2 x bfloat> @test_select_cc_bf16_f32(<2 x bfloat> %a, <2 x bfloat> %b,
; CHECK-LABEL: test_select_cc_bf16_f32(
; CHECK:       {
; CHECK-NEXT:    .reg .pred %p<3>;
; CHECK-NEXT:    .reg .b16 %rs<7>;
; CHECK-NEXT:    .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_bf16_f32_param_0];
; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_bf16_f32_param_2];
; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_bf16_f32_param_3];
; CHECK-NEXT:    setp.neu.f32 %p1, %r1, %r3;
; CHECK-NEXT:    setp.neu.f32 %p2, %r2, %r4;
; CHECK-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_bf16_f32_param_1];
; CHECK-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p2;
; CHECK-NEXT:    selp.b16 %rs6, %rs1, %rs3, %p1;
; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
; CHECK-NEXT:    ret;
                                          <2 x float> %c, <2 x float> %d) #0 {
  %cc = fcmp une <2 x float> %c, %d
  %r = select <2 x i1> %cc, <2 x bfloat> %a, <2 x bfloat> %b
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fptrunc_2xfloat(<2 x float> %a) #0 {
; CHECK-LABEL: test_fptrunc_2xfloat(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptrunc_2xfloat_param_0];
; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r3, %r2, %r1;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
; CHECK-NEXT:    ret;
  %r = fptrunc <2 x float> %a to <2 x bfloat>
  ret <2 x bfloat> %r
}

define <2 x float> @test_fpext_2xfloat(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_fpext_2xfloat(
; CHECK:       {
; CHECK-NEXT:    .reg .b16 %rs<3>;
; CHECK-NEXT:    .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fpext_2xfloat_param_0];
; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs2;
; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
; CHECK-NEXT:    ret;
  %r = fpext <2 x bfloat> %a to <2 x float>
  ret <2 x float> %r
}

define <2 x i16> @test_bitcast_2xbf16_to_2xi16(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_bitcast_2xbf16_to_2xi16(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_bitcast_2xbf16_to_2xi16_param_0];
; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
; CHECK-NEXT:    ret;
  %r = bitcast <2 x bfloat> %a to <2 x i16>
  ret <2 x i16> %r
}

define <2 x bfloat> @test_bitcast_2xi16_to_2xbf16(<2 x i16> %a) #0 {
; CHECK-LABEL: test_bitcast_2xi16_to_2xbf16(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_bitcast_2xi16_to_2xbf16_param_0];
; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
; CHECK-NEXT:    ret;
  %r = bitcast <2 x i16> %a to <2 x bfloat>
  ret <2 x bfloat> %r
}

declare <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.powi.f16(<2 x bfloat> %a, <2 x i32> %b) #0
declare <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.pow.f16(<2 x bfloat> %a, <2 x bfloat> %b) #0
declare <2 x bfloat> @llvm.exp.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.exp2.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.log.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.log10.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.log2.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.fma.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0
declare <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.minnum.f16(<2 x bfloat> %a, <2 x bfloat> %b) #0
declare <2 x bfloat> @llvm.maxnum.f16(<2 x bfloat> %a, <2 x bfloat> %b) #0
declare <2 x bfloat> @llvm.copysign.f16(<2 x bfloat> %a, <2 x bfloat> %b) #0
declare <2 x bfloat> @llvm.floor.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.ceil.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.trunc.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.rint.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.nearbyint.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a) #0
declare <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0

define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_sqrt(
; CHECK:       {
; CHECK-NEXT:    .reg .b16 %rs<3>;
; CHECK-NEXT:    .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_sqrt_param_0];
; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
; CHECK-NEXT:    sqrt.rn.f32 %r2, %r1;
; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs2;
; CHECK-NEXT:    sqrt.rn.f32 %r4, %r3;
; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fmuladd(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
; CHECK-LABEL: test_fmuladd(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<5>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_fmuladd_param_0];
; CHECK-NEXT:    ld.param.b32 %r2, [test_fmuladd_param_1];
; CHECK-NEXT:    ld.param.b32 %r3, [test_fmuladd_param_2];
; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_fabs(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<3>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_fabs_param_0];
; CHECK-NEXT:    abs.bf16x2 %r2, %r1;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; SM80-LABEL: test_fabs_add(
; SM80:       {
; SM80-NEXT:    .reg .b32 %r<7>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_0];
; SM80-NEXT:    mov.b32 %r2, 1065369472;
; SM80-NEXT:    fma.rn.bf16x2 %r3, %r1, %r2, %r1;
; SM80-NEXT:    ld.param.b32 %r4, [test_fabs_add_param_1];
; SM80-NEXT:    abs.bf16x2 %r5, %r3;
; SM80-NEXT:    fma.rn.bf16x2 %r6, %r5, %r2, %r4;
; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_fabs_add(
; SM90:       {
; SM90-NEXT:    .reg .b32 %r<6>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_0];
; SM90-NEXT:    add.rn.bf16x2 %r2, %r1, %r1;
; SM90-NEXT:    ld.param.b32 %r3, [test_fabs_add_param_1];
; SM90-NEXT:    abs.bf16x2 %r4, %r2;
; SM90-NEXT:    add.rn.bf16x2 %r5, %r4, %r3;
; SM90-NEXT:    st.param.b32 [func_retval0], %r5;
; SM90-NEXT:    ret;
  %s = fadd <2 x bfloat> %a, %a
  %r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %s)
  %d = fadd <2 x bfloat> %r, %b
  ret <2 x bfloat> %d
}

define <2 x bfloat> @test_minnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_minnum(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_minnum_param_0];
; CHECK-NEXT:    ld.param.b32 %r2, [test_minnum_param_1];
; CHECK-NEXT:    min.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.minnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_maxnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_maxnum(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<4>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_maxnum_param_0];
; CHECK-NEXT:    ld.param.b32 %r2, [test_maxnum_param_1];
; CHECK-NEXT:    max.bf16x2 %r3, %r1, %r2;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.maxnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_floor(
; SM80:       {
; SM80-NEXT:    .reg .b16 %rs<3>;
; SM80-NEXT:    .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
; SM80-NEXT:    cvt.rmi.f32.f32 %r2, %r1;
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
; SM80-NEXT:    cvt.rmi.f32.f32 %r4, %r3;
; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_floor(
; SM90:       {
; SM90-NEXT:    .reg .b16 %rs<5>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
; SM90-NEXT:    cvt.rmi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT:    cvt.rmi.bf16.bf16 %rs4, %rs1;
; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.floor.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_ceil(
; SM80:       {
; SM80-NEXT:    .reg .b16 %rs<3>;
; SM80-NEXT:    .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
; SM80-NEXT:    cvt.rpi.f32.f32 %r2, %r1;
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
; SM80-NEXT:    cvt.rpi.f32.f32 %r4, %r3;
; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_ceil(
; SM90:       {
; SM90-NEXT:    .reg .b16 %rs<5>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
; SM90-NEXT:    cvt.rpi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT:    cvt.rpi.bf16.bf16 %rs4, %rs1;
; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.ceil.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_trunc(
; SM80:       {
; SM80-NEXT:    .reg .b16 %rs<3>;
; SM80-NEXT:    .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
; SM80-NEXT:    cvt.rzi.f32.f32 %r2, %r1;
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
; SM80-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_trunc(
; SM90:       {
; SM90-NEXT:    .reg .b16 %rs<5>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
; SM90-NEXT:    cvt.rzi.bf16.bf16 %rs3, %rs2;
; SM90-NEXT:    cvt.rzi.bf16.bf16 %rs4, %rs1;
; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.trunc.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 {
; SM80-LABEL: test_rint(
; SM80:       {
; SM80-NEXT:    .reg .b16 %rs<3>;
; SM80-NEXT:    .reg .b32 %r<6>;
; SM80-EMPTY:
; SM80-NEXT:  // %bb.0:
; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
; SM80-NEXT:    cvt.rni.f32.f32 %r2, %r1;
; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
; SM80-NEXT:    cvt.rni.f32.f32 %r4, %r3;
; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
; SM80-NEXT:    ret;
;
; SM90-LABEL: test_rint(
; SM90:       {
; SM90-NEXT:    .reg .b16 %rs<5>;
; SM90-EMPTY:
; SM90-NEXT:  // %bb.0:
; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
; SM90-NEXT:    cvt.rni.bf16.bf16 %rs3, %rs2;
; SM90-NEXT:    cvt.rni.bf16.bf16 %rs4, %rs1;
; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
; SM90-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.rint.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
; CHECK-LABEL: test_round(
; CHECK:       {
; CHECK-NEXT:    .reg .pred %p<5>;
; CHECK-NEXT:    .reg .b16 %rs<3>;
; CHECK-NEXT:    .reg .b32 %r<20>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_round_param_0];
; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
; CHECK-NEXT:    and.b32 %r2, %r1, -2147483648;
; CHECK-NEXT:    or.b32 %r3, %r2, 1056964608;
; CHECK-NEXT:    add.rn.f32 %r4, %r1, %r3;
; CHECK-NEXT:    cvt.rzi.f32.f32 %r5, %r4;
; CHECK-NEXT:    abs.f32 %r6, %r1;
; CHECK-NEXT:    setp.gt.f32 %p1, %r6, 0f4B000000;
; CHECK-NEXT:    selp.f32 %r7, %r1, %r5, %p1;
; CHECK-NEXT:    cvt.rzi.f32.f32 %r8, %r1;
; CHECK-NEXT:    setp.lt.f32 %p2, %r6, 0f3F000000;
; CHECK-NEXT:    selp.f32 %r9, %r8, %r7, %p2;
; CHECK-NEXT:    cvt.f32.bf16 %r10, %rs2;
; CHECK-NEXT:    and.b32 %r11, %r10, -2147483648;
; CHECK-NEXT:    or.b32 %r12, %r11, 1056964608;
; CHECK-NEXT:    add.rn.f32 %r13, %r10, %r12;
; CHECK-NEXT:    cvt.rzi.f32.f32 %r14, %r13;
; CHECK-NEXT:    abs.f32 %r15, %r10;
; CHECK-NEXT:    setp.gt.f32 %p3, %r15, 0f4B000000;
; CHECK-NEXT:    selp.f32 %r16, %r10, %r14, %p3;
; CHECK-NEXT:    cvt.rzi.f32.f32 %r17, %r10;
; CHECK-NEXT:    setp.lt.f32 %p4, %r15, 0f3F000000;
; CHECK-NEXT:    selp.f32 %r18, %r17, %r16, %p4;
; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r19, %r18, %r9;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r19;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a)
  ret <2 x bfloat> %r
}

define <2 x bfloat> @test_copysign(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; CHECK-LABEL: test_copysign(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<6>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b32 %r1, [test_copysign_param_1];
; CHECK-NEXT:    and.b32 %r2, %r1, -2147450880;
; CHECK-NEXT:    ld.param.b32 %r3, [test_copysign_param_0];
; CHECK-NEXT:    and.b32 %r4, %r3, 2147450879;
; CHECK-NEXT:    or.b32 %r5, %r4, %r2;
; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
; CHECK-NEXT:    ret;
  %r = call <2 x bfloat> @llvm.copysign.f16(<2 x bfloat> %a, <2 x bfloat> %b)
  ret <2 x bfloat> %r
}

define void @test_store_bf16x2(ptr %p1, ptr %p2, <2 x bfloat> %v) {
; CHECK-LABEL: test_store_bf16x2(
; CHECK:       {
; CHECK-NEXT:    .reg .b32 %r<2>;
; CHECK-NEXT:    .reg .b64 %rd<3>;
; CHECK-EMPTY:
; CHECK-NEXT:  // %bb.0:
; CHECK-NEXT:    ld.param.b64 %rd1, [test_store_bf16x2_param_0];
; CHECK-NEXT:    ld.param.b32 %r1, [test_store_bf16x2_param_2];
; CHECK-NEXT:    st.b32 [%rd1], %r1;
; CHECK-NEXT:    ld.param.b64 %rd2, [test_store_bf16x2_param_1];
; CHECK-NEXT:    st.b32 [%rd2], 1065369472;
; CHECK-NEXT:    ret;
  store <2 x bfloat> %v, ptr %p1
  store <2 x bfloat> <bfloat 1.0, bfloat 1.0>, ptr %p2
  ret void
}