; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GCN,TRUE16 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GCN,FAKE16 %s

; TODO: Add global-isel when it can support bf16
define amdgpu_ps void @llvm_sqrt_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_sqrt_bf16_v:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_sqrt_bf16_e32 v2.l, v2.l
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_sqrt_bf16_v:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_sqrt_bf16_e32 v2, v2
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %sqrt = call bfloat @llvm.sqrt.bf16(bfloat %src)
  store bfloat %sqrt, ptr addrspace(1) %out, align 2
  ret void
}

define amdgpu_ps void @llvm_sqrt_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_sqrt_bf16_s:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_sqrt_bf16_e32 v2.l, s0
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_sqrt_bf16_s:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_sqrt_bf16_e32 v2, s0
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %sqrt = call bfloat @llvm.sqrt.bf16(bfloat %src)
  store bfloat %sqrt, ptr addrspace(1) %out, align 2
  ret void
}

define amdgpu_ps void @v_test_add_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_add_v2bf16_vv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, v3
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fadd <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_add_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_add_v2bf16_vs:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, s0
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fadd <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_add_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_add_v2bf16_ss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fadd <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_add_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_add_v2bf16_vc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, 2.0 op_sel_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fadd <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_add_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_add_v2bf16_vl:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, 0x42c83f80, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fadd <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_sub_v2bf16_vv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, v3 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_sub_v2bf16_vs:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, s0 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_sub_v2bf16_ss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, s0, s1 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> %a, %b
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_vc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, -2.0 op_sel_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_vl:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, 0xc2c8bf80, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_lv(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_lv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, 0x42c83f80, v2 neg_lo:[0,1] neg_hi:[0,1]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> <bfloat 1.0, bfloat 100.0>, %a
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_sub_v2bf16_iv(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_sub_v2bf16_iv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_add_bf16 v2, v2, 1.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %add = fsub <2 x bfloat> <bfloat 1.0, bfloat 1.0>, %a
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_mul_v2bf16_vv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v2, v2, v3
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul <2 x bfloat> %a, %b
  store <2 x bfloat> %mul, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_v2bf16_vs:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v2, v2, s0
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul <2 x bfloat> %a, %b
  store <2 x bfloat> %mul, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_v2bf16_ss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul <2 x bfloat> %a, %b
  store <2 x bfloat> %mul, ptr addrspace(1) %out
  ret void
}

; FIXME: We can do better folding inline constant instead of a literal.

define amdgpu_ps void @v_test_mul_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_v2bf16_vc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul <2 x bfloat> %a, <bfloat 0.5, bfloat 0.5>
  store <2 x bfloat> %mul, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_v2bf16_vl:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v2, 0x42c83f80, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
  store <2 x bfloat> %mul, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_min_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_min_v2bf16_vv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_min_num_bf16 v2, v2, v3
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %min, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_min_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_min_v2bf16_vs:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_min_num_bf16 v2, v2, s0
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %min, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_min_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_min_v2bf16_ss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_min_num_bf16 v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %min, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_min_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_min_v2bf16_vc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_min_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)
  store <2 x bfloat> %min, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_min_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_min_v2bf16_vl:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_min_num_bf16 v2, 0x42c83f80, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)
  store <2 x bfloat> %min, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_max_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {
; GCN-LABEL: v_test_max_v2bf16_vv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v2, v2, v3
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %max, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_max_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_max_v2bf16_vs:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v2, v2, s0
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %max, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_max_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_max_v2bf16_ss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
  store <2 x bfloat> %max, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_max_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_max_v2bf16_vc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)
  store <2 x bfloat> %max, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_max_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_max_v2bf16_vl:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v2, 0x42c83f80, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)
  store <2 x bfloat> %max, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps bfloat @test_clamp_bf16(bfloat %src) {
; TRUE16-LABEL: test_clamp_bf16:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT:    ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
; FAKE16-NEXT:    ; return to shader part epilog
  %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0)
  %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)
  ret bfloat %clamp
}

define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) {
; TRUE16-LABEL: test_clamp_bf16_s:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_mov_b16_e32 v0.l, s0
; TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; TRUE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT:    ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16_s:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_pk_max_num_bf16 v0, s0, s0 clamp
; FAKE16-NEXT:    ; return to shader part epilog
  %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0)
  %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)
  ret bfloat %clamp
}

define amdgpu_ps float @test_clamp_bf16_hi16(<2 x bfloat> %src) {
; TRUE16-LABEL: test_clamp_bf16_hi16:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
; TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; TRUE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
; TRUE16-NEXT:    ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16_hi16:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
; FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; FAKE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
; FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
; FAKE16-NEXT:    ; return to shader part epilog
  %bsrc = extractelement <2 x bfloat> %src, i32 1
  %max = call bfloat @llvm.maxnum.bf16(bfloat %bsrc, bfloat 0.0)
  %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)
  %res = fpext bfloat %clamp to float
  ret float %res
}

define amdgpu_ps float @test_clamp_bf16_hi16_s(<2 x bfloat> inreg %src) {
; TRUE16-LABEL: test_clamp_bf16_hi16_s:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    s_lshr_b32 s0, s0, 16
; TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; TRUE16-NEXT:    v_mov_b16_e32 v0.l, s0
; TRUE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
; TRUE16-NEXT:    ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16_hi16_s:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    s_lshr_b32 s0, s0, 16
; FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; FAKE16-NEXT:    v_pk_max_num_bf16 v0, s0, s0 clamp
; FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
; FAKE16-NEXT:    ; return to shader part epilog
  %bsrc = extractelement <2 x bfloat> %src, i32 1
  %max = call bfloat @llvm.maxnum.bf16(bfloat %bsrc, bfloat 0.0)
  %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)
  %res = fpext bfloat %clamp to float
  ret float %res
}

define amdgpu_ps float @test_clamp_v2bf16(<2 x bfloat> %src) {
; GCN-LABEL: test_clamp_v2bf16:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
; GCN-NEXT:    ; return to shader part epilog
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
  %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)
  %ret = bitcast <2 x bfloat> %clamp to float
  ret float %ret
}

define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) {
; GCN-LABEL: test_clamp_v2bf16_s:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_max_num_bf16 v0, s0, s0 clamp
; GCN-NEXT:    ; return to shader part epilog
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
  %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)
  %ret = bitcast <2 x bfloat> %clamp to float
  ret float %ret
}

define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
; TRUE16-LABEL: test_clamp_bf16_folding:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_exp_bf16_e32 v0.l, v0.l
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; TRUE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
; TRUE16-NEXT:    ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16_folding:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_exp_bf16_e32 v0, v0
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
; FAKE16-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
; FAKE16-NEXT:    ; return to shader part epilog
  %exp = call bfloat @llvm.exp2.bf16(bfloat %src)
  %max = call bfloat @llvm.maxnum.bf16(bfloat %exp, bfloat 0.0)
  %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)
  ret bfloat %clamp
}

define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloat> %src1) {
; GCN-LABEL: test_clamp_v2bf16_folding:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_mul_bf16 v0, v0, v1 clamp
; GCN-NEXT:    ; return to shader part epilog
  %mul = fmul <2 x bfloat> %src0, %src1
  %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %mul, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
  %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)
  %ret = bitcast <2 x bfloat> %clamp to float
  ret float %ret
}

define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_vvv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, v3, v4
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul contract <2 x bfloat> %a, %b
  %add = fadd contract <2 x bfloat> %mul, %c
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_vss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul contract <2 x bfloat> %a, %b
  %add = fadd contract <2 x bfloat> %mul, %c
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_mul_add_v2bf16_sss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_mov_b32_e32 v2, s2
; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GCN-NEXT:    v_pk_fma_bf16 v2, s0, s1, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul contract <2 x bfloat> %a, %b
  %add = fadd contract <2 x bfloat> %mul, %c
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_mul_add_v2bf16_vsc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul contract <2 x bfloat> %a, %b
  %add = fadd contract <2 x bfloat> %mul, <bfloat 0.5, bfloat 0.5>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_mul_add_v2bf16_vll:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    s_mov_b32 s0, 0x43484000
; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT:    v_pk_fma_bf16 v2, 0x42c83f80, v2, s0
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %mul = fmul contract <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
  %add = fadd contract <2 x bfloat> %mul, <bfloat 2.0, bfloat 200.0>
  store <2 x bfloat> %add, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_fma_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
; GCN-LABEL: v_test_fma_v2bf16_vvv:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, v3, v4
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
  store <2 x bfloat> %fma, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_fma_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_fma_v2bf16_vss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, s1
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
  store <2 x bfloat> %fma, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_fma_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
; GCN-LABEL: v_test_fma_v2bf16_sss:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_mov_b32_e32 v2, s2
; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
; GCN-NEXT:    v_pk_fma_bf16 v2, s0, s1, v2
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
  store <2 x bfloat> %fma, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_fma_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
; GCN-LABEL: v_test_fma_v2bf16_vsc:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)
  store <2 x bfloat> %fma, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @v_test_fma_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {
; GCN-LABEL: v_test_fma_v2bf16_vll:
; GCN:       ; %bb.0:
; GCN-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; GCN-NEXT:    v_nop
; GCN-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT:    s_mov_b32 s0, 0x42c83f80
; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, 0x43484000
; GCN-NEXT:    global_store_b32 v[0:1], v2, off
; GCN-NEXT:    s_endpgm
  %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, <2 x bfloat> <bfloat 2.0, bfloat 200.0>)
  store <2 x bfloat> %fma, ptr addrspace(1) %out
  ret void
}

define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_log2_bf16_v:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_log_bf16_e32 v2.l, v2.l
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_log2_bf16_v:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_log_bf16_e32 v2, v2
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %log = call bfloat @llvm.log2.bf16(bfloat %src)
  store bfloat %log, ptr addrspace(1) %out, align 2
  ret void
}

define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_log2_bf16_s:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_log_bf16_e32 v2.l, s0
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_log2_bf16_s:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_log_bf16_e32 v2, s0
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %log = call bfloat @llvm.log2.bf16(bfloat %src)
  store bfloat %log, ptr addrspace(1) %out, align 2
  ret void
}

define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) {
; TRUE16-LABEL: llvm_exp2_bf16_v:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_exp_bf16_e32 v2.l, v2.l
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_exp2_bf16_v:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_exp_bf16_e32 v2, v2
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %exp = call bfloat @llvm.exp2.bf16(bfloat %src)
  store bfloat %exp, ptr addrspace(1) %out, align 2
  ret void
}

define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {
; TRUE16-LABEL: llvm_exp2_bf16_s:
; TRUE16:       ; %bb.0:
; TRUE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; TRUE16-NEXT:    v_nop
; TRUE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT:    v_exp_bf16_e32 v2.l, s0
; TRUE16-NEXT:    global_store_b16 v[0:1], v2, off
; TRUE16-NEXT:    s_endpgm
;
; FAKE16-LABEL: llvm_exp2_bf16_s:
; FAKE16:       ; %bb.0:
; FAKE16-NEXT:    global_prefetch_b8 v0, s[0:1] scope:SCOPE_SE
; FAKE16-NEXT:    v_nop
; FAKE16-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT:    v_exp_bf16_e32 v2, s0
; FAKE16-NEXT:    global_store_b16 v[0:1], v2, off
; FAKE16-NEXT:    s_endpgm
  %exp = call bfloat @llvm.exp2.bf16(bfloat %src)
  store bfloat %exp, ptr addrspace(1) %out, align 2
  ret void
}

declare bfloat @llvm.minnum.bf16(bfloat, bfloat)
declare bfloat @llvm.maxnum.bf16(bfloat, bfloat)
declare <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)
declare bfloat @llvm.sqrt.bf16(bfloat)
declare bfloat @llvm.log2.bf16(bfloat)
declare bfloat @llvm.exp2.bf16(bfloat)