; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-W32 %s
; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-W64 %s

; DPP16 Row Share optimization depends on knowing the wavefront size, this run should skip the optimization
; RUN: opt -mtriple=amdgcn-- -mattr=+dpp -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-NO-WAVE-SIZE %s

define i32 @test_wave_shuffle_self_select(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %tid)
  ret i32 %res
}

; In the Row Share 0 case, the logic is the same with and without the or.
; In fact, the or will likely be optimized out before reaching the DPP
; optimization step anyway. So this case should work with or without the or
define i32 @test_wave_shuffle_dpp_row_share_0(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 112
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, 65520   ; 0xFFF0
  %share_0 = or i32 %masked, 0
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_0)
  ret i32 %res
}

define i32 @test_wave_shuffle_dpp_row_share_0_no_or(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 112
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, 65520   ; 0xFFF0
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %masked)
  ret i32 %res
}

define i32 @test_wave_shuffle_dpp_row_share_7(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 48
; CHECK-NO-WAVE-SIZE-NEXT:    [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, 48  ; 0x30
  %share_7 = or i32 %masked, 7
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
  ret i32 %res
}

define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[SHARE_7:%.*]] = or i32 [[TID]], 7
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %share_7 = or i32 %tid, 7
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
  ret i32 %res
}

; Doing both mbcnt.lo and mbcnt.hi works for both wave32 and wave64 because the
; mbcnt.hi is optimized away for wave32. However, ommitting mbcnt.hi should prevent
; wave64 from optimizing to dpp.
define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-W64-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 48
; CHECK-W64-NEXT:    [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
; CHECK-W64-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 48
; CHECK-NO-WAVE-SIZE-NEXT:    [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %tid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %masked = and i32 %tid, 65520   ; 0xFFF0
  %share_7 = or i32 %masked, 7
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
  ret i32 %res
}

; The mask requirements for wave32 and wave64 are slightly different since wave64
; has 4 rows. This test has a mask that should only be valid for wave32 to be
; optimized to dpp.
define i32 @test_wave_shuffle_dpp_row_share_w32_mask(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-W64-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-W64-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 16
; CHECK-W64-NEXT:    [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
; CHECK-W64-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; CHECK-NO-WAVE-SIZE-NEXT:    [[TID:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; CHECK-NO-WAVE-SIZE-NEXT:    [[MASKED:%.*]] = and i32 [[TID]], 16
; CHECK-NO-WAVE-SIZE-NEXT:    [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, 16  ; 0x10
  %share_7 = or i32 %masked, 7
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
  ret i32 %res
}

define i32 @test_wave_shuffle_not_quite_row_share(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W32-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 55)
; CHECK-W32-NEXT:    ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-W64-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 55)
; CHECK-W64-NEXT:    ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
; CHECK-NO-WAVE-SIZE-NEXT:    [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 55)
; CHECK-NO-WAVE-SIZE-NEXT:    ret i32 [[RES]]
;
  %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, 65280   ; 0xFF00
  %or_res = or i32 %masked, 55    ; 0x37
  %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %or_res)
  ret i32 %res
}