; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX9

declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
declare i64 @llvm.amdgcn.wave.shuffle.i64(i64, i32)
declare i16 @llvm.amdgcn.wave.shuffle.i16(i16, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
declare <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16>, i32)
declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
declare ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5), i32)
declare ptr @llvm.amdgcn.wave.shuffle.p0(ptr, i32)
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)

; Quad perm: lane ^ 1 swaps adjacent pairs.
define i32 @test_quad_perm_xor1_w32(i32 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Quad perm: lane ^ 3 reverses within each quad.
define i32 @test_quad_perm_xor3_w32(i32 %val) {
; GFX11-LABEL: @test_quad_perm_xor3_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 3
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor3_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 3
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 3
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Half-row mirror: reverse within every 8-lane group.
define i32 @test_half_row_mirror_w32(i32 %val) {
; GFX11-LABEL: @test_half_row_mirror_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_row_mirror_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 7
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_half_row_mirror_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 7
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low3 = and i32 %lane, 7
  %rev = sub i32 7, %low3
  %high = and i32 %lane, -8
  %idx = or i32 %high, %rev
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Full-row mirror: reverse within every 16-lane row.
define i32 @test_full_row_mirror_w32(i32 %val) {
; GFX11-LABEL: @test_full_row_mirror_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_full_row_mirror_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 15
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_full_row_mirror_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 15
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low4 = and i32 %lane, 15
  %rev = sub i32 15, %low4
  %high = and i32 %lane, -16
  %idx = or i32 %high, %rev
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Row rotate right by 3 within each 16-lane row.
define i32 @test_row_rotate_right3_w32(i32 %val) {
; GFX11-LABEL: @test_row_rotate_right3_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 15
; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_rotate_right3_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
; GFX9-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 15
; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low4 = and i32 %lane, 15
  %rot = add i32 %low4, 3
  %wrapped = and i32 %rot, 15
  %high = and i32 %lane, -16
  %idx = or i32 %high, %wrapped
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Row share: broadcast lane 3 within each 16-lane row.
define i32 @test_row_share3_w32(i32 %val) {
; GFX11-LABEL: @test_row_share3_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_share3_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_share3_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %high = and i32 %lane, -16
  %idx = or i32 %high, 3
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Row xmask: xor each lane with mask 5 within each 16-lane row.
define i32 @test_row_xmask5_w32(i32 %val) {
; GFX11-LABEL: @test_row_xmask5_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_xmask5_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 5
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_row_xmask5_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 5
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low4 = and i32 %lane, 15
  %xored = xor i32 %low4, 5
  %high = and i32 %lane, -16
  %idx = or i32 %high, %xored
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Half-wave swap (wave64 only): lane ^ 32.
define i32 @test_half_wave_swap_w64(i32 %val) {
; GFX11-LABEL: @test_half_wave_swap_w64(
; GFX11-NEXT:    [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT:    [[IDX:%.*]] = xor i32 [[LO]], 32
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_wave_swap_w64(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_half_wave_swap_w64(
; GFX9-NEXT:    [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 32
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %idx = xor i32 %lane, 32
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; DPP8: rotate left by 1 within each 8-lane group.
define i32 @test_dpp8_rotate1_w32(i32 %val) {
; GFX11-LABEL: @test_dpp8_rotate1_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 7
; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 56
; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_dpp8_rotate1_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
; GFX9-NEXT:    [[WRAPPED:%.*]] = and i32 [[ROT]], 7
; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 56
; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low3 = and i32 %lane, 7
  %rot = add i32 %low3, 1
  %wrapped = and i32 %rot, 7
  %high = and i32 %lane, -8
  %idx = or i32 %high, %wrapped
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
define i32 @test_permlane16_mul3_w32(i32 %val) {
; GFX11-LABEL: @test_permlane16_mul3_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
; GFX11-W64-NEXT:    [[PERM:%.*]] = and i32 [[MUL3]], 15
; GFX11-W64-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX11-W64-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlane16_mul3_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
; GFX9-NEXT:    [[PERM:%.*]] = and i32 [[MUL3]], 15
; GFX9-NEXT:    [[HIGH:%.*]] = and i32 [[LANE]], 48
; GFX9-NEXT:    [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %low4 = and i32 %lane, 15
  %mul3 = mul i32 %low4, 3
  %perm = and i32 %mul3, 15
  %high = and i32 %lane, -16
  %idx = or i32 %high, %perm
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; PermlaneX16: lane ^ 16. Cross-row swap; identity selectors.
define i32 @test_permlanex16_xor16_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor16_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor16_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 16
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor16_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 16
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 16
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; PermlaneX16: lane ^ 17. Cross-row + XOR-by-1 within row.
define i32 @test_permlanex16_xor17_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor17_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1732584193, i32 -271733879, i1 false, i1 false)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor17_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 17
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor17_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 17
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 17
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; PermlaneX16: lane ^ 31. Cross-row + reverse within row.
define i32 @test_permlanex16_xor31_w32(i32 %val) {
; GFX11-LABEL: @test_permlanex16_xor31_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 -1985229329, i32 19088743, i1 false, i1 false)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_permlanex16_xor31_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 31
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_permlanex16_xor31_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 31
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 31
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Float type should work the same as i32.
define float @test_quad_perm_xor1_float_w32(float %val) {
; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret float [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret float [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_float_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret float [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
  ret float %result
}

; LDS (32-bit) pointer should optimize the same as i32: the gate must not
; reject 32-bit pointer types via Type::getPrimitiveSizeInBits().
define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
; GFX11-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret ptr addrspace(3) [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret ptr addrspace(3) [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_ptr3_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret ptr addrspace(3) [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
  ret ptr addrspace(3) %result
}

; Scratch (32-bit) pointer, should optimize like other 32-bit pointers.
define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
; GFX11-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) poison, ptr addrspace(5) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret ptr addrspace(5) [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret ptr addrspace(5) [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret ptr addrspace(5) [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) %val, i32 %idx)
  ret ptr addrspace(5) %result
}

; <2 x i16> vector (32-bit), should optimize like i32.
define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
; GFX11-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.update.dpp.v2i16(<2 x i16> poison, <2 x i16> [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret <2 x i16> [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret <2 x i16> [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret <2 x i16> [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
  ret <2 x i16> %result
}

; Half-row-share with <2 x i16>: falls back to ds_swizzle on GFX9, using the bitcast path.
define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
; GFX11-LABEL: @test_half_row_share3_v2i16(
; GFX11-NEXT:    [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
; GFX11-NEXT:    ret <2 x i16> [[RESULT]]
;
; GFX11-W64-LABEL: @test_half_row_share3_v2i16(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
; GFX11-W64-NEXT:    ret <2 x i16> [[RESULT]]
;
; GFX9-LABEL: @test_half_row_share3_v2i16(
; GFX9-NEXT:    [[TMP1:%.*]] = bitcast <2 x i16> [[VAL:%.*]] to i32
; GFX9-NEXT:    [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 120)
; GFX9-NEXT:    [[RESULT:%.*]] = bitcast i32 [[TMP2]] to <2 x i16>
; GFX9-NEXT:    ret <2 x i16> [[RESULT]]
;
  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %masked = and i32 %tid, -8
  %idx = or i32 %masked, 3
  %result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
  ret <2 x i16> %result
}

; Negative: 64-bit type, should not optimize.
define i64 @test_quad_perm_xor1_i64_negative(i64 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX11-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT:    ret i64 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i64 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_i64_negative(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i64 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 %val, i32 %idx)
  ret i64 %result
}

; Negative: 64-bit pointer, should not optimize.
define ptr @test_quad_perm_xor1_p0_negative(ptr %val) {
; GFX11-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX11-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT:    [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT:    ret ptr [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret ptr [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_p0_negative(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret ptr [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr %val, i32 %idx)
  ret ptr %result
}

; Negative: 16-bit type, should not optimize.
define i16 @test_quad_perm_xor1_i16_negative(i16 %val) {
; GFX11-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX11-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-NEXT:    [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT:    ret i16 [[RESULT]]
;
; GFX11-W64-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i16 [[RESULT]]
;
; GFX9-LABEL: @test_quad_perm_xor1_i16_negative(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX9-NEXT:    [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i16 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %result = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 %val, i32 %idx)
  ret i16 %result
}

; Negative: poison index, should not optimize.
define i32 @test_poison_index_negative(i32 %val) {
; GFX11-LABEL: @test_poison_index_negative(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_poison_index_negative(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_poison_index_negative(
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 poison)
  ret i32 %result
}

; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_nonconstant_shuffle_w32(
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Identity shuffle: lane_id itself.
define i32 @test_identity_shuffle_w32(i32 %val) {
; GFX11-LABEL: @test_identity_shuffle_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 228, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_identity_shuffle_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_identity_shuffle_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
  ret i32 %result
}

; Negative: index out of range (lane + 32 >= 32 for wave32).
define i32 @test_out_of_range_w32(i32 %val) {
; GFX11-LABEL: @test_out_of_range_w32(
; GFX11-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_out_of_range_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_out_of_range_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = add i32 %lane, 32
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; ds_bpermute with a constant shuffle pattern (quad perm via byte address).
define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
; GFX11-LABEL: @test_bpermute_quad_xor1_w32(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_bpermute_quad_xor1_w32(
; GFX11-W64-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT:    [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX11-W64-NEXT:    [[ADDR:%.*]] = xor i32 [[IDX]], 4
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_bpermute_quad_xor1_w32(
; GFX9-NEXT:    [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX9-NEXT:    [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
; GFX9-NEXT:    [[ADDR:%.*]] = xor i32 [[IDX]], 4
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %idx = xor i32 %lane, 1
  %addr = shl i32 %idx, 2
  %result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
  ret i32 %result
}

; XOR-by-5 within each 16-lane row. On GFX10+ this matches DPP row_xmask;
; on GFX9 row_xmask is unavailable, so the generic ds_swizzle bitmask path
; takes over: AND=0x1F, OR=0, XOR=5 -> 5151.
define i32 @test_xor5_bitmask(i32 %val) {
; GFX11-LABEL: @test_xor5_bitmask(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_xor5_bitmask(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_xor5_bitmask(
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 5151)
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %idx = xor i32 %lane, 5
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; XOR-by-16 (cross-row). On GFX11 this matches permlanex16; on GFX9 neither
; permlanex16 nor row_xmask exist, so the bitmask path catches it:
; AND=0x1F, OR=0, XOR=16 -> 16415.
define i32 @test_xor16_bitmask(i32 %val) {
; GFX11-LABEL: @test_xor16_bitmask(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_xor16_bitmask(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_xor16_bitmask(
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 16415)
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %idx = xor i32 %lane, 16
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}

; Broadcast lane 5 within each 16-lane row (lanes 0..15 read lane 5, lanes
; 16..31 read lane 21, etc.). On GFX10+ row_share covers it; on GFX9 there
; is no row_share, so the bitmask path emits AND=0x10, OR=5, XOR=0 -> 176.
define i32 @test_broadcast_in_rows16_bitmask(i32 %val) {
; GFX11-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX11-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
; GFX11-NEXT:    ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX11-W64-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
; GFX11-W64-NEXT:    ret i32 [[RESULT]]
;
; GFX9-LABEL: @test_broadcast_in_rows16_bitmask(
; GFX9-NEXT:    [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 176)
; GFX9-NEXT:    ret i32 [[RESULT]]
;
  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
  %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
  %hi = and i32 %lane, -16
  %idx = or i32 %hi, 5
  %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
  ret i32 %result
}