derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
752 lines
36 KiB
LLVM
752 lines
36 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
|
|
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
|
|
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX9
|
|
|
|
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
|
|
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
|
|
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
|
|
declare i64 @llvm.amdgcn.wave.shuffle.i64(i64, i32)
|
|
declare i16 @llvm.amdgcn.wave.shuffle.i16(i16, i32)
|
|
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
|
|
declare <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16>, i32)
|
|
declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
|
|
declare ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5), i32)
|
|
declare ptr @llvm.amdgcn.wave.shuffle.p0(ptr, i32)
|
|
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
|
|
|
|
; Quad perm: lane ^ 1 swaps adjacent pairs.
|
|
define i32 @test_quad_perm_xor1_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Quad perm: lane ^ 3 reverses within each quad.
|
|
define i32 @test_quad_perm_xor3_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor3_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor3_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 3
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Half-row mirror: reverse within every 8-lane group.
|
|
define i32 @test_half_row_mirror_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_half_row_mirror_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_half_row_mirror_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_half_row_mirror_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low3 = and i32 %lane, 7
|
|
%rev = sub i32 7, %low3
|
|
%high = and i32 %lane, -8
|
|
%idx = or i32 %high, %rev
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Full-row mirror: reverse within every 16-lane row.
|
|
define i32 @test_full_row_mirror_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_full_row_mirror_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_full_row_mirror_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_full_row_mirror_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low4 = and i32 %lane, 15
|
|
%rev = sub i32 15, %low4
|
|
%high = and i32 %lane, -16
|
|
%idx = or i32 %high, %rev
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Row rotate right by 3 within each 16-lane row.
|
|
define i32 @test_row_rotate_right3_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_row_rotate_right3_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
|
|
; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
|
|
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_row_rotate_right3_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
|
|
; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
|
|
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low4 = and i32 %lane, 15
|
|
%rot = add i32 %low4, 3
|
|
%wrapped = and i32 %rot, 15
|
|
%high = and i32 %lane, -16
|
|
%idx = or i32 %high, %wrapped
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Row share: broadcast lane 3 within each 16-lane row.
|
|
define i32 @test_row_share3_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_row_share3_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_row_share3_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_row_share3_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%high = and i32 %lane, -16
|
|
%idx = or i32 %high, 3
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Row xmask: xor each lane with mask 5 within each 16-lane row.
|
|
define i32 @test_row_xmask5_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_row_xmask5_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_row_xmask5_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_row_xmask5_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low4 = and i32 %lane, 15
|
|
%xored = xor i32 %low4, 5
|
|
%high = and i32 %lane, -16
|
|
%idx = or i32 %high, %xored
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Half-wave swap (wave64 only): lane ^ 32.
|
|
define i32 @test_half_wave_swap_w64(i32 %val) {
|
|
; GFX11-LABEL: @test_half_wave_swap_w64(
|
|
; GFX11-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LO]], 32
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_half_wave_swap_w64(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_half_wave_swap_w64(
|
|
; GFX9-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 32
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
|
|
%idx = xor i32 %lane, 32
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; DPP8: rotate left by 1 within each 8-lane group.
|
|
define i32 @test_dpp8_rotate1_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_dpp8_rotate1_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
|
|
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_dpp8_rotate1_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
|
|
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
|
|
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low3 = and i32 %lane, 7
|
|
%rot = add i32 %low3, 1
|
|
%wrapped = and i32 %rot, 7
|
|
%high = and i32 %lane, -8
|
|
%idx = or i32 %high, %wrapped
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
|
|
define i32 @test_permlane16_mul3_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_permlane16_mul3_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
|
|
; GFX11-W64-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
|
|
; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_permlane16_mul3_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
|
|
; GFX9-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
|
|
; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
|
|
; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%low4 = and i32 %lane, 15
|
|
%mul3 = mul i32 %low4, 3
|
|
%perm = and i32 %mul3, 15
|
|
%high = and i32 %lane, -16
|
|
%idx = or i32 %high, %perm
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; PermlaneX16: lane ^ 16. Cross-row swap; identity selectors.
|
|
define i32 @test_permlanex16_xor16_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_permlanex16_xor16_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_permlanex16_xor16_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_permlanex16_xor16_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 16
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; PermlaneX16: lane ^ 17. Cross-row + XOR-by-1 within row.
|
|
define i32 @test_permlanex16_xor17_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_permlanex16_xor17_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1732584193, i32 -271733879, i1 false, i1 false)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_permlanex16_xor17_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_permlanex16_xor17_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 17
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; PermlaneX16: lane ^ 31. Cross-row + reverse within row.
|
|
define i32 @test_permlanex16_xor31_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_permlanex16_xor31_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 -1985229329, i32 19088743, i1 false, i1 false)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_permlanex16_xor31_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_permlanex16_xor31_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 31
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Float type should work the same as i32.
|
|
define float @test_quad_perm_xor1_float_w32(float %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret float [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret float [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_float_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret float [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
|
|
ret float %result
|
|
}
|
|
|
|
; LDS (32-bit) pointer should optimize the same as i32: the gate must not
|
|
; reject 32-bit pointer types via Type::getPrimitiveSizeInBits().
|
|
define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_ptr3_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret ptr addrspace(3) [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr3_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret ptr addrspace(3) [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_ptr3_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret ptr addrspace(3) [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
|
|
ret ptr addrspace(3) %result
|
|
}
|
|
|
|
; Scratch (32-bit) pointer, should optimize like other 32-bit pointers.
|
|
define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_ptr5_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) poison, ptr addrspace(5) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret ptr addrspace(5) [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr5_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret ptr addrspace(5) [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_ptr5_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret ptr addrspace(5) [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) %val, i32 %idx)
|
|
ret ptr addrspace(5) %result
|
|
}
|
|
|
|
; <2 x i16> vector (32-bit), should optimize like i32.
|
|
define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_v2i16_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.update.dpp.v2i16(<2 x i16> poison, <2 x i16> [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_v2i16_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_v2i16_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
|
|
ret <2 x i16> %result
|
|
}
|
|
|
|
; Half-row-share with <2 x i16>: falls back to ds_swizzle on GFX9, using the bitcast path.
|
|
define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
|
|
; GFX11-LABEL: @test_half_row_share3_v2i16(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
|
|
; GFX11-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_half_row_share3_v2i16(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
|
|
; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_half_row_share3_v2i16(
|
|
; GFX9-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[VAL:%.*]] to i32
|
|
; GFX9-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 120)
|
|
; GFX9-NEXT: [[RESULT:%.*]] = bitcast i32 [[TMP2]] to <2 x i16>
|
|
; GFX9-NEXT: ret <2 x i16> [[RESULT]]
|
|
;
|
|
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
|
|
%masked = and i32 %tid, -8
|
|
%idx = or i32 %masked, 3
|
|
%result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
|
|
ret <2 x i16> %result
|
|
}
|
|
|
|
; Negative: 64-bit type, should not optimize.
|
|
define i64 @test_quad_perm_xor1_i64_negative(i64 %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_i64_negative(
|
|
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-NEXT: ret i64 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_i64_negative(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i64 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_i64_negative(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i64 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 %val, i32 %idx)
|
|
ret i64 %result
|
|
}
|
|
|
|
; Negative: 64-bit pointer, should not optimize.
|
|
define ptr @test_quad_perm_xor1_p0_negative(ptr %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_p0_negative(
|
|
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-NEXT: ret ptr [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_p0_negative(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret ptr [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_p0_negative(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret ptr [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr %val, i32 %idx)
|
|
ret ptr %result
|
|
}
|
|
|
|
; Negative: 16-bit type, should not optimize.
|
|
define i16 @test_quad_perm_xor1_i16_negative(i16 %val) {
|
|
; GFX11-LABEL: @test_quad_perm_xor1_i16_negative(
|
|
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-NEXT: ret i16 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_quad_perm_xor1_i16_negative(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i16 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_quad_perm_xor1_i16_negative(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i16 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%result = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 %val, i32 %idx)
|
|
ret i16 %result
|
|
}
|
|
|
|
; Negative: poison index, should not optimize.
|
|
define i32 @test_poison_index_negative(i32 %val) {
|
|
; GFX11-LABEL: @test_poison_index_negative(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_poison_index_negative(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_poison_index_negative(
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 poison)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Negative: non-constant index, should not be optimized.
|
|
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
|
|
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_nonconstant_shuffle_w32(
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Identity shuffle: lane_id itself.
|
|
define i32 @test_identity_shuffle_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_identity_shuffle_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 228, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_identity_shuffle_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_identity_shuffle_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Negative: index out of range (lane + 32 >= 32 for wave32).
|
|
define i32 @test_out_of_range_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_out_of_range_w32(
|
|
; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_out_of_range_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_out_of_range_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = add i32 %lane, 32
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; ds_bpermute with a constant shuffle pattern (quad perm via byte address).
|
|
define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
|
|
; GFX11-LABEL: @test_bpermute_quad_xor1_w32(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_bpermute_quad_xor1_w32(
|
|
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX11-W64-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
|
|
; GFX11-W64-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_bpermute_quad_xor1_w32(
|
|
; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
; GFX9-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
|
|
; GFX9-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%idx = xor i32 %lane, 1
|
|
%addr = shl i32 %idx, 2
|
|
%result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
|
|
ret i32 %result
|
|
}
|
|
|
|
; XOR-by-5 within each 16-lane row. On GFX10+ this matches DPP row_xmask;
|
|
; on GFX9 row_xmask is unavailable, so the generic ds_swizzle bitmask path
|
|
; takes over: AND=0x1F, OR=0, XOR=5 -> 5151.
|
|
define i32 @test_xor5_bitmask(i32 %val) {
|
|
; GFX11-LABEL: @test_xor5_bitmask(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_xor5_bitmask(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_xor5_bitmask(
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 5151)
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
|
|
%idx = xor i32 %lane, 5
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; XOR-by-16 (cross-row). On GFX11 this matches permlanex16; on GFX9 neither
|
|
; permlanex16 nor row_xmask exist, so the bitmask path catches it:
|
|
; AND=0x1F, OR=0, XOR=16 -> 16415.
|
|
define i32 @test_xor16_bitmask(i32 %val) {
|
|
; GFX11-LABEL: @test_xor16_bitmask(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_xor16_bitmask(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_xor16_bitmask(
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 16415)
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
|
|
%idx = xor i32 %lane, 16
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|
|
|
|
; Broadcast lane 5 within each 16-lane row (lanes 0..15 read lane 5, lanes
|
|
; 16..31 read lane 21, etc.). On GFX10+ row_share covers it; on GFX9 there
|
|
; is no row_share, so the bitmask path emits AND=0x10, OR=5, XOR=0 -> 176.
|
|
define i32 @test_broadcast_in_rows16_bitmask(i32 %val) {
|
|
; GFX11-LABEL: @test_broadcast_in_rows16_bitmask(
|
|
; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
|
|
; GFX11-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX11-W64-LABEL: @test_broadcast_in_rows16_bitmask(
|
|
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
|
|
; GFX11-W64-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
; GFX9-LABEL: @test_broadcast_in_rows16_bitmask(
|
|
; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 176)
|
|
; GFX9-NEXT: ret i32 [[RESULT]]
|
|
;
|
|
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
|
|
%hi = and i32 %lane, -16
|
|
%idx = or i32 %hi, 5
|
|
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
|
|
ret i32 %result
|
|
}
|