derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1713 lines
90 KiB
C
1713 lines
90 KiB
C
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 5
|
|
// RUN: %clang_cc1 -internal-isystem %S/Inputs/include \
|
|
// RUN: -internal-isystem %S/../../lib/Headers/ \
|
|
// RUN: -triple amdgcn-amd-amdhsa -emit-llvm %s -o - \
|
|
// RUN: | FileCheck %s --check-prefix=AMDGPU
|
|
// RUN: %clang_cc1 -internal-isystem %S/Inputs/include \
|
|
// RUN: -internal-isystem %S/../../lib/Headers/ \
|
|
// RUN: -triple nvptx64-nvidia-cuda -target-feature +ptx63 \
|
|
// RUN: -emit-llvm %s -o - \
|
|
// RUN: | FileCheck %s --check-prefix=NVPTX
|
|
// RUN: %clang_cc1 -internal-isystem %S/Inputs/include \
|
|
// RUN: -internal-isystem %S/../../lib/Headers/ \
|
|
// RUN: -triple spirv64-- -emit-llvm %s -o - \
|
|
// RUN: | FileCheck %s --check-prefix=SPIRV
|
|
|
|
#include <gpuintrin.h>
|
|
|
|
__gpu_kernel void foo() {
|
|
__gpu_num_blocks_x();
|
|
__gpu_num_blocks_y();
|
|
__gpu_num_blocks_z();
|
|
__gpu_num_blocks(0);
|
|
__gpu_block_id_x();
|
|
__gpu_block_id_y();
|
|
__gpu_block_id_z();
|
|
__gpu_block_id(0);
|
|
__gpu_num_threads_x();
|
|
__gpu_num_threads_y();
|
|
__gpu_num_threads_z();
|
|
__gpu_num_threads(0);
|
|
__gpu_thread_id_x();
|
|
__gpu_thread_id_y();
|
|
__gpu_thread_id_z();
|
|
__gpu_thread_id(0);
|
|
__gpu_num_lanes();
|
|
__gpu_lane_id();
|
|
__gpu_lane_mask();
|
|
__gpu_read_first_lane_u32(-1, -1);
|
|
__gpu_read_first_lane_u64(-1, -1);
|
|
__gpu_ballot(-1, 1);
|
|
__gpu_sync_threads();
|
|
__gpu_sync_lane(-1);
|
|
__gpu_shuffle_idx_u32(-1, -1, -1, 0);
|
|
__gpu_first_lane_id(-1);
|
|
__gpu_is_first_in_lane(-1);
|
|
__gpu_prefix_scan_add_u32(~0, 1);
|
|
__gpu_exit();
|
|
}
|
|
// AMDGPU-LABEL: define protected amdgpu_kernel void @foo(
|
|
// AMDGPU-SAME: ) #[[ATTR0:[0-9]+]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR8:[0-9]+]]
|
|
// AMDGPU-NEXT: [[CALL1:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL3:%.*]] = call i32 @__gpu_num_blocks(i32 noundef 0) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL5:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL6:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL7:%.*]] = call i32 @__gpu_block_id(i32 noundef 0) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL8:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL9:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL10:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL11:%.*]] = call i32 @__gpu_num_threads(i32 noundef 0) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL12:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL13:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL14:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL15:%.*]] = call i32 @__gpu_thread_id(i32 noundef 0) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL16:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL17:%.*]] = call i32 @__gpu_lane_id() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL18:%.*]] = call i64 @__gpu_lane_mask() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL19:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef -1, i32 noundef -1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL20:%.*]] = call i64 @__gpu_read_first_lane_u64(i64 noundef -1, i64 noundef -1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL21:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR8]]
|
|
// AMDGPU-NEXT: call void @__gpu_sync_threads() #[[ATTR8]]
|
|
// AMDGPU-NEXT: call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL22:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL23:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL24:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL25:%.*]] = call i32 @__gpu_prefix_scan_add_u32(i64 noundef -1, i32 noundef 1) #[[ATTR8]]
|
|
// AMDGPU-NEXT: call void @__gpu_exit() #[[ATTR9:[0-9]+]]
|
|
// AMDGPU-NEXT: unreachable
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_x(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 12
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG2:![0-9]+]], !invariant.load [[META3:![0-9]+]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i64 0
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 12, i32 18
|
|
// AMDGPU-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP3]], i32 [[TMP8]]
|
|
// AMDGPU-NEXT: [[TMP10:%.*]] = load i16, ptr addrspace(4) [[TMP9]], align 2, !range [[RNG4:![0-9]+]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
|
|
// AMDGPU-NEXT: [[DIV:%.*]] = udiv i32 [[TMP2]], [[TMP11]]
|
|
// AMDGPU-NEXT: ret i32 [[DIV]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_y(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 16
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG2]], !invariant.load [[META3]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i64 4
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 14, i32 20
|
|
// AMDGPU-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP3]], i32 [[TMP8]]
|
|
// AMDGPU-NEXT: [[TMP10:%.*]] = load i16, ptr addrspace(4) [[TMP9]], align 2, !range [[RNG4]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
|
|
// AMDGPU-NEXT: [[DIV:%.*]] = udiv i32 [[TMP2]], [[TMP11]]
|
|
// AMDGPU-NEXT: ret i32 [[DIV]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks_z(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i32 20
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !range [[RNG2]], !invariant.load [[META3]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP3]], i64 8
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(4) [[TMP4]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = icmp ult i32 [[TMP6]], [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = select i1 [[TMP7]], i32 16, i32 22
|
|
// AMDGPU-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP3]], i32 [[TMP8]]
|
|
// AMDGPU-NEXT: [[TMP10:%.*]] = load i16, ptr addrspace(4) [[TMP9]], align 2, !range [[RNG4]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
|
|
// AMDGPU-NEXT: [[DIV:%.*]] = udiv i32 [[TMP2]], [[TMP11]]
|
|
// AMDGPU-NEXT: ret i32 [[DIV]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_blocks(
|
|
// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// AMDGPU-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// AMDGPU-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// AMDGPU-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// AMDGPU-NEXT: ]
|
|
// AMDGPU: [[SW_BB]]:
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN:.*]]
|
|
// AMDGPU: [[SW_BB1]]:
|
|
// AMDGPU-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL2]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_BB3]]:
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL4]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_DEFAULT]]:
|
|
// AMDGPU-NEXT: store i32 1, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[RETURN]]:
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_block_id_x(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_block_id_y(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_block_id_z(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_block_id(
|
|
// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// AMDGPU-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// AMDGPU-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// AMDGPU-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// AMDGPU-NEXT: ]
|
|
// AMDGPU: [[SW_BB]]:
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN:.*]]
|
|
// AMDGPU: [[SW_BB1]]:
|
|
// AMDGPU-NEXT: [[CALL2:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL2]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_BB3]]:
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL4]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_DEFAULT]]:
|
|
// AMDGPU-NEXT: store i32 0, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[RETURN]]:
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_x(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i64 0
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], [[TMP2]]
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i32 12, i32 18
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP0]], i32 [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = load i16, ptr addrspace(4) [[TMP6]], align 2, !range [[RNG4]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
|
|
// AMDGPU-NEXT: ret i32 [[TMP8]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_y(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i64 4
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.y()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], [[TMP2]]
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i32 14, i32 20
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP0]], i32 [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = load i16, ptr addrspace(4) [[TMP6]], align 2, !range [[RNG4]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
|
|
// AMDGPU-NEXT: ret i32 [[TMP8]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_threads_z(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call align 8 dereferenceable(256) ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[TMP0]], i64 8
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(4) [[TMP1]], align 4, !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.workgroup.id.z()
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = icmp ult i32 [[TMP3]], [[TMP2]]
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i32 16, i32 22
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[TMP0]], i32 [[TMP5]]
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = load i16, ptr addrspace(4) [[TMP6]], align 2, !range [[RNG4]], !invariant.load [[META3]], !noundef [[META3]]
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = zext i16 [[TMP7]] to i32
|
|
// AMDGPU-NEXT: ret i32 [[TMP8]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_threads(
|
|
// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// AMDGPU-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// AMDGPU-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// AMDGPU-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// AMDGPU-NEXT: ]
|
|
// AMDGPU: [[SW_BB]]:
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN:.*]]
|
|
// AMDGPU: [[SW_BB1]]:
|
|
// AMDGPU-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL2]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_BB3]]:
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL4]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_DEFAULT]]:
|
|
// AMDGPU-NEXT: store i32 1, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[RETURN]]:
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_x(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_y(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.y()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_thread_id_z(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.workitem.id.z()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_thread_id(
|
|
// AMDGPU-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__DIM_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__DIM_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// AMDGPU-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// AMDGPU-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// AMDGPU-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// AMDGPU-NEXT: ]
|
|
// AMDGPU: [[SW_BB]]:
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN:.*]]
|
|
// AMDGPU: [[SW_BB1]]:
|
|
// AMDGPU-NEXT: [[CALL2:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL2]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_BB3]]:
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL4]], ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[SW_DEFAULT]]:
|
|
// AMDGPU-NEXT: store i32 0, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: br label %[[RETURN]]
|
|
// AMDGPU: [[RETURN]]:
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(5) [[RETVAL]], align 4
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_num_lanes(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.wavefrontsize()
|
|
// AMDGPU-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_lane_id(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP0]])
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i64 @__gpu_lane_mask(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
|
|
// AMDGPU-NEXT: ret i64 [[TMP0]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_read_first_lane_u32(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: store i32 [[__X]], ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP0]])
|
|
// AMDGPU-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i64 @__gpu_read_first_lane_u64(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i64 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__X_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__HI:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LO:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__HI_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__HI]] to ptr
|
|
// AMDGPU-NEXT: [[__LO_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LO]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: store i64 [[__X]], ptr [[__X_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i64, ptr [[__X_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
|
|
// AMDGPU-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
|
|
// AMDGPU-NEXT: store i32 [[CONV]], ptr [[__HI_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i64, ptr [[__X_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[AND:%.*]] = and i64 [[TMP1]], 4294967295
|
|
// AMDGPU-NEXT: [[CONV1:%.*]] = trunc i64 [[AND]] to i32
|
|
// AMDGPU-NEXT: store i32 [[CONV1]], ptr [[__LO_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = load i32, ptr [[__HI_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP2]], i32 noundef [[TMP3]]) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CONV2:%.*]] = zext i32 [[CALL]] to i64
|
|
// AMDGPU-NEXT: [[SHL:%.*]] = shl i64 [[CONV2]], 32
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = load i32, ptr [[__LO_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CALL3:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]]) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CONV4:%.*]] = zext i32 [[CALL3]] to i64
|
|
// AMDGPU-NEXT: [[AND5:%.*]] = and i64 [[CONV4]], 4294967295
|
|
// AMDGPU-NEXT: [[OR:%.*]] = or i64 [[SHL]], [[AND5]]
|
|
// AMDGPU-NEXT: ret i64 [[OR]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i64 @__gpu_ballot(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i1 noundef zeroext [[__X:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__X_ADDR:%.*]] = alloca i8, align 1, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[STOREDV:%.*]] = zext i1 [[__X]] to i8
|
|
// AMDGPU-NEXT: store i8 [[STOREDV]], ptr [[__X_ADDR_ASCAST]], align 1
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i8, ptr [[__X_ADDR_ASCAST]], align 1
|
|
// AMDGPU-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[TMP1]], 0
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[LOADEDV]])
|
|
// AMDGPU-NEXT: [[AND:%.*]] = and i64 [[TMP0]], [[TMP2]]
|
|
// AMDGPU-NEXT: ret i64 [[AND]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal void @__gpu_sync_threads(
|
|
// AMDGPU-SAME: ) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: call void @llvm.amdgcn.s.barrier()
|
|
// AMDGPU-NEXT: fence syncscope("workgroup") seq_cst
|
|
// AMDGPU-NEXT: ret void
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal void @__gpu_sync_lane(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: call void @llvm.amdgcn.wave.barrier()
|
|
// AMDGPU-NEXT: ret void
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_shuffle_idx_u32(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__IDX:%.*]], i32 noundef [[__X:%.*]], i32 noundef [[__WIDTH:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__IDX_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__WIDTH_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__IDX_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__IDX_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__WIDTH_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__WIDTH_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__LANE_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: store i32 [[__IDX]], ptr [[__IDX_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: store i32 [[__X]], ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: store i32 [[__WIDTH]], ptr [[__WIDTH_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i32, ptr [[__IDX_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr [[__WIDTH_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[SUB:%.*]] = sub i32 [[TMP1]], 1
|
|
// AMDGPU-NEXT: [[NOT:%.*]] = xor i32 [[SUB]], -1
|
|
// AMDGPU-NEXT: [[AND:%.*]] = and i32 [[CALL]], [[NOT]]
|
|
// AMDGPU-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], [[AND]]
|
|
// AMDGPU-NEXT: store i32 [[ADD]], ptr [[__LANE_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i32, ptr [[__LANE_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[SHL:%.*]] = shl i32 [[TMP2]], 2
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[SHL]], i32 [[TMP3]])
|
|
// AMDGPU-NEXT: ret i32 [[TMP4]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i64 @__gpu_first_lane_id(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 1
|
|
// AMDGPU-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP0]], 0
|
|
// AMDGPU-NEXT: [[FFS:%.*]] = select i1 [[ISZERO]], i64 0, i64 [[TMP2]]
|
|
// AMDGPU-NEXT: [[CAST:%.*]] = trunc i64 [[FFS]] to i32
|
|
// AMDGPU-NEXT: [[SUB:%.*]] = sub nsw i32 [[CAST]], 1
|
|
// AMDGPU-NEXT: [[CONV:%.*]] = sext i32 [[SUB]] to i64
|
|
// AMDGPU-NEXT: ret i64 [[CONV]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal zeroext i1 @__gpu_is_first_in_lane(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CONV:%.*]] = zext i32 [[CALL]] to i64
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[CALL1:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef [[TMP0]]) #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CMP:%.*]] = icmp eq i64 [[CONV]], [[CALL1]]
|
|
// AMDGPU-NEXT: ret i1 [[CMP]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal i32 @__gpu_prefix_scan_add_u32(
|
|
// AMDGPU-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__BELOW:%.*]] = alloca i64, align 8, addrspace(5)
|
|
// AMDGPU-NEXT: [[__STEP:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__SRC:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__RESULT:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__I:%.*]] = alloca i32, align 4, addrspace(5)
|
|
// AMDGPU-NEXT: [[__LANE_MASK_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__LANE_MASK_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__X_ADDR]] to ptr
|
|
// AMDGPU-NEXT: [[__BELOW_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__BELOW]] to ptr
|
|
// AMDGPU-NEXT: [[__STEP_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__STEP]] to ptr
|
|
// AMDGPU-NEXT: [[__SRC_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__SRC]] to ptr
|
|
// AMDGPU-NEXT: [[__RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__RESULT]] to ptr
|
|
// AMDGPU-NEXT: [[__I_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[__I]] to ptr
|
|
// AMDGPU-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: store i32 [[__X]], ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[SH_PROM:%.*]] = zext i32 [[CALL]] to i64
|
|
// AMDGPU-NEXT: [[SHL:%.*]] = shl i64 1, [[SH_PROM]]
|
|
// AMDGPU-NEXT: [[SUB:%.*]] = sub i64 [[SHL]], 1
|
|
// AMDGPU-NEXT: [[AND:%.*]] = and i64 [[TMP0]], [[SUB]]
|
|
// AMDGPU-NEXT: store i64 [[AND]], ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: store i32 1, ptr [[__STEP_ASCAST]], align 4
|
|
// AMDGPU-NEXT: br label %[[FOR_COND:.*]]
|
|
// AMDGPU: [[FOR_COND]]:
|
|
// AMDGPU-NEXT: [[TMP1:%.*]] = load i32, ptr [[__STEP_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CALL1:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CMP:%.*]] = icmp ult i32 [[TMP1]], [[CALL1]]
|
|
// AMDGPU-NEXT: br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
|
|
// AMDGPU: [[FOR_BODY]]:
|
|
// AMDGPU-NEXT: [[TMP2:%.*]] = load i64, ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP2]], i1 true)
|
|
// AMDGPU-NEXT: [[CAST:%.*]] = trunc i64 [[TMP3]] to i32
|
|
// AMDGPU-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP2]], 0
|
|
// AMDGPU-NEXT: [[CLZG:%.*]] = select i1 [[ISZERO]], i32 64, i32 [[CAST]]
|
|
// AMDGPU-NEXT: [[SUB2:%.*]] = sub nsw i32 63, [[CLZG]]
|
|
// AMDGPU-NEXT: store i32 [[SUB2]], ptr [[__SRC_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP5:%.*]] = load i32, ptr [[__SRC_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP6:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CALL3:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR8]]
|
|
// AMDGPU-NEXT: [[CALL4:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]], i32 noundef [[TMP6]], i32 noundef [[CALL3]]) #[[ATTR8]]
|
|
// AMDGPU-NEXT: store i32 [[CALL4]], ptr [[__RESULT_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP7:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP8:%.*]] = load i64, ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TOBOOL:%.*]] = icmp ne i64 [[TMP8]], 0
|
|
// AMDGPU-NEXT: br i1 [[TOBOOL]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
|
|
// AMDGPU: [[COND_TRUE]]:
|
|
// AMDGPU-NEXT: [[TMP9:%.*]] = load i32, ptr [[__RESULT_ASCAST]], align 4
|
|
// AMDGPU-NEXT: br label %[[COND_END:.*]]
|
|
// AMDGPU: [[COND_FALSE]]:
|
|
// AMDGPU-NEXT: br label %[[COND_END]]
|
|
// AMDGPU: [[COND_END]]:
|
|
// AMDGPU-NEXT: [[COND:%.*]] = phi i32 [ [[TMP9]], %[[COND_TRUE]] ], [ 0, %[[COND_FALSE]] ]
|
|
// AMDGPU-NEXT: [[ADD:%.*]] = add i32 [[TMP7]], [[COND]]
|
|
// AMDGPU-NEXT: store i32 [[ADD]], ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: store i32 0, ptr [[__I_ASCAST]], align 4
|
|
// AMDGPU-NEXT: br label %[[FOR_COND5:.*]]
|
|
// AMDGPU: [[FOR_COND5]]:
|
|
// AMDGPU-NEXT: [[TMP10:%.*]] = load i32, ptr [[__I_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[TMP11:%.*]] = load i32, ptr [[__STEP_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[CMP6:%.*]] = icmp ult i32 [[TMP10]], [[TMP11]]
|
|
// AMDGPU-NEXT: br i1 [[CMP6]], label %[[FOR_BODY7:.*]], label %[[FOR_END:.*]]
|
|
// AMDGPU: [[FOR_BODY7]]:
|
|
// AMDGPU-NEXT: [[TMP12:%.*]] = load i64, ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[TMP13:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP12]], i1 true)
|
|
// AMDGPU-NEXT: [[CAST8:%.*]] = trunc i64 [[TMP13]] to i32
|
|
// AMDGPU-NEXT: [[ISZERO9:%.*]] = icmp eq i64 [[TMP12]], 0
|
|
// AMDGPU-NEXT: [[CLZG10:%.*]] = select i1 [[ISZERO9]], i32 0, i32 [[CAST8]]
|
|
// AMDGPU-NEXT: [[SUB11:%.*]] = sub nsw i32 63, [[CLZG10]]
|
|
// AMDGPU-NEXT: [[SH_PROM12:%.*]] = zext i32 [[SUB11]] to i64
|
|
// AMDGPU-NEXT: [[SHL13:%.*]] = shl i64 1, [[SH_PROM12]]
|
|
// AMDGPU-NEXT: [[TMP14:%.*]] = load i64, ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[AND14:%.*]] = and i64 [[SHL13]], [[TMP14]]
|
|
// AMDGPU-NEXT: [[TMP15:%.*]] = load i64, ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: [[XOR:%.*]] = xor i64 [[TMP15]], [[AND14]]
|
|
// AMDGPU-NEXT: store i64 [[XOR]], ptr [[__BELOW_ASCAST]], align 8
|
|
// AMDGPU-NEXT: br label %[[FOR_INC:.*]]
|
|
// AMDGPU: [[FOR_INC]]:
|
|
// AMDGPU-NEXT: [[TMP16:%.*]] = load i32, ptr [[__I_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[INC:%.*]] = add i32 [[TMP16]], 1
|
|
// AMDGPU-NEXT: store i32 [[INC]], ptr [[__I_ASCAST]], align 4
|
|
// AMDGPU-NEXT: br label %[[FOR_COND5]], !llvm.loop [[LOOP5:![0-9]+]]
|
|
// AMDGPU: [[FOR_END]]:
|
|
// AMDGPU-NEXT: br label %[[FOR_INC15:.*]]
|
|
// AMDGPU: [[FOR_INC15]]:
|
|
// AMDGPU-NEXT: [[TMP17:%.*]] = load i32, ptr [[__STEP_ASCAST]], align 4
|
|
// AMDGPU-NEXT: [[MUL:%.*]] = mul i32 [[TMP17]], 2
|
|
// AMDGPU-NEXT: store i32 [[MUL]], ptr [[__STEP_ASCAST]], align 4
|
|
// AMDGPU-NEXT: br label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
// AMDGPU: [[FOR_END16]]:
|
|
// AMDGPU-NEXT: [[TMP18:%.*]] = load i32, ptr [[__X_ADDR_ASCAST]], align 4
|
|
// AMDGPU-NEXT: ret i32 [[TMP18]]
|
|
//
|
|
//
|
|
// AMDGPU-LABEL: define internal void @__gpu_exit(
|
|
// AMDGPU-SAME: ) #[[ATTR1:[0-9]+]] {
|
|
// AMDGPU-NEXT: [[ENTRY:.*:]]
|
|
// AMDGPU-NEXT: call void @llvm.amdgcn.endpgm()
|
|
// AMDGPU-NEXT: ret void
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define protected ptx_kernel void @foo(
|
|
// NVPTX-SAME: ) #[[ATTR0:[0-9]+]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR6:[0-9]+]]
|
|
// NVPTX-NEXT: [[CALL1:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL3:%.*]] = call i32 @__gpu_num_blocks(i32 noundef 0) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL5:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL6:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL7:%.*]] = call i32 @__gpu_block_id(i32 noundef 0) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL8:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL9:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL10:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL11:%.*]] = call i32 @__gpu_num_threads(i32 noundef 0) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL12:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL13:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL14:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL15:%.*]] = call i32 @__gpu_thread_id(i32 noundef 0) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL16:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL17:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL18:%.*]] = call i64 @__gpu_lane_mask() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL19:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef -1, i32 noundef -1) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL20:%.*]] = call i64 @__gpu_read_first_lane_u64(i64 noundef -1, i64 noundef -1) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL21:%.*]] = call i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true) #[[ATTR6]]
|
|
// NVPTX-NEXT: call void @__gpu_sync_threads() #[[ATTR6]]
|
|
// NVPTX-NEXT: call void @__gpu_sync_lane(i64 noundef -1) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL22:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL23:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef -1) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL24:%.*]] = call zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL25:%.*]] = call i32 @__gpu_prefix_scan_add_u32(i64 noundef -1, i32 noundef 1) #[[ATTR6]]
|
|
// NVPTX-NEXT: call void @__gpu_exit() #[[ATTR7:[0-9]+]]
|
|
// NVPTX-NEXT: unreachable
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_x(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_y(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_blocks_z(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_blocks(
|
|
// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// NVPTX-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// NVPTX-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// NVPTX-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// NVPTX-NEXT: ]
|
|
// NVPTX: [[SW_BB]]:
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_blocks_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN:.*]]
|
|
// NVPTX: [[SW_BB1]]:
|
|
// NVPTX-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_blocks_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_BB3]]:
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_num_blocks_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_DEFAULT]]:
|
|
// NVPTX-NEXT: store i32 1, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[RETURN]]:
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_block_id_x(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_block_id_y(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_block_id_z(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_block_id(
|
|
// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// NVPTX-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// NVPTX-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// NVPTX-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// NVPTX-NEXT: ]
|
|
// NVPTX: [[SW_BB]]:
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_block_id_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN:.*]]
|
|
// NVPTX: [[SW_BB1]]:
|
|
// NVPTX-NEXT: [[CALL2:%.*]] = call i32 @__gpu_block_id_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_BB3]]:
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_block_id_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_DEFAULT]]:
|
|
// NVPTX-NEXT: store i32 0, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[RETURN]]:
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_threads_x(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_threads_y(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_threads_z(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_threads(
|
|
// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// NVPTX-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// NVPTX-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// NVPTX-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// NVPTX-NEXT: ]
|
|
// NVPTX: [[SW_BB]]:
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_threads_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN:.*]]
|
|
// NVPTX: [[SW_BB1]]:
|
|
// NVPTX-NEXT: [[CALL2:%.*]] = call i32 @__gpu_num_threads_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_BB3]]:
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_num_threads_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_DEFAULT]]:
|
|
// NVPTX-NEXT: store i32 1, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[RETURN]]:
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_thread_id_x(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_thread_id_y(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.y()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_thread_id_z(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.z()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_thread_id(
|
|
// NVPTX-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// NVPTX-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// NVPTX-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// NVPTX-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// NVPTX-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// NVPTX-NEXT: ]
|
|
// NVPTX: [[SW_BB]]:
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_thread_id_x() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN:.*]]
|
|
// NVPTX: [[SW_BB1]]:
|
|
// NVPTX-NEXT: [[CALL2:%.*]] = call i32 @__gpu_thread_id_y() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_BB3]]:
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_thread_id_z() #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[SW_DEFAULT]]:
|
|
// NVPTX-NEXT: store i32 0, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: br label %[[RETURN]]
|
|
// NVPTX: [[RETURN]]:
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// NVPTX-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_num_lanes(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.warpsize()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_lane_id(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.laneid()
|
|
// NVPTX-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i64 @__gpu_lane_mask(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.activemask()
|
|
// NVPTX-NEXT: [[CONV:%.*]] = zext i32 [[TMP0]] to i64
|
|
// NVPTX-NEXT: ret i64 [[CONV]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_read_first_lane_u32(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__MASK:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__ID:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
|
|
// NVPTX-NEXT: store i32 [[CONV]], ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = call i32 @llvm.cttz.i32(i32 [[TMP1]], i1 true)
|
|
// NVPTX-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 1
|
|
// NVPTX-NEXT: [[ISZERO:%.*]] = icmp eq i32 [[TMP1]], 0
|
|
// NVPTX-NEXT: [[FFS:%.*]] = select i1 [[ISZERO]], i32 0, i32 [[TMP3]]
|
|
// NVPTX-NEXT: [[SUB:%.*]] = sub nsw i32 [[FFS]], 1
|
|
// NVPTX-NEXT: store i32 [[SUB]], ptr [[__ID]], align 4
|
|
// NVPTX-NEXT: [[TMP4:%.*]] = load i32, ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP5:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP6:%.*]] = load i32, ptr [[__ID]], align 4
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[SUB1:%.*]] = sub i32 [[CALL]], 1
|
|
// NVPTX-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.shfl.sync.idx.i32(i32 [[TMP4]], i32 [[TMP5]], i32 [[TMP6]], i32 [[SUB1]])
|
|
// NVPTX-NEXT: ret i32 [[TMP7]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i64 @__gpu_read_first_lane_u64(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i64 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__X_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__HI:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__LO:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: store i64 [[__X]], ptr [[__X_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__X_ADDR]], align 8
|
|
// NVPTX-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
|
|
// NVPTX-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
|
|
// NVPTX-NEXT: store i32 [[CONV]], ptr [[__HI]], align 4
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i64, ptr [[__X_ADDR]], align 8
|
|
// NVPTX-NEXT: [[AND:%.*]] = and i64 [[TMP1]], 4294967295
|
|
// NVPTX-NEXT: [[CONV1:%.*]] = trunc i64 [[AND]] to i32
|
|
// NVPTX-NEXT: store i32 [[CONV1]], ptr [[__LO]], align 4
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP3:%.*]] = load i32, ptr [[__HI]], align 4
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP2]], i32 noundef [[TMP3]]) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CONV2:%.*]] = zext i32 [[CALL]] to i64
|
|
// NVPTX-NEXT: [[SHL:%.*]] = shl i64 [[CONV2]], 32
|
|
// NVPTX-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP5:%.*]] = load i32, ptr [[__LO]], align 4
|
|
// NVPTX-NEXT: [[CALL3:%.*]] = call i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]]) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CONV4:%.*]] = zext i32 [[CALL3]] to i64
|
|
// NVPTX-NEXT: [[AND5:%.*]] = and i64 [[CONV4]], 4294967295
|
|
// NVPTX-NEXT: [[OR:%.*]] = or i64 [[SHL]], [[AND5]]
|
|
// NVPTX-NEXT: ret i64 [[OR]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i64 @__gpu_ballot(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i1 noundef zeroext [[__X:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__X_ADDR:%.*]] = alloca i8, align 1
|
|
// NVPTX-NEXT: [[__MASK:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[STOREDV:%.*]] = zext i1 [[__X]] to i8
|
|
// NVPTX-NEXT: store i8 [[STOREDV]], ptr [[__X_ADDR]], align 1
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
|
|
// NVPTX-NEXT: store i32 [[CONV]], ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = load i8, ptr [[__X_ADDR]], align 1
|
|
// NVPTX-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[TMP2]], 0
|
|
// NVPTX-NEXT: [[TMP3:%.*]] = call i32 @llvm.nvvm.vote.ballot.sync(i32 [[TMP1]], i1 [[LOADEDV]])
|
|
// NVPTX-NEXT: [[CONV1:%.*]] = zext i32 [[TMP3]] to i64
|
|
// NVPTX-NEXT: ret i64 [[CONV1]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal void @__gpu_sync_threads(
|
|
// NVPTX-SAME: ) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: call void @llvm.nvvm.barrier.cta.sync.aligned.all(i32 0)
|
|
// NVPTX-NEXT: ret void
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal void @__gpu_sync_lane(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
|
|
// NVPTX-NEXT: call void @llvm.nvvm.bar.warp.sync(i32 [[CONV]])
|
|
// NVPTX-NEXT: ret void
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_shuffle_idx_u32(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__IDX:%.*]], i32 noundef [[__X:%.*]], i32 noundef [[__WIDTH:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__IDX_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__WIDTH_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__MASK:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__BITMASK:%.*]] = alloca i8, align 1
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: store i32 [[__IDX]], ptr [[__IDX_ADDR]], align 4
|
|
// NVPTX-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: store i32 [[__WIDTH]], ptr [[__WIDTH_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CONV:%.*]] = trunc i64 [[TMP0]] to i32
|
|
// NVPTX-NEXT: store i32 [[CONV]], ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[__IDX_ADDR]], align 4
|
|
// NVPTX-NEXT: [[SH_PROM:%.*]] = zext i32 [[TMP1]] to i64
|
|
// NVPTX-NEXT: [[SHL:%.*]] = shl i64 1, [[SH_PROM]]
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[AND:%.*]] = and i64 [[SHL]], [[TMP2]]
|
|
// NVPTX-NEXT: [[TOBOOL:%.*]] = icmp ne i64 [[AND]], 0
|
|
// NVPTX-NEXT: [[STOREDV:%.*]] = zext i1 [[TOBOOL]] to i8
|
|
// NVPTX-NEXT: store i8 [[STOREDV]], ptr [[__BITMASK]], align 1
|
|
// NVPTX-NEXT: [[TMP3:%.*]] = load i8, ptr [[__BITMASK]], align 1
|
|
// NVPTX-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[TMP3]], 0
|
|
// NVPTX-NEXT: [[CONV1:%.*]] = zext i1 [[LOADEDV]] to i32
|
|
// NVPTX-NEXT: [[SUB:%.*]] = sub nsw i32 0, [[CONV1]]
|
|
// NVPTX-NEXT: [[TMP4:%.*]] = load i32, ptr [[__MASK]], align 4
|
|
// NVPTX-NEXT: [[TMP5:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP6:%.*]] = load i32, ptr [[__IDX_ADDR]], align 4
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[TMP7:%.*]] = load i32, ptr [[__WIDTH_ADDR]], align 4
|
|
// NVPTX-NEXT: [[SUB2:%.*]] = sub i32 [[CALL]], [[TMP7]]
|
|
// NVPTX-NEXT: [[SHL3:%.*]] = shl i32 [[SUB2]], 8
|
|
// NVPTX-NEXT: [[OR:%.*]] = or i32 [[SHL3]], 31
|
|
// NVPTX-NEXT: [[TMP8:%.*]] = call i32 @llvm.nvvm.shfl.sync.idx.i32(i32 [[TMP4]], i32 [[TMP5]], i32 [[TMP6]], i32 [[OR]])
|
|
// NVPTX-NEXT: [[AND4:%.*]] = and i32 [[SUB]], [[TMP8]]
|
|
// NVPTX-NEXT: ret i32 [[AND4]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i64 @__gpu_first_lane_id(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 1
|
|
// NVPTX-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP0]], 0
|
|
// NVPTX-NEXT: [[FFS:%.*]] = select i1 [[ISZERO]], i64 0, i64 [[TMP2]]
|
|
// NVPTX-NEXT: [[CAST:%.*]] = trunc i64 [[FFS]] to i32
|
|
// NVPTX-NEXT: [[SUB:%.*]] = sub nsw i32 [[CAST]], 1
|
|
// NVPTX-NEXT: [[CONV:%.*]] = sext i32 [[SUB]] to i64
|
|
// NVPTX-NEXT: ret i64 [[CONV]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal zeroext i1 @__gpu_is_first_in_lane(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CONV:%.*]] = zext i32 [[CALL]] to i64
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CALL1:%.*]] = call i64 @__gpu_first_lane_id(i64 noundef [[TMP0]]) #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CMP:%.*]] = icmp eq i64 [[CONV]], [[CALL1]]
|
|
// NVPTX-NEXT: ret i1 [[CMP]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal i32 @__gpu_prefix_scan_add_u32(
|
|
// NVPTX-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__BELOW:%.*]] = alloca i64, align 8
|
|
// NVPTX-NEXT: [[__STEP:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__SRC:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__RESULT:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: [[__I:%.*]] = alloca i32, align 4
|
|
// NVPTX-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[CALL:%.*]] = call i32 @__gpu_lane_id() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[SH_PROM:%.*]] = zext i32 [[CALL]] to i64
|
|
// NVPTX-NEXT: [[SHL:%.*]] = shl i64 1, [[SH_PROM]]
|
|
// NVPTX-NEXT: [[SUB:%.*]] = sub i64 [[SHL]], 1
|
|
// NVPTX-NEXT: [[AND:%.*]] = and i64 [[TMP0]], [[SUB]]
|
|
// NVPTX-NEXT: store i64 [[AND]], ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: store i32 1, ptr [[__STEP]], align 4
|
|
// NVPTX-NEXT: br label %[[FOR_COND:.*]]
|
|
// NVPTX: [[FOR_COND]]:
|
|
// NVPTX-NEXT: [[TMP1:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// NVPTX-NEXT: [[CALL1:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CMP:%.*]] = icmp ult i32 [[TMP1]], [[CALL1]]
|
|
// NVPTX-NEXT: br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
|
|
// NVPTX: [[FOR_BODY]]:
|
|
// NVPTX-NEXT: [[TMP2:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP2]], i1 true)
|
|
// NVPTX-NEXT: [[CAST:%.*]] = trunc i64 [[TMP3]] to i32
|
|
// NVPTX-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP2]], 0
|
|
// NVPTX-NEXT: [[CLZG:%.*]] = select i1 [[ISZERO]], i32 64, i32 [[CAST]]
|
|
// NVPTX-NEXT: [[SUB2:%.*]] = sub nsw i32 63, [[CLZG]]
|
|
// NVPTX-NEXT: store i32 [[SUB2]], ptr [[__SRC]], align 4
|
|
// NVPTX-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// NVPTX-NEXT: [[TMP5:%.*]] = load i32, ptr [[__SRC]], align 4
|
|
// NVPTX-NEXT: [[TMP6:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[CALL3:%.*]] = call i32 @__gpu_num_lanes() #[[ATTR6]]
|
|
// NVPTX-NEXT: [[CALL4:%.*]] = call i32 @__gpu_shuffle_idx_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]], i32 noundef [[TMP6]], i32 noundef [[CALL3]]) #[[ATTR6]]
|
|
// NVPTX-NEXT: store i32 [[CALL4]], ptr [[__RESULT]], align 4
|
|
// NVPTX-NEXT: [[TMP7:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: [[TMP8:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: [[TOBOOL:%.*]] = icmp ne i64 [[TMP8]], 0
|
|
// NVPTX-NEXT: br i1 [[TOBOOL]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
|
|
// NVPTX: [[COND_TRUE]]:
|
|
// NVPTX-NEXT: [[TMP9:%.*]] = load i32, ptr [[__RESULT]], align 4
|
|
// NVPTX-NEXT: br label %[[COND_END:.*]]
|
|
// NVPTX: [[COND_FALSE]]:
|
|
// NVPTX-NEXT: br label %[[COND_END]]
|
|
// NVPTX: [[COND_END]]:
|
|
// NVPTX-NEXT: [[COND:%.*]] = phi i32 [ [[TMP9]], %[[COND_TRUE]] ], [ 0, %[[COND_FALSE]] ]
|
|
// NVPTX-NEXT: [[ADD:%.*]] = add i32 [[TMP7]], [[COND]]
|
|
// NVPTX-NEXT: store i32 [[ADD]], ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: store i32 0, ptr [[__I]], align 4
|
|
// NVPTX-NEXT: br label %[[FOR_COND5:.*]]
|
|
// NVPTX: [[FOR_COND5]]:
|
|
// NVPTX-NEXT: [[TMP10:%.*]] = load i32, ptr [[__I]], align 4
|
|
// NVPTX-NEXT: [[TMP11:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// NVPTX-NEXT: [[CMP6:%.*]] = icmp ult i32 [[TMP10]], [[TMP11]]
|
|
// NVPTX-NEXT: br i1 [[CMP6]], label %[[FOR_BODY7:.*]], label %[[FOR_END:.*]]
|
|
// NVPTX: [[FOR_BODY7]]:
|
|
// NVPTX-NEXT: [[TMP12:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: [[TMP13:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP12]], i1 true)
|
|
// NVPTX-NEXT: [[CAST8:%.*]] = trunc i64 [[TMP13]] to i32
|
|
// NVPTX-NEXT: [[ISZERO9:%.*]] = icmp eq i64 [[TMP12]], 0
|
|
// NVPTX-NEXT: [[CLZG10:%.*]] = select i1 [[ISZERO9]], i32 0, i32 [[CAST8]]
|
|
// NVPTX-NEXT: [[SUB11:%.*]] = sub nsw i32 63, [[CLZG10]]
|
|
// NVPTX-NEXT: [[SH_PROM12:%.*]] = zext i32 [[SUB11]] to i64
|
|
// NVPTX-NEXT: [[SHL13:%.*]] = shl i64 1, [[SH_PROM12]]
|
|
// NVPTX-NEXT: [[TMP14:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: [[AND14:%.*]] = and i64 [[SHL13]], [[TMP14]]
|
|
// NVPTX-NEXT: [[TMP15:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: [[XOR:%.*]] = xor i64 [[TMP15]], [[AND14]]
|
|
// NVPTX-NEXT: store i64 [[XOR]], ptr [[__BELOW]], align 8
|
|
// NVPTX-NEXT: br label %[[FOR_INC:.*]]
|
|
// NVPTX: [[FOR_INC]]:
|
|
// NVPTX-NEXT: [[TMP16:%.*]] = load i32, ptr [[__I]], align 4
|
|
// NVPTX-NEXT: [[INC:%.*]] = add i32 [[TMP16]], 1
|
|
// NVPTX-NEXT: store i32 [[INC]], ptr [[__I]], align 4
|
|
// NVPTX-NEXT: br label %[[FOR_COND5]], !llvm.loop [[LOOP1:![0-9]+]]
|
|
// NVPTX: [[FOR_END]]:
|
|
// NVPTX-NEXT: br label %[[FOR_INC15:.*]]
|
|
// NVPTX: [[FOR_INC15]]:
|
|
// NVPTX-NEXT: [[TMP17:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// NVPTX-NEXT: [[MUL:%.*]] = mul i32 [[TMP17]], 2
|
|
// NVPTX-NEXT: store i32 [[MUL]], ptr [[__STEP]], align 4
|
|
// NVPTX-NEXT: br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
// NVPTX: [[FOR_END16]]:
|
|
// NVPTX-NEXT: [[TMP18:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// NVPTX-NEXT: ret i32 [[TMP18]]
|
|
//
|
|
//
|
|
// NVPTX-LABEL: define internal void @__gpu_exit(
|
|
// NVPTX-SAME: ) #[[ATTR1:[0-9]+]] {
|
|
// NVPTX-NEXT: [[ENTRY:.*:]]
|
|
// NVPTX-NEXT: call void @llvm.nvvm.exit()
|
|
// NVPTX-NEXT: ret void
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define protected spir_kernel void @foo(
|
|
// SPIRV-SAME: ) #[[ATTR0:[0-9]+]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_num_blocks_x()
|
|
// SPIRV-NEXT: [[CALL1:%.*]] = call spir_func i32 @__gpu_num_blocks_y()
|
|
// SPIRV-NEXT: [[CALL2:%.*]] = call spir_func i32 @__gpu_num_blocks_z()
|
|
// SPIRV-NEXT: [[CALL3:%.*]] = call spir_func i32 @__gpu_num_blocks(i32 noundef 0)
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_block_id_x()
|
|
// SPIRV-NEXT: [[CALL5:%.*]] = call spir_func i32 @__gpu_block_id_y()
|
|
// SPIRV-NEXT: [[CALL6:%.*]] = call spir_func i32 @__gpu_block_id_z()
|
|
// SPIRV-NEXT: [[CALL7:%.*]] = call spir_func i32 @__gpu_block_id(i32 noundef 0)
|
|
// SPIRV-NEXT: [[CALL8:%.*]] = call spir_func i32 @__gpu_num_threads_x()
|
|
// SPIRV-NEXT: [[CALL9:%.*]] = call spir_func i32 @__gpu_num_threads_y()
|
|
// SPIRV-NEXT: [[CALL10:%.*]] = call spir_func i32 @__gpu_num_threads_z()
|
|
// SPIRV-NEXT: [[CALL11:%.*]] = call spir_func i32 @__gpu_num_threads(i32 noundef 0)
|
|
// SPIRV-NEXT: [[CALL12:%.*]] = call spir_func i32 @__gpu_thread_id_x()
|
|
// SPIRV-NEXT: [[CALL13:%.*]] = call spir_func i32 @__gpu_thread_id_y()
|
|
// SPIRV-NEXT: [[CALL14:%.*]] = call spir_func i32 @__gpu_thread_id_z()
|
|
// SPIRV-NEXT: [[CALL15:%.*]] = call spir_func i32 @__gpu_thread_id(i32 noundef 0)
|
|
// SPIRV-NEXT: [[CALL16:%.*]] = call spir_func i32 @__gpu_num_lanes()
|
|
// SPIRV-NEXT: [[CALL17:%.*]] = call spir_func i32 @__gpu_lane_id()
|
|
// SPIRV-NEXT: [[CALL18:%.*]] = call spir_func i64 @__gpu_lane_mask()
|
|
// SPIRV-NEXT: [[CALL19:%.*]] = call spir_func i32 @__gpu_read_first_lane_u32(i64 noundef -1, i32 noundef -1)
|
|
// SPIRV-NEXT: [[CALL20:%.*]] = call spir_func i64 @__gpu_read_first_lane_u64(i64 noundef -1, i64 noundef -1)
|
|
// SPIRV-NEXT: [[CALL21:%.*]] = call spir_func i64 @__gpu_ballot(i64 noundef -1, i1 noundef zeroext true)
|
|
// SPIRV-NEXT: call spir_func void @__gpu_sync_threads()
|
|
// SPIRV-NEXT: call spir_func void @__gpu_sync_lane(i64 noundef -1)
|
|
// SPIRV-NEXT: [[CALL22:%.*]] = call spir_func i32 @__gpu_shuffle_idx_u32(i64 noundef -1, i32 noundef -1, i32 noundef -1, i32 noundef 0)
|
|
// SPIRV-NEXT: [[CALL23:%.*]] = call spir_func i64 @__gpu_first_lane_id(i64 noundef -1)
|
|
// SPIRV-NEXT: [[CALL24:%.*]] = call spir_func zeroext i1 @__gpu_is_first_in_lane(i64 noundef -1)
|
|
// SPIRV-NEXT: [[CALL25:%.*]] = call spir_func i32 @__gpu_prefix_scan_add_u32(i64 noundef -1, i32 noundef 1)
|
|
// SPIRV-NEXT: call spir_func void @__gpu_exit() #[[ATTR7:[0-9]+]]
|
|
// SPIRV-NEXT: unreachable
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_blocks_x(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_NUM_WORKGROUPS:%.*]] = call i64 @llvm.spv.num.workgroups.i64(i32 0)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_NUM_WORKGROUPS]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_blocks_y(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_NUM_WORKGROUPS:%.*]] = call i64 @llvm.spv.num.workgroups.i64(i32 1)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_NUM_WORKGROUPS]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_blocks_z(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_NUM_WORKGROUPS:%.*]] = call i64 @llvm.spv.num.workgroups.i64(i32 2)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_NUM_WORKGROUPS]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_blocks(
|
|
// SPIRV-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// SPIRV-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// SPIRV-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// SPIRV-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// SPIRV-NEXT: ]
|
|
// SPIRV: [[SW_BB]]:
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_num_blocks_x()
|
|
// SPIRV-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN:.*]]
|
|
// SPIRV: [[SW_BB1]]:
|
|
// SPIRV-NEXT: [[CALL2:%.*]] = call spir_func i32 @__gpu_num_blocks_y()
|
|
// SPIRV-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_BB3]]:
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_num_blocks_z()
|
|
// SPIRV-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_DEFAULT]]:
|
|
// SPIRV-NEXT: store i32 1, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[RETURN]]:
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_block_id_x(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_GROUP_ID:%.*]] = call i64 @llvm.spv.group.id.i64(i32 0)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_GROUP_ID]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_block_id_y(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_GROUP_ID:%.*]] = call i64 @llvm.spv.group.id.i64(i32 1)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_GROUP_ID]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_block_id_z(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_GROUP_ID:%.*]] = call i64 @llvm.spv.group.id.i64(i32 2)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_GROUP_ID]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_block_id(
|
|
// SPIRV-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// SPIRV-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// SPIRV-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// SPIRV-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// SPIRV-NEXT: ]
|
|
// SPIRV: [[SW_BB]]:
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_block_id_x()
|
|
// SPIRV-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN:.*]]
|
|
// SPIRV: [[SW_BB1]]:
|
|
// SPIRV-NEXT: [[CALL2:%.*]] = call spir_func i32 @__gpu_block_id_y()
|
|
// SPIRV-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_BB3]]:
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_block_id_z()
|
|
// SPIRV-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_DEFAULT]]:
|
|
// SPIRV-NEXT: store i32 0, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[RETURN]]:
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_threads_x(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_WORKGROUP_SIZE:%.*]] = call i64 @llvm.spv.workgroup.size.i64(i32 0)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_WORKGROUP_SIZE]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_threads_y(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_WORKGROUP_SIZE:%.*]] = call i64 @llvm.spv.workgroup.size.i64(i32 1)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_WORKGROUP_SIZE]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_threads_z(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_WORKGROUP_SIZE:%.*]] = call i64 @llvm.spv.workgroup.size.i64(i32 2)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_WORKGROUP_SIZE]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_threads(
|
|
// SPIRV-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// SPIRV-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// SPIRV-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// SPIRV-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// SPIRV-NEXT: ]
|
|
// SPIRV: [[SW_BB]]:
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_num_threads_x()
|
|
// SPIRV-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN:.*]]
|
|
// SPIRV: [[SW_BB1]]:
|
|
// SPIRV-NEXT: [[CALL2:%.*]] = call spir_func i32 @__gpu_num_threads_y()
|
|
// SPIRV-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_BB3]]:
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_num_threads_z()
|
|
// SPIRV-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_DEFAULT]]:
|
|
// SPIRV-NEXT: store i32 1, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[RETURN]]:
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_thread_id_x(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_THREAD_ID_IN_GROUP:%.*]] = call i64 @llvm.spv.thread.id.in.group.i64(i32 0)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_THREAD_ID_IN_GROUP]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_thread_id_y(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_THREAD_ID_IN_GROUP:%.*]] = call i64 @llvm.spv.thread.id.in.group.i64(i32 1)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_THREAD_ID_IN_GROUP]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_thread_id_z(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[SPV_THREAD_ID_IN_GROUP:%.*]] = call i64 @llvm.spv.thread.id.in.group.i64(i32 2)
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SPV_THREAD_ID_IN_GROUP]] to i32
|
|
// SPIRV-NEXT: ret i32 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_thread_id(
|
|
// SPIRV-SAME: i32 noundef [[__DIM:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__DIM_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i32 [[__DIM]], ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__DIM_ADDR]], align 4
|
|
// SPIRV-NEXT: switch i32 [[TMP0]], label %[[SW_DEFAULT:.*]] [
|
|
// SPIRV-NEXT: i32 0, label %[[SW_BB:.*]]
|
|
// SPIRV-NEXT: i32 1, label %[[SW_BB1:.*]]
|
|
// SPIRV-NEXT: i32 2, label %[[SW_BB3:.*]]
|
|
// SPIRV-NEXT: ]
|
|
// SPIRV: [[SW_BB]]:
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_thread_id_x()
|
|
// SPIRV-NEXT: store i32 [[CALL]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN:.*]]
|
|
// SPIRV: [[SW_BB1]]:
|
|
// SPIRV-NEXT: [[CALL2:%.*]] = call spir_func i32 @__gpu_thread_id_y()
|
|
// SPIRV-NEXT: store i32 [[CALL2]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_BB3]]:
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_thread_id_z()
|
|
// SPIRV-NEXT: store i32 [[CALL4]], ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[SW_DEFAULT]]:
|
|
// SPIRV-NEXT: store i32 0, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: br label %[[RETURN]]
|
|
// SPIRV: [[RETURN]]:
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[RETVAL]], align 4
|
|
// SPIRV-NEXT: ret i32 [[TMP1]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_num_lanes(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = call i32 @llvm.spv.subgroup.size()
|
|
// SPIRV-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_lane_id(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = call i32 @llvm.spv.subgroup.local.invocation.id()
|
|
// SPIRV-NEXT: ret i32 [[TMP0]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i64 @__gpu_lane_mask(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__MASK:%.*]] = alloca <4 x i32>, align 16
|
|
// SPIRV-NEXT: [[REF_TMP:%.*]] = alloca <2 x i32>, align 8
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.spv.subgroup.ballot(i1 true)
|
|
// SPIRV-NEXT: store <4 x i32> [[TMP0]], ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <2 x i32> <i32 0, i32 1>
|
|
// SPIRV-NEXT: store <2 x i32> [[SHUFFLE]], ptr [[REF_TMP]], align 8
|
|
// SPIRV-NEXT: [[TMP3:%.*]] = load i64, ptr [[REF_TMP]], align 8
|
|
// SPIRV-NEXT: ret i64 [[TMP3]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_read_first_lane_u32(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i64 @__gpu_lane_mask()
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[CALL]], i1 true)
|
|
// SPIRV-NEXT: [[CAST:%.*]] = trunc i64 [[TMP1]] to i32
|
|
// SPIRV-NEXT: [[SPV_SHUFFLE:%.*]] = call i32 @llvm.spv.wave.readlane.i32(i32 [[TMP0]], i32 [[CAST]])
|
|
// SPIRV-NEXT: ret i32 [[SPV_SHUFFLE]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i64 @__gpu_read_first_lane_u64(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]], i64 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__X_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__HI:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__LO:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: store i64 [[__X]], ptr [[__X_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i64, ptr [[__X_ADDR]], align 8
|
|
// SPIRV-NEXT: [[SHR:%.*]] = lshr i64 [[TMP0]], 32
|
|
// SPIRV-NEXT: [[CONV:%.*]] = trunc i64 [[SHR]] to i32
|
|
// SPIRV-NEXT: store i32 [[CONV]], ptr [[__HI]], align 4
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i64, ptr [[__X_ADDR]], align 8
|
|
// SPIRV-NEXT: [[AND:%.*]] = and i64 [[TMP1]], 4294967295
|
|
// SPIRV-NEXT: [[CONV1:%.*]] = trunc i64 [[AND]] to i32
|
|
// SPIRV-NEXT: store i32 [[CONV1]], ptr [[__LO]], align 4
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP3:%.*]] = load i32, ptr [[__HI]], align 4
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP2]], i32 noundef [[TMP3]])
|
|
// SPIRV-NEXT: [[CONV2:%.*]] = zext i32 [[CALL]] to i64
|
|
// SPIRV-NEXT: [[SHL:%.*]] = shl i64 [[CONV2]], 32
|
|
// SPIRV-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP5:%.*]] = load i32, ptr [[__LO]], align 4
|
|
// SPIRV-NEXT: [[CALL3:%.*]] = call spir_func i32 @__gpu_read_first_lane_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]])
|
|
// SPIRV-NEXT: [[CONV4:%.*]] = zext i32 [[CALL3]] to i64
|
|
// SPIRV-NEXT: [[AND5:%.*]] = and i64 [[CONV4]], 4294967295
|
|
// SPIRV-NEXT: [[OR:%.*]] = or i64 [[SHL]], [[AND5]]
|
|
// SPIRV-NEXT: ret i64 [[OR]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i64 @__gpu_ballot(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]], i1 noundef zeroext [[__X:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__X_ADDR:%.*]] = alloca i8, align 1
|
|
// SPIRV-NEXT: [[__MASK:%.*]] = alloca <4 x i32>, align 16
|
|
// SPIRV-NEXT: [[REF_TMP:%.*]] = alloca <2 x i32>, align 8
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[STOREDV:%.*]] = zext i1 [[__X]] to i8
|
|
// SPIRV-NEXT: store i8 [[STOREDV]], ptr [[__X_ADDR]], align 1
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i8, ptr [[__X_ADDR]], align 1
|
|
// SPIRV-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[TMP0]], 0
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.spv.subgroup.ballot(i1 [[LOADEDV]])
|
|
// SPIRV-NEXT: store <4 x i32> [[TMP1]], ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr [[__MASK]], align 16
|
|
// SPIRV-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <2 x i32> <i32 0, i32 1>
|
|
// SPIRV-NEXT: store <2 x i32> [[SHUFFLE]], ptr [[REF_TMP]], align 8
|
|
// SPIRV-NEXT: [[TMP5:%.*]] = load i64, ptr [[REF_TMP]], align 8
|
|
// SPIRV-NEXT: [[AND:%.*]] = and i64 [[TMP2]], [[TMP5]]
|
|
// SPIRV-NEXT: ret i64 [[AND]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func void @__gpu_sync_threads(
|
|
// SPIRV-SAME: ) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: call void @llvm.spv.group.memory.barrier.with.group.sync()
|
|
// SPIRV-NEXT: ret void
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func void @__gpu_sync_lane(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: ret void
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_shuffle_idx_u32(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__IDX:%.*]], i32 noundef [[__X:%.*]], i32 noundef [[__WIDTH:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__IDX_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__WIDTH_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__LANE:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: store i32 [[__IDX]], ptr [[__IDX_ADDR]], align 4
|
|
// SPIRV-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: store i32 [[__WIDTH]], ptr [[__WIDTH_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i32, ptr [[__IDX_ADDR]], align 4
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_lane_id()
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[__WIDTH_ADDR]], align 4
|
|
// SPIRV-NEXT: [[SUB:%.*]] = sub i32 [[TMP1]], 1
|
|
// SPIRV-NEXT: [[NOT:%.*]] = xor i32 [[SUB]], -1
|
|
// SPIRV-NEXT: [[AND:%.*]] = and i32 [[CALL]], [[NOT]]
|
|
// SPIRV-NEXT: [[ADD:%.*]] = add i32 [[TMP0]], [[AND]]
|
|
// SPIRV-NEXT: store i32 [[ADD]], ptr [[__LANE]], align 4
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP3:%.*]] = load i32, ptr [[__LANE]], align 4
|
|
// SPIRV-NEXT: [[SPV_SHUFFLE:%.*]] = call i32 @llvm.spv.wave.readlane.i32(i32 [[TMP2]], i32 [[TMP3]])
|
|
// SPIRV-NEXT: ret i32 [[SPV_SHUFFLE]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i64 @__gpu_first_lane_id(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP0]], i1 true)
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 1
|
|
// SPIRV-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP0]], 0
|
|
// SPIRV-NEXT: [[FFS:%.*]] = select i1 [[ISZERO]], i64 0, i64 [[TMP2]]
|
|
// SPIRV-NEXT: [[CAST:%.*]] = trunc i64 [[FFS]] to i32
|
|
// SPIRV-NEXT: [[SUB:%.*]] = sub nsw i32 [[CAST]], 1
|
|
// SPIRV-NEXT: [[CONV:%.*]] = sext i32 [[SUB]] to i64
|
|
// SPIRV-NEXT: ret i64 [[CONV]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func zeroext i1 @__gpu_is_first_in_lane(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_lane_id()
|
|
// SPIRV-NEXT: [[CONV:%.*]] = zext i32 [[CALL]] to i64
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[CALL1:%.*]] = call spir_func i64 @__gpu_first_lane_id(i64 noundef [[TMP0]])
|
|
// SPIRV-NEXT: [[CMP:%.*]] = icmp eq i64 [[CONV]], [[CALL1]]
|
|
// SPIRV-NEXT: ret i1 [[CMP]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func i32 @__gpu_prefix_scan_add_u32(
|
|
// SPIRV-SAME: i64 noundef [[__LANE_MASK:%.*]], i32 noundef [[__X:%.*]]) #[[ATTR0]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: [[__LANE_MASK_ADDR:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__X_ADDR:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__BELOW:%.*]] = alloca i64, align 8
|
|
// SPIRV-NEXT: [[__STEP:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__SRC:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__RESULT:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: [[__I:%.*]] = alloca i32, align 4
|
|
// SPIRV-NEXT: store i64 [[__LANE_MASK]], ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: store i32 [[__X]], ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP0:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[CALL:%.*]] = call spir_func i32 @__gpu_lane_id()
|
|
// SPIRV-NEXT: [[SH_PROM:%.*]] = zext i32 [[CALL]] to i64
|
|
// SPIRV-NEXT: [[SHL:%.*]] = shl i64 1, [[SH_PROM]]
|
|
// SPIRV-NEXT: [[SUB:%.*]] = sub i64 [[SHL]], 1
|
|
// SPIRV-NEXT: [[AND:%.*]] = and i64 [[TMP0]], [[SUB]]
|
|
// SPIRV-NEXT: store i64 [[AND]], ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: store i32 1, ptr [[__STEP]], align 4
|
|
// SPIRV-NEXT: br label %[[FOR_COND:.*]]
|
|
// SPIRV: [[FOR_COND]]:
|
|
// SPIRV-NEXT: [[TMP1:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// SPIRV-NEXT: [[CALL1:%.*]] = call spir_func i32 @__gpu_num_lanes()
|
|
// SPIRV-NEXT: [[CMP:%.*]] = icmp ult i32 [[TMP1]], [[CALL1]]
|
|
// SPIRV-NEXT: br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
|
|
// SPIRV: [[FOR_BODY]]:
|
|
// SPIRV-NEXT: [[TMP2:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP2]], i1 true)
|
|
// SPIRV-NEXT: [[CAST:%.*]] = trunc i64 [[TMP3]] to i32
|
|
// SPIRV-NEXT: [[ISZERO:%.*]] = icmp eq i64 [[TMP2]], 0
|
|
// SPIRV-NEXT: [[CLZG:%.*]] = select i1 [[ISZERO]], i32 64, i32 [[CAST]]
|
|
// SPIRV-NEXT: [[SUB2:%.*]] = sub nsw i32 63, [[CLZG]]
|
|
// SPIRV-NEXT: store i32 [[SUB2]], ptr [[__SRC]], align 4
|
|
// SPIRV-NEXT: [[TMP4:%.*]] = load i64, ptr [[__LANE_MASK_ADDR]], align 8
|
|
// SPIRV-NEXT: [[TMP5:%.*]] = load i32, ptr [[__SRC]], align 4
|
|
// SPIRV-NEXT: [[TMP6:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[CALL3:%.*]] = call spir_func i32 @__gpu_num_lanes()
|
|
// SPIRV-NEXT: [[CALL4:%.*]] = call spir_func i32 @__gpu_shuffle_idx_u32(i64 noundef [[TMP4]], i32 noundef [[TMP5]], i32 noundef [[TMP6]], i32 noundef [[CALL3]])
|
|
// SPIRV-NEXT: store i32 [[CALL4]], ptr [[__RESULT]], align 4
|
|
// SPIRV-NEXT: [[TMP7:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: [[TMP8:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: [[TOBOOL:%.*]] = icmp ne i64 [[TMP8]], 0
|
|
// SPIRV-NEXT: br i1 [[TOBOOL]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
|
|
// SPIRV: [[COND_TRUE]]:
|
|
// SPIRV-NEXT: [[TMP9:%.*]] = load i32, ptr [[__RESULT]], align 4
|
|
// SPIRV-NEXT: br label %[[COND_END:.*]]
|
|
// SPIRV: [[COND_FALSE]]:
|
|
// SPIRV-NEXT: br label %[[COND_END]]
|
|
// SPIRV: [[COND_END]]:
|
|
// SPIRV-NEXT: [[COND:%.*]] = phi i32 [ [[TMP9]], %[[COND_TRUE]] ], [ 0, %[[COND_FALSE]] ]
|
|
// SPIRV-NEXT: [[ADD:%.*]] = add i32 [[TMP7]], [[COND]]
|
|
// SPIRV-NEXT: store i32 [[ADD]], ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: store i32 0, ptr [[__I]], align 4
|
|
// SPIRV-NEXT: br label %[[FOR_COND5:.*]]
|
|
// SPIRV: [[FOR_COND5]]:
|
|
// SPIRV-NEXT: [[TMP10:%.*]] = load i32, ptr [[__I]], align 4
|
|
// SPIRV-NEXT: [[TMP11:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// SPIRV-NEXT: [[CMP6:%.*]] = icmp ult i32 [[TMP10]], [[TMP11]]
|
|
// SPIRV-NEXT: br i1 [[CMP6]], label %[[FOR_BODY7:.*]], label %[[FOR_END:.*]]
|
|
// SPIRV: [[FOR_BODY7]]:
|
|
// SPIRV-NEXT: [[TMP12:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: [[TMP13:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP12]], i1 true)
|
|
// SPIRV-NEXT: [[CAST8:%.*]] = trunc i64 [[TMP13]] to i32
|
|
// SPIRV-NEXT: [[ISZERO9:%.*]] = icmp eq i64 [[TMP12]], 0
|
|
// SPIRV-NEXT: [[CLZG10:%.*]] = select i1 [[ISZERO9]], i32 0, i32 [[CAST8]]
|
|
// SPIRV-NEXT: [[SUB11:%.*]] = sub nsw i32 63, [[CLZG10]]
|
|
// SPIRV-NEXT: [[SH_PROM12:%.*]] = zext i32 [[SUB11]] to i64
|
|
// SPIRV-NEXT: [[SHL13:%.*]] = shl i64 1, [[SH_PROM12]]
|
|
// SPIRV-NEXT: [[TMP14:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: [[AND14:%.*]] = and i64 [[SHL13]], [[TMP14]]
|
|
// SPIRV-NEXT: [[TMP15:%.*]] = load i64, ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: [[XOR:%.*]] = xor i64 [[TMP15]], [[AND14]]
|
|
// SPIRV-NEXT: store i64 [[XOR]], ptr [[__BELOW]], align 8
|
|
// SPIRV-NEXT: br label %[[FOR_INC:.*]]
|
|
// SPIRV: [[FOR_INC]]:
|
|
// SPIRV-NEXT: [[TMP16:%.*]] = load i32, ptr [[__I]], align 4
|
|
// SPIRV-NEXT: [[INC:%.*]] = add i32 [[TMP16]], 1
|
|
// SPIRV-NEXT: store i32 [[INC]], ptr [[__I]], align 4
|
|
// SPIRV-NEXT: br label %[[FOR_COND5]], !llvm.loop [[LOOP1:![0-9]+]]
|
|
// SPIRV: [[FOR_END]]:
|
|
// SPIRV-NEXT: br label %[[FOR_INC15:.*]]
|
|
// SPIRV: [[FOR_INC15]]:
|
|
// SPIRV-NEXT: [[TMP17:%.*]] = load i32, ptr [[__STEP]], align 4
|
|
// SPIRV-NEXT: [[MUL:%.*]] = mul i32 [[TMP17]], 2
|
|
// SPIRV-NEXT: store i32 [[MUL]], ptr [[__STEP]], align 4
|
|
// SPIRV-NEXT: br label %[[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
// SPIRV: [[FOR_END16]]:
|
|
// SPIRV-NEXT: [[TMP18:%.*]] = load i32, ptr [[__X_ADDR]], align 4
|
|
// SPIRV-NEXT: ret i32 [[TMP18]]
|
|
//
|
|
//
|
|
// SPIRV-LABEL: define internal spir_func void @__gpu_exit(
|
|
// SPIRV-SAME: ) #[[ATTR1:[0-9]+]] {
|
|
// SPIRV-NEXT: [[ENTRY:.*:]]
|
|
// SPIRV-NEXT: call void @llvm.trap()
|
|
// SPIRV-NEXT: ret void
|
|
//
|
|
//.
|
|
// AMDGPU: [[RNG2]] = !{i32 1, i32 0}
|
|
// AMDGPU: [[META3]] = !{}
|
|
// AMDGPU: [[RNG4]] = !{i16 1, i16 1025}
|
|
// AMDGPU: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]]}
|
|
// AMDGPU: [[META6]] = !{!"llvm.loop.mustprogress"}
|
|
// AMDGPU: [[LOOP7]] = distinct !{[[LOOP7]], [[META6]]}
|
|
//.
|
|
// NVPTX: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]]}
|
|
// NVPTX: [[META2]] = !{!"llvm.loop.mustprogress"}
|
|
// NVPTX: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]]}
|
|
//.
|
|
// SPIRV: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]]}
|
|
// SPIRV: [[META2]] = !{!"llvm.loop.mustprogress"}
|
|
// SPIRV: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]]}
|
|
//.
|