derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
595 lines
22 KiB
C++
595 lines
22 KiB
C++
/*
|
|
* Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
|
|
*
|
|
* SPDX-License-Identifier: MIT
|
|
*/
|
|
|
|
#include "OCLBlitKernel.h"
|
|
|
|
#include <Timer.h>
|
|
#include <assert.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
|
|
#include "CL/cl.h"
|
|
|
|
const static cl_uint Stages = 4;
|
|
const static cl_uint ThreadsForCheck = 1 << Stages;
|
|
|
|
#define KERNEL_CODE(...) #__VA_ARGS__
|
|
|
|
const static char* strKernel =
|
|
KERNEL_CODE(
|
|
\n
|
|
\x23 if OCL20
|
|
\n
|
|
extern void __amd_scheduler(__global void *, __global void *, uint);
|
|
\n
|
|
\x23 endif
|
|
\n
|
|
extern void __amd_copyBufferToImage(
|
|
__global uint*, __write_only image2d_array_t, ulong4,
|
|
int4, int4, uint4, ulong4);
|
|
|
|
extern void __amd_copyImageToBuffer(
|
|
__read_only image2d_array_t, __global uint*, __global ushort*,
|
|
__global uchar*, int4, ulong4, int4, uint4, ulong4);
|
|
|
|
extern void __amd_copyImage(
|
|
__read_only image2d_array_t, __write_only image2d_array_t,
|
|
int4, int4, int4);
|
|
|
|
extern void __amd_copyImage1DA(
|
|
__read_only image2d_array_t, __write_only image2d_array_t,
|
|
int4, int4, int4);
|
|
|
|
extern void __amd_copyBufferRect(
|
|
__global uchar*, __global uchar*,
|
|
ulong4, ulong4, ulong4);
|
|
|
|
extern void __amd_copyBufferRectAligned(
|
|
__global uint*, __global uint*,
|
|
ulong4, ulong4, ulong4);
|
|
|
|
extern void __amd_copyBuffer(
|
|
__global uchar*, __global uchar*,
|
|
ulong, ulong, ulong, uint);
|
|
|
|
extern void __amd_copyBufferAligned(
|
|
__global uint*, __global uint*,
|
|
ulong, ulong, ulong, uint);
|
|
|
|
extern void __amd_fillBuffer(
|
|
__global uchar*, __global uint*, __constant uchar*,
|
|
uint, ulong, ulong);
|
|
|
|
extern void __amd_fillImage(
|
|
__write_only image2d_array_t,
|
|
float4, int4, uint4, int4, int4, uint);
|
|
|
|
__kernel void copyBufferToImage(
|
|
__global uint* src,
|
|
__write_only image2d_array_t dst,
|
|
ulong4 srcOrigin,
|
|
int4 dstOrigin,
|
|
int4 size,
|
|
uint4 format,
|
|
ulong4 pitch)
|
|
{
|
|
__amd_copyBufferToImage(src, dst, srcOrigin, dstOrigin, size, format, pitch);
|
|
}
|
|
|
|
__kernel void copyImageToBuffer(
|
|
__read_only image2d_array_t src,
|
|
__global uint* dstUInt,
|
|
__global ushort* dstUShort,
|
|
__global uchar* dstUChar,
|
|
int4 srcOrigin,
|
|
ulong4 dstOrigin,
|
|
int4 size,
|
|
uint4 format,
|
|
ulong4 pitch)
|
|
{
|
|
__amd_copyImageToBuffer(src, dstUInt, dstUShort, dstUChar,
|
|
srcOrigin, dstOrigin, size, format, pitch);
|
|
}
|
|
|
|
__kernel void copyImage(
|
|
__read_only image2d_array_t src,
|
|
__write_only image2d_array_t dst,
|
|
int4 srcOrigin,
|
|
int4 dstOrigin,
|
|
int4 size)
|
|
{
|
|
__amd_copyImage(src, dst, srcOrigin, dstOrigin, size);
|
|
}
|
|
|
|
__kernel void copyImage1DA(
|
|
__read_only image2d_array_t src,
|
|
__write_only image2d_array_t dst,
|
|
int4 srcOrigin,
|
|
int4 dstOrigin,
|
|
int4 size)
|
|
{
|
|
__amd_copyImage1DA(src, dst, srcOrigin, dstOrigin, size);
|
|
}
|
|
|
|
__kernel void copyBufferRect(
|
|
__global uchar* src,
|
|
__global uchar* dst,
|
|
ulong4 srcRect,
|
|
ulong4 dstRect,
|
|
ulong4 size)
|
|
{
|
|
__amd_copyBufferRect(src, dst, srcRect, dstRect, size);
|
|
}
|
|
|
|
__kernel void copyBufferRectAligned(
|
|
__global uint* src,
|
|
__global uint* dst,
|
|
ulong4 srcRect,
|
|
ulong4 dstRect,
|
|
ulong4 size)
|
|
{
|
|
__amd_copyBufferRectAligned(src, dst, srcRect, dstRect, size);
|
|
}
|
|
|
|
__kernel void copyBuffer(
|
|
__global uchar* srcI,
|
|
__global uchar* dstI,
|
|
ulong srcOrigin,
|
|
ulong dstOrigin,
|
|
ulong size,
|
|
uint remain)
|
|
{
|
|
__amd_copyBuffer(srcI, dstI, srcOrigin, dstOrigin, size, remain);
|
|
}
|
|
|
|
__kernel void copyBufferAligned(
|
|
__global uint* src,
|
|
__global uint* dst,
|
|
ulong srcOrigin,
|
|
ulong dstOrigin,
|
|
ulong size,
|
|
uint alignment)
|
|
{
|
|
__amd_copyBufferAligned(src, dst, srcOrigin, dstOrigin, size, alignment);
|
|
}
|
|
|
|
__kernel void fillBuffer(
|
|
__global uchar* bufUChar,
|
|
__global uint* bufUInt,
|
|
__constant uchar* pattern,
|
|
uint patternSize,
|
|
ulong offset,
|
|
ulong size)
|
|
{
|
|
__amd_fillBuffer(bufUChar, bufUInt, pattern, patternSize, offset, size);
|
|
}
|
|
|
|
__kernel void fillImage(
|
|
__write_only image2d_array_t image,
|
|
float4 patternFLOAT4,
|
|
int4 patternINT4,
|
|
uint4 patternUINT4,
|
|
int4 origin,
|
|
int4 size,
|
|
uint type)
|
|
{
|
|
__amd_fillImage(image, patternFLOAT4, patternINT4, patternUINT4,
|
|
origin, size, type);
|
|
}
|
|
\n
|
|
\x23 if OCL20
|
|
\n
|
|
typedef struct _HsaAqlDispatchPacket {
|
|
uint mix;
|
|
ushort workgroup_size[3];
|
|
ushort reserved2;
|
|
uint grid_size[3];
|
|
uint private_segment_size_bytes;
|
|
uint group_segment_size_bytes;
|
|
ulong kernel_object_address;
|
|
ulong kernel_arg_address;
|
|
ulong reserved3;
|
|
ulong completion_signal;
|
|
} HsaAqlDispatchPacket;
|
|
\n
|
|
// This is an OpenCLized hsa_control_directives_t
|
|
typedef struct _AmdControlDirectives {
|
|
ulong enabled_control_directives;
|
|
ushort enable_break_exceptions;
|
|
ushort enable_detect_exceptions;
|
|
uint max_dynamic_group_size;
|
|
ulong max_flat_grid_size;
|
|
uint max_flat_workgroup_size;
|
|
uchar required_dim;
|
|
uchar reserved1[3];
|
|
ulong required_grid_size[3];
|
|
uint required_workgroup_size[3];
|
|
uchar reserved2[60];
|
|
} AmdControlDirectives;
|
|
\n
|
|
// This is an OpenCLized amd_kernel_code_t
|
|
typedef struct _AmdKernelCode {
|
|
uint amd_kernel_code_version_major;
|
|
uint amd_kernel_code_version_minor;
|
|
ushort amd_machine_kind;
|
|
ushort amd_machine_version_major;
|
|
ushort amd_machine_version_minor;
|
|
ushort amd_machine_version_stepping;
|
|
long kernel_code_entry_byte_offset;
|
|
long kernel_code_prefetch_byte_offset;
|
|
ulong kernel_code_prefetch_byte_size;
|
|
ulong max_scratch_backing_memory_byte_size;
|
|
uint compute_pgm_rsrc1;
|
|
uint compute_pgm_rsrc2;
|
|
uint kernel_code_properties;
|
|
uint workitem_private_segment_byte_size;
|
|
uint workgroup_group_segment_byte_size;
|
|
uint gds_segment_byte_size;
|
|
ulong kernarg_segment_byte_size;
|
|
uint workgroup_fbarrier_count;
|
|
ushort wavefront_sgpr_count;
|
|
ushort workitem_vgpr_count;
|
|
ushort reserved_vgpr_first;
|
|
ushort reserved_vgpr_count;
|
|
ushort reserved_sgpr_first;
|
|
ushort reserved_sgpr_count;
|
|
ushort debug_wavefront_private_segment_offset_sgpr;
|
|
ushort debug_private_segment_buffer_sgpr;
|
|
uchar kernarg_segment_alignment;
|
|
uchar group_segment_alignment;
|
|
uchar private_segment_alignment;
|
|
uchar wavefront_size;
|
|
int call_convention;
|
|
uchar reserved1[12];
|
|
ulong runtime_loader_kernel_symbol;
|
|
AmdControlDirectives control_directives;
|
|
} AmdKernelCode;
|
|
\n
|
|
typedef struct _HwDispatchHeader {
|
|
uint writeData0; // CP WRITE_DATA write to rewind for memory
|
|
uint writeData1;
|
|
uint writeData2;
|
|
uint writeData3;
|
|
uint rewind; // REWIND execution
|
|
uint startExe; // valid bit
|
|
uint condExe0; // 0xC0032200 -- TYPE 3, COND_EXEC
|
|
uint condExe1; // 0x00000204 ----
|
|
uint condExe2; // 0x00000000 ----
|
|
uint condExe3; // 0x00000000 ----
|
|
uint condExe4; // 0x00000000 ----
|
|
} HwDispatchHeader;
|
|
\n
|
|
typedef struct _HwDispatch {
|
|
uint packet0; // 0xC0067602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (6 values)
|
|
uint offset0; // 0x00000204 ---- OFFSET
|
|
uint startX; // 0x00000000 ---- COMPUTE_START_X: START = 0x0
|
|
uint startY; // 0x00000000 ---- COMPUTE_START_Y: START = 0x0
|
|
uint startZ; // 0x00000000 ---- COMPUTE_START_Z: START = 0x0
|
|
uint wrkGrpSizeX; // 0x00000000 ---- COMPUTE_NUM_THREAD_X: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0
|
|
uint wrkGrpSizeY; // 0x00000000 ---- COMPUTE_NUM_THREAD_Y: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0
|
|
uint wrkGrpSizeZ; // 0x00000000 ---- COMPUTE_NUM_THREAD_Z: NUM_THREAD_FULL = 0x0, NUM_THREAD_PARTIAL = 0x0
|
|
uint packet1; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offset1; // 0x0000020C ---- OFFSET
|
|
uint isaLo; // 0x00000000 ---- COMPUTE_PGM_LO: DATA = 0x0
|
|
uint isaHi; // 0x00000000 ---- COMPUTE_PGM_HI: DATA = 0x0, INST_ATC__CI__VI = 0x0
|
|
uint packet2; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offset2; // 0x00000212 ---- OFFSET
|
|
uint resource1; // 0x00000000 ---- COMPUTE_PGM_RSRC1
|
|
uint resource2; // 0x00000000 ---- COMPUTE_PGM_RSRC2
|
|
uint packet3; // 0xc0017602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (1 value)
|
|
uint offset3; // 0x00000215 ---- OFFSET
|
|
uint pad31; // 0x000003ff ---- COMPUTE_RESOURCE_LIMITS
|
|
uint packet31; // 0xC0067602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (1 value)
|
|
uint offset31; // 0x00000218 ---- OFFSET
|
|
uint ringSize; // 0x00000000 ---- COMPUTE_TMPRING_SIZE: WAVES = 0x0, WAVESIZE = 0x0
|
|
uint user0; // 0xC0047602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (4 values)
|
|
uint offsUser0; // 0x00000240 ---- OFFSET
|
|
uint scratchLo; // 0x00000000 ---- COMPUTE_USER_DATA_0: DATA = 0x0
|
|
uint scratchHi; // 0x80000000 ---- COMPUTE_USER_DATA_1: DATA = 0x80000000
|
|
uint scratchSize; // 0x00000000 ---- COMPUTE_USER_DATA_2: DATA = 0x0
|
|
uint padUser; // 0x00EA7FAC ---- COMPUTE_USER_DATA_3: DATA = 0xEA7FAC
|
|
uint user1; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offsUser1; // 0x00000244 ---- OFFSET
|
|
uint aqlPtrLo; // 0x00000000 ---- COMPUTE_USER_DATA_4: DATA = 0x0
|
|
uint aqlPtrHi; // 0x00000000 ---- COMPUTE_USER_DATA_5: DATA = 0x0
|
|
uint user2; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offsUser2; // 0x00000246 ---- OFFSET
|
|
uint hsaQueueLo; // 0x00000000 ---- COMPUTE_USER_DATA_6: DATA = 0x0
|
|
uint hsaQueueHi; // 0x00000000 ---- COMPUTE_USER_DATA_7: DATA = 0x0
|
|
uint user3; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offsUser3; // 0x00000246 ---- OFFSET
|
|
uint argsLo; // 0x00000000 ---- COMPUTE_USER_DATA_8: DATA = 0x0
|
|
uint argsHi; // 0x00000000 ---- COMPUTE_USER_DATA_9: DATA = 0x0
|
|
uint copyData; // 0xC0044000 -- TYPE 3, COPY_DATA
|
|
uint copyDataFlags; // 0x00000405 ---- srcSel 0x5, destSel 0x4, countSel 0x0, wrConfirm 0x0, engineSel 0x0
|
|
uint scratchAddrLo; // 0x000201C4 ---- srcAddressLo
|
|
uint scratchAddrHi; // 0x00000000 ---- srcAddressHi
|
|
uint shPrivateLo; // 0x00002580 ---- dstAddressLo
|
|
uint shPrivateHi; // 0x00000000 ---- dstAddressHi
|
|
uint user4; // 0xC0027602 -- TYPE 3, SET_SH_REG, TYPE:COMPUTE (2 values)
|
|
uint offsUser4; // 0x00000248 ---- OFFSET
|
|
uint scratchOffs; // 0x00000000 ---- COMPUTE_USER_DATA_10: DATA = 0x0
|
|
uint privSize; // 0x00000030 ---- COMPUTE_USER_DATA_11: DATA = 0x30
|
|
uint packet4; // 0xC0031502 -- TYPE 3, DISPATCH_DIRECT, TYPE:COMPUTE
|
|
uint glbSizeX; // 0x00000000
|
|
uint glbSizeY; // 0x00000000
|
|
uint glbSizeZ; // 0x00000000
|
|
uint padd41; // 0x00000021
|
|
} HwDispatch;
|
|
\n
|
|
static const uint WavefrontSize = 64;
|
|
static const uint MaxWaveSize = 0x400;
|
|
static const uint UsrRegOffset = 0x240;
|
|
static const uint Pm4Nop = 0xC0001002;
|
|
static const uint Pm4UserRegs = 0xC0007602;
|
|
static const uint Pm4CopyReg = 0xC0044000;
|
|
static const uint PrivateSegEna = 0x1;
|
|
static const uint DispatchEna = 0x2;
|
|
static const uint QueuePtrEna = 0x4;
|
|
static const uint KernelArgEna = 0x8;
|
|
static const uint FlatScratchEna = 0x20;
|
|
\n
|
|
uint GetCmdTemplateHeaderSize() { return sizeof(HwDispatchHeader); }
|
|
\n
|
|
uint GetCmdTemplateDispatchSize() { return sizeof(HwDispatch); }
|
|
\n
|
|
void EmptyCmdTemplateDispatch(ulong cmdBuf)
|
|
{
|
|
volatile __global HwDispatch* dispatch = (volatile __global HwDispatch*)cmdBuf;
|
|
dispatch->glbSizeX = 0;
|
|
dispatch->glbSizeY = 0;
|
|
dispatch->glbSizeZ = 0;
|
|
}
|
|
\n
|
|
void RunCmdTemplateDispatch(
|
|
ulong cmdBuf,
|
|
__global HsaAqlDispatchPacket* aqlPkt,
|
|
ulong scratch,
|
|
ulong hsaQueue,
|
|
uint scratchSize,
|
|
uint scratchOffset,
|
|
uint numMaxWaves,
|
|
uint useATC)
|
|
\n
|
|
{
|
|
volatile __global HwDispatch* dispatch = (volatile __global HwDispatch*)cmdBuf;
|
|
uint usrRegCnt = 0;
|
|
|
|
// Program workgroup size
|
|
dispatch->wrkGrpSizeX = aqlPkt->workgroup_size[0];
|
|
dispatch->wrkGrpSizeY = aqlPkt->workgroup_size[1];
|
|
dispatch->wrkGrpSizeZ = aqlPkt->workgroup_size[2];
|
|
|
|
// ISA address
|
|
__global AmdKernelCode* kernelObj = (__global AmdKernelCode*)aqlPkt->kernel_object_address;
|
|
ulong isa = aqlPkt->kernel_object_address + kernelObj->kernel_code_entry_byte_offset;
|
|
|
|
dispatch->isaLo = (uint)(isa >> 8);
|
|
dispatch->isaHi = (uint)(isa >> 40) | (useATC ? 0x100 : 0);
|
|
|
|
// Program PGM resource registers
|
|
dispatch->resource1 = kernelObj->compute_pgm_rsrc1;
|
|
dispatch->resource2 = kernelObj->compute_pgm_rsrc2;
|
|
|
|
uint flags = kernelObj->kernel_code_properties;
|
|
uint privateSize = kernelObj->workitem_private_segment_byte_size;
|
|
|
|
uint ldsSize = aqlPkt->group_segment_size_bytes +
|
|
kernelObj->workgroup_group_segment_byte_size;
|
|
|
|
// Align up the LDS blocks 128 * 4(in DWORDs)
|
|
uint ldsBlocks = (ldsSize + 511) >> 9;
|
|
|
|
dispatch->resource2 |= (ldsBlocks << 15);
|
|
|
|
// Private/scratch segment was enabled
|
|
if (flags & PrivateSegEna) {
|
|
uint waveSize = privateSize * WavefrontSize;
|
|
// 256 DWRODs is the minimum for SQ
|
|
waveSize = max(MaxWaveSize, waveSize);
|
|
|
|
uint numWaves = scratchSize / waveSize;
|
|
|
|
numWaves = min(numWaves, numMaxWaves);
|
|
|
|
dispatch->ringSize = numWaves;
|
|
dispatch->ringSize |= (waveSize >> 10) << 12;
|
|
dispatch->user0 = Pm4UserRegs | (4 << 16);
|
|
dispatch->scratchLo = (uint)scratch;
|
|
dispatch->scratchHi = ((uint)(scratch >> 32)) | 0x80000000; // Enables swizzle
|
|
dispatch->scratchSize = scratchSize;
|
|
usrRegCnt += 4;
|
|
}
|
|
else {
|
|
dispatch->ringSize = 0;
|
|
dispatch->user0 = Pm4Nop | (4 << 16);
|
|
}
|
|
|
|
// Pointer to the AQL dispatch packet
|
|
dispatch->user1 = (flags & DispatchEna) ? (Pm4UserRegs | (2 << 16)) : (Pm4Nop | (2 << 16));
|
|
dispatch->offsUser1 = UsrRegOffset + usrRegCnt;
|
|
usrRegCnt += (flags & DispatchEna) ? 2 : 0;
|
|
ulong gpuAqlPtr = (ulong)aqlPkt;
|
|
dispatch->aqlPtrLo = (uint)gpuAqlPtr;
|
|
dispatch->aqlPtrHi = (uint)(gpuAqlPtr >> 32);
|
|
|
|
// Pointer to the AQL queue header
|
|
if (flags & QueuePtrEna) {
|
|
dispatch->user2 = Pm4UserRegs | (2 << 16);
|
|
dispatch->offsUser2 = UsrRegOffset + usrRegCnt;
|
|
usrRegCnt += 2;
|
|
dispatch->hsaQueueLo = (uint)hsaQueue;
|
|
dispatch->hsaQueueHi = (uint)(hsaQueue >> 32);
|
|
}
|
|
else {
|
|
dispatch->user2 = Pm4Nop | (2 << 16);
|
|
}
|
|
|
|
// Pointer to the AQL kernel arguments
|
|
dispatch->user3 = (flags & KernelArgEna) ? (Pm4UserRegs | (2 << 16)) : (Pm4Nop | (2 << 16));
|
|
dispatch->offsUser3 = UsrRegOffset + usrRegCnt;
|
|
usrRegCnt += (flags & KernelArgEna) ? 2 : 0;
|
|
dispatch->argsLo = (uint)aqlPkt->kernel_arg_address;
|
|
dispatch->argsHi = (uint)(aqlPkt->kernel_arg_address >> 32);
|
|
|
|
// Provide pointer to the private/scratch buffer for the flat address
|
|
if (flags & FlatScratchEna) {
|
|
dispatch->copyData = Pm4CopyReg;
|
|
dispatch->scratchAddrLo = (uint)((scratch - scratchOffset) >> 16);
|
|
dispatch->offsUser4 = UsrRegOffset + usrRegCnt;
|
|
dispatch->scratchOffs = scratchOffset;
|
|
dispatch->privSize = privateSize;
|
|
}
|
|
else {
|
|
dispatch->copyData = Pm4Nop | (8 << 16);
|
|
}
|
|
|
|
// Update the global launch grid
|
|
dispatch->glbSizeX = aqlPkt->grid_size[0];
|
|
dispatch->glbSizeY = aqlPkt->grid_size[1];
|
|
dispatch->glbSizeZ = aqlPkt->grid_size[2];
|
|
}
|
|
\n
|
|
__kernel void scheduler(
|
|
__global void * queue,
|
|
__global void * params,
|
|
uint paramIdx)
|
|
{
|
|
__amd_scheduler(queue, params, paramIdx);
|
|
}
|
|
\n
|
|
\x23 endif
|
|
\n
|
|
);
|
|
|
|
enum {
|
|
BlitCopyImage = 0,
|
|
BlitCopyImage1DA,
|
|
BlitCopyImageToBuffer,
|
|
BlitCopyBufferToImage,
|
|
BlitCopyBufferRect,
|
|
BlitCopyBufferRectAligned,
|
|
BlitCopyBuffer,
|
|
BlitCopyBufferAligned,
|
|
FillBuffer,
|
|
FillImage,
|
|
Scheduler,
|
|
BlitTotal
|
|
};
|
|
|
|
static const char* BlitName[BlitTotal] = {
|
|
"copyImage", "copyImage1DA", "copyImageToBuffer",
|
|
"copyBufferToImage", "copyBufferRect", "copyBufferRectAligned",
|
|
"copyBuffer", "copyBufferAligned", "fillBuffer",
|
|
"fillImage", "scheduler",
|
|
};
|
|
|
|
OCLBlitKernel::OCLBlitKernel() { _numSubTests = 1; }
|
|
|
|
OCLBlitKernel::~OCLBlitKernel() {}
|
|
|
|
void OCLBlitKernel::open(unsigned int test, char* units, double& conversion,
|
|
unsigned int deviceId) {
|
|
OCLTestImp::open(test, units, conversion, deviceId);
|
|
CHECK_RESULT((error_ != CL_SUCCESS), "Error opening test");
|
|
char dbuffer[1024] = {0};
|
|
CPerfCounter timer;
|
|
int sub = 0;
|
|
std::string options = "-cl-std=CL2.0 -DOCL20=1";
|
|
|
|
cl_device_type deviceType;
|
|
error_ = _wrapper->clGetDeviceInfo(devices_[deviceId], CL_DEVICE_TYPE, sizeof(deviceType),
|
|
&deviceType, NULL);
|
|
CHECK_RESULT((error_ != CL_SUCCESS), "CL_DEVICE_TYPE failed");
|
|
|
|
if (!(deviceType & CL_DEVICE_TYPE_GPU)) {
|
|
testDescString = "GPU device is required for this test!\n";
|
|
return;
|
|
}
|
|
|
|
size_t param_size = 0;
|
|
char* strVersion = 0;
|
|
error_ = _wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DEVICE_VERSION, 0, 0, ¶m_size);
|
|
CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
|
|
strVersion = new char[param_size];
|
|
error_ =
|
|
_wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DEVICE_VERSION, param_size, strVersion, 0);
|
|
CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
|
|
if (strVersion[7] < '2') {
|
|
options = "-DOCL20=0";
|
|
sub = 1;
|
|
delete strVersion;
|
|
testDescString = "Currently it works for OCL20 devices only!\n";
|
|
return;
|
|
}
|
|
delete strVersion;
|
|
|
|
error_ = _wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DRIVER_VERSION, 0, 0, ¶m_size);
|
|
CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
|
|
strVersion = new char[param_size];
|
|
error_ =
|
|
_wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DRIVER_VERSION, param_size, strVersion, 0);
|
|
CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
|
|
|
|
std::string sch = strKernel;
|
|
static const char AmdScheduler[] = "amd_scheduler";
|
|
static const char AmdSchedulerPal[] = "amd_scheduler_pal";
|
|
static const char AmdSchedulerROCm[] = "amd_scheduler_rocm";
|
|
const char* AmdSchedulerPatch = NULL;
|
|
size_t loc = 0;
|
|
|
|
if (NULL != strstr(strVersion, "LC")) {
|
|
if (NULL != strstr(strVersion, "PAL")) {
|
|
AmdSchedulerPatch = AmdSchedulerPal;
|
|
} else if (NULL != strstr(strVersion, "HSA")) {
|
|
AmdSchedulerPatch = AmdSchedulerROCm;
|
|
}
|
|
}
|
|
delete strVersion;
|
|
|
|
if (NULL != AmdSchedulerPatch) {
|
|
loc = sch.find(AmdScheduler);
|
|
sch.replace(loc, strlen(AmdScheduler), AmdSchedulerPatch);
|
|
loc = sch.find(AmdScheduler, (loc + strlen(AmdSchedulerPatch)));
|
|
sch.replace(loc, strlen(AmdScheduler), AmdSchedulerPatch);
|
|
}
|
|
|
|
timer.Reset();
|
|
timer.Start();
|
|
|
|
const char* strProgram = sch.c_str();
|
|
program_ = _wrapper->clCreateProgramWithSource(context_, 1, &strProgram, NULL, &error_);
|
|
CHECK_RESULT((error_ != CL_SUCCESS), "clCreateProgramWithSource() failed");
|
|
error_ = _wrapper->clBuildProgram(program_, 1, &devices_[deviceId], options.c_str(), NULL, NULL);
|
|
if (error_ != CL_SUCCESS) {
|
|
char programLog[1024];
|
|
_wrapper->clGetProgramBuildInfo(program_, devices_[deviceId], CL_PROGRAM_BUILD_LOG, 1024,
|
|
programLog, 0);
|
|
printf("\n%s\n", programLog);
|
|
fflush(stdout);
|
|
}
|
|
CHECK_RESULT((error_ != CL_SUCCESS), "clBuildProgram() failed");
|
|
|
|
cl_kernel kernels[BlitTotal];
|
|
for (int i = 0; i < BlitTotal - sub; ++i) {
|
|
kernels[i] = _wrapper->clCreateKernel(program_, BlitName[i], &error_);
|
|
CHECK_RESULT((error_ != CL_SUCCESS), "clCreateKernel() failed");
|
|
}
|
|
timer.Stop();
|
|
double sec = timer.GetElapsedTime();
|
|
|
|
time_ = (float)sec * 1000.f;
|
|
testDescString = "Blit kernel compilaiton time (ms):";
|
|
|
|
for (int i = 0; i < BlitTotal - sub; ++i) {
|
|
_wrapper->clReleaseKernel(kernels[i]);
|
|
}
|
|
}
|
|
|
|
void OCLBlitKernel::run(void) { _perfInfo = time_; }
|
|
|
|
unsigned int OCLBlitKernel::close(void) { return OCLTestImp::close(); }
|