derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
389 lines
14 KiB
C
389 lines
14 KiB
C
/*
|
|
* Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
|
|
*
|
|
* SPDX-License-Identifier: MIT
|
|
*/
|
|
|
|
static const char* local_atomics_histogram =
|
|
"#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable\n"
|
|
"#define MIN(a,b) ((a) < (b)) ? (a) : (b) \n"
|
|
"#define MAX(a,b) ((a) > (b)) ? (a) : (b) \n"
|
|
"__kernel __attribute__((reqd_work_group_size(256,1,1)))\n"
|
|
"void local_atomics_histogram(__global uint4 *Image,\n"
|
|
"__global uint *Histogram,\n"
|
|
"uint n4VectorsPerThread)\n"
|
|
"{\n"
|
|
" __local __attribute__((aligned(16))) uint subhists[NBANKS * NBINS];\n"
|
|
"\n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" uint ltid = get_local_id(0);\n"
|
|
" uint Stride = get_global_size(0);\n"
|
|
"\n"
|
|
" uint i, idx;\n"
|
|
" uint4 temp, temp2;\n"
|
|
" const uint shft = (uint) BITS_PER_PIX;\n"
|
|
" const uint msk = (uint) (NBINS-1);\n"
|
|
" uint offset = (uint) ltid % (uint) (NBANKS);\n"
|
|
"\n"
|
|
" uint lmem_items = NBANKS * NBINS;\n"
|
|
" uint lmem_items_per_thread;\n"
|
|
" uint lmem_max_threads;\n"
|
|
"\n"
|
|
" // parallel LDS clear\n"
|
|
" // first, calculate threads per item, at least 1:\n"
|
|
" lmem_max_threads = MIN( 1, get_local_size(0) / lmem_items );\n"
|
|
" // but no more than we have items:\n"
|
|
" lmem_max_threads = MAX( 1, lmem_max_threads / lmem_items );\n"
|
|
" // calculate threads total:\n"
|
|
" lmem_max_threads = lmem_items / lmem_max_threads;\n"
|
|
" // but no more than LDS banks:\n"
|
|
" lmem_max_threads = MIN( get_local_size(0), lmem_max_threads );\n"
|
|
"\n"
|
|
" lmem_items_per_thread = lmem_items / lmem_max_threads;\n"
|
|
"\n"
|
|
" // now, clear LDS\n"
|
|
" __local uint4 *p = (__local uint4 *) subhists;\n"
|
|
"\n"
|
|
" if( ltid < lmem_max_threads )\n"
|
|
" {\n"
|
|
" for(i=0, idx=ltid; i<lmem_items_per_thread/4; i++, "
|
|
"idx+=lmem_max_threads)\n"
|
|
" {\n"
|
|
" p[idx] = 0;\n"
|
|
" }\n"
|
|
" }\n"
|
|
"\n"
|
|
" barrier( CLK_LOCAL_MEM_FENCE );\n"
|
|
"\n"
|
|
" // read & scatter phase\n"
|
|
"\n"
|
|
" for( i=0, idx=tid; i<n4VectorsPerThread; i++, idx += Stride )\n"
|
|
" {\n"
|
|
" temp = Image[idx];\n"
|
|
" temp2 = (temp & msk) * (uint4) NBANKS + offset;\n"
|
|
"\n"
|
|
" (void) atom_inc( subhists + temp2.x );\n"
|
|
" (void) atom_inc( subhists + temp2.y );\n"
|
|
" (void) atom_inc( subhists + temp2.z );\n"
|
|
" (void) atom_inc( subhists + temp2.w );\n"
|
|
"\n"
|
|
" temp = temp >> shft;\n"
|
|
" temp2 = (temp & msk) * (uint4) NBANKS + offset;\n"
|
|
"\n"
|
|
" (void) atom_inc( subhists + temp2.x );\n"
|
|
" (void) atom_inc( subhists + temp2.y );\n"
|
|
" (void) atom_inc( subhists + temp2.z );\n"
|
|
" (void) atom_inc( subhists + temp2.w );\n"
|
|
"\n"
|
|
" temp = temp >> shft;\n"
|
|
" temp2 = (temp & msk) * (uint4) NBANKS + offset;\n"
|
|
"\n"
|
|
" (void) atom_inc( subhists + temp2.x );\n"
|
|
" (void) atom_inc( subhists + temp2.y );\n"
|
|
" (void) atom_inc( subhists + temp2.z );\n"
|
|
" (void) atom_inc( subhists + temp2.w );\n"
|
|
"\n"
|
|
" temp = temp >> shft;\n"
|
|
" temp2 = (temp & msk) * (uint4) NBANKS + offset;\n"
|
|
"\n"
|
|
" (void) atom_inc( subhists + temp2.x );\n"
|
|
" (void) atom_inc( subhists + temp2.y );\n"
|
|
" (void) atom_inc( subhists + temp2.z );\n"
|
|
" (void) atom_inc( subhists + temp2.w );\n"
|
|
" }\n"
|
|
"\n"
|
|
" barrier( CLK_LOCAL_MEM_FENCE );\n"
|
|
"\n"
|
|
" // reduce __local banks to single histogram per work-group\n"
|
|
"\n"
|
|
" if( ltid < NBINS )\n"
|
|
" {\n"
|
|
" uint bin = 0;\n"
|
|
" for( i=0; i<NBANKS; i++ )\n"
|
|
" {\n"
|
|
" bin += subhists[ (ltid * NBANKS) + i ];\n"
|
|
" }\n"
|
|
" Histogram[ (get_group_id(0) * NBINS) + ltid ] = bin;\n"
|
|
" }\n"
|
|
"}\n";
|
|
|
|
static const char* local_atomics_reduce =
|
|
" __kernel void local_atomics_reduce( __global uint *Histogram, uint "
|
|
"nSubHists )\n"
|
|
"{\n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" uint bin = 0;\n"
|
|
" // Reduce work-group histograms into single histogram,\n"
|
|
" // one thread for each bin.\n"
|
|
" for( int i=0; i < nSubHists; i++ )\n"
|
|
" bin += Histogram[ (i * NBINS) + tid ];\n"
|
|
" Histogram[ tid ] = bin;\n"
|
|
"}\n";
|
|
|
|
static const char* global_atomics_histogram =
|
|
"#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable\n"
|
|
"__kernel __attribute__((reqd_work_group_size(256,1,1)))\n"
|
|
"void global_atomics_histogram(uint ItemsPerThread,\n"
|
|
"__global uint *Input,\n"
|
|
"__global uint *Histogram)\n"
|
|
"{\n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" const uint shft = (uint) BITS_PER_PIX;\n"
|
|
" const uint msk = (uint) (NBINS-1);\n"
|
|
" uint Stride = get_global_size(0);\n"
|
|
" for( int i = 0; i < ItemsPerThread; i++)\n"
|
|
" {\n"
|
|
" uint temp = Input[tid];\n"
|
|
" atom_inc( &(Histogram[ (temp & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp & msk) ]) );\n"
|
|
" tid += Stride;"
|
|
" }\n"
|
|
"}\n";
|
|
|
|
static const char* global_vec4_atomics_histogram =
|
|
"#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable\n"
|
|
"__kernel __attribute__((reqd_work_group_size(256,1,1)))\n"
|
|
"void global_atomics_histogram(uint ItemsPerThread,\n"
|
|
"__global uint4 *Input,\n"
|
|
"__global uint *Histogram)\n"
|
|
"{\n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" const uint shft = (uint) BITS_PER_PIX;\n"
|
|
" const uint msk = (uint) (NBINS-1);\n"
|
|
" uint Stride = get_global_size(0);\n"
|
|
" for( int i = 0; i < ItemsPerThread; i++)\n"
|
|
" {\n"
|
|
" uint4 temp = Input[tid];\n"
|
|
" atom_inc( &(Histogram[ (temp.x & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.y & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.z & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.w & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp.x & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.y & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.z & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.w & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp.x & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.y & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.z & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.w & msk) ]) );\n"
|
|
" temp = temp >> shft;\n"
|
|
" atom_inc( &(Histogram[ (temp.x & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.y & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.z & msk) ]) );\n"
|
|
" atom_inc( &(Histogram[ (temp.w & msk) ]) );\n"
|
|
" tid += Stride;"
|
|
" }\n"
|
|
"}\n";
|
|
|
|
static const char* global_atomics_sum_reduction_all_to_zero =
|
|
"#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable\n"
|
|
" __kernel void global_atomics_sum_reduction_all_to_zero(uint "
|
|
"ItemsPerThread, __global uint *Input, __global int *Output )\n"
|
|
"{\n"
|
|
" uint sum = 0;\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" \n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" uint Stride = get_global_size(0);\n"
|
|
" for( int i = 0; i < ItemsPerThread; i++)\n"
|
|
" {\n"
|
|
" uint data = Input[tid];\n"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" tid += Stride;\n"
|
|
" }\n"
|
|
" atom_add( &(Output[0]), sum);\n"
|
|
"}\n";
|
|
|
|
static const char* global_atomics_sum_reduction_workgroup =
|
|
"#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics : enable\n"
|
|
" __kernel void global_atomics_sum_reduction_workgroup(uint "
|
|
"ItemsPerThread, __global uint *Input, __global int *Output )\n"
|
|
"{\n"
|
|
" uint sum = 0;\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" \n"
|
|
" uint tid = get_global_id(0);\n"
|
|
" uint Stride = get_global_size(0);\n"
|
|
" for( int i = 0; i < ItemsPerThread; i++)\n"
|
|
" {\n"
|
|
" uint data = Input[tid];\n"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" data = data >> shft;"
|
|
" sum += data & msk;\n"
|
|
" tid += Stride;\n"
|
|
" }\n"
|
|
" atom_add( &(Output[get_group_id(0)]), sum);\n"
|
|
"}\n";
|
|
|
|
static const char* local_reduction =
|
|
"__kernel void local_reduction(__global uint* input, __global uint* "
|
|
"output, __local uint* sdata)\n"
|
|
"{\n"
|
|
" // load shared mem\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" unsigned int tid = get_local_id(0);\n"
|
|
"\n"
|
|
" unsigned int localSize = get_local_size(0);\n"
|
|
" unsigned int stride = get_global_id(0) * 2;\n"
|
|
" unsigned int data1 = input[stride];\n"
|
|
" unsigned int data2 = input[stride + 1];\n"
|
|
" unsigned int sum = 0;\n"
|
|
" for( int i = 0; i < 4; i++)\n"
|
|
" {\n"
|
|
" sum += (data1 & msk) + (data2 & msk);\n"
|
|
" data1 = data1 >> shft;\n"
|
|
" data2 = data2 >> shft;\n"
|
|
" }\n"
|
|
" sdata[tid] = sum;"
|
|
"\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" // do reduction in shared mem\n"
|
|
" for(unsigned int s = localSize >> 1; s > 0; s >>= 1)\n"
|
|
" {\n"
|
|
" if(tid < s) \n"
|
|
" {\n"
|
|
" sdata[tid] += sdata[tid + s];\n"
|
|
" }\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" }\n"
|
|
"\n"
|
|
" // write result for this block to global mem\n"
|
|
" if(tid == 0) output[get_group_id(0)] = sdata[0];\n"
|
|
"}\n";
|
|
|
|
static const char* local_vec4_reduction =
|
|
"__kernel void local_reduction(__global uint4* input, __global uint4* "
|
|
"output, __local uint4* sdata)\n"
|
|
"{\n"
|
|
" // load shared mem\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" unsigned int tid = get_local_id(0);\n"
|
|
"\n"
|
|
" unsigned int localSize = get_local_size(0);\n"
|
|
" unsigned int stride = get_global_id(0) * 2;\n"
|
|
" uint4 data1 = input[stride];\n"
|
|
" uint4 data2 = input[stride + 1];\n"
|
|
" uint4 sum = 0;\n"
|
|
" for( int i = 0; i < 4; i++)\n"
|
|
" {\n"
|
|
" sum += (data1 & msk) + (data2 & msk);\n"
|
|
" data1 = data1 >> shft;\n"
|
|
" data2 = data2 >> shft;\n"
|
|
" }\n"
|
|
" sdata[tid] = sum;"
|
|
"\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" // do reduction in shared mem\n"
|
|
" for(unsigned int s = localSize >> 1; s > 0; s >>= 1)\n"
|
|
" {\n"
|
|
" if(tid < s) \n"
|
|
" {\n"
|
|
" sdata[tid] += sdata[tid + s];\n"
|
|
" }\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" }\n"
|
|
"\n"
|
|
" // write result for this block to global mem\n"
|
|
" if(tid == 0) output[get_group_id(0)] = sdata[0];\n"
|
|
"}\n";
|
|
|
|
static const char* local_atomics_reduction =
|
|
"#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable\n"
|
|
"__kernel void local_reduction(__global uint* input, __global uint* "
|
|
"output, __local uint* sdata)\n"
|
|
"{\n"
|
|
" // load shared mem\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" unsigned int tid = get_local_id(0);\n"
|
|
"\n"
|
|
" unsigned int localSize = get_local_size(0);\n"
|
|
" unsigned int stride = get_global_id(0) * 2;\n"
|
|
" unsigned int data1 = input[stride];\n"
|
|
" unsigned int data2 = input[stride + 1];\n"
|
|
" unsigned int sum = 0;\n"
|
|
" for( int i = 0; i < 4; i++)\n"
|
|
" {\n"
|
|
" sum += (data1 & msk) + (data2 & msk);\n"
|
|
" data1 = data1 >> shft;\n"
|
|
" data2 = data2 >> shft;\n"
|
|
" }\n"
|
|
" sdata[tid] = sum;"
|
|
"\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" // do reduction in shared mem\n"
|
|
" for(unsigned int s = localSize >> 1; s > 0; s >>= 1)\n"
|
|
" {\n"
|
|
" if(tid < s) \n"
|
|
" {\n"
|
|
" atom_add( &(sdata[tid]), sdata[tid + s]);\n"
|
|
" }\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" }\n"
|
|
"\n"
|
|
" // write result for this block to global mem\n"
|
|
" if(tid == 0) output[get_group_id(0)] = sdata[0];\n"
|
|
"}\n";
|
|
|
|
static const char* local_vec4_atomics_reduction =
|
|
"#pragma OPENCL EXTENSION cl_khr_local_int32_base_atomics : enable\n"
|
|
"__kernel void local_reduction(__global uint4* input, __global uint4* "
|
|
"output, __local uint4* sdata)\n"
|
|
"{\n"
|
|
" // load shared mem\n"
|
|
" const uint msk = (uint)3;\n"
|
|
" const uint shft = (uint)8;\n"
|
|
" unsigned int tid = get_local_id(0);\n"
|
|
"\n"
|
|
" unsigned int localSize = get_local_size(0);\n"
|
|
" unsigned int stride = get_global_id(0) * 2;\n"
|
|
" uint4 data1 = input[stride];\n"
|
|
" uint4 data2 = input[stride + 1];\n"
|
|
" uint4 sum = 0;\n"
|
|
" for( int i = 0; i < 4; i++)\n"
|
|
" {\n"
|
|
" sum += (data1 & msk) + (data2 & msk);\n"
|
|
" data1 = data1 >> shft;\n"
|
|
" data2 = data2 >> shft;\n"
|
|
" }\n"
|
|
" sdata[tid] = sum;"
|
|
"\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" // do reduction in shared mem\n"
|
|
" for(unsigned int s = localSize >> 1; s > 0; s >>= 1)\n"
|
|
" {\n"
|
|
" if(tid < s) \n"
|
|
" {\n"
|
|
" atom_add( &(sdata[tid]).x, sdata[tid + s].x);\n"
|
|
" atom_add( &(sdata[tid]).y, sdata[tid + s].y);\n"
|
|
" atom_add( &(sdata[tid]).z, sdata[tid + s].z);\n"
|
|
" atom_add( &(sdata[tid]).w, sdata[tid + s].w);\n"
|
|
" }\n"
|
|
" barrier(CLK_LOCAL_MEM_FENCE);\n"
|
|
" }\n"
|
|
"\n"
|
|
" // write result for this block to global mem\n"
|
|
" if(tid == 0) output[get_group_id(0)] = sdata[0];\n"
|
|
"}\n";
|