// MIT License // // Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. #include // hip header file #include #include #define WIDTH 1024 #define NUM (WIDTH * WIDTH) #define THREADS_PER_BLOCK_X 4 #define THREADS_PER_BLOCK_Y 4 #define ITER_NUM 16 * 1024 #define BLOCK_SIZE 1024 template void check(T result, char const* const func, const char* const file, int const line) { if(result) { fprintf(stderr, "Hip error at %s:%d code=%d(%s) \"%s\" \n", file, line, static_cast(result), hipGetErrorName(result), func); exit(EXIT_FAILURE); } } #define checkHipErrors(val) check((val), #val, __FILE__, __LINE__) __global__ void kernel_mult(const float varA, const float varB, float* result) { *result = varA * varB; } __global__ void kernel_add(const float varA, const float varB, float* result) { *result = varA + varB; } __global__ void target_kernel(float* out, float* in, const int width) { int x = blockDim.x * blockIdx.x + threadIdx.x; int y = blockDim.y * blockIdx.y + threadIdx.y; out[y * width + x] = in[x * width + y]; } __global__ void nested_kernel(float* out, float* in, const int width) { int x = blockDim.x * blockIdx.x + threadIdx.x; int y = blockDim.y * blockIdx.y + threadIdx.y; out[y * width + x] = in[x * width + y]; } #define ASM_LINE asm volatile("v_mov_b32 %0 %1\n" : "=v"(a) : "s"(c)) #define REPEAT_2(X) \ X; \ X #define REPEAT_4(X) \ REPEAT_2(X); \ REPEAT_2(X) #define REPEAT_8(X) \ REPEAT_4(X); \ REPEAT_4(X) #define REPEAT_16(X) \ REPEAT_8(X); \ REPEAT_8(X) #define REPEAT_32(X) \ REPEAT_16(X); \ REPEAT_16(X) #define REPEAT_64(X) \ REPEAT_32(X); \ REPEAT_32(X) #define REPEAT_100(X) \ REPEAT_64(X); \ REPEAT_32(X); \ REPEAT_4(X) // 64+32+4=100 __global__ void pc_sampling_kernel(const int c) { int a = 0; #pragma nounroll for(int i = 0; i < ITER_NUM; i++) { REPEAT_100(ASM_LINE); } } int main() { auto tid = roctx_thread_id_t{}; // get the thread id recognized by rocprofiler-sdk from roctx roctxGetThreadId(&tid); auto stream = hipStream_t{}; checkHipErrors(hipStreamCreate(&stream)); // pause API tracing roctxProfilerPause(tid); // Set up matrices float* gpuMatrix = nullptr; float* gpuTransposeMatrix = nullptr; float* Matrix = nullptr; Matrix = (float*) malloc(NUM * sizeof(float)); // initialize the input data for(auto i = 0; i < NUM; i++) { Matrix[i] = (float) i * 10.0f; } // allocate the memory on the device side checkHipErrors(hipMalloc((void**) &gpuMatrix, NUM * sizeof(float))); checkHipErrors(hipMalloc((void**) &gpuTransposeMatrix, NUM * sizeof(float))); // Memory transfer from host to device checkHipErrors(hipMemcpy(gpuMatrix, Matrix, NUM * sizeof(float), hipMemcpyHostToDevice)); checkHipErrors( hipMemcpy(gpuTransposeMatrix, gpuMatrix, NUM * sizeof(float), hipMemcpyDeviceToDevice)); // Set up simple kernels constexpr auto NUM_KERNELS = 64; constexpr size_t TARGET_KERNEL_ITERATIONS = NUM_KERNELS / 4; float* result = nullptr; float varA = 5.5; float varB = 11.7; uint32_t num_blocks = BLOCK_SIZE; checkHipErrors(hipMallocAsync(&result, sizeof(float), stream)); for(auto i = 0; i < NUM_KERNELS; ++i) { kernel_add<<>>(varA++, varB++, result); kernel_mult<<>>(varA++, varB++, result); // Run target kernel if(i % TARGET_KERNEL_ITERATIONS == 0) { roctxProfilerResume(tid); hipLaunchKernelGGL(target_kernel, dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y), dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y), 0, stream, gpuTransposeMatrix, gpuMatrix, WIDTH); // Use max(i, 1) to ensure at least 1 thread per block (i=0 would be invalid) int threads_per_block = (i > 0 ? i : 1); pc_sampling_kernel<<>>(threads_per_block); // Check for kernel launch errors checkHipErrors(hipGetLastError()); roctxProfilerPause(tid); } } // Nested test here roctxProfilerResume(tid); roctxProfilerResume(tid); roctxProfilerPause(tid); // Kernel should appear only with --selected-regions-ref-count hipLaunchKernelGGL(nested_kernel, dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y), dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y), 0, stream, gpuTransposeMatrix, gpuMatrix, WIDTH); roctxProfilerPause(tid); // Should not appear in trace hipLaunchKernelGGL(nested_kernel, dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y), dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y), 0, stream, gpuTransposeMatrix, gpuMatrix, WIDTH); // Wait for all kernels to complete before freeing memory checkHipErrors(hipStreamSynchronize(stream)); // free the resources on device side checkHipErrors(hipFree(result)); checkHipErrors(hipFree(gpuMatrix)); checkHipErrors(hipFree(gpuTransposeMatrix)); // destroy the stream checkHipErrors(hipStreamDestroy(stream)); // free the resources on host side free(Matrix); }