TheRock/rocm-systems/projects/clr/opencl/tests/ocltst/module/perf/OCLPerfDeviceEnqueue2.cpp

/*
 * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 *
 * SPDX-License-Identifier: MIT
 */

#include "OCLPerfDeviceEnqueue2.h"

#include <Timer.h>
#include <assert.h>
#include <stdio.h>
#include <string.h>

#include "CL/cl.h"

// Quiet pesky warnings
#ifdef WIN_OS
#define SNPRINTF sprintf_s
#else
#define SNPRINTF snprintf
#endif

#define KERNEL_CODE(...) #__VA_ARGS__

typedef struct {
  unsigned int threads;
} testStruct;

static testStruct testList[] = {
    {64}, {128}, {256}, {512}, {1024}, {2048}, {4096},
};

static unsigned int qsizeList[] = {
    16, 32, 64, 128, 256, 512,
};

static unsigned int levelList[] = {
    1,
    2,
    4,
    8,
};

const static char* strKernel = {KERNEL_CODE(
  \n __kernel void childKernel(__global uint* buf, uint level) {
  if (level) {
    queue_t def_q = get_default_queue();
    ndrange_t ndrange = ndrange_1D(64, 64);
    int gid = get_global_id(0);
    int lid = get_local_id(0);
    if (lid == 0) {
      int enq_res = enqueue_kernel(def_q, CLK_ENQUEUE_FLAGS_WAIT_KERNEL, ndrange, ^{
        childKernel(buf, level - 1);
      });
    }
  } else {
    int idx = get_global_id(0);
    if (idx < 0) {
      buf[idx] = 0;
    }
  }
}
  \n __kernel void parentKernel(__global uint* buf, uint level) {
  queue_t def_q = get_default_queue();
  ndrange_t ndrange = ndrange_1D(64, 64);
  int gid = get_global_id(0);

  if (level) {
    int enq_res = enqueue_kernel(def_q, CLK_ENQUEUE_FLAGS_WAIT_KERNEL, ndrange, ^{
      childKernel(buf, level - 1);
    });
  }
}
  \n)};

OCLPerfDeviceEnqueue2::OCLPerfDeviceEnqueue2() {
  subTests_level = sizeof(levelList) / sizeof(unsigned int);
  subTests_qsize = (sizeof(qsizeList) / sizeof(unsigned int));
  subTests_thread = sizeof(testList) / sizeof(testStruct);
  testListSize = subTests_thread;
  _numSubTests = subTests_level * subTests_qsize * subTests_thread;
  deviceQueue_ = NULL;
  failed_ = false;
  kernel2_ = NULL;
  level = 2;
  skip_ = false;
}

OCLPerfDeviceEnqueue2::~OCLPerfDeviceEnqueue2() {}

void OCLPerfDeviceEnqueue2::open(unsigned int test, char* units, double& conversion,
                                 unsigned int deviceId) {
  if (type_ == CL_DEVICE_TYPE_CPU) {
    return;
  }

  OCLTestImp::open(test, units, conversion, deviceId);
  CHECK_RESULT((error_ != CL_SUCCESS), "Error opening test");
  testID_ = test;

  threads = testList[testID_ / (subTests_qsize * subTests_level)].threads;
  queueSize = qsizeList[(testID_ / subTests_level) % subTests_qsize] * 1024;
  level = levelList[testID_ % subTests_level];

  size_t param_size = 0;
  char* strVersion = 0;
  error_ = _wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DEVICE_VERSION, 0, 0, &param_size);
  CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
  strVersion = new char[param_size];
  error_ =
      _wrapper->clGetDeviceInfo(devices_[_deviceId], CL_DEVICE_VERSION, param_size, strVersion, 0);
  CHECK_RESULT(error_ != CL_SUCCESS, "clGetDeviceInfo failed");
  if (strVersion[7] < '2') {
    failed_ = true;
    return;
  }
  delete strVersion;

  program_ = _wrapper->clCreateProgramWithSource(context_, 1, &strKernel, NULL, &error_);
  CHECK_RESULT((error_ != CL_SUCCESS), "clCreateProgramWithSource()  failed");

  error_ = _wrapper->clBuildProgram(program_, 1, &devices_[deviceId], "-cl-std=CL2.0", NULL, NULL);
  if (error_ != CL_SUCCESS) {
    char programLog[1024];
    _wrapper->clGetProgramBuildInfo(program_, devices_[deviceId], CL_PROGRAM_BUILD_LOG, 1024,
                                    programLog, 0);
    printf("\n%s\n", programLog);
    fflush(stdout);
  }
  CHECK_RESULT((error_ != CL_SUCCESS), "clBuildProgram() failed");

  kernel_ = _wrapper->clCreateKernel(program_, "parentKernel", &error_);
  CHECK_RESULT((error_ != CL_SUCCESS), "clCreateKernel() failed");

  kernel2_ = _wrapper->clCreateKernel(program_, "childKernel", &error_);
  CHECK_RESULT((error_ != CL_SUCCESS), "clCreateKernel() failed");

  cl_mem buffer;

  buffer = _wrapper->clCreateBuffer(context_, CL_MEM_ALLOC_HOST_PTR, 2048, NULL, &error_);
  CHECK_RESULT((error_ != CL_SUCCESS), "clCreateBuffer() failed");
  buffers_.push_back(buffer);

#if defined(CL_VERSION_2_0)
  const cl_queue_properties cprops[] = {
      CL_QUEUE_PROPERTIES,
      static_cast<cl_queue_properties>(CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE |
                                       CL_QUEUE_ON_DEVICE_DEFAULT | CL_QUEUE_ON_DEVICE),
      CL_QUEUE_SIZE, queueSize, 0};
  deviceQueue_ =
      _wrapper->clCreateCommandQueueWithProperties(context_, devices_[deviceId], cprops, &error_);
  CHECK_RESULT((error_ != CL_SUCCESS), "clCreateCommandQueueWithProperties() failed");
#else
  skip_ = true;
  testDescString = "DeviceEnqueue NOT supported for < 2.0 builds. Test Skipped.";
  return;
#endif
}

static void CL_CALLBACK notify_callback(const char* errinfo, const void* private_info, size_t cb,
                                        void* user_data) {}

void OCLPerfDeviceEnqueue2::run(void) {
  CPerfCounter timer;
  if (type_ == CL_DEVICE_TYPE_CPU) {
    return;
  }

  if (failed_) {
    return;
  }

  if (skip_) {
    return;
  }

  cl_mem buffer = buffers()[0];

  size_t gws[1] = {threads};
  size_t lws[1] = {64};

  error_ = _wrapper->clSetKernelArg(kernel_, 0, sizeof(cl_mem), &buffer);
  CHECK_RESULT((error_ != CL_SUCCESS), "clSetKernelArg() failed");

  error_ = _wrapper->clSetKernelArg(kernel_, 1, sizeof(unsigned int), &level);
  CHECK_RESULT((error_ != CL_SUCCESS), "clSetKernelArg() failed");

  error_ = _wrapper->clEnqueueNDRangeKernel(cmdQueues_[_deviceId], kernel_, 1, NULL, gws, lws, 0,
                                            NULL, NULL);
  CHECK_RESULT((error_ != CL_SUCCESS), "clEnqueueNDRangeKernel() failed");

  _wrapper->clFinish(cmdQueues_[_deviceId]);

  // Try to normalize the amount of work per test
  // unsigned int repeats = (4096 / threads) * 100 ;
  unsigned int repeats = (4096 / threads) * 10;
  // unsigned int repeats = 100;
  timer.Reset();
  timer.Start();
  for (unsigned int i = 0; i < repeats; i++) {
    error_ = _wrapper->clEnqueueNDRangeKernel(cmdQueues_[_deviceId], kernel_, 1, NULL, gws, lws, 0,
                                              NULL, NULL);
    CHECK_RESULT((error_ != CL_SUCCESS), "clEnqueueNDRangeKernel() failed");

    _wrapper->clFinish(cmdQueues_[_deviceId]);
  }
  timer.Stop();

  double sec = timer.GetElapsedTime();

  _perfInfo = (float)(threads * repeats * level) / (float)(sec * 1000000.);
  char buf[256];
  SNPRINTF(buf, sizeof(buf),
           "%5d threads spawning 64 threads, queue size %3dKB (Mdisp/s), level=%2d", threads,
           queueSize / 1024, level);
  testDescString = buf;
}

unsigned int OCLPerfDeviceEnqueue2::close(void) {
  // FIXME: Re-enable CPU test once bug 10143 is fixed.
  if (type_ == CL_DEVICE_TYPE_CPU) {
    return 0;
  }

  if (deviceQueue_) {
    error_ = _wrapper->clReleaseCommandQueue(deviceQueue_);
    CHECK_RESULT_NO_RETURN(error_ != CL_SUCCESS, "clReleaseCommandQueue failed");
  }
  if (kernel2_) {
    error_ = _wrapper->clReleaseKernel(kernel2_);
    CHECK_RESULT_NO_RETURN(error_ != CL_SUCCESS, "clReleaseKernel failed");
  }
  return OCLTestImp::close();
}