TheRock/rocm-systems/projects/clr/opencl/tests/ocltst/module/gl/OCLGLFenceSync.cpp

/*
 * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 *
 * SPDX-License-Identifier: MIT
 */

#include "OCLGLFenceSync.h"

#include <assert.h>
#include <stdio.h>
#include <string.h>
#include <time.h>

#include "Timer.h"
#ifndef WIN_OS
#include <GL/glx.h>
#endif

const static char* strKernel =
    "__kernel void glmulticontext_test( __global uint4 *source, __global uint4 "
    "*dest)   \n"
    "{                                                                         "
    "         \n"
    "    int  tid = get_global_id(0);                                          "
    "         \n"
    "    dest[ tid ] = source [ tid ] + (uint4)(1);                            "
    "         \n"
    "}                                                                         "
    "         \n";

OCLGLFenceSync::OCLGLFenceSync() {
  memset(contextData_, 0, sizeof(contextData_));
  _numSubTests = 2;
}

OCLGLFenceSync::~OCLGLFenceSync() {}

#ifdef WIN_OS
typedef GLsync(__stdcall* glFenceSyncPtr)(GLenum condition, GLbitfield flags);
typedef bool(__stdcall* glIsSyncPtr)(GLsync sync);
typedef void(__stdcall* glDeleteSyncPtr)(GLsync sync);
typedef GLenum(__stdcall* glClientWaitSyncPtr)(GLsync sync, GLbitfield flags, GLuint64 timeout);
typedef void(__stdcall* glWaitSyncPtr)(GLsync sync, GLbitfield flags, GLuint64 timeout);
typedef void(__stdcall* glGetInteger64vPtr)(GLenum pname, GLint64* params);
typedef void(__stdcall* glGetSyncivPtr)(GLsync sync, GLenum pname, GLsizei bufSize, GLsizei* length,
                                        GLint* values);
#else
typedef GLsync (*glFenceSyncPtr)(GLenum condition, GLbitfield flags);
typedef bool (*glIsSyncPtr)(GLsync sync);
typedef void (*glDeleteSyncPtr)(GLsync sync);
typedef GLenum (*glClientWaitSyncPtr)(GLsync sync, GLbitfield flags, GLuint64 timeout);
typedef void (*glWaitSyncPtr)(GLsync sync, GLbitfield flags, GLuint64 timeout);
typedef void (*glGetInteger64vPtr)(GLenum pname, GLint64* params);
typedef void (*glGetSyncivPtr)(GLsync sync, GLenum pname, GLsizei bufSize, GLsizei* length,
                               GLint* values);
#endif

typedef struct __GLsync* GLsync;

glFenceSyncPtr glFenceSyncFunc;

glIsSyncPtr glIsSyncFunc;

glDeleteSyncPtr glDeleteSyncFunc;

glClientWaitSyncPtr glClientWaitSyncFunc;

glWaitSyncPtr glWaitSyncFunc;

glGetInteger64vPtr glGetInteger64vFunc;

glGetSyncivPtr glGetSyncivFunc;

#define CHK_GL_ERR() printf("%s\n", gluErrorString(glGetError()))

#define cl_khr_gl_event 1

static void InitSyncFns() {
#ifdef WIN_OS
  glFenceSyncFunc = (glFenceSyncPtr)wglGetProcAddress("glFenceSync");
  glIsSyncFunc = (glIsSyncPtr)wglGetProcAddress("glIsSync");
  glDeleteSyncFunc = (glDeleteSyncPtr)wglGetProcAddress("glDeleteSync");
  glClientWaitSyncFunc = (glClientWaitSyncPtr)wglGetProcAddress("glClientWaitSync");
  glWaitSyncFunc = (glWaitSyncPtr)wglGetProcAddress("glWaitSync");
  glGetInteger64vFunc = (glGetInteger64vPtr)wglGetProcAddress("glGetInteger64v");
  glGetSyncivFunc = (glGetSyncivPtr)wglGetProcAddress("glGetSynciv");
#else
  glFenceSyncFunc = (glFenceSyncPtr)glXGetProcAddress((GLubyte*)"glFenceSync");
  glIsSyncFunc = (glIsSyncPtr)glXGetProcAddress((GLubyte*)"glIsSync");
  glDeleteSyncFunc = (glDeleteSyncPtr)glXGetProcAddress((GLubyte*)"glDeleteSync");
  glClientWaitSyncFunc = (glClientWaitSyncPtr)glXGetProcAddress((GLubyte*)"glClientWaitSync");
  glWaitSyncFunc = (glWaitSyncPtr)glXGetProcAddress((GLubyte*)"glWaitSync");
  glGetInteger64vFunc = (glGetInteger64vPtr)glXGetProcAddress((GLubyte*)"glGetInteger64v");
  glGetSyncivFunc = (glGetSyncivPtr)glXGetProcAddress((GLubyte*)"glGetSynciv");
#endif
}

#define USING_ARB_sync 1

typedef cl_event(CL_API_CALL* clCreateEventFromGLsyncKHR_fn)(cl_context context, GLsync sync,
                                                             cl_int* errCode_ret);

clCreateEventFromGLsyncKHR_fn clCreateEventFromGLsyncKHR_ptr;

/* Helper to determine if an extension is supported by a device */
int is_extension_available(cl_device_id device, const char* extensionName) {
  char* extString;
  size_t size = 0;
  int err;
  int result = -1;

  if ((err = clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &size))) {
    printf(
        "Error: failed to determine size of device extensions string (err = "
        "%d)\n",
        err);
    return -2;
  }

  if (0 == size) return -3;

  extString = (char*)malloc(size);
  if (NULL == extString) {
    printf(
        "Error: unable to allocate %ld byte buffer for extension string (err = "
        "%d)\n",
        (long)size, err);
    return -40;
  }

  if ((err = clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, size, extString, NULL))) {
    printf("Error: failed to obtain device extensions string (err = %d)\n", err);
    free(extString);
    return -5;
  }

  if (strstr(extString, extensionName)) result = 0;

  free(extString);
  return result;
}

void OCLGLFenceSync::open(unsigned int test, char* units, double& conversion,
                          unsigned int deviceId) {
  _openTest = test;

  // Initialize random number seed
  srand((unsigned int)time(NULL));

  OCLGLCommon::open(test, units, conversion, deviceId);
  if (_errorFlag) return;

  cl_context_properties properties[7] = {0};
  for (unsigned int i = 0; i < c_glContextCount; i++) {
    error_ = is_extension_available(devices_[_deviceId], "cl_khr_gl_event");
    if (error_ != CL_SUCCESS) {
      printf("Silent failure: cl_khr_gl_event extension not available (%d)\n", error_);
      extensionSupported_ = false;
      return;
    }
    extensionSupported_ = true;

    createGLContext(contextData_[i].glContext);
    getCLContextPropertiesFromGLContext(contextData_[i].glContext, properties);

    // Create new CL context from GL context
    contextData_[i].clContext =
        _wrapper->clCreateContext(properties, 1, &devices_[_deviceId], NULL, NULL, &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "clCreateContext() failed (%d)", error_);

    // Create command queue for new context
    contextData_[i].clCmdQueue =
        _wrapper->clCreateCommandQueue(contextData_[i].clContext, devices_[_deviceId], 0, &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "clCreateCommandQueue() failed (%d)", error_);

    // Build the kernel
    contextData_[i].clProgram = _wrapper->clCreateProgramWithSource(contextData_[i].clContext, 1,
                                                                    &strKernel, NULL, &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "clCreateProgramWithSource()  failed (%d)", error_);

    error_ = _wrapper->clBuildProgram(contextData_[i].clProgram, 1, &devices_[deviceId], NULL, NULL,
                                      NULL);
    if (error_ != CL_SUCCESS) {
      char programLog[1024];
      _wrapper->clGetProgramBuildInfo(contextData_[i].clProgram, devices_[deviceId],
                                      CL_PROGRAM_BUILD_LOG, 1024, programLog, 0);
      printf("\n%s\n", programLog);
      fflush(stdout);
    }
    CHECK_RESULT((error_ != CL_SUCCESS), "clBuildProgram() failed (%d)", error_);

    contextData_[i].clKernel =
        _wrapper->clCreateKernel(contextData_[i].clProgram, "glmulticontext_test", &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "clCreateKernel() failed (%d)", error_);
  }
}

void OCLGLFenceSync::run() {
  if (_errorFlag || !extensionSupported_) {
    return;
  }

  CPerfCounter timer;
  double sec;
  float perf;
  cl_uint4 inOutData[c_numOfElements] = {{{0}}};
  cl_uint4 expectedData[c_numOfElements] = {{{0}}};
  unsigned int m = sizeof(cl_uint4) / sizeof(cl_uint);
  int count = 0;
  // Initialize input data with random values
  for (unsigned int i = 0; i < c_numOfElements; i++) {
    for (unsigned int j = 0; j < m; j++) {
      inOutData[i].s[j] = (unsigned int)i;
      expectedData[i].s[j] = inOutData[i].s[j] + c_glContextCount;
    }
  }

  cl_event fenceEvent0 = NULL, fenceEvent = NULL;
  GLsync glFence0 = NULL, glFence = NULL;
  InitSyncFns();

  clCreateEventFromGLsyncKHR_ptr =
      (clCreateEventFromGLsyncKHR_fn)clGetExtensionFunctionAddress("clCreateEventFromGLsyncKHR");
  if (clCreateEventFromGLsyncKHR_ptr == NULL) {
    printf(
        "ERROR: Unable to run fence_sync test (clCreateEventFromGLsyncKHR "
        "function not discovered!)\n");
    return;
  }

  for (unsigned int i = 0; i < c_glContextCount; i++) {
    makeCurrent(contextData_[i].glContext);

    // Generate and Bind in & out OpenGL buffers
    GLuint inGLBuffer = 0, outGLBuffer = 0;
    glGenBuffers(1, &inGLBuffer);
    glGenBuffers(1, &outGLBuffer);

    glBindBuffer(GL_ARRAY_BUFFER, inGLBuffer);
    glBufferData(GL_ARRAY_BUFFER, c_numOfElements * sizeof(cl_uint4), inOutData, GL_STATIC_DRAW);

    glBindBuffer(GL_ARRAY_BUFFER, outGLBuffer);
    glBufferData(GL_ARRAY_BUFFER, c_numOfElements * sizeof(cl_uint4), NULL, GL_STATIC_DRAW);

    glBindBuffer(GL_ARRAY_BUFFER, 0);

    glFinish();

    // Checking if clWaitForEvents works
    switch (_openTest) {
      case 0:  // Using fence sync
        glFence0 = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
        CHECK_RESULT((glFence0 == NULL), "Unable to create GL fence");

        fenceEvent0 = clCreateEventFromGLsyncKHR_ptr(contextData_[i].clContext, glFence0, &error_);
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to create CL event from GL fence (%d)",
                     error_);

        error_ = clWaitForEvents(1, &fenceEvent0);
        CHECK_RESULT((error_ != CL_SUCCESS), "clWaitForEvents() failed (%d)", error_);
        break;
      default:
        glFinish();
        break;
    }

    if (fenceEvent != NULL) {
      clReleaseEvent(fenceEvent0);
      glDeleteSync(glFence0);
    }

    cl_event acqEvent1 = 0, acqEvent2 = 0, kernelEvent = 0, relEvent1 = 0, relEvent2 = 0;

    // Create input buffer from GL input buffer
    contextData_[i].inputBuffer = _wrapper->clCreateFromGLBuffer(
        contextData_[i].clContext, CL_MEM_READ_ONLY, inGLBuffer, &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "Unable to create input GL buffer (%d)", error_);

    // Create output buffer from GL output buffer
    contextData_[i].outputBuffer = _wrapper->clCreateFromGLBuffer(
        contextData_[i].clContext, CL_MEM_WRITE_ONLY, outGLBuffer, &error_);
    CHECK_RESULT((error_ != CL_SUCCESS), "Unable to create output GL buffer (%d)", error_);

    timer.Reset();
    switch (_openTest) {
      case 0:  // Using fence sync
        timer.Start();
        glFence = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
        timer.Stop();
        CHECK_RESULT((glFence == NULL), "Unable to create GL fence");

        timer.Start();
        fenceEvent = clCreateEventFromGLsyncKHR_ptr(contextData_[i].clContext, glFence, &error_);
        timer.Stop();
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to create CL event from GL fence (%d)",
                     error_);
        break;
      default:
        break;
    }

    error_ = _wrapper->clSetKernelArg(contextData_[i].clKernel, 0, sizeof(cl_mem),
                                      &(contextData_[i].inputBuffer));
    CHECK_RESULT((error_ != CL_SUCCESS), "clSetKernelArg() failed (%d)", error_);

    error_ = _wrapper->clSetKernelArg(contextData_[i].clKernel, 1, sizeof(cl_mem),
                                      &(contextData_[i].outputBuffer));
    CHECK_RESULT((error_ != CL_SUCCESS), "clSetKernelArg() failed (%d)", error_);

    switch (_openTest) {
      case 0:  // Using fence sync
        timer.Start();
        error_ = _wrapper->clEnqueueAcquireGLObjects(contextData_[i].clCmdQueue, 1,
                                                     &(contextData_[i].inputBuffer), 1, &fenceEvent,
                                                     &acqEvent1);
        timer.Stop();
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to acquire GL objects (%d)", error_);

        timer.Start();
        error_ = _wrapper->clEnqueueAcquireGLObjects(contextData_[i].clCmdQueue, 1,
                                                     &(contextData_[i].outputBuffer), 1,
                                                     &fenceEvent, &acqEvent2);
        timer.Stop();
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to acquire GL objects (%d)", error_);
        break;
      case 1:  // Using glFinish
        timer.Start();
        glFinish();
        timer.Stop();

        timer.Start();
        error_ = _wrapper->clEnqueueAcquireGLObjects(
            contextData_[i].clCmdQueue, 1, &(contextData_[i].inputBuffer), 0, NULL, &acqEvent1);
        timer.Stop();
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to acquire GL objects (%d)", error_);

        timer.Start();
        error_ = _wrapper->clEnqueueAcquireGLObjects(
            contextData_[i].clCmdQueue, 1, &(contextData_[i].outputBuffer), 0, NULL, &acqEvent2);
        timer.Stop();
        CHECK_RESULT((error_ != CL_SUCCESS), "Unable to acquire GL objects (%d)", error_);
        break;
      default:
        break;
    }

    size_t gws[1] = {c_numOfElements};
    cl_event evts[2] = {acqEvent1, acqEvent2};
    error_ = _wrapper->clEnqueueNDRangeKernel(contextData_[i].clCmdQueue, contextData_[i].clKernel,
                                              1, NULL, gws, NULL, 2, evts, &kernelEvent);
    CHECK_RESULT((error_ != CL_SUCCESS), "clEnqueueNDRangeKernel() failed (%d)", error_);

    error_ = _wrapper->clEnqueueReleaseGLObjects(
        contextData_[i].clCmdQueue, 1, &(contextData_[i].inputBuffer), 1, &kernelEvent, &relEvent1);
    CHECK_RESULT((error_ != CL_SUCCESS), "clEnqueueReleaseGLObjects failed (%d)", error_);

    error_ = _wrapper->clEnqueueReleaseGLObjects(contextData_[i].clCmdQueue, 1,
                                                 &(contextData_[i].outputBuffer), 1, &kernelEvent,
                                                 &relEvent2);
    CHECK_RESULT((error_ != CL_SUCCESS), "clEnqueueReleaseGLObjects failed (%d)", error_);

    evts[0] = relEvent1;
    evts[1] = relEvent2;
    error_ = clWaitForEvents(2, evts);
    CHECK_RESULT((error_ != CL_SUCCESS), "clWaitForEvents() failed (%d)", error_);

    glBindBuffer(GL_ARRAY_BUFFER, outGLBuffer);
    void* glMem = glMapBuffer(GL_ARRAY_BUFFER, GL_READ_ONLY);
    memcpy(inOutData, glMem, c_numOfElements * sizeof(cl_uint4));
    glUnmapBuffer(GL_ARRAY_BUFFER);

    _wrapper->clReleaseMemObject(contextData_[i].inputBuffer);
    _wrapper->clReleaseMemObject(contextData_[i].outputBuffer);

    // Delete GL buffers
    glBindBuffer(GL_ARRAY_BUFFER, 0);
    glDeleteBuffers(1, &inGLBuffer);
    inGLBuffer = 0;
    glDeleteBuffers(1, &outGLBuffer);
    outGLBuffer = 0;
  }

  sec = timer.GetElapsedTime();
  perf = (float)sec * 1000000;  // in microseconds
  _perfInfo = (float)perf;

  if (fenceEvent != NULL) {
    clReleaseEvent(fenceEvent);
    glDeleteSync(glFence);
  }

  // Compare expected output with actual data received
  for (unsigned int i = 0; i < c_numOfElements; i++) {
    for (unsigned int j = 0; j < m; j++) {
      if (inOutData[i].s[j] != expectedData[i].s[j]) {
        printf(
            "Element %u is incorrect!\t expected:[ %u, %u, %u, %u ] differs "
            "from actual:{%u, %u, %u, %u}\n",
            i, expectedData[i].s[0], expectedData[i].s[1], expectedData[i].s[2],
            expectedData[i].s[3], inOutData[i].s[0], inOutData[i].s[1], inOutData[i].s[2],
            inOutData[i].s[3]);

        count++;
      }
    }
  }
  if (count) printf("Number of elements wrong: %d\n", count);
}

unsigned int OCLGLFenceSync::close() {
  error_ = is_extension_available(devices_[_deviceId], "cl_khr_gl_event");
  if (error_ == CL_SUCCESS) {
    for (unsigned int i = 0; i < c_glContextCount; i++) {
      makeCurrent(contextData_[i].glContext);
      _wrapper->clReleaseKernel(contextData_[i].clKernel);
      _wrapper->clReleaseProgram(contextData_[i].clProgram);
      _wrapper->clReleaseCommandQueue(contextData_[i].clCmdQueue);
      _wrapper->clReleaseContext(contextData_[i].clContext);
      destroyGLContext(contextData_[i].glContext);
    }
  }

  return OCLGLCommon::close();
}