TheRock/rocm-systems/projects/clr/rocclr/platform/command.cpp

/*
 * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 *
 * SPDX-License-Identifier: MIT
 */

#include "platform/activity.hpp"
#include "platform/command.hpp"
#include "platform/commandqueue.hpp"
#include "device/device.hpp"
#include "platform/context.hpp"
#include "platform/kernel.hpp"
#include "thread/monitor.hpp"
#include "platform/memory.hpp"
#include "platform/agent.hpp"
#include "os/alloc.hpp"

#include <atomic>
#include <cstring>
#include <algorithm>

namespace amd {

// ================================================================================================
Event::Event(HostQueue& queue, bool profilingEnabled)
    : callbacks_(NULL),
      status_(CL_INT_MAX),
      hw_event_(nullptr),
      notify_event_(nullptr),
      device_(&queue.device()),
      profilingInfo_(profilingEnabled) {
  event_entry_scope_.store(Device::kCacheStateInvalid, std::memory_order_relaxed);
  notified_.clear();
}

// ================================================================================================
Event::Event()
    : callbacks_(NULL),
      status_(CL_SUBMITTED),
      hw_event_(nullptr),
      notify_event_(nullptr),
      device_(nullptr) {
  event_entry_scope_.store(Device::kCacheStateInvalid, std::memory_order_relaxed);
  notified_.clear();
}

// ================================================================================================
Event::~Event() {
  CallBackEntry* callback = callbacks_;
  while (callback != NULL) {
    CallBackEntry* next = callback->next_;
    delete callback;
    callback = next;
  }
  if (auto* notifyEvent = notify_event_.load(std::memory_order_acquire)) {
    notifyEvent->release();
  }
  // Destroy global HW event if available
  if ((hw_event_ != nullptr) && (device_ != nullptr)) {
    device_->ReleaseGlobalSignal(hw_event_);
  }
}

// ================================================================================================
AccumulateCommand::~AccumulateCommand() {
  // When an external owner (e.g. the graph signal pool) reclaims the signals,
  // do not destroy them here.
  if (!owns_hw_events_) {
    return;
  }
  // Release all retained HW events per device
  for (auto& device_events_pair : hw_events_) {
    Device* dev = device_events_pair.first;
    if (dev != nullptr) {
      for (void* hw_event : device_events_pair.second) {
        if (hw_event != nullptr) {
          dev->ReleaseGlobalSignal(hw_event);
        }
      }
    }
  }
}

// ================================================================================================
uint64_t Event::recordProfilingInfo(int32_t status, uint64_t timeStamp) {
  if (timeStamp == 0) {
    timeStamp = Os::timeNanos();
  }
  switch (status) {
    case CL_QUEUED:
      profilingInfo_.queued_ = timeStamp;
      break;
    case CL_SUBMITTED:
      profilingInfo_.submitted_ = timeStamp;
      break;
    case CL_RUNNING:
      profilingInfo_.start_ = timeStamp;
      break;
    default:
      profilingInfo_.end_ = timeStamp;
      break;
  }
  return timeStamp;
}

// Global epoch time since the first processed command
uint64_t epoch = 0;
std::once_flag epoch_init;
// ================================================================================================
bool Event::setStatus(int32_t status, uint64_t timeStamp) {
  assert(status <= CL_QUEUED && "invalid status");

  int32_t currentStatus = this->status();
  if (currentStatus <= CL_COMPLETE || currentStatus <= status) {
    // We can only move forward in the execution status.
    return false;
  }

  if (profilingInfo().enabled_) {
    timeStamp = recordProfilingInfo(status, timeStamp);
    std::call_once(epoch_init, [&]{ epoch = profilingInfo().queued_;});
  }

  if (amd::IS_HIP) {
    // HIP API doesn't have any event, associated with a callback. Hence the SW status of
    // the event is irrelevant, during the actual callback. At the same time HIP API requires
    // to finish the callback before HIP stream can continue. Hence runtime has to process
    // the callback first and then update the status.
    if (callbacks_ != (CallBackEntry*)0) {
      processCallbacks(status);
    }
    if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
      // Somebody else beat us to it, let them deal with the release/signal.
      return false;
    }
  } else {
    if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
      // Somebody else beat us to it, let them deal with the release/signal.
      return false;
    }
    if (callbacks_ != (CallBackEntry*)0) {
      processCallbacks(status);
    }
  }

  if (Agent::shouldPostEventEvents() && command().type() != 0) {
    Agent::postEventStatusChanged(as_cl(this), status, timeStamp + Os::offsetToEpochNanos());
  }

  if (status <= CL_COMPLETE) {
    // Before we notify the waiters that this event reached the CL_COMPLETE
    // status, we release all the resources associated with this instance.
    if (!IS_HIP) {
      releaseResources();
    }

    if (profilingInfo().enabled_) {
      amd::activity_prof::ReportActivity(command());
    }

    // Broadcast all the waiters.
    if (referenceCount() > 1) {
      signal();
    }

    if (profilingInfo().enabled_) {
      ClPrint(LOG_DETAIL_DEBUG, LOG_CMD, "Command %p complete (Wall: %ld, CPU: %ld, GPU: %ld us)",
              &command(), ((profilingInfo().end_ - epoch) / 1000),
              ((profilingInfo().submitted_ - profilingInfo().queued_) / 1000),
              ((profilingInfo().end_ - profilingInfo().start_) / 1000));
    } else {
      ClPrint(LOG_DETAIL_DEBUG, LOG_CMD, "Command %p complete", &command());
    }
    release();
  }

  return true;
}

// ================================================================================================
bool Event::resetStatus(int32_t status) {
  int32_t currentStatus = this->status();
  if (currentStatus != CL_COMPLETE) {
    ClPrint(LOG_ERROR, LOG_CMD, "Command is reset before complete current status :%d",
            currentStatus);
  }
  if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
    ClPrint(LOG_ERROR, LOG_CMD, "Failed to reset command status");
    return false;
  }
  notified_.clear();
  return true;
}

// ================================================================================================
bool Event::setCallback(int32_t status, Event::CallBackFunction callback, void* data,
                        bool blocking) {
  assert(status >= CL_COMPLETE && status <= CL_QUEUED && "Invalid status");

  CallBackEntry* entry = new CallBackEntry(status, callback, data, blocking);
  if (entry == NULL) {
    return false;
  }

  entry->next_ = callbacks_;
  while (!callbacks_.compare_exchange_weak(
      entry->next_, entry));  // Someone else is also updating the head of the linked list! reload.

  // Check if the event has already reached 'status'
  if (this->status() <= status && entry->callback_ != CallBackFunction(0)) {
    if (entry->callback_.exchange(NULL) != NULL) {
      callback(as_cl(this), status, entry->data_);
    }
  }

  return true;
}

// ================================================================================================
void Event::processCallbacks(int32_t status) const {
  cl_event event = const_cast<cl_event>(as_cl(this));
  const int32_t mask = (status > CL_COMPLETE) ? status : CL_COMPLETE;

  // For_each callback:
  CallBackEntry* entry;
  for (entry = callbacks_; entry != NULL; entry = entry->next_) {
    // If the entry's status matches the mask,
    if (entry->status_ == mask && entry->callback_ != CallBackFunction(0)) {
      // invoke the callback function.
      CallBackFunction callback = entry->callback_.exchange(NULL);
      if (callback != NULL) {
        callback(event, status, entry->data_);
      }
    }
  }
}

static constexpr bool kCpuWait = true;
// ================================================================================================
bool Event::awaitCompletion() {
  if (status() > CL_COMPLETE) {
    // Notifies the current command queue about waiting
    if (!notifyCmdQueue(kCpuWait)) {
      return false;
    }

    ClPrint(LOG_DETAIL_DEBUG, LOG_WAIT, "Waiting for event %p to complete, current status %d",
            this, status());
    auto* queue = command().queue();
    if ((queue != nullptr) && queue->vdev()->ActiveWait()) {
      // Active wait: yield in a tight loop. Bail out if the queue's worker
      // thread is no longer alive, otherwise this spins forever during
      // process teardown -- on Windows ExitProcess terminates worker threads
      // before DLL_PROCESS_DETACH, so __hipUnregisterFatBinary ->
      // SyncAllStreams -> finish() reaches here with no thread left to signal
      // completion (ROCm/TheRock#999). Only meaningful when a real worker
      // thread exists (handle != nullptr); direct dispatch has none.
      while (status() > CL_COMPLETE) {
        if (queue->thread().handle() != nullptr && !amd::Os::isThreadAlive(queue->thread())) {
          break;
        }
        amd::Os::yield();
      }
    } else {
      ScopedLock lock(lock_);

      // Wait until the status becomes CL_COMPLETE or negative. Bail out if the
      // worker thread is dead -- nobody would signal the condition variable
      // and we would block forever during teardown (see comment above).
      while (status() > CL_COMPLETE) {
        if (queue != nullptr && queue->thread().handle() != nullptr &&
            !amd::Os::isThreadAlive(queue->thread())) {
          break;
        }
        lock_.wait();
      }
    }
    if (status() == CL_COMPLETE) {
      ClPrint(LOG_DETAIL_DEBUG, LOG_WAIT, "Event %p wait completed", this);
    } else {
      ClPrint(LOG_DETAIL_DEBUG, LOG_WAIT,
              "Event %p wait aborted, worker thread no longer alive (status %d)", this, status());
    }
  }

  return status() == CL_COMPLETE;
}

// ================================================================================================
bool Event::notifyCmdQueue(bool cpu_wait) {
  HostQueue* queue = command().queue();
  if (AMD_DIRECT_DISPATCH) {
    ScopedLock l(notify_lock_);
    if ((status() > CL_COMPLETE) && (nullptr != queue) &&
        // If HW event was assigned, then notification can be ignored, since a barrier was issued
        // @note: Force the marker always in OCL for now, since OCL events require precise
        // sequence of the status update
        ((HwEvent() == nullptr) || !amd::IS_HIP) && (notify_event_ == nullptr)) {
      // Make sure the queue is draining the enqueued commands.
      amd::Command* command = new amd::Marker(*queue, false, nullWaitList, this, cpu_wait);
      if (command == NULL) {
        return false;
      }
      command->enqueue();
      // Save notification, associated with the current event
      notify_event_ = command;
    }
  } else {
    if ((status() > CL_COMPLETE) && (nullptr != queue) && !notified_.test_and_set()) {
      // Make sure the queue is draining the enqueued commands.
      amd::Command* command = new amd::Marker(*queue, false, nullWaitList, this);
      if (command == NULL) {
        notified_.clear();
        return false;
      }
      command->enqueue();
      command->release();
    }
  }
  return true;
}

const Event::EventWaitList Event::nullWaitList(0);

static bool IsActivityEnabledAndCommit(cl_command_type type) {
  auto op = amd::activity_prof::OperationId(type);
  if (amd::activity_prof::IsEnabled(op)) {
    amd::activity_prof::CommitRecord(op);
    return true;
  }
  return false;
}

// ================================================================================================
Command::Command(HostQueue& queue, cl_command_type type, const EventWaitList& eventWaitList,
                 uint32_t commandWaitBits, const Event* waitingEvent)
    : Event(queue, IsActivityEnabledAndCommit(type) ||
                       queue.properties().test(CL_QUEUE_PROFILING_ENABLE) ||
                       Agent::shouldPostEventEvents()),
      queue_(&queue),
      next_(nullptr),
      type_(type),
      waitingEvent_(waitingEvent),
      eventWaitList_(eventWaitList),
      commandWaitBits_(commandWaitBits) {
  // Retain the commands from the event wait list.
  for (const auto& event : eventWaitList) {
    event->retain();
  }
}

SysmemPool<ComputeCommand>* Command::command_pool_ = new SysmemPool<ComputeCommand>;
// ================================================================================================
void Command::operator delete(void* ptr) {
  if (DEBUG_CLR_SYSMEM_POOL) {
    command_pool_->Free(ptr);
  } else {
    ::operator delete(ptr);
  }
}

// ================================================================================================
void* Command::operator new(size_t size) {
  if (DEBUG_CLR_SYSMEM_POOL) {
    return command_pool_->Alloc(size);
  } else {
    return ::operator new(size);
  }
}

// ================================================================================================
void Command::releaseResources() {
  const Command::EventWaitList& events = eventWaitList();

  // Release the commands from the event wait list.
  for (const auto& event : events) {
    event->release();
  }
}

// ================================================================================================
void Command::enqueue() {
  assert(queue_ != NULL && "Cannot be enqueued");

  if (Agent::shouldPostEventEvents() && type_ != 0) {
    Agent::postEventCreate(as_cl(static_cast<Event*>(this)), type_);
  }

  ClPrint(LOG_DETAIL_DEBUG, LOG_CMD, "Command (%s) enqueued: %p to queue: %p",
          amd::activity_prof::getOclCommandKindString(this->type()), this, queue_);

  // Direct dispatch logic below will submit the command immediately, but the command status
  // update will occur later after flush() with a wait
  if (AMD_DIRECT_DISPATCH) {
    setStatus(CL_QUEUED);

    // Notify all commands about the waiter. Barrier will be sent in order to obtain
    // HSA signal for a wait on the current queue
    for (const auto& event : eventWaitList()) {
      if (!amd::IS_HIP && event->command().type() == CL_COMMAND_USER) {
        if (event->status() >= CL_COMPLETE) {
          reinterpret_cast<amd::UserEvent*>(event)->AddDependent(this);
        } else {
          setStatus(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
          return;
        }
      } else {
        event->notifyCmdQueue(!kCpuWait);
      }
    }

    // The batch update must be lock protected to avoid a race condition
    // when multiple threads submit/flush/update the batch at the same time
    std::scoped_lock sl(queue_->vdev()->execution());
    queue_->FormSubmissionBatch(this);

    // Enqueue flushes, except profiling markers to avoid frequent expensive callbacks
    if (((type() == 0) && profilingInfo().batch_flush_) || (type() == CL_COMMAND_MARKER) ||
        (type() == CL_COMMAND_TASK)) {
      // The current HSA signal tracking logic requires profiling enabled for the markers
      EnableProfiling();
      // Update batch head for the current marker. Hence the status of all commands can be
      // updated upon the marker completion
      SetBatchHead(queue_->GetSubmissionBatch());

      submit(*queue_->vdev());

      // The batch will be tracked with the marker now
      queue_->ResetSubmissionBatch();
    } else {
      submit(*queue_->vdev());
      queue_->FlushSubmissionBatch();
    }
  } else {
    queue_->append(*this);
    queue_->flush();
  }

  if ((queue_->device().settings().waitCommand_ && (type_ != 0)) ||
      ((commandWaitBits_ & 0x2) != 0)) {
    queue_->finish();
  }

  // set this queue status is active
  queue_->SetQueueStatus();
}

// ================================================================================================
const Context& Command::context() const { return queue_->context(); }

NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList,
                                           Kernel& kernel, const NDRangeContainer& sizes,
                                           uint32_t sharedMemBytes, uint32_t extraParam,
                                           uint32_t gridId, uint32_t numGrids, uint64_t prevGridSum,
                                           uint64_t allGridSum, uint32_t firstDevice,
                                           bool forceProfiling)
    : Command(queue, CL_COMMAND_NDRANGE_KERNEL, eventWaitList,
              AMD_SERIALIZE_KERNEL | (HIP_LAUNCH_BLOCKING << 1)),
      kernel_(kernel),
      sizes_(sizes),
      sharedMemBytes_(sharedMemBytes),
      extraParam_(extraParam),
      gridId_(gridId),
      numGrids_(numGrids),
      prevGridSum_(prevGridSum),
      allGridSum_(allGridSum),
      firstDevice_(firstDevice),
      parameters_(nullptr) {
  auto& device = queue.device();
  auto devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(device));
  if (cooperativeGroups()) {
    setNumWorkgroups();
  }

  // This optimization will set marker_ts_ but may not submit a batch.
  if (forceProfiling) {
    profilingInfo_.enabled_ = true;
    profilingInfo_.clear();
    profilingInfo_.correlation_id_ = activity_prof::correlation_id;
    profilingInfo_.marker_ts_ = true;
  }
  kernel_.retain();
}

void NDRangeKernelCommand::releaseResources() {
  kernel_.parameters().release(parameters_);
  DEBUG_ONLY(parameters_ = NULL);
  kernel_.release();
  Command::releaseResources();
}

NativeFnCommand::NativeFnCommand(HostQueue& queue, const EventWaitList& eventWaitList,
                                 void(CL_CALLBACK* nativeFn)(void*), const void* args,
                                 size_t argsSize, size_t numMemObjs, const cl_mem* memObjs,
                                 const void** memLocs)
    : Command(queue, CL_COMMAND_NATIVE_KERNEL, eventWaitList),
      nativeFn_(nativeFn),
      argsSize_(argsSize) {
  args_ = new char[argsSize_];
  if (args_ == NULL) {
    return;
  }
  ::memcpy(args_, args, argsSize_);

  memObjects_.resize(numMemObjs);
  memOffsets_.resize(numMemObjs);
  for (size_t i = 0; i < numMemObjs; ++i) {
    Memory* obj = as_amd(memObjs[i]);

    obj->retain();
    memObjects_[i] = obj;
    memOffsets_[i] = (const_address)memLocs[i] - (const_address)args;
  }
}

int32_t NativeFnCommand::invoke() {
  size_t numMemObjs = memObjects_.size();
  for (size_t i = 0; i < numMemObjs; ++i) {
    void* hostMemPtr = memObjects_[i]->getHostMem();
    if (hostMemPtr == NULL) {
      return CL_MEM_OBJECT_ALLOCATION_FAILURE;
    }
    *reinterpret_cast<void**>(&args_[memOffsets_[i]]) = hostMemPtr;
  }
  nativeFn_(args_);
  return CL_SUCCESS;
}

bool OneMemoryArgCommand::validatePeerMemory() {
  amd::Device* queue_device = &queue()->device();
  // Rocr backend maps memory from different devices by default and runtime doesn't need to track
  // extra memory objects.
  if (queue_device->settings().rocr_backend_) {
    const std::vector<Device*>& srcDevices = memory_->getContext().devices();
    if (!memory_->isArena() && srcDevices.size() == 1 && queue_device != srcDevices[0]) {
      // current device and source device are not same hence
      // explicit allow access is needed for P2P access
      device::Memory* mem = memory_->getDeviceMemory(*srcDevices[0]);
      if (!mem->getAllowedPeerAccess()) {
        void* dst = reinterpret_cast<void*>(mem->virtualAddress());
        bool status = srcDevices[0]->deviceAllowAccess(dst);
        mem->setAllowedPeerAccess(true);
        return status;
      }
    }
  }
  return true;
}

bool OneMemoryArgCommand::validateMemory() {
  // Runtime disables deferred memory allocation for single device.
  // Hence ignore memory validations
  if (queue()->context().devices().size() == 1) {
    return true;
  }
  device::Memory* mem = memory_->getDeviceMemory(queue()->device());
  if (NULL == mem) {
    LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory_->getSize());
    return false;
  }
  return true;
}

bool TwoMemoryArgsCommand::validatePeerMemory() {
  bool accessAllowed = true;
  amd::Device* queue_device = &queue()->device();
  // Explicite Allow access is needed when first time memory is accessed from other device.
  // Rules : Remote device has to provide access to current device
  // --------------------------------------------------------------------
  // Crr_Dev = src  | Allow access will be called for dst memory        |
  // --------------------------------------------------------------------
  // Crr_Dev = dst  | Allow access will be called for src memory        |
  // --------------------------------------------------------------------
  // Crr_dev = other| Allow access will be called for dst and src memory|
  // --------------------------------------------------------------------
  if (queue_device->settings().rocr_backend_) {
    const std::vector<Device*>& srcDevices = memory1_->getContext().devices();
    const std::vector<Device*>& dstDevices = memory2_->getContext().devices();
    // explicit allow access is needed for P2P access
    device::Memory* mem1 = memory1_->getDeviceMemory(*srcDevices[0]);
    if (!memory1_->isArena() && !mem1->getAllowedPeerAccess() && srcDevices.size() == 1) {
      void* src = reinterpret_cast<void*>(mem1->originalDeviceAddress());
      accessAllowed = srcDevices[0]->deviceAllowAccess(src);
      mem1->setAllowedPeerAccess(true);
    }

    device::Memory* mem2 = memory2_->getDeviceMemory(*dstDevices[0]);
    if (!memory2_->isArena() && !mem2->getAllowedPeerAccess() && dstDevices.size() == 1) {
      void* dst = reinterpret_cast<void*>(mem2->originalDeviceAddress());
      accessAllowed &= dstDevices[0]->deviceAllowAccess(dst);
      mem2->setAllowedPeerAccess(true);
    }
  }
  return accessAllowed;
}

bool TwoMemoryArgsCommand::validateMemory() {
  // Runtime disables deferred memory allocation for single device.
  // Hence ignore memory validations
  if (queue()->context().devices().size() == 1) {
    return true;
  }
  device::Memory* mem = memory1_->getDeviceMemory(queue()->device());
  if (NULL == mem) {
    LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory1_->getSize());
    return false;
  }
  mem = memory2_->getDeviceMemory(queue()->device());
  if (NULL == mem) {
    LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize());
    return false;
  }
  return true;
}
bool ReadMemoryCommand::isEntireMemory() const {
  return source().isEntirelyCovered(origin(), size());
}

bool WriteMemoryCommand::isEntireMemory() const {
  return destination().isEntirelyCovered(origin(), size());
}

bool SvmMapMemoryCommand::isEntireMemory() const {
  return getSvmMem()->isEntirelyCovered(origin(), size());
}

bool FillMemoryCommand::isEntireMemory() const {
  return memory().isEntirelyCovered(origin(), size());
}

bool CopyMemoryCommand::isEntireMemory() const {
  bool result = false;

  switch (type()) {
    case CL_COMMAND_COPY_IMAGE_TO_BUFFER: {
      Coord3D imageSize(size()[0] * size()[1] * size()[2] *
                        source().asImage()->getImageFormat().getElementSize());
      result = source().isEntirelyCovered(srcOrigin(), size()) &&
               destination().isEntirelyCovered(dstOrigin(), imageSize);
    } break;
    case CL_COMMAND_COPY_BUFFER_TO_IMAGE: {
      Coord3D imageSize(size()[0] * size()[1] * size()[2] *
                        destination().asImage()->getImageFormat().getElementSize());
      result = source().isEntirelyCovered(srcOrigin(), imageSize) &&
               destination().isEntirelyCovered(dstOrigin(), size());
    } break;
    case CL_COMMAND_COPY_BUFFER_RECT: {
      Coord3D rectSize(size()[0] * size()[1] * size()[2]);
      Coord3D srcOffs(srcRect().start_);
      Coord3D dstOffs(dstRect().start_);
      result = source().isEntirelyCovered(srcOffs, rectSize) &&
               destination().isEntirelyCovered(dstOffs, rectSize);
    } break;
    default:
      result = source().isEntirelyCovered(srcOrigin(), size()) &&
               destination().isEntirelyCovered(dstOrigin(), size());
      break;
  }
  return result;
}

bool MapMemoryCommand::isEntireMemory() const {
  return memory().isEntirelyCovered(origin(), size());
}

void UnmapMemoryCommand::releaseResources() {
  //! @todo This is a workaround to a deadlock on indirect map release.
  //! Remove this code when CAL will have a refcounter on memory.
  //! decIndMapCount() has to go back to submitUnmapMemory()
  device::Memory* mem = memory_->getDeviceMemory(queue()->device());
  if (NULL != mem) {
    mem->releaseIndirectMap();
  }

  OneMemoryArgCommand::releaseResources();
}

bool MigrateMemObjectsCommand::validateMemory() {
  // Runtime disables deferred memory allocation for single device.
  // Hence ignore memory validations
  if (queue()->context().devices().size() == 1) {
    return true;
  }
  for (const auto& it : memObjects_) {
    device::Memory* mem = it->getDeviceMemory(queue()->device());
    if (NULL == mem) {
      LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
      return false;
    }
  }
  return true;
}

// =================================================================================================
int32_t NDRangeKernelCommand::captureHIPArgsAndValidate(void** kernelParams, address kernArgs,
                                                      size_t kernArgsSize) {
  const amd::Device& device = queue()->device();
  // Validate the kernel before submission
  if (!queue()->device().validateKernel(kernel(), queue()->vdev(), cooperativeGroups())) {
    return CL_OUT_OF_RESOURCES;
  }

  parameters_ = kernel().parameters().alloc(*queue()->vdev());
  if (parameters_ == nullptr) {
    LogError("Cannot allocate memory for parameters_");
    return CL_OUT_OF_RESOURCES;
  }

  if (!kernel().parameters().captureHIPArgs(kernelParams, kernArgs, kernArgsSize, parameters_)) {
    LogError("Cannot capture and set the kernel parameters");
    return CL_OUT_OF_RESOURCES;
  }
  return CL_SUCCESS;
}

int32_t NDRangeKernelCommand::captureOpenCLArgsAndValidate() {
  const amd::Device& device = queue()->device();
  // Validate the kernel before submission
  if (!queue()->device().validateKernel(kernel(), queue()->vdev(), cooperativeGroups())) {
    return CL_OUT_OF_RESOURCES;
  }

  int32_t error;
  uint64_t lclMemSize = kernel().getDeviceKernel(device)->workGroupInfo()->localMemSize_;
  parameters_ = kernel().parameters().captureOpenCLArgs(*queue()->vdev(),
                                                        sharedMemBytes_ + lclMemSize, &error);
  return error;
}

bool ExtObjectsCommand::validateMemory() {
  // Always process GL objects, even if deferred allocations are disabled,
  // because processGLResource() calls OGL Acquire().
  bool retVal = true;
  for (const auto& it : memObjects_) {
    device::Memory* mem = it->getDeviceMemory(queue()->device());
    if (NULL == mem) {
      LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
      return false;
    }
    retVal = processGLResource(mem);
  }
  return retVal;
}

bool AcquireExtObjectsCommand::processGLResource(device::Memory* mem) {
  return mem->processGLResource(device::Memory::GLDecompressResource);
}

bool ReleaseExtObjectsCommand::processGLResource(device::Memory* mem) {
  return mem->processGLResource(device::Memory::GLInvalidateFBO);
}

bool MakeBuffersResidentCommand::validateMemory() {
  // Runtime disables deferred memory allocation for single device.
  // Hence ignore memory validations
  if (queue()->context().devices().size() == 1) {
    return true;
  }
  for (const auto& it : memObjects_) {
    device::Memory* mem = it->getDeviceMemory(queue()->device());
    if (NULL == mem) {
      LogPrintfError("Can't allocate memory size - 0x%08X bytes!", it->getSize());
      return false;
    }
  }
  return true;
}

bool ThreadTraceMemObjectsCommand::validateMemory() {
  // Runtime disables deferred memory allocation for single device.
  // Hence ignore memory validations
  if (queue()->context().devices().size() == 1) {
    return true;
  }
  for (auto it = memObjects_.cbegin(); it != memObjects_.cend(); it++) {
    device::Memory* mem = (*it)->getDeviceMemory(queue()->device());
    if (NULL == mem) {
      for (auto tmpIt = memObjects_.cbegin(); tmpIt != it; tmpIt++) {
        device::Memory* tmpMem = (*tmpIt)->getDeviceMemory(queue()->device());
        delete tmpMem;
      }
      LogPrintfError("Can't allocate memory size - 0x%08X bytes!", (*it)->getSize());
      return false;
    }
  }
  return true;
}

bool CopyMemoryP2PCommand::validateMemory() {
  amd::Device* queue_device = &queue()->device();

  // Rocr backend maps memory from different devices by default and runtime doesn't need to track
  // extra memory objects. Also P2P staging buffer always allocated
  if (queue_device->settings().rocr_backend_) {
    return validatePeerMemory();
  }

  const std::vector<Device*>& devices = memory1_->getContext().devices();
  if (devices.size() != 1) {
    LogError("Can't allocate memory object for P2P extension");
    return false;
  }
  device::Memory* mem = memory1_->getDeviceMemory(*devices[0]);
  if (nullptr == mem) {
    LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory1_->getSize());
    return false;
  }
  const std::vector<Device*>& devices2 = memory2_->getContext().devices();
  if (devices2.size() != 1) {
    LogError("Can't allocate memory object for P2P extension");
    return false;
  }
  mem = memory2_->getDeviceMemory(*devices2[0]);
  if (nullptr == mem) {
    LogPrintfError("Can't allocate memory size - 0x%08X bytes!", memory2_->getSize());
    return false;
  }
  bool p2pStaging = false;
  // Validate P2P memories on the current device, if any of them is null, then it's p2p staging
  if ((nullptr == memory1_->getDeviceMemory(queue()->device())) ||
      (nullptr == memory2_->getDeviceMemory(queue()->device()))) {
    p2pStaging = true;
  }

  if (devices[0]->P2PStage() != nullptr && p2pStaging) {
    std::scoped_lock lock(devices[0]->P2PStageOps());
    // Make sure runtime allocates memory on every device
    for (uint d = 0; d < devices[0]->GlbCtx().devices().size(); ++d) {
      device::Memory* mem =
          devices[0]->P2PStage()->getDeviceMemory(*devices[0]->GlbCtx().devices()[d]);
      if (nullptr == mem) {
        ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                "Cannot get P2P stage Device Memory for device: 0x%x \n",
                devices[0]->GlbCtx().devices()[d]);
        return false;
      }
    }
  }
  return true;
}

// ================================================================================================
bool SvmPrefetchAsyncCommand::validateMemory() {
  amd::Memory* svmMem = amd::MemObjMap::FindMemObj(dev_ptr());
  if (nullptr == svmMem) {
    LogPrintfError("SvmPrefetchAsync received unknown memory for prefetch: %p!", dev_ptr());
    return false;
  }
  return true;
}

}  // namespace amd