TheRock/rocm-systems/projects/clr/rocclr/device/rocm/rocdevice.cpp

/*
 * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
 *
 * SPDX-License-Identifier: MIT
 */

#include "cl.h"
#include "platform/program.hpp"
#include "platform/kernel.hpp"
#include "os/os.hpp"
#include "utils/debug.hpp"
#include "utils/flags.hpp"
#include "utils/options.hpp"
#include "utils/versions.hpp"
#include "thread/monitor.hpp"
#include "CL/cl_ext.h"

#include "vdi_common.hpp"
#include "device/comgrctx.hpp"
#include "device/devhostcall.hpp"
#include "device/rocm/rocdevice.hpp"
#include "device/rocm/rocblit.hpp"
#include "device/rocm/rocvirtual.hpp"
#include "device/rocm/rocprogram.hpp"
#include "device/rocm/rockernel.hpp"
#include "device/rocm/rocmemory.hpp"
#include "device/rocm/rocglinterop.hpp"
#include "device/rocm/rocsignal.hpp"
#include "platform/sampler.hpp"

#ifdef _WIN32
#include "device/rocm/rocd3d10interop.hpp"
#include "device/rocm/rocd3d11interop.hpp"
#include "platform/interop_d3d10.hpp"
#include "platform/interop_d3d11.hpp"
#endif

#if defined(__clang__)
#if __has_feature(address_sanitizer)
#include "device/rocm/rocurilocator.hpp"
#endif
#endif

#if defined(__linux__)
#include <climits>
#include <unistd.h>
#endif

#include <algorithm>
#include <chrono>
#include <cinttypes>
#include <cstring>
#include <fstream>
#include <iostream>
#include <iomanip>
#include <memory>
#include <sstream>
#include <thread>
#include <vector>

#define OPENCL_VERSION_STR XSTR(OPENCL_MAJOR) "." XSTR(OPENCL_MINOR)
#define OPENCL_C_VERSION_STR XSTR(OPENCL_C_MAJOR) "." XSTR(OPENCL_C_MINOR)


static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kNone) ==
                  static_cast<uint32_t>(HSA_ACCESS_PERMISSION_NONE),
              "Vmm Access Flag None mismatch with ROC-runtime!");
static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadOnly) ==
                  static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RO),
              "Vmm Access Flag Read mismatch with ROCr-runtime!");
static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadWrite) ==
                  static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RW),
              "Vmm Access Flag Read Write mismatch with ROC-runtime!");


namespace amd::device {
extern const char* HipExtraSourceCode;
extern const char* HipExtraSourceCodeNoGWS;
}  // namespace amd::device

namespace amd::roc {
bool roc::Device::isHsaInitialized_ = false;
std::vector<hsa_agent_t> roc::Device::gpu_agents_;
std::vector<AgentInfo> roc::Device::cpu_agents_;

address Device::mg_sync_ = nullptr;

bool NullDevice::create(const amd::Isa& isa) {
  if (!isa.runtimeRocSupported()) {
    LogPrintfError("Offline HSA device %s is not supported", isa.targetId());
    return false;
  }

  online_ = false;
  // Mark the device as GPU type
  info_.type_ = CL_DEVICE_TYPE_GPU;
  info_.vendorId_ = 0x1002;

  roc::Settings* hsaSettings = new roc::Settings();
  settings_ = hsaSettings;
  if (!hsaSettings || !hsaSettings->create(false, isa, isa.xnack() == amd::Isa::Feature::Enabled)) {
    LogPrintfError("Error creating settings for offline HSA device %s", isa.targetId());
    return false;
  }

  if (!ValidateComgr()) {
    LogPrintfError("Code object manager initialization failed for offline HSA device %s",
                   isa.targetId());
    return false;
  }

  if (!amd::Device::create(isa)) {
    LogPrintfError("Unable to setup offline HSA device %s", isa.targetId());
    return false;
  }

  // Report the device name
  ::strncpy(info_.name_, isa.targetId(), sizeof(info_.name_) - 1);
  info_.extensions_ = getExtensionString();
  info_.maxWorkGroupSize_ = hsaSettings->maxWorkGroupSize_;
  ::strncpy(info_.vendor_, "Advanced Micro Devices, Inc.", sizeof(info_.vendor_) - 1);
  info_.oclcVersion_ = "OpenCL C " OPENCL_C_VERSION_STR " ";
  info_.spirVersions_ = "";
  std::stringstream ss;
  ss << AMD_BUILD_STRING " (HSA,LC) [Offline]";
  ::strncpy(info_.driverVersion_, ss.str().c_str(), sizeof(info_.driverVersion_) - 1);
  info_.version_ = "OpenCL " OPENCL_VERSION_STR " ";
  return true;
}

Device::Device(hsa_agent_t bkendDevice)
    : mapCacheOps_(nullptr),
      mapCache_(nullptr),
      bkendDevice_(bkendDevice),
      pciDeviceId_(0),
      gpuvm_segment_max_alloc_(0),
      alloc_granularity_(0),
      xferQueue_(nullptr),
      freeMem_(0),
      hsa_exclusive_gpu_access_(false),
      numOfVgpus_(0),
      preferred_numa_node_(0),
      maxSdmaReadMask_(0),
      maxSdmaWriteMask_(0),
      sdma_engine_allocator_(*this),
      cpu_agent_info_(nullptr),
      numHwPipes_(4) {
  for (uint i = 0; i < QueuePriority::Total; ++i) {
    queuePool_.emplace_back();
  }

  group_segment_.handle = 0;
  gpuvm_segment_.handle = 0;
  gpu_fine_grained_segment_.handle = 0;
  gpu_ext_fine_grained_segment_.handle = 0;
  prefetch_signal_.handle = 0;
  isXgmi_ = false;
  cache_state_ = Device::CacheState::kCacheStateInvalid;
}

int Device::agentGlobalIndex(hsa_agent_t agent) {
  for (size_t gpu_idx = 0; gpu_idx < gpu_agents_.size(); ++gpu_idx) {
    if (gpu_agents_[gpu_idx].handle == agent.handle) {
      return static_cast<int>(gpu_idx);
    }
  }
  for (size_t cpu_idx = 0; cpu_idx < cpu_agents_.size(); ++cpu_idx) {
    if (cpu_agents_[cpu_idx].agent.handle == agent.handle) {
      return static_cast<int>(gpu_agents_.size() + cpu_idx);
    }
  }
  return -1;
}

void Device::setupCpuAgent() {
  int32_t numaDistance = std::numeric_limits<int32_t>::max();
  uint32_t index = 0;  // 0 as default
  auto size = cpu_agents_.size();
  for (uint32_t i = 0; i < size; i++) {
    std::vector<amd::Device::LinkAttrType> link_attrs;
    link_attrs.push_back(std::make_pair(LinkAttribute::kLinkDistance, 0));
    if (findLinkInfo(cpu_agents_[i].fine_grain_pool, &link_attrs)) {
      if (link_attrs[0].second < numaDistance) {
        numaDistance = link_attrs[0].second;
        index = i;
      }
    }
  }
  std::vector<amd::Device::LinkAttrType> link_attrs;
  link_attrs.push_back(std::make_pair(LinkAttribute::kLinkLinkType, 0));
  if (findLinkInfo(cpu_agents_[0].fine_grain_pool, &link_attrs)) {
    isXgmi_ = (link_attrs[0].second == HSA_AMD_LINK_INFO_TYPE_XGMI);
  }

  preferred_numa_node_ = index;
  cpu_agent_info_ = &cpu_agents_[index];

  ClPrint(amd::LOG_INFO, amd::LOG_INIT,
          "Numa selects cpu agent[%zu]=0x%zx(fine=0x%zx,"
          "coarse=0x%zx) for gpu agent=0x%zx CPU<->GPU XGMI=%d",
          index, cpu_agent_info_->agent.handle, cpu_agent_info_->fine_grain_pool.handle,
          cpu_agent_info_->coarse_grain_pool.handle, bkendDevice_.handle, isXgmi_);
}

void Device::checkAtomicSupport() {
  std::vector<amd::Device::LinkAttrType> link_attrs;
  link_attrs.push_back(std::make_pair(LinkAttribute::kLinkAtomicSupport, 0));
  if (findLinkInfo(cpu_agent_info_->fine_grain_pool, &link_attrs)) {
    if (link_attrs[0].second == 1) {
      info_.pcie_atomics_ = true;
    }
  }
}

Device::~Device() {
  // Drain the ROCr async-events thread before releasing any backend state.
  // This guards OCL teardown; for HIP the drain already ran via RuntimeTearDown,
  // so this is a safe no-op in that path.
  WaitForHsaAsyncHandlersIdle();

  // Release cached map targets
  for (uint i = 0; mapCache_ != nullptr && i < mapCache_->size(); ++i) {
    if ((*mapCache_)[i] != nullptr) {
      (*mapCache_)[i]->release();
    }
  }
  delete mapCache_;
  delete mapCacheOps_;

  if (nullptr != p2p_stage_) {
    p2p_stage_->release();
    p2p_stage_ = nullptr;
  }
  if (nullptr != mg_sync_) {
    GlbCtx().svmFree(mg_sync_);
    mg_sync_ = nullptr;
  }
  if (glb_ctx_ != nullptr) {
    glb_ctx_->release();
    glb_ctx_ = nullptr;
  }

  // Destroy transfer queue FIRST (before destroying queues in pool)
  // because its destructor will call releaseQueue()
  delete xferQueue_;
  xferQueue_ = nullptr;

  // Final drain of deferred destroys on the main thread before hsa_shut_down.
  DrainDeferredQueueDestroys();

  for (auto& it : queuePool_) {
    for (auto qIter = it.begin(); qIter != it.end();) {
      hsa_queue_t* queue = qIter->first;
      auto& qInfo = qIter->second;
      ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE, "Deleting hardware queue %p with refCount 0",
              queue->base_address);
      qIter = it.erase(qIter);
      Hsa::queue_destroy(queue);
    }
  }
  queuePool_.clear();

  delete blitProgram_;

  if (context_ != nullptr) {
    context_->release();
  }

  delete[] p2p_agents_list_;

  if (0 != prefetch_signal_.handle) {
    Hsa::signal_destroy(prefetch_signal_);
  }
}

void NullDevice::tearDown() {}

void Device::WaitForHsaAsyncHandlersIdle() {
  constexpr int kDrainTimeoutMs = 5000;
  const std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now();
  const std::chrono::steady_clock::duration fast_timeout =
      std::chrono::milliseconds(kDrainTimeoutMs);

  auto sumQueuedHandlers = [this]() -> uint64_t {
    uint64_t sum = 0;
    std::scoped_lock lock(vgpusAccess_);
    for (VirtualGPU* vgpu : vgpus_) {
      if (vgpu != nullptr) {
        sum += vgpu->QueuedAsyncHandlers().load(std::memory_order_acquire);
      }
    }
    return sum;
  };

  while (sumQueuedHandlers() != 0) {
    if (std::chrono::steady_clock::now() - start_time > fast_timeout) {
      const uint64_t remaining = sumQueuedHandlers();
      if (remaining != 0) {
        LogPrintfError(
            "WaitForHsaAsyncHandlersIdle: %d ms elapsed, VirtualGPU queued_async total=%llu; "
            "proceeding with device destruction.",
            kDrainTimeoutMs, static_cast<unsigned long long>(remaining));
      }
      return;
    }
    std::this_thread::sleep_for(std::chrono::milliseconds(2));
  }
}

bool NullDevice::init() {
  // Create offline devices for all ISAs not already associated with an online
  // device. This allows code objects to be compiled for all supported ISAs.
  std::vector<Device*> devices = getDevices(CL_DEVICE_TYPE_GPU, false);
  for (const amd::Isa* isa = amd::Isa::begin(); isa != amd::Isa::end(); isa++) {
    if (!isa->runtimeRocSupported()) {
      continue;
    }
    bool isOnline = false;
    // Check if the particular device is online
    for (size_t i = 0; i < devices.size(); i++) {
      if (&(devices[i]->isa()) == isa) {
        isOnline = true;
        break;
      }
    }
    if (isOnline) {
      continue;
    }
    std::unique_ptr<NullDevice> nullDevice(new NullDevice());
    if (!nullDevice) {
      LogPrintfError("Error allocating new instance of offline HSA device %s", isa->targetId());
      return false;
    }
    if (!nullDevice->create(*isa)) {
      LogPrintfError("Skipping creating new instance of offline HSA sevice %s", isa->targetId());
      continue;
    }
    nullDevice.release()->registerDevice();
  }
  return true;
}

NullDevice::~NullDevice() {}

hsa_status_t Device::iterateAgentCallback(hsa_agent_t agent, void* data) {
  hsa_device_type_t dev_type = HSA_DEVICE_TYPE_CPU;

  hsa_status_t stat = Hsa::agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &dev_type);

  if (stat != HSA_STATUS_SUCCESS) {
    LogPrintfError("HSA_AGENT_INFO_DEVICE failed with %x", stat);
    return stat;
  }

  if (dev_type == HSA_DEVICE_TYPE_CPU) {
    AgentInfo info = {agent, {0}, {0}, {0}};
    stat = Hsa::agent_iterate_memory_pools(agent, Device::iterateCpuMemoryPoolCallback,
                                           reinterpret_cast<void*>(&info));
    if (stat == HSA_STATUS_SUCCESS) {
      cpu_agents_.push_back(info);
    }
  } else if (dev_type == HSA_DEVICE_TYPE_GPU) {
    gpu_agents_.push_back(agent);
  }

  return stat;
}

hsa_ven_amd_loader_1_03_pfn_t Device::amd_loader_ext_table = {nullptr};

hsa_status_t Device::loaderQueryHostAddress(const void* device, const void** host) {
  return amd_loader_ext_table.hsa_ven_amd_loader_query_host_address
             ? amd_loader_ext_table.hsa_ven_amd_loader_query_host_address(device, host)
             : HSA_STATUS_ERROR;
}

// ================================================================================================
bool Device::init() {
  if (!Hsa::LoadLib()) {
    LogPrintfWarning("Failed to load rocr library!");
    return false;
  }

  hsa_status_t status = Hsa::init();

  // If there are no GPUs available, hsa_init will fail with HSA_STATUS_ERROR_OUT_OF_RESOURCES
  // but for NoGpu tests to pass, true needs to be returned
  constexpr bool kNoOfflineDevices = false;
  std::vector<amd::Device*> devices = getDevices(CL_DEVICE_TYPE_GPU, kNoOfflineDevices);
  if (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES && devices.size() == 0) {
    return true;
  }

  if (status != HSA_STATUS_SUCCESS) {
    LogPrintfError("hsa_init failed with %x", status);
    return false;
  }

  Hsa::system_get_major_extension_table(HSA_EXTENSION_AMD_LOADER, 1, sizeof(amd_loader_ext_table),
                                        &amd_loader_ext_table);

  status = Hsa::iterate_agents(iterateAgentCallback, nullptr);
  if (status != HSA_STATUS_SUCCESS) {
    LogPrintfError("hsa_iterate_agents failed with %x", status);
    return false;
  }

  std::string ordinals =
      amd::IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ? HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES)
                  : GPU_DEVICE_ORDINAL;
  if (ordinals[0] != '\0') {
    size_t pos = 0;
    std::vector<hsa_agent_t> valid_agents;
    std::set<size_t> valid_indexes;
    do {
      size_t end;
      bool deviceIdValid = true;
      end = ordinals.find_first_of(',', pos);
      if (end == std::string::npos) {
        end = ordinals.size();
      }
      std::string str_id = ordinals.substr(pos, end - pos);
      // If Uuid is specified, then convert it to index
      // Uuid is an Ascii string with a maximum of 21 chars including NULL
      // The string value is in the format GPU-<body>, <body> encodes UUID as a 16 chars hex
      if (str_id.find("GPU-") != std::string::npos) {
        for (int i = 0; i < gpu_agents_.size(); i++) {
          auto agent = gpu_agents_[i];
          char unique_id[32] = {0};
          if (HSA_STATUS_SUCCESS ==
              Hsa::agent_get_info(agent, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_UUID),
                                  unique_id)) {
            if (std::string_view(unique_id).find(str_id) != std::string_view::npos) {
              str_id = std::to_string(i);
              break;
            }
          }
        }
      }
      int index = atoi(str_id.c_str());
      if (index < 0 || static_cast<size_t>(index) >= gpu_agents_.size() ||
          str_id != std::to_string(index)) {
        deviceIdValid = false;
      }

      if (!deviceIdValid) {
        // Exit the loop as anything to the right of invalid deviceId
        // has to be discarded
        break;
      } else {
        if (valid_indexes.find(index) == valid_indexes.end()) {
          valid_agents.push_back(gpu_agents_[index]);
          valid_indexes.insert(index);
        }
      }
      pos = end + 1;
    } while (pos < ordinals.size());
    gpu_agents_ = valid_agents;
  }

  LogPrintfInfo("Initalizing runtime stack, Enumerated GPU agents = %lu", gpu_agents_.size());

  for (auto agent : gpu_agents_) {
    std::unique_ptr<Device> roc_device(new Device(agent));
    if (!roc_device) {
      LogError("Error creating new instance of Device on then heap.");
      continue;
    }

    if (!roc_device->create()) {
      LogError("Error creating new instance of Device.");
      continue;
    }

    // Setup System Memory to be Non-Coherent per user
    // request via environment variable. By default the
    // System Memory is setup to be Coherent
    if (roc_device->settings().enableNCMode_) {
      hsa_status_t err = Hsa::coherency_set_type(agent, HSA_AMD_COHERENCY_TYPE_NONCOHERENT);
      if (err != HSA_STATUS_SUCCESS) {
        LogError("Unable to set NC memory policy!");
        continue;
      }
    }

    // Check to see if a global CU mask is requested
    if (amd::IS_HIP && ROC_GLOBAL_CU_MASK[0] != '\0') {
      roc_device->getGlobalCUMask(ROC_GLOBAL_CU_MASK);
    }
    // Note: for now disable HSA path by default except for gfx942
    if (IS_WINDOWS && (GPU_ENABLE_PAL == 2) &&
        (std::string_view(roc_device->info().name_).find("gfx942") == std::string_view::npos)) {
      return false;
    }
    roc_device.release()->registerDevice();
  }

  // Query active devices only
  devices = getDevices(CL_DEVICE_TYPE_GPU, kNoOfflineDevices);
  if (devices.size() > 0) {
    bool p2p_available = false;
    // Loop through all available devices
    for (auto device1 : devices) {
      // Find all agents that can have access to the current device
      for (auto agent : static_cast<Device*>(device1)->p2pAgents()) {
        // Find cl_device_id associated with the current agent
        for (auto device2 : devices) {
          if (agent.handle == static_cast<Device*>(device2)->getBackendDevice().handle) {
            // Device2 can have access to device1
            device2->p2pDevices_.push_back(as_cl(device1));
            device1->p2p_access_devices_.push_back(device2);
            p2p_available = true;
          }
        }
      }
    }

    // Create a dummy context for internal memory allocations on all reported devices
    glb_ctx_ = new amd::Context(devices, amd::Context::Info());
    if (glb_ctx_ == nullptr) {
      LogError("glb_ctx failed");
      return false;
    }

    // Allocate a staging buffer for P2P emulation path
    if ((devices.size() >= 1) && !p2p_available) {
      amd::Buffer* buf =
          new (*glb_ctx_) amd::Buffer(*glb_ctx_, CL_MEM_ALLOC_HOST_PTR, kP2PStagingSize);
      if ((buf != nullptr) && buf->create()) {
        p2p_stage_ = buf;
      } else {
        delete buf;
        LogError("p2p stg buffer alloc failed");
        return false;
      }
    }

    // Allocate mgpu sync buffer for cooperative launches
    if (amd::IS_HIP) {
      mg_sync_ = reinterpret_cast<address>(
          glb_ctx_->svmAlloc(kMGInfoSizePerDevice * devices.size(), kMGInfoSizePerDevice,
                             (CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS)));
      if (mg_sync_ == nullptr) {
        LogError("mgpu sync buffer alloc failed");
        return false;
      }
    }
  }

  if (amd::IS_HIP) {
    RegisterBackendErrorCb();
  }

  return true;
}

extern const char* SchedulerSourceCode;

void Device::tearDown() {
  NullDevice::tearDown();
  Hsa::shut_down();
}

// ================================================================================================
bool Device::create() {
  char agent_name[64] = {0};
  if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_NAME, agent_name)) {
    LogError("Unable to get HSA device name");
    return false;
  }

  if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_,
                                                (hsa_agent_info_t)HSA_AMD_AGENT_INFO_CHIP_ID,
                                                &pciDeviceId_)) {
    LogPrintfError("Unable to get PCI ID of HSA device %s", agent_name);
    return false;
  }

  struct agent_isas_t {
    uint count;
    hsa_isa_t first_isa;
  } agent_isas = {0, {0}};
  if (HSA_STATUS_SUCCESS != Hsa::agent_iterate_isas(
                                bkendDevice_,
                                [](hsa_isa_t isa, void* data) {
                                  agent_isas_t* agent_isas = static_cast<agent_isas_t*>(data);
                                  if (agent_isas->count++ == 0) {
                                    agent_isas->first_isa = isa;
                                  }
                                  return HSA_STATUS_SUCCESS;
                                },
                                &agent_isas)) {
    LogPrintfError("Unable to iterate supported ISAs for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  uint32_t isa_name_length = 0;
  if (HSA_STATUS_SUCCESS != Hsa::isa_get_info_alt(agent_isas.first_isa,
                                                  (hsa_isa_info_t)HSA_ISA_INFO_NAME_LENGTH,
                                                  &isa_name_length)) {
    LogPrintfError("Unable to get ISA name length for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  std::vector<char> isa_name(isa_name_length + 1, '\0');
  if (HSA_STATUS_SUCCESS != Hsa::isa_get_info_alt(agent_isas.first_isa,
                                                  (hsa_isa_info_t)HSA_ISA_INFO_NAME,
                                                  isa_name.data())) {
    LogPrintfError("Unable to get ISA name for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  const amd::Isa* isa = amd::Isa::findIsa(isa_name.data());
  if (!isa || !isa->runtimeRocSupported()) {
    LogPrintfError("Unsupported HSA device %s (PCI ID %x) for ISA %s", agent_name, pciDeviceId_,
                   isa_name.data());
    return false;
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_PROFILE, &agent_profile_)) {
    LogPrintfError("Unable to get profile for HSA device %s (PCI ID %x)", agent_name, pciDeviceId_);
    return false;
  }

  uint32_t coop_groups = 0;
  // Check cooperative groups for HIP only
  if (amd::IS_HIP &&
      (HSA_STATUS_SUCCESS !=
       Hsa::agent_get_info(bkendDevice_,
                           static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_COOPERATIVE_QUEUES),
                           &coop_groups))) {
    LogPrintfError(
        "Unable to determine if cooperative queues are supported for HSA device %s (PCI ID %x)",
        agent_name, pciDeviceId_);
    return false;
  }

  setupCpuAgent();

  // Get Agent HDP Flush Register Memory
  hsa_amd_hdp_flush_t hdpInfo;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HDP_FLUSH),
                          &hdpInfo)) {
    LogPrintfError("Unable to determine HDP flush info for HSA device %s", agent_name);
    return false;
  }

  info_.hdpMemFlushCntl = hdpInfo.HDP_MEM_FLUSH_CNTL;
  info_.hdpRegFlushCntl = hdpInfo.HDP_REG_FLUSH_CNTL;

  // Create HSA settings
  assert(!settings_);
  roc::Settings* hsaSettings = new roc::Settings();
  settings_ = hsaSettings;
  if (!hsaSettings || !hsaSettings->create((agent_profile_ == HSA_PROFILE_FULL), *isa,
                                           isa->xnack() == amd::Isa::Feature::Enabled, coop_groups,
                                           isXgmi_)) {
    LogPrintfError("Unable to create settings for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  if (!ValidateComgr()) {
    LogPrintfError("Code object manager initialization failed for HSA device %s (PCI ID %x)",
                   agent_name, pciDeviceId_);
    return false;
  }

  if (!amd::Device::create(*isa)) {
    LogPrintfError("Unable to setup device for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  uint32_t hsa_bdf_id = 0;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_BDFID),
                          &hsa_bdf_id)) {
    LogPrintfError("Unable to determine BFD ID for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }

  info_.deviceTopology_.pcie.type = CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD;
  info_.deviceTopology_.pcie.bus = (hsa_bdf_id & (0xFF << 8)) >> 8;
  info_.deviceTopology_.pcie.device = (hsa_bdf_id & (0x1F << 3)) >> 3;
  info_.deviceTopology_.pcie.function = (hsa_bdf_id & 0x07);
  uint32_t pci_domain_id = 0;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_DOMAIN),
                          &pci_domain_id)) {
    LogPrintfError("Unable to determine domain ID for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }
  info_.pciDomainID = pci_domain_id;

#ifdef _WIN32
  // Extract adapter LUID for D3D interop validation
  hsa_status_t luid_status = Hsa::agent_get_info(
      bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_LUID),
      &deviceLuid_);
  luidValid_ = luid_status == HSA_STATUS_SUCCESS;
  if (!luidValid_) {
    LogWarning("Could not extract LUID from HSA agent - D3D interop may not work correctly");
  }
#endif

  if (populateOCLDeviceConstants() == false) {
    LogPrintfError("populateOCLDeviceConstants failed for HSA device %s (PCI ID %x)", agent_name,
                   pciDeviceId_);
    return false;
  }
  hsaSettings->limit_blit_wg_ = info().maxComputeUnits_;
  if (!flagIsDefault(DEBUG_CLR_LIMIT_BLIT_WG)) {
    hsaSettings->limit_blit_wg_ = std::max(DEBUG_CLR_LIMIT_BLIT_WG, 0x1U);
  }
  amd::Context::Info info = {0};
  std::vector<amd::Device*> devices{this};

  // Create a dummy context
  context_ = new amd::Context(devices, info);
  if (context_ == nullptr) {
    return false;
  }

  // Map Cache Lock
  mapCacheOps_ = new std::recursive_mutex();
  if (nullptr == mapCacheOps_) {
    return false;
  }

  mapCache_ = new std::vector<amd::Memory*>();
  if (mapCache_ == nullptr) {
    return false;
  }
  // Use just 1 entry by default for the map cache
  mapCache_->push_back(nullptr);

  // Create signal for HMM prefetch operation on device
  if (HSA_STATUS_SUCCESS != Hsa::signal_create(kInitSignalValueOne, 0, nullptr, &prefetch_signal_)) {
    return false;
  }

  if (AMD_LOG_LEVEL >= LOG_EXTRA_EXTRA_DEBUG) {
    uint8_t logMask[8] = {0};
    hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_AQL);
    hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_SDMA);
    hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_INFO);
    Hsa::enable_logging(logMask, outFile);
  }

  return true;
}

// ================================================================================================
device::Program* NullDevice::createProgram(amd::Program& owner, amd::option::Options* options) {
  device::Program* program = new roc::Program(*this, owner);

  if (program == nullptr) {
    LogError("Memory allocation has failed!");
  }

  return program;
}

bool Device::createBlitProgram() {
  bool result = true;
  std::string extraKernel;

  if (amd::IS_HIP) {
    if (settings().gwsInitSupported_) {
      extraKernel = device::HipExtraSourceCode;
    } else {
      extraKernel = device::HipExtraSourceCodeNoGWS;
    }
  } else {
    extraKernel = SchedulerSourceCode;
  }

  blitProgram_ = new BlitProgram(context_);
  // Create blit programs
  if (blitProgram_ == nullptr || !blitProgram_->create(this, extraKernel, "")) {
    delete blitProgram_;
    blitProgram_ = nullptr;
    LogError("Couldn't create blit kernels!");
    return false;
  }

  return result;
}

device::Program* Device::createProgram(amd::Program& owner, amd::option::Options* options) {
  device::Program* program = new roc::Program(*this, owner);

  if (program == nullptr) {
    LogError("Memory allocation has failed!");
  }

  return program;
}

hsa_status_t Device::iterateGpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, void* data) {
  if (data == nullptr) {
    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
  }

  hsa_region_segment_t segment_type = (hsa_region_segment_t)0;
  hsa_status_t stat =
      Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment_type);
  if (stat != HSA_STATUS_SUCCESS) {
    return stat;
  }

  // TODO: system and device local segment
  Device* dev = reinterpret_cast<Device*>(data);
  switch (segment_type) {
    case HSA_REGION_SEGMENT_GLOBAL: {
      if (dev->settings().enableLocalMemory_) {
        uint32_t global_flag = 0;
        hsa_status_t stat =
            Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &global_flag);
        if (stat != HSA_STATUS_SUCCESS) {
          return stat;
        }

        // If the flag set is ext scoped fine grain, break the loop
        if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) {
          dev->gpu_ext_fine_grained_segment_ = pool;
          break;
        }

        if ((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) != 0) {
          dev->gpu_fine_grained_segment_ = pool;
        } else if ((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) != 0) {
          dev->gpuvm_segment_ = pool;

          // If cpu agent cannot access this pool, the device does not support large bar.
          hsa_amd_memory_pool_access_t tmp{};
          Hsa::agent_memory_pool_get_info(dev->cpu_agent_info_->agent, pool,
                                          HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &tmp);

          if (tmp == HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED) {
            dev->info_.largeBar_ = false;
          } else {
            dev->info_.largeBar_ = ROC_ENABLE_LARGE_BAR;
          }

          // Query the recommended granularity for this pool.
          stat = Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
                                              &(dev->info_.virtualMemAllocGranularityMinimum_));
          if (stat != HSA_STATUS_SUCCESS) {
            LogPrintfError(
                "Cannot query HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE info"
                "failed with hsa_status: %d",
                stat);
          }
          // Query the recommended granularity for this pool.
          stat = Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_REC_GRANULE,
                                              &(dev->info_.virtualMemAllocGranularityRecommended_));
          if (stat != HSA_STATUS_SUCCESS) {
            LogPrintfError(
                "Cannot query HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_REC_GRANULE info"
                "failed with hsa_status: %d",
                stat);
          }
        }

        if (dev->gpuvm_segment_.handle == 0) {
          dev->gpuvm_segment_ = pool;
        }
      }
      break;
    }
    case HSA_REGION_SEGMENT_GROUP:
      dev->group_segment_ = pool;
      break;
    default:
      break;
  }

  return HSA_STATUS_SUCCESS;
}

hsa_status_t Device::iterateCpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, void* data) {
  if (data == nullptr) {
    LogError("CpuMemoryPoolCallback invalid args");
    return HSA_STATUS_ERROR_INVALID_ARGUMENT;
  }

  hsa_region_segment_t segment_type = (hsa_region_segment_t)0;
  hsa_status_t stat =
      Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment_type);
  if (stat != HSA_STATUS_SUCCESS) {
    LogPrintfError("HSA_AMD_MEMORY_POOL_INFO_SEGMENT query failed with %x", stat);
    return stat;
  }
  AgentInfo* agentInfo = reinterpret_cast<AgentInfo*>(data);

  switch (segment_type) {
    case HSA_REGION_SEGMENT_GLOBAL: {
      uint32_t global_flag = 0;
      stat =
          Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &global_flag);
      if (stat != HSA_STATUS_SUCCESS) {
        LogPrintfError("HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS query failed with %x", stat);
        break;
      }

      // If the flag set is ext scoped fine grain, break the loop
      if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) {
        agentInfo->ext_fine_grain_pool = pool;
        break;
      }

      if ((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) != 0) {
        if (agentInfo->fine_grain_pool.handle == 0) {
          agentInfo->fine_grain_pool = pool;
        } else if ((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) == 0) {
          // If the fine_grain_pool was already filled, but kern_args flag was not set over-write.
          // This means this is region-1(fine_grain only), so over-write this with memory pool set
          // from "fine_grain and kern_args".
          agentInfo->fine_grain_pool = pool;
        }
        guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) == 0),
                  "Memory Segment cannot be both coarse and fine grained");
      } else {
        // If the flag is not set to fine grained, then it is coarse_grained by default.
        agentInfo->coarse_grain_pool = pool;
        guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) != 0),
                  "Memory Segments that are not fine grained has to be coarse grained");
        guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) == 0),
                  "Memory Segment cannot be both coarse and fine grained");
        guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) == 0),
                  "Coarse grained memory segment cannot have kern_args tag");
      }

      if ((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) != 0) {
        agentInfo->kern_arg_pool = pool;
        guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) == 0),
                  "Coarse grained memory segment cannot have kern_args tag");
      }

      break;
    }
    default:
      break;
  }

  return stat;
}

bool Device::createSampler(const amd::Sampler& owner, device::Sampler** sampler) const {
  *sampler = nullptr;
  Sampler* gpuSampler = new Sampler(*this);
  if ((nullptr == gpuSampler) || !gpuSampler->create(owner)) {
    delete gpuSampler;
    return false;
  }
  *sampler = gpuSampler;
  return true;
}

void Sampler::fillSampleDescriptor(hsa_ext_sampler_descriptor_v2_t& samplerDescriptor,
                                   const amd::Sampler& sampler) const {
  samplerDescriptor.filter_mode = sampler.filterMode() == CL_FILTER_NEAREST
                                      ? HSA_EXT_SAMPLER_FILTER_MODE_NEAREST
                                      : HSA_EXT_SAMPLER_FILTER_MODE_LINEAR;
  switch (sampler.mipFilter()) {
    case CL_FILTER_NEAREST:
      samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NEAREST;
      break;
    case CL_FILTER_LINEAR:
      samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_LINEAR;
      break;
    default:
      samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NONE;
  }
  samplerDescriptor.coordinate_mode = sampler.normalizedCoords()
                                          ? HSA_EXT_SAMPLER_COORDINATE_MODE_NORMALIZED
                                          : HSA_EXT_SAMPLER_COORDINATE_MODE_UNNORMALIZED;
  for (int i = 0; i < 3; i++) {
    switch (sampler.addressingMode(i)) {
      case CL_ADDRESS_CLAMP_TO_EDGE:
        samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE;
        break;
      case CL_ADDRESS_REPEAT:
        samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_REPEAT;
        break;
      case CL_ADDRESS_CLAMP:
        samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_BORDER;
        break;
      case CL_ADDRESS_MIRRORED_REPEAT:
        samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_MIRRORED_REPEAT;
        break;
      case CL_ADDRESS_NONE:
        samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_UNDEFINED;
        break;
      default:
        return;
    }
  }
}

bool Sampler::create(const amd::Sampler& owner) {
  hsa_ext_sampler_descriptor_v2_t samplerDescriptor;
  fillSampleDescriptor(samplerDescriptor, owner);

  hsa_status_t status =
      Hsa::sampler_create(dev_.getBackendDevice(), &samplerDescriptor, &hsa_sampler);

  if (HSA_STATUS_SUCCESS != status) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_RESOURCE,
             "Sampler creation failed with status: %d", status);
    return false;
  }

  hwSrd_ = hsa_sampler.handle;
  hwState_ = reinterpret_cast<address>(hsa_sampler.handle);

  return true;
}

Sampler::~Sampler() { Hsa::sampler_destroy(dev_.getBackendDevice(), hsa_sampler); }

Memory* Device::getGpuMemory(amd::Memory* mem) const {
  return static_cast<roc::Memory*>(mem->getDeviceMemory(*this));
}

const bool Device::isFineGrainSupported() const {
  bool result = (info().svmCapabilities_ & CL_DEVICE_SVM_ATOMICS) != 0 ? true : false;
  if (result) {
    if (gpu_fine_grained_segment_.handle != 0) {
      return true;
    }
  }
  return false;
}
// ================================================================================================
bool Device::populateOCLDeviceConstants() {
  info_.available_ = true;

  ::strncpy(info_.name_, isa().targetId(), sizeof(info_.name_) - 1);
  char device_name[64] = {0};
  if (HSA_STATUS_SUCCESS == Hsa::agent_get_info(bkendDevice_,
                                                (hsa_agent_info_t)HSA_AMD_AGENT_INFO_PRODUCT_NAME,
                                                device_name)) {
    ::strncpy(info_.boardName_, device_name, sizeof(info_.boardName_) - 1);
  }

  char unique_id[32] = {0};
  if (HSA_STATUS_SUCCESS ==
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_UUID),
                         unique_id)) {
    // ROCr gives the UUID info in the format GPU-XXXX with length 20 bytes
    // Strip the first 4 bytes and store only the 16 bytes representing UUID
    std::memcpy(info_.uuid_, unique_id + 4, sizeof(info_.uuid_));
  }
  if (HSA_STATUS_SUCCESS ==
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_CUID),
                         unique_id)) {
    std::memcpy(info_.cuid_, unique_id, sizeof(info_.cuid_));
  }
  hsa_luid_t localUID = {0};
  if (HSA_STATUS_SUCCESS ==
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_LUID),
                          &localUID)) {
    info_.luidLowPart_ = localUID.low;
    info_.luidHighPart_ = localUID.high;
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          (amd::IS_HIP)
                              ? (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COOPERATIVE_COMPUTE_UNIT_COUNT
                              : (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT,
                          &info_.maxComputeUnits_)) {
    return false;
  }
  assert(info_.maxComputeUnits_ > 0);

  info_.maxComputeUnits_ =
      settings().enableWgpMode_ ? info_.maxComputeUnits_ / 2 : info_.maxComputeUnits_;

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT,
                          &info_.maxPhysicalComputeUnits_)) {
    return false;
  }
  assert(info_.maxPhysicalComputeUnits_ > 0);

  info_.maxPhysicalComputeUnits_ = settings().enableWgpMode_ ? info_.maxPhysicalComputeUnits_ / 2
                                                             : info_.maxPhysicalComputeUnits_;

  if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_,
                                                (hsa_agent_info_t)HSA_AMD_AGENT_INFO_CACHELINE_SIZE,
                                                &info_.globalMemCacheLineSize_)) {
    return false;
  }
  info_.globalMemCacheLineSize_ =
      (info_.globalMemCacheLineSize_ != 0) ? info_.globalMemCacheLineSize_ : 64;

  uint32_t cachesize[4] = {0};
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_CACHE_SIZE, cachesize)) {
    return false;
  }

  if (info_.globalMemCacheLineSize_ < 256 &&
      (isa().versionMajor() >= 13 ||
       (isa().versionMajor() == 12 && isa().versionMinor() >= 5))) {
    info_.globalMemCacheLineSize_ = 256;
  }

  {
    uint32_t maxPrefetchRegions = 0;
    if (HSA_STATUS_SUCCESS ==
        Hsa::agent_get_info(
            bkendDevice_,
            (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_DATA_PREFETCH_REGIONS,
            &maxPrefetchRegions)) {
      info_.maxDynDataPrefetchRegions_ = maxPrefetchRegions;
    } else {
      info_.maxDynDataPrefetchRegions_ = 0;
    }
  }

  assert(cachesize[0] > 0);
  info_.globalMemCacheSize_ = cachesize[0];

  info_.globalMemCacheType_ = CL_READ_WRITE_CACHE;

  info_.type_ = CL_DEVICE_TYPE_GPU;

  info_.extensions_ = getExtensionString();
  info_.nativeVectorWidthDouble_ = info_.preferredVectorWidthDouble_ =
      (settings().doublePrecision_) ? 1 : 0;

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY,
                          &info_.maxEngineClockFrequency_)) {
    return false;
  }

  if (!(isa().versionMajor() == 9 && isa().versionMinor() == 0 && isa().versionStepping() == 2)) {
    if (info_.maxEngineClockFrequency_ <= 0) {
      LogError("maxEngineClockFrequency_ is NOT positive!");
    }
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_MAX_FREQUENCY,
                          &info_.maxMemoryClockFrequency_)) {
    return false;
  }

  uint64_t wallClockFrequency = 0;  // in Hz
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_TIMESTAMP_FREQUENCY,
                          &wallClockFrequency)) {
    LogWarning("HSA_AMD_AGENT_INFO_TIMESTAMP_FREQUENCY cannot be queried. Ignored!");
  }
  info_.wallClockFrequency_ = static_cast<uint32_t>(wallClockFrequency / 1000);  // in KHz

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_DRIVER_NODE_ID),
                          &info_.driverNodeId_)) {
    return false;
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SDMA_ENG),
                          &info_.numSDMAengines_)) {
    return false;
  }

  uint64_t scratchLimitMax = 0;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_MAX,
                          &scratchLimitMax)) {
    LogWarning("HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_MAX cannot be queried!");
    return false;
  }
  info_.scratchLimitMin = 0;
  info_.scratchLimitMax = scratchLimitMax;

  checkAtomicSupport();

  assert(cpu_agent_info_->fine_grain_pool.handle != 0);
  if (HSA_STATUS_SUCCESS != Hsa::agent_iterate_memory_pools(
                                bkendDevice_, Device::iterateGpuMemoryPoolCallback, this)) {
    return false;
  }

  assert(group_segment_.handle != 0);

  for (auto agent : gpu_agents_) {
    if (agent.handle != bkendDevice_.handle) {
      hsa_status_t err;
      // Can another GPU (agent) have access to the current GPU memory pool (gpuvm_segment_)?
      hsa_amd_memory_pool_access_t access;
      err = Hsa::agent_memory_pool_get_info(agent, gpuvm_segment_,
                                            HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access);
      if (err != HSA_STATUS_SUCCESS) {
        continue;
      }

      // Find accessible p2p agents - i.e != HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED
      if (HSA_AMD_MEMORY_POOL_ACCESS_ALLOWED_BY_DEFAULT == access ||
          HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT == access) {
        // Agent can have access to the current gpuvm_segment_
        p2p_agents_.push_back(agent);
      }
    }
  }

  // Keep track of all P2P Agents in a Array including current device handle for IPC
  p2p_agents_list_ = new hsa_agent_t[1 + p2p_agents_.size()];
  p2p_agents_list_[0] = getBackendDevice();
  for (size_t agent_idx = 0; agent_idx < p2p_agents_.size(); ++agent_idx) {
    p2p_agents_list_[1 + agent_idx] = p2p_agents_[agent_idx];
  }

  size_t group_segment_size = 0;
  if (HSA_STATUS_SUCCESS != Hsa::memory_pool_get_info(group_segment_,
                                                      HSA_AMD_MEMORY_POOL_INFO_SIZE,
                                                      &group_segment_size)) {
    return false;
  }
  assert(group_segment_size > 0);

  // Find SDMA read mask
  if (HSA_STATUS_SUCCESS !=
      Hsa::memory_copy_engine_status(getCpuAgent(), getBackendDevice(), &maxSdmaReadMask_)) {
    return false;
  }
  assert(maxSdmaReadMask_ > 0 && "No SDMA engines available for Read");

  // Find SDMA write mask
  if (HSA_STATUS_SUCCESS !=
      Hsa::memory_copy_engine_status(getBackendDevice(), getCpuAgent(), &maxSdmaWriteMask_)) {
    return false;
  }
  assert(maxSdmaWriteMask_ > 0 && "No SDMA engines available for Write");

  info_.localMemSizePerCU_ = group_segment_size;
  info_.localMemSize_ = group_segment_size;

  info_.maxWorkItemDimensions_ = 3;

  uint8_t memory_properties[8];
  // Get the memory property from ROCr.
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_PROPERTIES,
                         memory_properties)) {
    LogError("HSA_AGENT_INFO_AMD_MEMORY_PROPERTIES query failed");
  }

  // Check if the device is APU
  if (hsa_flag_isset64(memory_properties, HSA_AMD_MEMORY_PROPERTY_AGENT_IS_APU)) {
    info_.hostUnifiedMemory_ = 1;
  }

  if (settings().enableLocalMemory_ && gpuvm_segment_.handle != 0) {
    size_t global_segment_size = 0;
    if (HSA_STATUS_SUCCESS != Hsa::memory_pool_get_info(gpuvm_segment_,
                                                        HSA_AMD_MEMORY_POOL_INFO_SIZE,
                                                        &global_segment_size)) {
      return false;
    }

    assert(global_segment_size > 0);
    info_.globalMemSize_ = (static_cast<uint64_t>(std::min(GPU_MAX_HEAP_SIZE, 100u)) *
                            static_cast<uint64_t>(global_segment_size)) /
                           100u;

    // For APU with vram size <= 512MiB, use a smaller single alloc percentage
    if (info_.globalMemSize_ <= 536870912) {
      if (flagIsDefault(GPU_SINGLE_ALLOC_PERCENT)) {
        GPU_SINGLE_ALLOC_PERCENT = 75;
      }
    }

    gpuvm_segment_max_alloc_ =
        uint64_t(info_.globalMemSize_ * std::min(GPU_SINGLE_ALLOC_PERCENT, 100u) / 100u);
    assert(gpuvm_segment_max_alloc_ > 0);

    info_.maxMemAllocSize_ = static_cast<uint64_t>(gpuvm_segment_max_alloc_);

    if (HSA_STATUS_SUCCESS !=
        Hsa::memory_pool_get_info(gpuvm_segment_, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
                                  &alloc_granularity_)) {
      return false;
    }

    assert(alloc_granularity_ > 0);
  } else {
    // We suppose half of physical memory can be used by GPU in APU system
    info_.globalMemSize_ = amd::Os::hostTotalPhysicalMemory() / 2;
    info_.globalMemSize_ = std::max(info_.globalMemSize_, uint64_t(1 * Gi));
    info_.globalMemSize_ = (static_cast<uint64_t>(std::min(GPU_MAX_HEAP_SIZE, 100u)) *
                            static_cast<uint64_t>(info_.globalMemSize_)) /
                           100u;

    info_.maxMemAllocSize_ =
        uint64_t(info_.globalMemSize_ * std::min(GPU_SINGLE_ALLOC_PERCENT, 100u) / 100u);

    if (HSA_STATUS_SUCCESS !=
        Hsa::memory_pool_get_info(cpu_agent_info_->fine_grain_pool,
                                  HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
                                  &alloc_granularity_)) {
      return false;
    }
  }

  freeMem_ = info_.globalMemSize_;

  // Make sure the max allocation size is not larger than the available memory size.
  info_.maxMemAllocSize_ = std::min(info_.maxMemAllocSize_, info_.globalMemSize_);
  info_.maxMemAllocSize_ = amd::alignDown(info_.maxMemAllocSize_, sizeof(uint64_t));

  // Maximum system memory allocation size allowed
  info_.maxPhysicalMemAllocSize_ = amd::Os::getPhysicalMemSize();

  // make sure we don't run anything over 8 params for now
  info_.maxParameterSize_ = 1024;

  uint32_t max_work_group_size = 0;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WORKGROUP_MAX_SIZE, &max_work_group_size)) {
    return false;
  }
  assert(max_work_group_size > 0);
  max_work_group_size =
      std::min(max_work_group_size, static_cast<uint32_t>(settings().maxWorkGroupSize_));
  info_.maxWorkGroupSize_ = max_work_group_size;

  uint16_t max_workgroup_size[3] = {0, 0, 0};
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WORKGROUP_MAX_DIM, &max_workgroup_size)) {
    return false;
  }
  assert(max_workgroup_size[0] != 0 && max_workgroup_size[1] != 0 && max_workgroup_size[2] != 0);

  uint16_t max_work_item_size = static_cast<uint16_t>(max_work_group_size);
  info_.maxWorkItemSizes_[0] = std::min(max_workgroup_size[0], max_work_item_size);
  info_.maxWorkItemSizes_[1] = std::min(max_workgroup_size[1], max_work_item_size);
  info_.maxWorkItemSizes_[2] = std::min(max_workgroup_size[2], max_work_item_size);
  info_.preferredWorkGroupSize_ = settings().preferredWorkGroupSize_;

  hsa_dim3_t grid_max_dim = {0, 0, 0};
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_GRID_MAX_DIM, &grid_max_dim)) {
    return false;
  }
  assert(grid_max_dim.x != 0 && grid_max_dim.y != 0 && grid_max_dim.z != 0);
  info_.maxGridDim_[0] = std::min(grid_max_dim.x, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
  info_.maxGridDim_[1] = std::min(grid_max_dim.y, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
  info_.maxGridDim_[2] = std::min(grid_max_dim.z, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));

  info_.nativeVectorWidthChar_ = info_.preferredVectorWidthChar_ = 4;
  info_.nativeVectorWidthShort_ = info_.preferredVectorWidthShort_ = 2;
  info_.nativeVectorWidthInt_ = info_.preferredVectorWidthInt_ = 1;
  info_.nativeVectorWidthLong_ = info_.preferredVectorWidthLong_ = 1;
  info_.nativeVectorWidthFloat_ = info_.preferredVectorWidthFloat_ = 1;

  if (agent_profile_ == HSA_PROFILE_FULL) {  // full-profile = participating in coherent memory,
                                             // base-profile = NUMA based non-coherent memory
    info_.hostUnifiedMemory_ = 1;
    info_.iommuv2_ = true;
  }
  info_.memBaseAddrAlign_ = 8 * (flagIsDefault(MEMOBJ_BASE_ADDR_ALIGN) ? sizeof(int64_t[16]) * 2
                                                                       : MEMOBJ_BASE_ADDR_ALIGN);
  info_.minDataTypeAlignSize_ = sizeof(int64_t[16]);

  info_.maxConstantArgs_ = 8;
  info_.preferredConstantBufferSize_ = 16 * Ki;
  info_.maxConstantBufferSize_ = info_.maxMemAllocSize_;
  info_.localMemType_ = CL_LOCAL;
  info_.errorCorrectionSupport_ = false;
  info_.profilingTimerResolution_ = 1;
  info_.littleEndian_ = true;
  info_.compilerAvailable_ = true;
  info_.executionCapabilities_ = CL_EXEC_KERNEL;
  info_.queueProperties_ = CL_QUEUE_PROFILING_ENABLE;
  info_.platform_ = AMD_PLATFORM;
  info_.profile_ = "FULL_PROFILE";
  ::strncpy(info_.vendor_, "Advanced Micro Devices, Inc.", sizeof(info_.vendor_) - 1);

  info_.addressBits_ = LP64_SWITCH(32, 64);
  info_.maxSamplers_ = 16;
  info_.bufferFromImageSupport_ = false;
  info_.oclcVersion_ = "OpenCL C " OPENCL_C_VERSION_STR " ";
  info_.spirVersions_ = "";

  uint16_t major, minor;
  if (Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_VERSION_MAJOR, &major) !=
          HSA_STATUS_SUCCESS ||
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_VERSION_MINOR, &minor) !=
          HSA_STATUS_SUCCESS) {
    return false;
  }
  std::stringstream ss;
  ss << AMD_BUILD_STRING " (HSA" << major << "." << minor << ",LC)";

  ::strncpy(info_.driverVersion_, ss.str().c_str(), sizeof(info_.driverVersion_) - 1);

  if (isa().versionMajor() >= 9) {
    info_.version_ =
        "OpenCL " /*OPENCL_VERSION_STR*/
        "2.0"
        " ";
  } else {
    info_.version_ =
        "OpenCL " /*OPENCL_VERSION_STR*/
        "1.2"
        " ";
  }

  info_.builtInKernels_ = "";
  info_.linkerAvailable_ = true;
  info_.preferredInteropUserSync_ = true;
  info_.printfBufferSize_ = PrintfDbg::WorkitemDebugSize * info().maxWorkGroupSize_;
  info_.vendorId_ = 0x1002;  // AMD's PCIe vendor id

  info_.maxGlobalVariableSize_ = static_cast<size_t>(info_.maxMemAllocSize_);
  info_.globalVariablePreferredTotalSize_ = static_cast<size_t>(info_.globalMemSize_);

  // Populate the single config setting.
  info_.singleFPConfig_ =
      CL_FP_ROUND_TO_NEAREST | CL_FP_ROUND_TO_ZERO | CL_FP_ROUND_TO_INF | CL_FP_INF_NAN | CL_FP_FMA;

  if (settings().doublePrecision_) {
    info_.doubleFPConfig_ = info_.singleFPConfig_ | CL_FP_DENORM;
    info_.singleFPConfig_ |= CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT;
  }

  if (settings().singleFpDenorm_) {
    info_.singleFPConfig_ |= CL_FP_DENORM;
  }

  if (settings().checkExtension(ClKhrFp16)) {
    info_.halfFPConfig_ = info_.singleFPConfig_;
  }

  info_.preferredPlatformAtomicAlignment_ = 0;
  info_.preferredGlobalAtomicAlignment_ = 0;
  info_.preferredLocalAtomicAlignment_ = 0;

  uint8_t hsa_extensions[128];
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_EXTENSIONS, hsa_extensions)) {
    return false;
  }

  assert(HSA_EXTENSION_IMAGES < 8);
  const bool image_is_supported = ((hsa_extensions[0] & (1 << HSA_EXTENSION_IMAGES)) != 0);
  if (image_is_supported) {
    // Images
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_SAMPLER_HANDLERS),
                            &info_.maxSamplers_)) {
      return false;
    }

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_IMAGE_RD_HANDLES),
                            &info_.maxReadImageArgs_)) {
      return false;
    }

    // TODO: no attribute for write image.
    info_.maxWriteImageArgs_ = 8;

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_IMAGE_RORW_HANDLES),
                            &info_.maxReadWriteImageArgs_)) {
      return false;
    }

    uint32_t image_max_dim[3];
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_2D_MAX_ELEMENTS),
                            &image_max_dim)) {
      return false;
    }

    info_.image2DMaxWidth_ = image_max_dim[0];
    info_.image2DMaxHeight_ = image_max_dim[1];

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_3D_MAX_ELEMENTS),
                            &image_max_dim)) {
      return false;
    }

    info_.image3DMaxWidth_ = image_max_dim[0];
    info_.image3DMaxHeight_ = image_max_dim[1];
    info_.image3DMaxDepth_ = image_max_dim[2];

    uint32_t max_array_size = 0;
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_ARRAY_MAX_LAYERS),
                            &max_array_size)) {
      return false;
    }

    info_.imageMaxArraySize_ = max_array_size;

    uint32_t max_image1da_width = 0;
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1DA_MAX_ELEMENTS),
                            &max_image1da_width)) {
      return false;
    }

    info_.image1DAMaxWidth_ = max_image1da_width;

    uint32_t max_image2da_width[2] = {0, 0};
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_2DA_MAX_ELEMENTS),
                            &max_image2da_width)) {
      return false;
    }

    info_.image2DAMaxWidth_[0] = max_image2da_width[0];
    info_.image2DAMaxWidth_[1] = max_image2da_width[1];

    uint32_t max_image1d_width = 0;
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1D_MAX_ELEMENTS),
                            &max_image1d_width)) {
      return false;
    }
    info_.image1DMaxWidth_ = max_image1d_width;

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1DB_MAX_ELEMENTS),
                            &image_max_dim)) {
      return false;
    }
    info_.imageMaxBufferSize_ = (amd::IS_HIP) ? image_max_dim[0] : (1 << 27);

    info_.imageSupport_ = (info_.maxReadWriteImageArgs_ > 0) ? true : false;
  }

  // These are properties of the device's linear memory layout, not the image
  // extension.  They must be set unconditionally so that hipMallocPitch /
  // hipMalloc3D produce correct pitch alignment even when IMAGE_SUPPORT is
  // off.  The PAL backend already sets them unconditionally.
  info_.imagePitchAlignment_ = 256;
  info_.imageBaseAddressAlignment_ = 256;
  info_.bufferFromImageSupport_ = false;

  // Enable SVM Capabilities of Hsa device. Ensure
  // user has not setup memory to be non-coherent
  info_.svmCapabilities_ = 0;
  if (!settings().enableNCMode_) {
    info_.svmCapabilities_ = CL_DEVICE_SVM_COARSE_GRAIN_BUFFER;
    info_.svmCapabilities_ |= CL_DEVICE_SVM_FINE_GRAIN_BUFFER;
    // Report fine-grain system only on full profile
    if (agent_profile_ == HSA_PROFILE_FULL) {
      info_.svmCapabilities_ |= CL_DEVICE_SVM_FINE_GRAIN_SYSTEM;
    }
    if (amd::IS_HIP) {
      if (info_.iommuv2_ || isa().versionMajor() >= 8) {
        info_.svmCapabilities_ |= CL_DEVICE_SVM_ATOMICS;
      }
    }
  }

  if (settings().checkExtension(ClAmdDeviceAttributeQuery)) {
    info_.simdWidth_ = isa().simdWidth();
    info_.simdInstructionWidth_ = isa().simdInstructionWidth();
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WAVEFRONT_SIZE, &info_.wavefrontWidth_)) {
      return false;
    }

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_WIDTH),
                            &info_.vramBusBitWidth_)) {
      return false;
    }

    info_.globalMemChannels_ = info_.vramBusBitWidth_ / 32;

    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU),
                            &info_.simdPerCU_)) {
      return false;
    }

    uint32_t max_waves_per_cu = 0;
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU),
                            &max_waves_per_cu)) {
      return false;
    }

    if (settings().enableWgpMode_) {
      info_.simdPerCU_ *= 2;
      max_waves_per_cu *= 2;
    }

    if (HSA_STATUS_SUCCESS !=
        hsa_agent_get_info(bkendDevice_,
                           static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SHADER_ENGINES),
                           &info_.numberOfShaderEngines_)) {
      return false;
    }

    info_.maxThreadsPerCU_ = info_.wavefrontWidth_ * max_waves_per_cu;
    uint32_t cache_sizes[4];
    /* FIXIT [skudchad] -  Seems like hardcoded in HSA backend so 0*/
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AGENT_INFO_CACHE_SIZE),
                            cache_sizes)) {
      return false;
    }

    uint32_t asic_revision = 0;
    if (HSA_STATUS_SUCCESS !=
        Hsa::agent_get_info(bkendDevice_,
                            static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_ASIC_REVISION),
                            &asic_revision)) {
      return false;
    }
    info_.asicRevision_ = asic_revision;

    info_.l2CacheSize_ = cache_sizes[1];
    info_.timeStampFrequency_ = 1000000;
    info_.globalMemChannelBanks_ = 4;
    info_.globalMemChannelBankWidth_ = isa().memChannelBankWidth();
    info_.localMemSizePerCU_ = isa().localMemSizePerCU();
    info_.localMemBanks_ = isa().localMemBanks();
    info_.numAsyncQueues_ = kMaxAsyncQueues;
    info_.numRTQueues_ = info_.numAsyncQueues_;
    info_.numRTCUs_ = info_.maxComputeUnits_;

    // TODO: set to true once thread trace support is available
    info_.threadTraceEnable_ = false;
    info_.pcieDeviceId_ = pciDeviceId_;
    info_.cooperativeGroups_ = settings().enableCoopGroups_;
    info_.cooperativeMultiDeviceGroups_ = settings().enableCoopMultiDeviceGroups_;
    // Enable StreamWrite and StreamWait for all devices
    info_.aqlBarrierValue_ = true;
    info_.movdir64b_ = amd::hasMovdir64b() && (DEBUG_CLR_USE_MOVDIR64B != 0);
  }

  info_.maxPipePacketSize_ = info_.maxMemAllocSize_;
  info_.maxPipeActiveReservations_ = 16;
  info_.maxPipeArgs_ = 16;

  info_.queueOnDeviceProperties_ =
      CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE | CL_QUEUE_PROFILING_ENABLE;
  info_.queueOnDevicePreferredSize_ = 256 * Ki;
  info_.queueOnDeviceMaxSize_ = 8 * Mi;
  info_.maxOnDeviceQueues_ = 1;
  info_.maxOnDeviceEvents_ = settings().numDeviceEvents_;

  std::string addressableNumVGPRs, totalNumVGPRs, vGPRAllocGranule;
  std::string isaName = isa().isaName();
  info_.availableVGPRs_ =
      amd::device::getValueFromIsaMeta(isaName, "AddressableNumVGPRs", addressableNumVGPRs)
      ? atoi(addressableNumVGPRs.c_str())
      : 0;
  info_.vgprsPerSimd_ = amd::device::getValueFromIsaMeta(isaName, "TotalNumVGPRs", totalNumVGPRs)
      ? atoi(totalNumVGPRs.c_str())
      : 0;
  info_.vgprAllocGranularity_ =
      amd::device::getValueFromIsaMeta(isaName, "VGPRAllocGranule", vGPRAllocGranule)
      ? atoi(vGPRAllocGranule.c_str())
      : 0;

  info_.availableRegistersPerCU_ = info_.vgprsPerSimd_ * info_.simdPerCU_ * info_.wavefrontWidth_;
  ClPrint(amd::LOG_INFO, amd::LOG_INIT,
          "addressableNumVGPRs=%u, totalNumVGPRs=%u, vGPRAllocGranule=%u,"
          " availableRegistersPerCU_=%u",
          info_.availableVGPRs_, info_.vgprsPerSimd_, info_.vgprAllocGranularity_,
          info_.availableRegistersPerCU_);

  std::string sgprValue;
  info_.availableSGPRs_ =
      (amd::device::getValueFromIsaMeta(isaName, "AddressableNumSGPRs", sgprValue))
      ? (atoi(sgprValue.c_str()))
      : 0;
  std::string imageSupport;
  if (amd::device::getValueFromIsaMeta(isaName, "ImageSupport", imageSupport)) {
    info_.imageSupport_ =
        (DEBUG_CLR_DISABLE_IMAGE || !image_is_supported) ? 0 : std::atoi(imageSupport.c_str());
    ClPrint(amd::LOG_INFO, amd::LOG_INIT, "imageSupport=%u", info_.imageSupport_);
  } else {
    LogInfo("Can not get image support info from ISA meta");
  }

  // Generic support for HMM interfaces
  if (HSA_STATUS_SUCCESS !=
      Hsa::system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &info_.hmmSupported_)) {
    LogError("HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED query failed. HMM will be disabled");
  }

  // This capability should be available with xnack enabled
  if (HSA_STATUS_SUCCESS != Hsa::system_get_info(HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT,
                                                 &info_.hmmCpuMemoryAccessible_)) {
    LogError("HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT query failed.");
  }

  // HMM specific capability for CPU direct access to device memory
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_SVM_DIRECT_HOST_ACCESS),
                          &info_.hmmDirectHostAccess_)) {
    LogError("HSA_AMD_AGENT_INFO_SVM_DIRECT_HOST_ACCESS query failed.");
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_XCC),
                          &info_.numberOfXccs_)) {
    LogError("HSA_AMD_AGENT_INFO_NUM_XCC query failed.");
  }

  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_PM4_EMULATION),
                          &pm4_emulation_)) {
    LogError("HSA_AMD_AGENT_INFO_PM4_EMULATION query failed.");
  }

  info_.hasExpertSchedMode_ = false;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE),
                          &info_.hasExpertSchedMode_)) {
    LogWarning("HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE query failed.");
  }

  ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Gfx Major/Minor/Stepping: %d/%d/%d, Device ID: 0x%x",
          isa().versionMajor(), isa().versionMinor(), isa().versionStepping(), pciDeviceId_);
  ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Using dev kernel arg wa = %d", settings().kernel_arg_impl_);
  ClPrint(amd::LOG_INFO, amd::LOG_INIT, "HMM support: %d, XNACK: %d, Direct host access: %d",
          info_.hmmSupported_, info_.hmmCpuMemoryAccessible_, info_.hmmDirectHostAccess_);
  ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Max SDMA Read Mask: 0x%x, Max SDMA Write Mask: 0x%x",
          maxSdmaReadMask_, maxSdmaWriteMask_);

  info_.globalCUMask_ = {};

  // Virtual memory Management Support, if set to true then the HW and SW Stack supports VMM.
  info_.virtualMemoryManagement_ = false;
  if (HIP_VMEM_MANAGE_SUPPORT) {
    if (HSA_STATUS_SUCCESS !=
        Hsa::system_get_info(
            static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED),
            &info_.virtualMemoryManagement_)) {
      LogError("HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED query failed ");
    }
  }
  HIP_MEM_POOL_USE_VM &= info_.virtualMemoryManagement_;

  // Check Support for Buffer sharing with dma_buf
  std::ignore = Hsa::system_get_info(
                    static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_DMABUF_SUPPORTED),
                    &info_.dmabufSupported_);
  // devices with no cluster support; max size is 0
  info_.clusterMaxSize_ = 0;

  hsa_status_t hsaStatus = Hsa::agent_get_info(
      bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_CLUSTER_MAX_SIZE),
      &info_.clusterMaxSize_);


  // this is required for clustered kernel launches; but it might not be supported in older rocr,
  // so invalid argument might no be necessarily an error
  if (HSA_STATUS_SUCCESS != hsaStatus && HSA_STATUS_ERROR_INVALID_ARGUMENT != hsaStatus) {
    LogError("HSA_AMD_AGENT_INFO_CLUSTER_MAX_SIZE query failed");
  } else {
    // this is a temporary override of the ROCr result (when there is cluster support). This line
    // will be removed as soon as ROCr provides the correct result
    info_.clusterMaxSize_ = info_.clusterMaxSize_ > 1?
                            info_.maxComputeUnits_ / info_.numberOfShaderEngines_ :
                            1;
  }

  info_.gpuDirectRdmaWithHipVmmSupported_ =
      info_.virtualMemoryManagement_ && info_.dmabufSupported_;

  if (isa().versionMajor() < 8) {
    info_.sgprsPerSimd_ = 512;
  } else if (isa().versionMajor() < 10) {
    info_.sgprsPerSimd_ = 800;
  } else {
    info_.sgprsPerSimd_ =
        std::numeric_limits<uint32_t>::max();  // gfx10+ does not share SGPRs between waves
  }

  // Flag for fabric handle support
  info_.fabric_handle_ = false;
  if (HSA_STATUS_SUCCESS !=
      Hsa::system_get_info(
          static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_FABRIC_HANDLES_SUPPORTED),
          &info_.fabric_handle_)) {
    LogError("HSA_AMD_SYSTEM_INFO_FABRIC_HANDLES_SUPPORTED query failed ");
  }

  return true;
}

// ================================================================================================
device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
  std::scoped_lock lock(vgpusAccess());

  bool profiling = (queue != nullptr) && queue->properties().test(CL_QUEUE_PROFILING_ENABLE);
  bool cooperative = false;
  bool dedicated_queue = (queue != nullptr) && queue->isDedicatedQueue() &&
                         (settings().dynamic_queues_ >= 2);

  // If amd command queue is null, then it's an internal device queue
  if (queue == nullptr) {
    // In HIP mode the device queue will be allocated for the cooperative launches only
    cooperative = amd::IS_HIP && settings().enableCoopGroups_;
    profiling = amd::IS_HIP;
  }
  // Initialization of heap and other resources occur during the command
  // queue creation time.
  const std::vector<uint32_t> defaultCuMask = {};
  bool q = (queue != nullptr);
  VirtualGPU* virtualDevice =
      new VirtualGPU(*this, profiling, cooperative, q ? queue->cuMask() : defaultCuMask,
                     q ? queue->priority() : amd::CommandQueue::Priority::Normal,
                     dedicated_queue);

  if (!virtualDevice->create()) {
    delete virtualDevice;
    return nullptr;
  }

  return virtualDevice;
}

bool Device::globalFreeMemory(size_t* freeMemory) const {
  const uint TotalFreeMemory = 0;
  const uint LargestFreeBlock = 1;
  uint64_t globalAvailMemory;
  // Queries memory available in bytes across all global pools owned by the agent
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_,
                          static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_AVAIL),
                          &globalAvailMemory)) {
    LogError("HSA_AMD_AGENT_INFO_MEMORY_AVAIL query failed.");
    return false;
  }

  globalAvailMemory = globalAvailMemory / Ki;
  if (globalAvailMemory > HIP_HIDDEN_FREE_MEM * Ki) {
    globalAvailMemory -= HIP_HIDDEN_FREE_MEM * Ki;
  } else {
    globalAvailMemory = 0;
  }

  freeMemory[TotalFreeMemory] = globalAvailMemory;
  // since there is no memory heap on ROCm, the biggest free block is
  // equal to total free local memory
  freeMemory[LargestFreeBlock] = freeMemory[TotalFreeMemory];

  return true;
}

bool Device::amdFileRead(amd::Os::FileDesc handle, void* devicePtr, uint64_t size, int64_t file_offset,
                      uint64_t* size_copied, int32_t* status) {
  hsa_amd_ais_file_handle_t fh{};
  hsa_status_t ret = HSA_STATUS_ERROR;
#if defined(_WIN32)
  fh.handle = handle;
#else
  fh.fd = handle;
  ret = Hsa::ais_file_read(fh, devicePtr, size, file_offset, size_copied, status);
#endif

  if (HSA_STATUS_SUCCESS != ret) {
    LogPrintfError("amdFileRead operation failed with err 0x%xh", ret);
    return false;
  }
  return true;
}

bool Device::amdFileWrite(amd::Os::FileDesc handle, void* devicePtr, uint64_t size, int64_t file_offset,
                       uint64_t* size_copied, int32_t* status) {
  hsa_amd_ais_file_handle_t fh{};
  hsa_status_t ret = HSA_STATUS_ERROR;
#if defined(_WIN32)
  fh.handle = handle;
#else
  fh.fd = handle;
  ret = Hsa::ais_file_write(fh, devicePtr, size, file_offset, size_copied, status);
#endif

  if (HSA_STATUS_SUCCESS != ret) {
    LogPrintfError("amdFileWrite operation failed with err 0x%xh", ret);
    return false;
  }
  return true;
}

// ================================================================================================
bool Device::bindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext,
                                bool validateOnly) {
  bool success = true;

#ifdef _WIN32
  // Handle D3D10 device binding
  if (flags & amd::Context::Flags::D3D10DeviceKhr) {
    void* d3d10Device = gfxDevice[amd::Context::DeviceFlagIdx::D3D10DeviceKhrIdx];
    if (!D3D10Interop::associateD3D10Device(this, static_cast<ID3D10Device*>(d3d10Device),
                                            gfxContext, validateOnly)) {
      if (!validateOnly) {
        LogError("Failed D3D10Interop::associateD3D10Device()");
      }
      success = false;
    }
  }

  // Handle D3D11 device binding
  if (flags & amd::Context::Flags::D3D11DeviceKhr) {
    void* d3d11Device = gfxDevice[amd::Context::DeviceFlagIdx::D3D11DeviceKhrIdx];
    if (!D3D11Interop::associateD3D11Device(this, static_cast<ID3D11Device*>(d3d11Device),
                                            gfxContext, validateOnly)) {
      if (!validateOnly) {
        LogError("Failed D3D11Interop::associateD3D11Device()");
      }
      success = false;
    }
  }
#endif  // _WIN32

  // Handle GL device binding (existing code)
  if (flags & amd::Context::Flags::GLDeviceKhr) {
    void* glDevice = gfxDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx];
    if (!GlInterop::glAssociate(this, flags, gfxContext, glDevice, validateOnly)) {
      if (!validateOnly) {
        LogError("Failed GlInterop::glAssociate()");
      }
      success = false;
    }
  }

  return success;
}

// ================================================================================================
bool Device::unbindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext,
                                  bool validateOnly) {
  bool success = true;

#ifdef _WIN32
  // Handle D3D10 device cleanup
  if (flags & amd::Context::Flags::D3D10DeviceKhr) {
    D3D10Interop::dissociateD3D10Device(this, gfxDevice, gfxContext);
  }

  // Handle D3D11 device cleanup
  if (flags & amd::Context::Flags::D3D11DeviceKhr) {
    D3D11Interop::dissociateD3D11Device(this, gfxDevice, gfxContext);
  }
#endif  // _WIN32

  // Handle GL device cleanup (existing code)
  if (flags & amd::Context::Flags::GLDeviceKhr) {
    void* glDevice = gfxDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx];
    // validateOnly never started a session; nothing to dissociate.
    if (glDevice != nullptr && !validateOnly) {
      if (!GlInterop::glDissociate(this, gfxContext, glDevice)) {
        LogWarning("Failed GlInterop::glDissociate()");
        success = false;
      }
    }
  }

  return success;
}

amd::Memory* Device::findMapTarget(size_t size) const {
  // Must be serialised for access
  std::scoped_lock lk(*mapCacheOps_);

  amd::Memory* map = nullptr;
  size_t minSize = 0;
  size_t maxSize = 0;
  uint mapId = mapCache_->size();
  uint releaseId = mapCache_->size();

  // Find if the list has a map target of appropriate size
  for (uint i = 0; i < mapCache_->size(); i++) {
    if ((*mapCache_)[i] != nullptr) {
      // Requested size is smaller than the entry size
      if (size < (*mapCache_)[i]->getSize()) {
        if ((minSize == 0) || (minSize > (*mapCache_)[i]->getSize())) {
          minSize = (*mapCache_)[i]->getSize();
          mapId = i;
        }
      }
      // Requeted size matches the entry size
      else if (size == (*mapCache_)[i]->getSize()) {
        mapId = i;
        break;
      } else {
        // Find the biggest map target in the list
        if (maxSize < (*mapCache_)[i]->getSize()) {
          maxSize = (*mapCache_)[i]->getSize();
          releaseId = i;
        }
      }
    }
  }

  // Check if we found any map target
  if (mapId < mapCache_->size()) {
    map = (*mapCache_)[mapId];
    (*mapCache_)[mapId] = nullptr;
  }
  // If cache is full, then release the biggest map target
  else if (releaseId < mapCache_->size()) {
    (*mapCache_)[releaseId]->release();
    (*mapCache_)[releaseId] = nullptr;
  }

  return map;
}

bool Device::addMapTarget(amd::Memory* memory) const {
  // Must be serialised for access
  std::scoped_lock lk(*mapCacheOps_);

  // the svm memory shouldn't be cached
  if (!memory->canBeCached()) {
    return false;
  }
  // Find if the list has a map target of appropriate size
  for (uint i = 0; i < mapCache_->size(); ++i) {
    if ((*mapCache_)[i] == nullptr) {
      (*mapCache_)[i] = memory;
      return true;
    }
  }

  // Add a new entry
  mapCache_->push_back(memory);

  return true;
}

Memory* Device::getRocMemory(amd::Memory* mem) const {
  return static_cast<roc::Memory*>(mem->getDeviceMemory(*this));
}

// ================================================================================================
device::Memory* Device::createMemory(amd::Memory& owner) const {
  roc::Memory* memory = nullptr;
  if (owner.asBuffer()) {
    memory = new roc::Buffer(*this, owner);
  } else if (owner.asImage()) {
    memory = new roc::Image(*this, owner);
  } else {
    LogError("Unknown memory type");
  }

  if (memory == nullptr) {
    return nullptr;
  }

  bool result = memory->create();

  if (!result) {
    LogError("Failed creating memory");
    delete memory;
    return nullptr;
  }

  if (isP2pEnabled()) {
    memory->setAllowedPeerAccess(true);
  }
  // Initialize if the memory is a pipe object
  if (owner.getType() == CL_MEM_OBJECT_PIPE) {
    // Pipe initialize in order read_idx, write_idx, end_idx. Refer clk_pipe_t structure.
    // Init with 3 DWORDS for 32bit addressing and 6 DWORDS for 64bit
    size_t pipeInit[3] = {0, 0, owner.asPipe()->getMaxNumPackets()};
    xferMgr().writeBuffer(pipeInit, *memory, amd::Coord3D(0), amd::Coord3D(sizeof(pipeInit)));
  }

  // Transfer data only if OCL context has one device.
  // Cache coherency layer will update data for multiple devices
  if (!memory->isHostMemDirectAccess() && owner.asImage() && (owner.parent() == nullptr) &&
      (owner.getMemFlags() & CL_MEM_COPY_HOST_PTR) && (owner.getContext().devices().size() == 1)) {
    // To avoid recurssive call to Device::createMemory, we perform
    // data transfer to the view of the image
    amd::Image* imageView = owner.asImage()->createView(
        owner.getContext(), owner.asImage()->getImageFormat(), xferQueue());

    if (imageView == nullptr) {
      LogError("[OCL] Fail to allocate view of image object");
      return nullptr;
    }

    Image* devImageView = new roc::Image(static_cast<const Device&>(*this), *imageView);
    if (devImageView == nullptr) {
      LogError("[OCL] Fail to allocate device mem object for the view");
      imageView->release();
      return nullptr;
    }

    if (devImageView != nullptr && !devImageView->createView(static_cast<roc::Image&>(*memory))) {
      LogError("[OCL] Fail to create device mem object for the view");
      delete devImageView;
      imageView->release();
      return nullptr;
    }

    imageView->replaceDeviceMemory(this, devImageView);

    // Copy data with the original pitch values, since runtime doesn't perform
    // extra sysmem allocation for one device
    const auto image = owner.asImage();
    result = xferMgr().writeImage(owner.getHostMem(), *devImageView, amd::Coord3D(0, 0, 0),
                                  imageView->getRegion(), image->getRowPitch(),
                                  image->getSlicePitch(), true);

    // Release host memory, since runtime copied data
    owner.setHostMem(nullptr);

    imageView->release();
  }

  // Prepin sysmem buffer for possible data synchronization between CPU and GPU
  if (!memory->isHostMemDirectAccess() &&
      // Pin memory for the parent object only
      (owner.parent() == nullptr) && (owner.getHostMem() != nullptr) &&
      (owner.getSvmPtr() == nullptr)) {
    memory->pinSystemMemory(owner.getHostMem(), owner.getSize());
  }

  if (!result) {
    delete memory;
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_RESOURCE, "Cannot Write Image");
    return nullptr;
  }

  return memory;
}

// ================================================================================================
device::Memory* Device::createMemory(size_t size, size_t alignment) const {
  auto buffer = new roc::Buffer(*this, size);
  static constexpr bool LocalAlloc = true;
  if ((buffer == nullptr) || !buffer->create(LocalAlloc)) {
    LogError("Couldn't allocate memory on device!");
    return nullptr;
  }
  return buffer;
}

// ================================================================================================
hsa_amd_memory_pool_t Device::getHostMemoryPool(MemorySegment mem_seg,
                                                const AgentInfo* agentInfo) const {
  if (agentInfo == nullptr) {
    agentInfo = cpu_agent_info_;
  }
  hsa_amd_memory_pool_t segment{0};
  switch (mem_seg) {
    case kKernArg: {
      if (settings().fgs_kernel_arg_) {
        segment = agentInfo->kern_arg_pool;
        break;
      }
      // Falls through on else case.
    }
    case kNoAtomics:
      // If runtime disables barrier, then all host allocations must have L2 disabled
      if (agentInfo->coarse_grain_pool.handle != 0) {
        segment = agentInfo->coarse_grain_pool;
        break;
      }
      // Falls through on else case.
    case kAtomics:
      segment = agentInfo->fine_grain_pool;
      break;
    case kUncachedAtomics:
    case kIoMemory:
      if (agentInfo->ext_fine_grain_pool.handle != 0) {
        ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                "Using extended fine grained access system memory pool");
        segment = agentInfo->ext_fine_grain_pool;
        break;
      }
    default:
      guarantee(false, "Invalid Memory Segment");
      break;
  }
  assert(segment.handle != 0);
  return segment;
}

// ================================================================================================
void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg,
                        const void* agentInfo, bool allowAllAgentsAccess) const {
  void* ptr = nullptr;
  uint32_t memFlags = 0;
  if (mem_seg == kKernArg) {
    memFlags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
  }

  hsa_amd_memory_pool_t pool =
      getHostMemoryPool(mem_seg, static_cast<const amd::roc::AgentInfo*>(agentInfo));
  hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, memFlags, &ptr);

  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
          "Allocate hsa host memory %p, size 0x%zx,"
          " numa_node = %d, mem_seg = %d",
          ptr, size, preferred_numa_node_, static_cast<int>(mem_seg));
  if (stat != HSA_STATUS_SUCCESS) {
    LogPrintfError("Fail allocation host memory with err %d", stat);
    return nullptr;
  }

  // Allow access to all GPU agents if the flag is set
  // otherwise only allow access to the local backend device.
  if (allowAllAgentsAccess) {
    stat = Hsa::agents_allow_access(gpu_agents_.size(), &gpu_agents_[0], nullptr, ptr);
  }
  else {
    stat = Hsa::agents_allow_access(1, &bkendDevice_, nullptr, ptr);
  }

  if (stat != HSA_STATUS_SUCCESS) {
    LogPrintfError("Fail hsa_amd_agents_allow_access with err %d", stat);
    hostFree(ptr, size);
    return nullptr;
  }

  return ptr;
}

// ================================================================================================
void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const {
  void* ptr = nullptr;
  auto numa_node_count = cpu_agents_.size(); // count of host numa nodes
  numa::NumaPolicy np(numa_node_count);
  if (!np.GetMemPolicy()) {
    return hostAlloc(size, alignment, mem_seg, cpu_agent_info_);
  }
  switch (np.GetPolicy()) {
    case numa::NumaPolicy::Policy::kPrefered:
    case numa::NumaPolicy::Policy::kBind:
      // We only care about the first CPU node
      for (unsigned int i = 0; i < numa_node_count; i++) {
        if (np.IsPolicySetAt(i)) {
          ptr = hostAlloc(size, alignment, mem_seg, &cpu_agents_[i]);
          break;
        }
      }
      break;
    default:
      //  All other modes fall back to default mode
      ptr = hostAlloc(size, alignment, mem_seg, cpu_agent_info_);
  }
  return ptr;
}

void* Device::hostLock(void* hostMem, size_t size, const MemorySegment memSegment) const {
  hsa_amd_memory_pool_t pool = getHostMemoryPool(memSegment);
  void* deviceMemory = nullptr;
  uint32_t memFlags = 0;
  if (memSegment == kIoMemory) {
    memFlags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
  }

  hsa_status_t status = Hsa::memory_lock_to_pool(
      hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, memFlags, &deviceMemory);
  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
          "Locking to pool %p, size 0x%zx, hostMem = %p,"
          " deviceMemory = %p, memSegment = %d",
          pool, size, hostMem, deviceMemory, static_cast<int>(memSegment));
  if (status != HSA_STATUS_SUCCESS) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_LOCK,
             "Failed to lock memory to pool, failed with hsa_status: %d", status);
    deviceMemory = nullptr;
  }
  return deviceMemory;
}

void Device::hostFree(void* ptr, size_t size) const { memFree(ptr, size); }

bool Device::deviceAllowAccess(void* ptr) const {
  std::lock_guard<std::mutex> lock(lock_allow_access_);
  if (!p2pAgents().empty()) {
    hsa_status_t stat =
        Hsa::agents_allow_access(p2pAgents().size(), p2pAgents().data(), nullptr, ptr);
    if (stat != HSA_STATUS_SUCCESS) {
      LogPrintfError("Allow p2p access failed - hsa_amd_agents_allow_access with err %d", stat);
      return false;
    }
  }
  return true;
}

bool Device::allowPeerAccess(device::Memory* memory) const {
  if (memory == nullptr) {
    return false;
  }
  if (!p2pAgents().empty()) {
    void* ptr = reinterpret_cast<void*>(memory->virtualAddress());
    hsa_agent_t agent = getBackendDevice();
    hsa_status_t stat = Hsa::agents_allow_access(1, &agent, nullptr, ptr);
    if (stat != HSA_STATUS_SUCCESS) {
      LogPrintfError("Allow p2p access failed - hsa_amd_agents_allow_access with err: %d", stat);
      return false;
    }
  }
  return true;
}

uint64_t Device::deviceVmemAlloc(size_t size, uint64_t flags) const {
  // PHYMEM: ROCCLR_MEM_HSA_UNCACHED is passed as HSA_AMD_MEMORY_POOL_UNCACHED_FLAG in |flags|.
  const bool uncached = (flags & HSA_AMD_MEMORY_POOL_UNCACHED_FLAG) != 0;
  const hsa_amd_memory_pool_t& pool = (uncached && gpu_ext_fine_grained_segment_.handle)
                                       ? gpu_ext_fine_grained_segment_ : gpuvm_segment_;
  LogPrintfError("VMEM alloc: pool (selected)=0x%x, gpu_ext_fine_grained=0x%x, gpuvm=0x%x",
                 pool.handle, gpu_ext_fine_grained_segment_.handle, gpuvm_segment_.handle);

  if (pool.handle == 0 || gpuvm_segment_max_alloc_ == 0) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
            "Invalid argument, pool_handle: 0x%x , max_alloc: %u", pool.handle,
            gpuvm_segment_max_alloc_);
    return 0;
  }

  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};

  // We only allow pinned memory at this time.
  hsa_status_t hsa_status = Hsa::vmem_handle_create(pool, size, MEMORY_TYPE_PINNED, 0,
                                                    &hsa_vmem_handle);

  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_handle_create! Failed with hsa status: %d", hsa_status);
    return 0;
  }

  return hsa_vmem_handle.handle;
}

void Device::deviceVmemRelease(uint64_t mem_handle) const {
  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
  hsa_vmem_handle.handle = mem_handle;

  hsa_status_t hsa_status = Hsa::vmem_handle_release(hsa_vmem_handle);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_handle_release! Failed with hsa status: %d", hsa_status);
  }
}

bool Device::hostVmemSupported(int numaNode) const {
  if (cpu_agents_.empty()) {
    return false;
  }
  int node = numaNode;
  if (node < 0 || static_cast<size_t>(node) >= cpu_agents_.size()) {
    node = static_cast<int>(numa::getCurrentNumaNode());
  }
  hsa_agent_t cpu = getCpuAgent(node);
  bool supported = false;
  hsa_status_t hsa_status = Hsa::agent_get_info(
      cpu, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HOST_ALLOC_DMABUF_SUPPORTED),
      &supported);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    // Older ROCr predates this query: treat host-NUMA VMM as unsupported.
    return false;
  }
  return supported;
}

uint64_t Device::hostVmemAlloc(size_t size, uint64_t flags, int numaNode) const {
  // numaNode < 0 (HostNumaCurrent) resolves to the calling thread's current node.
  int node = numaNode;
  if (node < 0) {
    node = static_cast<int>(numa::getCurrentNumaNode());
  }
  if (node < 0 || static_cast<size_t>(node) >= cpu_agents_.size()) {
    LogPrintfError("hostVmemAlloc: invalid NUMA node %d (cpu agents: %zu)", numaNode,
                   cpu_agents_.size());
    return 0;
  }

  if (!hostVmemSupported(node)) {
    LogError("hostVmemAlloc: host memory VMM not supported on this CPU agent");
    return 0;
  }

  // PHYMEM: ROCCLR_MEM_HSA_UNCACHED is passed as HSA_AMD_MEMORY_POOL_UNCACHED_FLAG in |flags|.
  const bool uncached = (flags & HSA_AMD_MEMORY_POOL_UNCACHED_FLAG) != 0;
  const MemorySegment mem_seg = uncached ? kUncachedAtomics : kAtomics;
  hsa_amd_memory_pool_t pool = getHostMemoryPool(mem_seg, &cpu_agents_[node]);
  if (pool.handle == 0) {
    LogPrintfError("hostVmemAlloc: no host memory pool for NUMA node %d", node);
    return 0;
  }

  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
  hsa_status_t hsa_status =
      Hsa::vmem_handle_create(pool, size, MEMORY_TYPE_PINNED, 0, &hsa_vmem_handle);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("hostVmemAlloc: hsa_amd_vmem_handle_create failed with status: %d", hsa_status);
    return 0;
  }

  return hsa_vmem_handle.handle;
}

void* Device::reserveMemory(size_t size, size_t alignment) const {
  void* ptr = nullptr;
  // Reserves non registered VA memory using HSA APIs.
  hsa_status_t status = Hsa::vmem_address_reserve_align(&ptr, size, 0, alignment,
                                                        HSA_AMD_VMEM_ADDRESS_NO_REGISTER);
  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Reserve hsa device memory %p, size 0x%zx", ptr, size);
  if (status != HSA_STATUS_SUCCESS) {
    LogError("Fail to reserve memory");
    return nullptr;
  }
  return ptr;
}

void Device::releaseMemory(void* ptr, size_t size) const {
  hsa_status_t hsa_status = Hsa::vmem_address_free(ptr, size);
  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Free hsa reserved memory %p", ptr);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogError("hsa_amd_vmem_address_free failed");
  }
}

void* Device::deviceLocalAlloc(size_t size, const AllocationFlags& flags, bool allowAllAgentsAccess) const {
  const hsa_amd_memory_pool_t& pool =
      (flags.pseudo_fine_grain_ && gpu_ext_fine_grained_segment_.handle)
          ? gpu_ext_fine_grained_segment_
      : (flags.atomics_ && gpu_fine_grained_segment_.handle) ? gpu_fine_grained_segment_
                                                             : gpuvm_segment_;

  if (pool.handle == 0 || gpuvm_segment_max_alloc_ == 0) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
            "Invalid argument, pool_handle: 0x%x , max_alloc: %u",
            pool.handle, gpuvm_segment_max_alloc_);
    return nullptr;
  }

  uint32_t hsa_mem_flags = 0;
  if (flags.contiguous_) {
    hsa_mem_flags = HSA_AMD_MEMORY_POOL_CONTIGUOUS_FLAG;
  }
  if (flags.executable_) {
    hsa_mem_flags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
  }
  if (flags.uncached_ && isa().versionMajor() == 12) {
    hsa_mem_flags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
  }

  void* ptr = nullptr;
  hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, hsa_mem_flags, &ptr);
  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
          "Allocate hsa device memory %p, size 0x%zx, hsa_mem_flags 0x%xh", ptr, size,
          hsa_mem_flags);
  if (stat != HSA_STATUS_SUCCESS) {
    LogError("Fail allocation local memory");
    return nullptr;
  }

  if (allowAllAgentsAccess) {
    if (isP2pEnabled() && deviceAllowAccess(ptr) == false) {
      LogError("Allow p2p access for memory allocation");
      memFree(ptr, size);
      return nullptr;
    }
  }

  return ptr;
}

void Device::memFree(void* ptr, size_t size) const {
  hsa_status_t stat = Hsa::memory_pool_free(ptr);
  ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Free hsa memory %p", ptr);
  if (stat != HSA_STATUS_SUCCESS) {
    LogError("Fail freeing local memory");
  }
}

void Device::updateFreeMemory(size_t size, bool free) {
  if (free) {
    freeMem_ += size;
  } else {
    if (size > freeMem_) {
      // To avoid underflow of the freeMem_
      // This can happen if the free mem tracked is inaccurate, as some allocations can happen
      // directly via ROCr
      ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS,
              "Free memory set to zero on device 0x%lx, requested size = 0x%zx, freeMem_ = 0x%zx",
              this, size, freeMem_.load());
      freeMem_ = 0;
      return;
    }
    freeMem_ -= size;
  }
  ClPrint(amd::LOG_INFO, amd::LOG_MEM, "Device=0x%lx, freeMem_ = 0x%zx", this, freeMem_.load());
}

// ================================================================================================
void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags,
                       void* svmPtr) const {
  amd::Memory* mem = nullptr;
  void* svmPtrUsed = reinterpret_cast<void*>(amd::Memory::MemoryType::kSvmMemoryPtr);

  if (nullptr != svmPtr) {
    // Find the existing amd::mem object
    mem = amd::MemObjMap::FindMemObj(svmPtr);
    if (mem != nullptr) {
      return mem->getSvmPtr();
    }
    if (flags & CL_MEM_USE_HOST_PTR) {
      svmPtrUsed = svmPtr;
    } else {
      ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM, "Cannot find svm_ptr: %p", svmPtr);
      return nullptr;
    }
  }

  // create a hidden buffer, which will allocated on the device later
  mem = new (context) amd::Buffer(context, flags, size, svmPtrUsed);
  if (mem == nullptr) {
    LogError("failed to create a svm mem object!");
    return nullptr;
  }

  if (!mem->create(nullptr)) {
    LogError("failed to create a svm hidden buffer!");
    mem->release();
    return nullptr;
  }
  // if the device supports SVM FGS, return the committed CPU address directly.
  Memory* gpuMem = getRocMemory(mem);
  if (gpuMem == nullptr) {
    LogError("failed to create GPU memory from svm hidden buffer!");
    return nullptr;
  }

  // add the information to context so that we can use it later.
  if (mem->getSvmPtr() != nullptr) {
    amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem);
  }
  return mem->getSvmPtr();
}

void* Device::virtualAlloc(void* req_addr, size_t size, size_t alignment) {
  void* vptr = nullptr;
  // Reserves the address using HSA APIs, with requested address.
  // There is no guarantee that we will get the requested address.
  hsa_status_t hsa_status =
      Hsa::vmem_address_reserve(&vptr, size, reinterpret_cast<uint64_t>(req_addr), 0);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_address_reserve. Failed with status: %d", hsa_status);
    return nullptr;
  }

  constexpr bool kParent = true;
  amd::Memory* mem = CreateVirtualBuffer(context(), vptr, size, -1, -1, kParent);
  if (mem == nullptr) {
    LogPrintfError("Cannot create Virtual Buffer for vptr: %p of size: %u", vptr, size);
  }

  return mem->getSvmPtr();
}

bool Device::virtualFree(void* addr) {
  amd::Memory* memObj = amd::MemObjMap::FindVirtualMemObj(addr);
  if (memObj == nullptr) {
    LogPrintfError("Cannot find the Virtual MemObj entry for this addr %p", addr);
    return false;
  }

  if (!memObj->getContext().devices()[0]->DestroyVirtualBuffer(memObj)) {
    return false;
  }

  hsa_status_t hsa_status = Hsa::vmem_address_free(memObj->getSvmPtr(), memObj->getSize());
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_address_free. Failed with status:%d", hsa_status);
    return false;
  }
  return true;
}

// ================================================================================================
// Direct synchronous map/unmap path bypassing VirtualMapCommand. Used by the
// non-async hipMemMap/hipMemUnmap entry points (wired up in subsequent
// commits). No execution() lock and no barrier packet: the HIP layer is
// responsible for draining access-device queues from the CPU side before
// calling, and hsa_amd_vmem_{map,unmap} are synchronous w.r.t. the CPU.
cl_int Device::virtualMap(void* va, size_t size, amd::Memory* phys) {
  if (phys == nullptr) {
    LogError("virtualMap: phys is nullptr");
    return CL_INVALID_VALUE;
  }

  amd::Memory* vaddr_sub_obj = MapMemObjBookkeeping(phys, va, size);
  if (vaddr_sub_obj == nullptr) {
    LogError("virtualMap: MapMemObjBookkeeping failed");
    return CL_INVALID_VALUE;
  }

  hsa_amd_vmem_alloc_handle_t opaque_hsa_handle;
  opaque_hsa_handle.handle = phys->getUserData().hsa_handle;
  hsa_status_t hsa_status = Hsa::vmem_map(vaddr_sub_obj->getSvmPtr(), size,
                                          vaddr_sub_obj->getOffset(), opaque_hsa_handle, 0);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("virtualMap: hsa_amd_vmem_map failed with status: %d", hsa_status);
    // Roll back the bookkeeping so the sub_obj/MemObjMap doesn't leak.
    // FinalizeMapMemObjBookkeeping was never called, so MemObjMap doesn't
    // contain va and the cross-links are not wired. Just tear down the
    // sub-buffer view directly.
    vaddr_sub_obj->getContext().devices()[0]->DestroyVirtualBuffer(vaddr_sub_obj);
    vaddr_sub_obj->release();
    return CL_OUT_OF_HOST_MEMORY;
  }

  constexpr bool kImportVmmForInterprocess = true;
  FinalizeMapMemObjBookkeeping(vaddr_sub_obj, phys, va, kImportVmmForInterprocess);
  return CL_SUCCESS;
}

// ================================================================================================
cl_int Device::virtualUnmap(void* va, size_t size) {
  amd::Memory* vaddr_sub_obj = amd::MemObjMap::FindMemObj(va);
  if (vaddr_sub_obj == nullptr) {
    LogPrintfError("virtualUnmap: no sub_obj for va: %p", va);
    return CL_INVALID_VALUE;
  }

  hsa_status_t hsa_status = Hsa::vmem_unmap(vaddr_sub_obj->getSvmPtr(), size);
  if (hsa_status != HSA_STATUS_SUCCESS) {
    LogPrintfError("virtualUnmap: hsa_amd_vmem_unmap failed with status: %d", hsa_status);
    return CL_INVALID_VALUE;
  }

  constexpr bool kDestroyVirtualBuffer = true;
  constexpr bool kReleaseSubObj = true;
  UnmapMemObjBookkeeping(vaddr_sub_obj, va, kDestroyVirtualBuffer, kReleaseSubObj);
  return CL_SUCCESS;
}

// ================================================================================================
bool Device::SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags,
                          VmmLocationType access_location, int numaNode) {
  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
  hsa_amd_memory_access_desc_t desc;
  desc.permissions = static_cast<hsa_access_permission_t>(access_flags);
  // Resolve the agent granted access: the GPU backend for device access, the
  // default CPU agent for generic/current host access, or the CPU agent of a
  // specific NUMA node for HostNuma. getCpuAgent(int) clamps an out-of-range node
  // to the default agent (the HIP layer validates the node id beforehand).
  switch (access_location) {
    case VmmLocationType::kDevice:
      desc.agent_handle = getBackendDevice();
      break;
    case VmmLocationType::kHostNuma:
      desc.agent_handle = getCpuAgent(numaNode);
      break;
    case VmmLocationType::kHostNumaCurrent:
      desc.agent_handle = getCpuAgent(static_cast<int>(numa::getCurrentNumaNode()));
      break;
    case VmmLocationType::kHost:
    default:
      desc.agent_handle = getCpuAgent();
      break;
  }

  if ((hsa_status = Hsa::vmem_set_access(va_addr, va_size, &desc, 1)) != HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_set_access. Failed with status:%d", hsa_status);
    return false;
  }

  return true;
}

bool Device::GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) const {
  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
  hsa_access_permission_t perms;

  size_t discard_offset = 0;
  amd::Memory* va_mem_obj = amd::MemObjMap::FindMemObj(va_addr, &discard_offset);
  if (va_mem_obj == nullptr) {
    LogPrintfError("Failed to get Memory Object for va_addr: %p", va_addr);
    return false;
  }

  if ((hsa_status = Hsa::vmem_get_access(va_mem_obj->getSvmPtr(), &perms, getBackendDevice())) !=
      HSA_STATUS_SUCCESS) {
    LogPrintfError("Failed hsa_amd_vmem_get_access. Failed with status:%d", hsa_status);
    return false;
  }

  *access_flags_ptr = static_cast<VmmAccess>(perms);

  return true;
}

// ================================================================================================
amd::Device::VmmExportStatus Device::ExportShareableVMMHandle(amd::Memory& amd_mem_obj, int flags,
                                                void* shareableHandle,
                                                amd::Memory::HandleType handle_type) {
  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
  hsa_vmem_handle.handle = amd_mem_obj.getUserData().hsa_handle;

  if (hsa_vmem_handle.handle == 0) {
    LogError("HSA Handle is not valid");
    return amd::Device::VmmExportStatus::kError;
  }

  if (handle_type == amd::Memory::HandleType::kHandleFabric) { //handle type fabric
    hsa_fabric_handle_t fabric_handle;
    if ((hsa_status = Hsa::vmem_export_fabric_handle(&fabric_handle,
                        hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) {
      LogPrintfError("Failed hsa_vmem_export_fabric_handle with status: %d \n", hsa_status);
      if (hsa_status == static_cast<hsa_status_t>(HSA_STATUS_ERROR_RESOURCE_NOT_READY)) {
        return amd::Device::VmmExportStatus::kResourceNotReady;
      }
      return amd::Device::VmmExportStatus::kError;
    }
    *(reinterpret_cast<hsa_fabric_handle_t*>(shareableHandle)) = fabric_handle;
  } else {
    int dmabuf_fd = 0;
    if ((hsa_status = Hsa::vmem_export_shareable_handle(&dmabuf_fd,
                        hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) {
      LogPrintfError("Failed hsa_vmem_export_shareable_handle with status: %d \n", hsa_status);
      return amd::Device::VmmExportStatus::kError;
    }
    *(reinterpret_cast<int*>(shareableHandle)) = dmabuf_fd;
  }

  return amd::Device::VmmExportStatus::kSuccess;
}

// ================================================================================================
bool Device::ImportShareableHSAHandle(void* osHandle, uint64_t* hsa_handle_ptr,
                                      amd::Memory::HandleType handle_type) const {
  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
  hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};

  if (hsa_handle_ptr == nullptr) {
    LogError("HSA Handle ptr is null");
    return false;
  }

  if (handle_type == amd::Memory::HandleType::kHandleFabric) {
    hsa_fabric_handle_t fabric_handle = *(reinterpret_cast<hsa_fabric_handle_t*>(osHandle));
    if ((hsa_status = Hsa::vmem_import_fabric_handle(fabric_handle, &hsa_vmem_handle))
                        != HSA_STATUS_SUCCESS) {
      LogPrintfError("Failed hsa_amd_vmem_import_fabric_handle with status: %d \n", hsa_status);
      return false;
    }
  } else {
    int dmabuf_fd = static_cast<int>(reinterpret_cast<uintptr_t>(osHandle));
    if ((hsa_status = Hsa::vmem_import_shareable_handle(dmabuf_fd, &hsa_vmem_handle))
                        != HSA_STATUS_SUCCESS) {
      LogPrintfError("Failed hsa_amd_vmem_import_shareable_handle with status: %d \n", hsa_status);
      return false;
    }
  }

  *hsa_handle_ptr = hsa_vmem_handle.handle;
  return true;
}

// ================================================================================================
amd::Memory* Device::ImportShareableVMMHandle(void* osHandle, amd::Memory::HandleType handle_type) {
  amd::Memory* amd_mem_obj = new (context())
      amd::Buffer(context(), ROCCLR_MEM_PHYMEM | ROCCLR_MEM_INTERPROCESS, 0, osHandle);
  if (amd_mem_obj == nullptr) {
    LogError("Cannot create memory object");
    return nullptr;
  }

  amd_mem_obj->getUserData().hsa_handle_type = handle_type;
  if (!amd_mem_obj->create(nullptr, false)) {
    LogError("Failed to create mem_obj from imported fd");
    amd_mem_obj->release();
    return nullptr;
  }

  return amd_mem_obj;
}

// ================================================================================================
bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count, amd::MemoryAdvice advice,
                                 bool first_alloc, bool use_cpu, int numa_id) const {
  if ((settings().hmmFlags_ & Settings::Hmm::EnableSvmTracking) && !first_alloc) {
    amd::Memory* svm_mem = amd::MemObjMap::FindMemObj(dev_ptr);
    if ((nullptr == svm_mem) || ((svm_mem->getMemFlags() & CL_MEM_ALLOC_HOST_PTR) == 0) ||
        // Validate the range of provided memory
        ((svm_mem->getSize() - (reinterpret_cast<const_address>(dev_ptr) -
                                reinterpret_cast<address>(svm_mem->getSvmPtr()))) < count)) {
      LogPrintfError("SetSvmAttributes received unknown memory for update: %p!", dev_ptr);
      return false;
    }
  }
  if (info().hmmSupported_) {
    std::vector<hsa_amd_svm_attribute_pair_t> attr;

    switch (advice) {
      case amd::MemoryAdvice::SetReadMostly:
        attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, true});
        break;
      case amd::MemoryAdvice::UnsetReadMostly:
        attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, false});
        break;
      case amd::MemoryAdvice::SetPreferredLocation:
        if (use_cpu) {
          attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, getCpuAgent(numa_id).handle});
        } else {
          attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, getBackendDevice().handle});
        }
        break;
      case amd::MemoryAdvice::UnsetPreferredLocation:
        // @note: 0 may cause a failure on old runtimes
        attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, 0});
        break;
      case amd::MemoryAdvice::SetAccessedBy: {
        const uint64_t attrib = (first_alloc) ? HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE
                                              : HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE;
        if (use_cpu) {
          attr.push_back({attrib, getCpuAgent().handle});
        } else {
          if (first_alloc) {
            // Provide access to all possible devices.
            //! @note: HMM should support automatic page table update with xnack enabled,
            //! but currently it doesn't and runtime explicitly enables access from all devices
            for (const auto dev : devices()) {
              // Skip null devices
              if (static_cast<Device*>(dev)->getBackendDevice().handle != 0) {
                attr.push_back({attrib, static_cast<Device*>(dev)->getBackendDevice().handle});
              }
            }
          } else {
            attr.push_back({attrib, getBackendDevice().handle});
          }
        }
        break;
      }
      case amd::MemoryAdvice::UnsetAccessedBy:
        // When unsetting we should use HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE for the agent
        attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, getBackendDevice().handle});
        break;
      case amd::MemoryAdvice::SetCoarseGrain:
        attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED});
        break;
      case amd::MemoryAdvice::UnsetCoarseGrain:
        attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED});
        break;
      default:
        return false;
        break;
    }

    hsa_status_t status =
        Hsa::svm_attributes_set(const_cast<void*>(dev_ptr), count, attr.data(), attr.size());
    if (status != HSA_STATUS_SUCCESS) {
      LogPrintfError("hsa_amd_svm_attributes_set() failed. Advice: %d, status: %d", advice, status);
      return false;
    }
  } else {
    LogWarning("hsa_amd_svm_attributes_set() is ignored, because no HMM support");
  }
  return true;
}

// ================================================================================================
bool Device::SetSvmAttributes(const void* dev_ptr, size_t count, amd::MemoryAdvice advice,
                              bool use_cpu, int numa_id) const {
  constexpr bool kFirstAlloc = false;
  return SetSvmAttributesInt(dev_ptr, count, advice, kFirstAlloc, use_cpu);
}

// ================================================================================================
bool Device::GetSvmAttributes(void** data, size_t* data_sizes, int* attributes,
                              size_t num_attributes, const void* dev_ptr, size_t count) const {
  amd::Memory* svm_mem = amd::MemObjMap::FindMemObj(dev_ptr);
  if (settings().hmmFlags_ & Settings::Hmm::EnableSvmTracking) {
    if ((nullptr == svm_mem) || ((svm_mem->getMemFlags() & CL_MEM_ALLOC_HOST_PTR) == 0) ||
        // Validate the range of provided memory
        ((svm_mem->getSize() - (reinterpret_cast<const_address>(dev_ptr) -
                                reinterpret_cast<address>(svm_mem->getSvmPtr()))) < count)) {
      LogPrintfError("GetSvmAttributes received unknown memory %p for state!", dev_ptr);
      return false;
    }
  }

  hsa_amd_pointer_info_t ptr_info = {};
  for (size_t i = 0; i < num_attributes; ++i) {
    if (attributes[i] == amd::MemRangeAttribute::CoherencyMode) {
      ptr_info.size = sizeof(hsa_amd_pointer_info_t);
      // Query ptr type to see if it's a HMM allocation
      hsa_status_t status =
          Hsa::pointer_info(const_cast<void*>(dev_ptr), &ptr_info, nullptr, nullptr, nullptr);
      // The call should never fail in ROCR, but just check for an error and continue
      if (status != HSA_STATUS_SUCCESS) {
        LogError("hsa_amd_pointer_info() failed");
      }

      // Check if it's a legacy non-HMM allocation and update query
      *reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE;
      if (ptr_info.type == HSA_EXT_POINTER_TYPE_HSA ||
          ptr_info.type == HSA_EXT_POINTER_TYPE_LOCKED ||
          ptr_info.type == HSA_EXT_POINTER_TYPE_IPC) {
        if (ptr_info.global_flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED) {
          *reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED;
        } else if (ptr_info.global_flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED) {
          *reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED;
        }
      }
      // If coherency is still indeterminate
      if (ptr_info.type == HSA_EXT_POINTER_TYPE_HSA_VMEM) {
        if (svm_mem != nullptr && (svm_mem->getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) {
          *reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED;
        } else {
          *reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED;
        }
      }
    }
  }

  if (info().hmmSupported_) {
    uint32_t accessed_by = 0;
    std::vector<hsa_amd_svm_attribute_pair_t> attr;

    for (size_t i = 0; i < num_attributes; ++i) {
      switch (attributes[i]) {
        case amd::MemRangeAttribute::ReadMostly:
          attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, 0});
          break;
        case amd::MemRangeAttribute::PreferredLocation:
          attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, 0});
          break;
        case amd::MemRangeAttribute::AccessedBy:
          accessed_by = attr.size();
          // Add all GPU devices into the query
          for (const auto agent : gpu_agents_) {
            attr.push_back({HSA_AMD_SVM_ATTRIB_ACCESS_QUERY, agent.handle});
          }
          // Add CPU devices
          for (const auto agent_info : cpu_agents_) {
            attr.push_back({HSA_AMD_SVM_ATTRIB_ACCESS_QUERY, agent_info.agent.handle});
          }
          accessed_by = attr.size() - accessed_by;
          break;
        case amd::MemRangeAttribute::LastPrefetchLocation:
          attr.push_back({HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION, 0});
          break;
        case amd::MemRangeAttribute::CoherencyMode:
          if (*reinterpret_cast<uint32_t*>(data[i]) == HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE) {
            attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, 0});
          }
          break;
        default:
          return false;
          break;
      }
    }
    // Only call svm_attributes_get if there are SVM attributes to query
    if (attr.empty()) {
      return true;
    }
    hsa_status_t status =
        Hsa::svm_attributes_get(const_cast<void*>(dev_ptr), count, attr.data(), attr.size());
    if (status != HSA_STATUS_SUCCESS) {
      LogError("hsa_amd_svm_attributes_get() failed");
      return false;
    }

    uint32_t idx = 0;
    uint32_t rocr_attr = 0;
    for (size_t i = 0; i < num_attributes; ++i) {
      const auto& it = attr[rocr_attr];
      switch (attributes[i]) {
        case amd::MemRangeAttribute::ReadMostly:
          if (data_sizes[idx] != sizeof(uint32_t)) {
            return false;
          }
          // Cast ROCr value into the hip format
          *reinterpret_cast<uint32_t*>(data[idx]) =
              (static_cast<uint32_t>(it.value) > 0) ? true : false;
          ++rocr_attr;
          break;
        // The logic should be identical for the both queries
        case amd::MemRangeAttribute::PreferredLocation:
        case amd::MemRangeAttribute::LastPrefetchLocation:
          if (data_sizes[idx] != sizeof(uint32_t)) {
            return false;
          }
          *reinterpret_cast<int32_t*>(data[idx]) = static_cast<int32_t>(amd::InvalidDeviceId);
          // Find device agent returned by ROCr
          for (auto& device : devices()) {
            if (static_cast<Device*>(device)->getBackendDevice().handle == it.value) {
              *reinterpret_cast<uint32_t*>(data[idx]) = static_cast<uint32_t>(device->index());
            }
          }
          // Find CPU agent returned by ROCr
          for (auto& agent_info : cpu_agents_) {
            if (agent_info.agent.handle == it.value) {
              *reinterpret_cast<int32_t*>(data[idx]) = static_cast<int32_t>(amd::CpuDeviceId);
            }
          }
          ++rocr_attr;
          break;
        case amd::MemRangeAttribute::AccessedBy: {
          uint32_t entry = 0;
          uint32_t device_count = data_sizes[idx] / 4;
          // Make sure it's multiple of 4
          if (data_sizes[idx] % 4 != 0) {
            return false;
          }
          for (uint32_t att = 0; att < accessed_by; ++att) {
            const auto& it = attr[rocr_attr + att];
            if (entry >= device_count) {
              // The size of the array is less than the amount of available devices
              break;
            }
            switch (it.attribute) {
              case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE:
              case HSA_AMD_SVM_ATTRIB_AGENT_NO_ACCESS:
                break;
              case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE:
                reinterpret_cast<int32_t*>(data[idx])[entry] =
                    static_cast<int32_t>(amd::InvalidDeviceId);
                // Find device agent returned by ROCr
                for (auto& device : devices()) {
                  if (static_cast<Device*>(device)->getBackendDevice().handle == it.value) {
                    reinterpret_cast<uint32_t*>(data[idx])[entry] =
                        static_cast<uint32_t>(device->index());
                  }
                }
                // Find CPU agent returned by ROCr
                for (auto& agent_info : cpu_agents_) {
                  if (agent_info.agent.handle == it.value) {
                    reinterpret_cast<int32_t*>(data[idx])[entry] =
                        static_cast<int32_t>(amd::CpuDeviceId);
                  }
                }
                ++entry;
                break;
              default:
                LogWarning("Unexpected result from HSA_AMD_SVM_ATTRIB_ACCESS_QUERY");
                break;
            }
          }
          rocr_attr += accessed_by;
          for (uint32_t i = entry; i < device_count; ++i) {
            reinterpret_cast<int32_t*>(data[idx])[i] = static_cast<int32_t>(amd::InvalidDeviceId);
          }
          break;
        }
        case amd::MemRangeAttribute::CoherencyMode:
          if (data_sizes[idx] != sizeof(uint32_t)) {
            return false;
          }
          // if ptr is HMM alloc then overwrite the values
          if (*reinterpret_cast<uint32_t*>(data[idx]) == HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE) {
            // Cast ROCr value into the hip format
            *reinterpret_cast<uint32_t*>(data[idx]) = static_cast<uint32_t>(it.value);
          }
          ++rocr_attr;
          break;
        default:
          return false;
          break;
      }
      // Find the next location in the query
      ++idx;
    }
  } else if (ptr_info.type == HSA_EXT_POINTER_TYPE_RESERVED_ADDR) {
    LogError("GetSvmAttributes() failed, because no HMM support");
    return false;
  }

  return true;
}

size_t Device::ScratchLimitCurrent() const {
  uint64_t scratchLimitCurrent = 0;
  hsa_status_t ret =
      Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_CURRENT,
                         &scratchLimitCurrent);
  if (HSA_STATUS_SUCCESS != ret) {
    LogPrintfError("HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_CURRENT cannot be queried! Err: 0x%xh", ret);
    return 0;
  }
  return static_cast<size_t>(scratchLimitCurrent);
};

bool Device::UpdateScratchLimitCurrent(size_t limit) const {
  hsa_status_t ret = Hsa::agent_set_async_scratch_limit(bkendDevice_, limit);
  if (HSA_STATUS_SUCCESS != ret) {
    LogPrintfError("hsa_amd_agent_set_async_scratch_limit(%zu) failed with err 0x%xh", limit, ret);
    return false;
  }
  return true;
};

// ================================================================================================
bool Device::SvmAllocInit(void* memory, size_t size) const {
  amd::MemoryAdvice advice = amd::MemoryAdvice::SetAccessedBy;
  constexpr bool kFirstAlloc = true;
  if (!SetSvmAttributesInt(memory, size, advice, kFirstAlloc)) {
    return false;
  }

  if ((settings().hmmFlags_ & Settings::Hmm::EnableMallocPrefetch) == 0) {
    return true;
  }

  if (info().hmmSupported_) {
    // Initialize signal for the barrier
    Hsa::signal_store_relaxed(prefetch_signal_, kInitSignalValueOne);

    // Initiate a prefetch command which should force memory update in HMM
    hsa_status_t status =
        Hsa::svm_prefetch_async(memory, size, getBackendDevice(), 0, nullptr, prefetch_signal_);
    if (status != HSA_STATUS_SUCCESS) {
      LogError("hsa_amd_svm_prefetch_async() failed");
      return false;
    }

    // Wait for the prefetch
    if (!WaitForSignal(prefetch_signal_)) {
      LogError("Barrier packet submission failed");
      return false;
    }
  } else {
    LogWarning("Early prefetch failed, because no HMM support");
  }

  return true;
}

// ================================================================================================
void Device::svmFree(void* ptr) const {
  amd::Memory* svmMem = amd::MemObjMap::FindMemObj(ptr);
  if (nullptr != svmMem) {
    amd::MemObjMap::RemoveMemObj(svmMem->getSvmPtr());
    svmMem->release();
  }
}

// ================================================================================================
VirtualGPU* Device::xferQueue() const {
  Device* thisDevice = const_cast<Device*>(this);
  std::call_once(xferQueueOnce_, [thisDevice]() {
    // Create virtual device for internal memory transfer
    thisDevice->xferQueue_ = reinterpret_cast<VirtualGPU*>(thisDevice->createVirtualDevice());
    if (!thisDevice->xferQueue_) {
      LogError("Couldn't create the device transfer manager!");
      return;
    }
    if (thisDevice->xferQueue_->gpu_queue() == nullptr) {
      thisDevice->xferQueue_->SetGpuQueue(
          thisDevice->AcquireActiveQueue(amd::CommandQueue::Priority::Normal));
    }
  });
  if (!xferQueue_) {
    LogError("Couldn't create the device transfer manager!");
    return nullptr;
  }
  xferQueue_->enableSyncBlit();
  return xferQueue_;
}

// ================================================================================================
bool Device::SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput,
                          cl_set_device_clock_mode_output_amd* pSetClockModeOutput) {
  bool result = true;
  return result;
}

// ================================================================================================
bool Device::IsHwEventReady(const amd::Event& event, bool wait, amd::SyncPolicy policy) const {
  void* hw_event =
      (event.NotifyEvent() != nullptr) ? event.NotifyEvent()->HwEvent() : event.HwEvent();
  if (hw_event == nullptr) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_SIG, "No HW event");
    return false;
  } else if (wait) {
    // hipEventBlockingSync
    // when set the CPU gives up host thread for other work
    // when not set the CPU enters a busy-wait on the event to occur
    constexpr int kHipEventBlockingSync = 0x1;
    bool active_wait =
        !((policy == amd::SyncPolicy::Blocking) & kHipEventBlockingSync) && ActiveWait();
    bool yield = (policy == amd::SyncPolicy::Yield);
    return WaitForSignal(reinterpret_cast<ProfilingSignal*>(hw_event)->signal_, active_wait, yield);
  }

  auto signal = reinterpret_cast<ProfilingSignal*>(hw_event)->signal_;
  ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Check HW event = 0x%lx", signal.handle);

  return (Hsa::signal_load_relaxed(signal) == 0);
}

// ================================================================================================
void Device::getHwEventTime(const amd::Event& event, uint64_t* start, uint64_t* end) const {
  void* hw_event =
      (event.NotifyEvent() != nullptr) ? event.NotifyEvent()->HwEvent() : event.HwEvent();
  if (hw_event == nullptr) {
    ClPrint(amd::LOG_INFO, amd::LOG_SIG, "No HW event to read time");
    *start = *end = 0;
  } else {
    fetchSignalTime(reinterpret_cast<ProfilingSignal*>(hw_event)->signal_, getBackendDevice(),
                    start, end);
  }
}

// ================================================================================================
hsa_queue_t* Device::getQueueFromPool(const uint qIndex, bool force_reuse,
                                      hsa_queue_t* preferred,
                                      const std::unordered_set<uint64_t>* excluded_ids) {
  // Only reuse queues when we've reached the maximum limit, unless forced
  // Below the limit, return nullptr to allow creating new queues
  if (!force_reuse && queuePool_[qIndex].size() < settings().max_hw_queues_) {
    return nullptr;
  }

  // We've hit the limit, must reuse - find the queue with lowest load metric
  if (qIndex < QueuePriority::Total && queuePool_[qIndex].size() > 0) {
    // Best-effort preferred queue hint: for graph stream stability
    // Skip preferred if it's in the excluded set
    if (preferred != nullptr) {
      bool preferred_excluded = excluded_ids && excluded_ids->count(preferred->id) > 0;
      if (!preferred_excluded) {
        auto it = queuePool_[qIndex].find(preferred);
        if (it != queuePool_[qIndex].end()) {
          it->second.refCount++;
          ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
                  "Reusing preferred queue: %p refCount: %d",
                  it->first->base_address, it->second.refCount);
          return it->first;
        }
      }
    }

    typedef decltype(queuePool_)::value_type::const_reference PoolRef;

    // Select queue based on dynamic_queues_ mode
    decltype(queuePool_[qIndex].begin()) lowest;
    uint32_t mode = settings().dynamic_queues_;

    // gfx9XX pipe distribution: queues map to pipes via queue_id % num_pipes
    const bool pipe_dist = settings().queue_pipe_dist_;
    const uint32_t num_pipes = numHwPipes_;

    lowest = std::min_element(
        queuePool_[qIndex].begin(), queuePool_[qIndex].end(),
        [mode, pipe_dist, num_pipes, excluded_ids](PoolRef first_candidate,
                                                   PoolRef second_candidate) {
          // Exclusion filtering: prefer non-excluded queues over excluded ones
          bool first_is_excluded =
              excluded_ids && excluded_ids->count(first_candidate.first->id) > 0;
          bool second_is_excluded =
              excluded_ids && excluded_ids->count(second_candidate.first->id) > 0;
          if (first_is_excluded != second_is_excluded) return second_is_excluded;

          // Always use an unshared regular queue before sharing an active queue. On devices
          // with coarse-grained read pointer updates, an idle queue can retain a nonzero
          // apparent depth and must not lose to a blocked queue with a smaller reported depth.
          bool first_is_unshared =
              first_candidate.second.refCount == 0 &&
              !first_candidate.second.hasDedicatedQueue_;
          bool second_is_unshared =
              second_candidate.second.refCount == 0 &&
              !second_candidate.second.hasDedicatedQueue_;
          if (first_is_unshared != second_is_unshared) return first_is_unshared;

          if (mode >= 1) {
            // Mode 1+: Advanced weighted metric with dedicated queue penalty
            // Metric = dedicated_queue_penalty + (depth << 4) + refCount
            uint64_t first_metric =
                first_candidate.second.GetLoadMetric(first_candidate.first, mode);
            uint64_t second_metric =
                second_candidate.second.GetLoadMetric(second_candidate.first, mode);

            if (first_metric == second_metric && pipe_dist) {
              // gfx9XX pipe distribution: prefer lower pipe IDs for consistent distribution
              uint64_t first_pipe = first_candidate.first->id % num_pipes;
              uint64_t second_pipe = second_candidate.first->id % num_pipes;
              return first_pipe < second_pipe;
            }
            return first_metric < second_metric;
          } else {
            // Mode 0: Simple refCount-based selection
            return first_candidate.second.refCount < second_candidate.second.refCount;
          }
        });

    lowest->second.refCount++;
    ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
            "Selected queue (mode=%u): %p refCount: %d, depth: %lu, metric: %lu, pipe: %d%s%s",
            mode, lowest->first->base_address, lowest->second.refCount,
            QueueInfo::GetHwQueueDepth(lowest->first),
            lowest->second.GetLoadMetric(lowest->first, mode),
            pipe_dist ? (lowest->first->id % num_pipes) : -1,
            force_reuse ? " (forced)" : "",
            (excluded_ids && excluded_ids->count(lowest->first->id) > 0)
                ? " (excluded-fallback)" : "");
    return lowest->first;
  }
  return nullptr;
}

// ================================================================================================
hsa_queue_t* Device::AcquireActiveQueue(amd::CommandQueue::Priority priority,
                                        hsa_queue_t* preferred,
                                        const std::unordered_set<uint64_t>* excluded_ids) {
  uint32_t queue_size = ROC_AQL_QUEUE_SIZE;
  auto queue = acquireQueue(queue_size, false, std::vector<uint32_t>{},
                            priority, true, false, preferred, excluded_ids);
  return queue;
}

// ================================================================================================
hsa_queue_t* Device::acquireQueue(uint32_t queue_size_hint, bool coop_queue,
                                  const std::vector<uint32_t>& cuMask,
                                  amd::CommandQueue::Priority priority, bool managed,
                                  bool dedicated_queue, hsa_queue_t* preferred,
                                  const std::unordered_set<uint64_t>* excluded_ids,
                                  void** metadata_ring_buffer) {
  // App-thread context: flush queues deferred from the async thread once they
  // reach the batch threshold, bounding live deferred queues.
  if (DeferredQueueCount() >= kDeferredQueueDrainThreshold) {
    DrainDeferredQueueDestroys();
  }

  hsa_amd_queue_priority_t queue_priority;
  uint qIndex;
  switch (priority) {
    case amd::CommandQueue::Priority::Low:
      queue_priority = HSA_AMD_QUEUE_PRIORITY_LOW;
      qIndex = QueuePriority::Low;
      break;
    case amd::CommandQueue::Priority::High:
      queue_priority = HSA_AMD_QUEUE_PRIORITY_HIGH;
      qIndex = QueuePriority::High;
      break;
    case amd::CommandQueue::Priority::Normal:
    case amd::CommandQueue::Priority::Medium:
    default:
      queue_priority = HSA_AMD_QUEUE_PRIORITY_NORMAL;
      qIndex = QueuePriority::Normal;
      break;
  }

  // If flag set, force all streams to normal priority
  // This means that GPU_MAX_HW_QUEUES may need to be incremented
  // to account for the additional normal priority queues
  if (DEBUG_HIP_IGNORE_STREAM_PRIORITY) {
    queue_priority = HSA_AMD_QUEUE_PRIORITY_NORMAL;
    qIndex = QueuePriority::Normal;
  }

  { // Lock
    amd::ScopedLock l(active_queue_access_);

    assert(queuePool_[QueuePriority::Low].size() <= settings().max_hw_queues_ ||
           queuePool_[QueuePriority::Normal].size() <= settings().max_hw_queues_ ||
           queuePool_[QueuePriority::High].size() <= settings().max_hw_queues_);

    ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
            "Number of allocated hardware queues with low priority: %d,"
            " with normal priority: %d, with high priority: %d, maximum per priority is: %d",
            queuePool_[QueuePriority::Low].size(), queuePool_[QueuePriority::Normal].size(),
            queuePool_[QueuePriority::High].size(), settings().max_hw_queues_);

    // If we have reached the max number of queues, reuse an existing queue with the matching queue
    // priority, choosing the one with the least number of users. Note: Don't attempt to reuse the
    // cooperative queue, since it's single per device.

    // num_queues_[qIndex] tracks persistent (non-managed) queues per priority.
    // When the total queues (managed + non-managed) exceed max_hw_queues_, we must reuse existing
    // queues. 'managed' streams do not increment num_queues_, allowing them to use the
    // pool without permanently consuming slots. ReleaseActiveQueue() uses this counter to
    // decide when to start reclaiming queues.
    if (!coop_queue && (cuMask.size() == 0) &&
        (queuePool_[qIndex].size() >= settings().max_hw_queues_)) {
      hsa_queue_t* queue = getQueueFromPool(qIndex, false, preferred, excluded_ids);
      if (queue != nullptr) {
        if (!managed) {
          num_queues_[qIndex]++;
        }
        return queue;
      }
    }
  } // Lock release

  // Create a new queue.
  uint32_t queue_max_packets = 0;
  if (HSA_STATUS_SUCCESS !=
      Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_QUEUE_MAX_SIZE, &queue_max_packets)) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE, "Cannot get hsa agent info");
    return nullptr;
  }
  auto queue_size = (queue_max_packets < queue_size_hint) ? queue_max_packets : queue_size_hint;

  auto queue_type = HSA_QUEUE_TYPE_MULTI;

  if (coop_queue) {
    queue_type = HSA_QUEUE_TYPE_COOPERATIVE;
  }

  // Resolve the final CU mask (merge custom + global, expand for WGP mode)
  std::vector<uint32_t> final_mask;
  if (cuMask.size() != 0 || info_.globalCUMask_.size() != 0) {
    std::vector<uint32_t> mask;
    if (cuMask.size() != 0 && info_.globalCUMask_.size() == 0) {
      mask = cuMask;
    } else if (cuMask.size() != 0 && info_.globalCUMask_.size() != 0) {
      for (unsigned int i = 0; i < std::min(cuMask.size(), info_.globalCUMask_.size()); i++) {
        mask.push_back(cuMask[i] & info_.globalCUMask_[i]);
      }
      bool zeroCUMask = true;
      for (auto m : mask) {
        if (m != 0) {
          zeroCUMask = false;
          break;
        }
      }
      if (zeroCUMask) {
        mask = info_.globalCUMask_;
      }
    } else {
      mask = info_.globalCUMask_;
    }

    if (settings().enableWgpMode_) {
      final_mask.resize(mask.size() * 2, 0);
      for (size_t i = 0; i < mask.size(); i++) {
        for (int j = 0; j < 16; j++) {
          if (((mask[i] >> j) & 0x1) == 0x1) {
            final_mask[2 * i] |= (0x3 << (2 * j));
          }
          if (((mask[i] >> (16 + j)) & 0x1) == 0x1) {
            final_mask[2 * i + 1] |= (0x3 << (2 * j));
          }
        }
      }
    } else {
      final_mask = mask;
    }
  }

  // Build the queue creation descriptor with priority, CU mask, and flags.
  // settings().aql_device_ring_buf_ controls queue ring buffer placement (enabled
  // by default on gfx94x/gfx125x, overridable via DEBUG_CLR_AQL_DEV_QUEUE):
  //   false - system memory
  //   true  - ring buffer in device memory if Large BAR is available.
  const bool device_mem_ring_buf =
      settings().aql_device_ring_buf_ && info_.largeBar_ && Hsa::amd_queue_create_available();
  hsa_amd_queue_create_desc_t desc = {};
  desc.version = HSA_AMD_QUEUE_CREATE_DESC_VERSION;
  desc.engine_type = HSA_AMD_QUEUE_ENGINE_COMPUTE;
  desc.queue_size_bytes = queue_size * sizeof(hsa_kernel_dispatch_packet_t);
  desc.priority = queue_priority;
  desc.callback = callbackQueue;
  desc.callback_data = this;
  desc.engine.compute.type = queue_type;
  desc.engine.compute.private_segment_size = HSA_AMD_PRIVATE_SEGMENT_SIZE_DEFAULT;
  if (device_mem_ring_buf) {
    desc.flags = static_cast<hsa_amd_queue_create_flag_t>(
        desc.flags | HSA_AMD_QUEUE_CREATE_DEVICE_MEM_RING_BUF);
  }
  if (!final_mask.empty()) {
    desc.engine.compute.cu_mask_count = static_cast<uint32_t>(final_mask.size() * 32);
    desc.engine.compute.cu_mask = final_mask.data();
  }

  while (Hsa::amd_queue_create(bkendDevice_, &desc, 1) != HSA_STATUS_SUCCESS) {
    queue_size >>= 1;
    desc.queue_size_bytes = queue_size * sizeof(hsa_kernel_dispatch_packet_t);
    if (queue_size < 64) {
      LogError("Device::acquireQueue: hsa_amd_queue_create failed!");
      if (!coop_queue && (cuMask.size() == 0)) {
        amd::ScopedLock l(active_queue_access_);
        if (queuePool_[qIndex].size() > 0) {
          bool kForceReuse = true;
          return getQueueFromPool(qIndex, kForceReuse);
        }
      }
      ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE,
              "Device::acquireQueue: hsa_amd_queue_create failed!");
      return nullptr;
    }
  }

  hsa_queue_t* queue = desc.queue;
  if (IsLogEnabled(amd::LOG_INFO, amd::LOG_QUEUE)) {
    if (!final_mask.empty()) {
      std::stringstream ss;
      ss << std::hex;
      for (int i = final_mask.size() - 1; i >= 0; i--) {
        ss << std::setfill('0') << std::setw(8) << final_mask[i];
      }
      ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
              "Created SWq=%p HWq=%p size %d priority %d cooperative %i flags 0x%x "
              "device_mem_ring_buf %i CU mask 0x%s",
              queue, queue->base_address, queue_size, queue_priority, coop_queue, desc.flags,
              device_mem_ring_buf, ss.str().c_str());
    } else {
      ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
              "Created SWq=%p HWq=%p size %d priority %d cooperative %i flags 0x%x "
              "device_mem_ring_buf %i",
              queue, queue->base_address, queue_size, queue_priority, coop_queue, desc.flags,
              device_mem_ring_buf);
    }
  }

  Hsa::profiling_set_profiler_enabled(queue, 1);

  if (!metadata_version_queried_) {
    uint8_t major = 0, minor = 0;
    hsa_amd_queue_get_info(queue,
        HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_DISPATCH_PKT_VERSION_MAJOR, &major);
    hsa_amd_queue_get_info(queue,
        HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_DISPATCH_PKT_VERSION_MINOR, &minor);
    if (major < (1 << 3) && minor < (1 << 5)) {
      metadata_version_header_ =
          (static_cast<uint32_t>(major) << HSA_AMD_METADATA_PACKET_HEADER_VERSION_MAJOR) |
          (static_cast<uint32_t>(minor) << HSA_AMD_METADATA_PACKET_HEADER_VERSION_MINOR);
    }
    metadata_version_queried_ = true;
  }

  // Query extras for the newly-created queue and insert into the lookup map
  auto populateExtras = [&]() {
    QueueExtras extras;
    extras.deviceMemRingBuf = (desc.flags & HSA_AMD_QUEUE_CREATE_DEVICE_MEM_RING_BUF) != 0;
    hsa_amd_queue_get_info(queue, HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_RING_BUFFER,
                           &extras.metadataRingBuffer);
    if (DEBUG_CLR_DIRECT_DOORBELL) {
      uint64_t db_id = 0;
      if (hsa_amd_queue_get_info(queue, HSA_AMD_QUEUE_INFO_DOORBELL_ID, &db_id) ==
              HSA_STATUS_SUCCESS &&
          db_id != 0) {
        extras.doorbellPtr = reinterpret_cast<volatile uint64_t*>(db_id);
      }
    }
    queue_extras_[queue] = extras;
  };

  if (!final_mask.empty()) {
    amd::ScopedLock l(active_queue_access_);
    populateExtras();
    return queue;
  }

  if (coop_queue) {
    amd::ScopedLock l(active_queue_access_);
    populateExtras();
    return queue;
  }

  // Add queue to the pool (including dedicated queues)
  amd::ScopedLock l(active_queue_access_);
  auto result = queuePool_[qIndex].emplace(std::make_pair(queue, QueueInfo()));
  assert(result.second && "QueueInfo already exists");
  auto& qInfo = result.first->second;
  qInfo.refCount = 1;
  qInfo.hasDedicatedQueue_ = dedicated_queue;
  populateExtras();
  ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "acquireQueue refCount: %p (%d) %s",
          result.first->first->base_address, result.first->second.refCount,
          dedicated_queue ? "(dedicated)" : "");
  if (!managed && (cuMask.size() == 0)) {
    num_queues_[qIndex]++;
  }
  return queue;
}

// ================================================================================================
bool Device::ReleaseActiveQueue(hsa_queue_t* queue, amd::CommandQueue::Priority priority) {
  uint qIndex;
  switch (priority) {
    case amd::CommandQueue::Priority::Low:
      qIndex = QueuePriority::Low;
      break;
    case amd::CommandQueue::Priority::High:
      qIndex = QueuePriority::High;
      break;
    case amd::CommandQueue::Priority::Normal:
    case amd::CommandQueue::Priority::Medium:
    default:
      qIndex = QueuePriority::Normal;
      break;
  }
  // Release a queue if the total number of allocated queues exceeds the max possible
  if (num_queues_[qIndex].load() > settings().max_hw_queues_) {
    releaseQueue(queue, std::vector<uint32_t>{}, false, true);
    return true;
  } else {
    return false;
  }
}

// ================================================================================================
void Device::releaseQueue(hsa_queue_t* queue, const std::vector<uint32_t>& cuMask, bool coop_queue,
                          bool managed) {
  // Defer cleanup operations outside the lock
  {  // Lock
    amd::ScopedLock l(active_queue_access_);
    for (uint qIndex = 0; qIndex < queuePool_.size(); ++qIndex) {
      auto& it = queuePool_[qIndex];
      auto qIter = it.find(queue);
      if (qIter != it.end()) {
        if (!managed && (cuMask.size() == 0)) {
          num_queues_[qIndex]--;
        }
        auto& qInfo = qIter->second;
        assert(qInfo.refCount > 0);
        qInfo.refCount--;
        ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "releaseQueue refCount:%p (%d)",
                qIter->first->base_address, qIter->second.refCount);
        break;  // Found and processed the queue
      }
    }
  }  // Lock release

  // hsa queues with cumask set and coop queues are not being reused. Hence, if the app uses such
  // queues, we need to destroy them when the queue is released.
  if (!cuMask.empty() || coop_queue) {
    // On the async-events thread a non-coop ~AqlQueue self-deadlocks; defer it to an app thread.
    // Coop queues are recycled synchronously via GWSRelease, so never defer them.
    if (InAsyncSignalHandler() && !coop_queue) {
      ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deferring CG enabled hardware queue %p destroy",
              queue->base_address);
      std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
      deferredQueueDestroy_.push_back(queue);
    } else {
      ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deleting CG enabled hardware queue %p ",
              queue->base_address);
      Hsa::queue_destroy(queue);
    }
  }
}

size_t Device::DeferredQueueCount() {
  std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
  return deferredQueueDestroy_.size();
}

void Device::DrainDeferredQueueDestroys() {
  // Swap out the batch under lock, then destroy without holding it: queue_destroy
  // is blocking and may run runtime callbacks.
  std::vector<hsa_queue_t*> pending;
  {
    std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
    pending.swap(deferredQueueDestroy_);
  }
  for (hsa_queue_t* queue : pending) {
    ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deleting deferred hardware queue %p", queue->base_address);
    Hsa::queue_destroy(queue);
  }
}

// ================================================================================================
Device::QueueExtras Device::GetQueueExtras(hsa_queue_t* queue) {
  amd::ScopedLock l(active_queue_access_);
  auto it = queue_extras_.find(queue);
  return (it != queue_extras_.end()) ? it->second : QueueExtras{};
}

bool Device::findLinkInfo(const amd::Device& other_device, std::vector<LinkAttrType>* link_attrs) {
  return findLinkInfo((static_cast<const roc::Device*>(&other_device))->gpuvm_segment_, link_attrs);
}

bool Device::findLinkInfo(const hsa_amd_memory_pool_t& pool,
                          std::vector<LinkAttrType>* link_attrs) {
  if ((!pool.handle) || (link_attrs == nullptr)) {
    return false;
  }

  // Retrieve the hops between 2 devices.
  int32_t hops = 0;
  hsa_status_t hsa_status = Hsa::agent_memory_pool_get_info(
      bkendDevice_, pool, HSA_AMD_AGENT_MEMORY_POOL_INFO_NUM_LINK_HOPS, &hops);

  if (hsa_status != HSA_STATUS_SUCCESS) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
             "Cannot get hops info, hsa failed with status: %d", hsa_status);
    return false;
  }

  if (hops < 0) {
    return false;
  }

  // The pool is on its agent
  if (hops == 0) {
    for (auto& link_attr : (*link_attrs)) {
      switch (link_attr.first) {
        case kLinkLinkType: {
          // No link, so type is meaningless,
          // caller should ignore it
          link_attr.second = -1;
          break;
        }
        case kLinkHopCount: {
          // no hop
          link_attr.second = 0;
          break;
        }
        case kLinkDistance: {
          // distance is zero, if no hops
          link_attr.second = 0;
          break;
        }
        case kLinkAtomicSupport: {
          // atomic support if its on the same agent
          link_attr.second = 1;
          break;
        }
        default: {
          ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                   "Invalid LinkAttribute: %d ", link_attr.first);
          return false;
        }
      }
    }
    return true;
  }

  // Retrieve link info on the pool.
  std::vector<hsa_amd_memory_pool_link_info_t> link_info(hops);
  hsa_status = Hsa::agent_memory_pool_get_info(
      bkendDevice_, pool, HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO, link_info.data());

  if (hsa_status != HSA_STATUS_SUCCESS) {
    ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
             "Cannot retrieve link info, hsa failed with status: %d", hsa_status);
    return false;
  }

  for (auto& link_attr : (*link_attrs)) {
    switch (link_attr.first) {
      case kLinkLinkType: {
        link_attr.second = static_cast<int32_t>(link_info[0].link_type);
        break;
      }
      case kLinkHopCount: {
        uint32_t distance = 0;
        // Because of Rocrs limitation hops is set to 1 always between two different devices
        // If Rocr Changes the behaviour revisit this logic
        for (size_t hop_idx = 0; hop_idx < static_cast<size_t>(hops); ++hop_idx) {
          distance += link_info[hop_idx].numa_distance;
        }
        uint32_t oneHopDistance = (link_info[0].link_type == HSA_AMD_LINK_INFO_TYPE_XGMI) ? 13 : 20;
        link_attr.second = static_cast<int32_t>(distance / oneHopDistance);
        break;
      }
      case kLinkDistance: {
        uint32_t distance = 0;
        // Sum of distances between hops
        for (size_t hop_idx = 0; hop_idx < static_cast<size_t>(hops); ++hop_idx) {
          distance += link_info[hop_idx].numa_distance;
        }
        link_attr.second = static_cast<int32_t>(distance);
        break;
      }
      case kLinkAtomicSupport: {
        // if either of the atomic is supported
        link_attr.second = static_cast<int32_t>(link_info[0].atomic_support_64bit ||
                                                link_info[0].atomic_support_32bit);
        break;
      }
      default: {
        ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
                 "Invalid LinkAttribute: %d ", link_attr.first);
        return false;
      }
    }
  }

  return true;
}

// ================================================================================================
void Device::getGlobalCUMask(std::string_view cuMaskStr) {
  if (cuMaskStr.length() != 0) {
    std::string_view pre = cuMaskStr.substr(0, 2);
    if (pre.compare("0x") == 0 || pre.compare("0X") == 0) {
      cuMaskStr = cuMaskStr.substr(2, cuMaskStr.length());
    }

    int end = cuMaskStr.length();

    // the number of current physical CUs compressed in 4-bits
    size_t compPhysicalCUs = static_cast<size_t>(
        (settings().enableWgpMode_ ? info_.maxComputeUnits_ * 2 : info_.maxComputeUnits_) / 4);

    // the number of final available compute units after applying the requested CU mask
    uint32_t availCUs = 0;

    // read numCharToRead characters (8 or less) from the cuMask string each time, convert
    // it into hex, and store it into the globalCUMask_. If the length of the cuMask string
    // is more than the compressed physical available CUs, ignore the rest
    for (unsigned i = 0; i < std::min(cuMaskStr.length(), compPhysicalCUs); i += 8) {
      int numCharToRead = (i + 8 <= compPhysicalCUs) ? 8 : compPhysicalCUs - 8;
      std::string_view temp =
          cuMaskStr.substr(std::max(0, end - numCharToRead), std::min(numCharToRead, end));
      end -= numCharToRead;
      char *endOfConverted = nullptr;
      unsigned long ul = std::strtoul(temp.data(), &endOfConverted, 16);
      if (endOfConverted == temp.data()) { // no conversion could be performed
        info_.globalCUMask_ = {};
        availCUs = 0;
        break;
      }
      info_.globalCUMask_.push_back(static_cast<uint32_t>(ul));
      // count number of set bits in ul to find the number of active CUs
      // in each iteration
      while (ul) {
        ul &= (ul - 1);
        availCUs++;
      }
    }
    // update the maxComputeUnits_ based on the requested CU mask
    if (availCUs != 0 && availCUs < compPhysicalCUs * 4) {
      info_.maxComputeUnits_ = settings().enableWgpMode_ ? availCUs / 2 : availCUs;
    } else {
      info_.globalCUMask_ = {};
    }
  } else {
    info_.globalCUMask_ = {};
  }
}

// ================================================================================================
device::Signal* Device::createSignal() const { return new roc::Signal(); }

// ================================================================================================
device::Signal* Device::createIpcSignal() const { return new roc::IpcSignal(); }

// ================================================================================================
static std::string GetLocalHostName() {
#if defined(__linux__)
  char buf[HOST_NAME_MAX + 1];
  buf[HOST_NAME_MAX] = '\0';
  if (gethostname(buf, sizeof(buf)) == 0) return buf;
#endif
  return "";
}

// ================================================================================================
hsa_status_t Device::BackendErrorCallBackHandler(const hsa_amd_event_t* event, void* data) {
  cl_int gpu_error = CL_SUCCESS;
  switch (event->event_type) {
    case HSA_AMD_GPU_MEMORY_FAULT_EVENT: {
      gpu_error = CL_INVALID_MEM_OBJECT;
      LogError("Memory Fault Error");
      std::string hostnameStr = GetLocalHostName();
      std::string host_tag = hostnameStr.empty() ? "" : "host: " + hostnameStr + ", ";
      for (auto* amd_dev : amd::Device::devices()) {
        if (amd_dev->type() != CL_DEVICE_TYPE_GPU) continue;
        Device* roc_dev = reinterpret_cast<Device*>(amd_dev);
        for (auto it : roc_dev->vgpus()) {
          roc::VirtualGPU* vgpu = reinterpret_cast<roc::VirtualGPU*>(it);
          bool faulted = false;
          if (vgpu->gpu_queue() != nullptr &&
              hsa_amd_queue_get_info(vgpu->gpu_queue(),
                                     HSA_AMD_QUEUE_INFO_VM_FAULT_STATUS,
                                     &faulted) == HSA_STATUS_SUCCESS &&
              faulted) {
            std::string kernelName = vgpu->AnalyzeAqlQueue();
            const char* kname = kernelName.c_str();
            ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
                    "Memory Fault Error [%sGPU index: %u, "
                    "faulting addr: 0x%" PRIx64 ", kernel: %s]",
                    host_tag.c_str(), roc_dev->index(),
                    event->memory_fault.virtual_address, kname);
          }
        }
      }
      break;
    }
    case HSA_AMD_GPU_HW_EXCEPTION_EVENT:
      gpu_error = CL_INVALID_OPERATION;
      LogError("HW Exception Error");
      break;
    case HSA_AMD_GPU_MEMORY_ERROR_EVENT:
      gpu_error = CL_DEVICE_NOT_AVAILABLE;
      LogError("GPU Memory Error");
      break;
    case HSA_AMD_SYSTEM_SHUTDOWN_EVENT:
      // This is not a fatal error just ignore it.
      return HSA_STATUS_SUCCESS;
    default:
      gpu_error = CL_DEVICE_NOT_AVAILABLE;
      LogError("Unknown Event Type ");
      break;
  }

  // Execute the default handler if a GPU core file should be generated ...
  if (amd::Os::DumpCoreFile() || !HIP_SKIP_ABORT_ON_GPU_ERROR) {
    return HSA_STATUS_ERROR;
  }

  gpu_error_.store(gpu_error, std::memory_order_relaxed);
  return HSA_STATUS_SUCCESS;
}

// ================================================================================================
void Device::RegisterBackendErrorCb() {
  // Register ROCclr Error Callback
  hsa_status_t hsa_error = HSA_STATUS_SUCCESS;
  hsa_error = Hsa::register_system_event_handler(BackendErrorCallBackHandler, nullptr);
  if (hsa_error != HSA_STATUS_SUCCESS) {
    LogError("Cannot Register Call back event handler");
  }
}
// ================================================================================================
amd::Memory* Device::GetArenaMemObj(const void* ptr, size_t& offset, size_t size) {
  // Only create arena_mem_object if CPU memory is accessible from HMM
  // or if runtime received an interop from another ROCr's client
  // Disable arena for XNACK
  hsa_amd_pointer_info_t ptr_info = {};
  ptr_info.size = sizeof(hsa_amd_pointer_info_t);
  if (!IsValidAllocation(ptr, size, &ptr_info)) {
    return nullptr;
  }

  if (arena_mem_obj_ == nullptr) {
    arena_mem_obj_ = new (context()) amd::ArenaMemory(context());
    if ((arena_mem_obj_ != nullptr) && !arena_mem_obj_->create(nullptr)) {
      LogError("Arena Memory Creation failed!");
      arena_mem_obj_->release();
      arena_mem_obj_ = nullptr;
    }
    if (arena_mem_obj_ == nullptr) {
      return arena_mem_obj_;
    }
  }

  // Calculate the offset of the pointer.
  const void* dev_ptr = reinterpret_cast<void*>(
      arena_mem_obj_->getDeviceMemory(*arena_mem_obj_->getContext().devices()[0])
          ->virtualAddress());
  offset = reinterpret_cast<size_t>(ptr) - reinterpret_cast<size_t>(dev_ptr);

  return arena_mem_obj_;
}

// ================================================================================================
void Device::ReleaseGlobalSignal(void* signal) const {
  if (signal != nullptr) {
    reinterpret_cast<ProfilingSignal*>(signal)->release();
  }
}

// ================================================================================================
void Device::RetainGlobalSignal(void* signal) const {
  if (signal != nullptr) {
    reinterpret_cast<ProfilingSignal*>(signal)->retain();
  }
}

// ================================================================================================
bool Device::CreateHwEvents(int count, std::vector<void*>& hw_events) const {
  hw_events.resize(count, nullptr);
  for (int i = 0; i < count; ++i) {
    ProfilingSignal* ps = new ProfilingSignal();
    if (HSA_STATUS_SUCCESS !=
        Hsa::signal_create(1, 0, nullptr, HSA_AMD_SIGNAL_AMD_GPU_ONLY, &ps->signal_)) {
      delete ps;
      for (int j = 0; j < i; ++j) {
        reinterpret_cast<ProfilingSignal*>(hw_events[j])->release();
        hw_events[j] = nullptr;
      }
      return false;
    }
    hw_events[i] = ps;
  }
  return true;
}

// ================================================================================================
void Device::DestroyHwEvent(void* hw_event) const {
  ReleaseGlobalSignal(hw_event);
}

// ================================================================================================
void Device::ResetHwEvents(const std::vector<void*>& hw_events) const {
  // Re-arm pooled signals for reuse by a new graph launch. The caller
  // guarantees these signals belong to a completed (drained) launch, so this
  // cannot corrupt an in-flight launch. Avoids signal_create/destroy on the
  // hot launch path.
  for (void* hw_event : hw_events) {
    if (hw_event != nullptr) {
      auto* ps = reinterpret_cast<ProfilingSignal*>(hw_event);
      Hsa::signal_silent_store_relaxed(ps->signal_, 1);
      ps->flags_.done_ = true;
      ps->ResetCachedTiming();
    }
  }
}

// ================================================================================================
void Device::QuiesceHwEvents(const std::vector<void*>& hw_events) const {
  // Pooled signals rest in the armed state (value 1). Before destruction, store
  // the completed value (0) so ~ProfilingSignal does not block waiting on a
  // signal that is armed but idle (no GPU work will ever drain it).
  for (void* hw_event : hw_events) {
    if (hw_event != nullptr) {
      auto* ps = reinterpret_cast<ProfilingSignal*>(hw_event);
      Hsa::signal_silent_store_relaxed(ps->signal_, 0);
    }
  }
}

// ================================================================================================
uint8_t* Device::CreateBarrierPacket() const {
  static constexpr uint16_t kBarrierNopHeader =
      (HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE) |
      (1 << HSA_PACKET_HEADER_BARRIER) |
      (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) |
      (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE);

  static_assert(sizeof(hsa_barrier_and_packet_t) == 64, "AQL packet size must be 64 bytes");
  auto* raw = new uint8_t[64]();
  auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
  pkt->header = kBarrierNopHeader;
  return raw;
}

// ================================================================================================
void Device::ApplyHwEventPatches(const std::vector<HwEventPatch>& patches,
                                 const std::vector<void*>& hw_events) const {
  for (const auto& patch : patches) {
    auto* ps = reinterpret_cast<ProfilingSignal*>(hw_events[patch.hw_event_index]);
    hsa_signal_t sig = ps->signal_;

    // Patch the flat buffer copy (dispatched to GPU) directly.
    // The original dispatchPackets pointer is retained for UpdateAQLPacket matching.
    uint8_t* raw = patch.flat_packet ? patch.flat_packet : patch.packet;

    if (patch.dep_slot == HwEventPatch::kExtDispatchDepSignal) {
      // Patch dep_signal in hsa_amd_ext_kernel_dispatch_packet_t via offset-based
      static constexpr size_t kExtDepSignalOffset =
          offsetof(hsa_amd_ext_kernel_dispatch_packet_t, dep_signal);
      memcpy(raw + kExtDepSignalOffset, &sig, sizeof(sig));
    } else if (patch.dep_slot == HwEventPatch::kCompletionSignal) {
      auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
      pkt->completion_signal = sig;

      // Prepare this signal for profiling: mark it as active and classify
      // the packet type so checkGpuTime → addTimestamps only fires for
      // kernel dispatches (not synthetic barriers).
      ps->flags_.done_ = false;
      // Record the queue this patched dispatch signal runs on (resolved from the
      // owning segment's stream at launch) so profiling attributes it to the
      // right stream rather than the graph launch stream.
      ps->queue_index_ = patch.queue_index;
      uint16_t hdr;
      memcpy(&hdr, patch.packet, sizeof(hdr));
      uint8_t pktType = hdr & ((1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1);
      // A kernel dispatch could be a vendor-specific ext-kernel-dispatch
      // packet, identified by amd_format (byte 2).  Classify it as a dispatch so
      // the patched last-node completion signal contributes its GPU timing like
      // every other graph kernel node.
      const uint8_t amdFormat = patch.packet[2];
      ps->flags_.isPacketDispatch_ =
          (pktType == HSA_PACKET_TYPE_KERNEL_DISPATCH) ||
          (pktType == HSA_PACKET_TYPE_VENDOR_SPECIFIC &&
           amdFormat == HSA_AMD_PACKET_TYPE_EXT_KERNEL_DISPATCH);
    } else {
      // dep_slot >= 0: patch a barrier's dependency signal slot (cross-segment wait)
      auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
      pkt->dep_signal[patch.dep_slot] = sig;
    }
  }
}

// ================================================================================================
bool Device::CreateUserEvent(amd::UserEvent* event) const {
  std::unique_ptr<ProfilingSignal> signal(new ProfilingSignal());
  if ((signal == nullptr) ||
      (HSA_STATUS_SUCCESS != Hsa::signal_create(0, 0, nullptr, &signal->signal_))) {
    return false;
  }
  Hsa::signal_silent_store_relaxed(signal->signal_, kInitSignalValueOne);
  event->SetHwEvent(signal.release());
  return true;
}

// ================================================================================================
void Device::SetUserEvent(amd::UserEvent* event) const {
  auto signal = reinterpret_cast<ProfilingSignal*>(event->HwEvent());
  assert(signal != nullptr && "Can't have user event without hw event!");
  Hsa::signal_silent_store_relaxed(signal->signal_, 0);
}

// ================================================================================================
bool Device::IsValidAllocation(const void* dev_ptr, size_t size, hsa_amd_pointer_info_t* ptr_info) {
  // Query ptr type to see if it's a HMM allocation
  hsa_status_t status =
      Hsa::pointer_info(const_cast<void*>(dev_ptr), ptr_info, nullptr, nullptr, nullptr);
  // The call should never fail in ROCR, but just check for an error and continue
  if (status != HSA_STATUS_SUCCESS) {
    LogError("hsa_amd_pointer_info() failed");
  }

  if (ptr_info->type == HSA_EXT_POINTER_TYPE_RESERVED_ADDR) {
    return false;
  }

  // Return false for pinned memory. A true return may result in a race because
  // ROCclr may attempt to do a pin/copy/unpin underneath in a multithreaded environment
  if (ptr_info->type == HSA_EXT_POINTER_TYPE_LOCKED) {
    return false;
  }

  if (ptr_info->type != HSA_EXT_POINTER_TYPE_UNKNOWN) {
    if ((size != 0) && ((reinterpret_cast<const_address>(dev_ptr) -
                         reinterpret_cast<const_address>(ptr_info->agentBaseAddress)) > size)) {
      return false;
    }
    return true;
  }
  return false;
}

// ================================================================================================
void Device::HiddenHeapAlloc(const VirtualGPU& gpu) {
  auto HeapAllocOnly = [this, &gpu]() -> bool {
    // Allocate initial heap for device memory allocator
    static constexpr size_t HeapBufferSize = 128 * Ki;
    heap_buffer_ = createMemory(HeapBufferSize);
    if (initial_heap_size_ != 0) {
      initial_heap_size_ = amd::alignUp(initial_heap_size_, 2 * Mi);
      initial_heap_buffer_ = createMemory(initial_heap_size_);
    }
    if (heap_buffer_ == nullptr) {
      LogError("Heap buffer allocation failed!");
      return false;
    }
    return true;
  };
  std::call_once(heap_allocated_, HeapAllocOnly);
}

// ================================================================================================
void Device::HiddenHeapInit(const VirtualGPU& gpu) {
  auto HeapZeroOut = [this, &gpu]() -> bool {
    static constexpr size_t HeapBufferSize = 128 * Ki;
    bool result = static_cast<const KernelBlitManager&>(gpu.blitMgr())
                      .initHeap(heap_buffer_, initial_heap_buffer_, HeapBufferSize,
                                initial_heap_size_ / (2 * Mi));

    return result;
  };
  std::call_once(heap_initialized_, HeapZeroOut);
}

// ================================================================================================
uint32_t Device::SdmaEngineAllocator::AllocateEngine(VirtualGPU* vgpu, HwQueueEngine engine_type,
                                                      hsa_agent_t peerAgent, hsa_agent_t copyAgent) {
  std::scoped_lock lock(lock_);

  // Get valid engine mask based on operation type (read vs write)
  uint32_t validEngineMask = (engine_type == HwQueueEngine::SdmaD2H)
                              ? device_.maxSdmaReadMask_
                              : device_.maxSdmaWriteMask_;

  // Simple round-robin path if all engines have equal bandwidth
  // Disabled by default - use preferred engine logic for current GPUs
  constexpr bool kUseSimpleRR = false;

  if (kUseSimpleRR) {
    // Simple round-robin: just cycle through valid engines
    // This will be enabled for future GPUs where engine selection doesn't matter
    if (validEngineMask == 0) {
      ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
              "No valid SDMA engines for VirtualGPU %p", vgpu);
      return 0;
    }

    // Cycle through bit positions, find next valid engine
    uint32_t start_bit = next_rr_engine_.fetch_add(1, std::memory_order_relaxed);
    uint32_t selected_mask = 0;

    // Try up to 32 positions to find a valid engine
    for (uint32_t i = 0; i < 32; ++i) {
      uint32_t bit = (start_bit + i) % 32;
      uint32_t mask = 1u << bit;
      if (validEngineMask & mask) {
        selected_mask = mask;
        break;
      }
    }

    vgpu_to_engine_[vgpu] = selected_mask;

    ClPrint(amd::LOG_INFO, amd::LOG_COPY,
            "Assigned SDMA engine (simple RR) to VirtualGPU %p: mask=0x%x, engine_type=%d",
            vgpu, selected_mask, engine_type);

    return selected_mask;
  }

  // Current path: Query HSA for engine status and preferences
  uint32_t freeEngineMask = 0;
  uint32_t preferredMask = 0;
  hsa_status_t status = HSA_STATUS_SUCCESS;

  // Query current engine status
  status = Hsa::memory_copy_engine_status(peerAgent, copyAgent, &freeEngineMask);
  if (status == HSA_STATUS_SUCCESS) {
    // Query preferred (high-bandwidth) engines
    status = Hsa::memory_get_preferred_copy_engine(peerAgent, copyAgent, &preferredMask);
  }

  // Constrain to valid engines
  freeEngineMask &= validEngineMask;
  preferredMask &= validEngineMask;

  ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
          "Engine query for VirtualGPU %p: status=%x, free_mask=0x%x, preferred_mask=0x%x, "
          "valid_mask=0x%x, engine_type=%d",
          vgpu, status, freeEngineMask, preferredMask, validEngineMask, engine_type);

  uint32_t candidate_mask = 0;
  uint32_t allocated_mask = 0;

  // For inter-GPU copies, strongly prefer the recommended engines
  bool is_inter_gpu = (engine_type == HwQueueEngine::SdmaP2P);

  if (is_inter_gpu && (preferredMask != 0)) {
    // Inter-GPU: prioritize preferredMask, even if engines are already allocated
    candidate_mask = validEngineMask & preferredMask;

    ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
            "Inter-GPU copy for VirtualGPU %p: prioritizing preferred engines, "
            "candidate_mask=0x%x",
            vgpu, candidate_mask);
  } else {
    // Regular read/write/intra: enforce exclusivity (don't share engines)
    // Build a mask of engines already allocated to other VirtualGPUs
    for (const auto& pair : vgpu_to_engine_) {
      allocated_mask |= pair.second;
    }

    uint32_t available_mask = validEngineMask & ~allocated_mask;

    if (available_mask == 0) {
      ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
              "No unallocated SDMA engines available for VirtualGPU %p, engine_type=%d "
              "(valid_mask=0x%x, allocated_mask=0x%x)",
              vgpu, engine_type, validEngineMask, allocated_mask);
      return 0;
    }

    // Prefer high-bandwidth (recommended) engines if available
    candidate_mask = available_mask & preferredMask;
    if (candidate_mask == 0) {
      candidate_mask = available_mask;
    }
  }

  if (candidate_mask == 0) {
    ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
            "No candidate SDMA engines for VirtualGPU %p, engine_type=%d",
            vgpu, engine_type);
    return 0;
  }

  // Select the lowest bit (first available engine)
  uint32_t selected_mask = candidate_mask & (~candidate_mask + 1);

  // Update the map
  vgpu_to_engine_[vgpu] = selected_mask;

  ClPrint(amd::LOG_INFO, amd::LOG_COPY,
          "Assigned SDMA engine to VirtualGPU %p: mask=0x%x, engine_type=%d, "
          "valid_mask=0x%x, preferred_mask=0x%x, allocated_mask=0x%x, is_inter_gpu=%d",
          vgpu, selected_mask, engine_type, validEngineMask, preferredMask,
          allocated_mask, is_inter_gpu);

  return selected_mask;
}

// ================================================================================================
void Device::SdmaEngineAllocator::ReleaseEngine(VirtualGPU* vgpu) {
  std::scoped_lock lock(lock_);

  auto it = vgpu_to_engine_.find(vgpu);
  if (it != vgpu_to_engine_.end()) {
    ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
            "Released SDMA engine for VirtualGPU %p: mask=0x%x",
            vgpu, it->second);
    vgpu_to_engine_.erase(it);
  }
}

// ================================================================================================
void Device::AddKernel(Kernel& gpuKernel) const {
  std::scoped_lock lock(vgpusAccess());
  kernel_map_.insert({gpuKernel.KernelCodeHandle(), gpuKernel});
}

// ================================================================================================
void Device::RemoveKernel(Kernel& gpuKernel) const {
  if (gpuKernel.KernelCodeHandle() != 0) {
    std::scoped_lock lock(vgpusAccess());
    auto it = kernel_map_.find(gpuKernel.KernelCodeHandle());
    if (it != kernel_map_.end()) {
      // Remove the old mapping
      kernel_map_.erase(it);
    }
  }
}

// ================================================================================================
ProfilingSignal::~ProfilingSignal() {
  if (signal_.handle != 0) {
    if (Hsa::signal_load_relaxed(signal_) > 0
        && !(HIP_SKIP_ABORT_ON_GPU_ERROR && amd::Device::IsGPUInError())) {
      LogError("Runtime shouldn't destroy a signal that is still busy!");
      if (Hsa::signal_wait_scacquire(signal_, HSA_SIGNAL_CONDITION_LT, kInitSignalValueOne,
                                    kUnlimitedWait, HSA_WAIT_STATE_BLOCKED) != 0) {
      }
    }
    Hsa::signal_destroy(signal_);
  }
}

// ================================================================================================
cl_int ConvertHSAErrorIntoCLError(hsa_status_t hsa_status) {
  cl_int cl_error = CL_SUCCESS;
  switch (hsa_status) {
    case HSA_STATUS_ERROR_OUT_OF_RESOURCES:
      cl_error = CL_OUT_OF_RESOURCES;
      break;
    case HSA_STATUS_ERROR_EXCEPTION:
      cl_error = CL_INVALID_OPERATION;
      break;
    case HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS:
      cl_error = CL_INVALID_ARG_VALUE;
      break;
    case HSA_STATUS_ERROR_INVALID_ALLOCATION:
      cl_error = CL_MEM_OBJECT_ALLOCATION_FAILURE;
      break;
    case HSA_STATUS_ERROR_INVALID_CODE_OBJECT:
      cl_error = CL_INVALID_PROGRAM;
      break;
    case HSA_STATUS_ERROR_INVALID_PACKET_FORMAT:
      cl_error = CL_INVALID_OPERATION;
      break;
    case HSA_STATUS_ERROR_INVALID_ARGUMENT:
      cl_error = CL_INVALID_ARG_VALUE;
      break;
    case HSA_STATUS_ERROR_INVALID_ISA:
      cl_error = CL_INVALID_KERNEL;
      break;
    case (hsa_status_t)HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION:
      cl_error = CL_BUILD_PROGRAM_FAILURE;
      break;
    case (hsa_status_t)HSA_STATUS_ERROR_MEMORY_FAULT:
      cl_error = CL_INVALID_MEM_OBJECT;
      break;
    case (hsa_status_t)HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION:
      cl_error = CL_INVALID_MEM_OBJECT;
      break;
    case (hsa_status_t)HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS:
      cl_error = CL_OUT_OF_RESOURCES;
      break;
    case HSA_STATUS_ERROR:
    default:
      cl_error = CL_DEVICE_NOT_AVAILABLE;
      break;
  }
  return cl_error;
}

// ================================================================================================
void callbackQueue(hsa_status_t status, hsa_queue_t* queue, void* data) {
  if (status != HSA_STATUS_SUCCESS && status != HSA_STATUS_INFO_BREAK) {
    Device* dev = reinterpret_cast<Device*>(data);
    std::string hostnameStr = GetLocalHostName();
    std::string host_tag = hostnameStr.empty() ? "" : "host: " + hostnameStr + ", ";
    std::string kernelName;
    for (auto it : dev->vgpus()) {
      roc::VirtualGPU* vgpu = reinterpret_cast<roc::VirtualGPU*>(it);
      if (vgpu->gpu_queue() == queue) {
        kernelName = vgpu->AnalyzeAqlQueue();
      }
    }
    // Abort on device exceptions.
    const char* errorMsg = 0;
    Hsa::status_string(status, &errorMsg);
    const char* kname = kernelName.c_str();
    if (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES) {
      size_t global_available_mem = 0;
      if (HSA_STATUS_SUCCESS !=
          Hsa::agent_get_info(dev->getBackendDevice(),
                             static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_AVAIL),
                             &global_available_mem)) {
        LogError("HSA_AMD_AGENT_INFO_MEMORY_AVAIL query failed.");
      }
      ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
              "Callback: Queue %p Aborting with error : %s Code: 0x%x Available Free mem : %zu MB"
              " [%sGPU index: %u, kernel: %s]",
              queue->base_address, errorMsg, status, global_available_mem / Mi,
              host_tag.c_str(), dev->index(), kname);
    } else {
      ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
              "Callback: Queue %p aborting with error : %s code: 0x%x"
              " [%sGPU index: %u, kernel: %s]",
              queue->base_address, errorMsg, status, host_tag.c_str(), dev->index(), kname);
    }

    // Core dumps generally provide limited value for OOM or register overflow,
    // so do not let DumpCoreFile() be the reason to abort in those cases. These
    // errors should still honor HIP_SKIP_ABORT_ON_GPU_ERROR consistently.
    const bool is_oom = (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES);
    // HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS is a dispatch-time
    // failure (kernel exceeds hardware register limits), not a corruption.
    // Treat it as recoverable rather than aborting.
    const bool invalid_dispatch =
        (status == (hsa_status_t)HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS);
    const bool should_abort =
      (is_oom || invalid_dispatch)
        ? !HIP_SKIP_ABORT_ON_GPU_ERROR
        : (amd::Os::DumpCoreFile() || !HIP_SKIP_ABORT_ON_GPU_ERROR);

    if (should_abort) {
      abort();
    }
    amd::Device::gpu_error_.store(ConvertHSAErrorIntoCLError(status), std::memory_order_relaxed);
  }
}

// ================================================================================================
#if defined(__clang__)
#if __has_feature(address_sanitizer)
device::UriLocator* Device::createUriLocator() const { return new roc::UriLocator(); }
#endif
#endif

// ================================================================================================
namespace {

inline bool MapHandleType(amd::ExternalSemaphoreHandleType t,
                          hsa_amd_external_semaphore_handle_type_t* out) {
  switch (t) {
    case amd::ExternalSemaphoreHandleType::OpaqueFd:
      *out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD;
      return true;
    case amd::ExternalSemaphoreHandleType::OpaqueWin32:
      *out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32;
      return true;
    case amd::ExternalSemaphoreHandleType::OpaqueWin32Kmt:
      *out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT;
      return true;
    default:
      // Reject amd:: enum values with no HSA counterpart
      // (D3D11/D3D12Fence, NvSciSync, KeyedMutex*, TimelineSemaphore*).
      // Types we do map (e.g. OpaqueFd, OpaqueWin32Kmt) may still fail
      // at libhsakmt import time on platforms without KMD support;
      // capability ground-truth lives in the lower layer.
      return false;
  }
}

}  // namespace

// ================================================================================================
bool Device::importExtSemaphore(void** extSemaphore, const amd::Os::FileDesc& handle,
                                amd::ExternalSemaphoreHandleType sem_handle_type) {
  if (extSemaphore == nullptr) return false;

  hsa_amd_external_semaphore_handle_descriptor_t desc = {};
  if (!MapHandleType(sem_handle_type, &desc.type)) return false;

  // Populate the descriptor union by the *mapped* HSA type, not by
  // build platform: amd::Os::FileDesc is void* on Windows and int on
  // POSIX, but the HSA layer reads the union member dictated by
  // desc.type. Reading a member that wasn't written is undefined
  // behaviour, so reject combinations where the platform's FileDesc
  // shape doesn't match the requested handle type.
  switch (desc.type) {
    case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32:
    case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT:
#if defined(_WIN32)
      desc.handle.win32_handle = handle;
      break;
#else
      // Win32 NT handles aren't routable through a non-Windows KMD.
      return false;
#endif
    case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD:
#if defined(_WIN32)
      // POSIX FDs aren't routable through WDDM.
      return false;
#else
      desc.handle.fd = handle;
      break;
#endif
    default:
      return false;
  }

  // Heap-allocated holder so the caller gets an opaque pointer for
  // DestroyExtSemaphore. Storing the hsa_amd_external_semaphore_t
  // preserves the syncobj identity that submitExternalSemaphoreCmd
  // will consume once the queue signal/wait submission path lands.
  auto* holder = new hsa_amd_external_semaphore_t{};

  hsa_status_t s = hsa_amd_external_semaphore_handle_open(bkendDevice_, &desc, holder);
  if (s != HSA_STATUS_SUCCESS) {
    delete holder;
    return false;
  }

  *extSemaphore = holder;
  return true;
}

// ================================================================================================
void Device::DestroyExtSemaphore(void* extSemaphore) {
  if (extSemaphore == nullptr) return;
  auto* holder = static_cast<hsa_amd_external_semaphore_t*>(extSemaphore);
  hsa_amd_external_semaphore_handle_close(*holder);
  delete holder;
}

}  // namespace amd::roc