TheRock/compiler/amd-llvm/offload/plugins-nextgen/level_zero/src/L0Device.cpp

//===--- Level Zero Target RTL Implementation -----------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// GenericDevice instatiation for SPIR-V/Xe machine.
//
//===----------------------------------------------------------------------===//

#include "L0Device.h"
#include "L0Defs.h"
#include "L0Interop.h"
#include "L0Plugin.h"
#include "L0Program.h"
#include "L0Trace.h"

#include "GlobalHandler.h"
#include "OffloadAPI.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Object/ELF.h"

namespace llvm::omp::target::plugin {

// clang-format off
/// Mapping from device arch to GPU runtime's device identifiers.
static struct {
  DeviceArchTy arch;
  PCIIdTy ids[10];
} DeviceArchMap[] = {{DeviceArchTy::DeviceArch_Gen,
                      {PCIIdTy::SKL,
                       PCIIdTy::KBL,
                       PCIIdTy::CFL, PCIIdTy::CFL_2,
                       PCIIdTy::ICX,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_Gen,
                      {PCIIdTy::TGL, PCIIdTy::TGL_2,
                       PCIIdTy::DG1,
                       PCIIdTy::RKL,
                       PCIIdTy::ADLS,
                       PCIIdTy::RTL,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_XeLPG,
                      {PCIIdTy::MTL,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_XeHPC,
                      {PCIIdTy::PVC,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_XeHPG,
                      {PCIIdTy::DG2_ATS_M,
                       PCIIdTy::DG2_ATS_M_2,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_Xe2LP,
                      {PCIIdTy::LNL,
                       PCIIdTy::None}},
                     {DeviceArchTy::DeviceArch_Xe2HP,
                      {PCIIdTy::BMG,
                       PCIIdTy::None}},
};
constexpr int DeviceArchMapSize = sizeof(DeviceArchMap) / sizeof(DeviceArchMap[0]);
// clang-format on

DeviceArchTy L0DeviceTy::computeArch() const {
  const auto PCIDeviceId = getPCIId();
  if (PCIDeviceId == 0) {
    ODBG(OLDT_Device) << "Warning: Cannot decide device arch for " << getName()
                      << ".";
    return DeviceArchTy::DeviceArch_None;
  }

  for (int ArchIndex = 0; ArchIndex < DeviceArchMapSize; ArchIndex++) {
    for (int i = 0;; i++) {
      const auto Id = DeviceArchMap[ArchIndex].ids[i];
      if (Id == PCIIdTy::None)
        break;
      auto maskedId = static_cast<PCIIdTy>(PCIDeviceId & 0xFF00);
      if (maskedId == Id)
        return DeviceArchMap[ArchIndex].arch; // Exact match or prefix match.
    }
  }

  ODBG(OLDT_Device) << "Warning: Cannot decide device arch for " << getName()
                    << ".";
  return DeviceArchTy::DeviceArch_None;
}

bool L0DeviceTy::isDeviceIPorNewer(uint32_t Version) const {
  ze_device_ip_version_ext_t IPVersion{};
  IPVersion.stype = ZE_STRUCTURE_TYPE_DEVICE_IP_VERSION_EXT;
  IPVersion.pNext = nullptr;
  ze_device_properties_t DevicePR{};
  DevicePR.stype = ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES;
  DevicePR.pNext = &IPVersion;
  CALL_ZE_RET(false, zeDeviceGetProperties, zeDevice, &DevicePR);
  return IPVersion.ipVersion >= Version;
}

/// Get default compute group ordinal. Returns Ordinal-NumQueues pair.
std::pair<uint32_t, uint32_t> L0DeviceTy::findComputeOrdinal() {
  std::pair<uint32_t, uint32_t> Ordinal{MaxOrdinal, 0};
  uint32_t Count = 0;
  const auto zeDevice = getZeDevice();
  CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
              nullptr);
  ze_command_queue_group_properties_t Init{
      ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0};
  std::vector<ze_command_queue_group_properties_t> Properties(Count, Init);
  CALL_ZE_RET(Ordinal, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
              Properties.data());
  for (uint32_t I = 0; I < Count; I++) {
    // TODO: add a separate set of ordinals for compute queue groups which
    // support cooperative kernels.
    if (Properties[I].flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) {
      Ordinal.first = I;
      Ordinal.second = Properties[I].numQueues;
      break;
    }
  }
  if (Ordinal.first == MaxOrdinal)
    ODBG(OLDT_Device) << "Error: no command queues are found";

  return Ordinal;
}

/// Check if device supports cooperative kernels by checking if any command
/// queue group has the cooperative kernels flag set.
bool L0DeviceTy::checkCooperativeKernelSupport() {
  uint32_t Count = 0;
  const auto zeDevice = getZeDevice();
  CALL_ZE_RET(false, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
              nullptr);

  std::vector<ze_command_queue_group_properties_t> Properties(
      Count,
      {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES, nullptr, 0, 0, 0});
  CALL_ZE_RET(false, zeDeviceGetCommandQueueGroupProperties, zeDevice, &Count,
              Properties.data());

  for (auto &Property : Properties)
    if (Property.flags &
        ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COOPERATIVE_KERNELS)
      return true;

  return false;
}

void L0DeviceTy::reportDeviceInfo() const {
  ODBG_OS(OLDT_Device, [&](llvm::raw_ostream &O) {
    O << "Device " << DeviceId << " information\n"
      << "-- Name                         : " << getName() << "\n"
      << "-- PCI ID                       : "
      << llvm::format("0x%" PRIx32, getPCIId()) << "\n"
      << "-- UUID                         : " << getUuid().data() << "\n"
      << "-- Number of total EUs          : " << getNumEUs() << "\n"
      << "-- Number of threads per EU     : " << getNumThreadsPerEU() << "\n"
      << "-- EU SIMD width                : " << getSIMDWidth() << "\n"
      << "-- Number of EUs per subslice   : " << getNumEUsPerSubslice() << "\n"
      << "-- Number of subslices per slice: " << getNumSubslicesPerSlice()
      << "\n"
      << "-- Number of slices             : " << getNumSlices() << "\n"
      << "-- Local memory size (bytes)    : " << getMaxSharedLocalMemory()
      << "\n"
      << "-- Global memory size (bytes)   : " << getGlobalMemorySize() << "\n"
      << "-- Cache size (bytes)           : " << getCacheSize() << "\n"
      << "-- Max clock frequency (MHz)    : " << getClockRate() << "\n";
  });
}

Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
  const auto &Options = getPlugin().getOptions();

  uint32_t Count = 1;
  const auto zeDevice = getZeDevice();
  CALL_ZE_RET_ERROR(zeDeviceGetProperties, zeDevice, &DeviceProperties);
  CALL_ZE_RET_ERROR(zeDeviceGetComputeProperties, zeDevice, &ComputeProperties);
  CALL_ZE_RET_ERROR(zeDeviceGetMemoryProperties, zeDevice, &Count,
                    &MemoryProperties);
  CALL_ZE_RET_ERROR(zeDeviceGetCacheProperties, zeDevice, &Count,
                    &CacheProperties);
  CALL_ZE_RET_ERROR(zeDeviceGetModuleProperties, zeDevice, &ModuleProperties);

  DeviceName = std::string(DeviceProperties.name);

  ODBG(OLDT_Device) << "Found a GPU device, Name = " << DeviceProperties.name;

  DeviceArch = computeArch();
  // Default allocation kind for this device.
  AllocKind = isDiscreteDevice() ? TARGET_ALLOC_DEVICE : TARGET_ALLOC_SHARED;

  ze_kernel_indirect_access_flags_t Flags =
      (AllocKind == TARGET_ALLOC_DEVICE)
          ? ZE_KERNEL_INDIRECT_ACCESS_FLAG_DEVICE
          : ZE_KERNEL_INDIRECT_ACCESS_FLAG_SHARED;
  IndirectAccessFlags = Flags;

  // Get the UUID.
  std::string uid;
  for (int n = 0; n < ZE_MAX_DEVICE_UUID_SIZE; n++)
    uid += std::to_string(DeviceProperties.uuid.id[n]);
  DeviceUuid = std::move(uid);

  ComputeOrdinal = findComputeOrdinal();
  QueueCache.setCommandMode(getPlugin().getOptions().CommandMode);

  SupportsCooperativeKernels = checkCooperativeKernelSupport();

  if (auto Err = MemAllocator.initDevicePools(*this, Options))
    return Err;
  l0Context.getHostMemAllocator().updateMaxAllocSize(*this);
  reportDeviceInfo();
  return Plugin::success();
}

Error L0DeviceTy::deinitImpl() {
  for (auto &PGM : Programs)
    if (auto Err = PGM.deinit())
      return Err;
  if (auto Err = QueueCache.deinit())
    return Err;
  return MemAllocator.deinit();
}

Expected<DeviceImageTy *>
L0DeviceTy::loadBinaryImpl(std::unique_ptr<MemoryBuffer> &&TgtImage,
                           int32_t ImageId) {
  auto *PGM = getProgramFromImage(TgtImage->getMemBufferRef());
  if (PGM) {
    // Program already exists.
    return PGM;
  }

  INFO(OMP_INFOTYPE_PLUGIN_KERNEL, getDeviceId(),
       "Device %" PRId32 ": Loading binary from " DPxMOD "\n", getDeviceId(),
       DPxPTR(TgtImage->getBufferStart()));

  const auto &Options = getPlugin().getOptions();
  std::string CompilationOptions(Options.CompilationOptions);
  CompilationOptions += " " + Options.UserCompilationOptions;

  INFO(OMP_INFOTYPE_PLUGIN_KERNEL, getDeviceId(),
       "Base L0 module compilation options: %s\n", CompilationOptions.c_str());

  CompilationOptions += " ";
  CompilationOptions += Options.InternalCompilationOptions;

  L0ProgramBuilderTy Builder(*this, std::move(TgtImage));
  if (auto Err = Builder.buildModules(CompilationOptions))
    return std::move(Err);

  auto ProgramOrErr = addProgram(ImageId, Builder);
  if (!ProgramOrErr)
    return ProgramOrErr.takeError();
  auto &Program = *ProgramOrErr;

  if (auto Err = Program.loadModuleKernels())
    return std::move(Err);

  return &Program;
}

Error L0DeviceTy::unloadBinaryImpl(DeviceImageTy *Image) {
  // Ignoring for now.
  // TODO: call properly L0Program unload.
  return Plugin::success();
}

Expected<L0QueueTy *>
L0DeviceTy::getOrCreateQueue(__tgt_async_info *AsyncInfo) {
  L0QueueTy *Queue = static_cast<L0QueueTy *>(AsyncInfo->Queue);
  if (!Queue) {
    auto NewQueueOrErr = QueueCache.getQueue();
    if (!NewQueueOrErr)
      return NewQueueOrErr.takeError();
    Queue = *NewQueueOrErr;
    AsyncInfo->Queue = Queue;
  }
  return Queue;
}

Error L0DeviceTy::synchronizeImpl(__tgt_async_info &AsyncInfo,
                                  bool ReleaseQueue) {

  L0QueueTy *Queue = static_cast<L0QueueTy *>(AsyncInfo.Queue);
  if (!Queue)
    return Plugin::success();

  Error SyncErr = Queue->synchronize();

  if (ReleaseQueue) {
    releaseQueue(Queue);
    getStagingBuffer().reset();
    AsyncInfo.Queue = nullptr;
  }

  return SyncErr;
}

Expected<bool>
L0DeviceTy::hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) {
  L0QueueTy *Queue = AsyncInfoWrapper.getQueueAs<L0QueueTy *>();
  if (!Queue)
    return false;
  return Queue->hasPendingWork();
}

Error L0DeviceTy::queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
                                 bool *IsQueueWorkCompleted) {
  L0QueueTy *Queue = static_cast<L0QueueTy *>(AsyncInfo.Queue);
  bool WorkCompleted = true;

  if (Queue) {
    auto PendingWorkOrErr = Queue->hasPendingWork();
    if (!PendingWorkOrErr)
      return PendingWorkOrErr.takeError();
    WorkCompleted = !*PendingWorkOrErr;
  }

  if (IsQueueWorkCompleted)
    *IsQueueWorkCompleted = WorkCompleted;

  if (!WorkCompleted || !Queue)
    return Plugin::success();

  if (ReleaseQueue) {
    releaseQueue(Queue);
    getStagingBuffer().reset();
    AsyncInfo.Queue = nullptr;
  }

  return Plugin::success();
}

Expected<void *> L0DeviceTy::allocate(size_t Size, void *HstPtr,
                                      TargetAllocTy Kind) {
  return dataAlloc(Size, /*Align=*/0, Kind,
                   /*Offset=*/0, /*UserAlloc=*/HstPtr == nullptr,
                   /*DevMalloc=*/false);
}

Error L0DeviceTy::free(void *TgtPtr, TargetAllocTy Kind) {
  return dataDelete(TgtPtr);
}

Error L0DeviceTy::dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
                                 AsyncInfoWrapperTy &AsyncInfoWrapper) {
  if (Size == 0)
    return Plugin::success();

  __tgt_async_info *AsyncInfo = AsyncInfoWrapper;
  auto AsyncQueueOrErr = getOrCreateQueue(AsyncInfo);
  if (!AsyncQueueOrErr)
    return AsyncQueueOrErr.takeError();
  auto *AsyncQueue = *AsyncQueueOrErr;

  if (auto Err = AsyncQueue->dataSubmit(TgtPtr, HstPtr, Size))
    return Err;
  ODBG(OLDT_DataTransfer) << "Device " << getDeviceId() << ": Submitted "
                          << Size
                          << " bytes from host to device (hst:" << HstPtr
                          << ") -> (tgt:" << TgtPtr << ").";
  return Plugin::success();
}

Error L0DeviceTy::dataRetrieveImpl(void *HstPtr, const void *TgtPtr,
                                   int64_t Size,
                                   AsyncInfoWrapperTy &AsyncInfoWrapper) {
  if (Size == 0)
    return Plugin::success();

  __tgt_async_info *AsyncInfo = AsyncInfoWrapper;
  assert(AsyncInfo && "AsyncInfo must be provided for data retrieval");

  const auto DeviceId = getDeviceId();
  auto QueueOrErr = getOrCreateQueue(AsyncInfo);
  if (!QueueOrErr)
    return QueueOrErr.takeError();
  auto *Queue = *QueueOrErr;

  if (auto Err = Queue->dataRetrieve(HstPtr, TgtPtr, Size))
    return Err;
  ODBG(OLDT_DataTransfer) << "Device " << DeviceId << ": Retrieved " << Size
                          << " bytes from device to host (tgt:" << TgtPtr
                          << ") -> (hst:" << HstPtr << ").";
  return Plugin::success();
}

Error L0DeviceTy::dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstDev,
                                   void *DstPtr, int64_t Size,
                                   AsyncInfoWrapperTy &AsyncInfoWrapper) {
  if (auto Err =
          enqueueMemCopy(DstPtr, SrcPtr, Size,
                         static_cast<__tgt_async_info *>(AsyncInfoWrapper)))
    return Err;
  return Plugin::success();
}

Error L0DeviceTy::initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) {
  auto QueueOrErr = getOrCreateQueue(AsyncInfoWrapper);
  return QueueOrErr ? Plugin::success() : QueueOrErr.takeError();
}

const char *L0DeviceTy::getArchCStr() const {
  switch (getDeviceArch()) {
  case DeviceArchTy::DeviceArch_Gen:
    return "Intel GPU Xe";
  case DeviceArchTy::DeviceArch_XeLPG:
    return "Intel GPU Xe LPG";
  case DeviceArchTy::DeviceArch_XeHPC:
    return "Intel GPU Xe HPC";
  case DeviceArchTy::DeviceArch_XeHPG:
    return "Intel GPU Xe HPG";
  case DeviceArchTy::DeviceArch_Xe2LP:
    return "Intel GPU Xe2 LP";
  case DeviceArchTy::DeviceArch_Xe2HP:
    return "Intel GPU Xe HP";
  case DeviceArchTy::DeviceArch_x86_64:
    return "Intel X86 64";
  default:
    return "Intel GPU Unknown";
  }
}

static const char *DriverVersionToStrTable[] = {
    "1.0", "1.1", "1.2", "1.3",  "1.4",  "1.5", "1.6",
    "1.7", "1.8", "1.9", "1.10", "1.11", "1.12"};
constexpr size_t DriverVersionToStrTableSize =
    sizeof(DriverVersionToStrTable) / sizeof(DriverVersionToStrTable[0]);

Expected<InfoTreeNode> L0DeviceTy::obtainInfoImpl() {
  InfoTreeNode Info;
  Info.add("Device Number", getDeviceId());
  Info.add("Device Name", getNameCStr(), "", DeviceInfo::NAME);
  Info.add("Product Name", getArchCStr(), "", DeviceInfo::PRODUCT_NAME);
  Info.add("Device Type", "GPU", "", DeviceInfo::TYPE);
  Info.add("Vendor", "Intel", "", DeviceInfo::VENDOR);
  Info.add("Vendor ID", getVendorId(), "", DeviceInfo::VENDOR_ID);
  auto DriverVersion = getDriverAPIVersion();
  if (DriverVersion < DriverVersionToStrTableSize)
    Info.add("Driver Version", DriverVersionToStrTable[DriverVersion], "",
             DeviceInfo::DRIVER_VERSION);
  else
    Info.add("Driver Version", "Unknown", "", DeviceInfo::DRIVER_VERSION);
  Info.add("Device PCI ID", getPCIId());
  Info.add("Device UUID", getUuid().data());
  Info.add("Number of total EUs", getNumEUs(), "",
           DeviceInfo::NUM_COMPUTE_UNITS);
  Info.add("Number of threads per EU", getNumThreadsPerEU());
  Info.add("EU SIMD width", getSIMDWidth());
  Info.add("Number of EUs per subslice", getNumEUsPerSubslice());
  Info.add("Number of subslices per slice", getNumSubslicesPerSlice());
  Info.add("Number of slices", getNumSlices());
  Info.add("Max Group size", getMaxGroupSize(), "",
           DeviceInfo::MAX_WORK_GROUP_SIZE);
  auto &MaxGroupSize =
      *Info.add("Workgroup Max Size per Dimension", std::monostate{}, "",
                DeviceInfo::MAX_WORK_GROUP_SIZE_PER_DIMENSION);
  MaxGroupSize.add("x", getMaxGroupSizeX());
  MaxGroupSize.add("y", getMaxGroupSizeY());
  MaxGroupSize.add("z", getMaxGroupSizeZ());
  Info.add("Maximum Grid Dimensions", getMaxGroupSize() * getMaxGroupCount(),
           "", DeviceInfo::MAX_WORK_SIZE);
  auto &MaxSize = *Info.add("Grid Size per Dimension", std::monostate{}, "",
                            DeviceInfo::MAX_WORK_SIZE_PER_DIMENSION);
  MaxSize.add("x", getMaxGroupSizeX() * getMaxGroupCountX());
  MaxSize.add("y", getMaxGroupSizeY() * getMaxGroupCountY());
  MaxSize.add("z", getMaxGroupSizeZ() * getMaxGroupCountZ());

  Info.add("Local memory size (bytes)", getMaxSharedLocalMemory(), "",
           DeviceInfo::WORK_GROUP_LOCAL_MEM_SIZE);
  Info.add("Global memory size (bytes)", getGlobalMemorySize(), "",
           DeviceInfo::GLOBAL_MEM_SIZE);
  Info.add("Cache size (bytes)", getCacheSize());
  Info.add("Max Memory Allocation Size (bytes)", getMaxMemAllocSize(), "",
           DeviceInfo::MAX_MEM_ALLOC_SIZE);
  Info.add("Max clock frequency (MHz)", getClockRate(), "",
           DeviceInfo::MAX_CLOCK_FREQUENCY);
  Info.add("Max memory clock frequency (MHz)", getMemoryClockRate(), "",
           DeviceInfo::MEMORY_CLOCK_RATE);
  Info.add("Memory Address Size", uint64_t{64u}, "bits",
           DeviceInfo::ADDRESS_BITS);

  // FP64 (Double precision).
  Info.add("Double FP Support", supportsFP64(), "",
           DeviceInfo::DOUBLE_FP_SUPPORT);
  ol_device_fp_capability_flags_t DoubleFPCapabilities = 0;
  ze_device_fp_flags_t ZeDoubleFPFlags = getFP64Flags();
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_DENORM)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_DENORM;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_INF_NAN)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_INF)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_FMA)
    DoubleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_FMA;
  if (ZeDoubleFPFlags & ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT)
    DoubleFPCapabilities |=
        OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT;
  Info.add("Double FP Capabilities", DoubleFPCapabilities, "",
           DeviceInfo::DOUBLE_FP_CONFIG);

  // FP16 (Half precision).
  Info.add("Half FP Support", supportsFP16(), "", DeviceInfo::HALF_FP_SUPPORT);
  ol_device_fp_capability_flags_t HalfFPCapabilities = 0;
  ze_device_fp_flags_t ZeHalfFPFlags = getFP16Flags();
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_DENORM)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_DENORM;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_INF_NAN)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_INF)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_FMA)
    HalfFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_FMA;
  if (ZeHalfFPFlags & ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT)
    HalfFPCapabilities |=
        OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT;
  Info.add("Half FP Capabilities", HalfFPCapabilities, "",
           DeviceInfo::HALF_FP_CONFIG);

  // FP32 (Single FP).
  Info.add("Single FP Support", true, "", DeviceInfo::SINGLE_FP_SUPPORT);
  ol_device_fp_capability_flags_t SingleFPCapabilities = 0;
  ze_device_fp_flags_t ZeSingleFPFlags = getFP32Flags();
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_DENORM)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_DENORM;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_INF_NAN)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_NEAREST)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_ZERO)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_ROUND_TO_INF)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_FMA)
    SingleFPCapabilities |= OL_DEVICE_FP_CAPABILITY_FLAG_FMA;
  if (ZeSingleFPFlags & ZE_DEVICE_FP_FLAG_ROUNDED_DIVIDE_SQRT)
    SingleFPCapabilities |=
        OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT;
  Info.add("Single FP Capabilities", SingleFPCapabilities, "",
           DeviceInfo::SINGLE_FP_CONFIG);

  Info.add("Cooperative launch support", SupportsCooperativeKernels, "",
           DeviceInfo::COOPERATIVE_LAUNCH_SUPPORT);
  return Info;
}

Expected<GenericKernelTy &> L0DeviceTy::constructKernel(const char *Name) {
  // Allocate and construct the L0 kernel.
  L0KernelTy *L0Kernel = getPlugin().allocate<L0KernelTy>();
  if (!L0Kernel)
    return Plugin::error(ErrorCode::UNKNOWN,
                         "Failed to allocate memory for L0 kernel");

  new (L0Kernel) L0KernelTy(Name);

  return *L0Kernel;
}

uint32_t L0DeviceTy::getMemAllocType(const void *Ptr) const {
  ze_memory_allocation_properties_t properties = {
      ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES,
      nullptr,                // Extension.
      ZE_MEMORY_TYPE_UNKNOWN, // Type.
      0,                      // Id.
      0,                      // Page size.
  };

  ze_result_t rc;
  CALL_ZE(rc, zeMemGetAllocProperties, getZeContext(), Ptr, &properties,
          nullptr);

  if (rc == ZE_RESULT_ERROR_INVALID_ARGUMENT)
    return ZE_MEMORY_TYPE_UNKNOWN;
  else
    return properties.type;
}

interop_spec_t L0DeviceTy::selectInteropPreference(int32_t InteropType,
                                                   int32_t NumPrefers,
                                                   interop_spec_t *Prefers) {
  // no supported preference found, set default to level_zero,
  // non-ordered unless is targetsync.
  return interop_spec_t{
      tgt_fr_level_zero,
      {InteropType == kmp_interop_type_targetsync /*inorder*/, 0},
      0};
}

Expected<OmpInteropTy> L0DeviceTy::createInterop(int32_t InteropContext,
                                                 interop_spec_t &InteropSpec) {
  auto Ret = new omp_interop_val_t(
      DeviceId, static_cast<kmp_interop_type_t>(InteropContext));
  Ret->fr_id = tgt_fr_level_zero;
  Ret->vendor_id = omp_vendor_intel;

  if (InteropContext == kmp_interop_type_target ||
      InteropContext == kmp_interop_type_targetsync) {
    Ret->device_info.Platform = getZeDriver();
    Ret->device_info.Device = getZeDevice();
    Ret->device_info.Context = getZeContext();
  }

  Ret->rtl_property = new L0Interop::Property();
  if (InteropContext == kmp_interop_type_targetsync) {
    Ret->async_info = new __tgt_async_info();

    // Ensure cleanup on error
    llvm::scope_exit CleanupOnError([&]() {
      if (Ret->async_info)
        delete Ret->async_info;
      if (Ret->rtl_property)
        delete static_cast<L0Interop::Property *>(Ret->rtl_property);
      delete Ret;
    });

    auto L0 = static_cast<L0Interop::Property *>(Ret->rtl_property);

    bool InOrder = InteropSpec.attrs.inorder;
    Ret->attrs.inorder = InOrder;
    auto CmdListOrErr = createImmCmdList(InOrder);
    if (!CmdListOrErr)
      return CmdListOrErr.takeError();
    Ret->async_info->Queue = *CmdListOrErr;
    L0->ImmCmdList = *CmdListOrErr;

    CleanupOnError.release();
  }

  return Ret;
}

Error L0DeviceTy::releaseInterop(OmpInteropTy Interop) {
  const auto DeviceId = getDeviceId();

  if (!Interop || Interop->device_id != static_cast<intptr_t>(DeviceId)) {
    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                         "Invalid/inconsistent OpenMP interop " DPxMOD "\n",
                         DPxPTR(Interop));
  }
  auto L0 = static_cast<L0Interop::Property *>(Interop->rtl_property);
  if (Interop->async_info && Interop->async_info->Queue) {
    auto ImmCmdList = L0->ImmCmdList;
    CALL_ZE_RET_ERROR(zeCommandListDestroy, ImmCmdList);
  }
  delete L0;
  delete Interop;

  return Plugin::success();
}

/// Enqueue fill command.
Error L0DeviceTy::enqueueMemFill(void *Ptr, const void *Pattern,
                                 size_t PatternSize, size_t Size,
                                 __tgt_async_info *AsyncInfo) {
  auto QueueOrErr = getOrCreateQueue(AsyncInfo);
  if (!QueueOrErr)
    return QueueOrErr.takeError();
  L0QueueTy *AsyncQueue = *QueueOrErr;
  return AsyncQueue->memoryFill(Ptr, Pattern, PatternSize, Size);
}

Error L0DeviceTy::dataFillImpl(void *TgtPtr, const void *PatternPtr,
                               int64_t PatternSize, int64_t Size,
                               AsyncInfoWrapperTy &AsyncInfoWrapper) {
  return enqueueMemFill(TgtPtr, PatternPtr, PatternSize, Size,
                        AsyncInfoWrapper);
}

Expected<void *> L0DeviceTy::dataAlloc(size_t Size, size_t Align, int32_t Kind,
                                       intptr_t Offset, bool UserAlloc,
                                       bool DevMalloc, uint32_t MemAdvice,
                                       AllocOptionTy AllocOpt) {

  const bool UseDedicatedPool =
      (AllocOpt == AllocOptionTy::ALLOC_OPT_REDUCTION_SCRATCH) ||
      (AllocOpt == AllocOptionTy::ALLOC_OPT_REDUCTION_COUNTER);
  if (Kind == TARGET_ALLOC_DEFAULT) {
    if (UserAlloc)
      Kind = TARGET_ALLOC_DEVICE;
    else if (AllocOpt == AllocOptionTy::ALLOC_OPT_HOST_MEM)
      Kind = TARGET_ALLOC_HOST;
    else if (UseDedicatedPool)
      Kind = TARGET_ALLOC_DEVICE;
    else
      Kind = getAllocKind();
  }
  auto &Allocator = getMemAllocator(Kind);
  return Allocator.alloc(Size, Align, Kind, Offset, UserAlloc, DevMalloc,
                         MemAdvice, AllocOpt);
}

Error L0DeviceTy::dataDelete(void *Ptr) {
  auto &Allocator = getMemAllocator(Ptr);
  return Allocator.dealloc(Ptr);
}

Error L0DeviceTy::makeMemoryResident(void *Mem, size_t Size) {
  CALL_ZE_RET_ERROR(zeContextMakeMemoryResident, getZeContext(), getZeDevice(),
                    Mem, Size);
  return Plugin::success();
}

/// Create an immediate command list.
Expected<ze_command_list_handle_t>
L0DeviceTy::createImmCmdList(uint32_t Ordinal, uint32_t Index, bool InOrder) {
  ze_command_queue_flags_t Flags = InOrder ? ZE_COMMAND_QUEUE_FLAG_IN_ORDER : 0;
  ze_command_queue_desc_t Desc{ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC,
                               nullptr,
                               Ordinal,
                               Index,
                               Flags | ZE_COMMAND_QUEUE_FLAG_COPY_OFFLOAD_HINT,
                               ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS,
                               ZE_COMMAND_QUEUE_PRIORITY_NORMAL};
  ze_command_list_handle_t CmdList = nullptr;
  CALL_ZE_RET_ERROR(zeCommandListCreateImmediate, getZeContext(), getZeDevice(),
                    &Desc, &CmdList);
  ODBG(OLDT_Device) << "Created an immediate command list " << CmdList
                    << " (Ordinal: " << Ordinal << ", Index: " << Index
                    << ", Flags: " << Flags << ") for device " << getZeIdCStr();
  return CmdList;
}

Error L0DeviceTy::dataFence(__tgt_async_info *Async) {
  auto QueueOrErr = getOrCreateQueue(Async);
  if (!QueueOrErr)
    return QueueOrErr.takeError();
  L0QueueTy *Queue = *QueueOrErr;
  return Queue->dataFence();
}

Expected<bool> L0DeviceTy::isAccessiblePtrImpl(const void *Ptr, size_t Size) {
  if (!Ptr || Size == 0)
    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                         "Invalid input to %s (Ptr = %p, Size = %zu)", __func__,
                         Ptr, Size);
  return getMemAllocator(Ptr).contains(Ptr, Size);
}

Error L0DeviceTy::callGlobalConstructors(GenericPluginTy &Plugin,
                                         DeviceImageTy &Image) {
  return callGlobalCtorDtorCommon(Plugin, Image, /*IsCtor=*/true);
}

Error L0DeviceTy::callGlobalDestructors(GenericPluginTy &Plugin,
                                        DeviceImageTy &Image) {
  return callGlobalCtorDtorCommon(Plugin, Image, /*IsCtor=*/false);
}

Error L0DeviceTy::callGlobalCtorDtorCommon(GenericPluginTy &Plugin,
                                           DeviceImageTy &Image, bool IsCtor) {
  const char *KernelName = IsCtor ? "spirv$device$init" : "spirv$device$fini";

  // Check if a kernel was generated to run constructor or destructors.
  // It should be created by the 'spirv-lower-ctor-dtor' pass.
  GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
  if (!Handler.isSymbolInImage(*this, Image, KernelName))
    return Plugin::success();

  // Instead of returning errors directly, we capture them and provide
  // more of context about this routine.
  auto HandleErr = [&](Error Err) {
    std::string Buffer;
    llvm::raw_string_ostream(Buffer)
        << "failed to call global " << (IsCtor ? "constructors" : "destructors")
        << " in the image";
    return Plugin::error(ErrorCode::INVALID_BINARY, std::move(Err),
                         Buffer.c_str());
  };

  // The SPIR-V backend cannot handle creating the ctor / dtor array
  // automatically so we must create it ourselves. The backend will emit
  // several globals that contain function pointers we can call. These are
  // prefixed with a __init_array_object_ or __fini_array_object_.
  auto ELFObjOrErr = Handler.getELFObjectFile(Image);
  if (!ELFObjOrErr)
    return HandleErr(ELFObjOrErr.takeError());

  using FuncNameAndPriority = std::pair<StringRef, uint16_t>;
  SmallVector<FuncNameAndPriority> Funcs;
  for (ELFSymbolRef Sym : (*ELFObjOrErr)->symbols()) {
    auto NameOrErr = Sym.getName();
    if (!NameOrErr)
      return HandleErr(NameOrErr.takeError());

    if (!NameOrErr->starts_with(IsCtor ? "__init_array_object_"
                                       : "__fini_array_object_"))
      continue;

    uint16_t Priority;
    if (NameOrErr->rsplit('_').second.getAsInteger(10, Priority))
      return Plugin::error(
          ErrorCode::INVALID_BINARY,
          "failed to call global %s in the image: invalid priority",
          IsCtor ? "constructors" : "destructors");

    Funcs.emplace_back(*NameOrErr, Priority);
  }

  if (Funcs.empty()) {
    ODBG(OLDT_Module) << KernelName << " found in the image but no "
                      << (IsCtor ? "constructors" : "destructors")
                      << " found in the image.";
    return Plugin::success();
  }

  // Sort the created array to be in priority order.
  llvm::sort(Funcs,
             [](const auto &X, const auto &Y) { return X.second < Y.second; });

  auto BufferOrErr = allocate(Funcs.size() * sizeof(void *),
                              /*HostPtr=*/nullptr, TARGET_ALLOC_DEVICE);
  if (!BufferOrErr)
    return HandleErr(BufferOrErr.takeError());

  void *Buffer = *BufferOrErr;
  if (!Buffer)
    return Plugin::error(
        ErrorCode::OUT_OF_RESOURCES,
        "failed to allocate memory for global buffer to run %s",
        IsCtor ? "constructors" : "destructors");

  auto CleanupBufferAndErr = [&](Error RetErr) {
    if (auto Err = free(Buffer, TARGET_ALLOC_DEVICE)) {
      return joinErrors(std::move(RetErr), std::move(Err));
    }
    return RetErr;
  };

  auto *GlobalPtrStart = reinterpret_cast<uintptr_t *>(Buffer);
  auto *GlobalPtrStop = reinterpret_cast<uintptr_t *>(Buffer) + Funcs.size();

  SmallVector<void *> FunctionPtrs(Funcs.size());
  size_t Idx = 0;
  for (auto [Name, Priority] : Funcs) {
    GlobalTy FunctionAddr(Name.str(), sizeof(void *), &FunctionPtrs[Idx++]);
    if (auto Err = Handler.readGlobalFromDevice(*this, Image, FunctionAddr))
      return CleanupBufferAndErr(std::move(Err));
  }

  if (auto Err = dataSubmit(GlobalPtrStart, FunctionPtrs.data(),
                            FunctionPtrs.size() * sizeof(void *),
                            /*AsyncInfo=*/nullptr))
    return CleanupBufferAndErr(std::move(Err));

  GlobalTy StartGlobal(IsCtor ? "__init_array_start" : "__fini_array_start",
                       sizeof(void *), &GlobalPtrStart);
  if (auto Err = Handler.writeGlobalToDevice(*this, Image, StartGlobal))
    return CleanupBufferAndErr(std::move(Err));

  GlobalTy StopGlobal(IsCtor ? "__init_array_end" : "__fini_array_end",
                      sizeof(void *), &GlobalPtrStop);
  if (auto Err = Handler.writeGlobalToDevice(*this, Image, StopGlobal))
    return CleanupBufferAndErr(std::move(Err));

  // Call the generated kernel to execute the constructors or destructors.
  auto KernelOrErr = constructKernel(KernelName);
  if (!KernelOrErr)
    return CleanupBufferAndErr(KernelOrErr.takeError());

  GenericKernelTy &L0Kernel = *KernelOrErr;
  if (auto Err = L0Kernel.init(*this, Image))
    return CleanupBufferAndErr(std::move(Err));

  AsyncInfoWrapperTy AsyncInfoWrapper(*this, /*AsyncInfoPtr=*/nullptr);

  KernelArgsTy KernelArgs{};
  uint32_t NumBlocksAndThreads[3] = {1u, 1u, 1u};
  auto Err =
      L0Kernel.launchImpl(*this, NumBlocksAndThreads, NumBlocksAndThreads, 0,
                          KernelArgs, KernelLaunchParamsTy{}, AsyncInfoWrapper);

  AsyncInfoWrapper.finalize(Err);
  return CleanupBufferAndErr(std::move(Err));
}

} // namespace llvm::omp::target::plugin