derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
4562 lines
170 KiB
C++
4562 lines
170 KiB
C++
/*
|
|
* Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
|
|
*
|
|
* SPDX-License-Identifier: MIT
|
|
*/
|
|
|
|
#include "cl.h"
|
|
#include "platform/program.hpp"
|
|
#include "platform/kernel.hpp"
|
|
#include "os/os.hpp"
|
|
#include "utils/debug.hpp"
|
|
#include "utils/flags.hpp"
|
|
#include "utils/options.hpp"
|
|
#include "utils/versions.hpp"
|
|
#include "thread/monitor.hpp"
|
|
#include "CL/cl_ext.h"
|
|
|
|
#include "vdi_common.hpp"
|
|
#include "device/comgrctx.hpp"
|
|
#include "device/devhostcall.hpp"
|
|
#include "device/rocm/rocdevice.hpp"
|
|
#include "device/rocm/rocblit.hpp"
|
|
#include "device/rocm/rocvirtual.hpp"
|
|
#include "device/rocm/rocprogram.hpp"
|
|
#include "device/rocm/rockernel.hpp"
|
|
#include "device/rocm/rocmemory.hpp"
|
|
#include "device/rocm/rocglinterop.hpp"
|
|
#include "device/rocm/rocsignal.hpp"
|
|
#include "platform/sampler.hpp"
|
|
|
|
#ifdef _WIN32
|
|
#include "device/rocm/rocd3d10interop.hpp"
|
|
#include "device/rocm/rocd3d11interop.hpp"
|
|
#include "platform/interop_d3d10.hpp"
|
|
#include "platform/interop_d3d11.hpp"
|
|
#endif
|
|
|
|
#if defined(__clang__)
|
|
#if __has_feature(address_sanitizer)
|
|
#include "device/rocm/rocurilocator.hpp"
|
|
#endif
|
|
#endif
|
|
|
|
#if defined(__linux__)
|
|
#include <climits>
|
|
#include <unistd.h>
|
|
#endif
|
|
|
|
#include <algorithm>
|
|
#include <chrono>
|
|
#include <cinttypes>
|
|
#include <cstring>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <iomanip>
|
|
#include <memory>
|
|
#include <sstream>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
#define OPENCL_VERSION_STR XSTR(OPENCL_MAJOR) "." XSTR(OPENCL_MINOR)
|
|
#define OPENCL_C_VERSION_STR XSTR(OPENCL_C_MAJOR) "." XSTR(OPENCL_C_MINOR)
|
|
|
|
|
|
static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kNone) ==
|
|
static_cast<uint32_t>(HSA_ACCESS_PERMISSION_NONE),
|
|
"Vmm Access Flag None mismatch with ROC-runtime!");
|
|
static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadOnly) ==
|
|
static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RO),
|
|
"Vmm Access Flag Read mismatch with ROCr-runtime!");
|
|
static_assert(static_cast<uint32_t>(amd::Device::VmmAccess::kReadWrite) ==
|
|
static_cast<uint32_t>(HSA_ACCESS_PERMISSION_RW),
|
|
"Vmm Access Flag Read Write mismatch with ROC-runtime!");
|
|
|
|
|
|
namespace amd::device {
|
|
extern const char* HipExtraSourceCode;
|
|
extern const char* HipExtraSourceCodeNoGWS;
|
|
} // namespace amd::device
|
|
|
|
namespace amd::roc {
|
|
bool roc::Device::isHsaInitialized_ = false;
|
|
std::vector<hsa_agent_t> roc::Device::gpu_agents_;
|
|
std::vector<AgentInfo> roc::Device::cpu_agents_;
|
|
|
|
address Device::mg_sync_ = nullptr;
|
|
|
|
bool NullDevice::create(const amd::Isa& isa) {
|
|
if (!isa.runtimeRocSupported()) {
|
|
LogPrintfError("Offline HSA device %s is not supported", isa.targetId());
|
|
return false;
|
|
}
|
|
|
|
online_ = false;
|
|
// Mark the device as GPU type
|
|
info_.type_ = CL_DEVICE_TYPE_GPU;
|
|
info_.vendorId_ = 0x1002;
|
|
|
|
roc::Settings* hsaSettings = new roc::Settings();
|
|
settings_ = hsaSettings;
|
|
if (!hsaSettings || !hsaSettings->create(false, isa, isa.xnack() == amd::Isa::Feature::Enabled)) {
|
|
LogPrintfError("Error creating settings for offline HSA device %s", isa.targetId());
|
|
return false;
|
|
}
|
|
|
|
if (!ValidateComgr()) {
|
|
LogPrintfError("Code object manager initialization failed for offline HSA device %s",
|
|
isa.targetId());
|
|
return false;
|
|
}
|
|
|
|
if (!amd::Device::create(isa)) {
|
|
LogPrintfError("Unable to setup offline HSA device %s", isa.targetId());
|
|
return false;
|
|
}
|
|
|
|
// Report the device name
|
|
::strncpy(info_.name_, isa.targetId(), sizeof(info_.name_) - 1);
|
|
info_.extensions_ = getExtensionString();
|
|
info_.maxWorkGroupSize_ = hsaSettings->maxWorkGroupSize_;
|
|
::strncpy(info_.vendor_, "Advanced Micro Devices, Inc.", sizeof(info_.vendor_) - 1);
|
|
info_.oclcVersion_ = "OpenCL C " OPENCL_C_VERSION_STR " ";
|
|
info_.spirVersions_ = "";
|
|
std::stringstream ss;
|
|
ss << AMD_BUILD_STRING " (HSA,LC) [Offline]";
|
|
::strncpy(info_.driverVersion_, ss.str().c_str(), sizeof(info_.driverVersion_) - 1);
|
|
info_.version_ = "OpenCL " OPENCL_VERSION_STR " ";
|
|
return true;
|
|
}
|
|
|
|
Device::Device(hsa_agent_t bkendDevice)
|
|
: mapCacheOps_(nullptr),
|
|
mapCache_(nullptr),
|
|
bkendDevice_(bkendDevice),
|
|
pciDeviceId_(0),
|
|
gpuvm_segment_max_alloc_(0),
|
|
alloc_granularity_(0),
|
|
xferQueue_(nullptr),
|
|
freeMem_(0),
|
|
hsa_exclusive_gpu_access_(false),
|
|
numOfVgpus_(0),
|
|
preferred_numa_node_(0),
|
|
maxSdmaReadMask_(0),
|
|
maxSdmaWriteMask_(0),
|
|
sdma_engine_allocator_(*this),
|
|
cpu_agent_info_(nullptr),
|
|
numHwPipes_(4) {
|
|
for (uint i = 0; i < QueuePriority::Total; ++i) {
|
|
queuePool_.emplace_back();
|
|
}
|
|
|
|
group_segment_.handle = 0;
|
|
gpuvm_segment_.handle = 0;
|
|
gpu_fine_grained_segment_.handle = 0;
|
|
gpu_ext_fine_grained_segment_.handle = 0;
|
|
prefetch_signal_.handle = 0;
|
|
isXgmi_ = false;
|
|
cache_state_ = Device::CacheState::kCacheStateInvalid;
|
|
}
|
|
|
|
int Device::agentGlobalIndex(hsa_agent_t agent) {
|
|
for (size_t gpu_idx = 0; gpu_idx < gpu_agents_.size(); ++gpu_idx) {
|
|
if (gpu_agents_[gpu_idx].handle == agent.handle) {
|
|
return static_cast<int>(gpu_idx);
|
|
}
|
|
}
|
|
for (size_t cpu_idx = 0; cpu_idx < cpu_agents_.size(); ++cpu_idx) {
|
|
if (cpu_agents_[cpu_idx].agent.handle == agent.handle) {
|
|
return static_cast<int>(gpu_agents_.size() + cpu_idx);
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
void Device::setupCpuAgent() {
|
|
int32_t numaDistance = std::numeric_limits<int32_t>::max();
|
|
uint32_t index = 0; // 0 as default
|
|
auto size = cpu_agents_.size();
|
|
for (uint32_t i = 0; i < size; i++) {
|
|
std::vector<amd::Device::LinkAttrType> link_attrs;
|
|
link_attrs.push_back(std::make_pair(LinkAttribute::kLinkDistance, 0));
|
|
if (findLinkInfo(cpu_agents_[i].fine_grain_pool, &link_attrs)) {
|
|
if (link_attrs[0].second < numaDistance) {
|
|
numaDistance = link_attrs[0].second;
|
|
index = i;
|
|
}
|
|
}
|
|
}
|
|
std::vector<amd::Device::LinkAttrType> link_attrs;
|
|
link_attrs.push_back(std::make_pair(LinkAttribute::kLinkLinkType, 0));
|
|
if (findLinkInfo(cpu_agents_[0].fine_grain_pool, &link_attrs)) {
|
|
isXgmi_ = (link_attrs[0].second == HSA_AMD_LINK_INFO_TYPE_XGMI);
|
|
}
|
|
|
|
preferred_numa_node_ = index;
|
|
cpu_agent_info_ = &cpu_agents_[index];
|
|
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT,
|
|
"Numa selects cpu agent[%zu]=0x%zx(fine=0x%zx,"
|
|
"coarse=0x%zx) for gpu agent=0x%zx CPU<->GPU XGMI=%d",
|
|
index, cpu_agent_info_->agent.handle, cpu_agent_info_->fine_grain_pool.handle,
|
|
cpu_agent_info_->coarse_grain_pool.handle, bkendDevice_.handle, isXgmi_);
|
|
}
|
|
|
|
void Device::checkAtomicSupport() {
|
|
std::vector<amd::Device::LinkAttrType> link_attrs;
|
|
link_attrs.push_back(std::make_pair(LinkAttribute::kLinkAtomicSupport, 0));
|
|
if (findLinkInfo(cpu_agent_info_->fine_grain_pool, &link_attrs)) {
|
|
if (link_attrs[0].second == 1) {
|
|
info_.pcie_atomics_ = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
Device::~Device() {
|
|
// Drain the ROCr async-events thread before releasing any backend state.
|
|
// This guards OCL teardown; for HIP the drain already ran via RuntimeTearDown,
|
|
// so this is a safe no-op in that path.
|
|
WaitForHsaAsyncHandlersIdle();
|
|
|
|
// Release cached map targets
|
|
for (uint i = 0; mapCache_ != nullptr && i < mapCache_->size(); ++i) {
|
|
if ((*mapCache_)[i] != nullptr) {
|
|
(*mapCache_)[i]->release();
|
|
}
|
|
}
|
|
delete mapCache_;
|
|
delete mapCacheOps_;
|
|
|
|
if (nullptr != p2p_stage_) {
|
|
p2p_stage_->release();
|
|
p2p_stage_ = nullptr;
|
|
}
|
|
if (nullptr != mg_sync_) {
|
|
GlbCtx().svmFree(mg_sync_);
|
|
mg_sync_ = nullptr;
|
|
}
|
|
if (glb_ctx_ != nullptr) {
|
|
glb_ctx_->release();
|
|
glb_ctx_ = nullptr;
|
|
}
|
|
|
|
// Destroy transfer queue FIRST (before destroying queues in pool)
|
|
// because its destructor will call releaseQueue()
|
|
delete xferQueue_;
|
|
xferQueue_ = nullptr;
|
|
|
|
// Final drain of deferred destroys on the main thread before hsa_shut_down.
|
|
DrainDeferredQueueDestroys();
|
|
|
|
for (auto& it : queuePool_) {
|
|
for (auto qIter = it.begin(); qIter != it.end();) {
|
|
hsa_queue_t* queue = qIter->first;
|
|
auto& qInfo = qIter->second;
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE, "Deleting hardware queue %p with refCount 0",
|
|
queue->base_address);
|
|
qIter = it.erase(qIter);
|
|
Hsa::queue_destroy(queue);
|
|
}
|
|
}
|
|
queuePool_.clear();
|
|
|
|
delete blitProgram_;
|
|
|
|
if (context_ != nullptr) {
|
|
context_->release();
|
|
}
|
|
|
|
delete[] p2p_agents_list_;
|
|
|
|
if (0 != prefetch_signal_.handle) {
|
|
Hsa::signal_destroy(prefetch_signal_);
|
|
}
|
|
}
|
|
|
|
void NullDevice::tearDown() {}
|
|
|
|
void Device::WaitForHsaAsyncHandlersIdle() {
|
|
constexpr int kDrainTimeoutMs = 5000;
|
|
const std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now();
|
|
const std::chrono::steady_clock::duration fast_timeout =
|
|
std::chrono::milliseconds(kDrainTimeoutMs);
|
|
|
|
auto sumQueuedHandlers = [this]() -> uint64_t {
|
|
uint64_t sum = 0;
|
|
std::scoped_lock lock(vgpusAccess_);
|
|
for (VirtualGPU* vgpu : vgpus_) {
|
|
if (vgpu != nullptr) {
|
|
sum += vgpu->QueuedAsyncHandlers().load(std::memory_order_acquire);
|
|
}
|
|
}
|
|
return sum;
|
|
};
|
|
|
|
while (sumQueuedHandlers() != 0) {
|
|
if (std::chrono::steady_clock::now() - start_time > fast_timeout) {
|
|
const uint64_t remaining = sumQueuedHandlers();
|
|
if (remaining != 0) {
|
|
LogPrintfError(
|
|
"WaitForHsaAsyncHandlersIdle: %d ms elapsed, VirtualGPU queued_async total=%llu; "
|
|
"proceeding with device destruction.",
|
|
kDrainTimeoutMs, static_cast<unsigned long long>(remaining));
|
|
}
|
|
return;
|
|
}
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
}
|
|
}
|
|
|
|
bool NullDevice::init() {
|
|
// Create offline devices for all ISAs not already associated with an online
|
|
// device. This allows code objects to be compiled for all supported ISAs.
|
|
std::vector<Device*> devices = getDevices(CL_DEVICE_TYPE_GPU, false);
|
|
for (const amd::Isa* isa = amd::Isa::begin(); isa != amd::Isa::end(); isa++) {
|
|
if (!isa->runtimeRocSupported()) {
|
|
continue;
|
|
}
|
|
bool isOnline = false;
|
|
// Check if the particular device is online
|
|
for (size_t i = 0; i < devices.size(); i++) {
|
|
if (&(devices[i]->isa()) == isa) {
|
|
isOnline = true;
|
|
break;
|
|
}
|
|
}
|
|
if (isOnline) {
|
|
continue;
|
|
}
|
|
std::unique_ptr<NullDevice> nullDevice(new NullDevice());
|
|
if (!nullDevice) {
|
|
LogPrintfError("Error allocating new instance of offline HSA device %s", isa->targetId());
|
|
return false;
|
|
}
|
|
if (!nullDevice->create(*isa)) {
|
|
LogPrintfError("Skipping creating new instance of offline HSA sevice %s", isa->targetId());
|
|
continue;
|
|
}
|
|
nullDevice.release()->registerDevice();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
NullDevice::~NullDevice() {}
|
|
|
|
hsa_status_t Device::iterateAgentCallback(hsa_agent_t agent, void* data) {
|
|
hsa_device_type_t dev_type = HSA_DEVICE_TYPE_CPU;
|
|
|
|
hsa_status_t stat = Hsa::agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &dev_type);
|
|
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("HSA_AGENT_INFO_DEVICE failed with %x", stat);
|
|
return stat;
|
|
}
|
|
|
|
if (dev_type == HSA_DEVICE_TYPE_CPU) {
|
|
AgentInfo info = {agent, {0}, {0}, {0}};
|
|
stat = Hsa::agent_iterate_memory_pools(agent, Device::iterateCpuMemoryPoolCallback,
|
|
reinterpret_cast<void*>(&info));
|
|
if (stat == HSA_STATUS_SUCCESS) {
|
|
cpu_agents_.push_back(info);
|
|
}
|
|
} else if (dev_type == HSA_DEVICE_TYPE_GPU) {
|
|
gpu_agents_.push_back(agent);
|
|
}
|
|
|
|
return stat;
|
|
}
|
|
|
|
hsa_ven_amd_loader_1_03_pfn_t Device::amd_loader_ext_table = {nullptr};
|
|
|
|
hsa_status_t Device::loaderQueryHostAddress(const void* device, const void** host) {
|
|
return amd_loader_ext_table.hsa_ven_amd_loader_query_host_address
|
|
? amd_loader_ext_table.hsa_ven_amd_loader_query_host_address(device, host)
|
|
: HSA_STATUS_ERROR;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::init() {
|
|
if (!Hsa::LoadLib()) {
|
|
LogPrintfWarning("Failed to load rocr library!");
|
|
return false;
|
|
}
|
|
|
|
hsa_status_t status = Hsa::init();
|
|
|
|
// If there are no GPUs available, hsa_init will fail with HSA_STATUS_ERROR_OUT_OF_RESOURCES
|
|
// but for NoGpu tests to pass, true needs to be returned
|
|
constexpr bool kNoOfflineDevices = false;
|
|
std::vector<amd::Device*> devices = getDevices(CL_DEVICE_TYPE_GPU, kNoOfflineDevices);
|
|
if (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES && devices.size() == 0) {
|
|
return true;
|
|
}
|
|
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("hsa_init failed with %x", status);
|
|
return false;
|
|
}
|
|
|
|
Hsa::system_get_major_extension_table(HSA_EXTENSION_AMD_LOADER, 1, sizeof(amd_loader_ext_table),
|
|
&amd_loader_ext_table);
|
|
|
|
status = Hsa::iterate_agents(iterateAgentCallback, nullptr);
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("hsa_iterate_agents failed with %x", status);
|
|
return false;
|
|
}
|
|
|
|
std::string ordinals =
|
|
amd::IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ? HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES)
|
|
: GPU_DEVICE_ORDINAL;
|
|
if (ordinals[0] != '\0') {
|
|
size_t pos = 0;
|
|
std::vector<hsa_agent_t> valid_agents;
|
|
std::set<size_t> valid_indexes;
|
|
do {
|
|
size_t end;
|
|
bool deviceIdValid = true;
|
|
end = ordinals.find_first_of(',', pos);
|
|
if (end == std::string::npos) {
|
|
end = ordinals.size();
|
|
}
|
|
std::string str_id = ordinals.substr(pos, end - pos);
|
|
// If Uuid is specified, then convert it to index
|
|
// Uuid is an Ascii string with a maximum of 21 chars including NULL
|
|
// The string value is in the format GPU-<body>, <body> encodes UUID as a 16 chars hex
|
|
if (str_id.find("GPU-") != std::string::npos) {
|
|
for (int i = 0; i < gpu_agents_.size(); i++) {
|
|
auto agent = gpu_agents_[i];
|
|
char unique_id[32] = {0};
|
|
if (HSA_STATUS_SUCCESS ==
|
|
Hsa::agent_get_info(agent, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_UUID),
|
|
unique_id)) {
|
|
if (std::string_view(unique_id).find(str_id) != std::string_view::npos) {
|
|
str_id = std::to_string(i);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
int index = atoi(str_id.c_str());
|
|
if (index < 0 || static_cast<size_t>(index) >= gpu_agents_.size() ||
|
|
str_id != std::to_string(index)) {
|
|
deviceIdValid = false;
|
|
}
|
|
|
|
if (!deviceIdValid) {
|
|
// Exit the loop as anything to the right of invalid deviceId
|
|
// has to be discarded
|
|
break;
|
|
} else {
|
|
if (valid_indexes.find(index) == valid_indexes.end()) {
|
|
valid_agents.push_back(gpu_agents_[index]);
|
|
valid_indexes.insert(index);
|
|
}
|
|
}
|
|
pos = end + 1;
|
|
} while (pos < ordinals.size());
|
|
gpu_agents_ = valid_agents;
|
|
}
|
|
|
|
LogPrintfInfo("Initalizing runtime stack, Enumerated GPU agents = %lu", gpu_agents_.size());
|
|
|
|
for (auto agent : gpu_agents_) {
|
|
std::unique_ptr<Device> roc_device(new Device(agent));
|
|
if (!roc_device) {
|
|
LogError("Error creating new instance of Device on then heap.");
|
|
continue;
|
|
}
|
|
|
|
if (!roc_device->create()) {
|
|
LogError("Error creating new instance of Device.");
|
|
continue;
|
|
}
|
|
|
|
// Setup System Memory to be Non-Coherent per user
|
|
// request via environment variable. By default the
|
|
// System Memory is setup to be Coherent
|
|
if (roc_device->settings().enableNCMode_) {
|
|
hsa_status_t err = Hsa::coherency_set_type(agent, HSA_AMD_COHERENCY_TYPE_NONCOHERENT);
|
|
if (err != HSA_STATUS_SUCCESS) {
|
|
LogError("Unable to set NC memory policy!");
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Check to see if a global CU mask is requested
|
|
if (amd::IS_HIP && ROC_GLOBAL_CU_MASK[0] != '\0') {
|
|
roc_device->getGlobalCUMask(ROC_GLOBAL_CU_MASK);
|
|
}
|
|
// Note: for now disable HSA path by default except for gfx942
|
|
if (IS_WINDOWS && (GPU_ENABLE_PAL == 2) &&
|
|
(std::string_view(roc_device->info().name_).find("gfx942") == std::string_view::npos)) {
|
|
return false;
|
|
}
|
|
roc_device.release()->registerDevice();
|
|
}
|
|
|
|
// Query active devices only
|
|
devices = getDevices(CL_DEVICE_TYPE_GPU, kNoOfflineDevices);
|
|
if (devices.size() > 0) {
|
|
bool p2p_available = false;
|
|
// Loop through all available devices
|
|
for (auto device1 : devices) {
|
|
// Find all agents that can have access to the current device
|
|
for (auto agent : static_cast<Device*>(device1)->p2pAgents()) {
|
|
// Find cl_device_id associated with the current agent
|
|
for (auto device2 : devices) {
|
|
if (agent.handle == static_cast<Device*>(device2)->getBackendDevice().handle) {
|
|
// Device2 can have access to device1
|
|
device2->p2pDevices_.push_back(as_cl(device1));
|
|
device1->p2p_access_devices_.push_back(device2);
|
|
p2p_available = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Create a dummy context for internal memory allocations on all reported devices
|
|
glb_ctx_ = new amd::Context(devices, amd::Context::Info());
|
|
if (glb_ctx_ == nullptr) {
|
|
LogError("glb_ctx failed");
|
|
return false;
|
|
}
|
|
|
|
// Allocate a staging buffer for P2P emulation path
|
|
if ((devices.size() >= 1) && !p2p_available) {
|
|
amd::Buffer* buf =
|
|
new (*glb_ctx_) amd::Buffer(*glb_ctx_, CL_MEM_ALLOC_HOST_PTR, kP2PStagingSize);
|
|
if ((buf != nullptr) && buf->create()) {
|
|
p2p_stage_ = buf;
|
|
} else {
|
|
delete buf;
|
|
LogError("p2p stg buffer alloc failed");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Allocate mgpu sync buffer for cooperative launches
|
|
if (amd::IS_HIP) {
|
|
mg_sync_ = reinterpret_cast<address>(
|
|
glb_ctx_->svmAlloc(kMGInfoSizePerDevice * devices.size(), kMGInfoSizePerDevice,
|
|
(CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS)));
|
|
if (mg_sync_ == nullptr) {
|
|
LogError("mgpu sync buffer alloc failed");
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (amd::IS_HIP) {
|
|
RegisterBackendErrorCb();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
extern const char* SchedulerSourceCode;
|
|
|
|
void Device::tearDown() {
|
|
NullDevice::tearDown();
|
|
Hsa::shut_down();
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::create() {
|
|
char agent_name[64] = {0};
|
|
if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_NAME, agent_name)) {
|
|
LogError("Unable to get HSA device name");
|
|
return false;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_,
|
|
(hsa_agent_info_t)HSA_AMD_AGENT_INFO_CHIP_ID,
|
|
&pciDeviceId_)) {
|
|
LogPrintfError("Unable to get PCI ID of HSA device %s", agent_name);
|
|
return false;
|
|
}
|
|
|
|
struct agent_isas_t {
|
|
uint count;
|
|
hsa_isa_t first_isa;
|
|
} agent_isas = {0, {0}};
|
|
if (HSA_STATUS_SUCCESS != Hsa::agent_iterate_isas(
|
|
bkendDevice_,
|
|
[](hsa_isa_t isa, void* data) {
|
|
agent_isas_t* agent_isas = static_cast<agent_isas_t*>(data);
|
|
if (agent_isas->count++ == 0) {
|
|
agent_isas->first_isa = isa;
|
|
}
|
|
return HSA_STATUS_SUCCESS;
|
|
},
|
|
&agent_isas)) {
|
|
LogPrintfError("Unable to iterate supported ISAs for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
uint32_t isa_name_length = 0;
|
|
if (HSA_STATUS_SUCCESS != Hsa::isa_get_info_alt(agent_isas.first_isa,
|
|
(hsa_isa_info_t)HSA_ISA_INFO_NAME_LENGTH,
|
|
&isa_name_length)) {
|
|
LogPrintfError("Unable to get ISA name length for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
std::vector<char> isa_name(isa_name_length + 1, '\0');
|
|
if (HSA_STATUS_SUCCESS != Hsa::isa_get_info_alt(agent_isas.first_isa,
|
|
(hsa_isa_info_t)HSA_ISA_INFO_NAME,
|
|
isa_name.data())) {
|
|
LogPrintfError("Unable to get ISA name for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
const amd::Isa* isa = amd::Isa::findIsa(isa_name.data());
|
|
if (!isa || !isa->runtimeRocSupported()) {
|
|
LogPrintfError("Unsupported HSA device %s (PCI ID %x) for ISA %s", agent_name, pciDeviceId_,
|
|
isa_name.data());
|
|
return false;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_PROFILE, &agent_profile_)) {
|
|
LogPrintfError("Unable to get profile for HSA device %s (PCI ID %x)", agent_name, pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
uint32_t coop_groups = 0;
|
|
// Check cooperative groups for HIP only
|
|
if (amd::IS_HIP &&
|
|
(HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_COOPERATIVE_QUEUES),
|
|
&coop_groups))) {
|
|
LogPrintfError(
|
|
"Unable to determine if cooperative queues are supported for HSA device %s (PCI ID %x)",
|
|
agent_name, pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
setupCpuAgent();
|
|
|
|
// Get Agent HDP Flush Register Memory
|
|
hsa_amd_hdp_flush_t hdpInfo;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HDP_FLUSH),
|
|
&hdpInfo)) {
|
|
LogPrintfError("Unable to determine HDP flush info for HSA device %s", agent_name);
|
|
return false;
|
|
}
|
|
|
|
info_.hdpMemFlushCntl = hdpInfo.HDP_MEM_FLUSH_CNTL;
|
|
info_.hdpRegFlushCntl = hdpInfo.HDP_REG_FLUSH_CNTL;
|
|
|
|
// Create HSA settings
|
|
assert(!settings_);
|
|
roc::Settings* hsaSettings = new roc::Settings();
|
|
settings_ = hsaSettings;
|
|
if (!hsaSettings || !hsaSettings->create((agent_profile_ == HSA_PROFILE_FULL), *isa,
|
|
isa->xnack() == amd::Isa::Feature::Enabled, coop_groups,
|
|
isXgmi_)) {
|
|
LogPrintfError("Unable to create settings for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
if (!ValidateComgr()) {
|
|
LogPrintfError("Code object manager initialization failed for HSA device %s (PCI ID %x)",
|
|
agent_name, pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
if (!amd::Device::create(*isa)) {
|
|
LogPrintfError("Unable to setup device for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
uint32_t hsa_bdf_id = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_BDFID),
|
|
&hsa_bdf_id)) {
|
|
LogPrintfError("Unable to determine BFD ID for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
|
|
info_.deviceTopology_.pcie.type = CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD;
|
|
info_.deviceTopology_.pcie.bus = (hsa_bdf_id & (0xFF << 8)) >> 8;
|
|
info_.deviceTopology_.pcie.device = (hsa_bdf_id & (0x1F << 3)) >> 3;
|
|
info_.deviceTopology_.pcie.function = (hsa_bdf_id & 0x07);
|
|
uint32_t pci_domain_id = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_DOMAIN),
|
|
&pci_domain_id)) {
|
|
LogPrintfError("Unable to determine domain ID for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
info_.pciDomainID = pci_domain_id;
|
|
|
|
#ifdef _WIN32
|
|
// Extract adapter LUID for D3D interop validation
|
|
hsa_status_t luid_status = Hsa::agent_get_info(
|
|
bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_LUID),
|
|
&deviceLuid_);
|
|
luidValid_ = luid_status == HSA_STATUS_SUCCESS;
|
|
if (!luidValid_) {
|
|
LogWarning("Could not extract LUID from HSA agent - D3D interop may not work correctly");
|
|
}
|
|
#endif
|
|
|
|
if (populateOCLDeviceConstants() == false) {
|
|
LogPrintfError("populateOCLDeviceConstants failed for HSA device %s (PCI ID %x)", agent_name,
|
|
pciDeviceId_);
|
|
return false;
|
|
}
|
|
hsaSettings->limit_blit_wg_ = info().maxComputeUnits_;
|
|
if (!flagIsDefault(DEBUG_CLR_LIMIT_BLIT_WG)) {
|
|
hsaSettings->limit_blit_wg_ = std::max(DEBUG_CLR_LIMIT_BLIT_WG, 0x1U);
|
|
}
|
|
amd::Context::Info info = {0};
|
|
std::vector<amd::Device*> devices{this};
|
|
|
|
// Create a dummy context
|
|
context_ = new amd::Context(devices, info);
|
|
if (context_ == nullptr) {
|
|
return false;
|
|
}
|
|
|
|
// Map Cache Lock
|
|
mapCacheOps_ = new std::recursive_mutex();
|
|
if (nullptr == mapCacheOps_) {
|
|
return false;
|
|
}
|
|
|
|
mapCache_ = new std::vector<amd::Memory*>();
|
|
if (mapCache_ == nullptr) {
|
|
return false;
|
|
}
|
|
// Use just 1 entry by default for the map cache
|
|
mapCache_->push_back(nullptr);
|
|
|
|
// Create signal for HMM prefetch operation on device
|
|
if (HSA_STATUS_SUCCESS != Hsa::signal_create(kInitSignalValueOne, 0, nullptr, &prefetch_signal_)) {
|
|
return false;
|
|
}
|
|
|
|
if (AMD_LOG_LEVEL >= LOG_EXTRA_EXTRA_DEBUG) {
|
|
uint8_t logMask[8] = {0};
|
|
hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_AQL);
|
|
hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_SDMA);
|
|
hsa_flag_set64(logMask, HSA_AMD_LOG_FLAG_INFO);
|
|
Hsa::enable_logging(logMask, outFile);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
device::Program* NullDevice::createProgram(amd::Program& owner, amd::option::Options* options) {
|
|
device::Program* program = new roc::Program(*this, owner);
|
|
|
|
if (program == nullptr) {
|
|
LogError("Memory allocation has failed!");
|
|
}
|
|
|
|
return program;
|
|
}
|
|
|
|
bool Device::createBlitProgram() {
|
|
bool result = true;
|
|
std::string extraKernel;
|
|
|
|
if (amd::IS_HIP) {
|
|
if (settings().gwsInitSupported_) {
|
|
extraKernel = device::HipExtraSourceCode;
|
|
} else {
|
|
extraKernel = device::HipExtraSourceCodeNoGWS;
|
|
}
|
|
} else {
|
|
extraKernel = SchedulerSourceCode;
|
|
}
|
|
|
|
blitProgram_ = new BlitProgram(context_);
|
|
// Create blit programs
|
|
if (blitProgram_ == nullptr || !blitProgram_->create(this, extraKernel, "")) {
|
|
delete blitProgram_;
|
|
blitProgram_ = nullptr;
|
|
LogError("Couldn't create blit kernels!");
|
|
return false;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
device::Program* Device::createProgram(amd::Program& owner, amd::option::Options* options) {
|
|
device::Program* program = new roc::Program(*this, owner);
|
|
|
|
if (program == nullptr) {
|
|
LogError("Memory allocation has failed!");
|
|
}
|
|
|
|
return program;
|
|
}
|
|
|
|
hsa_status_t Device::iterateGpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, void* data) {
|
|
if (data == nullptr) {
|
|
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
hsa_region_segment_t segment_type = (hsa_region_segment_t)0;
|
|
hsa_status_t stat =
|
|
Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment_type);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
return stat;
|
|
}
|
|
|
|
// TODO: system and device local segment
|
|
Device* dev = reinterpret_cast<Device*>(data);
|
|
switch (segment_type) {
|
|
case HSA_REGION_SEGMENT_GLOBAL: {
|
|
if (dev->settings().enableLocalMemory_) {
|
|
uint32_t global_flag = 0;
|
|
hsa_status_t stat =
|
|
Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &global_flag);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
return stat;
|
|
}
|
|
|
|
// If the flag set is ext scoped fine grain, break the loop
|
|
if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) {
|
|
dev->gpu_ext_fine_grained_segment_ = pool;
|
|
break;
|
|
}
|
|
|
|
if ((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) != 0) {
|
|
dev->gpu_fine_grained_segment_ = pool;
|
|
} else if ((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) != 0) {
|
|
dev->gpuvm_segment_ = pool;
|
|
|
|
// If cpu agent cannot access this pool, the device does not support large bar.
|
|
hsa_amd_memory_pool_access_t tmp{};
|
|
Hsa::agent_memory_pool_get_info(dev->cpu_agent_info_->agent, pool,
|
|
HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &tmp);
|
|
|
|
if (tmp == HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED) {
|
|
dev->info_.largeBar_ = false;
|
|
} else {
|
|
dev->info_.largeBar_ = ROC_ENABLE_LARGE_BAR;
|
|
}
|
|
|
|
// Query the recommended granularity for this pool.
|
|
stat = Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
|
|
&(dev->info_.virtualMemAllocGranularityMinimum_));
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError(
|
|
"Cannot query HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE info"
|
|
"failed with hsa_status: %d",
|
|
stat);
|
|
}
|
|
// Query the recommended granularity for this pool.
|
|
stat = Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_REC_GRANULE,
|
|
&(dev->info_.virtualMemAllocGranularityRecommended_));
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError(
|
|
"Cannot query HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_REC_GRANULE info"
|
|
"failed with hsa_status: %d",
|
|
stat);
|
|
}
|
|
}
|
|
|
|
if (dev->gpuvm_segment_.handle == 0) {
|
|
dev->gpuvm_segment_ = pool;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case HSA_REGION_SEGMENT_GROUP:
|
|
dev->group_segment_ = pool;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return HSA_STATUS_SUCCESS;
|
|
}
|
|
|
|
hsa_status_t Device::iterateCpuMemoryPoolCallback(hsa_amd_memory_pool_t pool, void* data) {
|
|
if (data == nullptr) {
|
|
LogError("CpuMemoryPoolCallback invalid args");
|
|
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
|
}
|
|
|
|
hsa_region_segment_t segment_type = (hsa_region_segment_t)0;
|
|
hsa_status_t stat =
|
|
Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_SEGMENT, &segment_type);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("HSA_AMD_MEMORY_POOL_INFO_SEGMENT query failed with %x", stat);
|
|
return stat;
|
|
}
|
|
AgentInfo* agentInfo = reinterpret_cast<AgentInfo*>(data);
|
|
|
|
switch (segment_type) {
|
|
case HSA_REGION_SEGMENT_GLOBAL: {
|
|
uint32_t global_flag = 0;
|
|
stat =
|
|
Hsa::memory_pool_get_info(pool, HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS, &global_flag);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("HSA_AMD_MEMORY_POOL_INFO_GLOBAL_FLAGS query failed with %x", stat);
|
|
break;
|
|
}
|
|
|
|
// If the flag set is ext scoped fine grain, break the loop
|
|
if ((global_flag & HSA_REGION_GLOBAL_FLAG_EXTENDED_SCOPE_FINE_GRAINED) != 0) {
|
|
agentInfo->ext_fine_grain_pool = pool;
|
|
break;
|
|
}
|
|
|
|
if ((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) != 0) {
|
|
if (agentInfo->fine_grain_pool.handle == 0) {
|
|
agentInfo->fine_grain_pool = pool;
|
|
} else if ((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) == 0) {
|
|
// If the fine_grain_pool was already filled, but kern_args flag was not set over-write.
|
|
// This means this is region-1(fine_grain only), so over-write this with memory pool set
|
|
// from "fine_grain and kern_args".
|
|
agentInfo->fine_grain_pool = pool;
|
|
}
|
|
guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) == 0),
|
|
"Memory Segment cannot be both coarse and fine grained");
|
|
} else {
|
|
// If the flag is not set to fine grained, then it is coarse_grained by default.
|
|
agentInfo->coarse_grain_pool = pool;
|
|
guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) != 0),
|
|
"Memory Segments that are not fine grained has to be coarse grained");
|
|
guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_FINE_GRAINED) == 0),
|
|
"Memory Segment cannot be both coarse and fine grained");
|
|
guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) == 0),
|
|
"Coarse grained memory segment cannot have kern_args tag");
|
|
}
|
|
|
|
if ((global_flag & HSA_REGION_GLOBAL_FLAG_KERNARG) != 0) {
|
|
agentInfo->kern_arg_pool = pool;
|
|
guarantee(((global_flag & HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED) == 0),
|
|
"Coarse grained memory segment cannot have kern_args tag");
|
|
}
|
|
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return stat;
|
|
}
|
|
|
|
bool Device::createSampler(const amd::Sampler& owner, device::Sampler** sampler) const {
|
|
*sampler = nullptr;
|
|
Sampler* gpuSampler = new Sampler(*this);
|
|
if ((nullptr == gpuSampler) || !gpuSampler->create(owner)) {
|
|
delete gpuSampler;
|
|
return false;
|
|
}
|
|
*sampler = gpuSampler;
|
|
return true;
|
|
}
|
|
|
|
void Sampler::fillSampleDescriptor(hsa_ext_sampler_descriptor_v2_t& samplerDescriptor,
|
|
const amd::Sampler& sampler) const {
|
|
samplerDescriptor.filter_mode = sampler.filterMode() == CL_FILTER_NEAREST
|
|
? HSA_EXT_SAMPLER_FILTER_MODE_NEAREST
|
|
: HSA_EXT_SAMPLER_FILTER_MODE_LINEAR;
|
|
switch (sampler.mipFilter()) {
|
|
case CL_FILTER_NEAREST:
|
|
samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NEAREST;
|
|
break;
|
|
case CL_FILTER_LINEAR:
|
|
samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_LINEAR;
|
|
break;
|
|
default:
|
|
samplerDescriptor.mipmap_filter_mode = HSA_EXT_SAMPLER_FILTER_MODE_NONE;
|
|
}
|
|
samplerDescriptor.coordinate_mode = sampler.normalizedCoords()
|
|
? HSA_EXT_SAMPLER_COORDINATE_MODE_NORMALIZED
|
|
: HSA_EXT_SAMPLER_COORDINATE_MODE_UNNORMALIZED;
|
|
for (int i = 0; i < 3; i++) {
|
|
switch (sampler.addressingMode(i)) {
|
|
case CL_ADDRESS_CLAMP_TO_EDGE:
|
|
samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE;
|
|
break;
|
|
case CL_ADDRESS_REPEAT:
|
|
samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_REPEAT;
|
|
break;
|
|
case CL_ADDRESS_CLAMP:
|
|
samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_CLAMP_TO_BORDER;
|
|
break;
|
|
case CL_ADDRESS_MIRRORED_REPEAT:
|
|
samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_MIRRORED_REPEAT;
|
|
break;
|
|
case CL_ADDRESS_NONE:
|
|
samplerDescriptor.address_modes[i] = HSA_EXT_SAMPLER_ADDRESSING_MODE_UNDEFINED;
|
|
break;
|
|
default:
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool Sampler::create(const amd::Sampler& owner) {
|
|
hsa_ext_sampler_descriptor_v2_t samplerDescriptor;
|
|
fillSampleDescriptor(samplerDescriptor, owner);
|
|
|
|
hsa_status_t status =
|
|
Hsa::sampler_create(dev_.getBackendDevice(), &samplerDescriptor, &hsa_sampler);
|
|
|
|
if (HSA_STATUS_SUCCESS != status) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_RESOURCE,
|
|
"Sampler creation failed with status: %d", status);
|
|
return false;
|
|
}
|
|
|
|
hwSrd_ = hsa_sampler.handle;
|
|
hwState_ = reinterpret_cast<address>(hsa_sampler.handle);
|
|
|
|
return true;
|
|
}
|
|
|
|
Sampler::~Sampler() { Hsa::sampler_destroy(dev_.getBackendDevice(), hsa_sampler); }
|
|
|
|
Memory* Device::getGpuMemory(amd::Memory* mem) const {
|
|
return static_cast<roc::Memory*>(mem->getDeviceMemory(*this));
|
|
}
|
|
|
|
const bool Device::isFineGrainSupported() const {
|
|
bool result = (info().svmCapabilities_ & CL_DEVICE_SVM_ATOMICS) != 0 ? true : false;
|
|
if (result) {
|
|
if (gpu_fine_grained_segment_.handle != 0) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
// ================================================================================================
|
|
bool Device::populateOCLDeviceConstants() {
|
|
info_.available_ = true;
|
|
|
|
::strncpy(info_.name_, isa().targetId(), sizeof(info_.name_) - 1);
|
|
char device_name[64] = {0};
|
|
if (HSA_STATUS_SUCCESS == Hsa::agent_get_info(bkendDevice_,
|
|
(hsa_agent_info_t)HSA_AMD_AGENT_INFO_PRODUCT_NAME,
|
|
device_name)) {
|
|
::strncpy(info_.boardName_, device_name, sizeof(info_.boardName_) - 1);
|
|
}
|
|
|
|
char unique_id[32] = {0};
|
|
if (HSA_STATUS_SUCCESS ==
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_UUID),
|
|
unique_id)) {
|
|
// ROCr gives the UUID info in the format GPU-XXXX with length 20 bytes
|
|
// Strip the first 4 bytes and store only the 16 bytes representing UUID
|
|
std::memcpy(info_.uuid_, unique_id + 4, sizeof(info_.uuid_));
|
|
}
|
|
if (HSA_STATUS_SUCCESS ==
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_CUID),
|
|
unique_id)) {
|
|
std::memcpy(info_.cuid_, unique_id, sizeof(info_.cuid_));
|
|
}
|
|
hsa_luid_t localUID = {0};
|
|
if (HSA_STATUS_SUCCESS ==
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_LUID),
|
|
&localUID)) {
|
|
info_.luidLowPart_ = localUID.low;
|
|
info_.luidHighPart_ = localUID.high;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
(amd::IS_HIP)
|
|
? (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COOPERATIVE_COMPUTE_UNIT_COUNT
|
|
: (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT,
|
|
&info_.maxComputeUnits_)) {
|
|
return false;
|
|
}
|
|
assert(info_.maxComputeUnits_ > 0);
|
|
|
|
info_.maxComputeUnits_ =
|
|
settings().enableWgpMode_ ? info_.maxComputeUnits_ / 2 : info_.maxComputeUnits_;
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT,
|
|
&info_.maxPhysicalComputeUnits_)) {
|
|
return false;
|
|
}
|
|
assert(info_.maxPhysicalComputeUnits_ > 0);
|
|
|
|
info_.maxPhysicalComputeUnits_ = settings().enableWgpMode_ ? info_.maxPhysicalComputeUnits_ / 2
|
|
: info_.maxPhysicalComputeUnits_;
|
|
|
|
if (HSA_STATUS_SUCCESS != Hsa::agent_get_info(bkendDevice_,
|
|
(hsa_agent_info_t)HSA_AMD_AGENT_INFO_CACHELINE_SIZE,
|
|
&info_.globalMemCacheLineSize_)) {
|
|
return false;
|
|
}
|
|
info_.globalMemCacheLineSize_ =
|
|
(info_.globalMemCacheLineSize_ != 0) ? info_.globalMemCacheLineSize_ : 64;
|
|
|
|
uint32_t cachesize[4] = {0};
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_CACHE_SIZE, cachesize)) {
|
|
return false;
|
|
}
|
|
|
|
if (info_.globalMemCacheLineSize_ < 256 &&
|
|
(isa().versionMajor() >= 13 ||
|
|
(isa().versionMajor() == 12 && isa().versionMinor() >= 5))) {
|
|
info_.globalMemCacheLineSize_ = 256;
|
|
}
|
|
|
|
{
|
|
uint32_t maxPrefetchRegions = 0;
|
|
if (HSA_STATUS_SUCCESS ==
|
|
Hsa::agent_get_info(
|
|
bkendDevice_,
|
|
(hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_DATA_PREFETCH_REGIONS,
|
|
&maxPrefetchRegions)) {
|
|
info_.maxDynDataPrefetchRegions_ = maxPrefetchRegions;
|
|
} else {
|
|
info_.maxDynDataPrefetchRegions_ = 0;
|
|
}
|
|
}
|
|
|
|
assert(cachesize[0] > 0);
|
|
info_.globalMemCacheSize_ = cachesize[0];
|
|
|
|
info_.globalMemCacheType_ = CL_READ_WRITE_CACHE;
|
|
|
|
info_.type_ = CL_DEVICE_TYPE_GPU;
|
|
|
|
info_.extensions_ = getExtensionString();
|
|
info_.nativeVectorWidthDouble_ = info_.preferredVectorWidthDouble_ =
|
|
(settings().doublePrecision_) ? 1 : 0;
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY,
|
|
&info_.maxEngineClockFrequency_)) {
|
|
return false;
|
|
}
|
|
|
|
if (!(isa().versionMajor() == 9 && isa().versionMinor() == 0 && isa().versionStepping() == 2)) {
|
|
if (info_.maxEngineClockFrequency_ <= 0) {
|
|
LogError("maxEngineClockFrequency_ is NOT positive!");
|
|
}
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_MAX_FREQUENCY,
|
|
&info_.maxMemoryClockFrequency_)) {
|
|
return false;
|
|
}
|
|
|
|
uint64_t wallClockFrequency = 0; // in Hz
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_TIMESTAMP_FREQUENCY,
|
|
&wallClockFrequency)) {
|
|
LogWarning("HSA_AMD_AGENT_INFO_TIMESTAMP_FREQUENCY cannot be queried. Ignored!");
|
|
}
|
|
info_.wallClockFrequency_ = static_cast<uint32_t>(wallClockFrequency / 1000); // in KHz
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_DRIVER_NODE_ID),
|
|
&info_.driverNodeId_)) {
|
|
return false;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SDMA_ENG),
|
|
&info_.numSDMAengines_)) {
|
|
return false;
|
|
}
|
|
|
|
uint64_t scratchLimitMax = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_MAX,
|
|
&scratchLimitMax)) {
|
|
LogWarning("HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_MAX cannot be queried!");
|
|
return false;
|
|
}
|
|
info_.scratchLimitMin = 0;
|
|
info_.scratchLimitMax = scratchLimitMax;
|
|
|
|
checkAtomicSupport();
|
|
|
|
assert(cpu_agent_info_->fine_grain_pool.handle != 0);
|
|
if (HSA_STATUS_SUCCESS != Hsa::agent_iterate_memory_pools(
|
|
bkendDevice_, Device::iterateGpuMemoryPoolCallback, this)) {
|
|
return false;
|
|
}
|
|
|
|
assert(group_segment_.handle != 0);
|
|
|
|
for (auto agent : gpu_agents_) {
|
|
if (agent.handle != bkendDevice_.handle) {
|
|
hsa_status_t err;
|
|
// Can another GPU (agent) have access to the current GPU memory pool (gpuvm_segment_)?
|
|
hsa_amd_memory_pool_access_t access;
|
|
err = Hsa::agent_memory_pool_get_info(agent, gpuvm_segment_,
|
|
HSA_AMD_AGENT_MEMORY_POOL_INFO_ACCESS, &access);
|
|
if (err != HSA_STATUS_SUCCESS) {
|
|
continue;
|
|
}
|
|
|
|
// Find accessible p2p agents - i.e != HSA_AMD_MEMORY_POOL_ACCESS_NEVER_ALLOWED
|
|
if (HSA_AMD_MEMORY_POOL_ACCESS_ALLOWED_BY_DEFAULT == access ||
|
|
HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT == access) {
|
|
// Agent can have access to the current gpuvm_segment_
|
|
p2p_agents_.push_back(agent);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Keep track of all P2P Agents in a Array including current device handle for IPC
|
|
p2p_agents_list_ = new hsa_agent_t[1 + p2p_agents_.size()];
|
|
p2p_agents_list_[0] = getBackendDevice();
|
|
for (size_t agent_idx = 0; agent_idx < p2p_agents_.size(); ++agent_idx) {
|
|
p2p_agents_list_[1 + agent_idx] = p2p_agents_[agent_idx];
|
|
}
|
|
|
|
size_t group_segment_size = 0;
|
|
if (HSA_STATUS_SUCCESS != Hsa::memory_pool_get_info(group_segment_,
|
|
HSA_AMD_MEMORY_POOL_INFO_SIZE,
|
|
&group_segment_size)) {
|
|
return false;
|
|
}
|
|
assert(group_segment_size > 0);
|
|
|
|
// Find SDMA read mask
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::memory_copy_engine_status(getCpuAgent(), getBackendDevice(), &maxSdmaReadMask_)) {
|
|
return false;
|
|
}
|
|
assert(maxSdmaReadMask_ > 0 && "No SDMA engines available for Read");
|
|
|
|
// Find SDMA write mask
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::memory_copy_engine_status(getBackendDevice(), getCpuAgent(), &maxSdmaWriteMask_)) {
|
|
return false;
|
|
}
|
|
assert(maxSdmaWriteMask_ > 0 && "No SDMA engines available for Write");
|
|
|
|
info_.localMemSizePerCU_ = group_segment_size;
|
|
info_.localMemSize_ = group_segment_size;
|
|
|
|
info_.maxWorkItemDimensions_ = 3;
|
|
|
|
uint8_t memory_properties[8];
|
|
// Get the memory property from ROCr.
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_PROPERTIES,
|
|
memory_properties)) {
|
|
LogError("HSA_AGENT_INFO_AMD_MEMORY_PROPERTIES query failed");
|
|
}
|
|
|
|
// Check if the device is APU
|
|
if (hsa_flag_isset64(memory_properties, HSA_AMD_MEMORY_PROPERTY_AGENT_IS_APU)) {
|
|
info_.hostUnifiedMemory_ = 1;
|
|
}
|
|
|
|
if (settings().enableLocalMemory_ && gpuvm_segment_.handle != 0) {
|
|
size_t global_segment_size = 0;
|
|
if (HSA_STATUS_SUCCESS != Hsa::memory_pool_get_info(gpuvm_segment_,
|
|
HSA_AMD_MEMORY_POOL_INFO_SIZE,
|
|
&global_segment_size)) {
|
|
return false;
|
|
}
|
|
|
|
assert(global_segment_size > 0);
|
|
info_.globalMemSize_ = (static_cast<uint64_t>(std::min(GPU_MAX_HEAP_SIZE, 100u)) *
|
|
static_cast<uint64_t>(global_segment_size)) /
|
|
100u;
|
|
|
|
// For APU with vram size <= 512MiB, use a smaller single alloc percentage
|
|
if (info_.globalMemSize_ <= 536870912) {
|
|
if (flagIsDefault(GPU_SINGLE_ALLOC_PERCENT)) {
|
|
GPU_SINGLE_ALLOC_PERCENT = 75;
|
|
}
|
|
}
|
|
|
|
gpuvm_segment_max_alloc_ =
|
|
uint64_t(info_.globalMemSize_ * std::min(GPU_SINGLE_ALLOC_PERCENT, 100u) / 100u);
|
|
assert(gpuvm_segment_max_alloc_ > 0);
|
|
|
|
info_.maxMemAllocSize_ = static_cast<uint64_t>(gpuvm_segment_max_alloc_);
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::memory_pool_get_info(gpuvm_segment_, HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
|
|
&alloc_granularity_)) {
|
|
return false;
|
|
}
|
|
|
|
assert(alloc_granularity_ > 0);
|
|
} else {
|
|
// We suppose half of physical memory can be used by GPU in APU system
|
|
info_.globalMemSize_ = amd::Os::hostTotalPhysicalMemory() / 2;
|
|
info_.globalMemSize_ = std::max(info_.globalMemSize_, uint64_t(1 * Gi));
|
|
info_.globalMemSize_ = (static_cast<uint64_t>(std::min(GPU_MAX_HEAP_SIZE, 100u)) *
|
|
static_cast<uint64_t>(info_.globalMemSize_)) /
|
|
100u;
|
|
|
|
info_.maxMemAllocSize_ =
|
|
uint64_t(info_.globalMemSize_ * std::min(GPU_SINGLE_ALLOC_PERCENT, 100u) / 100u);
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::memory_pool_get_info(cpu_agent_info_->fine_grain_pool,
|
|
HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE,
|
|
&alloc_granularity_)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
freeMem_ = info_.globalMemSize_;
|
|
|
|
// Make sure the max allocation size is not larger than the available memory size.
|
|
info_.maxMemAllocSize_ = std::min(info_.maxMemAllocSize_, info_.globalMemSize_);
|
|
info_.maxMemAllocSize_ = amd::alignDown(info_.maxMemAllocSize_, sizeof(uint64_t));
|
|
|
|
// Maximum system memory allocation size allowed
|
|
info_.maxPhysicalMemAllocSize_ = amd::Os::getPhysicalMemSize();
|
|
|
|
// make sure we don't run anything over 8 params for now
|
|
info_.maxParameterSize_ = 1024;
|
|
|
|
uint32_t max_work_group_size = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WORKGROUP_MAX_SIZE, &max_work_group_size)) {
|
|
return false;
|
|
}
|
|
assert(max_work_group_size > 0);
|
|
max_work_group_size =
|
|
std::min(max_work_group_size, static_cast<uint32_t>(settings().maxWorkGroupSize_));
|
|
info_.maxWorkGroupSize_ = max_work_group_size;
|
|
|
|
uint16_t max_workgroup_size[3] = {0, 0, 0};
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WORKGROUP_MAX_DIM, &max_workgroup_size)) {
|
|
return false;
|
|
}
|
|
assert(max_workgroup_size[0] != 0 && max_workgroup_size[1] != 0 && max_workgroup_size[2] != 0);
|
|
|
|
uint16_t max_work_item_size = static_cast<uint16_t>(max_work_group_size);
|
|
info_.maxWorkItemSizes_[0] = std::min(max_workgroup_size[0], max_work_item_size);
|
|
info_.maxWorkItemSizes_[1] = std::min(max_workgroup_size[1], max_work_item_size);
|
|
info_.maxWorkItemSizes_[2] = std::min(max_workgroup_size[2], max_work_item_size);
|
|
info_.preferredWorkGroupSize_ = settings().preferredWorkGroupSize_;
|
|
|
|
hsa_dim3_t grid_max_dim = {0, 0, 0};
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_GRID_MAX_DIM, &grid_max_dim)) {
|
|
return false;
|
|
}
|
|
assert(grid_max_dim.x != 0 && grid_max_dim.y != 0 && grid_max_dim.z != 0);
|
|
info_.maxGridDim_[0] = std::min(grid_max_dim.x, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
|
|
info_.maxGridDim_[1] = std::min(grid_max_dim.y, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
|
|
info_.maxGridDim_[2] = std::min(grid_max_dim.z, static_cast<uint32_t>(std::numeric_limits<int32_t>::max()));
|
|
|
|
info_.nativeVectorWidthChar_ = info_.preferredVectorWidthChar_ = 4;
|
|
info_.nativeVectorWidthShort_ = info_.preferredVectorWidthShort_ = 2;
|
|
info_.nativeVectorWidthInt_ = info_.preferredVectorWidthInt_ = 1;
|
|
info_.nativeVectorWidthLong_ = info_.preferredVectorWidthLong_ = 1;
|
|
info_.nativeVectorWidthFloat_ = info_.preferredVectorWidthFloat_ = 1;
|
|
|
|
if (agent_profile_ == HSA_PROFILE_FULL) { // full-profile = participating in coherent memory,
|
|
// base-profile = NUMA based non-coherent memory
|
|
info_.hostUnifiedMemory_ = 1;
|
|
info_.iommuv2_ = true;
|
|
}
|
|
info_.memBaseAddrAlign_ = 8 * (flagIsDefault(MEMOBJ_BASE_ADDR_ALIGN) ? sizeof(int64_t[16]) * 2
|
|
: MEMOBJ_BASE_ADDR_ALIGN);
|
|
info_.minDataTypeAlignSize_ = sizeof(int64_t[16]);
|
|
|
|
info_.maxConstantArgs_ = 8;
|
|
info_.preferredConstantBufferSize_ = 16 * Ki;
|
|
info_.maxConstantBufferSize_ = info_.maxMemAllocSize_;
|
|
info_.localMemType_ = CL_LOCAL;
|
|
info_.errorCorrectionSupport_ = false;
|
|
info_.profilingTimerResolution_ = 1;
|
|
info_.littleEndian_ = true;
|
|
info_.compilerAvailable_ = true;
|
|
info_.executionCapabilities_ = CL_EXEC_KERNEL;
|
|
info_.queueProperties_ = CL_QUEUE_PROFILING_ENABLE;
|
|
info_.platform_ = AMD_PLATFORM;
|
|
info_.profile_ = "FULL_PROFILE";
|
|
::strncpy(info_.vendor_, "Advanced Micro Devices, Inc.", sizeof(info_.vendor_) - 1);
|
|
|
|
info_.addressBits_ = LP64_SWITCH(32, 64);
|
|
info_.maxSamplers_ = 16;
|
|
info_.bufferFromImageSupport_ = false;
|
|
info_.oclcVersion_ = "OpenCL C " OPENCL_C_VERSION_STR " ";
|
|
info_.spirVersions_ = "";
|
|
|
|
uint16_t major, minor;
|
|
if (Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_VERSION_MAJOR, &major) !=
|
|
HSA_STATUS_SUCCESS ||
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_VERSION_MINOR, &minor) !=
|
|
HSA_STATUS_SUCCESS) {
|
|
return false;
|
|
}
|
|
std::stringstream ss;
|
|
ss << AMD_BUILD_STRING " (HSA" << major << "." << minor << ",LC)";
|
|
|
|
::strncpy(info_.driverVersion_, ss.str().c_str(), sizeof(info_.driverVersion_) - 1);
|
|
|
|
if (isa().versionMajor() >= 9) {
|
|
info_.version_ =
|
|
"OpenCL " /*OPENCL_VERSION_STR*/
|
|
"2.0"
|
|
" ";
|
|
} else {
|
|
info_.version_ =
|
|
"OpenCL " /*OPENCL_VERSION_STR*/
|
|
"1.2"
|
|
" ";
|
|
}
|
|
|
|
info_.builtInKernels_ = "";
|
|
info_.linkerAvailable_ = true;
|
|
info_.preferredInteropUserSync_ = true;
|
|
info_.printfBufferSize_ = PrintfDbg::WorkitemDebugSize * info().maxWorkGroupSize_;
|
|
info_.vendorId_ = 0x1002; // AMD's PCIe vendor id
|
|
|
|
info_.maxGlobalVariableSize_ = static_cast<size_t>(info_.maxMemAllocSize_);
|
|
info_.globalVariablePreferredTotalSize_ = static_cast<size_t>(info_.globalMemSize_);
|
|
|
|
// Populate the single config setting.
|
|
info_.singleFPConfig_ =
|
|
CL_FP_ROUND_TO_NEAREST | CL_FP_ROUND_TO_ZERO | CL_FP_ROUND_TO_INF | CL_FP_INF_NAN | CL_FP_FMA;
|
|
|
|
if (settings().doublePrecision_) {
|
|
info_.doubleFPConfig_ = info_.singleFPConfig_ | CL_FP_DENORM;
|
|
info_.singleFPConfig_ |= CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT;
|
|
}
|
|
|
|
if (settings().singleFpDenorm_) {
|
|
info_.singleFPConfig_ |= CL_FP_DENORM;
|
|
}
|
|
|
|
if (settings().checkExtension(ClKhrFp16)) {
|
|
info_.halfFPConfig_ = info_.singleFPConfig_;
|
|
}
|
|
|
|
info_.preferredPlatformAtomicAlignment_ = 0;
|
|
info_.preferredGlobalAtomicAlignment_ = 0;
|
|
info_.preferredLocalAtomicAlignment_ = 0;
|
|
|
|
uint8_t hsa_extensions[128];
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_EXTENSIONS, hsa_extensions)) {
|
|
return false;
|
|
}
|
|
|
|
assert(HSA_EXTENSION_IMAGES < 8);
|
|
const bool image_is_supported = ((hsa_extensions[0] & (1 << HSA_EXTENSION_IMAGES)) != 0);
|
|
if (image_is_supported) {
|
|
// Images
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_SAMPLER_HANDLERS),
|
|
&info_.maxSamplers_)) {
|
|
return false;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_IMAGE_RD_HANDLES),
|
|
&info_.maxReadImageArgs_)) {
|
|
return false;
|
|
}
|
|
|
|
// TODO: no attribute for write image.
|
|
info_.maxWriteImageArgs_ = 8;
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_MAX_IMAGE_RORW_HANDLES),
|
|
&info_.maxReadWriteImageArgs_)) {
|
|
return false;
|
|
}
|
|
|
|
uint32_t image_max_dim[3];
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_2D_MAX_ELEMENTS),
|
|
&image_max_dim)) {
|
|
return false;
|
|
}
|
|
|
|
info_.image2DMaxWidth_ = image_max_dim[0];
|
|
info_.image2DMaxHeight_ = image_max_dim[1];
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_3D_MAX_ELEMENTS),
|
|
&image_max_dim)) {
|
|
return false;
|
|
}
|
|
|
|
info_.image3DMaxWidth_ = image_max_dim[0];
|
|
info_.image3DMaxHeight_ = image_max_dim[1];
|
|
info_.image3DMaxDepth_ = image_max_dim[2];
|
|
|
|
uint32_t max_array_size = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_ARRAY_MAX_LAYERS),
|
|
&max_array_size)) {
|
|
return false;
|
|
}
|
|
|
|
info_.imageMaxArraySize_ = max_array_size;
|
|
|
|
uint32_t max_image1da_width = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1DA_MAX_ELEMENTS),
|
|
&max_image1da_width)) {
|
|
return false;
|
|
}
|
|
|
|
info_.image1DAMaxWidth_ = max_image1da_width;
|
|
|
|
uint32_t max_image2da_width[2] = {0, 0};
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_2DA_MAX_ELEMENTS),
|
|
&max_image2da_width)) {
|
|
return false;
|
|
}
|
|
|
|
info_.image2DAMaxWidth_[0] = max_image2da_width[0];
|
|
info_.image2DAMaxWidth_[1] = max_image2da_width[1];
|
|
|
|
uint32_t max_image1d_width = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1D_MAX_ELEMENTS),
|
|
&max_image1d_width)) {
|
|
return false;
|
|
}
|
|
info_.image1DMaxWidth_ = max_image1d_width;
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_EXT_AGENT_INFO_IMAGE_1DB_MAX_ELEMENTS),
|
|
&image_max_dim)) {
|
|
return false;
|
|
}
|
|
info_.imageMaxBufferSize_ = (amd::IS_HIP) ? image_max_dim[0] : (1 << 27);
|
|
|
|
info_.imageSupport_ = (info_.maxReadWriteImageArgs_ > 0) ? true : false;
|
|
}
|
|
|
|
// These are properties of the device's linear memory layout, not the image
|
|
// extension. They must be set unconditionally so that hipMallocPitch /
|
|
// hipMalloc3D produce correct pitch alignment even when IMAGE_SUPPORT is
|
|
// off. The PAL backend already sets them unconditionally.
|
|
info_.imagePitchAlignment_ = 256;
|
|
info_.imageBaseAddressAlignment_ = 256;
|
|
info_.bufferFromImageSupport_ = false;
|
|
|
|
// Enable SVM Capabilities of Hsa device. Ensure
|
|
// user has not setup memory to be non-coherent
|
|
info_.svmCapabilities_ = 0;
|
|
if (!settings().enableNCMode_) {
|
|
info_.svmCapabilities_ = CL_DEVICE_SVM_COARSE_GRAIN_BUFFER;
|
|
info_.svmCapabilities_ |= CL_DEVICE_SVM_FINE_GRAIN_BUFFER;
|
|
// Report fine-grain system only on full profile
|
|
if (agent_profile_ == HSA_PROFILE_FULL) {
|
|
info_.svmCapabilities_ |= CL_DEVICE_SVM_FINE_GRAIN_SYSTEM;
|
|
}
|
|
if (amd::IS_HIP) {
|
|
if (info_.iommuv2_ || isa().versionMajor() >= 8) {
|
|
info_.svmCapabilities_ |= CL_DEVICE_SVM_ATOMICS;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (settings().checkExtension(ClAmdDeviceAttributeQuery)) {
|
|
info_.simdWidth_ = isa().simdWidth();
|
|
info_.simdInstructionWidth_ = isa().simdInstructionWidth();
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_WAVEFRONT_SIZE, &info_.wavefrontWidth_)) {
|
|
return false;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_WIDTH),
|
|
&info_.vramBusBitWidth_)) {
|
|
return false;
|
|
}
|
|
|
|
info_.globalMemChannels_ = info_.vramBusBitWidth_ / 32;
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU),
|
|
&info_.simdPerCU_)) {
|
|
return false;
|
|
}
|
|
|
|
uint32_t max_waves_per_cu = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU),
|
|
&max_waves_per_cu)) {
|
|
return false;
|
|
}
|
|
|
|
if (settings().enableWgpMode_) {
|
|
info_.simdPerCU_ *= 2;
|
|
max_waves_per_cu *= 2;
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
hsa_agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SHADER_ENGINES),
|
|
&info_.numberOfShaderEngines_)) {
|
|
return false;
|
|
}
|
|
|
|
info_.maxThreadsPerCU_ = info_.wavefrontWidth_ * max_waves_per_cu;
|
|
uint32_t cache_sizes[4];
|
|
/* FIXIT [skudchad] - Seems like hardcoded in HSA backend so 0*/
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AGENT_INFO_CACHE_SIZE),
|
|
cache_sizes)) {
|
|
return false;
|
|
}
|
|
|
|
uint32_t asic_revision = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_ASIC_REVISION),
|
|
&asic_revision)) {
|
|
return false;
|
|
}
|
|
info_.asicRevision_ = asic_revision;
|
|
|
|
info_.l2CacheSize_ = cache_sizes[1];
|
|
info_.timeStampFrequency_ = 1000000;
|
|
info_.globalMemChannelBanks_ = 4;
|
|
info_.globalMemChannelBankWidth_ = isa().memChannelBankWidth();
|
|
info_.localMemSizePerCU_ = isa().localMemSizePerCU();
|
|
info_.localMemBanks_ = isa().localMemBanks();
|
|
info_.numAsyncQueues_ = kMaxAsyncQueues;
|
|
info_.numRTQueues_ = info_.numAsyncQueues_;
|
|
info_.numRTCUs_ = info_.maxComputeUnits_;
|
|
|
|
// TODO: set to true once thread trace support is available
|
|
info_.threadTraceEnable_ = false;
|
|
info_.pcieDeviceId_ = pciDeviceId_;
|
|
info_.cooperativeGroups_ = settings().enableCoopGroups_;
|
|
info_.cooperativeMultiDeviceGroups_ = settings().enableCoopMultiDeviceGroups_;
|
|
// Enable StreamWrite and StreamWait for all devices
|
|
info_.aqlBarrierValue_ = true;
|
|
info_.movdir64b_ = amd::hasMovdir64b() && (DEBUG_CLR_USE_MOVDIR64B != 0);
|
|
}
|
|
|
|
info_.maxPipePacketSize_ = info_.maxMemAllocSize_;
|
|
info_.maxPipeActiveReservations_ = 16;
|
|
info_.maxPipeArgs_ = 16;
|
|
|
|
info_.queueOnDeviceProperties_ =
|
|
CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE | CL_QUEUE_PROFILING_ENABLE;
|
|
info_.queueOnDevicePreferredSize_ = 256 * Ki;
|
|
info_.queueOnDeviceMaxSize_ = 8 * Mi;
|
|
info_.maxOnDeviceQueues_ = 1;
|
|
info_.maxOnDeviceEvents_ = settings().numDeviceEvents_;
|
|
|
|
std::string addressableNumVGPRs, totalNumVGPRs, vGPRAllocGranule;
|
|
std::string isaName = isa().isaName();
|
|
info_.availableVGPRs_ =
|
|
amd::device::getValueFromIsaMeta(isaName, "AddressableNumVGPRs", addressableNumVGPRs)
|
|
? atoi(addressableNumVGPRs.c_str())
|
|
: 0;
|
|
info_.vgprsPerSimd_ = amd::device::getValueFromIsaMeta(isaName, "TotalNumVGPRs", totalNumVGPRs)
|
|
? atoi(totalNumVGPRs.c_str())
|
|
: 0;
|
|
info_.vgprAllocGranularity_ =
|
|
amd::device::getValueFromIsaMeta(isaName, "VGPRAllocGranule", vGPRAllocGranule)
|
|
? atoi(vGPRAllocGranule.c_str())
|
|
: 0;
|
|
|
|
info_.availableRegistersPerCU_ = info_.vgprsPerSimd_ * info_.simdPerCU_ * info_.wavefrontWidth_;
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT,
|
|
"addressableNumVGPRs=%u, totalNumVGPRs=%u, vGPRAllocGranule=%u,"
|
|
" availableRegistersPerCU_=%u",
|
|
info_.availableVGPRs_, info_.vgprsPerSimd_, info_.vgprAllocGranularity_,
|
|
info_.availableRegistersPerCU_);
|
|
|
|
std::string sgprValue;
|
|
info_.availableSGPRs_ =
|
|
(amd::device::getValueFromIsaMeta(isaName, "AddressableNumSGPRs", sgprValue))
|
|
? (atoi(sgprValue.c_str()))
|
|
: 0;
|
|
std::string imageSupport;
|
|
if (amd::device::getValueFromIsaMeta(isaName, "ImageSupport", imageSupport)) {
|
|
info_.imageSupport_ =
|
|
(DEBUG_CLR_DISABLE_IMAGE || !image_is_supported) ? 0 : std::atoi(imageSupport.c_str());
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "imageSupport=%u", info_.imageSupport_);
|
|
} else {
|
|
LogInfo("Can not get image support info from ISA meta");
|
|
}
|
|
|
|
// Generic support for HMM interfaces
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &info_.hmmSupported_)) {
|
|
LogError("HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED query failed. HMM will be disabled");
|
|
}
|
|
|
|
// This capability should be available with xnack enabled
|
|
if (HSA_STATUS_SUCCESS != Hsa::system_get_info(HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT,
|
|
&info_.hmmCpuMemoryAccessible_)) {
|
|
LogError("HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT query failed.");
|
|
}
|
|
|
|
// HMM specific capability for CPU direct access to device memory
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_SVM_DIRECT_HOST_ACCESS),
|
|
&info_.hmmDirectHostAccess_)) {
|
|
LogError("HSA_AMD_AGENT_INFO_SVM_DIRECT_HOST_ACCESS query failed.");
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_XCC),
|
|
&info_.numberOfXccs_)) {
|
|
LogError("HSA_AMD_AGENT_INFO_NUM_XCC query failed.");
|
|
}
|
|
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_PM4_EMULATION),
|
|
&pm4_emulation_)) {
|
|
LogError("HSA_AMD_AGENT_INFO_PM4_EMULATION query failed.");
|
|
}
|
|
|
|
info_.hasExpertSchedMode_ = false;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE),
|
|
&info_.hasExpertSchedMode_)) {
|
|
LogWarning("HSA_AMD_AGENT_INFO_HAS_EXPERT_SCHED_MODE query failed.");
|
|
}
|
|
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Gfx Major/Minor/Stepping: %d/%d/%d, Device ID: 0x%x",
|
|
isa().versionMajor(), isa().versionMinor(), isa().versionStepping(), pciDeviceId_);
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Using dev kernel arg wa = %d", settings().kernel_arg_impl_);
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "HMM support: %d, XNACK: %d, Direct host access: %d",
|
|
info_.hmmSupported_, info_.hmmCpuMemoryAccessible_, info_.hmmDirectHostAccess_);
|
|
ClPrint(amd::LOG_INFO, amd::LOG_INIT, "Max SDMA Read Mask: 0x%x, Max SDMA Write Mask: 0x%x",
|
|
maxSdmaReadMask_, maxSdmaWriteMask_);
|
|
|
|
info_.globalCUMask_ = {};
|
|
|
|
// Virtual memory Management Support, if set to true then the HW and SW Stack supports VMM.
|
|
info_.virtualMemoryManagement_ = false;
|
|
if (HIP_VMEM_MANAGE_SUPPORT) {
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::system_get_info(
|
|
static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED),
|
|
&info_.virtualMemoryManagement_)) {
|
|
LogError("HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED query failed ");
|
|
}
|
|
}
|
|
HIP_MEM_POOL_USE_VM &= info_.virtualMemoryManagement_;
|
|
|
|
// Check Support for Buffer sharing with dma_buf
|
|
std::ignore = Hsa::system_get_info(
|
|
static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_DMABUF_SUPPORTED),
|
|
&info_.dmabufSupported_);
|
|
// devices with no cluster support; max size is 0
|
|
info_.clusterMaxSize_ = 0;
|
|
|
|
hsa_status_t hsaStatus = Hsa::agent_get_info(
|
|
bkendDevice_, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_CLUSTER_MAX_SIZE),
|
|
&info_.clusterMaxSize_);
|
|
|
|
|
|
// this is required for clustered kernel launches; but it might not be supported in older rocr,
|
|
// so invalid argument might no be necessarily an error
|
|
if (HSA_STATUS_SUCCESS != hsaStatus && HSA_STATUS_ERROR_INVALID_ARGUMENT != hsaStatus) {
|
|
LogError("HSA_AMD_AGENT_INFO_CLUSTER_MAX_SIZE query failed");
|
|
} else {
|
|
// this is a temporary override of the ROCr result (when there is cluster support). This line
|
|
// will be removed as soon as ROCr provides the correct result
|
|
info_.clusterMaxSize_ = info_.clusterMaxSize_ > 1?
|
|
info_.maxComputeUnits_ / info_.numberOfShaderEngines_ :
|
|
1;
|
|
}
|
|
|
|
info_.gpuDirectRdmaWithHipVmmSupported_ =
|
|
info_.virtualMemoryManagement_ && info_.dmabufSupported_;
|
|
|
|
if (isa().versionMajor() < 8) {
|
|
info_.sgprsPerSimd_ = 512;
|
|
} else if (isa().versionMajor() < 10) {
|
|
info_.sgprsPerSimd_ = 800;
|
|
} else {
|
|
info_.sgprsPerSimd_ =
|
|
std::numeric_limits<uint32_t>::max(); // gfx10+ does not share SGPRs between waves
|
|
}
|
|
|
|
// Flag for fabric handle support
|
|
info_.fabric_handle_ = false;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::system_get_info(
|
|
static_cast<hsa_system_info_t>(HSA_AMD_SYSTEM_INFO_FABRIC_HANDLES_SUPPORTED),
|
|
&info_.fabric_handle_)) {
|
|
LogError("HSA_AMD_SYSTEM_INFO_FABRIC_HANDLES_SUPPORTED query failed ");
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
|
|
std::scoped_lock lock(vgpusAccess());
|
|
|
|
bool profiling = (queue != nullptr) && queue->properties().test(CL_QUEUE_PROFILING_ENABLE);
|
|
bool cooperative = false;
|
|
bool dedicated_queue = (queue != nullptr) && queue->isDedicatedQueue() &&
|
|
(settings().dynamic_queues_ >= 2);
|
|
|
|
// If amd command queue is null, then it's an internal device queue
|
|
if (queue == nullptr) {
|
|
// In HIP mode the device queue will be allocated for the cooperative launches only
|
|
cooperative = amd::IS_HIP && settings().enableCoopGroups_;
|
|
profiling = amd::IS_HIP;
|
|
}
|
|
// Initialization of heap and other resources occur during the command
|
|
// queue creation time.
|
|
const std::vector<uint32_t> defaultCuMask = {};
|
|
bool q = (queue != nullptr);
|
|
VirtualGPU* virtualDevice =
|
|
new VirtualGPU(*this, profiling, cooperative, q ? queue->cuMask() : defaultCuMask,
|
|
q ? queue->priority() : amd::CommandQueue::Priority::Normal,
|
|
dedicated_queue);
|
|
|
|
if (!virtualDevice->create()) {
|
|
delete virtualDevice;
|
|
return nullptr;
|
|
}
|
|
|
|
return virtualDevice;
|
|
}
|
|
|
|
bool Device::globalFreeMemory(size_t* freeMemory) const {
|
|
const uint TotalFreeMemory = 0;
|
|
const uint LargestFreeBlock = 1;
|
|
uint64_t globalAvailMemory;
|
|
// Queries memory available in bytes across all global pools owned by the agent
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_,
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_AVAIL),
|
|
&globalAvailMemory)) {
|
|
LogError("HSA_AMD_AGENT_INFO_MEMORY_AVAIL query failed.");
|
|
return false;
|
|
}
|
|
|
|
globalAvailMemory = globalAvailMemory / Ki;
|
|
if (globalAvailMemory > HIP_HIDDEN_FREE_MEM * Ki) {
|
|
globalAvailMemory -= HIP_HIDDEN_FREE_MEM * Ki;
|
|
} else {
|
|
globalAvailMemory = 0;
|
|
}
|
|
|
|
freeMemory[TotalFreeMemory] = globalAvailMemory;
|
|
// since there is no memory heap on ROCm, the biggest free block is
|
|
// equal to total free local memory
|
|
freeMemory[LargestFreeBlock] = freeMemory[TotalFreeMemory];
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Device::amdFileRead(amd::Os::FileDesc handle, void* devicePtr, uint64_t size, int64_t file_offset,
|
|
uint64_t* size_copied, int32_t* status) {
|
|
hsa_amd_ais_file_handle_t fh{};
|
|
hsa_status_t ret = HSA_STATUS_ERROR;
|
|
#if defined(_WIN32)
|
|
fh.handle = handle;
|
|
#else
|
|
fh.fd = handle;
|
|
ret = Hsa::ais_file_read(fh, devicePtr, size, file_offset, size_copied, status);
|
|
#endif
|
|
|
|
if (HSA_STATUS_SUCCESS != ret) {
|
|
LogPrintfError("amdFileRead operation failed with err 0x%xh", ret);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool Device::amdFileWrite(amd::Os::FileDesc handle, void* devicePtr, uint64_t size, int64_t file_offset,
|
|
uint64_t* size_copied, int32_t* status) {
|
|
hsa_amd_ais_file_handle_t fh{};
|
|
hsa_status_t ret = HSA_STATUS_ERROR;
|
|
#if defined(_WIN32)
|
|
fh.handle = handle;
|
|
#else
|
|
fh.fd = handle;
|
|
ret = Hsa::ais_file_write(fh, devicePtr, size, file_offset, size_copied, status);
|
|
#endif
|
|
|
|
if (HSA_STATUS_SUCCESS != ret) {
|
|
LogPrintfError("amdFileWrite operation failed with err 0x%xh", ret);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::bindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext,
|
|
bool validateOnly) {
|
|
bool success = true;
|
|
|
|
#ifdef _WIN32
|
|
// Handle D3D10 device binding
|
|
if (flags & amd::Context::Flags::D3D10DeviceKhr) {
|
|
void* d3d10Device = gfxDevice[amd::Context::DeviceFlagIdx::D3D10DeviceKhrIdx];
|
|
if (!D3D10Interop::associateD3D10Device(this, static_cast<ID3D10Device*>(d3d10Device),
|
|
gfxContext, validateOnly)) {
|
|
if (!validateOnly) {
|
|
LogError("Failed D3D10Interop::associateD3D10Device()");
|
|
}
|
|
success = false;
|
|
}
|
|
}
|
|
|
|
// Handle D3D11 device binding
|
|
if (flags & amd::Context::Flags::D3D11DeviceKhr) {
|
|
void* d3d11Device = gfxDevice[amd::Context::DeviceFlagIdx::D3D11DeviceKhrIdx];
|
|
if (!D3D11Interop::associateD3D11Device(this, static_cast<ID3D11Device*>(d3d11Device),
|
|
gfxContext, validateOnly)) {
|
|
if (!validateOnly) {
|
|
LogError("Failed D3D11Interop::associateD3D11Device()");
|
|
}
|
|
success = false;
|
|
}
|
|
}
|
|
#endif // _WIN32
|
|
|
|
// Handle GL device binding (existing code)
|
|
if (flags & amd::Context::Flags::GLDeviceKhr) {
|
|
void* glDevice = gfxDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx];
|
|
if (!GlInterop::glAssociate(this, flags, gfxContext, glDevice, validateOnly)) {
|
|
if (!validateOnly) {
|
|
LogError("Failed GlInterop::glAssociate()");
|
|
}
|
|
success = false;
|
|
}
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::unbindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext,
|
|
bool validateOnly) {
|
|
bool success = true;
|
|
|
|
#ifdef _WIN32
|
|
// Handle D3D10 device cleanup
|
|
if (flags & amd::Context::Flags::D3D10DeviceKhr) {
|
|
D3D10Interop::dissociateD3D10Device(this, gfxDevice, gfxContext);
|
|
}
|
|
|
|
// Handle D3D11 device cleanup
|
|
if (flags & amd::Context::Flags::D3D11DeviceKhr) {
|
|
D3D11Interop::dissociateD3D11Device(this, gfxDevice, gfxContext);
|
|
}
|
|
#endif // _WIN32
|
|
|
|
// Handle GL device cleanup (existing code)
|
|
if (flags & amd::Context::Flags::GLDeviceKhr) {
|
|
void* glDevice = gfxDevice[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx];
|
|
// validateOnly never started a session; nothing to dissociate.
|
|
if (glDevice != nullptr && !validateOnly) {
|
|
if (!GlInterop::glDissociate(this, gfxContext, glDevice)) {
|
|
LogWarning("Failed GlInterop::glDissociate()");
|
|
success = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
amd::Memory* Device::findMapTarget(size_t size) const {
|
|
// Must be serialised for access
|
|
std::scoped_lock lk(*mapCacheOps_);
|
|
|
|
amd::Memory* map = nullptr;
|
|
size_t minSize = 0;
|
|
size_t maxSize = 0;
|
|
uint mapId = mapCache_->size();
|
|
uint releaseId = mapCache_->size();
|
|
|
|
// Find if the list has a map target of appropriate size
|
|
for (uint i = 0; i < mapCache_->size(); i++) {
|
|
if ((*mapCache_)[i] != nullptr) {
|
|
// Requested size is smaller than the entry size
|
|
if (size < (*mapCache_)[i]->getSize()) {
|
|
if ((minSize == 0) || (minSize > (*mapCache_)[i]->getSize())) {
|
|
minSize = (*mapCache_)[i]->getSize();
|
|
mapId = i;
|
|
}
|
|
}
|
|
// Requeted size matches the entry size
|
|
else if (size == (*mapCache_)[i]->getSize()) {
|
|
mapId = i;
|
|
break;
|
|
} else {
|
|
// Find the biggest map target in the list
|
|
if (maxSize < (*mapCache_)[i]->getSize()) {
|
|
maxSize = (*mapCache_)[i]->getSize();
|
|
releaseId = i;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if we found any map target
|
|
if (mapId < mapCache_->size()) {
|
|
map = (*mapCache_)[mapId];
|
|
(*mapCache_)[mapId] = nullptr;
|
|
}
|
|
// If cache is full, then release the biggest map target
|
|
else if (releaseId < mapCache_->size()) {
|
|
(*mapCache_)[releaseId]->release();
|
|
(*mapCache_)[releaseId] = nullptr;
|
|
}
|
|
|
|
return map;
|
|
}
|
|
|
|
bool Device::addMapTarget(amd::Memory* memory) const {
|
|
// Must be serialised for access
|
|
std::scoped_lock lk(*mapCacheOps_);
|
|
|
|
// the svm memory shouldn't be cached
|
|
if (!memory->canBeCached()) {
|
|
return false;
|
|
}
|
|
// Find if the list has a map target of appropriate size
|
|
for (uint i = 0; i < mapCache_->size(); ++i) {
|
|
if ((*mapCache_)[i] == nullptr) {
|
|
(*mapCache_)[i] = memory;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// Add a new entry
|
|
mapCache_->push_back(memory);
|
|
|
|
return true;
|
|
}
|
|
|
|
Memory* Device::getRocMemory(amd::Memory* mem) const {
|
|
return static_cast<roc::Memory*>(mem->getDeviceMemory(*this));
|
|
}
|
|
|
|
// ================================================================================================
|
|
device::Memory* Device::createMemory(amd::Memory& owner) const {
|
|
roc::Memory* memory = nullptr;
|
|
if (owner.asBuffer()) {
|
|
memory = new roc::Buffer(*this, owner);
|
|
} else if (owner.asImage()) {
|
|
memory = new roc::Image(*this, owner);
|
|
} else {
|
|
LogError("Unknown memory type");
|
|
}
|
|
|
|
if (memory == nullptr) {
|
|
return nullptr;
|
|
}
|
|
|
|
bool result = memory->create();
|
|
|
|
if (!result) {
|
|
LogError("Failed creating memory");
|
|
delete memory;
|
|
return nullptr;
|
|
}
|
|
|
|
if (isP2pEnabled()) {
|
|
memory->setAllowedPeerAccess(true);
|
|
}
|
|
// Initialize if the memory is a pipe object
|
|
if (owner.getType() == CL_MEM_OBJECT_PIPE) {
|
|
// Pipe initialize in order read_idx, write_idx, end_idx. Refer clk_pipe_t structure.
|
|
// Init with 3 DWORDS for 32bit addressing and 6 DWORDS for 64bit
|
|
size_t pipeInit[3] = {0, 0, owner.asPipe()->getMaxNumPackets()};
|
|
xferMgr().writeBuffer(pipeInit, *memory, amd::Coord3D(0), amd::Coord3D(sizeof(pipeInit)));
|
|
}
|
|
|
|
// Transfer data only if OCL context has one device.
|
|
// Cache coherency layer will update data for multiple devices
|
|
if (!memory->isHostMemDirectAccess() && owner.asImage() && (owner.parent() == nullptr) &&
|
|
(owner.getMemFlags() & CL_MEM_COPY_HOST_PTR) && (owner.getContext().devices().size() == 1)) {
|
|
// To avoid recurssive call to Device::createMemory, we perform
|
|
// data transfer to the view of the image
|
|
amd::Image* imageView = owner.asImage()->createView(
|
|
owner.getContext(), owner.asImage()->getImageFormat(), xferQueue());
|
|
|
|
if (imageView == nullptr) {
|
|
LogError("[OCL] Fail to allocate view of image object");
|
|
return nullptr;
|
|
}
|
|
|
|
Image* devImageView = new roc::Image(static_cast<const Device&>(*this), *imageView);
|
|
if (devImageView == nullptr) {
|
|
LogError("[OCL] Fail to allocate device mem object for the view");
|
|
imageView->release();
|
|
return nullptr;
|
|
}
|
|
|
|
if (devImageView != nullptr && !devImageView->createView(static_cast<roc::Image&>(*memory))) {
|
|
LogError("[OCL] Fail to create device mem object for the view");
|
|
delete devImageView;
|
|
imageView->release();
|
|
return nullptr;
|
|
}
|
|
|
|
imageView->replaceDeviceMemory(this, devImageView);
|
|
|
|
// Copy data with the original pitch values, since runtime doesn't perform
|
|
// extra sysmem allocation for one device
|
|
const auto image = owner.asImage();
|
|
result = xferMgr().writeImage(owner.getHostMem(), *devImageView, amd::Coord3D(0, 0, 0),
|
|
imageView->getRegion(), image->getRowPitch(),
|
|
image->getSlicePitch(), true);
|
|
|
|
// Release host memory, since runtime copied data
|
|
owner.setHostMem(nullptr);
|
|
|
|
imageView->release();
|
|
}
|
|
|
|
// Prepin sysmem buffer for possible data synchronization between CPU and GPU
|
|
if (!memory->isHostMemDirectAccess() &&
|
|
// Pin memory for the parent object only
|
|
(owner.parent() == nullptr) && (owner.getHostMem() != nullptr) &&
|
|
(owner.getSvmPtr() == nullptr)) {
|
|
memory->pinSystemMemory(owner.getHostMem(), owner.getSize());
|
|
}
|
|
|
|
if (!result) {
|
|
delete memory;
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_RESOURCE, "Cannot Write Image");
|
|
return nullptr;
|
|
}
|
|
|
|
return memory;
|
|
}
|
|
|
|
// ================================================================================================
|
|
device::Memory* Device::createMemory(size_t size, size_t alignment) const {
|
|
auto buffer = new roc::Buffer(*this, size);
|
|
static constexpr bool LocalAlloc = true;
|
|
if ((buffer == nullptr) || !buffer->create(LocalAlloc)) {
|
|
LogError("Couldn't allocate memory on device!");
|
|
return nullptr;
|
|
}
|
|
return buffer;
|
|
}
|
|
|
|
// ================================================================================================
|
|
hsa_amd_memory_pool_t Device::getHostMemoryPool(MemorySegment mem_seg,
|
|
const AgentInfo* agentInfo) const {
|
|
if (agentInfo == nullptr) {
|
|
agentInfo = cpu_agent_info_;
|
|
}
|
|
hsa_amd_memory_pool_t segment{0};
|
|
switch (mem_seg) {
|
|
case kKernArg: {
|
|
if (settings().fgs_kernel_arg_) {
|
|
segment = agentInfo->kern_arg_pool;
|
|
break;
|
|
}
|
|
// Falls through on else case.
|
|
}
|
|
case kNoAtomics:
|
|
// If runtime disables barrier, then all host allocations must have L2 disabled
|
|
if (agentInfo->coarse_grain_pool.handle != 0) {
|
|
segment = agentInfo->coarse_grain_pool;
|
|
break;
|
|
}
|
|
// Falls through on else case.
|
|
case kAtomics:
|
|
segment = agentInfo->fine_grain_pool;
|
|
break;
|
|
case kUncachedAtomics:
|
|
case kIoMemory:
|
|
if (agentInfo->ext_fine_grain_pool.handle != 0) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Using extended fine grained access system memory pool");
|
|
segment = agentInfo->ext_fine_grain_pool;
|
|
break;
|
|
}
|
|
default:
|
|
guarantee(false, "Invalid Memory Segment");
|
|
break;
|
|
}
|
|
assert(segment.handle != 0);
|
|
return segment;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void* Device::hostAlloc(size_t size, size_t alignment, MemorySegment mem_seg,
|
|
const void* agentInfo, bool allowAllAgentsAccess) const {
|
|
void* ptr = nullptr;
|
|
uint32_t memFlags = 0;
|
|
if (mem_seg == kKernArg) {
|
|
memFlags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
|
|
}
|
|
|
|
hsa_amd_memory_pool_t pool =
|
|
getHostMemoryPool(mem_seg, static_cast<const amd::roc::AgentInfo*>(agentInfo));
|
|
hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, memFlags, &ptr);
|
|
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
|
|
"Allocate hsa host memory %p, size 0x%zx,"
|
|
" numa_node = %d, mem_seg = %d",
|
|
ptr, size, preferred_numa_node_, static_cast<int>(mem_seg));
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Fail allocation host memory with err %d", stat);
|
|
return nullptr;
|
|
}
|
|
|
|
// Allow access to all GPU agents if the flag is set
|
|
// otherwise only allow access to the local backend device.
|
|
if (allowAllAgentsAccess) {
|
|
stat = Hsa::agents_allow_access(gpu_agents_.size(), &gpu_agents_[0], nullptr, ptr);
|
|
}
|
|
else {
|
|
stat = Hsa::agents_allow_access(1, &bkendDevice_, nullptr, ptr);
|
|
}
|
|
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Fail hsa_amd_agents_allow_access with err %d", stat);
|
|
hostFree(ptr, size);
|
|
return nullptr;
|
|
}
|
|
|
|
return ptr;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void* Device::hostNumaAlloc(size_t size, size_t alignment, MemorySegment mem_seg) const {
|
|
void* ptr = nullptr;
|
|
auto numa_node_count = cpu_agents_.size(); // count of host numa nodes
|
|
numa::NumaPolicy np(numa_node_count);
|
|
if (!np.GetMemPolicy()) {
|
|
return hostAlloc(size, alignment, mem_seg, cpu_agent_info_);
|
|
}
|
|
switch (np.GetPolicy()) {
|
|
case numa::NumaPolicy::Policy::kPrefered:
|
|
case numa::NumaPolicy::Policy::kBind:
|
|
// We only care about the first CPU node
|
|
for (unsigned int i = 0; i < numa_node_count; i++) {
|
|
if (np.IsPolicySetAt(i)) {
|
|
ptr = hostAlloc(size, alignment, mem_seg, &cpu_agents_[i]);
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
// All other modes fall back to default mode
|
|
ptr = hostAlloc(size, alignment, mem_seg, cpu_agent_info_);
|
|
}
|
|
return ptr;
|
|
}
|
|
|
|
void* Device::hostLock(void* hostMem, size_t size, const MemorySegment memSegment) const {
|
|
hsa_amd_memory_pool_t pool = getHostMemoryPool(memSegment);
|
|
void* deviceMemory = nullptr;
|
|
uint32_t memFlags = 0;
|
|
if (memSegment == kIoMemory) {
|
|
memFlags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
|
|
}
|
|
|
|
hsa_status_t status = Hsa::memory_lock_to_pool(
|
|
hostMem, size, const_cast<hsa_agent_t*>(&bkendDevice_), 1, pool, memFlags, &deviceMemory);
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
|
|
"Locking to pool %p, size 0x%zx, hostMem = %p,"
|
|
" deviceMemory = %p, memSegment = %d",
|
|
pool, size, hostMem, deviceMemory, static_cast<int>(memSegment));
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_LOCK,
|
|
"Failed to lock memory to pool, failed with hsa_status: %d", status);
|
|
deviceMemory = nullptr;
|
|
}
|
|
return deviceMemory;
|
|
}
|
|
|
|
void Device::hostFree(void* ptr, size_t size) const { memFree(ptr, size); }
|
|
|
|
bool Device::deviceAllowAccess(void* ptr) const {
|
|
std::lock_guard<std::mutex> lock(lock_allow_access_);
|
|
if (!p2pAgents().empty()) {
|
|
hsa_status_t stat =
|
|
Hsa::agents_allow_access(p2pAgents().size(), p2pAgents().data(), nullptr, ptr);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Allow p2p access failed - hsa_amd_agents_allow_access with err %d", stat);
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool Device::allowPeerAccess(device::Memory* memory) const {
|
|
if (memory == nullptr) {
|
|
return false;
|
|
}
|
|
if (!p2pAgents().empty()) {
|
|
void* ptr = reinterpret_cast<void*>(memory->virtualAddress());
|
|
hsa_agent_t agent = getBackendDevice();
|
|
hsa_status_t stat = Hsa::agents_allow_access(1, &agent, nullptr, ptr);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Allow p2p access failed - hsa_amd_agents_allow_access with err: %d", stat);
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
uint64_t Device::deviceVmemAlloc(size_t size, uint64_t flags) const {
|
|
// PHYMEM: ROCCLR_MEM_HSA_UNCACHED is passed as HSA_AMD_MEMORY_POOL_UNCACHED_FLAG in |flags|.
|
|
const bool uncached = (flags & HSA_AMD_MEMORY_POOL_UNCACHED_FLAG) != 0;
|
|
const hsa_amd_memory_pool_t& pool = (uncached && gpu_ext_fine_grained_segment_.handle)
|
|
? gpu_ext_fine_grained_segment_ : gpuvm_segment_;
|
|
LogPrintfError("VMEM alloc: pool (selected)=0x%x, gpu_ext_fine_grained=0x%x, gpuvm=0x%x",
|
|
pool.handle, gpu_ext_fine_grained_segment_.handle, gpuvm_segment_.handle);
|
|
|
|
if (pool.handle == 0 || gpuvm_segment_max_alloc_ == 0) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Invalid argument, pool_handle: 0x%x , max_alloc: %u", pool.handle,
|
|
gpuvm_segment_max_alloc_);
|
|
return 0;
|
|
}
|
|
|
|
hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
|
|
|
|
// We only allow pinned memory at this time.
|
|
hsa_status_t hsa_status = Hsa::vmem_handle_create(pool, size, MEMORY_TYPE_PINNED, 0,
|
|
&hsa_vmem_handle);
|
|
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_handle_create! Failed with hsa status: %d", hsa_status);
|
|
return 0;
|
|
}
|
|
|
|
return hsa_vmem_handle.handle;
|
|
}
|
|
|
|
void Device::deviceVmemRelease(uint64_t mem_handle) const {
|
|
hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
|
|
hsa_vmem_handle.handle = mem_handle;
|
|
|
|
hsa_status_t hsa_status = Hsa::vmem_handle_release(hsa_vmem_handle);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_handle_release! Failed with hsa status: %d", hsa_status);
|
|
}
|
|
}
|
|
|
|
bool Device::hostVmemSupported(int numaNode) const {
|
|
if (cpu_agents_.empty()) {
|
|
return false;
|
|
}
|
|
int node = numaNode;
|
|
if (node < 0 || static_cast<size_t>(node) >= cpu_agents_.size()) {
|
|
node = static_cast<int>(numa::getCurrentNumaNode());
|
|
}
|
|
hsa_agent_t cpu = getCpuAgent(node);
|
|
bool supported = false;
|
|
hsa_status_t hsa_status = Hsa::agent_get_info(
|
|
cpu, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_HOST_ALLOC_DMABUF_SUPPORTED),
|
|
&supported);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
// Older ROCr predates this query: treat host-NUMA VMM as unsupported.
|
|
return false;
|
|
}
|
|
return supported;
|
|
}
|
|
|
|
uint64_t Device::hostVmemAlloc(size_t size, uint64_t flags, int numaNode) const {
|
|
// numaNode < 0 (HostNumaCurrent) resolves to the calling thread's current node.
|
|
int node = numaNode;
|
|
if (node < 0) {
|
|
node = static_cast<int>(numa::getCurrentNumaNode());
|
|
}
|
|
if (node < 0 || static_cast<size_t>(node) >= cpu_agents_.size()) {
|
|
LogPrintfError("hostVmemAlloc: invalid NUMA node %d (cpu agents: %zu)", numaNode,
|
|
cpu_agents_.size());
|
|
return 0;
|
|
}
|
|
|
|
if (!hostVmemSupported(node)) {
|
|
LogError("hostVmemAlloc: host memory VMM not supported on this CPU agent");
|
|
return 0;
|
|
}
|
|
|
|
// PHYMEM: ROCCLR_MEM_HSA_UNCACHED is passed as HSA_AMD_MEMORY_POOL_UNCACHED_FLAG in |flags|.
|
|
const bool uncached = (flags & HSA_AMD_MEMORY_POOL_UNCACHED_FLAG) != 0;
|
|
const MemorySegment mem_seg = uncached ? kUncachedAtomics : kAtomics;
|
|
hsa_amd_memory_pool_t pool = getHostMemoryPool(mem_seg, &cpu_agents_[node]);
|
|
if (pool.handle == 0) {
|
|
LogPrintfError("hostVmemAlloc: no host memory pool for NUMA node %d", node);
|
|
return 0;
|
|
}
|
|
|
|
hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
|
|
hsa_status_t hsa_status =
|
|
Hsa::vmem_handle_create(pool, size, MEMORY_TYPE_PINNED, 0, &hsa_vmem_handle);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("hostVmemAlloc: hsa_amd_vmem_handle_create failed with status: %d", hsa_status);
|
|
return 0;
|
|
}
|
|
|
|
return hsa_vmem_handle.handle;
|
|
}
|
|
|
|
void* Device::reserveMemory(size_t size, size_t alignment) const {
|
|
void* ptr = nullptr;
|
|
// Reserves non registered VA memory using HSA APIs.
|
|
hsa_status_t status = Hsa::vmem_address_reserve_align(&ptr, size, 0, alignment,
|
|
HSA_AMD_VMEM_ADDRESS_NO_REGISTER);
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Reserve hsa device memory %p, size 0x%zx", ptr, size);
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogError("Fail to reserve memory");
|
|
return nullptr;
|
|
}
|
|
return ptr;
|
|
}
|
|
|
|
void Device::releaseMemory(void* ptr, size_t size) const {
|
|
hsa_status_t hsa_status = Hsa::vmem_address_free(ptr, size);
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Free hsa reserved memory %p", ptr);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogError("hsa_amd_vmem_address_free failed");
|
|
}
|
|
}
|
|
|
|
void* Device::deviceLocalAlloc(size_t size, const AllocationFlags& flags, bool allowAllAgentsAccess) const {
|
|
const hsa_amd_memory_pool_t& pool =
|
|
(flags.pseudo_fine_grain_ && gpu_ext_fine_grained_segment_.handle)
|
|
? gpu_ext_fine_grained_segment_
|
|
: (flags.atomics_ && gpu_fine_grained_segment_.handle) ? gpu_fine_grained_segment_
|
|
: gpuvm_segment_;
|
|
|
|
if (pool.handle == 0 || gpuvm_segment_max_alloc_ == 0) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Invalid argument, pool_handle: 0x%x , max_alloc: %u",
|
|
pool.handle, gpuvm_segment_max_alloc_);
|
|
return nullptr;
|
|
}
|
|
|
|
uint32_t hsa_mem_flags = 0;
|
|
if (flags.contiguous_) {
|
|
hsa_mem_flags = HSA_AMD_MEMORY_POOL_CONTIGUOUS_FLAG;
|
|
}
|
|
if (flags.executable_) {
|
|
hsa_mem_flags |= HSA_AMD_MEMORY_POOL_EXECUTABLE_FLAG;
|
|
}
|
|
if (flags.uncached_ && isa().versionMajor() == 12) {
|
|
hsa_mem_flags |= HSA_AMD_MEMORY_POOL_UNCACHED_FLAG;
|
|
}
|
|
|
|
void* ptr = nullptr;
|
|
hsa_status_t stat = Hsa::memory_pool_allocate(pool, size, hsa_mem_flags, &ptr);
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM,
|
|
"Allocate hsa device memory %p, size 0x%zx, hsa_mem_flags 0x%xh", ptr, size,
|
|
hsa_mem_flags);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogError("Fail allocation local memory");
|
|
return nullptr;
|
|
}
|
|
|
|
if (allowAllAgentsAccess) {
|
|
if (isP2pEnabled() && deviceAllowAccess(ptr) == false) {
|
|
LogError("Allow p2p access for memory allocation");
|
|
memFree(ptr, size);
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
return ptr;
|
|
}
|
|
|
|
void Device::memFree(void* ptr, size_t size) const {
|
|
hsa_status_t stat = Hsa::memory_pool_free(ptr);
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_MEM, "Free hsa memory %p", ptr);
|
|
if (stat != HSA_STATUS_SUCCESS) {
|
|
LogError("Fail freeing local memory");
|
|
}
|
|
}
|
|
|
|
void Device::updateFreeMemory(size_t size, bool free) {
|
|
if (free) {
|
|
freeMem_ += size;
|
|
} else {
|
|
if (size > freeMem_) {
|
|
// To avoid underflow of the freeMem_
|
|
// This can happen if the free mem tracked is inaccurate, as some allocations can happen
|
|
// directly via ROCr
|
|
ClPrint(amd::LOG_ERROR, amd::LOG_ALWAYS,
|
|
"Free memory set to zero on device 0x%lx, requested size = 0x%zx, freeMem_ = 0x%zx",
|
|
this, size, freeMem_.load());
|
|
freeMem_ = 0;
|
|
return;
|
|
}
|
|
freeMem_ -= size;
|
|
}
|
|
ClPrint(amd::LOG_INFO, amd::LOG_MEM, "Device=0x%lx, freeMem_ = 0x%zx", this, freeMem_.load());
|
|
}
|
|
|
|
// ================================================================================================
|
|
void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags,
|
|
void* svmPtr) const {
|
|
amd::Memory* mem = nullptr;
|
|
void* svmPtrUsed = reinterpret_cast<void*>(amd::Memory::MemoryType::kSvmMemoryPtr);
|
|
|
|
if (nullptr != svmPtr) {
|
|
// Find the existing amd::mem object
|
|
mem = amd::MemObjMap::FindMemObj(svmPtr);
|
|
if (mem != nullptr) {
|
|
return mem->getSvmPtr();
|
|
}
|
|
if (flags & CL_MEM_USE_HOST_PTR) {
|
|
svmPtrUsed = svmPtr;
|
|
} else {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM, "Cannot find svm_ptr: %p", svmPtr);
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
// create a hidden buffer, which will allocated on the device later
|
|
mem = new (context) amd::Buffer(context, flags, size, svmPtrUsed);
|
|
if (mem == nullptr) {
|
|
LogError("failed to create a svm mem object!");
|
|
return nullptr;
|
|
}
|
|
|
|
if (!mem->create(nullptr)) {
|
|
LogError("failed to create a svm hidden buffer!");
|
|
mem->release();
|
|
return nullptr;
|
|
}
|
|
// if the device supports SVM FGS, return the committed CPU address directly.
|
|
Memory* gpuMem = getRocMemory(mem);
|
|
if (gpuMem == nullptr) {
|
|
LogError("failed to create GPU memory from svm hidden buffer!");
|
|
return nullptr;
|
|
}
|
|
|
|
// add the information to context so that we can use it later.
|
|
if (mem->getSvmPtr() != nullptr) {
|
|
amd::MemObjMap::AddMemObj(mem->getSvmPtr(), mem);
|
|
}
|
|
return mem->getSvmPtr();
|
|
}
|
|
|
|
void* Device::virtualAlloc(void* req_addr, size_t size, size_t alignment) {
|
|
void* vptr = nullptr;
|
|
// Reserves the address using HSA APIs, with requested address.
|
|
// There is no guarantee that we will get the requested address.
|
|
hsa_status_t hsa_status =
|
|
Hsa::vmem_address_reserve(&vptr, size, reinterpret_cast<uint64_t>(req_addr), 0);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_address_reserve. Failed with status: %d", hsa_status);
|
|
return nullptr;
|
|
}
|
|
|
|
constexpr bool kParent = true;
|
|
amd::Memory* mem = CreateVirtualBuffer(context(), vptr, size, -1, -1, kParent);
|
|
if (mem == nullptr) {
|
|
LogPrintfError("Cannot create Virtual Buffer for vptr: %p of size: %u", vptr, size);
|
|
}
|
|
|
|
return mem->getSvmPtr();
|
|
}
|
|
|
|
bool Device::virtualFree(void* addr) {
|
|
amd::Memory* memObj = amd::MemObjMap::FindVirtualMemObj(addr);
|
|
if (memObj == nullptr) {
|
|
LogPrintfError("Cannot find the Virtual MemObj entry for this addr %p", addr);
|
|
return false;
|
|
}
|
|
|
|
if (!memObj->getContext().devices()[0]->DestroyVirtualBuffer(memObj)) {
|
|
return false;
|
|
}
|
|
|
|
hsa_status_t hsa_status = Hsa::vmem_address_free(memObj->getSvmPtr(), memObj->getSize());
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_address_free. Failed with status:%d", hsa_status);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
// Direct synchronous map/unmap path bypassing VirtualMapCommand. Used by the
|
|
// non-async hipMemMap/hipMemUnmap entry points (wired up in subsequent
|
|
// commits). No execution() lock and no barrier packet: the HIP layer is
|
|
// responsible for draining access-device queues from the CPU side before
|
|
// calling, and hsa_amd_vmem_{map,unmap} are synchronous w.r.t. the CPU.
|
|
cl_int Device::virtualMap(void* va, size_t size, amd::Memory* phys) {
|
|
if (phys == nullptr) {
|
|
LogError("virtualMap: phys is nullptr");
|
|
return CL_INVALID_VALUE;
|
|
}
|
|
|
|
amd::Memory* vaddr_sub_obj = MapMemObjBookkeeping(phys, va, size);
|
|
if (vaddr_sub_obj == nullptr) {
|
|
LogError("virtualMap: MapMemObjBookkeeping failed");
|
|
return CL_INVALID_VALUE;
|
|
}
|
|
|
|
hsa_amd_vmem_alloc_handle_t opaque_hsa_handle;
|
|
opaque_hsa_handle.handle = phys->getUserData().hsa_handle;
|
|
hsa_status_t hsa_status = Hsa::vmem_map(vaddr_sub_obj->getSvmPtr(), size,
|
|
vaddr_sub_obj->getOffset(), opaque_hsa_handle, 0);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("virtualMap: hsa_amd_vmem_map failed with status: %d", hsa_status);
|
|
// Roll back the bookkeeping so the sub_obj/MemObjMap doesn't leak.
|
|
// FinalizeMapMemObjBookkeeping was never called, so MemObjMap doesn't
|
|
// contain va and the cross-links are not wired. Just tear down the
|
|
// sub-buffer view directly.
|
|
vaddr_sub_obj->getContext().devices()[0]->DestroyVirtualBuffer(vaddr_sub_obj);
|
|
vaddr_sub_obj->release();
|
|
return CL_OUT_OF_HOST_MEMORY;
|
|
}
|
|
|
|
constexpr bool kImportVmmForInterprocess = true;
|
|
FinalizeMapMemObjBookkeeping(vaddr_sub_obj, phys, va, kImportVmmForInterprocess);
|
|
return CL_SUCCESS;
|
|
}
|
|
|
|
// ================================================================================================
|
|
cl_int Device::virtualUnmap(void* va, size_t size) {
|
|
amd::Memory* vaddr_sub_obj = amd::MemObjMap::FindMemObj(va);
|
|
if (vaddr_sub_obj == nullptr) {
|
|
LogPrintfError("virtualUnmap: no sub_obj for va: %p", va);
|
|
return CL_INVALID_VALUE;
|
|
}
|
|
|
|
hsa_status_t hsa_status = Hsa::vmem_unmap(vaddr_sub_obj->getSvmPtr(), size);
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("virtualUnmap: hsa_amd_vmem_unmap failed with status: %d", hsa_status);
|
|
return CL_INVALID_VALUE;
|
|
}
|
|
|
|
constexpr bool kDestroyVirtualBuffer = true;
|
|
constexpr bool kReleaseSubObj = true;
|
|
UnmapMemObjBookkeeping(vaddr_sub_obj, va, kDestroyVirtualBuffer, kReleaseSubObj);
|
|
return CL_SUCCESS;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::SetMemAccess(void* va_addr, size_t va_size, VmmAccess access_flags,
|
|
VmmLocationType access_location, int numaNode) {
|
|
hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
|
|
hsa_amd_memory_access_desc_t desc;
|
|
desc.permissions = static_cast<hsa_access_permission_t>(access_flags);
|
|
// Resolve the agent granted access: the GPU backend for device access, the
|
|
// default CPU agent for generic/current host access, or the CPU agent of a
|
|
// specific NUMA node for HostNuma. getCpuAgent(int) clamps an out-of-range node
|
|
// to the default agent (the HIP layer validates the node id beforehand).
|
|
switch (access_location) {
|
|
case VmmLocationType::kDevice:
|
|
desc.agent_handle = getBackendDevice();
|
|
break;
|
|
case VmmLocationType::kHostNuma:
|
|
desc.agent_handle = getCpuAgent(numaNode);
|
|
break;
|
|
case VmmLocationType::kHostNumaCurrent:
|
|
desc.agent_handle = getCpuAgent(static_cast<int>(numa::getCurrentNumaNode()));
|
|
break;
|
|
case VmmLocationType::kHost:
|
|
default:
|
|
desc.agent_handle = getCpuAgent();
|
|
break;
|
|
}
|
|
|
|
if ((hsa_status = Hsa::vmem_set_access(va_addr, va_size, &desc, 1)) != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_set_access. Failed with status:%d", hsa_status);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool Device::GetMemAccess(void* va_addr, VmmAccess* access_flags_ptr) const {
|
|
hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
|
|
hsa_access_permission_t perms;
|
|
|
|
size_t discard_offset = 0;
|
|
amd::Memory* va_mem_obj = amd::MemObjMap::FindMemObj(va_addr, &discard_offset);
|
|
if (va_mem_obj == nullptr) {
|
|
LogPrintfError("Failed to get Memory Object for va_addr: %p", va_addr);
|
|
return false;
|
|
}
|
|
|
|
if ((hsa_status = Hsa::vmem_get_access(va_mem_obj->getSvmPtr(), &perms, getBackendDevice())) !=
|
|
HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_get_access. Failed with status:%d", hsa_status);
|
|
return false;
|
|
}
|
|
|
|
*access_flags_ptr = static_cast<VmmAccess>(perms);
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
amd::Device::VmmExportStatus Device::ExportShareableVMMHandle(amd::Memory& amd_mem_obj, int flags,
|
|
void* shareableHandle,
|
|
amd::Memory::HandleType handle_type) {
|
|
hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
|
|
hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
|
|
hsa_vmem_handle.handle = amd_mem_obj.getUserData().hsa_handle;
|
|
|
|
if (hsa_vmem_handle.handle == 0) {
|
|
LogError("HSA Handle is not valid");
|
|
return amd::Device::VmmExportStatus::kError;
|
|
}
|
|
|
|
if (handle_type == amd::Memory::HandleType::kHandleFabric) { //handle type fabric
|
|
hsa_fabric_handle_t fabric_handle;
|
|
if ((hsa_status = Hsa::vmem_export_fabric_handle(&fabric_handle,
|
|
hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_vmem_export_fabric_handle with status: %d \n", hsa_status);
|
|
if (hsa_status == static_cast<hsa_status_t>(HSA_STATUS_ERROR_RESOURCE_NOT_READY)) {
|
|
return amd::Device::VmmExportStatus::kResourceNotReady;
|
|
}
|
|
return amd::Device::VmmExportStatus::kError;
|
|
}
|
|
*(reinterpret_cast<hsa_fabric_handle_t*>(shareableHandle)) = fabric_handle;
|
|
} else {
|
|
int dmabuf_fd = 0;
|
|
if ((hsa_status = Hsa::vmem_export_shareable_handle(&dmabuf_fd,
|
|
hsa_vmem_handle, flags)) != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_vmem_export_shareable_handle with status: %d \n", hsa_status);
|
|
return amd::Device::VmmExportStatus::kError;
|
|
}
|
|
*(reinterpret_cast<int*>(shareableHandle)) = dmabuf_fd;
|
|
}
|
|
|
|
return amd::Device::VmmExportStatus::kSuccess;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::ImportShareableHSAHandle(void* osHandle, uint64_t* hsa_handle_ptr,
|
|
amd::Memory::HandleType handle_type) const {
|
|
hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
|
|
hsa_amd_vmem_alloc_handle_t hsa_vmem_handle{};
|
|
|
|
if (hsa_handle_ptr == nullptr) {
|
|
LogError("HSA Handle ptr is null");
|
|
return false;
|
|
}
|
|
|
|
if (handle_type == amd::Memory::HandleType::kHandleFabric) {
|
|
hsa_fabric_handle_t fabric_handle = *(reinterpret_cast<hsa_fabric_handle_t*>(osHandle));
|
|
if ((hsa_status = Hsa::vmem_import_fabric_handle(fabric_handle, &hsa_vmem_handle))
|
|
!= HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_import_fabric_handle with status: %d \n", hsa_status);
|
|
return false;
|
|
}
|
|
} else {
|
|
int dmabuf_fd = static_cast<int>(reinterpret_cast<uintptr_t>(osHandle));
|
|
if ((hsa_status = Hsa::vmem_import_shareable_handle(dmabuf_fd, &hsa_vmem_handle))
|
|
!= HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("Failed hsa_amd_vmem_import_shareable_handle with status: %d \n", hsa_status);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
*hsa_handle_ptr = hsa_vmem_handle.handle;
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
amd::Memory* Device::ImportShareableVMMHandle(void* osHandle, amd::Memory::HandleType handle_type) {
|
|
amd::Memory* amd_mem_obj = new (context())
|
|
amd::Buffer(context(), ROCCLR_MEM_PHYMEM | ROCCLR_MEM_INTERPROCESS, 0, osHandle);
|
|
if (amd_mem_obj == nullptr) {
|
|
LogError("Cannot create memory object");
|
|
return nullptr;
|
|
}
|
|
|
|
amd_mem_obj->getUserData().hsa_handle_type = handle_type;
|
|
if (!amd_mem_obj->create(nullptr, false)) {
|
|
LogError("Failed to create mem_obj from imported fd");
|
|
amd_mem_obj->release();
|
|
return nullptr;
|
|
}
|
|
|
|
return amd_mem_obj;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count, amd::MemoryAdvice advice,
|
|
bool first_alloc, bool use_cpu, int numa_id) const {
|
|
if ((settings().hmmFlags_ & Settings::Hmm::EnableSvmTracking) && !first_alloc) {
|
|
amd::Memory* svm_mem = amd::MemObjMap::FindMemObj(dev_ptr);
|
|
if ((nullptr == svm_mem) || ((svm_mem->getMemFlags() & CL_MEM_ALLOC_HOST_PTR) == 0) ||
|
|
// Validate the range of provided memory
|
|
((svm_mem->getSize() - (reinterpret_cast<const_address>(dev_ptr) -
|
|
reinterpret_cast<address>(svm_mem->getSvmPtr()))) < count)) {
|
|
LogPrintfError("SetSvmAttributes received unknown memory for update: %p!", dev_ptr);
|
|
return false;
|
|
}
|
|
}
|
|
if (info().hmmSupported_) {
|
|
std::vector<hsa_amd_svm_attribute_pair_t> attr;
|
|
|
|
switch (advice) {
|
|
case amd::MemoryAdvice::SetReadMostly:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, true});
|
|
break;
|
|
case amd::MemoryAdvice::UnsetReadMostly:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, false});
|
|
break;
|
|
case amd::MemoryAdvice::SetPreferredLocation:
|
|
if (use_cpu) {
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, getCpuAgent(numa_id).handle});
|
|
} else {
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, getBackendDevice().handle});
|
|
}
|
|
break;
|
|
case amd::MemoryAdvice::UnsetPreferredLocation:
|
|
// @note: 0 may cause a failure on old runtimes
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, 0});
|
|
break;
|
|
case amd::MemoryAdvice::SetAccessedBy: {
|
|
const uint64_t attrib = (first_alloc) ? HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE
|
|
: HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE;
|
|
if (use_cpu) {
|
|
attr.push_back({attrib, getCpuAgent().handle});
|
|
} else {
|
|
if (first_alloc) {
|
|
// Provide access to all possible devices.
|
|
//! @note: HMM should support automatic page table update with xnack enabled,
|
|
//! but currently it doesn't and runtime explicitly enables access from all devices
|
|
for (const auto dev : devices()) {
|
|
// Skip null devices
|
|
if (static_cast<Device*>(dev)->getBackendDevice().handle != 0) {
|
|
attr.push_back({attrib, static_cast<Device*>(dev)->getBackendDevice().handle});
|
|
}
|
|
}
|
|
} else {
|
|
attr.push_back({attrib, getBackendDevice().handle});
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case amd::MemoryAdvice::UnsetAccessedBy:
|
|
// When unsetting we should use HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE for the agent
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, getBackendDevice().handle});
|
|
break;
|
|
case amd::MemoryAdvice::SetCoarseGrain:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED});
|
|
break;
|
|
case amd::MemoryAdvice::UnsetCoarseGrain:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED});
|
|
break;
|
|
default:
|
|
return false;
|
|
break;
|
|
}
|
|
|
|
hsa_status_t status =
|
|
Hsa::svm_attributes_set(const_cast<void*>(dev_ptr), count, attr.data(), attr.size());
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogPrintfError("hsa_amd_svm_attributes_set() failed. Advice: %d, status: %d", advice, status);
|
|
return false;
|
|
}
|
|
} else {
|
|
LogWarning("hsa_amd_svm_attributes_set() is ignored, because no HMM support");
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::SetSvmAttributes(const void* dev_ptr, size_t count, amd::MemoryAdvice advice,
|
|
bool use_cpu, int numa_id) const {
|
|
constexpr bool kFirstAlloc = false;
|
|
return SetSvmAttributesInt(dev_ptr, count, advice, kFirstAlloc, use_cpu);
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::GetSvmAttributes(void** data, size_t* data_sizes, int* attributes,
|
|
size_t num_attributes, const void* dev_ptr, size_t count) const {
|
|
amd::Memory* svm_mem = amd::MemObjMap::FindMemObj(dev_ptr);
|
|
if (settings().hmmFlags_ & Settings::Hmm::EnableSvmTracking) {
|
|
if ((nullptr == svm_mem) || ((svm_mem->getMemFlags() & CL_MEM_ALLOC_HOST_PTR) == 0) ||
|
|
// Validate the range of provided memory
|
|
((svm_mem->getSize() - (reinterpret_cast<const_address>(dev_ptr) -
|
|
reinterpret_cast<address>(svm_mem->getSvmPtr()))) < count)) {
|
|
LogPrintfError("GetSvmAttributes received unknown memory %p for state!", dev_ptr);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
hsa_amd_pointer_info_t ptr_info = {};
|
|
for (size_t i = 0; i < num_attributes; ++i) {
|
|
if (attributes[i] == amd::MemRangeAttribute::CoherencyMode) {
|
|
ptr_info.size = sizeof(hsa_amd_pointer_info_t);
|
|
// Query ptr type to see if it's a HMM allocation
|
|
hsa_status_t status =
|
|
Hsa::pointer_info(const_cast<void*>(dev_ptr), &ptr_info, nullptr, nullptr, nullptr);
|
|
// The call should never fail in ROCR, but just check for an error and continue
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogError("hsa_amd_pointer_info() failed");
|
|
}
|
|
|
|
// Check if it's a legacy non-HMM allocation and update query
|
|
*reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE;
|
|
if (ptr_info.type == HSA_EXT_POINTER_TYPE_HSA ||
|
|
ptr_info.type == HSA_EXT_POINTER_TYPE_LOCKED ||
|
|
ptr_info.type == HSA_EXT_POINTER_TYPE_IPC) {
|
|
if (ptr_info.global_flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_COARSE_GRAINED) {
|
|
*reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED;
|
|
} else if (ptr_info.global_flags & HSA_AMD_MEMORY_POOL_GLOBAL_FLAG_FINE_GRAINED) {
|
|
*reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED;
|
|
}
|
|
}
|
|
// If coherency is still indeterminate
|
|
if (ptr_info.type == HSA_EXT_POINTER_TYPE_HSA_VMEM) {
|
|
if (svm_mem != nullptr && (svm_mem->getMemFlags() & CL_MEM_SVM_FINE_GRAIN_BUFFER)) {
|
|
*reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED;
|
|
} else {
|
|
*reinterpret_cast<uint32_t*>(data[i]) = HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (info().hmmSupported_) {
|
|
uint32_t accessed_by = 0;
|
|
std::vector<hsa_amd_svm_attribute_pair_t> attr;
|
|
|
|
for (size_t i = 0; i < num_attributes; ++i) {
|
|
switch (attributes[i]) {
|
|
case amd::MemRangeAttribute::ReadMostly:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_READ_MOSTLY, 0});
|
|
break;
|
|
case amd::MemRangeAttribute::PreferredLocation:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION, 0});
|
|
break;
|
|
case amd::MemRangeAttribute::AccessedBy:
|
|
accessed_by = attr.size();
|
|
// Add all GPU devices into the query
|
|
for (const auto agent : gpu_agents_) {
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_ACCESS_QUERY, agent.handle});
|
|
}
|
|
// Add CPU devices
|
|
for (const auto agent_info : cpu_agents_) {
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_ACCESS_QUERY, agent_info.agent.handle});
|
|
}
|
|
accessed_by = attr.size() - accessed_by;
|
|
break;
|
|
case amd::MemRangeAttribute::LastPrefetchLocation:
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION, 0});
|
|
break;
|
|
case amd::MemRangeAttribute::CoherencyMode:
|
|
if (*reinterpret_cast<uint32_t*>(data[i]) == HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE) {
|
|
attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, 0});
|
|
}
|
|
break;
|
|
default:
|
|
return false;
|
|
break;
|
|
}
|
|
}
|
|
// Only call svm_attributes_get if there are SVM attributes to query
|
|
if (attr.empty()) {
|
|
return true;
|
|
}
|
|
hsa_status_t status =
|
|
Hsa::svm_attributes_get(const_cast<void*>(dev_ptr), count, attr.data(), attr.size());
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogError("hsa_amd_svm_attributes_get() failed");
|
|
return false;
|
|
}
|
|
|
|
uint32_t idx = 0;
|
|
uint32_t rocr_attr = 0;
|
|
for (size_t i = 0; i < num_attributes; ++i) {
|
|
const auto& it = attr[rocr_attr];
|
|
switch (attributes[i]) {
|
|
case amd::MemRangeAttribute::ReadMostly:
|
|
if (data_sizes[idx] != sizeof(uint32_t)) {
|
|
return false;
|
|
}
|
|
// Cast ROCr value into the hip format
|
|
*reinterpret_cast<uint32_t*>(data[idx]) =
|
|
(static_cast<uint32_t>(it.value) > 0) ? true : false;
|
|
++rocr_attr;
|
|
break;
|
|
// The logic should be identical for the both queries
|
|
case amd::MemRangeAttribute::PreferredLocation:
|
|
case amd::MemRangeAttribute::LastPrefetchLocation:
|
|
if (data_sizes[idx] != sizeof(uint32_t)) {
|
|
return false;
|
|
}
|
|
*reinterpret_cast<int32_t*>(data[idx]) = static_cast<int32_t>(amd::InvalidDeviceId);
|
|
// Find device agent returned by ROCr
|
|
for (auto& device : devices()) {
|
|
if (static_cast<Device*>(device)->getBackendDevice().handle == it.value) {
|
|
*reinterpret_cast<uint32_t*>(data[idx]) = static_cast<uint32_t>(device->index());
|
|
}
|
|
}
|
|
// Find CPU agent returned by ROCr
|
|
for (auto& agent_info : cpu_agents_) {
|
|
if (agent_info.agent.handle == it.value) {
|
|
*reinterpret_cast<int32_t*>(data[idx]) = static_cast<int32_t>(amd::CpuDeviceId);
|
|
}
|
|
}
|
|
++rocr_attr;
|
|
break;
|
|
case amd::MemRangeAttribute::AccessedBy: {
|
|
uint32_t entry = 0;
|
|
uint32_t device_count = data_sizes[idx] / 4;
|
|
// Make sure it's multiple of 4
|
|
if (data_sizes[idx] % 4 != 0) {
|
|
return false;
|
|
}
|
|
for (uint32_t att = 0; att < accessed_by; ++att) {
|
|
const auto& it = attr[rocr_attr + att];
|
|
if (entry >= device_count) {
|
|
// The size of the array is less than the amount of available devices
|
|
break;
|
|
}
|
|
switch (it.attribute) {
|
|
case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE:
|
|
case HSA_AMD_SVM_ATTRIB_AGENT_NO_ACCESS:
|
|
break;
|
|
case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE:
|
|
reinterpret_cast<int32_t*>(data[idx])[entry] =
|
|
static_cast<int32_t>(amd::InvalidDeviceId);
|
|
// Find device agent returned by ROCr
|
|
for (auto& device : devices()) {
|
|
if (static_cast<Device*>(device)->getBackendDevice().handle == it.value) {
|
|
reinterpret_cast<uint32_t*>(data[idx])[entry] =
|
|
static_cast<uint32_t>(device->index());
|
|
}
|
|
}
|
|
// Find CPU agent returned by ROCr
|
|
for (auto& agent_info : cpu_agents_) {
|
|
if (agent_info.agent.handle == it.value) {
|
|
reinterpret_cast<int32_t*>(data[idx])[entry] =
|
|
static_cast<int32_t>(amd::CpuDeviceId);
|
|
}
|
|
}
|
|
++entry;
|
|
break;
|
|
default:
|
|
LogWarning("Unexpected result from HSA_AMD_SVM_ATTRIB_ACCESS_QUERY");
|
|
break;
|
|
}
|
|
}
|
|
rocr_attr += accessed_by;
|
|
for (uint32_t i = entry; i < device_count; ++i) {
|
|
reinterpret_cast<int32_t*>(data[idx])[i] = static_cast<int32_t>(amd::InvalidDeviceId);
|
|
}
|
|
break;
|
|
}
|
|
case amd::MemRangeAttribute::CoherencyMode:
|
|
if (data_sizes[idx] != sizeof(uint32_t)) {
|
|
return false;
|
|
}
|
|
// if ptr is HMM alloc then overwrite the values
|
|
if (*reinterpret_cast<uint32_t*>(data[idx]) == HSA_AMD_SVM_GLOBAL_FLAG_INDETERMINATE) {
|
|
// Cast ROCr value into the hip format
|
|
*reinterpret_cast<uint32_t*>(data[idx]) = static_cast<uint32_t>(it.value);
|
|
}
|
|
++rocr_attr;
|
|
break;
|
|
default:
|
|
return false;
|
|
break;
|
|
}
|
|
// Find the next location in the query
|
|
++idx;
|
|
}
|
|
} else if (ptr_info.type == HSA_EXT_POINTER_TYPE_RESERVED_ADDR) {
|
|
LogError("GetSvmAttributes() failed, because no HMM support");
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
size_t Device::ScratchLimitCurrent() const {
|
|
uint64_t scratchLimitCurrent = 0;
|
|
hsa_status_t ret =
|
|
Hsa::agent_get_info(bkendDevice_, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_CURRENT,
|
|
&scratchLimitCurrent);
|
|
if (HSA_STATUS_SUCCESS != ret) {
|
|
LogPrintfError("HSA_AMD_AGENT_INFO_SCRATCH_LIMIT_CURRENT cannot be queried! Err: 0x%xh", ret);
|
|
return 0;
|
|
}
|
|
return static_cast<size_t>(scratchLimitCurrent);
|
|
};
|
|
|
|
bool Device::UpdateScratchLimitCurrent(size_t limit) const {
|
|
hsa_status_t ret = Hsa::agent_set_async_scratch_limit(bkendDevice_, limit);
|
|
if (HSA_STATUS_SUCCESS != ret) {
|
|
LogPrintfError("hsa_amd_agent_set_async_scratch_limit(%zu) failed with err 0x%xh", limit, ret);
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
// ================================================================================================
|
|
bool Device::SvmAllocInit(void* memory, size_t size) const {
|
|
amd::MemoryAdvice advice = amd::MemoryAdvice::SetAccessedBy;
|
|
constexpr bool kFirstAlloc = true;
|
|
if (!SetSvmAttributesInt(memory, size, advice, kFirstAlloc)) {
|
|
return false;
|
|
}
|
|
|
|
if ((settings().hmmFlags_ & Settings::Hmm::EnableMallocPrefetch) == 0) {
|
|
return true;
|
|
}
|
|
|
|
if (info().hmmSupported_) {
|
|
// Initialize signal for the barrier
|
|
Hsa::signal_store_relaxed(prefetch_signal_, kInitSignalValueOne);
|
|
|
|
// Initiate a prefetch command which should force memory update in HMM
|
|
hsa_status_t status =
|
|
Hsa::svm_prefetch_async(memory, size, getBackendDevice(), 0, nullptr, prefetch_signal_);
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogError("hsa_amd_svm_prefetch_async() failed");
|
|
return false;
|
|
}
|
|
|
|
// Wait for the prefetch
|
|
if (!WaitForSignal(prefetch_signal_)) {
|
|
LogError("Barrier packet submission failed");
|
|
return false;
|
|
}
|
|
} else {
|
|
LogWarning("Early prefetch failed, because no HMM support");
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::svmFree(void* ptr) const {
|
|
amd::Memory* svmMem = amd::MemObjMap::FindMemObj(ptr);
|
|
if (nullptr != svmMem) {
|
|
amd::MemObjMap::RemoveMemObj(svmMem->getSvmPtr());
|
|
svmMem->release();
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
VirtualGPU* Device::xferQueue() const {
|
|
Device* thisDevice = const_cast<Device*>(this);
|
|
std::call_once(xferQueueOnce_, [thisDevice]() {
|
|
// Create virtual device for internal memory transfer
|
|
thisDevice->xferQueue_ = reinterpret_cast<VirtualGPU*>(thisDevice->createVirtualDevice());
|
|
if (!thisDevice->xferQueue_) {
|
|
LogError("Couldn't create the device transfer manager!");
|
|
return;
|
|
}
|
|
if (thisDevice->xferQueue_->gpu_queue() == nullptr) {
|
|
thisDevice->xferQueue_->SetGpuQueue(
|
|
thisDevice->AcquireActiveQueue(amd::CommandQueue::Priority::Normal));
|
|
}
|
|
});
|
|
if (!xferQueue_) {
|
|
LogError("Couldn't create the device transfer manager!");
|
|
return nullptr;
|
|
}
|
|
xferQueue_->enableSyncBlit();
|
|
return xferQueue_;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput,
|
|
cl_set_device_clock_mode_output_amd* pSetClockModeOutput) {
|
|
bool result = true;
|
|
return result;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::IsHwEventReady(const amd::Event& event, bool wait, amd::SyncPolicy policy) const {
|
|
void* hw_event =
|
|
(event.NotifyEvent() != nullptr) ? event.NotifyEvent()->HwEvent() : event.HwEvent();
|
|
if (hw_event == nullptr) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_SIG, "No HW event");
|
|
return false;
|
|
} else if (wait) {
|
|
// hipEventBlockingSync
|
|
// when set the CPU gives up host thread for other work
|
|
// when not set the CPU enters a busy-wait on the event to occur
|
|
constexpr int kHipEventBlockingSync = 0x1;
|
|
bool active_wait =
|
|
!((policy == amd::SyncPolicy::Blocking) & kHipEventBlockingSync) && ActiveWait();
|
|
bool yield = (policy == amd::SyncPolicy::Yield);
|
|
return WaitForSignal(reinterpret_cast<ProfilingSignal*>(hw_event)->signal_, active_wait, yield);
|
|
}
|
|
|
|
auto signal = reinterpret_cast<ProfilingSignal*>(hw_event)->signal_;
|
|
ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Check HW event = 0x%lx", signal.handle);
|
|
|
|
return (Hsa::signal_load_relaxed(signal) == 0);
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::getHwEventTime(const amd::Event& event, uint64_t* start, uint64_t* end) const {
|
|
void* hw_event =
|
|
(event.NotifyEvent() != nullptr) ? event.NotifyEvent()->HwEvent() : event.HwEvent();
|
|
if (hw_event == nullptr) {
|
|
ClPrint(amd::LOG_INFO, amd::LOG_SIG, "No HW event to read time");
|
|
*start = *end = 0;
|
|
} else {
|
|
fetchSignalTime(reinterpret_cast<ProfilingSignal*>(hw_event)->signal_, getBackendDevice(),
|
|
start, end);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
hsa_queue_t* Device::getQueueFromPool(const uint qIndex, bool force_reuse,
|
|
hsa_queue_t* preferred,
|
|
const std::unordered_set<uint64_t>* excluded_ids) {
|
|
// Only reuse queues when we've reached the maximum limit, unless forced
|
|
// Below the limit, return nullptr to allow creating new queues
|
|
if (!force_reuse && queuePool_[qIndex].size() < settings().max_hw_queues_) {
|
|
return nullptr;
|
|
}
|
|
|
|
// We've hit the limit, must reuse - find the queue with lowest load metric
|
|
if (qIndex < QueuePriority::Total && queuePool_[qIndex].size() > 0) {
|
|
// Best-effort preferred queue hint: for graph stream stability
|
|
// Skip preferred if it's in the excluded set
|
|
if (preferred != nullptr) {
|
|
bool preferred_excluded = excluded_ids && excluded_ids->count(preferred->id) > 0;
|
|
if (!preferred_excluded) {
|
|
auto it = queuePool_[qIndex].find(preferred);
|
|
if (it != queuePool_[qIndex].end()) {
|
|
it->second.refCount++;
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
|
|
"Reusing preferred queue: %p refCount: %d",
|
|
it->first->base_address, it->second.refCount);
|
|
return it->first;
|
|
}
|
|
}
|
|
}
|
|
|
|
typedef decltype(queuePool_)::value_type::const_reference PoolRef;
|
|
|
|
// Select queue based on dynamic_queues_ mode
|
|
decltype(queuePool_[qIndex].begin()) lowest;
|
|
uint32_t mode = settings().dynamic_queues_;
|
|
|
|
// gfx9XX pipe distribution: queues map to pipes via queue_id % num_pipes
|
|
const bool pipe_dist = settings().queue_pipe_dist_;
|
|
const uint32_t num_pipes = numHwPipes_;
|
|
|
|
lowest = std::min_element(
|
|
queuePool_[qIndex].begin(), queuePool_[qIndex].end(),
|
|
[mode, pipe_dist, num_pipes, excluded_ids](PoolRef first_candidate,
|
|
PoolRef second_candidate) {
|
|
// Exclusion filtering: prefer non-excluded queues over excluded ones
|
|
bool first_is_excluded =
|
|
excluded_ids && excluded_ids->count(first_candidate.first->id) > 0;
|
|
bool second_is_excluded =
|
|
excluded_ids && excluded_ids->count(second_candidate.first->id) > 0;
|
|
if (first_is_excluded != second_is_excluded) return second_is_excluded;
|
|
|
|
// Always use an unshared regular queue before sharing an active queue. On devices
|
|
// with coarse-grained read pointer updates, an idle queue can retain a nonzero
|
|
// apparent depth and must not lose to a blocked queue with a smaller reported depth.
|
|
bool first_is_unshared =
|
|
first_candidate.second.refCount == 0 &&
|
|
!first_candidate.second.hasDedicatedQueue_;
|
|
bool second_is_unshared =
|
|
second_candidate.second.refCount == 0 &&
|
|
!second_candidate.second.hasDedicatedQueue_;
|
|
if (first_is_unshared != second_is_unshared) return first_is_unshared;
|
|
|
|
if (mode >= 1) {
|
|
// Mode 1+: Advanced weighted metric with dedicated queue penalty
|
|
// Metric = dedicated_queue_penalty + (depth << 4) + refCount
|
|
uint64_t first_metric =
|
|
first_candidate.second.GetLoadMetric(first_candidate.first, mode);
|
|
uint64_t second_metric =
|
|
second_candidate.second.GetLoadMetric(second_candidate.first, mode);
|
|
|
|
if (first_metric == second_metric && pipe_dist) {
|
|
// gfx9XX pipe distribution: prefer lower pipe IDs for consistent distribution
|
|
uint64_t first_pipe = first_candidate.first->id % num_pipes;
|
|
uint64_t second_pipe = second_candidate.first->id % num_pipes;
|
|
return first_pipe < second_pipe;
|
|
}
|
|
return first_metric < second_metric;
|
|
} else {
|
|
// Mode 0: Simple refCount-based selection
|
|
return first_candidate.second.refCount < second_candidate.second.refCount;
|
|
}
|
|
});
|
|
|
|
lowest->second.refCount++;
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
|
|
"Selected queue (mode=%u): %p refCount: %d, depth: %lu, metric: %lu, pipe: %d%s%s",
|
|
mode, lowest->first->base_address, lowest->second.refCount,
|
|
QueueInfo::GetHwQueueDepth(lowest->first),
|
|
lowest->second.GetLoadMetric(lowest->first, mode),
|
|
pipe_dist ? (lowest->first->id % num_pipes) : -1,
|
|
force_reuse ? " (forced)" : "",
|
|
(excluded_ids && excluded_ids->count(lowest->first->id) > 0)
|
|
? " (excluded-fallback)" : "");
|
|
return lowest->first;
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
// ================================================================================================
|
|
hsa_queue_t* Device::AcquireActiveQueue(amd::CommandQueue::Priority priority,
|
|
hsa_queue_t* preferred,
|
|
const std::unordered_set<uint64_t>* excluded_ids) {
|
|
uint32_t queue_size = ROC_AQL_QUEUE_SIZE;
|
|
auto queue = acquireQueue(queue_size, false, std::vector<uint32_t>{},
|
|
priority, true, false, preferred, excluded_ids);
|
|
return queue;
|
|
}
|
|
|
|
// ================================================================================================
|
|
hsa_queue_t* Device::acquireQueue(uint32_t queue_size_hint, bool coop_queue,
|
|
const std::vector<uint32_t>& cuMask,
|
|
amd::CommandQueue::Priority priority, bool managed,
|
|
bool dedicated_queue, hsa_queue_t* preferred,
|
|
const std::unordered_set<uint64_t>* excluded_ids,
|
|
void** metadata_ring_buffer) {
|
|
// App-thread context: flush queues deferred from the async thread once they
|
|
// reach the batch threshold, bounding live deferred queues.
|
|
if (DeferredQueueCount() >= kDeferredQueueDrainThreshold) {
|
|
DrainDeferredQueueDestroys();
|
|
}
|
|
|
|
hsa_amd_queue_priority_t queue_priority;
|
|
uint qIndex;
|
|
switch (priority) {
|
|
case amd::CommandQueue::Priority::Low:
|
|
queue_priority = HSA_AMD_QUEUE_PRIORITY_LOW;
|
|
qIndex = QueuePriority::Low;
|
|
break;
|
|
case amd::CommandQueue::Priority::High:
|
|
queue_priority = HSA_AMD_QUEUE_PRIORITY_HIGH;
|
|
qIndex = QueuePriority::High;
|
|
break;
|
|
case amd::CommandQueue::Priority::Normal:
|
|
case amd::CommandQueue::Priority::Medium:
|
|
default:
|
|
queue_priority = HSA_AMD_QUEUE_PRIORITY_NORMAL;
|
|
qIndex = QueuePriority::Normal;
|
|
break;
|
|
}
|
|
|
|
// If flag set, force all streams to normal priority
|
|
// This means that GPU_MAX_HW_QUEUES may need to be incremented
|
|
// to account for the additional normal priority queues
|
|
if (DEBUG_HIP_IGNORE_STREAM_PRIORITY) {
|
|
queue_priority = HSA_AMD_QUEUE_PRIORITY_NORMAL;
|
|
qIndex = QueuePriority::Normal;
|
|
}
|
|
|
|
{ // Lock
|
|
amd::ScopedLock l(active_queue_access_);
|
|
|
|
assert(queuePool_[QueuePriority::Low].size() <= settings().max_hw_queues_ ||
|
|
queuePool_[QueuePriority::Normal].size() <= settings().max_hw_queues_ ||
|
|
queuePool_[QueuePriority::High].size() <= settings().max_hw_queues_);
|
|
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
|
|
"Number of allocated hardware queues with low priority: %d,"
|
|
" with normal priority: %d, with high priority: %d, maximum per priority is: %d",
|
|
queuePool_[QueuePriority::Low].size(), queuePool_[QueuePriority::Normal].size(),
|
|
queuePool_[QueuePriority::High].size(), settings().max_hw_queues_);
|
|
|
|
// If we have reached the max number of queues, reuse an existing queue with the matching queue
|
|
// priority, choosing the one with the least number of users. Note: Don't attempt to reuse the
|
|
// cooperative queue, since it's single per device.
|
|
|
|
// num_queues_[qIndex] tracks persistent (non-managed) queues per priority.
|
|
// When the total queues (managed + non-managed) exceed max_hw_queues_, we must reuse existing
|
|
// queues. 'managed' streams do not increment num_queues_, allowing them to use the
|
|
// pool without permanently consuming slots. ReleaseActiveQueue() uses this counter to
|
|
// decide when to start reclaiming queues.
|
|
if (!coop_queue && (cuMask.size() == 0) &&
|
|
(queuePool_[qIndex].size() >= settings().max_hw_queues_)) {
|
|
hsa_queue_t* queue = getQueueFromPool(qIndex, false, preferred, excluded_ids);
|
|
if (queue != nullptr) {
|
|
if (!managed) {
|
|
num_queues_[qIndex]++;
|
|
}
|
|
return queue;
|
|
}
|
|
}
|
|
} // Lock release
|
|
|
|
// Create a new queue.
|
|
uint32_t queue_max_packets = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(bkendDevice_, HSA_AGENT_INFO_QUEUE_MAX_SIZE, &queue_max_packets)) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE, "Cannot get hsa agent info");
|
|
return nullptr;
|
|
}
|
|
auto queue_size = (queue_max_packets < queue_size_hint) ? queue_max_packets : queue_size_hint;
|
|
|
|
auto queue_type = HSA_QUEUE_TYPE_MULTI;
|
|
|
|
if (coop_queue) {
|
|
queue_type = HSA_QUEUE_TYPE_COOPERATIVE;
|
|
}
|
|
|
|
// Resolve the final CU mask (merge custom + global, expand for WGP mode)
|
|
std::vector<uint32_t> final_mask;
|
|
if (cuMask.size() != 0 || info_.globalCUMask_.size() != 0) {
|
|
std::vector<uint32_t> mask;
|
|
if (cuMask.size() != 0 && info_.globalCUMask_.size() == 0) {
|
|
mask = cuMask;
|
|
} else if (cuMask.size() != 0 && info_.globalCUMask_.size() != 0) {
|
|
for (unsigned int i = 0; i < std::min(cuMask.size(), info_.globalCUMask_.size()); i++) {
|
|
mask.push_back(cuMask[i] & info_.globalCUMask_[i]);
|
|
}
|
|
bool zeroCUMask = true;
|
|
for (auto m : mask) {
|
|
if (m != 0) {
|
|
zeroCUMask = false;
|
|
break;
|
|
}
|
|
}
|
|
if (zeroCUMask) {
|
|
mask = info_.globalCUMask_;
|
|
}
|
|
} else {
|
|
mask = info_.globalCUMask_;
|
|
}
|
|
|
|
if (settings().enableWgpMode_) {
|
|
final_mask.resize(mask.size() * 2, 0);
|
|
for (size_t i = 0; i < mask.size(); i++) {
|
|
for (int j = 0; j < 16; j++) {
|
|
if (((mask[i] >> j) & 0x1) == 0x1) {
|
|
final_mask[2 * i] |= (0x3 << (2 * j));
|
|
}
|
|
if (((mask[i] >> (16 + j)) & 0x1) == 0x1) {
|
|
final_mask[2 * i + 1] |= (0x3 << (2 * j));
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
final_mask = mask;
|
|
}
|
|
}
|
|
|
|
// Build the queue creation descriptor with priority, CU mask, and flags.
|
|
// settings().aql_device_ring_buf_ controls queue ring buffer placement (enabled
|
|
// by default on gfx94x/gfx125x, overridable via DEBUG_CLR_AQL_DEV_QUEUE):
|
|
// false - system memory
|
|
// true - ring buffer in device memory if Large BAR is available.
|
|
const bool device_mem_ring_buf =
|
|
settings().aql_device_ring_buf_ && info_.largeBar_ && Hsa::amd_queue_create_available();
|
|
hsa_amd_queue_create_desc_t desc = {};
|
|
desc.version = HSA_AMD_QUEUE_CREATE_DESC_VERSION;
|
|
desc.engine_type = HSA_AMD_QUEUE_ENGINE_COMPUTE;
|
|
desc.queue_size_bytes = queue_size * sizeof(hsa_kernel_dispatch_packet_t);
|
|
desc.priority = queue_priority;
|
|
desc.callback = callbackQueue;
|
|
desc.callback_data = this;
|
|
desc.engine.compute.type = queue_type;
|
|
desc.engine.compute.private_segment_size = HSA_AMD_PRIVATE_SEGMENT_SIZE_DEFAULT;
|
|
if (device_mem_ring_buf) {
|
|
desc.flags = static_cast<hsa_amd_queue_create_flag_t>(
|
|
desc.flags | HSA_AMD_QUEUE_CREATE_DEVICE_MEM_RING_BUF);
|
|
}
|
|
if (!final_mask.empty()) {
|
|
desc.engine.compute.cu_mask_count = static_cast<uint32_t>(final_mask.size() * 32);
|
|
desc.engine.compute.cu_mask = final_mask.data();
|
|
}
|
|
|
|
while (Hsa::amd_queue_create(bkendDevice_, &desc, 1) != HSA_STATUS_SUCCESS) {
|
|
queue_size >>= 1;
|
|
desc.queue_size_bytes = queue_size * sizeof(hsa_kernel_dispatch_packet_t);
|
|
if (queue_size < 64) {
|
|
LogError("Device::acquireQueue: hsa_amd_queue_create failed!");
|
|
if (!coop_queue && (cuMask.size() == 0)) {
|
|
amd::ScopedLock l(active_queue_access_);
|
|
if (queuePool_[qIndex].size() > 0) {
|
|
bool kForceReuse = true;
|
|
return getQueueFromPool(qIndex, kForceReuse);
|
|
}
|
|
}
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_QUEUE,
|
|
"Device::acquireQueue: hsa_amd_queue_create failed!");
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
hsa_queue_t* queue = desc.queue;
|
|
if (IsLogEnabled(amd::LOG_INFO, amd::LOG_QUEUE)) {
|
|
if (!final_mask.empty()) {
|
|
std::stringstream ss;
|
|
ss << std::hex;
|
|
for (int i = final_mask.size() - 1; i >= 0; i--) {
|
|
ss << std::setfill('0') << std::setw(8) << final_mask[i];
|
|
}
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
|
|
"Created SWq=%p HWq=%p size %d priority %d cooperative %i flags 0x%x "
|
|
"device_mem_ring_buf %i CU mask 0x%s",
|
|
queue, queue->base_address, queue_size, queue_priority, coop_queue, desc.flags,
|
|
device_mem_ring_buf, ss.str().c_str());
|
|
} else {
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE,
|
|
"Created SWq=%p HWq=%p size %d priority %d cooperative %i flags 0x%x "
|
|
"device_mem_ring_buf %i",
|
|
queue, queue->base_address, queue_size, queue_priority, coop_queue, desc.flags,
|
|
device_mem_ring_buf);
|
|
}
|
|
}
|
|
|
|
Hsa::profiling_set_profiler_enabled(queue, 1);
|
|
|
|
if (!metadata_version_queried_) {
|
|
uint8_t major = 0, minor = 0;
|
|
hsa_amd_queue_get_info(queue,
|
|
HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_DISPATCH_PKT_VERSION_MAJOR, &major);
|
|
hsa_amd_queue_get_info(queue,
|
|
HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_DISPATCH_PKT_VERSION_MINOR, &minor);
|
|
if (major < (1 << 3) && minor < (1 << 5)) {
|
|
metadata_version_header_ =
|
|
(static_cast<uint32_t>(major) << HSA_AMD_METADATA_PACKET_HEADER_VERSION_MAJOR) |
|
|
(static_cast<uint32_t>(minor) << HSA_AMD_METADATA_PACKET_HEADER_VERSION_MINOR);
|
|
}
|
|
metadata_version_queried_ = true;
|
|
}
|
|
|
|
// Query extras for the newly-created queue and insert into the lookup map
|
|
auto populateExtras = [&]() {
|
|
QueueExtras extras;
|
|
extras.deviceMemRingBuf = (desc.flags & HSA_AMD_QUEUE_CREATE_DEVICE_MEM_RING_BUF) != 0;
|
|
hsa_amd_queue_get_info(queue, HSA_AMD_QUEUE_INFO_PREFETCH_METADATA_RING_BUFFER,
|
|
&extras.metadataRingBuffer);
|
|
if (DEBUG_CLR_DIRECT_DOORBELL) {
|
|
uint64_t db_id = 0;
|
|
if (hsa_amd_queue_get_info(queue, HSA_AMD_QUEUE_INFO_DOORBELL_ID, &db_id) ==
|
|
HSA_STATUS_SUCCESS &&
|
|
db_id != 0) {
|
|
extras.doorbellPtr = reinterpret_cast<volatile uint64_t*>(db_id);
|
|
}
|
|
}
|
|
queue_extras_[queue] = extras;
|
|
};
|
|
|
|
if (!final_mask.empty()) {
|
|
amd::ScopedLock l(active_queue_access_);
|
|
populateExtras();
|
|
return queue;
|
|
}
|
|
|
|
if (coop_queue) {
|
|
amd::ScopedLock l(active_queue_access_);
|
|
populateExtras();
|
|
return queue;
|
|
}
|
|
|
|
// Add queue to the pool (including dedicated queues)
|
|
amd::ScopedLock l(active_queue_access_);
|
|
auto result = queuePool_[qIndex].emplace(std::make_pair(queue, QueueInfo()));
|
|
assert(result.second && "QueueInfo already exists");
|
|
auto& qInfo = result.first->second;
|
|
qInfo.refCount = 1;
|
|
qInfo.hasDedicatedQueue_ = dedicated_queue;
|
|
populateExtras();
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "acquireQueue refCount: %p (%d) %s",
|
|
result.first->first->base_address, result.first->second.refCount,
|
|
dedicated_queue ? "(dedicated)" : "");
|
|
if (!managed && (cuMask.size() == 0)) {
|
|
num_queues_[qIndex]++;
|
|
}
|
|
return queue;
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::ReleaseActiveQueue(hsa_queue_t* queue, amd::CommandQueue::Priority priority) {
|
|
uint qIndex;
|
|
switch (priority) {
|
|
case amd::CommandQueue::Priority::Low:
|
|
qIndex = QueuePriority::Low;
|
|
break;
|
|
case amd::CommandQueue::Priority::High:
|
|
qIndex = QueuePriority::High;
|
|
break;
|
|
case amd::CommandQueue::Priority::Normal:
|
|
case amd::CommandQueue::Priority::Medium:
|
|
default:
|
|
qIndex = QueuePriority::Normal;
|
|
break;
|
|
}
|
|
// Release a queue if the total number of allocated queues exceeds the max possible
|
|
if (num_queues_[qIndex].load() > settings().max_hw_queues_) {
|
|
releaseQueue(queue, std::vector<uint32_t>{}, false, true);
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::releaseQueue(hsa_queue_t* queue, const std::vector<uint32_t>& cuMask, bool coop_queue,
|
|
bool managed) {
|
|
// Defer cleanup operations outside the lock
|
|
{ // Lock
|
|
amd::ScopedLock l(active_queue_access_);
|
|
for (uint qIndex = 0; qIndex < queuePool_.size(); ++qIndex) {
|
|
auto& it = queuePool_[qIndex];
|
|
auto qIter = it.find(queue);
|
|
if (qIter != it.end()) {
|
|
if (!managed && (cuMask.size() == 0)) {
|
|
num_queues_[qIndex]--;
|
|
}
|
|
auto& qInfo = qIter->second;
|
|
assert(qInfo.refCount > 0);
|
|
qInfo.refCount--;
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "releaseQueue refCount:%p (%d)",
|
|
qIter->first->base_address, qIter->second.refCount);
|
|
break; // Found and processed the queue
|
|
}
|
|
}
|
|
} // Lock release
|
|
|
|
// hsa queues with cumask set and coop queues are not being reused. Hence, if the app uses such
|
|
// queues, we need to destroy them when the queue is released.
|
|
if (!cuMask.empty() || coop_queue) {
|
|
// On the async-events thread a non-coop ~AqlQueue self-deadlocks; defer it to an app thread.
|
|
// Coop queues are recycled synchronously via GWSRelease, so never defer them.
|
|
if (InAsyncSignalHandler() && !coop_queue) {
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deferring CG enabled hardware queue %p destroy",
|
|
queue->base_address);
|
|
std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
|
|
deferredQueueDestroy_.push_back(queue);
|
|
} else {
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deleting CG enabled hardware queue %p ",
|
|
queue->base_address);
|
|
Hsa::queue_destroy(queue);
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t Device::DeferredQueueCount() {
|
|
std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
|
|
return deferredQueueDestroy_.size();
|
|
}
|
|
|
|
void Device::DrainDeferredQueueDestroys() {
|
|
// Swap out the batch under lock, then destroy without holding it: queue_destroy
|
|
// is blocking and may run runtime callbacks.
|
|
std::vector<hsa_queue_t*> pending;
|
|
{
|
|
std::scoped_lock<std::mutex> lock(deferredQueueDestroyLock_);
|
|
pending.swap(deferredQueueDestroy_);
|
|
}
|
|
for (hsa_queue_t* queue : pending) {
|
|
ClPrint(amd::LOG_INFO, amd::LOG_QUEUE, "Deleting deferred hardware queue %p", queue->base_address);
|
|
Hsa::queue_destroy(queue);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
Device::QueueExtras Device::GetQueueExtras(hsa_queue_t* queue) {
|
|
amd::ScopedLock l(active_queue_access_);
|
|
auto it = queue_extras_.find(queue);
|
|
return (it != queue_extras_.end()) ? it->second : QueueExtras{};
|
|
}
|
|
|
|
bool Device::findLinkInfo(const amd::Device& other_device, std::vector<LinkAttrType>* link_attrs) {
|
|
return findLinkInfo((static_cast<const roc::Device*>(&other_device))->gpuvm_segment_, link_attrs);
|
|
}
|
|
|
|
bool Device::findLinkInfo(const hsa_amd_memory_pool_t& pool,
|
|
std::vector<LinkAttrType>* link_attrs) {
|
|
if ((!pool.handle) || (link_attrs == nullptr)) {
|
|
return false;
|
|
}
|
|
|
|
// Retrieve the hops between 2 devices.
|
|
int32_t hops = 0;
|
|
hsa_status_t hsa_status = Hsa::agent_memory_pool_get_info(
|
|
bkendDevice_, pool, HSA_AMD_AGENT_MEMORY_POOL_INFO_NUM_LINK_HOPS, &hops);
|
|
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Cannot get hops info, hsa failed with status: %d", hsa_status);
|
|
return false;
|
|
}
|
|
|
|
if (hops < 0) {
|
|
return false;
|
|
}
|
|
|
|
// The pool is on its agent
|
|
if (hops == 0) {
|
|
for (auto& link_attr : (*link_attrs)) {
|
|
switch (link_attr.first) {
|
|
case kLinkLinkType: {
|
|
// No link, so type is meaningless,
|
|
// caller should ignore it
|
|
link_attr.second = -1;
|
|
break;
|
|
}
|
|
case kLinkHopCount: {
|
|
// no hop
|
|
link_attr.second = 0;
|
|
break;
|
|
}
|
|
case kLinkDistance: {
|
|
// distance is zero, if no hops
|
|
link_attr.second = 0;
|
|
break;
|
|
}
|
|
case kLinkAtomicSupport: {
|
|
// atomic support if its on the same agent
|
|
link_attr.second = 1;
|
|
break;
|
|
}
|
|
default: {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Invalid LinkAttribute: %d ", link_attr.first);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Retrieve link info on the pool.
|
|
std::vector<hsa_amd_memory_pool_link_info_t> link_info(hops);
|
|
hsa_status = Hsa::agent_memory_pool_get_info(
|
|
bkendDevice_, pool, HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO, link_info.data());
|
|
|
|
if (hsa_status != HSA_STATUS_SUCCESS) {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Cannot retrieve link info, hsa failed with status: %d", hsa_status);
|
|
return false;
|
|
}
|
|
|
|
for (auto& link_attr : (*link_attrs)) {
|
|
switch (link_attr.first) {
|
|
case kLinkLinkType: {
|
|
link_attr.second = static_cast<int32_t>(link_info[0].link_type);
|
|
break;
|
|
}
|
|
case kLinkHopCount: {
|
|
uint32_t distance = 0;
|
|
// Because of Rocrs limitation hops is set to 1 always between two different devices
|
|
// If Rocr Changes the behaviour revisit this logic
|
|
for (size_t hop_idx = 0; hop_idx < static_cast<size_t>(hops); ++hop_idx) {
|
|
distance += link_info[hop_idx].numa_distance;
|
|
}
|
|
uint32_t oneHopDistance = (link_info[0].link_type == HSA_AMD_LINK_INFO_TYPE_XGMI) ? 13 : 20;
|
|
link_attr.second = static_cast<int32_t>(distance / oneHopDistance);
|
|
break;
|
|
}
|
|
case kLinkDistance: {
|
|
uint32_t distance = 0;
|
|
// Sum of distances between hops
|
|
for (size_t hop_idx = 0; hop_idx < static_cast<size_t>(hops); ++hop_idx) {
|
|
distance += link_info[hop_idx].numa_distance;
|
|
}
|
|
link_attr.second = static_cast<int32_t>(distance);
|
|
break;
|
|
}
|
|
case kLinkAtomicSupport: {
|
|
// if either of the atomic is supported
|
|
link_attr.second = static_cast<int32_t>(link_info[0].atomic_support_64bit ||
|
|
link_info[0].atomic_support_32bit);
|
|
break;
|
|
}
|
|
default: {
|
|
ClPrint(amd::LOG_DETAIL_DEBUG, amd::LOG_MEM,
|
|
"Invalid LinkAttribute: %d ", link_attr.first);
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::getGlobalCUMask(std::string_view cuMaskStr) {
|
|
if (cuMaskStr.length() != 0) {
|
|
std::string_view pre = cuMaskStr.substr(0, 2);
|
|
if (pre.compare("0x") == 0 || pre.compare("0X") == 0) {
|
|
cuMaskStr = cuMaskStr.substr(2, cuMaskStr.length());
|
|
}
|
|
|
|
int end = cuMaskStr.length();
|
|
|
|
// the number of current physical CUs compressed in 4-bits
|
|
size_t compPhysicalCUs = static_cast<size_t>(
|
|
(settings().enableWgpMode_ ? info_.maxComputeUnits_ * 2 : info_.maxComputeUnits_) / 4);
|
|
|
|
// the number of final available compute units after applying the requested CU mask
|
|
uint32_t availCUs = 0;
|
|
|
|
// read numCharToRead characters (8 or less) from the cuMask string each time, convert
|
|
// it into hex, and store it into the globalCUMask_. If the length of the cuMask string
|
|
// is more than the compressed physical available CUs, ignore the rest
|
|
for (unsigned i = 0; i < std::min(cuMaskStr.length(), compPhysicalCUs); i += 8) {
|
|
int numCharToRead = (i + 8 <= compPhysicalCUs) ? 8 : compPhysicalCUs - 8;
|
|
std::string_view temp =
|
|
cuMaskStr.substr(std::max(0, end - numCharToRead), std::min(numCharToRead, end));
|
|
end -= numCharToRead;
|
|
char *endOfConverted = nullptr;
|
|
unsigned long ul = std::strtoul(temp.data(), &endOfConverted, 16);
|
|
if (endOfConverted == temp.data()) { // no conversion could be performed
|
|
info_.globalCUMask_ = {};
|
|
availCUs = 0;
|
|
break;
|
|
}
|
|
info_.globalCUMask_.push_back(static_cast<uint32_t>(ul));
|
|
// count number of set bits in ul to find the number of active CUs
|
|
// in each iteration
|
|
while (ul) {
|
|
ul &= (ul - 1);
|
|
availCUs++;
|
|
}
|
|
}
|
|
// update the maxComputeUnits_ based on the requested CU mask
|
|
if (availCUs != 0 && availCUs < compPhysicalCUs * 4) {
|
|
info_.maxComputeUnits_ = settings().enableWgpMode_ ? availCUs / 2 : availCUs;
|
|
} else {
|
|
info_.globalCUMask_ = {};
|
|
}
|
|
} else {
|
|
info_.globalCUMask_ = {};
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
device::Signal* Device::createSignal() const { return new roc::Signal(); }
|
|
|
|
// ================================================================================================
|
|
device::Signal* Device::createIpcSignal() const { return new roc::IpcSignal(); }
|
|
|
|
// ================================================================================================
|
|
static std::string GetLocalHostName() {
|
|
#if defined(__linux__)
|
|
char buf[HOST_NAME_MAX + 1];
|
|
buf[HOST_NAME_MAX] = '\0';
|
|
if (gethostname(buf, sizeof(buf)) == 0) return buf;
|
|
#endif
|
|
return "";
|
|
}
|
|
|
|
// ================================================================================================
|
|
hsa_status_t Device::BackendErrorCallBackHandler(const hsa_amd_event_t* event, void* data) {
|
|
cl_int gpu_error = CL_SUCCESS;
|
|
switch (event->event_type) {
|
|
case HSA_AMD_GPU_MEMORY_FAULT_EVENT: {
|
|
gpu_error = CL_INVALID_MEM_OBJECT;
|
|
LogError("Memory Fault Error");
|
|
std::string hostnameStr = GetLocalHostName();
|
|
std::string host_tag = hostnameStr.empty() ? "" : "host: " + hostnameStr + ", ";
|
|
for (auto* amd_dev : amd::Device::devices()) {
|
|
if (amd_dev->type() != CL_DEVICE_TYPE_GPU) continue;
|
|
Device* roc_dev = reinterpret_cast<Device*>(amd_dev);
|
|
for (auto it : roc_dev->vgpus()) {
|
|
roc::VirtualGPU* vgpu = reinterpret_cast<roc::VirtualGPU*>(it);
|
|
bool faulted = false;
|
|
if (vgpu->gpu_queue() != nullptr &&
|
|
hsa_amd_queue_get_info(vgpu->gpu_queue(),
|
|
HSA_AMD_QUEUE_INFO_VM_FAULT_STATUS,
|
|
&faulted) == HSA_STATUS_SUCCESS &&
|
|
faulted) {
|
|
std::string kernelName = vgpu->AnalyzeAqlQueue();
|
|
const char* kname = kernelName.c_str();
|
|
ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
|
|
"Memory Fault Error [%sGPU index: %u, "
|
|
"faulting addr: 0x%" PRIx64 ", kernel: %s]",
|
|
host_tag.c_str(), roc_dev->index(),
|
|
event->memory_fault.virtual_address, kname);
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
case HSA_AMD_GPU_HW_EXCEPTION_EVENT:
|
|
gpu_error = CL_INVALID_OPERATION;
|
|
LogError("HW Exception Error");
|
|
break;
|
|
case HSA_AMD_GPU_MEMORY_ERROR_EVENT:
|
|
gpu_error = CL_DEVICE_NOT_AVAILABLE;
|
|
LogError("GPU Memory Error");
|
|
break;
|
|
case HSA_AMD_SYSTEM_SHUTDOWN_EVENT:
|
|
// This is not a fatal error just ignore it.
|
|
return HSA_STATUS_SUCCESS;
|
|
default:
|
|
gpu_error = CL_DEVICE_NOT_AVAILABLE;
|
|
LogError("Unknown Event Type ");
|
|
break;
|
|
}
|
|
|
|
// Execute the default handler if a GPU core file should be generated ...
|
|
if (amd::Os::DumpCoreFile() || !HIP_SKIP_ABORT_ON_GPU_ERROR) {
|
|
return HSA_STATUS_ERROR;
|
|
}
|
|
|
|
gpu_error_.store(gpu_error, std::memory_order_relaxed);
|
|
return HSA_STATUS_SUCCESS;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::RegisterBackendErrorCb() {
|
|
// Register ROCclr Error Callback
|
|
hsa_status_t hsa_error = HSA_STATUS_SUCCESS;
|
|
hsa_error = Hsa::register_system_event_handler(BackendErrorCallBackHandler, nullptr);
|
|
if (hsa_error != HSA_STATUS_SUCCESS) {
|
|
LogError("Cannot Register Call back event handler");
|
|
}
|
|
}
|
|
// ================================================================================================
|
|
amd::Memory* Device::GetArenaMemObj(const void* ptr, size_t& offset, size_t size) {
|
|
// Only create arena_mem_object if CPU memory is accessible from HMM
|
|
// or if runtime received an interop from another ROCr's client
|
|
// Disable arena for XNACK
|
|
hsa_amd_pointer_info_t ptr_info = {};
|
|
ptr_info.size = sizeof(hsa_amd_pointer_info_t);
|
|
if (!IsValidAllocation(ptr, size, &ptr_info)) {
|
|
return nullptr;
|
|
}
|
|
|
|
if (arena_mem_obj_ == nullptr) {
|
|
arena_mem_obj_ = new (context()) amd::ArenaMemory(context());
|
|
if ((arena_mem_obj_ != nullptr) && !arena_mem_obj_->create(nullptr)) {
|
|
LogError("Arena Memory Creation failed!");
|
|
arena_mem_obj_->release();
|
|
arena_mem_obj_ = nullptr;
|
|
}
|
|
if (arena_mem_obj_ == nullptr) {
|
|
return arena_mem_obj_;
|
|
}
|
|
}
|
|
|
|
// Calculate the offset of the pointer.
|
|
const void* dev_ptr = reinterpret_cast<void*>(
|
|
arena_mem_obj_->getDeviceMemory(*arena_mem_obj_->getContext().devices()[0])
|
|
->virtualAddress());
|
|
offset = reinterpret_cast<size_t>(ptr) - reinterpret_cast<size_t>(dev_ptr);
|
|
|
|
return arena_mem_obj_;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::ReleaseGlobalSignal(void* signal) const {
|
|
if (signal != nullptr) {
|
|
reinterpret_cast<ProfilingSignal*>(signal)->release();
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::RetainGlobalSignal(void* signal) const {
|
|
if (signal != nullptr) {
|
|
reinterpret_cast<ProfilingSignal*>(signal)->retain();
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::CreateHwEvents(int count, std::vector<void*>& hw_events) const {
|
|
hw_events.resize(count, nullptr);
|
|
for (int i = 0; i < count; ++i) {
|
|
ProfilingSignal* ps = new ProfilingSignal();
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::signal_create(1, 0, nullptr, HSA_AMD_SIGNAL_AMD_GPU_ONLY, &ps->signal_)) {
|
|
delete ps;
|
|
for (int j = 0; j < i; ++j) {
|
|
reinterpret_cast<ProfilingSignal*>(hw_events[j])->release();
|
|
hw_events[j] = nullptr;
|
|
}
|
|
return false;
|
|
}
|
|
hw_events[i] = ps;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::DestroyHwEvent(void* hw_event) const {
|
|
ReleaseGlobalSignal(hw_event);
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::ResetHwEvents(const std::vector<void*>& hw_events) const {
|
|
// Re-arm pooled signals for reuse by a new graph launch. The caller
|
|
// guarantees these signals belong to a completed (drained) launch, so this
|
|
// cannot corrupt an in-flight launch. Avoids signal_create/destroy on the
|
|
// hot launch path.
|
|
for (void* hw_event : hw_events) {
|
|
if (hw_event != nullptr) {
|
|
auto* ps = reinterpret_cast<ProfilingSignal*>(hw_event);
|
|
Hsa::signal_silent_store_relaxed(ps->signal_, 1);
|
|
ps->flags_.done_ = true;
|
|
ps->ResetCachedTiming();
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::QuiesceHwEvents(const std::vector<void*>& hw_events) const {
|
|
// Pooled signals rest in the armed state (value 1). Before destruction, store
|
|
// the completed value (0) so ~ProfilingSignal does not block waiting on a
|
|
// signal that is armed but idle (no GPU work will ever drain it).
|
|
for (void* hw_event : hw_events) {
|
|
if (hw_event != nullptr) {
|
|
auto* ps = reinterpret_cast<ProfilingSignal*>(hw_event);
|
|
Hsa::signal_silent_store_relaxed(ps->signal_, 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
uint8_t* Device::CreateBarrierPacket() const {
|
|
static constexpr uint16_t kBarrierNopHeader =
|
|
(HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE) |
|
|
(1 << HSA_PACKET_HEADER_BARRIER) |
|
|
(HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) |
|
|
(HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE);
|
|
|
|
static_assert(sizeof(hsa_barrier_and_packet_t) == 64, "AQL packet size must be 64 bytes");
|
|
auto* raw = new uint8_t[64]();
|
|
auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
|
|
pkt->header = kBarrierNopHeader;
|
|
return raw;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::ApplyHwEventPatches(const std::vector<HwEventPatch>& patches,
|
|
const std::vector<void*>& hw_events) const {
|
|
for (const auto& patch : patches) {
|
|
auto* ps = reinterpret_cast<ProfilingSignal*>(hw_events[patch.hw_event_index]);
|
|
hsa_signal_t sig = ps->signal_;
|
|
|
|
// Patch the flat buffer copy (dispatched to GPU) directly.
|
|
// The original dispatchPackets pointer is retained for UpdateAQLPacket matching.
|
|
uint8_t* raw = patch.flat_packet ? patch.flat_packet : patch.packet;
|
|
|
|
if (patch.dep_slot == HwEventPatch::kExtDispatchDepSignal) {
|
|
// Patch dep_signal in hsa_amd_ext_kernel_dispatch_packet_t via offset-based
|
|
static constexpr size_t kExtDepSignalOffset =
|
|
offsetof(hsa_amd_ext_kernel_dispatch_packet_t, dep_signal);
|
|
memcpy(raw + kExtDepSignalOffset, &sig, sizeof(sig));
|
|
} else if (patch.dep_slot == HwEventPatch::kCompletionSignal) {
|
|
auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
|
|
pkt->completion_signal = sig;
|
|
|
|
// Prepare this signal for profiling: mark it as active and classify
|
|
// the packet type so checkGpuTime → addTimestamps only fires for
|
|
// kernel dispatches (not synthetic barriers).
|
|
ps->flags_.done_ = false;
|
|
// Record the queue this patched dispatch signal runs on (resolved from the
|
|
// owning segment's stream at launch) so profiling attributes it to the
|
|
// right stream rather than the graph launch stream.
|
|
ps->queue_index_ = patch.queue_index;
|
|
uint16_t hdr;
|
|
memcpy(&hdr, patch.packet, sizeof(hdr));
|
|
uint8_t pktType = hdr & ((1 << HSA_PACKET_HEADER_WIDTH_TYPE) - 1);
|
|
// A kernel dispatch could be a vendor-specific ext-kernel-dispatch
|
|
// packet, identified by amd_format (byte 2). Classify it as a dispatch so
|
|
// the patched last-node completion signal contributes its GPU timing like
|
|
// every other graph kernel node.
|
|
const uint8_t amdFormat = patch.packet[2];
|
|
ps->flags_.isPacketDispatch_ =
|
|
(pktType == HSA_PACKET_TYPE_KERNEL_DISPATCH) ||
|
|
(pktType == HSA_PACKET_TYPE_VENDOR_SPECIFIC &&
|
|
amdFormat == HSA_AMD_PACKET_TYPE_EXT_KERNEL_DISPATCH);
|
|
} else {
|
|
// dep_slot >= 0: patch a barrier's dependency signal slot (cross-segment wait)
|
|
auto* pkt = reinterpret_cast<hsa_barrier_and_packet_t*>(raw);
|
|
pkt->dep_signal[patch.dep_slot] = sig;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::CreateUserEvent(amd::UserEvent* event) const {
|
|
std::unique_ptr<ProfilingSignal> signal(new ProfilingSignal());
|
|
if ((signal == nullptr) ||
|
|
(HSA_STATUS_SUCCESS != Hsa::signal_create(0, 0, nullptr, &signal->signal_))) {
|
|
return false;
|
|
}
|
|
Hsa::signal_silent_store_relaxed(signal->signal_, kInitSignalValueOne);
|
|
event->SetHwEvent(signal.release());
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::SetUserEvent(amd::UserEvent* event) const {
|
|
auto signal = reinterpret_cast<ProfilingSignal*>(event->HwEvent());
|
|
assert(signal != nullptr && "Can't have user event without hw event!");
|
|
Hsa::signal_silent_store_relaxed(signal->signal_, 0);
|
|
}
|
|
|
|
// ================================================================================================
|
|
bool Device::IsValidAllocation(const void* dev_ptr, size_t size, hsa_amd_pointer_info_t* ptr_info) {
|
|
// Query ptr type to see if it's a HMM allocation
|
|
hsa_status_t status =
|
|
Hsa::pointer_info(const_cast<void*>(dev_ptr), ptr_info, nullptr, nullptr, nullptr);
|
|
// The call should never fail in ROCR, but just check for an error and continue
|
|
if (status != HSA_STATUS_SUCCESS) {
|
|
LogError("hsa_amd_pointer_info() failed");
|
|
}
|
|
|
|
if (ptr_info->type == HSA_EXT_POINTER_TYPE_RESERVED_ADDR) {
|
|
return false;
|
|
}
|
|
|
|
// Return false for pinned memory. A true return may result in a race because
|
|
// ROCclr may attempt to do a pin/copy/unpin underneath in a multithreaded environment
|
|
if (ptr_info->type == HSA_EXT_POINTER_TYPE_LOCKED) {
|
|
return false;
|
|
}
|
|
|
|
if (ptr_info->type != HSA_EXT_POINTER_TYPE_UNKNOWN) {
|
|
if ((size != 0) && ((reinterpret_cast<const_address>(dev_ptr) -
|
|
reinterpret_cast<const_address>(ptr_info->agentBaseAddress)) > size)) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::HiddenHeapAlloc(const VirtualGPU& gpu) {
|
|
auto HeapAllocOnly = [this, &gpu]() -> bool {
|
|
// Allocate initial heap for device memory allocator
|
|
static constexpr size_t HeapBufferSize = 128 * Ki;
|
|
heap_buffer_ = createMemory(HeapBufferSize);
|
|
if (initial_heap_size_ != 0) {
|
|
initial_heap_size_ = amd::alignUp(initial_heap_size_, 2 * Mi);
|
|
initial_heap_buffer_ = createMemory(initial_heap_size_);
|
|
}
|
|
if (heap_buffer_ == nullptr) {
|
|
LogError("Heap buffer allocation failed!");
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
std::call_once(heap_allocated_, HeapAllocOnly);
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::HiddenHeapInit(const VirtualGPU& gpu) {
|
|
auto HeapZeroOut = [this, &gpu]() -> bool {
|
|
static constexpr size_t HeapBufferSize = 128 * Ki;
|
|
bool result = static_cast<const KernelBlitManager&>(gpu.blitMgr())
|
|
.initHeap(heap_buffer_, initial_heap_buffer_, HeapBufferSize,
|
|
initial_heap_size_ / (2 * Mi));
|
|
|
|
return result;
|
|
};
|
|
std::call_once(heap_initialized_, HeapZeroOut);
|
|
}
|
|
|
|
// ================================================================================================
|
|
uint32_t Device::SdmaEngineAllocator::AllocateEngine(VirtualGPU* vgpu, HwQueueEngine engine_type,
|
|
hsa_agent_t peerAgent, hsa_agent_t copyAgent) {
|
|
std::scoped_lock lock(lock_);
|
|
|
|
// Get valid engine mask based on operation type (read vs write)
|
|
uint32_t validEngineMask = (engine_type == HwQueueEngine::SdmaD2H)
|
|
? device_.maxSdmaReadMask_
|
|
: device_.maxSdmaWriteMask_;
|
|
|
|
// Simple round-robin path if all engines have equal bandwidth
|
|
// Disabled by default - use preferred engine logic for current GPUs
|
|
constexpr bool kUseSimpleRR = false;
|
|
|
|
if (kUseSimpleRR) {
|
|
// Simple round-robin: just cycle through valid engines
|
|
// This will be enabled for future GPUs where engine selection doesn't matter
|
|
if (validEngineMask == 0) {
|
|
ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
|
|
"No valid SDMA engines for VirtualGPU %p", vgpu);
|
|
return 0;
|
|
}
|
|
|
|
// Cycle through bit positions, find next valid engine
|
|
uint32_t start_bit = next_rr_engine_.fetch_add(1, std::memory_order_relaxed);
|
|
uint32_t selected_mask = 0;
|
|
|
|
// Try up to 32 positions to find a valid engine
|
|
for (uint32_t i = 0; i < 32; ++i) {
|
|
uint32_t bit = (start_bit + i) % 32;
|
|
uint32_t mask = 1u << bit;
|
|
if (validEngineMask & mask) {
|
|
selected_mask = mask;
|
|
break;
|
|
}
|
|
}
|
|
|
|
vgpu_to_engine_[vgpu] = selected_mask;
|
|
|
|
ClPrint(amd::LOG_INFO, amd::LOG_COPY,
|
|
"Assigned SDMA engine (simple RR) to VirtualGPU %p: mask=0x%x, engine_type=%d",
|
|
vgpu, selected_mask, engine_type);
|
|
|
|
return selected_mask;
|
|
}
|
|
|
|
// Current path: Query HSA for engine status and preferences
|
|
uint32_t freeEngineMask = 0;
|
|
uint32_t preferredMask = 0;
|
|
hsa_status_t status = HSA_STATUS_SUCCESS;
|
|
|
|
// Query current engine status
|
|
status = Hsa::memory_copy_engine_status(peerAgent, copyAgent, &freeEngineMask);
|
|
if (status == HSA_STATUS_SUCCESS) {
|
|
// Query preferred (high-bandwidth) engines
|
|
status = Hsa::memory_get_preferred_copy_engine(peerAgent, copyAgent, &preferredMask);
|
|
}
|
|
|
|
// Constrain to valid engines
|
|
freeEngineMask &= validEngineMask;
|
|
preferredMask &= validEngineMask;
|
|
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
|
|
"Engine query for VirtualGPU %p: status=%x, free_mask=0x%x, preferred_mask=0x%x, "
|
|
"valid_mask=0x%x, engine_type=%d",
|
|
vgpu, status, freeEngineMask, preferredMask, validEngineMask, engine_type);
|
|
|
|
uint32_t candidate_mask = 0;
|
|
uint32_t allocated_mask = 0;
|
|
|
|
// For inter-GPU copies, strongly prefer the recommended engines
|
|
bool is_inter_gpu = (engine_type == HwQueueEngine::SdmaP2P);
|
|
|
|
if (is_inter_gpu && (preferredMask != 0)) {
|
|
// Inter-GPU: prioritize preferredMask, even if engines are already allocated
|
|
candidate_mask = validEngineMask & preferredMask;
|
|
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
|
|
"Inter-GPU copy for VirtualGPU %p: prioritizing preferred engines, "
|
|
"candidate_mask=0x%x",
|
|
vgpu, candidate_mask);
|
|
} else {
|
|
// Regular read/write/intra: enforce exclusivity (don't share engines)
|
|
// Build a mask of engines already allocated to other VirtualGPUs
|
|
for (const auto& pair : vgpu_to_engine_) {
|
|
allocated_mask |= pair.second;
|
|
}
|
|
|
|
uint32_t available_mask = validEngineMask & ~allocated_mask;
|
|
|
|
if (available_mask == 0) {
|
|
ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
|
|
"No unallocated SDMA engines available for VirtualGPU %p, engine_type=%d "
|
|
"(valid_mask=0x%x, allocated_mask=0x%x)",
|
|
vgpu, engine_type, validEngineMask, allocated_mask);
|
|
return 0;
|
|
}
|
|
|
|
// Prefer high-bandwidth (recommended) engines if available
|
|
candidate_mask = available_mask & preferredMask;
|
|
if (candidate_mask == 0) {
|
|
candidate_mask = available_mask;
|
|
}
|
|
}
|
|
|
|
if (candidate_mask == 0) {
|
|
ClPrint(amd::LOG_WARNING, amd::LOG_COPY,
|
|
"No candidate SDMA engines for VirtualGPU %p, engine_type=%d",
|
|
vgpu, engine_type);
|
|
return 0;
|
|
}
|
|
|
|
// Select the lowest bit (first available engine)
|
|
uint32_t selected_mask = candidate_mask & (~candidate_mask + 1);
|
|
|
|
// Update the map
|
|
vgpu_to_engine_[vgpu] = selected_mask;
|
|
|
|
ClPrint(amd::LOG_INFO, amd::LOG_COPY,
|
|
"Assigned SDMA engine to VirtualGPU %p: mask=0x%x, engine_type=%d, "
|
|
"valid_mask=0x%x, preferred_mask=0x%x, allocated_mask=0x%x, is_inter_gpu=%d",
|
|
vgpu, selected_mask, engine_type, validEngineMask, preferredMask,
|
|
allocated_mask, is_inter_gpu);
|
|
|
|
return selected_mask;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::SdmaEngineAllocator::ReleaseEngine(VirtualGPU* vgpu) {
|
|
std::scoped_lock lock(lock_);
|
|
|
|
auto it = vgpu_to_engine_.find(vgpu);
|
|
if (it != vgpu_to_engine_.end()) {
|
|
ClPrint(amd::LOG_DEBUG, amd::LOG_COPY,
|
|
"Released SDMA engine for VirtualGPU %p: mask=0x%x",
|
|
vgpu, it->second);
|
|
vgpu_to_engine_.erase(it);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::AddKernel(Kernel& gpuKernel) const {
|
|
std::scoped_lock lock(vgpusAccess());
|
|
kernel_map_.insert({gpuKernel.KernelCodeHandle(), gpuKernel});
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::RemoveKernel(Kernel& gpuKernel) const {
|
|
if (gpuKernel.KernelCodeHandle() != 0) {
|
|
std::scoped_lock lock(vgpusAccess());
|
|
auto it = kernel_map_.find(gpuKernel.KernelCodeHandle());
|
|
if (it != kernel_map_.end()) {
|
|
// Remove the old mapping
|
|
kernel_map_.erase(it);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
ProfilingSignal::~ProfilingSignal() {
|
|
if (signal_.handle != 0) {
|
|
if (Hsa::signal_load_relaxed(signal_) > 0
|
|
&& !(HIP_SKIP_ABORT_ON_GPU_ERROR && amd::Device::IsGPUInError())) {
|
|
LogError("Runtime shouldn't destroy a signal that is still busy!");
|
|
if (Hsa::signal_wait_scacquire(signal_, HSA_SIGNAL_CONDITION_LT, kInitSignalValueOne,
|
|
kUnlimitedWait, HSA_WAIT_STATE_BLOCKED) != 0) {
|
|
}
|
|
}
|
|
Hsa::signal_destroy(signal_);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
cl_int ConvertHSAErrorIntoCLError(hsa_status_t hsa_status) {
|
|
cl_int cl_error = CL_SUCCESS;
|
|
switch (hsa_status) {
|
|
case HSA_STATUS_ERROR_OUT_OF_RESOURCES:
|
|
cl_error = CL_OUT_OF_RESOURCES;
|
|
break;
|
|
case HSA_STATUS_ERROR_EXCEPTION:
|
|
cl_error = CL_INVALID_OPERATION;
|
|
break;
|
|
case HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS:
|
|
cl_error = CL_INVALID_ARG_VALUE;
|
|
break;
|
|
case HSA_STATUS_ERROR_INVALID_ALLOCATION:
|
|
cl_error = CL_MEM_OBJECT_ALLOCATION_FAILURE;
|
|
break;
|
|
case HSA_STATUS_ERROR_INVALID_CODE_OBJECT:
|
|
cl_error = CL_INVALID_PROGRAM;
|
|
break;
|
|
case HSA_STATUS_ERROR_INVALID_PACKET_FORMAT:
|
|
cl_error = CL_INVALID_OPERATION;
|
|
break;
|
|
case HSA_STATUS_ERROR_INVALID_ARGUMENT:
|
|
cl_error = CL_INVALID_ARG_VALUE;
|
|
break;
|
|
case HSA_STATUS_ERROR_INVALID_ISA:
|
|
cl_error = CL_INVALID_KERNEL;
|
|
break;
|
|
case (hsa_status_t)HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION:
|
|
cl_error = CL_BUILD_PROGRAM_FAILURE;
|
|
break;
|
|
case (hsa_status_t)HSA_STATUS_ERROR_MEMORY_FAULT:
|
|
cl_error = CL_INVALID_MEM_OBJECT;
|
|
break;
|
|
case (hsa_status_t)HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION:
|
|
cl_error = CL_INVALID_MEM_OBJECT;
|
|
break;
|
|
case (hsa_status_t)HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS:
|
|
cl_error = CL_OUT_OF_RESOURCES;
|
|
break;
|
|
case HSA_STATUS_ERROR:
|
|
default:
|
|
cl_error = CL_DEVICE_NOT_AVAILABLE;
|
|
break;
|
|
}
|
|
return cl_error;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void callbackQueue(hsa_status_t status, hsa_queue_t* queue, void* data) {
|
|
if (status != HSA_STATUS_SUCCESS && status != HSA_STATUS_INFO_BREAK) {
|
|
Device* dev = reinterpret_cast<Device*>(data);
|
|
std::string hostnameStr = GetLocalHostName();
|
|
std::string host_tag = hostnameStr.empty() ? "" : "host: " + hostnameStr + ", ";
|
|
std::string kernelName;
|
|
for (auto it : dev->vgpus()) {
|
|
roc::VirtualGPU* vgpu = reinterpret_cast<roc::VirtualGPU*>(it);
|
|
if (vgpu->gpu_queue() == queue) {
|
|
kernelName = vgpu->AnalyzeAqlQueue();
|
|
}
|
|
}
|
|
// Abort on device exceptions.
|
|
const char* errorMsg = 0;
|
|
Hsa::status_string(status, &errorMsg);
|
|
const char* kname = kernelName.c_str();
|
|
if (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES) {
|
|
size_t global_available_mem = 0;
|
|
if (HSA_STATUS_SUCCESS !=
|
|
Hsa::agent_get_info(dev->getBackendDevice(),
|
|
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_MEMORY_AVAIL),
|
|
&global_available_mem)) {
|
|
LogError("HSA_AMD_AGENT_INFO_MEMORY_AVAIL query failed.");
|
|
}
|
|
ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
|
|
"Callback: Queue %p Aborting with error : %s Code: 0x%x Available Free mem : %zu MB"
|
|
" [%sGPU index: %u, kernel: %s]",
|
|
queue->base_address, errorMsg, status, global_available_mem / Mi,
|
|
host_tag.c_str(), dev->index(), kname);
|
|
} else {
|
|
ClPrint(amd::LOG_NONE, amd::LOG_ALWAYS,
|
|
"Callback: Queue %p aborting with error : %s code: 0x%x"
|
|
" [%sGPU index: %u, kernel: %s]",
|
|
queue->base_address, errorMsg, status, host_tag.c_str(), dev->index(), kname);
|
|
}
|
|
|
|
// Core dumps generally provide limited value for OOM or register overflow,
|
|
// so do not let DumpCoreFile() be the reason to abort in those cases. These
|
|
// errors should still honor HIP_SKIP_ABORT_ON_GPU_ERROR consistently.
|
|
const bool is_oom = (status == HSA_STATUS_ERROR_OUT_OF_RESOURCES);
|
|
// HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS is a dispatch-time
|
|
// failure (kernel exceeds hardware register limits), not a corruption.
|
|
// Treat it as recoverable rather than aborting.
|
|
const bool invalid_dispatch =
|
|
(status == (hsa_status_t)HSA_STATUS_ERROR_INVALID_DISPATCH_PARAMETERS);
|
|
const bool should_abort =
|
|
(is_oom || invalid_dispatch)
|
|
? !HIP_SKIP_ABORT_ON_GPU_ERROR
|
|
: (amd::Os::DumpCoreFile() || !HIP_SKIP_ABORT_ON_GPU_ERROR);
|
|
|
|
if (should_abort) {
|
|
abort();
|
|
}
|
|
amd::Device::gpu_error_.store(ConvertHSAErrorIntoCLError(status), std::memory_order_relaxed);
|
|
}
|
|
}
|
|
|
|
// ================================================================================================
|
|
#if defined(__clang__)
|
|
#if __has_feature(address_sanitizer)
|
|
device::UriLocator* Device::createUriLocator() const { return new roc::UriLocator(); }
|
|
#endif
|
|
#endif
|
|
|
|
// ================================================================================================
|
|
namespace {
|
|
|
|
inline bool MapHandleType(amd::ExternalSemaphoreHandleType t,
|
|
hsa_amd_external_semaphore_handle_type_t* out) {
|
|
switch (t) {
|
|
case amd::ExternalSemaphoreHandleType::OpaqueFd:
|
|
*out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD;
|
|
return true;
|
|
case amd::ExternalSemaphoreHandleType::OpaqueWin32:
|
|
*out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32;
|
|
return true;
|
|
case amd::ExternalSemaphoreHandleType::OpaqueWin32Kmt:
|
|
*out = HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT;
|
|
return true;
|
|
default:
|
|
// Reject amd:: enum values with no HSA counterpart
|
|
// (D3D11/D3D12Fence, NvSciSync, KeyedMutex*, TimelineSemaphore*).
|
|
// Types we do map (e.g. OpaqueFd, OpaqueWin32Kmt) may still fail
|
|
// at libhsakmt import time on platforms without KMD support;
|
|
// capability ground-truth lives in the lower layer.
|
|
return false;
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// ================================================================================================
|
|
bool Device::importExtSemaphore(void** extSemaphore, const amd::Os::FileDesc& handle,
|
|
amd::ExternalSemaphoreHandleType sem_handle_type) {
|
|
if (extSemaphore == nullptr) return false;
|
|
|
|
hsa_amd_external_semaphore_handle_descriptor_t desc = {};
|
|
if (!MapHandleType(sem_handle_type, &desc.type)) return false;
|
|
|
|
// Populate the descriptor union by the *mapped* HSA type, not by
|
|
// build platform: amd::Os::FileDesc is void* on Windows and int on
|
|
// POSIX, but the HSA layer reads the union member dictated by
|
|
// desc.type. Reading a member that wasn't written is undefined
|
|
// behaviour, so reject combinations where the platform's FileDesc
|
|
// shape doesn't match the requested handle type.
|
|
switch (desc.type) {
|
|
case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32:
|
|
case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT:
|
|
#if defined(_WIN32)
|
|
desc.handle.win32_handle = handle;
|
|
break;
|
|
#else
|
|
// Win32 NT handles aren't routable through a non-Windows KMD.
|
|
return false;
|
|
#endif
|
|
case HSA_AMD_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD:
|
|
#if defined(_WIN32)
|
|
// POSIX FDs aren't routable through WDDM.
|
|
return false;
|
|
#else
|
|
desc.handle.fd = handle;
|
|
break;
|
|
#endif
|
|
default:
|
|
return false;
|
|
}
|
|
|
|
// Heap-allocated holder so the caller gets an opaque pointer for
|
|
// DestroyExtSemaphore. Storing the hsa_amd_external_semaphore_t
|
|
// preserves the syncobj identity that submitExternalSemaphoreCmd
|
|
// will consume once the queue signal/wait submission path lands.
|
|
auto* holder = new hsa_amd_external_semaphore_t{};
|
|
|
|
hsa_status_t s = hsa_amd_external_semaphore_handle_open(bkendDevice_, &desc, holder);
|
|
if (s != HSA_STATUS_SUCCESS) {
|
|
delete holder;
|
|
return false;
|
|
}
|
|
|
|
*extSemaphore = holder;
|
|
return true;
|
|
}
|
|
|
|
// ================================================================================================
|
|
void Device::DestroyExtSemaphore(void* extSemaphore) {
|
|
if (extSemaphore == nullptr) return;
|
|
auto* holder = static_cast<hsa_amd_external_semaphore_t*>(extSemaphore);
|
|
hsa_amd_external_semaphore_handle_close(*holder);
|
|
delete holder;
|
|
}
|
|
|
|
} // namespace amd::roc
|