derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1200 lines
56 KiB
C++
1200 lines
56 KiB
C++
// MIT License
|
|
//
|
|
// Copyright (c) 2017-2025 Advanced Micro Devices, Inc.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
// THE SOFTWARE.
|
|
|
|
#ifndef SRC_PM4_PMC_BUILDER_H_
|
|
#define SRC_PM4_PMC_BUILDER_H_
|
|
|
|
#include <stdint.h>
|
|
|
|
#include <algorithm>
|
|
#include <set>
|
|
#include <map>
|
|
#include <string>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "lib/aqlprofile/aqlprofile.hpp" // aqlprofile_cu_bitmap_t (gated aql_profile_v2.h)
|
|
#include "lib/aqlprofile/def/gpu_block_info.h"
|
|
#include "lib/aqlprofile/pm4/cmd_config.h"
|
|
#include "lib/aqlprofile/util/hsa_rsrc_factory.h" // AgentInfo
|
|
|
|
namespace pm4_builder
|
|
{
|
|
// MI300 UMC constants
|
|
constexpr uint32_t VIRTUALXCCID_SELECT = 0;
|
|
constexpr uint32_t UMC_MASTER_XCC = 2;
|
|
constexpr uint32_t MAX_AID = 4;
|
|
constexpr uint32_t UMC_USR_BIT = 34 - 2;
|
|
constexpr uint32_t UMC_AID_BIT = 32 - 2;
|
|
constexpr uint32_t UMC_SAMPLE_BYTE_SIZE = 8;
|
|
|
|
constexpr size_t SPI_SPECIAL_CNT = 0x1000000;
|
|
inline bool
|
|
SPISkip(size_t block, size_t id)
|
|
{
|
|
return (block & CounterBlockSPIAttr) != 0 && id >= SPI_SPECIAL_CNT;
|
|
}
|
|
|
|
enum GCMode
|
|
{
|
|
GC_MODE_XCD_ID_MASK = 0xFF,
|
|
GC_MODE_XCD = 0x100,
|
|
GC_MODE_AID = 0x200,
|
|
GC_MODE_AID_WITH_XCD_INDEX = 0x400,
|
|
GC_MODE_ALL = GC_MODE_XCD | GC_MODE_AID,
|
|
};
|
|
|
|
class CmdBuffer;
|
|
class CmdBuilder;
|
|
|
|
// helper class for building PrecExec packet
|
|
template <typename Builder>
|
|
class PrecExecBuilder
|
|
{
|
|
public:
|
|
PrecExecBuilder(Builder& _builder, CmdBuffer* cmd_buffer, uint32_t target_xcc, bool is_mi300)
|
|
: cmd_buffer_(cmd_buffer)
|
|
, builder{_builder}
|
|
, is_mi300_(is_mi300)
|
|
, target_xcc_(target_xcc)
|
|
{
|
|
if(is_mi300_)
|
|
{
|
|
// PRED_EXEC applies to MI300 only
|
|
pos_ = cmd_buffer->DwSize();
|
|
builder.BuildPredExecPacket(cmd_buffer, target_xcc_, 0);
|
|
initial_buff_size_ = cmd_buffer->DwSize();
|
|
}
|
|
}
|
|
|
|
~PrecExecBuilder()
|
|
{
|
|
if(is_mi300_)
|
|
{
|
|
// PRED_EXEC applies to MI300 only
|
|
CmdBuffer pred_exec;
|
|
// update first PRED_EXEC packet to its correct value
|
|
builder.BuildPredExecPacket(
|
|
&pred_exec, target_xcc_, cmd_buffer_->DwSize() - initial_buff_size_);
|
|
const uint32_t* data = (const uint32_t*) pred_exec.Data();
|
|
|
|
for(size_t i = 0; i < pred_exec.DwSize(); ++i)
|
|
cmd_buffer_->Assign(pos_ + i, data[i]);
|
|
}
|
|
}
|
|
|
|
private:
|
|
CmdBuffer* cmd_buffer_{nullptr};
|
|
Builder& builder;
|
|
bool is_mi300_{false};
|
|
uint32_t target_xcc_{0};
|
|
int pos_{0};
|
|
int initial_buff_size_{0};
|
|
};
|
|
|
|
// PMC PM4 commands builder virtual interface
|
|
class PmcBuilder
|
|
{
|
|
public:
|
|
PmcBuilder() {}
|
|
virtual ~PmcBuilder() {}
|
|
// Generate enable profiling commands
|
|
virtual void Enable(CmdBuffer* cmd_buffer) = 0;
|
|
// Generate disable profiling commands
|
|
virtual void Disable(CmdBuffer* cmd_buffer) = 0;
|
|
// Generate wait for GPU idle commands
|
|
virtual void WaitIdle(CmdBuffer* cmd_buffer) = 0;
|
|
// Generate start profiling commands.
|
|
virtual void Start(CmdBuffer* cmd_buffer, const counters_vector& counters_vec) = 0;
|
|
// Generate stop profiling commands.
|
|
// Return actual required data buffer size.
|
|
virtual void Stop(CmdBuffer* cmd_buffer, const counters_vector& counters_vec) = 0;
|
|
// Generate read profiling commands.
|
|
// Return actual required data buffer size.
|
|
virtual uint32_t Read(CmdBuffer* cmd_buffer,
|
|
const counters_vector& counters_vec,
|
|
void* data_buffer) = 0;
|
|
virtual int GetNumWGPs() = 0;
|
|
};
|
|
|
|
// PMC PM4 commands builder template
|
|
template <typename Builder, typename Primitives, bool concurrent>
|
|
class GpuPmcBuilder
|
|
: public PmcBuilder
|
|
, protected Primitives
|
|
{
|
|
private:
|
|
typedef uint32_t reg_addr_t;
|
|
// Shader Engines number on the GPU
|
|
uint32_t se_number_;
|
|
uint32_t wgp_per_sa_;
|
|
uint32_t sarrays_per_se_;
|
|
// XCC number on the GPU
|
|
uint32_t xcc_number_;
|
|
uint32_t xcc_per_aid_;
|
|
Builder builder;
|
|
// TODO: Temporary patch for gfx1250's asymmetric CU design, will remove
|
|
// after CU mask support is added to agent_info
|
|
bool asymmetric_cu_patch;
|
|
|
|
// Per-(SE, SA) CU-activity mask (2 CU bits per WGP) from the DRM cu_bitmap.
|
|
// bIsWGPcounter11 reads active WGPs and zero-fills harvested ones (reading a
|
|
// harvested WGP aliases onto an active one). Indexed by physical WGP slot, so
|
|
// no remap is needed and the emitted sample count stays == GetNumWGPs().
|
|
// Synthesized fully-active when no bitmap is present (V0/V1 agents).
|
|
std::vector<std::vector<uint32_t>> sa_cu_mask_;
|
|
|
|
void DebugTrace(uint32_t value)
|
|
{
|
|
CmdBuffer cmd_buffer;
|
|
uint32_t header[2] = {0, value};
|
|
APPEND_COMMAND_WRAPPER((&cmd_buffer), header);
|
|
}
|
|
|
|
// Reg-info table getting helper
|
|
const CounterRegInfo* get_reg_table(const counter_des_t& counter_des)
|
|
{
|
|
const auto* block_info = counter_des.block_info;
|
|
const auto& block_des = counter_des.block_des;
|
|
auto base_index = block_des.index;
|
|
if((block_info->attr & CounterBlockAidAttr) && (xcc_number_ > 1))
|
|
// MI300 all AID style instances fold back to per AID counter_reg_info
|
|
base_index %= (block_info->instance_count / MAX_AID);
|
|
base_index = (block_info->attr & CounterBlockExplInstAttr)
|
|
? base_index * block_info->counter_count
|
|
: 0;
|
|
return &(block_info->counter_reg_info[base_index]);
|
|
}
|
|
|
|
uint32_t GetAidNumber() const { return (xcc_number_ > 1) ? 4 : 1; }
|
|
|
|
uint32_t GetTargetAid(const counter_des_t& counter_des) const
|
|
{
|
|
const auto num_aid = GetAidNumber();
|
|
const auto num_instance = counter_des.block_info->instance_count;
|
|
const auto num_instance_per_aid = num_instance / num_aid;
|
|
const auto instance_index = counter_des.block_des.index;
|
|
const auto target_aid_index = instance_index / num_instance_per_aid;
|
|
|
|
return target_aid_index;
|
|
}
|
|
|
|
// helper function to convert a 32-bit address to a 64-bit SMN address.
|
|
// Returns the address seen by UMC_MASTER_XCC of register at reg_addr on target_aid_index.
|
|
uint64_t get_smn_addr(uint64_t addr, uint32_t target_aid_index, bool use_aid = true)
|
|
{
|
|
if((xcc_number_ > 1) && use_aid)
|
|
addr |= ((uint64_t) 1 << UMC_USR_BIT) | ((uint64_t) target_aid_index << UMC_AID_BIT);
|
|
return addr;
|
|
}
|
|
|
|
uint64_t get_smn_addr(const Register& reg, uint32_t target_aid_index, bool use_aid = true)
|
|
{
|
|
return get_smn_addr(builder.get_addr(reg), target_aid_index, use_aid);
|
|
}
|
|
|
|
// start counters for rpb-block like instances
|
|
void start_generic_mc_counters(CmdBuffer* cmd_buffer,
|
|
const std::set<uint64_t>& instances,
|
|
bool use_aid = true)
|
|
{
|
|
// insert master XCC PRED_EXEC packet here if it is MI300
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, (xcc_number_ > 1) && use_aid);
|
|
for(const auto& control_addr : instances)
|
|
{
|
|
// rpb instance clear
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::mc_reset_value());
|
|
// rpb instance enable
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::mc_start_value());
|
|
}
|
|
}
|
|
|
|
void SetGrbmGfxIndex(CmdBuffer* cmd_buffer, uint32_t value, GCMode gc_mode = GC_MODE_XCD)
|
|
{
|
|
uint32_t xcc_id;
|
|
if(gc_mode & GC_MODE_XCD)
|
|
builder.BuildWriteUConfigRegPacket(cmd_buffer, Primitives::GRBM_GFX_INDEX_ADDR, value);
|
|
if(gc_mode & GC_MODE_AID)
|
|
{
|
|
for(xcc_id = 0; xcc_id < xcc_number_; xcc_id += xcc_per_aid_)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, xcc_id, xcc_number_ > 1);
|
|
builder.BuildWritePConfigRegPacketToChiplet(cmd_buffer,
|
|
Primitives::GRBMA_GFX_INDEX_ADDR,
|
|
value,
|
|
static_cast<ChipletId>(xcc_id));
|
|
}
|
|
}
|
|
if(gc_mode & GC_MODE_AID_WITH_XCD_INDEX)
|
|
{
|
|
xcc_id = gc_mode & GC_MODE_XCD_ID_MASK;
|
|
// We don't need PrecExec for this case because the caller will program PrecExec
|
|
builder.BuildWritePConfigRegPacketToChiplet(cmd_buffer,
|
|
Primitives::GRBMA_GFX_INDEX_ADDR,
|
|
value,
|
|
static_cast<ChipletId>(xcc_id));
|
|
}
|
|
}
|
|
|
|
void SetGrbmBroadcast(CmdBuffer* cmd_buffer, GCMode gc_mode)
|
|
{
|
|
SetGrbmGfxIndex(cmd_buffer, Primitives::grbm_broadcast_value(), gc_mode);
|
|
}
|
|
|
|
void SetPerfmonCntl(CmdBuffer* cmd_buffer, uint32_t value, uint32_t attr)
|
|
{
|
|
if(attr & CounterBlockCpmonAttr)
|
|
builder.BuildWriteUConfigRegPacket(cmd_buffer, Primitives::CP_PERFMON_CNTL_ADDR, value);
|
|
if(attr & CounterBlockGrbmaAttr)
|
|
{
|
|
for(uint32_t xcc_id = 0; xcc_id < xcc_number_; xcc_id += xcc_per_aid_)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, xcc_id, xcc_number_ > 1);
|
|
builder.BuildWritePConfigRegPacketToChiplet(cmd_buffer,
|
|
Primitives::AID_PERFMON_CNTL_ADDR,
|
|
value,
|
|
static_cast<ChipletId>(xcc_id));
|
|
}
|
|
}
|
|
}
|
|
|
|
uint32_t GetInstanceIndex(uint32_t instance_index, const GpuBlockInfo* block_info)
|
|
{
|
|
// GLARB blocks require special instance handling, so we encode instance_count into
|
|
// instance_index. This won't impact GPUs without GLARB blocks
|
|
return (block_info->attr & CounterBlockGlarbAttr)
|
|
? (instance_index | (block_info->instance_count << 16))
|
|
: instance_index;
|
|
}
|
|
|
|
// Populate sa_cu_mask_[se][sa] from the agent cu_bitmap; on GFX11 also grow
|
|
// wgp_per_sa_ to the physical WGP span (highest active WGP + 1) so the read
|
|
// loop's fixed [0, wgp_per_sa_) bound reaches a middle-harvested WGP (e.g.
|
|
// gfx1151 active {0,1,2,4} -> span 5). GFX12 keeps its cu_num-derived count.
|
|
//
|
|
// Each logical (se, sa) maps to a raw bitmap slot: when logical SEs exceed
|
|
// the bitmap's SE dimension, upper SEs fold into higher SA columns, mirroring
|
|
// the kernel's gfx_v11_0_get_cu_info packing cu_bitmap[i%4][j+(i/4)*sh_per_se]
|
|
// (validated on Navi31, 6 SE; else SE 4-5 would miss the bitmap). Out-of-window
|
|
// or empty entries fall back to fully active (cu_bitmap.bits is fixed-size;
|
|
// out-of-window reads would be UB).
|
|
void build_sa_cu_mask(const AgentInfo* agent_info)
|
|
{
|
|
// Fits Navi31 (6 x 2), MI200 (8 x 1), and any future SE x SA-per-SE
|
|
// topology too large for a hardcoded kMaxSA.
|
|
sa_cu_mask_.assign(se_number_, std::vector<uint32_t>(sarrays_per_se_, 0u));
|
|
|
|
constexpr uint32_t kMaxWgpPerSa = 16;
|
|
constexpr uint32_t bitmap_se_lim = std::extent_v<decltype(aqlprofile_cu_bitmap_t::bits), 0>;
|
|
constexpr uint32_t bitmap_sa_lim = std::extent_v<decltype(aqlprofile_cu_bitmap_t::bits), 1>;
|
|
// Highest active WGP index across all SAs (physical span = index + 1).
|
|
uint32_t max_wgp_index = 0;
|
|
for(uint32_t se = 0; se < se_number_; ++se)
|
|
{
|
|
for(uint32_t sa = 0; sa < sarrays_per_se_; ++sa)
|
|
{
|
|
const uint32_t raw_se = se % bitmap_se_lim;
|
|
const uint32_t raw_sa = sa + (se / bitmap_se_lim) * sarrays_per_se_;
|
|
|
|
uint32_t cu_bm = (raw_se < bitmap_se_lim && raw_sa < bitmap_sa_lim)
|
|
? agent_info->cu_bitmap.bits[raw_se][raw_sa]
|
|
: 0u;
|
|
if(cu_bm == 0)
|
|
{
|
|
// No bitmap: synthesize a fully-active mask. Guards avoid UB:
|
|
// clz(0), and (1u << 32) when wgp_per_sa_ spans the full
|
|
// 32-bit window (16 WGPs * 2 CU bits).
|
|
if(wgp_per_sa_ == 0) continue;
|
|
cu_bm = (wgp_per_sa_ >= kMaxWgpPerSa) ? ~0u : ((1u << (wgp_per_sa_ * 2u)) - 1u);
|
|
}
|
|
sa_cu_mask_.at(se).at(sa) = cu_bm;
|
|
|
|
const uint32_t max_cu_bit = 31u - __builtin_clz(cu_bm);
|
|
const uint32_t max_wgp = max_cu_bit / 2u;
|
|
if(max_wgp > max_wgp_index) max_wgp_index = max_wgp;
|
|
}
|
|
}
|
|
|
|
if constexpr(Primitives::GFXIP_LEVEL == 11)
|
|
wgp_per_sa_ = std::max(wgp_per_sa_, max_wgp_index + 1);
|
|
}
|
|
|
|
public:
|
|
explicit GpuPmcBuilder(const AgentInfo* agent_info)
|
|
: PmcBuilder()
|
|
, se_number_(agent_info->se_num / agent_info->xcc_num)
|
|
, sarrays_per_se_(agent_info->shader_arrays_per_se)
|
|
, xcc_number_(agent_info->xcc_num)
|
|
, xcc_per_aid_(agent_info->xcc_per_aid)
|
|
, builder(acquire_ip_offset_table(agent_info))
|
|
{
|
|
this->wgp_per_sa_ = (agent_info->cu_num / 2 + sarrays_per_se_ * se_number_ - 1) /
|
|
(se_number_ * sarrays_per_se_);
|
|
this->wgp_per_sa_ /= agent_info->xcc_num;
|
|
|
|
build_sa_cu_mask(agent_info);
|
|
|
|
// Due to MI300 CP firmware issue we need to use mem_mapped_register mode to patch for GCEA
|
|
// hang. Otherwise both perfcounters mode and mem_mapped_register mode should work.
|
|
builder.bUsePerfCounterMode = (xcc_number_ > 1) ? false : true;
|
|
this->asymmetric_cu_patch = strncmp(agent_info->name, "gfx1250", 7) ? false : true;
|
|
}
|
|
|
|
int GetNumWGPs() override
|
|
{
|
|
if(Primitives::GFXIP_LEVEL >= 11) return wgp_per_sa_;
|
|
return 1;
|
|
};
|
|
|
|
// Build PMC enable PM4 commands - enable CP counting for a specific queue
|
|
void Enable(CmdBuffer* cmd_buffer)
|
|
{
|
|
// Program Compute Perfcount Enable register to support perf counting
|
|
builder.BuildWriteShRegPacket(cmd_buffer,
|
|
Primitives::COMPUTE_PERFCOUNT_ENABLE_ADDR,
|
|
Primitives::cp_perfcount_enable_value());
|
|
}
|
|
// Build PMC disable PM4 commands - enable CP counting for a specific queue
|
|
void Disable(CmdBuffer* cmd_buffer)
|
|
{
|
|
// Program Compute Perfcount Enable register to support perf counting
|
|
builder.BuildWriteShRegPacket(cmd_buffer,
|
|
Primitives::COMPUTE_PERFCOUNT_ENABLE_ADDR,
|
|
Primitives::cp_perfcount_disable_value());
|
|
}
|
|
// Build PMC waite-idle PM4 commands - enable CP counting for a specific queue
|
|
void WaitIdle(CmdBuffer* cmd_buffer)
|
|
{
|
|
// Program Compute Perfcount WaiteIdle register to support perf counting
|
|
builder.BuildWriteWaitIdlePacket(cmd_buffer);
|
|
}
|
|
|
|
// Build PMC start PM4 commands
|
|
void Start(CmdBuffer* cmd_buffer, const counters_vector& counters_vec) override
|
|
{
|
|
// Issue barrier command
|
|
if(!concurrent) builder.BuildWriteWaitIdlePacket(cmd_buffer);
|
|
GCMode gc_mode_global =
|
|
(counters_vec.get_attr() & CounterBlockGrbmaAttr) ? GC_MODE_ALL : GC_MODE_XCD;
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode_global);
|
|
// Disable RLC Perfmon Clock Gating
|
|
// On Vega this is needed to collect Perf Cntrs
|
|
if(Primitives::GFXIP_LEVEL == 9)
|
|
builder.BuildWriteUConfigRegPacket(
|
|
cmd_buffer, Primitives::RLC_PERFMON_CLK_CNTL_ADDR, 1);
|
|
// Reset perf counters
|
|
SetPerfmonCntl(
|
|
cmd_buffer, Primitives::cp_perfmon_cntl_reset_value(), counters_vec.get_attr());
|
|
// Enable SQ Counter Control enable performance counter in graphics pipeline if implied
|
|
Primitives::validate_counters(counters_vec.get_attr());
|
|
if(counters_vec.get_attr() & CounterBlockTcAttr)
|
|
{
|
|
builder.BuildWriteUConfigRegPacket(cmd_buffer,
|
|
Primitives::SQ_PERFCOUNTER_CTRL_ADDR,
|
|
Primitives::sq_control_enable_value());
|
|
}
|
|
if(Primitives::GFXIP_LEVEL >= 11 &&
|
|
(counters_vec.get_attr() & (CounterBlockTcAttr | CounterBlockSqAttr)))
|
|
{
|
|
builder.BuildWriteUConfigRegPacket(cmd_buffer,
|
|
Primitives::SQ_PERFCOUNTER_CTRL2_ADDR,
|
|
Primitives::sq_control2_enable_value());
|
|
}
|
|
#if defined(_GFX10_PRIMITIVES_H_) || defined(_GFX11_PRIMITIVES_H_)
|
|
// Clear and enable GUS counters
|
|
if(counters_vec.get_attr() & CounterBlockGusAttr)
|
|
{
|
|
builder.BuildWriteConfigRegPacket(cmd_buffer,
|
|
Primitives::GUS_PERFCOUNTER_RSLT_CNTL_ADDR,
|
|
Primitives::gus_disable_clear_value());
|
|
builder.BuildWriteConfigRegPacket(cmd_buffer,
|
|
Primitives::GUS_PERFCOUNTER_RSLT_CNTL_ADDR,
|
|
Primitives::gus_start_value());
|
|
}
|
|
#endif
|
|
|
|
// SDMA mask
|
|
// sdma performance monitor control value accumulator
|
|
std::pair<reg_addr_t, uint32_t>
|
|
sdma_select_accumulator[Primitives::SDMA_COUNTER_BLOCK_NUM_INSTANCES];
|
|
uint32_t sdma_mask = 0;
|
|
std::map<uint32_t, uint64_t> sdmas;
|
|
bool is_mi100 = false;
|
|
// UMC channels and their control register (for enable/disable) per channel
|
|
std::map<uint32_t, uint64_t> umcchs;
|
|
// RPB/ATC are per AID block like UMC above, we save its control register (for
|
|
// enable/disable) per AID instance
|
|
std::set<uint64_t> rpbs;
|
|
std::set<uint64_t> atcs;
|
|
std::set<uint64_t> perf_cnt;
|
|
// Programming perf counters
|
|
for(const auto& counter_des : counters_vec)
|
|
{
|
|
const auto* block_info = counter_des.block_info;
|
|
const auto& block_des = counter_des.block_des;
|
|
const auto* reg_table = get_reg_table(counter_des);
|
|
const auto& reg_info = reg_table[counter_des.index];
|
|
|
|
if(SPISkip(block_info->attr, counter_des.id))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// std:: cout << std::hex << "block id("<<block_des.id<<") index("<<block_des.index<<")
|
|
// counter id ("<<counter_des.id
|
|
// <<") index("<<counter_des.index<<") sel-addr("<<reg_info.select_addr<<")"
|
|
// << std::endl;
|
|
|
|
// Set GRBM index to access proper block instance
|
|
//
|
|
// TODO: In order to get different event for different instance with WGP counter blocks,
|
|
// we
|
|
// need to loop through WGP instead of blindly broadcast instance. Fortunately,
|
|
// this is not a common practice
|
|
const uint32_t grbm_value =
|
|
(block_info->instance_count > 1 && !(block_info->attr & CounterBlockWgpAttr))
|
|
? Primitives::grbm_inst_index_value(
|
|
GetInstanceIndex(block_des.index, block_info))
|
|
: Primitives::grbm_broadcast_value();
|
|
GCMode gc_mode = (block_info->attr & CounterBlockGrbmaAttr) ? GC_MODE_ALL : GC_MODE_XCD;
|
|
SetGrbmGfxIndex(cmd_buffer, grbm_value, gc_mode);
|
|
// Reset counters
|
|
if(block_info->attr & CounterBlockMcAttr)
|
|
{
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::mc_reset_value());
|
|
}
|
|
if(block_info->attr & CounterBlockCleanAttr)
|
|
{
|
|
for(uint32_t i = 0; i < block_info->counter_count; ++i)
|
|
{
|
|
builder.BuildWriteConfigRegPacket(
|
|
cmd_buffer, block_info->counter_reg_info[i].register_addr_lo, 0);
|
|
builder.BuildWriteConfigRegPacket(
|
|
cmd_buffer, block_info->counter_reg_info[i].register_addr_hi, 0);
|
|
}
|
|
}
|
|
|
|
// Setup counters
|
|
if(block_info->select_value != NULL && !(block_info->attr & CounterBlockPerfCntAttr))
|
|
{
|
|
auto select_addr = reg_info.select_addr;
|
|
auto value = block_info->select_value(counter_des);
|
|
if(block_info->attr & CounterBlockGrbmaAttr)
|
|
{
|
|
for(uint32_t xcc_id = 0; xcc_id < xcc_number_; xcc_id += xcc_per_aid_)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, xcc_id, xcc_number_ > 1);
|
|
builder.BuildWritePConfigRegPacketToChiplet(
|
|
cmd_buffer, select_addr, value, static_cast<ChipletId>(xcc_id));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
builder.BuildWriteConfigRegPacket(cmd_buffer, select_addr, value);
|
|
}
|
|
}
|
|
if(block_info->attr & CounterBlockSdmaAttr)
|
|
{
|
|
const auto sdma_index = counter_des.block_des.index;
|
|
is_mi100 = (reg_info.control_addr.offset == 0) ? true : false;
|
|
if(is_mi100)
|
|
{
|
|
// MI100: A SDMA instance shares a common control register for PERF_SEL and
|
|
// ENABLE/CLEAR.
|
|
sdma_mask |= 1u << sdma_index;
|
|
sdma_select_accumulator[sdma_index].first =
|
|
builder.get_addr(reg_info.select_addr);
|
|
sdma_select_accumulator[sdma_index].second |=
|
|
Primitives::sdma_select_value(counter_des);
|
|
}
|
|
else
|
|
{
|
|
// MI200 and MI300 have separate select and control registers
|
|
const auto sdma_index = counter_des.block_des.index;
|
|
const auto target_aid_index =
|
|
sdma_index / (block_info->instance_count / MAX_AID);
|
|
// sdma enable/clear/stop is programmed per instance and saved in sdmas
|
|
sdmas.insert(
|
|
{sdma_index, get_smn_addr(reg_info.control_addr, target_aid_index)});
|
|
|
|
if(xcc_number_ > 1)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x4B30 >> 2, 0), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x6330 >> 2, 0), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x651B0 >> 2, 0), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x661B0 >> 2, 0), 0x04000100);
|
|
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x4B30 >> 2, 1), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x6330 >> 2, 1), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x651B0 >> 2, 1), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x661B0 >> 2, 1), 0x04000100);
|
|
|
|
if(xcc_number_ > 4)
|
|
{
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x4B30 >> 2, 2), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x6330 >> 2, 2), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x651B0 >> 2, 2), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x661B0 >> 2, 2), 0x04000100);
|
|
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x4B30 >> 2, 3), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x6330 >> 2, 3), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x651B0 >> 2, 3), 0x04000100);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, get_smn_addr(0x661B0 >> 2, 3), 0x04000100);
|
|
}
|
|
}
|
|
|
|
// insert master XCC PRED_EXEC packet here if it is MI300
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
|
|
// sdma counter select is programmed per performance counter
|
|
uint64_t select_addr = get_smn_addr(reg_info.select_addr, target_aid_index);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, select_addr, Primitives::sdma_select_value(counter_des));
|
|
}
|
|
}
|
|
if(block_info->attr & (CounterBlockAidAttr | CounterBlockPerfCntAttr))
|
|
{
|
|
bool use_aid = bool(block_info->attr & CounterBlockAidAttr);
|
|
const auto target_aid_index = use_aid ? GetTargetAid(counter_des) : 0;
|
|
const auto instance_index = counter_des.block_des.index;
|
|
|
|
// insert master XCC PRED_EXEC packet here if it is MI300
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, (xcc_number_ > 1) && use_aid);
|
|
|
|
// umc counter select per UMC counter
|
|
uint64_t select_addr =
|
|
get_smn_addr(reg_info.select_addr, target_aid_index, use_aid);
|
|
uint64_t control_addr =
|
|
get_smn_addr(reg_info.control_addr, target_aid_index, use_aid);
|
|
|
|
if(block_info->attr & CounterBlockUmcAttr)
|
|
{
|
|
// skip
|
|
}
|
|
if(block_info->attr & CounterBlockPerfCntAttr)
|
|
{
|
|
if(block_info->attr & CounterBlockRpbAttr)
|
|
rpbs.insert(control_addr);
|
|
else if(block_info->attr & CounterBlockAtcAttr)
|
|
atcs.insert(control_addr);
|
|
else
|
|
perf_cnt.insert(control_addr);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, select_addr, block_info->select_value(counter_des));
|
|
}
|
|
}
|
|
// Start counters
|
|
if(block_info->attr & CounterBlockMcAttr)
|
|
{
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::mc_config_value(counter_des));
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::mc_start_value());
|
|
}
|
|
// Configure SQ block
|
|
if(block_info->attr & CounterBlockSqAttr)
|
|
{
|
|
if(Primitives::GFXIP_LEVEL == 9)
|
|
builder.BuildWriteUConfigRegPacket(cmd_buffer,
|
|
Primitives::SQ_PERFCOUNTER_MASK_ADDR,
|
|
Primitives::sq_mask_value(counter_des));
|
|
builder.BuildWriteUConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::sq_control_value(counter_des));
|
|
}
|
|
#if defined(_GFX10_PRIMITIVES_H_) || defined(_GFX11_PRIMITIVES_H_)
|
|
// Configure GUS block
|
|
if(block_info->attr & CounterBlockGusAttr)
|
|
builder.BuildWriteConfigRegPacket(
|
|
cmd_buffer, reg_info.select_addr, Primitives::gus_select_value(counter_des));
|
|
#endif
|
|
}
|
|
// SDMA start for all SDMA chnnels/instances recorded earlier
|
|
if(sdma_mask != 0)
|
|
{
|
|
// MI100
|
|
for(uint32_t sdma_index = 0, mask = sdma_mask; mask != 0; sdma_index++, mask >>= 1)
|
|
{
|
|
if(mask & 1)
|
|
{
|
|
builder.BuildWritePConfigRegPacket(cmd_buffer,
|
|
sdma_select_accumulator[sdma_index].first,
|
|
Primitives::sdma_disable_clear_value());
|
|
builder.BuildWritePConfigRegPacket(cmd_buffer,
|
|
sdma_select_accumulator[sdma_index].first,
|
|
sdma_select_accumulator[sdma_index].second);
|
|
}
|
|
}
|
|
}
|
|
if(!sdmas.empty())
|
|
{
|
|
// MI200 and MI300
|
|
// insert master XCC PRED_EXEC packet here if it is MI300
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
|
|
for(const auto& i : sdmas)
|
|
{
|
|
uint32_t sdma_index = i.first;
|
|
uint64_t control_addr = i.second;
|
|
// sdma per channel/instance clear
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::sdma_disable_clear_value());
|
|
|
|
// sdma per channel/instance enable
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::sdma_enable_value());
|
|
}
|
|
}
|
|
|
|
// RPB start for all RPB instances
|
|
if(!rpbs.empty()) start_generic_mc_counters(cmd_buffer, rpbs);
|
|
|
|
// ATC start is treated the same as RPB instance
|
|
if(!atcs.empty()) start_generic_mc_counters(cmd_buffer, atcs);
|
|
|
|
if(!perf_cnt.empty()) start_generic_mc_counters(cmd_buffer, perf_cnt, false);
|
|
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode_global);
|
|
// Program Compute Perfcount Enable register to support perf counting
|
|
builder.BuildWriteShRegPacket(cmd_buffer,
|
|
Primitives::COMPUTE_PERFCOUNT_ENABLE_ADDR,
|
|
Primitives::cp_perfcount_enable_value());
|
|
// Reset the counter list
|
|
SetPerfmonCntl(
|
|
cmd_buffer, Primitives::cp_perfmon_cntl_reset_value(), counters_vec.get_attr());
|
|
// Start the counter list
|
|
SetPerfmonCntl(
|
|
cmd_buffer, Primitives::cp_perfmon_cntl_start_value(), counters_vec.get_attr());
|
|
// Issue barrier command to apply the commands to configure perfcounters
|
|
if(!concurrent) builder.BuildWriteWaitIdlePacket(cmd_buffer);
|
|
}
|
|
|
|
// Build PMC read PM4 packets
|
|
uint32_t ReadXccPackets(CmdBuffer* cmd_buffer,
|
|
const counters_vector& counters_vec,
|
|
uint32_t* buf,
|
|
uint32_t& read_counter,
|
|
GCMode gc_mode = GC_MODE_XCD)
|
|
{
|
|
uint32_t xcc_id = gc_mode & GC_MODE_XCD_ID_MASK;
|
|
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode);
|
|
|
|
if(Primitives::GFXIP_LEVEL == 10)
|
|
{
|
|
for(auto& elem : counters_vec)
|
|
{
|
|
if((elem.block_info->attr & CounterBlockGRBMAttr) == 0) continue;
|
|
const auto& reg_info = get_reg_table(elem)[elem.index];
|
|
builder.BuildCopyCounterDataPacket(
|
|
cmd_buffer, reg_info.register_addr_lo, reg_info.register_addr_hi, buf, 3);
|
|
break;
|
|
}
|
|
}
|
|
|
|
builder.BuildWriteWaitIdlePacket(cmd_buffer);
|
|
|
|
#if defined(_GFX10_PRIMITIVES_H_) || defined(_GFX11_PRIMITIVES_H_)
|
|
// Stop GUS counters
|
|
if(counters_vec.get_attr() & CounterBlockGusAttr)
|
|
builder.BuildWriteConfigRegPacket(cmd_buffer,
|
|
Primitives::GUS_PERFCOUNTER_RSLT_CNTL_ADDR,
|
|
Primitives::gus_stop_value());
|
|
#endif
|
|
// SDMA mask
|
|
uint32_t sdma_mask = 0;
|
|
// UMC mask
|
|
uint32_t umc_mask = 0;
|
|
// Iterate through the list of blocks to create PM4 packets to read counter values
|
|
for(const auto& counter_des : counters_vec)
|
|
{
|
|
const auto* block_info = counter_des.block_info;
|
|
const auto& block_des = counter_des.block_des;
|
|
const auto* reg_table = get_reg_table(counter_des);
|
|
const auto& reg_info = reg_table[counter_des.index];
|
|
|
|
// Skip AID mode counters
|
|
if(block_info->attr & CounterBlockAidAttr) continue;
|
|
|
|
// Keep PerfCnt for XCD mode, skip it for AIGC mode
|
|
if((block_info->attr & CounterBlockPerfCntAttr) && (gc_mode & GC_MODE_XCD))
|
|
{
|
|
// Choose which counter to read
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::mc_config_value(counter_des));
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
3);
|
|
read_counter += 2;
|
|
continue;
|
|
}
|
|
|
|
if(bool(block_info->attr & CounterBlockGrbmaAttr) !=
|
|
bool(gc_mode & GC_MODE_AID_WITH_XCD_INDEX))
|
|
continue;
|
|
|
|
if(SPISkip(block_info->attr, counter_des.id))
|
|
{
|
|
read_counter += 2 * se_number_; // Skip two 64-bit SPI counters per SE
|
|
continue;
|
|
}
|
|
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode);
|
|
|
|
if(block_info->attr & CounterBlockMcAttr)
|
|
{
|
|
const uint32_t grbm_value = (block_info->instance_count > 1)
|
|
? Primitives::grbm_inst_index_value(block_des.index)
|
|
: Primitives::grbm_broadcast_value();
|
|
SetGrbmGfxIndex(cmd_buffer, grbm_value);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, reg_info.control_addr, Primitives::mc_config_value(counter_des));
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
3);
|
|
read_counter += 2;
|
|
}
|
|
else if(block_info->attr & CounterBlockSdmaAttr)
|
|
{
|
|
// Stop SDMA: this code path applies only to non-MI300
|
|
if(reg_info.control_addr.offset == 0)
|
|
{
|
|
// MI100: stopped per instance
|
|
const uint32_t mask = 1u << counter_des.block_des.index;
|
|
if((sdma_mask & mask) == 0)
|
|
{
|
|
sdma_mask |= mask;
|
|
auto control_addr = (reg_info.control_addr.offset == 0)
|
|
? reg_info.select_addr
|
|
: reg_info.control_addr;
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::sdma_stop_value(counter_des));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// MI200: stopped per counter to choose which counter to read
|
|
builder.BuildWritePConfigRegPacket(cmd_buffer,
|
|
reg_info.control_addr,
|
|
Primitives::sdma_stop_value(counter_des));
|
|
}
|
|
// Read SDMA
|
|
uint32_t dw_mask = 0x1;
|
|
if(reg_info.register_addr_hi.offset != 0) dw_mask = 0x3;
|
|
if(buf != nullptr)
|
|
{
|
|
buf[read_counter] = 0;
|
|
buf[read_counter + 1] = 0;
|
|
}
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
dw_mask);
|
|
read_counter += 2;
|
|
}
|
|
else if(block_info->attr & CounterBlockUmcAttr)
|
|
{
|
|
// skip
|
|
}
|
|
else
|
|
{
|
|
const uint32_t se_end_index =
|
|
(block_info->attr & CounterBlockSeAttr) ? se_number_ : 1;
|
|
const uint32_t sa_end_index =
|
|
(block_info->attr & CounterBlockSaAttr) ? sarrays_per_se_ : 1;
|
|
for(uint32_t se_index = 0; se_index < se_end_index; ++se_index)
|
|
for(uint32_t sarray = 0; sarray < sa_end_index; ++sarray)
|
|
{
|
|
uint32_t grbm_value = Primitives::grbm_broadcast_value();
|
|
if((block_info->instance_count > 1) &&
|
|
(block_info->attr & CounterBlockSaAttr))
|
|
{
|
|
grbm_value = Primitives::grbm_inst_se_sh_index_value(
|
|
block_des.index, se_index, sarray);
|
|
}
|
|
else if((block_info->instance_count > 1) &&
|
|
(block_info->attr & CounterBlockSeAttr))
|
|
{
|
|
grbm_value =
|
|
Primitives::grbm_inst_se_index_value(block_des.index, se_index);
|
|
}
|
|
else if(block_info->instance_count > 1)
|
|
{
|
|
grbm_value = Primitives::grbm_inst_index_value(
|
|
GetInstanceIndex(block_des.index, block_info));
|
|
}
|
|
else if(block_info->attr & CounterBlockSeAttr)
|
|
{
|
|
grbm_value = Primitives::grbm_se_index_value(se_index);
|
|
}
|
|
|
|
bool bIsWGPcounter11 = Primitives::GFXIP_LEVEL == 11 &&
|
|
(block_info->attr & CounterBlockWgpAttr);
|
|
bool bIsWGPcounter12 = Primitives::GFXIP_LEVEL >= 12 &&
|
|
(block_info->attr & CounterBlockWgpAttr);
|
|
|
|
if(bIsWGPcounter11)
|
|
{
|
|
// Read active WGPs at their physical slot; zero-fill
|
|
// harvested ones (see sa_cu_mask_). Fixed [0, wgp_per_sa_)
|
|
// bound == GetNumWGPs(), so no remap and the reader
|
|
// stays in sync.
|
|
const uint32_t cu_bm = sa_cu_mask_.at(se_index).at(sarray);
|
|
for(uint32_t wgp = 0; wgp < wgp_per_sa_; ++wgp)
|
|
{
|
|
if(cu_bm & (3u << (wgp * 2)))
|
|
{
|
|
if(block_info->instance_count > 1)
|
|
grbm_value = Primitives::grbm_inst_se_sh_wgp_index_value(
|
|
block_des.index, se_index, sarray, wgp);
|
|
else
|
|
grbm_value = Primitives::grbm_se_sh_wgp_index_value(
|
|
se_index, sarray, wgp);
|
|
SetGrbmGfxIndex(cmd_buffer, grbm_value);
|
|
uint32_t dw_mask = reg_info.register_addr_hi.offset ? 3 : 1;
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
dw_mask);
|
|
if(buf && (dw_mask == 1)) buf[read_counter + 1] = 0;
|
|
}
|
|
else if(buf != nullptr)
|
|
{
|
|
buf[read_counter] = 0;
|
|
buf[read_counter + 1] = 0;
|
|
}
|
|
read_counter += 2;
|
|
}
|
|
}
|
|
else if(bIsWGPcounter12)
|
|
{
|
|
for(uint32_t wgp = 0; wgp < wgp_per_sa_; wgp++)
|
|
{
|
|
// TODO: This patch is needed to avoid soft-hang for some WGP
|
|
// blocks, will remove after CU mask support is added to
|
|
// agent_info
|
|
if(asymmetric_cu_patch && sarray == 1 && wgp == 8)
|
|
{
|
|
if(buf != nullptr)
|
|
{
|
|
buf[read_counter] = 0;
|
|
buf[read_counter + 1] = 0;
|
|
}
|
|
read_counter += 2;
|
|
continue;
|
|
}
|
|
if(block_info->instance_count > 1)
|
|
grbm_value = Primitives::grbm_inst_se_sh_wgp_index_value(
|
|
block_des.index, se_index, sarray, wgp);
|
|
else
|
|
grbm_value = Primitives::grbm_se_sh_wgp_index_value(
|
|
se_index, sarray, wgp);
|
|
SetGrbmGfxIndex(cmd_buffer, grbm_value);
|
|
uint32_t dw_mask = reg_info.register_addr_hi.offset ? 3 : 1;
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
dw_mask);
|
|
if(buf && (dw_mask == 1)) buf[read_counter + 1] = 0;
|
|
read_counter += 2;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
SetGrbmGfxIndex(cmd_buffer, grbm_value, gc_mode);
|
|
if(block_info->attr & CounterBlockGrbmaAttr)
|
|
builder.BuildCopyCounterDataPacketFromChiplet(
|
|
cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
3,
|
|
static_cast<ChipletId>(xcc_id));
|
|
else
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
reg_info.register_addr_lo,
|
|
reg_info.register_addr_hi,
|
|
buf + read_counter,
|
|
3);
|
|
read_counter += 2;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode);
|
|
// Return amount of data to read
|
|
return read_counter * sizeof(uint32_t);
|
|
}
|
|
|
|
// Build PMC stop PM4 commands
|
|
void Stop(CmdBuffer* cmd_buffer, const counters_vector& counters_vec) override
|
|
{
|
|
GCMode gc_mode =
|
|
(counters_vec.get_attr() & CounterBlockGrbmaAttr) ? GC_MODE_ALL : GC_MODE_XCD;
|
|
// Reset Grbm to its default state - broadcast
|
|
SetGrbmBroadcast(cmd_buffer, gc_mode);
|
|
|
|
uint32_t sdma_mask = 0;
|
|
if(counters_vec.get_attr() & (CounterBlockAidAttr | CounterBlockPerfCntAttr))
|
|
{
|
|
for(const auto& counter_des : counters_vec)
|
|
{
|
|
const auto* block_info = counter_des.block_info;
|
|
const auto& block_des = counter_des.block_des;
|
|
const auto* reg_table = get_reg_table(counter_des);
|
|
const auto& reg_info = reg_table[counter_des.index];
|
|
|
|
if(block_info->attr & CounterBlockAidAttr)
|
|
{
|
|
// MI300 AID blocks: UMC/RPB/ATC/SDMA event insert master XCC PRED_EXEC packet
|
|
// here
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
|
|
const auto target_aid_index = GetTargetAid(counter_des);
|
|
uint64_t smn_control_addr =
|
|
get_smn_addr(reg_info.control_addr, target_aid_index);
|
|
|
|
if(block_info->attr & CounterBlockUmcAttr)
|
|
{
|
|
// Stop UMC
|
|
}
|
|
else if(block_info->attr & (CounterBlockRpbAttr | CounterBlockAtcAttr))
|
|
{
|
|
// Stop RPB/ATC
|
|
builder.BuildWritePConfigRegPacket(cmd_buffer, smn_control_addr, 0);
|
|
}
|
|
else if(block_info->attr & CounterBlockSdmaAttr)
|
|
{
|
|
// Stop SDMA
|
|
if(reg_info.control_addr.offset == 0)
|
|
{
|
|
// MI100: stopped per instance
|
|
const uint32_t mask = 1u << counter_des.block_des.index;
|
|
if((sdma_mask & mask) == 0)
|
|
{
|
|
sdma_mask |= mask;
|
|
auto control_addr = (reg_info.control_addr.offset == 0)
|
|
? reg_info.select_addr
|
|
: reg_info.control_addr;
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer,
|
|
control_addr,
|
|
Primitives::sdma_stop_value(counter_des));
|
|
}
|
|
}
|
|
else if(xcc_number_ > 1)
|
|
{
|
|
// MI300 SDMA event: insert master XCC PRED_EXEC packet here
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer,
|
|
smn_control_addr,
|
|
Primitives::sdma_stop_value(counter_des));
|
|
}
|
|
else
|
|
{
|
|
// MI200: stopped per counter to choose which counter to read
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer,
|
|
reg_info.control_addr,
|
|
Primitives::sdma_stop_value(counter_des));
|
|
}
|
|
}
|
|
}
|
|
else if(block_info->attr & CounterBlockPerfCntAttr)
|
|
{
|
|
// Stop Per-XCD PerfCnt
|
|
builder.BuildWritePConfigRegPacket(cmd_buffer, reg_info.control_addr, 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Issue barrier command to wait commands to complete
|
|
SetPerfmonCntl(
|
|
cmd_buffer, Primitives::cp_perfmon_cntl_stop_value(), counters_vec.get_attr());
|
|
|
|
// Enable RLC Perfmon Clock Gating. On Vega this
|
|
// was disabled during Perf Cntrs collection session
|
|
if(Primitives::GFXIP_LEVEL == 9)
|
|
builder.BuildWriteUConfigRegPacket(
|
|
cmd_buffer, Primitives::RLC_PERFMON_CLK_CNTL_ADDR, 0);
|
|
|
|
builder.BuildWriteWaitIdlePacket(cmd_buffer);
|
|
}
|
|
|
|
// Build PMC read PM4 commands
|
|
uint32_t Read(CmdBuffer* cmd_buffer,
|
|
const counters_vector& counters_vec,
|
|
void* data_buffer) override
|
|
{
|
|
uint32_t* buf = reinterpret_cast<uint32_t*>(data_buffer);
|
|
uint32_t read_counter = 0;
|
|
auto counters_attr = counters_vec.get_attr();
|
|
|
|
SetPerfmonCntl(
|
|
cmd_buffer, Primitives::cp_perfmon_cntl_read_value(), counters_vec.get_attr());
|
|
|
|
// counters have UMC events: MI300 Loop over MI300 XCCs for each counter_des
|
|
if(counters_attr & CounterBlockAidAttr)
|
|
{
|
|
for(const auto& counter_des : counters_vec)
|
|
{
|
|
const auto* block_info = counter_des.block_info;
|
|
const auto& block_des = counter_des.block_des;
|
|
const auto* reg_table = get_reg_table(counter_des);
|
|
const auto& reg_info = reg_table[counter_des.index];
|
|
|
|
if(block_info->attr & CounterBlockUmcAttr)
|
|
{
|
|
// skip
|
|
}
|
|
else if(block_info->attr & CounterBlockSdmaAttr)
|
|
{
|
|
// insert master XCC PRED_EXEC packet accordingly
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
|
|
const auto sdma_index = counter_des.block_des.index;
|
|
const auto target_aid_index = sdma_index >> 2;
|
|
|
|
// Read SDMA
|
|
uint32_t dw_mask = 0x1;
|
|
if(reg_info.register_addr_hi.offset != 0)
|
|
{
|
|
// MI200 and MI300 both have register_addr_hi
|
|
// Select which SDMA perf counter to read
|
|
auto smn_control_addr =
|
|
get_smn_addr(reg_info.control_addr, target_aid_index);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, smn_control_addr, Primitives::sdma_stop_value(counter_des));
|
|
dw_mask = 0x3;
|
|
}
|
|
auto smn_register_addr_lo =
|
|
get_smn_addr(reg_info.register_addr_lo, target_aid_index);
|
|
auto smn_register_addr_hi =
|
|
get_smn_addr(reg_info.register_addr_hi, target_aid_index);
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
smn_register_addr_lo,
|
|
smn_register_addr_hi,
|
|
buf + read_counter,
|
|
dw_mask);
|
|
read_counter += 2;
|
|
}
|
|
else if((block_info->attr & CounterBlockAidAttr))
|
|
{
|
|
// Read UMC/ATC/RPB
|
|
// insert master XCC PRED_EXEC packet accordingly
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, VIRTUALXCCID_SELECT, xcc_number_ > 1);
|
|
|
|
const auto target_aid_index = GetTargetAid(counter_des);
|
|
if(counters_attr & (CounterBlockRpbAttr | CounterBlockAtcAttr))
|
|
{
|
|
// For RPB program to choose which counter to read
|
|
uint64_t control_addr =
|
|
get_smn_addr(reg_info.control_addr, target_aid_index);
|
|
builder.BuildWritePConfigRegPacket(
|
|
cmd_buffer, control_addr, Primitives::mc_config_value(counter_des));
|
|
}
|
|
auto smn_register_addr_lo =
|
|
get_smn_addr(reg_info.register_addr_lo, target_aid_index);
|
|
auto smn_register_addr_hi =
|
|
get_smn_addr(reg_info.register_addr_hi, target_aid_index);
|
|
builder.BuildCopyCounterDataPacket(cmd_buffer,
|
|
smn_register_addr_lo,
|
|
smn_register_addr_hi,
|
|
buf + read_counter,
|
|
3);
|
|
read_counter += 2;
|
|
}
|
|
}
|
|
}
|
|
for(uint32_t xcc_selected = 0; xcc_selected < xcc_number_; ++xcc_selected)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, xcc_selected, xcc_number_ > 1);
|
|
ReadXccPackets(cmd_buffer, counters_vec, buf, read_counter);
|
|
}
|
|
// AIGC blocks
|
|
if(counters_vec.get_attr() & CounterBlockGrbmaAttr)
|
|
{
|
|
for(uint32_t xcc_selected = 0; xcc_selected < xcc_number_; xcc_selected += xcc_per_aid_)
|
|
{
|
|
PrecExecBuilder<Builder> prec_exec_builder(
|
|
builder, cmd_buffer, xcc_selected, xcc_number_ > 1);
|
|
GCMode gc_mode = (GCMode)(GC_MODE_AID_WITH_XCD_INDEX | xcc_selected);
|
|
ReadXccPackets(cmd_buffer, counters_vec, buf, read_counter, gc_mode);
|
|
}
|
|
}
|
|
|
|
builder.BuildCacheFlushPacket(
|
|
cmd_buffer, reinterpret_cast<size_t>(buf), read_counter * sizeof(uint32_t));
|
|
|
|
// Return amount of data to read
|
|
return read_counter * sizeof(uint32_t);
|
|
}
|
|
};
|
|
|
|
}; // namespace pm4_builder
|
|
|
|
#endif // SRC_PM4_PMC_BUILDER_H_
|