derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1352 lines
63 KiB
Python
1352 lines
63 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) Advanced Micro Devices. All rights reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
# this software and associated documentation files (the "Software"), to deal in
|
|
# the Software without restriction, including without limitation the rights to
|
|
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
# subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
|
|
import copy
|
|
import json
|
|
import logging
|
|
import time
|
|
|
|
from amdsmi_helpers import AMDSMIHelpers
|
|
|
|
from amdsmi import amdsmi_exception, amdsmi_interface
|
|
|
|
|
|
class MonitorCommands:
|
|
def monitor(
|
|
self,
|
|
args,
|
|
multiple_devices=False,
|
|
watching_output=False,
|
|
gpu=None,
|
|
watch=None,
|
|
watch_time=None,
|
|
iterations=None,
|
|
power_usage=None,
|
|
temperature=None,
|
|
base_board_temps=None,
|
|
gpu_board_temps=None,
|
|
gfx_util=None,
|
|
mem_util=None,
|
|
encoder=None,
|
|
decoder=None,
|
|
ecc=None,
|
|
vram_usage=None,
|
|
pcie=None,
|
|
process=None,
|
|
violation=None,
|
|
nic=None,
|
|
switch=None,
|
|
brcm_nic=None,
|
|
brcm_switch=None,
|
|
):
|
|
"""Populate a table with each GPU as an index to rows of targeted data
|
|
|
|
Args:
|
|
args (Namespace): Namespace containing the parsed CLI args
|
|
multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False.
|
|
gpu (device_handle, optional): device_handle for target device. Defaults to None.
|
|
nic (device_handle, optional): device_handle for target nic device. Defaults to None.
|
|
switch (device_handle, optional): device_handle for target switch device. Defaults to None.
|
|
watch (bool, optional): Value override for args.watch. Defaults to None.
|
|
watch_time (int, optional): Value override for args.watch_time. Defaults to None.
|
|
iterations (int, optional): Value override for args.iterations. Defaults to None.
|
|
power_usage (bool, optional): Value override for args.power_usage. Defaults to None.
|
|
temperature (bool, optional): Value override for args.temperature. Defaults to None.
|
|
base_board_temps (bool, optional): Value override for args.base_board_temps. Defaults to None.
|
|
gpu_board_temps (bool, optional): Value override for args.gpu_board_temps. Defaults to None.
|
|
gfx (bool, optional): Value override for args.gfx. Defaults to None.
|
|
mem_util (bool, optional): Value override for args.mem. Defaults to None.
|
|
encoder (bool, optional): Value override for args.encoder. Defaults to None.
|
|
decoder (bool, optional): Value override for args.decoder. Defaults to None.
|
|
ecc (bool, optional): Value override for args.ecc. Defaults to None.
|
|
vram_usage (bool, optional): Value override for args.vram_usage. Defaults to None.
|
|
pcie (bool, optional): Value override for args.pcie. Defaults to None.
|
|
process (bool, optional): Value override for args.process. Defaults to None.
|
|
violation (bool, optional): Value override for args.violation. Defaults to None.
|
|
brcm_nic (bool, optional): Value override for args.brcm_nic. Defaults to None.
|
|
brcm_switch (bool, optional): Value override for args.brcm_switch. Defaults to None.
|
|
|
|
Raises:
|
|
ValueError: Value error if no gpu value is provided
|
|
IndexError: Index error if gpu list is empty
|
|
|
|
Return:
|
|
Nothing
|
|
"""
|
|
# Set args.* to passed in arguments
|
|
if gpu:
|
|
args.gpu = gpu
|
|
if watch:
|
|
args.watch = watch
|
|
if watch_time:
|
|
args.watch_time = watch_time
|
|
if iterations:
|
|
args.iterations = iterations
|
|
if nic:
|
|
args.nic = nic
|
|
if switch:
|
|
args.switch = switch
|
|
|
|
# monitor args
|
|
if power_usage:
|
|
args.power_usage = power_usage
|
|
if temperature:
|
|
args.temperature = temperature
|
|
if base_board_temps:
|
|
args.base_board_temps = base_board_temps
|
|
if gpu_board_temps:
|
|
args.gpu_board_temps = gpu_board_temps
|
|
if gfx_util:
|
|
args.gfx = gfx_util
|
|
if mem_util:
|
|
args.mem = mem_util
|
|
if encoder:
|
|
args.encoder = encoder
|
|
if decoder:
|
|
args.decoder = decoder
|
|
if ecc:
|
|
args.ecc = ecc
|
|
if vram_usage:
|
|
args.vram_usage = vram_usage
|
|
if pcie:
|
|
args.pcie = pcie
|
|
if process:
|
|
args.process = process
|
|
if self.helpers.is_brcm_nic_initialized() and (
|
|
brcm_nic or getattr(args, "brcm_nic", False)
|
|
):
|
|
self.metric_nic(
|
|
args,
|
|
multiple_devices,
|
|
watching_output,
|
|
watch,
|
|
watch_time,
|
|
iterations,
|
|
nic=args.nic,
|
|
nic_temperature=args.temperature,
|
|
)
|
|
return
|
|
if self.helpers.is_brcm_switch_initialized() and (
|
|
brcm_switch or getattr(args, "brcm_switch", False)
|
|
):
|
|
self.metric_switch(
|
|
args,
|
|
multiple_devices,
|
|
watching_output,
|
|
watch,
|
|
watch_time,
|
|
iterations,
|
|
switch=args.switch,
|
|
)
|
|
return
|
|
if not self.helpers.is_virtual_os():
|
|
if violation:
|
|
args.violation = violation
|
|
else:
|
|
args.violation = False # Disable violation for virtual OS
|
|
|
|
# Handle No GPU passed
|
|
if args.gpu == None:
|
|
args.gpu = self.device_handles
|
|
|
|
if not self.group_check_printed:
|
|
self.helpers.check_required_groups()
|
|
self.group_check_printed = True
|
|
|
|
# If all arguments are False, the print all values
|
|
# Don't include process in this logic as it's an optional edge case
|
|
if not any(
|
|
[
|
|
args.power_usage,
|
|
args.temperature,
|
|
args.base_board_temps,
|
|
args.gpu_board_temps,
|
|
args.gfx,
|
|
args.mem,
|
|
args.encoder,
|
|
args.decoder,
|
|
args.ecc,
|
|
args.vram_usage,
|
|
args.pcie,
|
|
args.violation,
|
|
]
|
|
):
|
|
args.power_usage = args.temperature = args.gfx = args.mem = args.encoder = (
|
|
args.decoder
|
|
) = args.vram_usage = True
|
|
# set extra args for default output filtering
|
|
args.default_output = True
|
|
else:
|
|
if not hasattr(args, "default_output"):
|
|
args.default_output = False
|
|
|
|
# Handle watch logic, will only enter this block once
|
|
if args.watch:
|
|
self.helpers.handle_watch(args=args, subcommand=self.monitor, logger=self.logger)
|
|
return
|
|
|
|
# Handle multiple GPUs
|
|
if isinstance(args.gpu, list):
|
|
if len(args.gpu) > 1:
|
|
# Deepcopy gpus as recursion will destroy the gpu list
|
|
stored_gpus = []
|
|
for gpu in args.gpu:
|
|
stored_gpus.append(gpu)
|
|
|
|
# When --sort-by-pid, suppress per-GPU process collection in the
|
|
# recursive monitor() calls. Use an instance attribute so args is
|
|
# not mutated (exception-safe with try/finally).
|
|
sort_by_pid = getattr(args, "sort_by_pid", False) and args.process
|
|
self._monitor_suppress_process_collection = sort_by_pid
|
|
|
|
try:
|
|
# Store output from multiple devices without printing to console
|
|
for device_handle in args.gpu:
|
|
self.monitor(
|
|
args,
|
|
multiple_devices=True,
|
|
watching_output=watching_output,
|
|
gpu=device_handle,
|
|
)
|
|
finally:
|
|
self._monitor_suppress_process_collection = False
|
|
|
|
# Reload original gpus
|
|
args.gpu = stored_gpus
|
|
|
|
dual_csv_output = False
|
|
if args.process:
|
|
if self.logger.is_csv_format():
|
|
dual_csv_output = True
|
|
|
|
# When --sort-by-pid, route PID-grouped data through the logger:
|
|
# - JSON/CSV: rows appended to multiple_device_output
|
|
# - human-readable: ASCII table stored in _sort_by_pid_secondary_table
|
|
if sort_by_pid:
|
|
self._sort_by_pid_secondary_table = None
|
|
self._monitor_inject_sort_by_pid(args, watching_output)
|
|
|
|
# Flush the output
|
|
self.logger.print_output(
|
|
multiple_device_enabled=True,
|
|
watching_output=watching_output,
|
|
tabular=True,
|
|
dual_csv_output=dual_csv_output,
|
|
)
|
|
|
|
# Human-readable PID-grouped table: route via logger destination
|
|
# so --file redirection is honored (only populated in HR mode).
|
|
if sort_by_pid and self._sort_by_pid_secondary_table:
|
|
self._write_via_logger_destination(self._sort_by_pid_secondary_table)
|
|
|
|
# Add output to total watch output and clear multiple device output
|
|
if watching_output:
|
|
self.logger.store_watch_output(multiple_device_enabled=True)
|
|
|
|
return
|
|
elif len(args.gpu) == 1:
|
|
args.gpu = args.gpu[0]
|
|
else:
|
|
raise IndexError("args.gpu should not be an empty list")
|
|
|
|
monitor_values = {}
|
|
|
|
# Get gpu_id for logging
|
|
gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu)
|
|
|
|
# Reset the table header and store the timestamp if watch output is enabled
|
|
self.logger.table_header = "GPU"
|
|
if watching_output:
|
|
self.logger.store_output(args.gpu, "timestamp", int(time.time()))
|
|
self.logger.table_header = "TIMESTAMP".rjust(10) + " " + self.logger.table_header
|
|
|
|
if args.loglevel == "DEBUG":
|
|
try:
|
|
# Get GPU Metrics table version
|
|
gpu_metric_version_info = amdsmi_interface.amdsmi_get_gpu_metrics_header_info(
|
|
args.gpu
|
|
)
|
|
gpu_metric_version_str = json.dumps(gpu_metric_version_info, indent=4)
|
|
logging.debug(
|
|
"GPU Metrics table Version for GPU %s | %s", gpu_id, gpu_metric_version_str
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"#4 - Unable to load GPU Metrics table version for %s | %s",
|
|
gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
|
|
try:
|
|
# Get GPU Metrics table
|
|
gpu_metric_debug_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu)
|
|
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"#5 - Unable to load GPU Metrics table for %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
is_partition_metrics = False # True if we get the metrics from xcp_metrics file (amdsmi_get_gpu_partition_metrics_info)
|
|
# get metric info only once per gpu, this will speed up data output
|
|
try:
|
|
# Get GPU Metrics table
|
|
gpu_metrics_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu)
|
|
if args.loglevel == "DEBUG":
|
|
gpu_metric_debug_info = json.dumps(gpu_metrics_info, indent=4)
|
|
logging.debug("GPU Metrics table for GPU %s | %s", gpu_id, gpu_metric_debug_info)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
gpu_metrics_info = amdsmi_interface._NA_amdsmi_get_gpu_metrics_info()
|
|
logging.debug(
|
|
"Unable to load GPU Metrics table for %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
# Workaround for XCP (partition) metrics not providing num_partition in v1.9+/v1.1+
|
|
# Provides original formatting for earlier metric versions
|
|
partition_metric_info = self.helpers._get_metric_version_and_partition_info(
|
|
gpu_metrics_info, is_partition_metrics, gpu_id, args.gpu
|
|
)
|
|
partition_id = partition_metric_info["partition_id"]
|
|
num_partition = partition_metric_info["num_partition"]
|
|
|
|
# Update logger for XCP display (only if applicable)
|
|
self.logger.table_header += "XCP".rjust(5, " ")
|
|
self.logger.store_output(
|
|
args.gpu, "xcp", partition_id
|
|
) # Store partition_id initially; can be updated via num_xcp
|
|
|
|
# Store the pcie_bw values due to possible increase in bandwidth due to repeated gpu_metrics calls
|
|
if args.pcie:
|
|
try:
|
|
pcie_info = amdsmi_interface.amdsmi_get_pcie_info(args.gpu)["pcie_metric"]
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
pcie_info = "N/A"
|
|
logging.debug(
|
|
"Failed to get pci bandwidth on gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
power_unit = "W"
|
|
|
|
# Resume regular ordering of values
|
|
if args.power_usage:
|
|
try:
|
|
if gpu_metrics_info["current_socket_power"] != "N/A":
|
|
monitor_values["power_usage"] = gpu_metrics_info["current_socket_power"]
|
|
else: # Fallback to average_socket_power for older gpu_metrics versions
|
|
monitor_values["power_usage"] = gpu_metrics_info["average_socket_power"]
|
|
|
|
if (
|
|
self.logger.is_human_readable_format()
|
|
and monitor_values["power_usage"] != "N/A"
|
|
):
|
|
monitor_values["power_usage"] = f"{monitor_values['power_usage']} {power_unit}"
|
|
if self.logger.is_json_format() and monitor_values["power_usage"] != "N/A":
|
|
monitor_values["power_usage"] = {
|
|
"value": monitor_values["power_usage"],
|
|
"unit": power_unit,
|
|
}
|
|
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["power_usage"] = "N/A"
|
|
logging.debug("Failed to get power usage on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "POWER".rjust(7)
|
|
|
|
if args.power_usage and not args.default_output:
|
|
# Get Current Power Cap
|
|
try:
|
|
# assume that we're always asking for ppt0 for quick checks like this
|
|
power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu, 0)
|
|
monitor_values["max_power"] = power_cap_info[
|
|
"power_cap"
|
|
] # Get current power cap (`power_cap`) socket is set to
|
|
# `max_power_cap`, is the maximum value it can be set to
|
|
monitor_values["max_power"] = self.helpers.convert_SI_unit(
|
|
monitor_values["max_power"], AMDSMIHelpers.SI_Unit.MICRO
|
|
)
|
|
|
|
if self.logger.is_human_readable_format() and monitor_values["max_power"] != "N/A":
|
|
monitor_values["max_power"] = f"{monitor_values['max_power']} {power_unit}"
|
|
if self.logger.is_json_format() and monitor_values["max_power"] != "N/A":
|
|
monitor_values["max_power"] = {
|
|
"value": monitor_values["max_power"],
|
|
"unit": power_unit,
|
|
}
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["max_power"] = "N/A"
|
|
logging.debug(
|
|
"Failed to get power cap info for gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
self.logger.table_header += "PWR_CAP".rjust(9)
|
|
|
|
if args.temperature:
|
|
try:
|
|
temperature = gpu_metrics_info["temperature_hotspot"]
|
|
# Fallback to APU GFX temperature if hotspot is N/A
|
|
if temperature == "N/A":
|
|
temperature = gpu_metrics_info.get("apu_metrics.temperature_gfx", "N/A")
|
|
if temperature != "N/A":
|
|
temperature = round(temperature)
|
|
monitor_values["hotspot_temperature"] = temperature
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["hotspot_temperature"] = "N/A"
|
|
logging.debug("Failed to get hotspot temperature on gpu %s | %s", gpu_id, e)
|
|
|
|
try:
|
|
temperature = gpu_metrics_info["temperature_mem"]
|
|
monitor_values["memory_temperature"] = temperature
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["memory_temperature"] = "N/A"
|
|
logging.debug("Failed to get memory temperature on gpu %s | %s", gpu_id, e)
|
|
|
|
temp_unit_human_readable = "\N{DEGREE SIGN}C"
|
|
temp_unit_json = "C"
|
|
if monitor_values["hotspot_temperature"] != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["hotspot_temperature"] = (
|
|
f"{monitor_values['hotspot_temperature']} {temp_unit_human_readable}"
|
|
)
|
|
if self.logger.is_json_format():
|
|
monitor_values["hotspot_temperature"] = {
|
|
"value": monitor_values["hotspot_temperature"],
|
|
"unit": temp_unit_json,
|
|
}
|
|
if monitor_values["memory_temperature"] != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["memory_temperature"] = (
|
|
f"{monitor_values['memory_temperature']} {temp_unit_human_readable}"
|
|
)
|
|
if self.logger.is_json_format():
|
|
monitor_values["memory_temperature"] = {
|
|
"value": monitor_values["memory_temperature"],
|
|
"unit": temp_unit_json,
|
|
}
|
|
|
|
self.logger.table_header += "GPU_T".rjust(8)
|
|
self.logger.table_header += "MEM_T".rjust(8)
|
|
|
|
if args.gpu_board_temps:
|
|
try:
|
|
gpu_board_temp_dict = self.helpers.get_gpu_board_temperatures(
|
|
args.gpu, gpu_id, self.logger
|
|
)
|
|
|
|
temp_unit_json = "C"
|
|
# Add GPU board sensor headers
|
|
if gpu_board_temp_dict:
|
|
for temp_sensor in sorted(gpu_board_temp_dict.keys()):
|
|
self.logger.table_header += f"{temp_sensor}".rjust(
|
|
max(len(temp_sensor) + 2, 7)
|
|
)
|
|
for temp_type, temp_value in gpu_board_temp_dict.items():
|
|
if self.logger.is_json_format() and isinstance(temp_value, dict):
|
|
temp_value["unit"] = temp_unit_json
|
|
monitor_values[temp_type] = temp_value
|
|
except Exception as e:
|
|
logging.debug("Failed to get GPU board temperatures on gpu %s | %s", gpu_id, e)
|
|
|
|
if args.base_board_temps:
|
|
try:
|
|
base_board_temp_dict = self.helpers.get_base_board_temperatures(
|
|
args.gpu, gpu_id, self.logger
|
|
)
|
|
|
|
temp_unit_json = "C"
|
|
# Add base board sensor headers
|
|
if base_board_temp_dict:
|
|
for temp_sensor in sorted(base_board_temp_dict.keys()):
|
|
self.logger.table_header += f"{temp_sensor}".rjust(
|
|
max(len(temp_sensor) + 2, 7)
|
|
)
|
|
for temp_type, temp_value in base_board_temp_dict.items():
|
|
if self.logger.is_json_format() and isinstance(temp_value, dict):
|
|
temp_value["unit"] = temp_unit_json
|
|
monitor_values[temp_type] = temp_value
|
|
except Exception as e:
|
|
logging.debug("Failed to get base board temperatures on gpu %s | %s", gpu_id, e)
|
|
|
|
if args.gfx:
|
|
try:
|
|
gfx_clk = gpu_metrics_info["current_gfxclk"]
|
|
# Fallback to APU current GFX clock, then average
|
|
if gfx_clk == "N/A":
|
|
gfx_clk = gpu_metrics_info.get("apu_metrics.current_gfxclk", "N/A")
|
|
if gfx_clk == "N/A":
|
|
gfx_clk = gpu_metrics_info.get("apu_metrics.average_gfxclk_frequency", "N/A")
|
|
monitor_values["gfx_clk"] = gfx_clk
|
|
freq_unit = "MHz"
|
|
if gfx_clk != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["gfx_clk"] = f"{monitor_values['gfx_clk']} {freq_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["gfx_clk"] = {
|
|
"value": monitor_values["gfx_clk"],
|
|
"unit": freq_unit,
|
|
}
|
|
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["gfx_clk"] = "N/A"
|
|
logging.debug("Failed to get gfx clock on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "GFX_CLK".rjust(10)
|
|
|
|
try:
|
|
gfx_util = gpu_metrics_info["average_gfx_activity"]
|
|
activity_unit = "%"
|
|
if gfx_util != "N/A":
|
|
monitor_values["gfx"] = gfx_util
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["gfx"] = f"{monitor_values['gfx']} {activity_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["gfx"] = {"value": monitor_values["gfx"], "unit": activity_unit}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["gfx"] = "N/A"
|
|
logging.debug("Failed to get gfx utilization on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "GFX%".rjust(7)
|
|
|
|
if args.mem:
|
|
try:
|
|
mem_util = gpu_metrics_info["average_umc_activity"]
|
|
activity_unit = "%"
|
|
if mem_util != "N/A":
|
|
monitor_values["mem"] = mem_util
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["mem"] = f"{monitor_values['mem']} {activity_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["mem"] = {"value": monitor_values["mem"], "unit": activity_unit}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["mem"] = "N/A"
|
|
logging.debug("Failed to get mem utilization on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "MEM%".rjust(7)
|
|
|
|
# don't populate mem clock on default output
|
|
if not args.default_output:
|
|
try:
|
|
mem_clock = gpu_metrics_info["current_uclk"]
|
|
# Fallback to APU current UCLK, then average
|
|
if mem_clock == "N/A":
|
|
mem_clock = gpu_metrics_info.get("apu_metrics.current_uclk", "N/A")
|
|
if mem_clock == "N/A":
|
|
mem_clock = gpu_metrics_info.get(
|
|
"apu_metrics.average_uclk_frequency", "N/A"
|
|
)
|
|
monitor_values["mem_clock"] = mem_clock
|
|
freq_unit = "MHz"
|
|
if mem_clock != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["mem_clock"] = (
|
|
f"{monitor_values['mem_clock']} {freq_unit}"
|
|
)
|
|
if self.logger.is_json_format():
|
|
monitor_values["mem_clock"] = {
|
|
"value": monitor_values["mem_clock"],
|
|
"unit": freq_unit,
|
|
}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["mem_clock"] = "N/A"
|
|
logging.debug("Failed to get mem clock on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "MEM_CLOCK".rjust(11)
|
|
|
|
if args.encoder:
|
|
# TODO: The encoding utilization is in progress for Navi. Note: MI3x ASICs only support decoding.
|
|
try:
|
|
# Get List of vcn activity values
|
|
encoder_util = "N/A" # Not yet implemented
|
|
encoding_activity_avg = []
|
|
for value in encoder_util:
|
|
if isinstance(value, int):
|
|
encoding_activity_avg.append(value)
|
|
|
|
# Averaging the possible encoding activity values
|
|
if encoding_activity_avg:
|
|
encoding_activity_avg = round(
|
|
sum(encoding_activity_avg) / len(encoding_activity_avg)
|
|
)
|
|
else:
|
|
encoding_activity_avg = "N/A"
|
|
|
|
monitor_values["encoder"] = encoding_activity_avg
|
|
|
|
activity_unit = "%"
|
|
if monitor_values["encoder"] != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["encoder"] = f"{monitor_values['encoder']} {activity_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["encoder"] = {
|
|
"value": monitor_values["encoder"],
|
|
"unit": activity_unit,
|
|
}
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["encoder"] = "N/A"
|
|
logging.debug(
|
|
"Failed to get encoder utilization on gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
self.logger.table_header += "ENC%".rjust(7)
|
|
|
|
if args.decoder:
|
|
try:
|
|
# Get List of vcn activity values
|
|
# Note: MI3x ASICs only support decoding, so the vcn_activity/vcn_busy
|
|
# is used for decoding activity.
|
|
decoder_util = gpu_metrics_info["vcn_activity"]
|
|
if (
|
|
gpu_metrics_info["vcn_activity"][0] == "N/A"
|
|
and gpu_metrics_info["xcp_stats.vcn_busy"][partition_id][0] != "N/A"
|
|
):
|
|
decoder_util = gpu_metrics_info["xcp_stats.vcn_busy"][partition_id]
|
|
decoding_activity_avg = self.helpers.average_flattened_ints(
|
|
decoder_util, context="decoder_util"
|
|
)
|
|
monitor_values["decoder"] = decoding_activity_avg
|
|
|
|
activity_unit = "%"
|
|
if monitor_values["decoder"] != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["decoder"] = f"{monitor_values['decoder']} {activity_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["decoder"] = {
|
|
"value": monitor_values["decoder"],
|
|
"unit": activity_unit,
|
|
}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["decoder"] = "N/A"
|
|
logging.debug("Failed to get decoder utilization on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "DEC%".rjust(7)
|
|
|
|
if (args.encoder or args.decoder) and not args.default_output:
|
|
try:
|
|
vclock = gpu_metrics_info["current_vclk0"]
|
|
# Fallback to APU average VCLK if current is N/A
|
|
if vclock == "N/A":
|
|
vclock = gpu_metrics_info.get("apu_metrics.average_vclk_frequency", "N/A")
|
|
monitor_values["vclock"] = vclock
|
|
|
|
freq_unit = "MHz"
|
|
if vclock != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["vclock"] = f"{monitor_values['vclock']} {freq_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["vclock"] = {
|
|
"value": monitor_values["vclock"],
|
|
"unit": freq_unit,
|
|
}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["vclock"] = "N/A"
|
|
logging.debug("Failed to get dclock on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "VCLOCK".rjust(10)
|
|
|
|
try:
|
|
dclock = gpu_metrics_info["current_dclk0"]
|
|
monitor_values["dclock"] = dclock
|
|
|
|
freq_unit = "MHz"
|
|
if dclock != "N/A":
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["dclock"] = f"{monitor_values['dclock']} {freq_unit}"
|
|
if self.logger.is_json_format():
|
|
monitor_values["dclock"] = {
|
|
"value": monitor_values["dclock"],
|
|
"unit": freq_unit,
|
|
}
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
monitor_values["dclock"] = "N/A"
|
|
logging.debug("Failed to get dclock on gpu %s | %s", gpu_id, e)
|
|
|
|
self.logger.table_header += "DCLOCK".rjust(10)
|
|
|
|
if args.ecc:
|
|
try:
|
|
ecc = amdsmi_interface.amdsmi_get_gpu_total_ecc_count(args.gpu)
|
|
monitor_values["single_bit_ecc"] = ecc["correctable_count"]
|
|
monitor_values["double_bit_ecc"] = ecc["uncorrectable_count"]
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["ecc"] = "N/A"
|
|
logging.debug("Failed to get ecc on gpu %s | %s", gpu_id, e.get_error_info())
|
|
|
|
self.logger.table_header += "SINGLE_ECC".rjust(12)
|
|
self.logger.table_header += "DOUBLE_ECC".rjust(12)
|
|
|
|
try:
|
|
pcie_metric = amdsmi_interface.amdsmi_get_pcie_info(args.gpu)["pcie_metric"]
|
|
logging.debug("PCIE Metric for %s | %s", gpu_id, pcie_metric)
|
|
monitor_values["pcie_replay"] = pcie_metric["pcie_replay_count"]
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["pcie_replay"] = "N/A"
|
|
logging.debug(
|
|
"Failed to get gpu_metrics pcie replay counter on gpu %s | %s",
|
|
gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
|
|
if monitor_values["pcie_replay"] == "N/A":
|
|
try:
|
|
pcie_replay = amdsmi_interface.amdsmi_get_gpu_pci_replay_counter(args.gpu)
|
|
monitor_values["pcie_replay"] = pcie_replay
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"Failed to get sysfs pcie replay counter on gpu %s | %s",
|
|
gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
|
|
self.logger.table_header += "PCIE_REPLAY".rjust(13)
|
|
|
|
if args.vram_usage and not args.default_output:
|
|
mem_type, mem_type_name = self.helpers.get_apu_memory_type_and_name(args.gpu, gpu_id)
|
|
|
|
try:
|
|
mem_used = amdsmi_interface.amdsmi_get_gpu_memory_usage(args.gpu, mem_type) // (
|
|
1024 * 1024
|
|
)
|
|
mem_total = amdsmi_interface.amdsmi_get_gpu_memory_total(args.gpu, mem_type) // (
|
|
1024 * 1024
|
|
)
|
|
monitor_values["vram_used"] = mem_used
|
|
monitor_values["vram_free"] = mem_total - mem_used
|
|
monitor_values["vram_total"] = mem_total
|
|
if mem_total != 0:
|
|
monitor_values["vram_percent"] = round((mem_used / mem_total) * 100, 2)
|
|
else:
|
|
monitor_values["vram_percent"] = "N/A"
|
|
|
|
mem_usage_unit = "MB"
|
|
mem_percent_unit = "%"
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["vram_used"] = f"{monitor_values['vram_used']} {mem_usage_unit}"
|
|
monitor_values["vram_free"] = f"{monitor_values['vram_free']} {mem_usage_unit}"
|
|
monitor_values["vram_total"] = (
|
|
f"{monitor_values['vram_total']} {mem_usage_unit}"
|
|
)
|
|
monitor_values["vram_percent"] = (
|
|
f"{monitor_values['vram_percent']} {mem_percent_unit}"
|
|
)
|
|
if self.logger.is_json_format():
|
|
monitor_values["vram_used"] = {
|
|
"value": monitor_values["vram_used"],
|
|
"unit": mem_usage_unit,
|
|
}
|
|
monitor_values["vram_free"] = {
|
|
"value": monitor_values["vram_free"],
|
|
"unit": mem_usage_unit,
|
|
}
|
|
monitor_values["vram_total"] = {
|
|
"value": monitor_values["vram_total"],
|
|
"unit": mem_usage_unit,
|
|
}
|
|
monitor_values["vram_percent"] = {
|
|
"value": monitor_values["vram_percent"],
|
|
"unit": mem_percent_unit,
|
|
}
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["vram_used"] = "N/A"
|
|
monitor_values["vram_free"] = "N/A"
|
|
monitor_values["vram_total"] = "N/A"
|
|
monitor_values["vram_percent"] = "N/A"
|
|
logging.debug(
|
|
"Failed to get %s memory usage on gpu %s | %s",
|
|
mem_type_name.lower(),
|
|
gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
|
|
# Use appropriate headers based on memory type
|
|
self.logger.table_header += f"{mem_type_name}_USED".rjust(11)
|
|
self.logger.table_header += f"{mem_type_name}_FREE".rjust(12)
|
|
self.logger.table_header += f"{mem_type_name}_TOTAL".rjust(12)
|
|
self.logger.table_header += f"{mem_type_name}%".rjust(9)
|
|
|
|
if args.vram_usage and args.default_output:
|
|
mem_type, mem_type_name = self.helpers.get_apu_memory_type_and_name(args.gpu, gpu_id)
|
|
|
|
try:
|
|
mem_used = amdsmi_interface.amdsmi_get_gpu_memory_usage(args.gpu, mem_type) // (
|
|
1024 * 1024
|
|
)
|
|
mem_total = amdsmi_interface.amdsmi_get_gpu_memory_total(args.gpu, mem_type) // (
|
|
1024 * 1024
|
|
)
|
|
mem_usage_unit = "GB"
|
|
if self.logger.is_json_format():
|
|
monitor_values["vram_used"] = {
|
|
"value": round(mem_used / 1024, 1),
|
|
"unit": mem_usage_unit,
|
|
}
|
|
monitor_values["vram_total"] = {
|
|
"value": round(mem_total / 1024, 1),
|
|
"unit": mem_usage_unit,
|
|
}
|
|
elif self.logger.is_csv_format():
|
|
monitor_values["vram_used"] = round(mem_used / 1024, 1)
|
|
monitor_values["vram_total"] = round(mem_total / 1024, 1)
|
|
else:
|
|
monitor_values["vram_usage"] = (
|
|
f"{mem_used / 1024:5.1f}/{mem_total / 1024:5.1f} {mem_usage_unit}".rjust(
|
|
16, " "
|
|
)
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
if self.logger.is_json_format():
|
|
monitor_values["vram_used"] = "N/A"
|
|
monitor_values["vram_total"] = "N/A"
|
|
else:
|
|
monitor_values["vram_usage"] = "N/A"
|
|
logging.debug(
|
|
"Failed to get %s memory usage on gpu %s | %s",
|
|
mem_type_name.lower(),
|
|
gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
|
|
# Use appropriate header based on memory type
|
|
header_name = f"{mem_type_name}_USAGE"
|
|
self.logger.table_header += header_name.rjust(16)
|
|
|
|
if args.pcie:
|
|
if pcie_info != "N/A":
|
|
pcie_bw_unit = "Mb/s"
|
|
monitor_values["pcie_bw"] = self.helpers.unit_format(
|
|
self.logger, pcie_info["pcie_bandwidth"], pcie_bw_unit
|
|
)
|
|
else:
|
|
monitor_values["pcie_bw"] = pcie_info
|
|
|
|
self.logger.table_header += "PCIE_BW".rjust(12)
|
|
|
|
# initialize dual_csv_format; applicable to process only
|
|
dual_csv_output = False
|
|
|
|
# Store process list separately
|
|
# _monitor_suppress_process_collection is set by the multi-GPU dispatcher
|
|
# when --sort-by-pid is active; per-GPU collection is replaced by the
|
|
# PID-grouped aggregation rendered after the loop.
|
|
if args.process and not getattr(self, "_monitor_suppress_process_collection", False):
|
|
# Populate initial processes
|
|
try:
|
|
process_list = amdsmi_interface.amdsmi_get_gpu_process_list(args.gpu)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM:
|
|
raise PermissionError("Command requires elevation") from e
|
|
logging.debug(
|
|
"Failed to get process list for gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
raise e
|
|
|
|
try:
|
|
num_compute_units = amdsmi_interface.amdsmi_get_gpu_asic_info(args.gpu)[
|
|
"num_compute_units"
|
|
]
|
|
except (KeyError, amdsmi_exception.AmdSmiLibraryException) as e:
|
|
num_compute_units = "N/A"
|
|
logging.debug(
|
|
"Failed to get num compute units for gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
|
|
# Clean processes dictionary
|
|
filtered_process_values = []
|
|
for process_info in process_list:
|
|
process_info.pop("engine_usage") # Remove 'engine_usage' value
|
|
process_info["mem_usage"] = process_info.pop("mem")
|
|
process_info["cu_occupancy"] = process_info.pop("cu_occupancy")
|
|
process_info["sdma_usage"] = process_info.pop("sdma_usage")
|
|
process_info["evicted_time"] = process_info.pop("evicted_time")
|
|
|
|
memory_usage_unit = "B"
|
|
evicted_time_unit = "ms"
|
|
sdma_usage_unit = "us"
|
|
|
|
if self.logger.is_human_readable_format():
|
|
process_info["mem_usage"] = self.helpers.convert_bytes_to_readable(
|
|
process_info["mem_usage"]
|
|
)
|
|
for usage_metric in process_info["memory_usage"]:
|
|
process_info["memory_usage"][usage_metric] = (
|
|
self.helpers.convert_bytes_to_readable(
|
|
process_info["memory_usage"][usage_metric]
|
|
)
|
|
)
|
|
memory_usage_unit = ""
|
|
|
|
process_info["mem_usage"] = self.helpers.unit_format(
|
|
self.logger, process_info["mem_usage"], memory_usage_unit
|
|
)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
process_info["evicted_time"] = self.helpers.convert_time_to_readable(
|
|
process_info["evicted_time"], "ms"
|
|
)
|
|
else:
|
|
process_info["evicted_time"] = self.helpers.unit_format(
|
|
self.logger, process_info["evicted_time"], evicted_time_unit
|
|
)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
process_info["sdma_usage"] = self.helpers.convert_time_to_readable(
|
|
process_info["sdma_usage"], "us"
|
|
)
|
|
else:
|
|
process_info["sdma_usage"] = self.helpers.unit_format(
|
|
self.logger, process_info["sdma_usage"], sdma_usage_unit
|
|
)
|
|
|
|
for usage_metric in process_info["memory_usage"]:
|
|
process_info["memory_usage"][usage_metric] = self.helpers.unit_format(
|
|
self.logger, process_info["memory_usage"][usage_metric], memory_usage_unit
|
|
)
|
|
|
|
if "cu_occupancy" in process_info:
|
|
try:
|
|
cu_occupancy = process_info["cu_occupancy"]
|
|
if (
|
|
num_compute_units != "N/A"
|
|
and num_compute_units > 0
|
|
and cu_occupancy != "N/A"
|
|
):
|
|
cu_percentage = round((cu_occupancy / num_compute_units) * 100, 1)
|
|
process_info["cu_occupancy"] = self.helpers.unit_format(
|
|
self.logger, cu_percentage, "%"
|
|
)
|
|
else:
|
|
process_info["cu_occupancy"] = "N/A"
|
|
except Exception as e:
|
|
process_info["cu_occupancy"] = "N/A"
|
|
logging.debug(
|
|
"Failed to calculate cu_occupancy percentage for GPU %s | %s",
|
|
gpu_id,
|
|
str(e),
|
|
)
|
|
|
|
filtered_process_values.append({"process_info": process_info})
|
|
|
|
# If no processes are populated then we populate an N/A placeholder
|
|
if not filtered_process_values:
|
|
logging.debug("Monitor - Failed to detect any process on gpu %s", gpu_id)
|
|
filtered_process_values.append({"process_info": "N/A"})
|
|
|
|
for index, process in enumerate(filtered_process_values):
|
|
if process["process_info"] == "N/A":
|
|
filtered_process_values[index]["process_info"] = "No running processes detected"
|
|
|
|
# Build the process table's title and header
|
|
self.logger.secondary_table_title = "PROCESS INFO"
|
|
self.logger.secondary_table_header = (
|
|
"GPU".rjust(3)
|
|
+ "NAME".rjust(19)
|
|
+ "PID".rjust(9)
|
|
+ "GTT_MEM".rjust(10)
|
|
+ "CPU_MEM".rjust(10)
|
|
+ "VRAM_MEM".rjust(10)
|
|
+ "MEM_USG".rjust(10)
|
|
+ "CU%".rjust(9)
|
|
+ "SDMA".rjust(8)
|
|
+ "EVICT".rjust(8)
|
|
)
|
|
|
|
if watching_output:
|
|
self.logger.secondary_table_header = (
|
|
"TIMESTAMP".rjust(10) + " " + self.logger.secondary_table_header
|
|
)
|
|
|
|
logging.debug(f"Monitor - Process Info for GPU {gpu_id} | {filtered_process_values}")
|
|
|
|
if self.logger.is_json_format():
|
|
self.logger.store_output(args.gpu, "process_list", filtered_process_values)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
# Print out process in flattened format
|
|
# The logger detects if process list is present and pulls it out and prints
|
|
# that table with timestamp, gpu, and prints headers separately
|
|
self.logger.store_output(args.gpu, "process_list", filtered_process_values)
|
|
|
|
if self.logger.is_csv_format():
|
|
dual_csv_output = True
|
|
# The logger detects if process list is present and pulls it out and prints
|
|
# that table with timestamp, gpu, and prints headers separately
|
|
self.logger.store_output(args.gpu, "process_list", filtered_process_values)
|
|
|
|
###################
|
|
### XCP Metrics ###
|
|
###################
|
|
# Must come after process list - XCP detail is a multi-dimensional array, which is displayed
|
|
# in tabular format with XCP values for same gpu shown on multiple lines.
|
|
if args.violation:
|
|
violation_status = {
|
|
"pviol": "N/A",
|
|
"tviol": "N/A",
|
|
"tviol_active": "N/A",
|
|
"phot_tviol": "N/A",
|
|
"vr_tviol": "N/A",
|
|
"hbm_tviol": "N/A",
|
|
"gfx_clkviol": "N/A",
|
|
"gfxclk_pviol": "N/A",
|
|
"gfxclk_tviol": "N/A",
|
|
"gfxclk_totalviol": "N/A",
|
|
"low_utilviol": "N/A",
|
|
}
|
|
try:
|
|
violations = amdsmi_interface.amdsmi_get_violation_status(args.gpu)
|
|
violation_status["pviol"] = violations["per_ppt_pwr"]
|
|
violation_status["tviol"] = violations["per_socket_thrm"]
|
|
violation_status["tviol_active"] = violations["active_socket_thrm"]
|
|
violation_status["phot_tviol"] = violations["per_prochot_thrm"]
|
|
violation_status["vr_tviol"] = violations["per_vr_thrm"]
|
|
violation_status["hbm_tviol"] = violations["per_hbm_thrm"]
|
|
violation_status["gfx_clkviol"] = violations["per_gfx_clk_below_host_limit"]
|
|
violation_status["gfxclk_pviol"] = violations["per_gfx_clk_below_host_limit_pwr"]
|
|
violation_status["gfxclk_tviol"] = violations["per_gfx_clk_below_host_limit_thm"]
|
|
violation_status["gfxclk_totalviol"] = violations[
|
|
"per_gfx_clk_below_host_limit_total"
|
|
]
|
|
violation_status["low_utilviol"] = violations["per_low_utilization"]
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
monitor_values["pviol"] = violation_status["pviol"]
|
|
monitor_values["tviol"] = violation_status["tviol"]
|
|
monitor_values["tviol_active"] = violation_status["tviol_active"]
|
|
monitor_values["phot_tviol"] = violation_status["phot_tviol"]
|
|
monitor_values["vr_tviol"] = violation_status["vr_tviol"]
|
|
monitor_values["hbm_tviol"] = violation_status["hbm_tviol"]
|
|
monitor_values["gfx_clkviol"] = violation_status["gfx_clkviol"]
|
|
monitor_values["gfxclk_pviol"] = violation_status["gfxclk_pviol"]
|
|
monitor_values["gfxclk_tviol"] = violation_status["gfxclk_tviol"]
|
|
monitor_values["gfxclk_totalviol"] = violation_status["gfxclk_totalviol"]
|
|
monitor_values["low_utilviol"] = violation_status["low_utilviol"]
|
|
logging.debug(
|
|
"Failed to get violation status on gpu %s | %s", gpu_id, e.get_error_info()
|
|
)
|
|
violation_status_unit = "%"
|
|
kPVIOL_MAX_WIDTH = 7
|
|
kTVIOL_MAX_WIDTH = 7
|
|
kTVIOL_ACTIVE_MAX_WIDTH = 14
|
|
kPHOT_MAX_WIDTH = 12
|
|
kVR_MAX_WIDTH = 10
|
|
kHBM_MAX_WIDTH = 11
|
|
kGFXC_MAX_WIDTH = 13
|
|
kGFXC_PVIOL_MAX_WIDTH = 58
|
|
kGFXC_TVIOL_MAX_WIDTH = kGFXC_PVIOL_MAX_WIDTH
|
|
kGFXC_TOTALVIOL_MAX_WIDTH = kGFXC_PVIOL_MAX_WIDTH
|
|
kLOW_UTILVIOL_MAX_WIDTH = kGFXC_PVIOL_MAX_WIDTH
|
|
|
|
for key, value in violation_status.items():
|
|
if not isinstance(value, list):
|
|
if value != "N/A":
|
|
if key == "tviol_active" or key == "xcp":
|
|
monitor_values[key] = value
|
|
else:
|
|
monitor_values[key] = self.helpers.unit_format(
|
|
self.logger, violation_status[key], violation_status_unit
|
|
)
|
|
else:
|
|
monitor_values[key] = violation_status[key]
|
|
else:
|
|
if num_partition != "N/A":
|
|
# these are one after another, in order to display each in sub-sections
|
|
new_xcp_dict = {}
|
|
for current_xcp in range(num_partition):
|
|
new_xcp_dict[f"xcp_{current_xcp}"] = self.helpers.unit_format(
|
|
self.logger, value[current_xcp], "%"
|
|
)
|
|
monitor_values[key] = new_xcp_dict
|
|
else:
|
|
monitor_values[key] = value[0] if value else "N/A"
|
|
# save deep copy of monitor values, used later to grab xcp specific values
|
|
monitor_values_deepcopy = copy.deepcopy(monitor_values)
|
|
|
|
self.logger.table_header += "PVIOL".rjust(kPVIOL_MAX_WIDTH, " ")
|
|
self.logger.table_header += "TVIOL".rjust(kTVIOL_MAX_WIDTH, " ")
|
|
self.logger.table_header += "TVIOL_ACTIVE".rjust(kTVIOL_ACTIVE_MAX_WIDTH, " ")
|
|
self.logger.table_header += "PHOT_TVIOL".rjust(kPHOT_MAX_WIDTH, " ")
|
|
self.logger.table_header += "VR_TVIOL".rjust(kVR_MAX_WIDTH, " ")
|
|
self.logger.table_header += "HBM_TVIOL".rjust(kHBM_MAX_WIDTH, " ")
|
|
self.logger.table_header += "GFX_CLKVIOL".rjust(kGFXC_MAX_WIDTH, " ")
|
|
self.logger.table_header += "GFXCLK_PVIOL".rjust(kGFXC_PVIOL_MAX_WIDTH, " ")
|
|
self.logger.table_header += "GFXCLK_TVIOL".rjust(kGFXC_TVIOL_MAX_WIDTH, " ")
|
|
self.logger.table_header += "GFXCLK_TOTALVIOL".rjust(kGFXC_TOTALVIOL_MAX_WIDTH, " ")
|
|
self.logger.table_header += "LOW_UTILVIOL".rjust(kLOW_UTILVIOL_MAX_WIDTH, " ")
|
|
|
|
# Print/capture by XCPs
|
|
if num_partition != "N/A" and partition_id == 0:
|
|
current_xcp = 0
|
|
while current_xcp in range(num_partition) or current_xcp == 0:
|
|
if not multiple_devices and watching_output and current_xcp == 0:
|
|
# Need to clear output for single device, otherwise while watching output
|
|
# XCP detail will continue stacking on top of each other
|
|
self.logger.clear_multiple_devices_output()
|
|
|
|
if watching_output:
|
|
self.logger.store_output(args.gpu, "timestamp", int(time.time()))
|
|
|
|
if current_xcp != 0: # set all other values without XCP stats to N/A
|
|
self.logger.store_output(args.gpu, "xcp", current_xcp)
|
|
monitor_values["pviol"] = "N/A"
|
|
monitor_values["tviol"] = "N/A"
|
|
monitor_values["tviol_active"] = "N/A"
|
|
monitor_values["phot_tviol"] = "N/A"
|
|
monitor_values["vr_tviol"] = "N/A"
|
|
monitor_values["hbm_tviol"] = "N/A"
|
|
monitor_values["gfx_clkviol"] = "N/A"
|
|
for (
|
|
k,
|
|
_,
|
|
) in monitor_values.items(): # change other keys to "N/A" since we should have all applicable XCP stats
|
|
# eg. amd-smi monitor -p -t -V should only show XCP info for violations
|
|
# below primary device
|
|
if k != "xcp" and k not in [
|
|
"gfxclk_pviol",
|
|
"gfxclk_tviol",
|
|
"gfxclk_totalviol",
|
|
"low_utilviol",
|
|
]:
|
|
monitor_values[k] = "N/A"
|
|
|
|
if isinstance(monitor_values_deepcopy["gfxclk_pviol"], dict):
|
|
monitor_values["gfxclk_pviol"] = monitor_values_deepcopy["gfxclk_pviol"][
|
|
f"xcp_{current_xcp}"
|
|
]
|
|
if isinstance(monitor_values_deepcopy["gfxclk_tviol"], dict):
|
|
monitor_values["gfxclk_tviol"] = monitor_values_deepcopy["gfxclk_tviol"][
|
|
f"xcp_{current_xcp}"
|
|
]
|
|
if isinstance(monitor_values_deepcopy["gfxclk_totalviol"], dict):
|
|
monitor_values["gfxclk_totalviol"] = monitor_values_deepcopy[
|
|
"gfxclk_totalviol"
|
|
][f"xcp_{current_xcp}"]
|
|
if isinstance(monitor_values_deepcopy["low_utilviol"], dict):
|
|
monitor_values["low_utilviol"] = monitor_values_deepcopy["low_utilviol"][
|
|
f"xcp_{current_xcp}"
|
|
]
|
|
|
|
if self.logger.is_human_readable_format():
|
|
monitor_values["pviol"] = monitor_values["pviol"]
|
|
monitor_values["tviol"] = monitor_values["tviol"]
|
|
monitor_values["phot_tviol"] = monitor_values["phot_tviol"]
|
|
monitor_values["vr_tviol"] = monitor_values["vr_tviol"]
|
|
monitor_values["hbm_tviol"] = monitor_values["hbm_tviol"]
|
|
monitor_values["gfx_clkviol"] = monitor_values["gfx_clkviol"]
|
|
monitor_values["gfxclk_pviol"] = str(
|
|
monitor_values["gfxclk_pviol"]
|
|
).replace("'", "")
|
|
monitor_values["gfxclk_tviol"] = str(
|
|
monitor_values["gfxclk_tviol"]
|
|
).replace("'", "")
|
|
monitor_values["gfxclk_totalviol"] = str(
|
|
monitor_values["gfxclk_totalviol"]
|
|
).replace("'", "")
|
|
monitor_values["low_utilviol"] = str(
|
|
monitor_values["low_utilviol"]
|
|
).replace("'", "")
|
|
self.logger.store_output(args.gpu, "values", monitor_values)
|
|
self.logger.store_multiple_device_output()
|
|
current_xcp += 1
|
|
else:
|
|
self.logger.store_output(args.gpu, "xcp", partition_id)
|
|
self.logger.store_output(args.gpu, "values", monitor_values)
|
|
|
|
# Store typical output for all commands (XCP data will be handled separately, eg. violation status)
|
|
if not args.violation:
|
|
self.logger.store_output(args.gpu, "values", monitor_values)
|
|
|
|
# Now handling the single gpu case only
|
|
if multiple_devices:
|
|
self.logger.store_multiple_device_output()
|
|
return
|
|
|
|
if (
|
|
watching_output and not self.logger.destination == "stdout"
|
|
): # End of single gpu add to watch_output
|
|
self.logger.store_watch_output(multiple_device_enabled=False)
|
|
|
|
if args.violation:
|
|
# Print violation status for single gpu, which have different xcp information per 1 gpu
|
|
self.logger.print_output(
|
|
multiple_device_enabled=True,
|
|
watching_output=watching_output,
|
|
tabular=True,
|
|
dual_csv_output=dual_csv_output,
|
|
)
|
|
else:
|
|
# Print the output for single gpu, which currently does not have multiple xcp information
|
|
self.logger.print_output(
|
|
multiple_device_enabled=False,
|
|
watching_output=watching_output,
|
|
tabular=True,
|
|
dual_csv_output=dual_csv_output,
|
|
)
|
|
|
|
def _write_via_logger_destination(self, text):
|
|
"""Write text to the logger's destination (stdout or output file).
|
|
|
|
Used for content that does not fit the logger's structured output path
|
|
but must still honor --file redirection.
|
|
"""
|
|
if self.logger.destination == "stdout":
|
|
print(text)
|
|
else:
|
|
with self.logger.destination.open("a", encoding="utf-8") as f:
|
|
f.write(text + "\n")
|
|
|
|
def _monitor_inject_sort_by_pid(self, args, watching_output):
|
|
"""Render PID-grouped process info for monitor --process --sort-by-pid.
|
|
|
|
For JSON/CSV the rows are pushed into the logger's multiple_device_output
|
|
so they flow through print_output() like every other monitor row. For
|
|
human-readable output an ASCII table is stored in
|
|
self._sort_by_pid_secondary_table; the caller is responsible for writing
|
|
it via _write_via_logger_destination after the primary table.
|
|
"""
|
|
handles = args.gpu if isinstance(args.gpu, list) else [args.gpu]
|
|
try:
|
|
pid_list = amdsmi_interface.amdsmi_get_gpu_process_list_by_pid(handles)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug("Failed to get process list by pid | %s", e.get_error_info())
|
|
return
|
|
|
|
# Unit string constants — hoisted out of the loop (style cleanup)
|
|
memory_usage_unit_default = "B"
|
|
evicted_time_unit = "ms"
|
|
sdma_usage_unit = "us"
|
|
|
|
# Build one entry per PID+GPU combination, grouped so all GPUs for a PID
|
|
# appear together.
|
|
filtered = []
|
|
for proc in pid_list:
|
|
for gpu_entry in proc["gpus"]:
|
|
memory_usage_unit = memory_usage_unit_default
|
|
|
|
mem_usage = gpu_entry["mem"]
|
|
mem_dict = {
|
|
"gtt_mem": gpu_entry["memory_usage"]["gtt_mem"],
|
|
"cpu_mem": gpu_entry["memory_usage"]["cpu_mem"],
|
|
"vram_mem": gpu_entry["memory_usage"]["vram_mem"],
|
|
}
|
|
|
|
if self.logger.is_human_readable_format():
|
|
mem_usage = self.helpers.convert_bytes_to_readable(mem_usage)
|
|
for k in mem_dict:
|
|
mem_dict[k] = self.helpers.convert_bytes_to_readable(mem_dict[k])
|
|
memory_usage_unit = ""
|
|
|
|
mem_usage = self.helpers.unit_format(self.logger, mem_usage, memory_usage_unit)
|
|
for k in mem_dict:
|
|
mem_dict[k] = self.helpers.unit_format(
|
|
self.logger, mem_dict[k], memory_usage_unit
|
|
)
|
|
|
|
cu = gpu_entry["cu_occupancy"]
|
|
if cu == "N/A":
|
|
cu_str = "N/A"
|
|
else:
|
|
cu_str = self.helpers.unit_format(self.logger, cu, "")
|
|
|
|
if self.logger.is_human_readable_format():
|
|
sdma = self.helpers.convert_time_to_readable(gpu_entry["sdma_usage"], "us")
|
|
evict = self.helpers.convert_time_to_readable(gpu_entry["evicted_time"], "ms")
|
|
else:
|
|
sdma = self.helpers.unit_format(
|
|
self.logger, gpu_entry["sdma_usage"], sdma_usage_unit
|
|
)
|
|
evict = self.helpers.unit_format(
|
|
self.logger, gpu_entry["evicted_time"], evicted_time_unit
|
|
)
|
|
|
|
info = {
|
|
"name": proc["name"],
|
|
"pid": proc["pid"],
|
|
"memory_usage": mem_dict,
|
|
"mem_usage": mem_usage,
|
|
"cu_occupancy": cu_str,
|
|
"sdma_usage": sdma,
|
|
"evicted_time": evict,
|
|
}
|
|
filtered.append({"gpu_index": gpu_entry["gpu_index"], "process_info": info})
|
|
|
|
if not filtered:
|
|
filtered.append({"gpu_index": "N/A", "process_info": "No running processes detected"})
|
|
|
|
# JSON / CSV: push rows into the logger's multiple_device_output so they
|
|
# flow through print_output() — never write a raw ASCII tail that would
|
|
# corrupt structured output.
|
|
if self.logger.is_json_format() or self.logger.is_csv_format():
|
|
for item in filtered:
|
|
info = item["process_info"]
|
|
if isinstance(info, str):
|
|
row = {"gpu_index": item["gpu_index"], "message": info}
|
|
else:
|
|
row = {
|
|
"gpu_index": item["gpu_index"],
|
|
"pid": info["pid"],
|
|
"name": info["name"],
|
|
"mem_usage": info.get("mem_usage", "N/A"),
|
|
"cu_occupancy": info.get("cu_occupancy", "N/A"),
|
|
"sdma_usage": info.get("sdma_usage", "N/A"),
|
|
"evicted_time": info.get("evicted_time", "N/A"),
|
|
}
|
|
mu = info.get("memory_usage", {})
|
|
if isinstance(mu, dict):
|
|
if self.logger.is_csv_format():
|
|
row.update(self.logger.flatten_dict(mu))
|
|
else:
|
|
row["memory_usage"] = mu
|
|
self.logger.output = row
|
|
self.logger.store_multiple_device_output()
|
|
return
|
|
|
|
# Human-readable: build the ASCII secondary table; the caller writes it
|
|
# via _write_via_logger_destination after print_output().
|
|
header = (
|
|
"\nPROCESS INFO (grouped by PID):\n"
|
|
+ "GPU".rjust(3)
|
|
+ "NAME".rjust(19)
|
|
+ "PID".rjust(9)
|
|
+ "GTT_MEM".rjust(10)
|
|
+ "CPU_MEM".rjust(10)
|
|
+ "VRAM_MEM".rjust(10)
|
|
+ "MEM_USG".rjust(10)
|
|
+ "CU%".rjust(9)
|
|
+ "SDMA".rjust(8)
|
|
+ "EVICT".rjust(8)
|
|
)
|
|
rows = []
|
|
for item in filtered:
|
|
info = item["process_info"]
|
|
if isinstance(info, str):
|
|
rows.append(" " + info)
|
|
continue
|
|
name = str(info.get("name", "N/A")).split("/")[-1][:17]
|
|
row = str(item["gpu_index"]).rjust(3)
|
|
row += name.rjust(19)
|
|
row += str(info["pid"]).rjust(9)
|
|
mu = info.get("memory_usage", {})
|
|
row += str(mu.get("gtt_mem", "N/A")).rjust(10)
|
|
row += str(mu.get("cpu_mem", "N/A")).rjust(10)
|
|
row += str(mu.get("vram_mem", "N/A")).rjust(10)
|
|
row += str(info.get("mem_usage", "N/A")).rjust(10)
|
|
row += str(info.get("cu_occupancy", "N/A")).rjust(9)
|
|
row += str(info.get("sdma_usage", "N/A")).rjust(8)
|
|
row += str(info.get("evicted_time", "N/A")).rjust(8)
|
|
rows.append(row)
|
|
|
|
self._sort_by_pid_secondary_table = header + "\n" + "\n".join(rows)
|