derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1106 lines
50 KiB
Python
1106 lines
50 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (C) Advanced Micro Devices. All rights reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
# this software and associated documentation files (the "Software"), to deal in
|
|
# the Software without restriction, including without limitation the rights to
|
|
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
# subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
|
|
import logging
|
|
|
|
from amdsmi import amdsmi_exception, amdsmi_interface
|
|
|
|
|
|
class TopologyCommands:
|
|
def topology_nic(
|
|
self,
|
|
args,
|
|
multiple_devices=False,
|
|
gpu=None,
|
|
nic=None,
|
|
nic_topo=None,
|
|
nic_switch=None,
|
|
multiple_device_enabled=None,
|
|
switch=None,
|
|
):
|
|
"""Get topology information for target gpus
|
|
params:
|
|
args - argparser args to pass to subcommand
|
|
multiple_devices (bool) - True if checking for multiple devices
|
|
gpu (device_handle) - device_handle for target device
|
|
nic (device_handle) - device_handle for target device
|
|
nic_topo (bool) - True if checking for connectivity between nic and gpu devices
|
|
nic_switch (bool) - True if checking for gpu, nic and switch device's affinity and parent switch
|
|
switch (device_handle) - device_handle for target device
|
|
|
|
return:
|
|
Nothing
|
|
"""
|
|
# Set args.* to passed in arguments
|
|
if gpu:
|
|
args.gpu = gpu
|
|
if nic:
|
|
args.nic = nic
|
|
if nic_topo:
|
|
args.nic_topo = nic_topo
|
|
if nic_switch:
|
|
args.nic_switch = nic_switch
|
|
if switch:
|
|
args.switch = switch
|
|
|
|
if not self.group_check_printed:
|
|
self.helpers.check_required_groups()
|
|
self.group_check_printed = True
|
|
|
|
is_single_nic_request = False # -N option
|
|
is_single_switch_request = False # --switch option
|
|
is_single_gpu_request = False # -g option
|
|
|
|
gpucount = 0
|
|
niccount = 0
|
|
switchcount = 0
|
|
|
|
# The topology parser conditionally registers --nic / --switch based on
|
|
# is_brcm_nic_initialized() / is_brcm_switch_initialized(). On hosts
|
|
# where one is gated off, the corresponding argparse attribute is
|
|
# absent on the namespace; getattr() guards the unconditional access
|
|
# below to prevent AttributeError on single-subsystem hosts.
|
|
if getattr(args, "nic", None) is None:
|
|
args.nic = self.device_handles_brcm_nics
|
|
if not isinstance(args.nic, list):
|
|
args.nic = [args.nic]
|
|
if len(args.nic) == 1:
|
|
is_single_nic_request = True
|
|
niccount = len(args.nic)
|
|
|
|
if getattr(args, "switch", None) is None:
|
|
args.switch = self.device_handles_switchs
|
|
if not isinstance(args.switch, list):
|
|
args.switch = [args.switch]
|
|
if len(args.switch) == 1:
|
|
is_single_switch_request = True
|
|
switchcount = len(args.switch)
|
|
|
|
if args.gpu is None:
|
|
args.gpu = self.device_handles
|
|
if not isinstance(args.gpu, list):
|
|
args.gpu = [args.gpu]
|
|
if len(args.gpu) == 1:
|
|
is_single_gpu_request = True
|
|
gpucount = len(args.gpu)
|
|
|
|
# Clear the table header
|
|
self.logger.table_header = "".rjust(12)
|
|
|
|
if args.nic_topo:
|
|
topo_dict = {}
|
|
|
|
# Loop through each NIC to get its BDF and corresponding GPU statuses
|
|
for idx, dest_nic in enumerate(args.nic):
|
|
# Get NIC ID and BDF
|
|
nic_bdf = ""
|
|
nic_info = amdsmi_interface.amdsmi_get_nic_info(dest_nic)
|
|
if nic_info:
|
|
nic_bdf = nic_info["bdf"]
|
|
nic_id = self.helpers.get_nic_id_from_device_handle(dest_nic)
|
|
|
|
# List to store the GPU statuses for this NIC
|
|
gpu_statuses_for_nic = []
|
|
|
|
# Loop through each GPU to determine its status
|
|
for gpu_dest in args.gpu:
|
|
gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest)
|
|
gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest)
|
|
try:
|
|
status = amdsmi_interface.amdsmi_get_nic_gpu_topo_info(dest_nic, gpu_dest)
|
|
except amdsmi_exception.AmdSmiLibraryException:
|
|
status = "N/A"
|
|
gpu_statuses_for_nic.append((gpu_bdf, status)) # Store BDF and status as tuple
|
|
|
|
# Store NIC BDF and associated GPU statuses in the dictionary
|
|
topo_dict[nic_bdf] = gpu_statuses_for_nic
|
|
|
|
# Prepare tabular output for logger
|
|
tabular_output = []
|
|
|
|
# Add header row for GPU BDFs
|
|
if self.logger.is_human_readable_format():
|
|
header_row = {"brcm_nic": "", "bdf": "".rjust(19)}
|
|
else:
|
|
header_row = {}
|
|
|
|
gpu_bdfs = [] # List to store GPU BDFs for the header
|
|
for gpu_dest in args.gpu:
|
|
gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest)
|
|
gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest)
|
|
if self.logger.is_human_readable_format():
|
|
header_row[f"GPU BDF_{gpu_bdf}"] = f"{gpu_bdf}".rjust(20)
|
|
else:
|
|
header_row[f"gpu_{gpu_id}"] = f"{gpu_bdf}"
|
|
gpu_bdfs.append(gpu_bdf) # Store GPU BDF for later reference
|
|
|
|
# Add the header row
|
|
tabular_output.append(header_row)
|
|
|
|
# Add NIC rows with their associated GPU statuses
|
|
for idx, (nic_bdf, gpu_info) in enumerate(topo_dict.items()):
|
|
if not is_single_nic_request:
|
|
if self.logger.is_human_readable_format():
|
|
nic_row = {
|
|
"brcm_nic": f"BRCM_NIC{idx}".ljust(12),
|
|
"bdf": f"{nic_bdf}".ljust(18),
|
|
}
|
|
else:
|
|
nic_row = {"brcm_nic": f"BRCM_NIC{idx}", "bdf": f"{nic_bdf}"}
|
|
else: # nic_id get stored in the initial iteration
|
|
if self.logger.is_human_readable_format():
|
|
nic_row = {
|
|
"brcm_nic": f"BRCM_NIC{nic_id}".ljust(12),
|
|
"bdf": f"{nic_bdf}".ljust(18),
|
|
}
|
|
else:
|
|
nic_row = {"brcm_nic": f"BRCM_NIC{nic_id}", "bdf": f"{nic_bdf}"}
|
|
|
|
# Add GPU BDFs and statuses in the row
|
|
for gpu_idx, (gpu_bdf, status) in enumerate(gpu_info):
|
|
if self.logger.is_human_readable_format():
|
|
nic_row[f"GPU{gpu_idx} Status"] = status.ljust(20)
|
|
else:
|
|
nic_row[f"gpu_{gpu_idx}_topo"] = status
|
|
# Add the NIC row to the table
|
|
tabular_output.append(nic_row)
|
|
|
|
# Use the logger to display the table
|
|
|
|
# Construct the table header with GPU column names (adjusting for multiple GPUs)
|
|
gpu_columns = [f"GPU{idx} " for idx in range(len(gpu_bdfs))]
|
|
gpu_status_columns = [f"GPU{idx} Status" for idx in range(len(gpu_bdfs))]
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.table_header = f"{'Device'.ljust(30)}" + " ".join(
|
|
gpu.ljust(18) for gpu in gpu_columns
|
|
)
|
|
else:
|
|
self.logger.table_header = f"{'Device'}" + "".join(gpu for gpu in gpu_columns)
|
|
|
|
# Output the table
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "NIC-GPU ACCESS TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
# Populate the legend output
|
|
legend_parts = [
|
|
"\n\nLegend:",
|
|
" PCIe = gpu->nic are in same switch and numa",
|
|
" X-NUMA=gpu->nic are in different or same switch and across NUMA",
|
|
" NUMA= gpu->nic are in different or same switch and same NUMA",
|
|
]
|
|
legend_output = "\n".join(legend_parts)
|
|
|
|
if self.logger.destination == "stdout":
|
|
print(legend_output)
|
|
else:
|
|
with self.logger.destination.open("a", encoding="utf-8") as output_file:
|
|
output_file.write(legend_output + "\n")
|
|
|
|
return
|
|
|
|
if args.nic_switch:
|
|
# Prepare the table's header and data
|
|
tabular_output = []
|
|
|
|
# Add header row for BDF, NUMA, and CPU Affinity
|
|
header_row = {"Device": "", "bdf": "", "NUMA": "", "SWITCH": "", "CPU Affinity": ""}
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output.append(header_row)
|
|
|
|
if is_single_nic_request:
|
|
gpucount = 0
|
|
niccount = 1
|
|
switchcount = 0
|
|
if is_single_switch_request:
|
|
gpucount = 0
|
|
niccount = 0
|
|
switchcount = 1
|
|
if is_single_gpu_request:
|
|
gpucount = 1
|
|
niccount = 0
|
|
switchcount = 0
|
|
|
|
# First, add GPU information
|
|
if gpucount > 0:
|
|
for gpu_idx, gpu_dest in enumerate(args.gpu):
|
|
gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest)
|
|
gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest)
|
|
CPU_Affinity = amdsmi_interface.amdsmi_get_gpu_topo_cpu_affinity(gpu_dest)
|
|
numa_node = amdsmi_interface.amdsmi_get_gpu_topo_numa_affinity(gpu_dest)
|
|
switch_bdf = amdsmi_interface.amdsmi_get_root_switch(gpu_bdf)
|
|
|
|
# Add GPU row to the table
|
|
if self.logger.is_human_readable_format():
|
|
device_row = {
|
|
"Device": f"GPU{gpu_id}".ljust(17),
|
|
"bdf": f"{gpu_bdf}".rjust(2),
|
|
"NUMA": f"{numa_node}".rjust(8).ljust(20),
|
|
"SWITCH": f"{switch_bdf}".rjust(8).ljust(20),
|
|
"CPU Affinity": f"{CPU_Affinity}".ljust(20),
|
|
}
|
|
else:
|
|
device_row = {
|
|
"Device": f"GPU{gpu_id}",
|
|
"bdf": gpu_bdf,
|
|
"NUMA": numa_node,
|
|
"SWITCH": switch_bdf,
|
|
"CPU Affinity": CPU_Affinity,
|
|
}
|
|
tabular_output.append(device_row)
|
|
|
|
## Then, add NIC information
|
|
if niccount > 0:
|
|
for nic_idx, nic_dest in enumerate(args.nic):
|
|
nic_id = self.helpers.get_nic_id_from_device_handle(nic_dest)
|
|
nic_bdf = ""
|
|
nic_info = amdsmi_interface.amdsmi_get_nic_info(nic_dest)
|
|
if nic_info:
|
|
nic_bdf = nic_info["bdf"]
|
|
CPU_Affinity = amdsmi_interface.amdsmi_get_nic_topo_cpu_affinity(nic_dest)
|
|
numa_node = amdsmi_interface.amdsmi_get_nic_topo_numa_affinity(nic_dest)
|
|
switch_bdf = amdsmi_interface.amdsmi_get_root_switch(nic_bdf)
|
|
|
|
# Add NIC row to the table
|
|
if self.logger.is_human_readable_format():
|
|
device_row = {
|
|
"Device": f"BRCM_NIC{nic_id}".ljust(17),
|
|
"bdf": f"{nic_bdf}".rjust(2),
|
|
"NUMA": f"{numa_node}".rjust(8).ljust(20),
|
|
"SWITCH": f"{switch_bdf}".rjust(8).ljust(20),
|
|
"CPU Affinity": f"{CPU_Affinity}".ljust(20),
|
|
}
|
|
else:
|
|
device_row = {
|
|
"Device": f"BRCM_NIC{nic_id}",
|
|
"bdf": nic_bdf,
|
|
"NUMA": numa_node,
|
|
"SWITCH": switch_bdf,
|
|
"CPU Affinity": CPU_Affinity,
|
|
}
|
|
tabular_output.append(device_row)
|
|
|
|
## Then, add SWITCH information
|
|
if switchcount > 0:
|
|
for switch_idx, switch_dest in enumerate(args.switch):
|
|
switch_id = self.helpers.get_switch_id_from_device_handle(switch_dest)
|
|
switch_bdf = amdsmi_interface.amdsmi_get_switch_device_bdf(switch_dest)
|
|
CPU_Affinity = amdsmi_interface.amdsmi_get_switch_topo_cpu_affinity(switch_dest)
|
|
numa_node = amdsmi_interface.amdsmi_get_switch_topo_numa_affinity(switch_dest)
|
|
pSwitch_bdf = "N/A"
|
|
|
|
# Add NIC row to the table
|
|
if self.logger.is_human_readable_format():
|
|
device_row = {
|
|
"Device": f"BRCM_SWITCH{switch_id}".ljust(17),
|
|
"bdf": f"{switch_bdf}".rjust(2),
|
|
"NUMA": f"{numa_node}".rjust(8).ljust(20),
|
|
"SWITCH": f"{pSwitch_bdf}".rjust(8).ljust(20),
|
|
"CPU Affinity": f"{CPU_Affinity}".ljust(20),
|
|
}
|
|
else:
|
|
device_row = {
|
|
"Device": f"BRCM_SWITCH{switch_id}",
|
|
"bdf": switch_bdf,
|
|
"NUMA": numa_node,
|
|
"SWITCH": pSwitch_bdf,
|
|
"CPU Affinity": CPU_Affinity,
|
|
}
|
|
tabular_output.append(device_row)
|
|
|
|
# Display the table using the logger
|
|
self.logger.table_title = "AFFINITY TABLE"
|
|
self.logger.table_header = (
|
|
"Device".ljust(17)
|
|
+ "bdf".ljust(17)
|
|
+ "NUMA".ljust(15)
|
|
+ "SWITCH".ljust(20)
|
|
+ "CPU Affinity".ljust(17)
|
|
)
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
return
|
|
|
|
def topology(
|
|
self,
|
|
args,
|
|
multiple_devices=False,
|
|
gpu=None,
|
|
access=None,
|
|
weight=None,
|
|
hops=None,
|
|
link_type=None,
|
|
numa_bw=None,
|
|
coherent=None,
|
|
atomics=None,
|
|
dma=None,
|
|
bi_dir=None,
|
|
nic=None,
|
|
nic_topo=None,
|
|
nic_switch=None,
|
|
multiple_device_enabled=None,
|
|
switch=None,
|
|
):
|
|
"""Get topology information for target gpus
|
|
params:
|
|
args - argparser args to pass to subcommand
|
|
multiple_devices (bool) - True if checking for multiple devices
|
|
gpu (device_handle) - device_handle for target device
|
|
access (bool) - Value override for args.access
|
|
weight (bool) - Value override for args.weight
|
|
hops (bool) - Value override for args.hops
|
|
type (bool) - Value override for args.type
|
|
numa_bw (bool) - Value override for args.numa_bw
|
|
coherent (bool) - Value override for args.coherent
|
|
atomics (bool) - Value override for args.atomics
|
|
dma (bool) - Value override for args.dma
|
|
bi_dir (bool) - Value override for args.bi_dir
|
|
nic (device_handle) - device_handle for target device
|
|
nic_topo (bool) - True if checking for connectivity between nic and gpu devices
|
|
nic_switch (bool) - True if checking for gpu, nic and switch device's affinity and parent switch
|
|
switch (device_handle) - device_handle for target device
|
|
return:
|
|
Nothing
|
|
"""
|
|
# Set args.* to passed in arguments
|
|
if gpu:
|
|
args.gpu = gpu
|
|
if access:
|
|
args.access = access
|
|
if weight:
|
|
args.weight = weight
|
|
if hops:
|
|
args.hops = hops
|
|
if link_type:
|
|
args.link_type = link_type
|
|
if numa_bw:
|
|
args.numa_bw = numa_bw
|
|
if coherent:
|
|
args.coherent = coherent
|
|
if atomics:
|
|
args.atomics = atomics
|
|
if dma:
|
|
args.dma = dma
|
|
if bi_dir:
|
|
args.bi_dir = bi_dir
|
|
if nic:
|
|
args.nic = nic
|
|
if switch:
|
|
args.switch = switch
|
|
# nic_topo / nic_switch are only registered on the parser when their
|
|
# respective subsystem is initialized, so guard every namespace access
|
|
# with getattr() to stay safe on single-subsystem hosts.
|
|
nic_topo_req = getattr(args, "nic_topo", False)
|
|
nic_switch_req = getattr(args, "nic_switch", False)
|
|
if (self.helpers.is_brcm_nic_initialized() and (nic_topo or nic_topo_req)) or (
|
|
self.helpers.is_brcm_switch_initialized() and (nic_switch or nic_switch_req)
|
|
):
|
|
self.topology_nic(
|
|
args,
|
|
multiple_devices,
|
|
args.gpu,
|
|
getattr(args, "nic", None),
|
|
nic_topo_req,
|
|
nic_switch_req,
|
|
multiple_device_enabled,
|
|
getattr(args, "switch", None),
|
|
)
|
|
return
|
|
|
|
# Handle No GPU passed
|
|
if args.gpu == None:
|
|
args.gpu = self.device_handles
|
|
|
|
if not isinstance(args.gpu, list):
|
|
args.gpu = [args.gpu]
|
|
|
|
# Handle all args being false
|
|
if not any(
|
|
[
|
|
args.access,
|
|
args.weight,
|
|
args.hops,
|
|
args.link_type,
|
|
args.numa_bw,
|
|
args.coherent,
|
|
args.atomics,
|
|
args.dma,
|
|
args.bi_dir,
|
|
]
|
|
):
|
|
args.access = args.weight = args.hops = args.link_type = args.numa_bw = (
|
|
args.coherent
|
|
) = args.atomics = args.dma = args.bi_dir = True
|
|
|
|
# Clear the table header
|
|
self.logger.table_header = "".rjust(12)
|
|
|
|
if not self.group_check_printed:
|
|
self.helpers.check_required_groups()
|
|
self.group_check_printed = True
|
|
|
|
p2p_status_cache = {}
|
|
|
|
def get_cached_p2p_status(src_gpu, dest_gpu):
|
|
# Get P2P status with caching to avoid duplicate calls
|
|
src_gpu_id = self.helpers.get_gpu_id_from_device_handle(src_gpu)
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
key = (src_gpu_id, dest_gpu_id)
|
|
|
|
if key not in p2p_status_cache:
|
|
try:
|
|
if src_gpu == dest_gpu:
|
|
p2p_status_cache[key] = {
|
|
"cap": {
|
|
"is_iolink_coherent": -1,
|
|
"is_iolink_atomics_32bit": -1,
|
|
"is_iolink_atomics_64bit": -1,
|
|
"is_iolink_dma": -1,
|
|
"is_iolink_bi_directional": -1,
|
|
}
|
|
}
|
|
else:
|
|
p2p_status_cache[key] = amdsmi_interface.amdsmi_topo_get_p2p_status(
|
|
src_gpu, dest_gpu
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"Failed to get link status for %s to %s | %s",
|
|
src_gpu_id,
|
|
dest_gpu_id,
|
|
e.get_error_info(),
|
|
)
|
|
p2p_status_cache[key] = {
|
|
"cap": {
|
|
"is_iolink_coherent": -1,
|
|
"is_iolink_atomics_32bit": -1,
|
|
"is_iolink_atomics_64bit": -1,
|
|
"is_iolink_dma": -1,
|
|
"is_iolink_bi_directional": -1,
|
|
}
|
|
}
|
|
|
|
return p2p_status_cache[key]
|
|
|
|
# Populate the possible gpus
|
|
topo_values = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_id = self.helpers.get_gpu_id_from_device_handle(src_gpu)
|
|
topo_values.append({"gpu": src_gpu_id})
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
topo_values[src_gpu_index]["bdf"] = src_gpu_bdf
|
|
self.logger.table_header += src_gpu_bdf.rjust(13)
|
|
|
|
if not self.logger.is_json_format():
|
|
continue # below is for JSON format only
|
|
|
|
##########################
|
|
# JSON formatting start #
|
|
##########################
|
|
links = []
|
|
# create json obj for data alignment
|
|
# dest_gpu_links = {
|
|
# "gpu": GPU #
|
|
# "bdf": BDF identification
|
|
# "weight": 0 - self (current node); weight >= 0 correlated with hops (GPU-CPU, GPU-GPU, GPU-CPU-CPU-GPU, etc..)
|
|
# "link_status": "ENABLED" - devices linked; "DISABLED" - devices not linked; Correlated to access
|
|
# "link_type": "SELF" - current node, "PCIE", "XGMI", "N/A" - no link,"UNKNOWN" - unidentified link type
|
|
# "num_hops": num_hops - # of hops between devices
|
|
# "bandwidth": numa_bw - The NUMA "minimum bandwidth-maximum bandwidth" between src and dest nodes
|
|
# "N/A" - self node or not connected devices
|
|
# "coherent": coherent - Coherent / Non-Coherent io links
|
|
# "atomics": atomics - 32 and 64-bit atomic io link capability between nodes
|
|
# "dma": dma - P2P direct memory access (DMA) link capability between nodes
|
|
# "bi_dir": bi_dir - P2P bi-directional link capability between nodes
|
|
# }
|
|
|
|
for dest_gpu_index, dest_gpu in enumerate(args.gpu):
|
|
link_type = "SELF"
|
|
if src_gpu != dest_gpu:
|
|
link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[
|
|
"type"
|
|
]
|
|
if isinstance(link_type, int):
|
|
if link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_INTERNAL:
|
|
link_type = "UNKNOWN"
|
|
elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_PCIE:
|
|
link_type = "PCIE"
|
|
elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI:
|
|
link_type = "XGMI"
|
|
else:
|
|
link_type = "N/A"
|
|
|
|
numa_bw = "N/A"
|
|
if src_gpu != dest_gpu:
|
|
try:
|
|
bw_dict = amdsmi_interface.amdsmi_get_minmax_bandwidth_between_processors(
|
|
src_gpu, dest_gpu
|
|
)
|
|
numa_bw = f"{bw_dict['min_bandwidth']}-{bw_dict['max_bandwidth']}"
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"Failed to get min max bandwidth for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
weight = 0
|
|
num_hops = 0
|
|
if src_gpu != dest_gpu:
|
|
weight = amdsmi_interface.amdsmi_topo_get_link_weight(src_gpu, dest_gpu)
|
|
num_hops = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)["hops"]
|
|
link_status = amdsmi_interface.amdsmi_is_P2P_accessible(src_gpu, dest_gpu)
|
|
if link_status:
|
|
link_status = "ENABLED"
|
|
else:
|
|
link_status = "DISABLED"
|
|
|
|
link_coherent = "SELF"
|
|
link_atomics = "SELF"
|
|
link_dma = "SELF"
|
|
link_bi_dir = "SELF"
|
|
|
|
if src_gpu != dest_gpu:
|
|
try:
|
|
cap = get_cached_p2p_status(src_gpu, dest_gpu)["cap"]
|
|
link_coherent = (
|
|
"C"
|
|
if cap["is_iolink_coherent"] == 1
|
|
else "NC"
|
|
if cap["is_iolink_coherent"] == 0
|
|
else "N/A"
|
|
)
|
|
link_atomics = (
|
|
"64,32"
|
|
if cap["is_iolink_atomics_32bit"] == 1
|
|
and cap["is_iolink_atomics_64bit"] == 1
|
|
else "32"
|
|
if cap["is_iolink_atomics_32bit"] == 1
|
|
else "64"
|
|
if cap["is_iolink_atomics_64bit"] == 1
|
|
else "N/A"
|
|
)
|
|
link_dma = (
|
|
"T"
|
|
if cap["is_iolink_dma"] == 1
|
|
else "F"
|
|
if cap["is_iolink_dma"] == 0
|
|
else "N/A"
|
|
)
|
|
link_bi_dir = (
|
|
"T"
|
|
if cap["is_iolink_bi_directional"] == 1
|
|
else "F"
|
|
if cap["is_iolink_bi_directional"] == 0
|
|
else "N/A"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
logging.debug(
|
|
"Failed to get link status for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
# link_status = amdsmi_is_P2P_accessible(src,dest)
|
|
dest_gpu_links = {
|
|
"gpu": self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
"bdf": amdsmi_interface.amdsmi_get_gpu_device_bdf(dest_gpu),
|
|
"weight": weight,
|
|
"link_status": link_status,
|
|
"link_type": link_type,
|
|
"num_hops": num_hops,
|
|
"bandwidth": numa_bw,
|
|
"coherent": link_coherent,
|
|
"atomics": link_atomics,
|
|
"dma": link_dma,
|
|
"bi_dir": link_bi_dir,
|
|
}
|
|
if not args.access:
|
|
del dest_gpu_links["link_status"]
|
|
if not args.weight:
|
|
del dest_gpu_links["weight"]
|
|
if not args.link_type:
|
|
del dest_gpu_links["link_type"]
|
|
if not args.hops:
|
|
del dest_gpu_links["num_hops"]
|
|
if not args.numa_bw:
|
|
del dest_gpu_links["bandwidth"]
|
|
if not args.coherent:
|
|
del dest_gpu_links["coherent"]
|
|
if not args.atomics:
|
|
del dest_gpu_links["atomics"]
|
|
if not args.dma:
|
|
del dest_gpu_links["dma"]
|
|
if not args.bi_dir:
|
|
del dest_gpu_links["bi_dir"]
|
|
links.append(dest_gpu_links)
|
|
dest_end = dest_gpu_index + 1 == len(args.gpu)
|
|
isEndOfSrc = src_gpu_index + 1 == len(args.gpu)
|
|
if dest_end:
|
|
topo_values[src_gpu_index]["links"] = links
|
|
continue
|
|
if isEndOfSrc:
|
|
self.logger.multiple_device_output = topo_values
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
return
|
|
##########################
|
|
# JSON formatting end #
|
|
##########################
|
|
|
|
if args.access:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_links = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
try:
|
|
dest_gpu_link_status = amdsmi_interface.amdsmi_is_P2P_accessible(
|
|
src_gpu, dest_gpu
|
|
)
|
|
if dest_gpu_link_status:
|
|
src_gpu_links[dest_gpu_key] = "ENABLED"
|
|
else:
|
|
src_gpu_links[dest_gpu_key] = "DISABLED"
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_links[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link status for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["link_accessibility"] = src_gpu_links
|
|
|
|
tabular_output_dict.update(src_gpu_links)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "ACCESS TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.weight:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_weight = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_weight[dest_gpu_key] = 0
|
|
continue
|
|
|
|
try:
|
|
dest_gpu_link_weight = amdsmi_interface.amdsmi_topo_get_link_weight(
|
|
src_gpu, dest_gpu
|
|
)
|
|
src_gpu_weight[dest_gpu_key] = dest_gpu_link_weight
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_weight[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link weight for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["weight"] = src_gpu_weight
|
|
|
|
tabular_output_dict.update(src_gpu_weight)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "WEIGHT TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.hops:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_hops = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_hops[dest_gpu_key] = 0
|
|
continue
|
|
|
|
try:
|
|
dest_gpu_hops = amdsmi_interface.amdsmi_topo_get_link_type(
|
|
src_gpu, dest_gpu
|
|
)["hops"]
|
|
src_gpu_hops[dest_gpu_key] = dest_gpu_hops
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_hops[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link hops for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["hops"] = src_gpu_hops
|
|
|
|
tabular_output_dict.update(src_gpu_hops)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "HOPS TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.link_type:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_link_type = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_link_type[dest_gpu_key] = "SELF"
|
|
continue
|
|
try:
|
|
link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[
|
|
"type"
|
|
]
|
|
if isinstance(link_type, int):
|
|
if (
|
|
link_type
|
|
== amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_INTERNAL
|
|
):
|
|
src_gpu_link_type[dest_gpu_key] = "UNKNOWN"
|
|
elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_PCIE:
|
|
src_gpu_link_type[dest_gpu_key] = "PCIE"
|
|
elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI:
|
|
src_gpu_link_type[dest_gpu_key] = "XGMI"
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_link_type[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link type for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["link_type"] = src_gpu_link_type
|
|
|
|
tabular_output_dict.update(src_gpu_link_type)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "LINK TYPE TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.numa_bw:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_link_type = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_link_type[dest_gpu_key] = "N/A"
|
|
continue
|
|
|
|
try:
|
|
link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[
|
|
"type"
|
|
]
|
|
if isinstance(link_type, int):
|
|
if link_type != amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI:
|
|
# non_xgmi = True
|
|
src_gpu_link_type[dest_gpu_key] = "N/A"
|
|
continue
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_link_type[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link type for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
try:
|
|
bw_dict = amdsmi_interface.amdsmi_get_minmax_bandwidth_between_processors(
|
|
src_gpu, dest_gpu
|
|
)
|
|
src_gpu_link_type[dest_gpu_key] = (
|
|
f"{bw_dict['min_bandwidth']}-{bw_dict['max_bandwidth']}"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_link_type[dest_gpu_key] = e.get_error_info()
|
|
logging.debug(
|
|
"Failed to get min max bandwidth for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["numa_bandwidth"] = src_gpu_link_type
|
|
|
|
tabular_output_dict.update(src_gpu_link_type)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "NUMA BW TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.coherent:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_coherent = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_coherent[dest_gpu_key] = "SELF"
|
|
continue
|
|
try:
|
|
iolink_coherent = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][
|
|
"is_iolink_coherent"
|
|
]
|
|
src_gpu_coherent[dest_gpu_key] = (
|
|
"C" if iolink_coherent == 1 else "NC" if iolink_coherent == 0 else "N/A"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_coherent[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link coherent for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["coherent"] = src_gpu_coherent
|
|
|
|
tabular_output_dict.update(src_gpu_coherent)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "CACHE COHERENCY TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.atomics:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_atomics = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_atomics[dest_gpu_key] = "SELF"
|
|
continue
|
|
try:
|
|
cap = get_cached_p2p_status(src_gpu, dest_gpu)["cap"]
|
|
src_gpu_atomics[dest_gpu_key] = (
|
|
"64,32"
|
|
if cap["is_iolink_atomics_32bit"] == 1
|
|
and cap["is_iolink_atomics_64bit"] == 1
|
|
else "32"
|
|
if cap["is_iolink_atomics_32bit"] == 1
|
|
else "64"
|
|
if cap["is_iolink_atomics_64bit"] == 1
|
|
else "N/A"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_atomics[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link atomics for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["atomics"] = src_gpu_atomics
|
|
|
|
tabular_output_dict.update(src_gpu_atomics)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "ATOMICS TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.dma:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_dma = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_dma[dest_gpu_key] = "SELF"
|
|
continue
|
|
try:
|
|
iolink_dma = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][
|
|
"is_iolink_dma"
|
|
]
|
|
src_gpu_dma[dest_gpu_key] = (
|
|
"T" if iolink_dma == 1 else "F" if iolink_dma == 0 else "N/A"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_dma[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link dma for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["dma"] = src_gpu_dma
|
|
|
|
tabular_output_dict.update(src_gpu_dma)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "DMA TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if args.bi_dir:
|
|
tabular_output = []
|
|
for src_gpu_index, src_gpu in enumerate(args.gpu):
|
|
src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu)
|
|
if self.logger.is_human_readable_format():
|
|
tabular_output_dict = {"gpu": f"{src_gpu_bdf} "}
|
|
else:
|
|
tabular_output_dict = {"gpu": src_gpu_bdf}
|
|
src_gpu_bi_dir = {}
|
|
for dest_gpu in args.gpu:
|
|
dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu)
|
|
dest_gpu_key = f"gpu_{dest_gpu_id}"
|
|
|
|
if src_gpu == dest_gpu:
|
|
src_gpu_bi_dir[dest_gpu_key] = "SELF"
|
|
continue
|
|
try:
|
|
iolink_bi_dir = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][
|
|
"is_iolink_bi_directional"
|
|
]
|
|
src_gpu_bi_dir[dest_gpu_key] = (
|
|
"T" if iolink_bi_dir == 1 else "F" if iolink_bi_dir == 0 else "N/A"
|
|
)
|
|
except amdsmi_exception.AmdSmiLibraryException as e:
|
|
src_gpu_bi_dir[dest_gpu_key] = "N/A"
|
|
logging.debug(
|
|
"Failed to get link bi-directional for %s to %s | %s",
|
|
self.helpers.get_gpu_id_from_device_handle(src_gpu),
|
|
self.helpers.get_gpu_id_from_device_handle(dest_gpu),
|
|
e.get_error_info(),
|
|
)
|
|
|
|
topo_values[src_gpu_index]["bi_dir"] = src_gpu_bi_dir
|
|
|
|
tabular_output_dict.update(src_gpu_bi_dir)
|
|
tabular_output.append(tabular_output_dict)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
self.logger.multiple_device_output = tabular_output
|
|
self.logger.table_title = "BI-DIRECTIONAL TABLE"
|
|
self.logger.print_output(multiple_device_enabled=True, tabular=True)
|
|
|
|
if self.logger.is_human_readable_format():
|
|
# Populate the legend output
|
|
legend_parts = [
|
|
"\n\nLegend:",
|
|
" SELF = Current GPU",
|
|
" ENABLED / DISABLED = Link is enabled or disabled",
|
|
" N/A = Not supported",
|
|
" T/F = True / False",
|
|
" C/NC = Coherent / Non-Coherent io links",
|
|
" 64,32 = 64 bit and 32 bit atomic support",
|
|
" <BW from>-<BW to>",
|
|
]
|
|
legend_output = "\n".join(legend_parts)
|
|
|
|
if self.logger.destination == "stdout":
|
|
print(legend_output)
|
|
else:
|
|
with self.logger.destination.open("a", encoding="utf-8") as output_file:
|
|
output_file.write(legend_output + "\n")
|
|
|
|
self.logger.multiple_device_output = topo_values
|
|
|
|
if self.logger.is_csv_format():
|
|
new_output = []
|
|
for elem in self.logger.multiple_device_output:
|
|
new_output.append(self.logger.flatten_dict(elem, topology_override=True))
|
|
self.logger.multiple_device_output = new_output
|
|
|
|
if not self.logger.is_human_readable_format():
|
|
self.logger.print_output(multiple_device_enabled=True)
|