#!/usr/bin/env python3 # # Copyright (C) Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in # the Software without restriction, including without limitation the rights to # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of # the Software, and to permit persons to whom the Software is furnished to do so, # subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import logging from amdsmi import amdsmi_exception, amdsmi_interface class TopologyCommands: def topology_nic( self, args, multiple_devices=False, gpu=None, nic=None, nic_topo=None, nic_switch=None, multiple_device_enabled=None, switch=None, ): """Get topology information for target gpus params: args - argparser args to pass to subcommand multiple_devices (bool) - True if checking for multiple devices gpu (device_handle) - device_handle for target device nic (device_handle) - device_handle for target device nic_topo (bool) - True if checking for connectivity between nic and gpu devices nic_switch (bool) - True if checking for gpu, nic and switch device's affinity and parent switch switch (device_handle) - device_handle for target device return: Nothing """ # Set args.* to passed in arguments if gpu: args.gpu = gpu if nic: args.nic = nic if nic_topo: args.nic_topo = nic_topo if nic_switch: args.nic_switch = nic_switch if switch: args.switch = switch if not self.group_check_printed: self.helpers.check_required_groups() self.group_check_printed = True is_single_nic_request = False # -N option is_single_switch_request = False # --switch option is_single_gpu_request = False # -g option gpucount = 0 niccount = 0 switchcount = 0 # The topology parser conditionally registers --nic / --switch based on # is_brcm_nic_initialized() / is_brcm_switch_initialized(). On hosts # where one is gated off, the corresponding argparse attribute is # absent on the namespace; getattr() guards the unconditional access # below to prevent AttributeError on single-subsystem hosts. if getattr(args, "nic", None) is None: args.nic = self.device_handles_brcm_nics if not isinstance(args.nic, list): args.nic = [args.nic] if len(args.nic) == 1: is_single_nic_request = True niccount = len(args.nic) if getattr(args, "switch", None) is None: args.switch = self.device_handles_switchs if not isinstance(args.switch, list): args.switch = [args.switch] if len(args.switch) == 1: is_single_switch_request = True switchcount = len(args.switch) if args.gpu is None: args.gpu = self.device_handles if not isinstance(args.gpu, list): args.gpu = [args.gpu] if len(args.gpu) == 1: is_single_gpu_request = True gpucount = len(args.gpu) # Clear the table header self.logger.table_header = "".rjust(12) if args.nic_topo: topo_dict = {} # Loop through each NIC to get its BDF and corresponding GPU statuses for idx, dest_nic in enumerate(args.nic): # Get NIC ID and BDF nic_bdf = "" nic_info = amdsmi_interface.amdsmi_get_nic_info(dest_nic) if nic_info: nic_bdf = nic_info["bdf"] nic_id = self.helpers.get_nic_id_from_device_handle(dest_nic) # List to store the GPU statuses for this NIC gpu_statuses_for_nic = [] # Loop through each GPU to determine its status for gpu_dest in args.gpu: gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest) gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest) try: status = amdsmi_interface.amdsmi_get_nic_gpu_topo_info(dest_nic, gpu_dest) except amdsmi_exception.AmdSmiLibraryException: status = "N/A" gpu_statuses_for_nic.append((gpu_bdf, status)) # Store BDF and status as tuple # Store NIC BDF and associated GPU statuses in the dictionary topo_dict[nic_bdf] = gpu_statuses_for_nic # Prepare tabular output for logger tabular_output = [] # Add header row for GPU BDFs if self.logger.is_human_readable_format(): header_row = {"brcm_nic": "", "bdf": "".rjust(19)} else: header_row = {} gpu_bdfs = [] # List to store GPU BDFs for the header for gpu_dest in args.gpu: gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest) gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest) if self.logger.is_human_readable_format(): header_row[f"GPU BDF_{gpu_bdf}"] = f"{gpu_bdf}".rjust(20) else: header_row[f"gpu_{gpu_id}"] = f"{gpu_bdf}" gpu_bdfs.append(gpu_bdf) # Store GPU BDF for later reference # Add the header row tabular_output.append(header_row) # Add NIC rows with their associated GPU statuses for idx, (nic_bdf, gpu_info) in enumerate(topo_dict.items()): if not is_single_nic_request: if self.logger.is_human_readable_format(): nic_row = { "brcm_nic": f"BRCM_NIC{idx}".ljust(12), "bdf": f"{nic_bdf}".ljust(18), } else: nic_row = {"brcm_nic": f"BRCM_NIC{idx}", "bdf": f"{nic_bdf}"} else: # nic_id get stored in the initial iteration if self.logger.is_human_readable_format(): nic_row = { "brcm_nic": f"BRCM_NIC{nic_id}".ljust(12), "bdf": f"{nic_bdf}".ljust(18), } else: nic_row = {"brcm_nic": f"BRCM_NIC{nic_id}", "bdf": f"{nic_bdf}"} # Add GPU BDFs and statuses in the row for gpu_idx, (gpu_bdf, status) in enumerate(gpu_info): if self.logger.is_human_readable_format(): nic_row[f"GPU{gpu_idx} Status"] = status.ljust(20) else: nic_row[f"gpu_{gpu_idx}_topo"] = status # Add the NIC row to the table tabular_output.append(nic_row) # Use the logger to display the table # Construct the table header with GPU column names (adjusting for multiple GPUs) gpu_columns = [f"GPU{idx} " for idx in range(len(gpu_bdfs))] gpu_status_columns = [f"GPU{idx} Status" for idx in range(len(gpu_bdfs))] if self.logger.is_human_readable_format(): self.logger.table_header = f"{'Device'.ljust(30)}" + " ".join( gpu.ljust(18) for gpu in gpu_columns ) else: self.logger.table_header = f"{'Device'}" + "".join(gpu for gpu in gpu_columns) # Output the table self.logger.multiple_device_output = tabular_output self.logger.table_title = "NIC-GPU ACCESS TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if self.logger.is_human_readable_format(): # Populate the legend output legend_parts = [ "\n\nLegend:", " PCIe = gpu->nic are in same switch and numa", " X-NUMA=gpu->nic are in different or same switch and across NUMA", " NUMA= gpu->nic are in different or same switch and same NUMA", ] legend_output = "\n".join(legend_parts) if self.logger.destination == "stdout": print(legend_output) else: with self.logger.destination.open("a", encoding="utf-8") as output_file: output_file.write(legend_output + "\n") return if args.nic_switch: # Prepare the table's header and data tabular_output = [] # Add header row for BDF, NUMA, and CPU Affinity header_row = {"Device": "", "bdf": "", "NUMA": "", "SWITCH": "", "CPU Affinity": ""} if self.logger.is_human_readable_format(): tabular_output.append(header_row) if is_single_nic_request: gpucount = 0 niccount = 1 switchcount = 0 if is_single_switch_request: gpucount = 0 niccount = 0 switchcount = 1 if is_single_gpu_request: gpucount = 1 niccount = 0 switchcount = 0 # First, add GPU information if gpucount > 0: for gpu_idx, gpu_dest in enumerate(args.gpu): gpu_id = self.helpers.get_gpu_id_from_device_handle(gpu_dest) gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(gpu_dest) CPU_Affinity = amdsmi_interface.amdsmi_get_gpu_topo_cpu_affinity(gpu_dest) numa_node = amdsmi_interface.amdsmi_get_gpu_topo_numa_affinity(gpu_dest) switch_bdf = amdsmi_interface.amdsmi_get_root_switch(gpu_bdf) # Add GPU row to the table if self.logger.is_human_readable_format(): device_row = { "Device": f"GPU{gpu_id}".ljust(17), "bdf": f"{gpu_bdf}".rjust(2), "NUMA": f"{numa_node}".rjust(8).ljust(20), "SWITCH": f"{switch_bdf}".rjust(8).ljust(20), "CPU Affinity": f"{CPU_Affinity}".ljust(20), } else: device_row = { "Device": f"GPU{gpu_id}", "bdf": gpu_bdf, "NUMA": numa_node, "SWITCH": switch_bdf, "CPU Affinity": CPU_Affinity, } tabular_output.append(device_row) ## Then, add NIC information if niccount > 0: for nic_idx, nic_dest in enumerate(args.nic): nic_id = self.helpers.get_nic_id_from_device_handle(nic_dest) nic_bdf = "" nic_info = amdsmi_interface.amdsmi_get_nic_info(nic_dest) if nic_info: nic_bdf = nic_info["bdf"] CPU_Affinity = amdsmi_interface.amdsmi_get_nic_topo_cpu_affinity(nic_dest) numa_node = amdsmi_interface.amdsmi_get_nic_topo_numa_affinity(nic_dest) switch_bdf = amdsmi_interface.amdsmi_get_root_switch(nic_bdf) # Add NIC row to the table if self.logger.is_human_readable_format(): device_row = { "Device": f"BRCM_NIC{nic_id}".ljust(17), "bdf": f"{nic_bdf}".rjust(2), "NUMA": f"{numa_node}".rjust(8).ljust(20), "SWITCH": f"{switch_bdf}".rjust(8).ljust(20), "CPU Affinity": f"{CPU_Affinity}".ljust(20), } else: device_row = { "Device": f"BRCM_NIC{nic_id}", "bdf": nic_bdf, "NUMA": numa_node, "SWITCH": switch_bdf, "CPU Affinity": CPU_Affinity, } tabular_output.append(device_row) ## Then, add SWITCH information if switchcount > 0: for switch_idx, switch_dest in enumerate(args.switch): switch_id = self.helpers.get_switch_id_from_device_handle(switch_dest) switch_bdf = amdsmi_interface.amdsmi_get_switch_device_bdf(switch_dest) CPU_Affinity = amdsmi_interface.amdsmi_get_switch_topo_cpu_affinity(switch_dest) numa_node = amdsmi_interface.amdsmi_get_switch_topo_numa_affinity(switch_dest) pSwitch_bdf = "N/A" # Add NIC row to the table if self.logger.is_human_readable_format(): device_row = { "Device": f"BRCM_SWITCH{switch_id}".ljust(17), "bdf": f"{switch_bdf}".rjust(2), "NUMA": f"{numa_node}".rjust(8).ljust(20), "SWITCH": f"{pSwitch_bdf}".rjust(8).ljust(20), "CPU Affinity": f"{CPU_Affinity}".ljust(20), } else: device_row = { "Device": f"BRCM_SWITCH{switch_id}", "bdf": switch_bdf, "NUMA": numa_node, "SWITCH": pSwitch_bdf, "CPU Affinity": CPU_Affinity, } tabular_output.append(device_row) # Display the table using the logger self.logger.table_title = "AFFINITY TABLE" self.logger.table_header = ( "Device".ljust(17) + "bdf".ljust(17) + "NUMA".ljust(15) + "SWITCH".ljust(20) + "CPU Affinity".ljust(17) ) self.logger.multiple_device_output = tabular_output self.logger.print_output(multiple_device_enabled=True, tabular=True) return def topology( self, args, multiple_devices=False, gpu=None, access=None, weight=None, hops=None, link_type=None, numa_bw=None, coherent=None, atomics=None, dma=None, bi_dir=None, nic=None, nic_topo=None, nic_switch=None, multiple_device_enabled=None, switch=None, ): """Get topology information for target gpus params: args - argparser args to pass to subcommand multiple_devices (bool) - True if checking for multiple devices gpu (device_handle) - device_handle for target device access (bool) - Value override for args.access weight (bool) - Value override for args.weight hops (bool) - Value override for args.hops type (bool) - Value override for args.type numa_bw (bool) - Value override for args.numa_bw coherent (bool) - Value override for args.coherent atomics (bool) - Value override for args.atomics dma (bool) - Value override for args.dma bi_dir (bool) - Value override for args.bi_dir nic (device_handle) - device_handle for target device nic_topo (bool) - True if checking for connectivity between nic and gpu devices nic_switch (bool) - True if checking for gpu, nic and switch device's affinity and parent switch switch (device_handle) - device_handle for target device return: Nothing """ # Set args.* to passed in arguments if gpu: args.gpu = gpu if access: args.access = access if weight: args.weight = weight if hops: args.hops = hops if link_type: args.link_type = link_type if numa_bw: args.numa_bw = numa_bw if coherent: args.coherent = coherent if atomics: args.atomics = atomics if dma: args.dma = dma if bi_dir: args.bi_dir = bi_dir if nic: args.nic = nic if switch: args.switch = switch # nic_topo / nic_switch are only registered on the parser when their # respective subsystem is initialized, so guard every namespace access # with getattr() to stay safe on single-subsystem hosts. nic_topo_req = getattr(args, "nic_topo", False) nic_switch_req = getattr(args, "nic_switch", False) if (self.helpers.is_brcm_nic_initialized() and (nic_topo or nic_topo_req)) or ( self.helpers.is_brcm_switch_initialized() and (nic_switch or nic_switch_req) ): self.topology_nic( args, multiple_devices, args.gpu, getattr(args, "nic", None), nic_topo_req, nic_switch_req, multiple_device_enabled, getattr(args, "switch", None), ) return # Handle No GPU passed if args.gpu == None: args.gpu = self.device_handles if not isinstance(args.gpu, list): args.gpu = [args.gpu] # Handle all args being false if not any( [ args.access, args.weight, args.hops, args.link_type, args.numa_bw, args.coherent, args.atomics, args.dma, args.bi_dir, ] ): args.access = args.weight = args.hops = args.link_type = args.numa_bw = ( args.coherent ) = args.atomics = args.dma = args.bi_dir = True # Clear the table header self.logger.table_header = "".rjust(12) if not self.group_check_printed: self.helpers.check_required_groups() self.group_check_printed = True p2p_status_cache = {} def get_cached_p2p_status(src_gpu, dest_gpu): # Get P2P status with caching to avoid duplicate calls src_gpu_id = self.helpers.get_gpu_id_from_device_handle(src_gpu) dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) key = (src_gpu_id, dest_gpu_id) if key not in p2p_status_cache: try: if src_gpu == dest_gpu: p2p_status_cache[key] = { "cap": { "is_iolink_coherent": -1, "is_iolink_atomics_32bit": -1, "is_iolink_atomics_64bit": -1, "is_iolink_dma": -1, "is_iolink_bi_directional": -1, } } else: p2p_status_cache[key] = amdsmi_interface.amdsmi_topo_get_p2p_status( src_gpu, dest_gpu ) except amdsmi_exception.AmdSmiLibraryException as e: logging.debug( "Failed to get link status for %s to %s | %s", src_gpu_id, dest_gpu_id, e.get_error_info(), ) p2p_status_cache[key] = { "cap": { "is_iolink_coherent": -1, "is_iolink_atomics_32bit": -1, "is_iolink_atomics_64bit": -1, "is_iolink_dma": -1, "is_iolink_bi_directional": -1, } } return p2p_status_cache[key] # Populate the possible gpus topo_values = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_id = self.helpers.get_gpu_id_from_device_handle(src_gpu) topo_values.append({"gpu": src_gpu_id}) src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) topo_values[src_gpu_index]["bdf"] = src_gpu_bdf self.logger.table_header += src_gpu_bdf.rjust(13) if not self.logger.is_json_format(): continue # below is for JSON format only ########################## # JSON formatting start # ########################## links = [] # create json obj for data alignment # dest_gpu_links = { # "gpu": GPU # # "bdf": BDF identification # "weight": 0 - self (current node); weight >= 0 correlated with hops (GPU-CPU, GPU-GPU, GPU-CPU-CPU-GPU, etc..) # "link_status": "ENABLED" - devices linked; "DISABLED" - devices not linked; Correlated to access # "link_type": "SELF" - current node, "PCIE", "XGMI", "N/A" - no link,"UNKNOWN" - unidentified link type # "num_hops": num_hops - # of hops between devices # "bandwidth": numa_bw - The NUMA "minimum bandwidth-maximum bandwidth" between src and dest nodes # "N/A" - self node or not connected devices # "coherent": coherent - Coherent / Non-Coherent io links # "atomics": atomics - 32 and 64-bit atomic io link capability between nodes # "dma": dma - P2P direct memory access (DMA) link capability between nodes # "bi_dir": bi_dir - P2P bi-directional link capability between nodes # } for dest_gpu_index, dest_gpu in enumerate(args.gpu): link_type = "SELF" if src_gpu != dest_gpu: link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[ "type" ] if isinstance(link_type, int): if link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_INTERNAL: link_type = "UNKNOWN" elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_PCIE: link_type = "PCIE" elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI: link_type = "XGMI" else: link_type = "N/A" numa_bw = "N/A" if src_gpu != dest_gpu: try: bw_dict = amdsmi_interface.amdsmi_get_minmax_bandwidth_between_processors( src_gpu, dest_gpu ) numa_bw = f"{bw_dict['min_bandwidth']}-{bw_dict['max_bandwidth']}" except amdsmi_exception.AmdSmiLibraryException as e: logging.debug( "Failed to get min max bandwidth for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) weight = 0 num_hops = 0 if src_gpu != dest_gpu: weight = amdsmi_interface.amdsmi_topo_get_link_weight(src_gpu, dest_gpu) num_hops = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)["hops"] link_status = amdsmi_interface.amdsmi_is_P2P_accessible(src_gpu, dest_gpu) if link_status: link_status = "ENABLED" else: link_status = "DISABLED" link_coherent = "SELF" link_atomics = "SELF" link_dma = "SELF" link_bi_dir = "SELF" if src_gpu != dest_gpu: try: cap = get_cached_p2p_status(src_gpu, dest_gpu)["cap"] link_coherent = ( "C" if cap["is_iolink_coherent"] == 1 else "NC" if cap["is_iolink_coherent"] == 0 else "N/A" ) link_atomics = ( "64,32" if cap["is_iolink_atomics_32bit"] == 1 and cap["is_iolink_atomics_64bit"] == 1 else "32" if cap["is_iolink_atomics_32bit"] == 1 else "64" if cap["is_iolink_atomics_64bit"] == 1 else "N/A" ) link_dma = ( "T" if cap["is_iolink_dma"] == 1 else "F" if cap["is_iolink_dma"] == 0 else "N/A" ) link_bi_dir = ( "T" if cap["is_iolink_bi_directional"] == 1 else "F" if cap["is_iolink_bi_directional"] == 0 else "N/A" ) except amdsmi_exception.AmdSmiLibraryException as e: logging.debug( "Failed to get link status for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) # link_status = amdsmi_is_P2P_accessible(src,dest) dest_gpu_links = { "gpu": self.helpers.get_gpu_id_from_device_handle(dest_gpu), "bdf": amdsmi_interface.amdsmi_get_gpu_device_bdf(dest_gpu), "weight": weight, "link_status": link_status, "link_type": link_type, "num_hops": num_hops, "bandwidth": numa_bw, "coherent": link_coherent, "atomics": link_atomics, "dma": link_dma, "bi_dir": link_bi_dir, } if not args.access: del dest_gpu_links["link_status"] if not args.weight: del dest_gpu_links["weight"] if not args.link_type: del dest_gpu_links["link_type"] if not args.hops: del dest_gpu_links["num_hops"] if not args.numa_bw: del dest_gpu_links["bandwidth"] if not args.coherent: del dest_gpu_links["coherent"] if not args.atomics: del dest_gpu_links["atomics"] if not args.dma: del dest_gpu_links["dma"] if not args.bi_dir: del dest_gpu_links["bi_dir"] links.append(dest_gpu_links) dest_end = dest_gpu_index + 1 == len(args.gpu) isEndOfSrc = src_gpu_index + 1 == len(args.gpu) if dest_end: topo_values[src_gpu_index]["links"] = links continue if isEndOfSrc: self.logger.multiple_device_output = topo_values self.logger.print_output(multiple_device_enabled=True, tabular=True) return ########################## # JSON formatting end # ########################## if args.access: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_links = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" try: dest_gpu_link_status = amdsmi_interface.amdsmi_is_P2P_accessible( src_gpu, dest_gpu ) if dest_gpu_link_status: src_gpu_links[dest_gpu_key] = "ENABLED" else: src_gpu_links[dest_gpu_key] = "DISABLED" except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_links[dest_gpu_key] = "N/A" logging.debug( "Failed to get link status for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["link_accessibility"] = src_gpu_links tabular_output_dict.update(src_gpu_links) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "ACCESS TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.weight: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_weight = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_weight[dest_gpu_key] = 0 continue try: dest_gpu_link_weight = amdsmi_interface.amdsmi_topo_get_link_weight( src_gpu, dest_gpu ) src_gpu_weight[dest_gpu_key] = dest_gpu_link_weight except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_weight[dest_gpu_key] = "N/A" logging.debug( "Failed to get link weight for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["weight"] = src_gpu_weight tabular_output_dict.update(src_gpu_weight) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "WEIGHT TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.hops: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_hops = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_hops[dest_gpu_key] = 0 continue try: dest_gpu_hops = amdsmi_interface.amdsmi_topo_get_link_type( src_gpu, dest_gpu )["hops"] src_gpu_hops[dest_gpu_key] = dest_gpu_hops except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_hops[dest_gpu_key] = "N/A" logging.debug( "Failed to get link hops for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["hops"] = src_gpu_hops tabular_output_dict.update(src_gpu_hops) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "HOPS TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.link_type: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_link_type = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_link_type[dest_gpu_key] = "SELF" continue try: link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[ "type" ] if isinstance(link_type, int): if ( link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_INTERNAL ): src_gpu_link_type[dest_gpu_key] = "UNKNOWN" elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_PCIE: src_gpu_link_type[dest_gpu_key] = "PCIE" elif link_type == amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI: src_gpu_link_type[dest_gpu_key] = "XGMI" except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = "N/A" logging.debug( "Failed to get link type for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["link_type"] = src_gpu_link_type tabular_output_dict.update(src_gpu_link_type) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "LINK TYPE TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.numa_bw: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_link_type = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_link_type[dest_gpu_key] = "N/A" continue try: link_type = amdsmi_interface.amdsmi_topo_get_link_type(src_gpu, dest_gpu)[ "type" ] if isinstance(link_type, int): if link_type != amdsmi_interface.amdsmi_wrapper.AMDSMI_LINK_TYPE_XGMI: # non_xgmi = True src_gpu_link_type[dest_gpu_key] = "N/A" continue except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = "N/A" logging.debug( "Failed to get link type for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) try: bw_dict = amdsmi_interface.amdsmi_get_minmax_bandwidth_between_processors( src_gpu, dest_gpu ) src_gpu_link_type[dest_gpu_key] = ( f"{bw_dict['min_bandwidth']}-{bw_dict['max_bandwidth']}" ) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = e.get_error_info() logging.debug( "Failed to get min max bandwidth for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["numa_bandwidth"] = src_gpu_link_type tabular_output_dict.update(src_gpu_link_type) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "NUMA BW TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.coherent: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_coherent = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_coherent[dest_gpu_key] = "SELF" continue try: iolink_coherent = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][ "is_iolink_coherent" ] src_gpu_coherent[dest_gpu_key] = ( "C" if iolink_coherent == 1 else "NC" if iolink_coherent == 0 else "N/A" ) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_coherent[dest_gpu_key] = "N/A" logging.debug( "Failed to get link coherent for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["coherent"] = src_gpu_coherent tabular_output_dict.update(src_gpu_coherent) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "CACHE COHERENCY TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.atomics: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_atomics = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_atomics[dest_gpu_key] = "SELF" continue try: cap = get_cached_p2p_status(src_gpu, dest_gpu)["cap"] src_gpu_atomics[dest_gpu_key] = ( "64,32" if cap["is_iolink_atomics_32bit"] == 1 and cap["is_iolink_atomics_64bit"] == 1 else "32" if cap["is_iolink_atomics_32bit"] == 1 else "64" if cap["is_iolink_atomics_64bit"] == 1 else "N/A" ) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_atomics[dest_gpu_key] = "N/A" logging.debug( "Failed to get link atomics for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["atomics"] = src_gpu_atomics tabular_output_dict.update(src_gpu_atomics) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "ATOMICS TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.dma: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_dma = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_dma[dest_gpu_key] = "SELF" continue try: iolink_dma = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][ "is_iolink_dma" ] src_gpu_dma[dest_gpu_key] = ( "T" if iolink_dma == 1 else "F" if iolink_dma == 0 else "N/A" ) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_dma[dest_gpu_key] = "N/A" logging.debug( "Failed to get link dma for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["dma"] = src_gpu_dma tabular_output_dict.update(src_gpu_dma) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "DMA TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if args.bi_dir: tabular_output = [] for src_gpu_index, src_gpu in enumerate(args.gpu): src_gpu_bdf = amdsmi_interface.amdsmi_get_gpu_device_bdf(src_gpu) if self.logger.is_human_readable_format(): tabular_output_dict = {"gpu": f"{src_gpu_bdf} "} else: tabular_output_dict = {"gpu": src_gpu_bdf} src_gpu_bi_dir = {} for dest_gpu in args.gpu: dest_gpu_id = self.helpers.get_gpu_id_from_device_handle(dest_gpu) dest_gpu_key = f"gpu_{dest_gpu_id}" if src_gpu == dest_gpu: src_gpu_bi_dir[dest_gpu_key] = "SELF" continue try: iolink_bi_dir = get_cached_p2p_status(src_gpu, dest_gpu)["cap"][ "is_iolink_bi_directional" ] src_gpu_bi_dir[dest_gpu_key] = ( "T" if iolink_bi_dir == 1 else "F" if iolink_bi_dir == 0 else "N/A" ) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_bi_dir[dest_gpu_key] = "N/A" logging.debug( "Failed to get link bi-directional for %s to %s | %s", self.helpers.get_gpu_id_from_device_handle(src_gpu), self.helpers.get_gpu_id_from_device_handle(dest_gpu), e.get_error_info(), ) topo_values[src_gpu_index]["bi_dir"] = src_gpu_bi_dir tabular_output_dict.update(src_gpu_bi_dir) tabular_output.append(tabular_output_dict) if self.logger.is_human_readable_format(): self.logger.multiple_device_output = tabular_output self.logger.table_title = "BI-DIRECTIONAL TABLE" self.logger.print_output(multiple_device_enabled=True, tabular=True) if self.logger.is_human_readable_format(): # Populate the legend output legend_parts = [ "\n\nLegend:", " SELF = Current GPU", " ENABLED / DISABLED = Link is enabled or disabled", " N/A = Not supported", " T/F = True / False", " C/NC = Coherent / Non-Coherent io links", " 64,32 = 64 bit and 32 bit atomic support", " -", ] legend_output = "\n".join(legend_parts) if self.logger.destination == "stdout": print(legend_output) else: with self.logger.destination.open("a", encoding="utf-8") as output_file: output_file.write(legend_output + "\n") self.logger.multiple_device_output = topo_values if self.logger.is_csv_format(): new_output = [] for elem in self.logger.multiple_device_output: new_output.append(self.logger.flatten_dict(elem, topology_override=True)) self.logger.multiple_device_output = new_output if not self.logger.is_human_readable_format(): self.logger.print_output(multiple_device_enabled=True)