#!/usr/bin/env python3 # MIT License # # Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import csv import sys import pytest import re import os import glob import json from pathlib import Path OCCUPANCY_FIELDS = ( "time", "cu", "simd", "wave_id", "start", "kernel_id", "me_id", "pipe_id", "is_ext", "workgroup_id", "cluster_id", ) OCCUPANCY_VERSION = "3.1.0" EVENT_FIELDS = ( "kind", "time", "type", "me_id", "pipe_id", "flags", "payload", "byte_offset", ) DISPATCH_FIELDS = ( "kind", "time", "me_id", "pipe_id", "kernel_id", "kernel_name", "entry_point", "user_sgprs", "vgprs", "sgprs", "lds_size", "thread_dim_x", "thread_dim_y", "thread_dim_z", "dispatch_pkt_addr", "byte_offset", "flags", ) def assert_json_int(value, field): assert isinstance(value, int) and not isinstance( value, bool ), f"{field} must be an integer" def find_occupancy_files(output_path): pattern = os.path.join(output_path, "ui_output_*", "occupancy.json") return sorted(Path(path) for path in glob.glob(pattern)) def validate_occupancy_rows(occupancy_data, occupancy_file): assert ( occupancy_data.get("version") == OCCUPANCY_VERSION ), f"occupancy version mismatch in {occupancy_file}" assert ( "occupancy_fields" in occupancy_data ), f"occupancy_fields missing from {occupancy_file}" fields = occupancy_data["occupancy_fields"] assert isinstance( fields, list ), f"occupancy_fields must be a list in {occupancy_file}" for field in OCCUPANCY_FIELDS: assert ( field in fields ), f"{field} missing from occupancy_fields in {occupancy_file}" found_rows = False for se, rows in occupancy_data.items(): if not str(se).isdigit(): continue assert isinstance(rows, list), f"occupancy SE {se} must be a list" for row in rows: found_rows = True assert isinstance(row, list), f"occupancy row for SE {se} must be a list" assert len(row) >= len( fields ), f"occupancy row for SE {se} has {len(row)} fields, expected {len(fields)}" assert found_rows, f"No occupancy rows found in {occupancy_file}" def validate_trace_event(event, occupancy_file): for field in EVENT_FIELDS: assert field in event, f"{field} missing from event record in {occupancy_file}" assert event["kind"] == "event" assert_json_int(event["time"], "event.time") assert_json_int(event["type"], "event.type") assert_json_int(event["me_id"], "event.me_id") assert_json_int(event["pipe_id"], "event.pipe_id") assert_json_int(event["flags"], "event.flags") assert_json_int(event["payload"], "event.payload") assert_json_int(event["byte_offset"], "event.byte_offset") assert "name" not in event assert "type_name" not in event def validate_dispatch_event(dispatch, dispatches, occupancy_file): for field in DISPATCH_FIELDS: assert ( field in dispatch ), f"{field} missing from dispatch record in {occupancy_file}" assert dispatch["kind"] == "dispatch" for field in ( "time", "me_id", "pipe_id", "kernel_id", "user_sgprs", "vgprs", "sgprs", "lds_size", "thread_dim_x", "thread_dim_y", "thread_dim_z", "dispatch_pkt_addr", "byte_offset", "flags", ): assert_json_int(dispatch[field], f"dispatch.{field}") assert dispatch["vgprs"] > 0 assert dispatch["sgprs"] > 0 assert dispatch["thread_dim_x"] > 0 assert dispatch["thread_dim_y"] > 0 assert dispatch["thread_dim_z"] > 0 assert isinstance(dispatch["kernel_name"], str) assert len(dispatch["kernel_name"]) > 0 entry_point = dispatch["entry_point"] assert isinstance(entry_point, dict) assert_json_int(entry_point["address"], "dispatch.entry_point.address") assert_json_int(entry_point["code_object_id"], "dispatch.entry_point.code_object_id") dispatch_id = str(dispatch["kernel_id"]) assert dispatch_id in dispatches assert dispatches[dispatch_id] == dispatch["kernel_name"] def read_att_stats_kernel_names(output_path): traced_kernel_names = set() stats_files = list(Path(output_path).glob("stats_*.csv")) assert len(stats_files) > 0, f"No stats_*.csv files found in {output_path}" for stats_file in stats_files: with open(stats_file, "r") as f: reader = csv.reader(f) header = next(reader, None) assert header is not None, f"Empty stats CSV: {stats_file}" for row in reader: if len(row) >= 3 and row[2].startswith("; "): traced_kernel_names.add(row[2][2:].strip()) if len(row) >= 8 and row[7].strip(): traced_kernel_names.add(row[7].strip()) assert len(traced_kernel_names) > 0, "No kernel names found in stats CSV files" return traced_kernel_names def test_json_data(json_data): data = json_data["rocprofiler-sdk-tool"] strings = data["strings"] assert "att_filenames" in strings.keys() att_files = data["strings"]["att_filenames"] assert len(att_files) > 0 def test_code_object_memory(code_object_file_path, json_data, output_path): data = json_data["rocprofiler-sdk-tool"] tool_memory_load = data["strings"]["code_object_snapshot_filenames"] gfx_pattern = "gfx[a-z0-9]+" match = re.search(gfx_pattern, tool_memory_load[1]) assert match != None gpu_name = match.group(0) read_bytes = lambda filename: open(os.path.join(output_path, filename), "rb").read() # Loads all saved code objects tool_memory = [read_bytes(saved) for saved in tool_memory_load[1:]] found = False for hsa_file in code_object_file_path["hsa_memory_load"]: m = re.search(gfx_pattern, hsa_file) assert m != None gpu = m.group(0) if gpu == gpu_name: found = True hsa_memory_bytes = open(hsa_file, "rb").read() # Checks if hsa_file is one of the saved code objects assert any([hsa_memory_bytes == fs for fs in tool_memory]) break assert found == True def test_perfcounter_target_cu(output_path, request): """ Test that when --att-perfcounter-target-cu is specified, all perfcounter events are tagged only for the target CU (or no events at all). """ # Get the target CU from pytest command line if provided target_cu = request.config.getoption("--target-cu", default=None) if target_cu is None: pytest.skip("--target-cu not specified, skipping perfcounter target CU test") target_cu = int(target_cu) # Find all perfcounter JSON files pattern = os.path.join(output_path, "ui_output_*", "se*_perfcounter.json") perfcounter_files = glob.glob(pattern) print(perfcounter_files) # It's acceptable to have no perfcounter files (target CU may not have any events) for pc_file in perfcounter_files: with open(pc_file, "r", encoding="utf-8") as f: json_data = json.load(f) data = json_data.get("data", []) # Check that all events are for the target CU for event in data: # CU is at index 5 event_cu = event[5] assert event_cu == target_cu, ( f"Found perfcounter event for CU {event_cu}, " f"but target CU is {target_cu} in file {pc_file}" ) def test_occupancy_event_tracing_fields(att_occupancy_event_trace_out_dir_path): occupancy_files = find_occupancy_files(att_occupancy_event_trace_out_dir_path) assert ( occupancy_files ), f"No occupancy.json files found under {att_occupancy_event_trace_out_dir_path}" found_event = False found_dispatch = False for occupancy_file in occupancy_files: with open(occupancy_file, "r", encoding="utf-8") as f: occupancy_data = json.load(f) validate_occupancy_rows(occupancy_data, occupancy_file) dispatches = occupancy_data.get("dispatches", {}) assert isinstance( dispatches, dict ), f"dispatches must be an object in {occupancy_file}" assert dispatches, f"dispatches is empty in {occupancy_file}" events = occupancy_data.get("events", {}) assert isinstance(events, dict), f"events must be an object in {occupancy_file}" assert events, f"events is empty in {occupancy_file}" for se, records in events.items(): assert isinstance(records, list), f"events[{se}] must be a list" assert records, f"events[{se}] is empty in {occupancy_file}" for record in records: assert isinstance(record, dict), f"event record must be an object" kind = record.get("kind") if kind == "event": validate_trace_event(record, occupancy_file) found_event = True elif kind == "dispatch": validate_dispatch_event(record, dispatches, occupancy_file) found_dispatch = True else: assert False, f"Unexpected event kind {kind} in {occupancy_file}" assert found_event, "No event records found in occupancy.json" assert found_dispatch, "No dispatch records found in occupancy.json" def test_realtime_clock(output_path): def verify_sorted(timestamps): # Sort by shader_clock (index 0) timestamps_sorted = sorted(timestamps, key=lambda ts: ts[0]) # Ensure realtime clock is non-decreasing assert all( curr[1] >= prev[1] for prev, curr in zip(timestamps_sorted, timestamps_sorted[1:]) ) def verify_gfxclock(timestamps, rt_frequency): delta_shader_clock = timestamps[-1][0] - timestamps[0][0] delta_realtime_ts = timestamps[-1][1] - timestamps[0][1] gfxclock = rt_frequency * delta_shader_clock / delta_realtime_ts # gfxclock must be positive assert gfxclock > 0 # gfxclock must be <10GHz assert gfxclock < 1e10 pattern = os.path.join(output_path, "ui_output_*", "realtime.json") for rt_file in glob.glob(pattern): with open(rt_file, "r", encoding="utf-8") as f: json_file = json.load(f) frequency = json_file["metadata"]["frequency"] # frequency = 0 means aqlprofile is not instrumented if frequency > 0: for key, value in json_file.items(): # Exclude metadata and single-clock timestamps if "metadata" not in key and len(value) >= 2: verify_sorted(value) verify_gfxclock(value, frequency) def test_other_simd_data(att_other_simd_out_dir_path): def find_ui_output_dirs(att_other_simd_out_dir_path): matches = [ p for p in Path(att_other_simd_out_dir_path).glob("ui_output_agent_*") if p.is_dir() ] return matches def find_other_simd_files(other_simd_files_path): root = Path(other_simd_files_path) if not root.is_dir(): return [] matches = [p for p in root.glob("other_simd_se*") if p.is_file()] return matches att_ui_dispatch_dirs = find_ui_output_dirs(att_other_simd_out_dir_path) # more than 1 dirs must be generated by rocprofv3. assert len(att_ui_dispatch_dirs) > 0, "ui_output_agent_* dirs not found." for ui_dispatch_dir in att_ui_dispatch_dirs: with open(ui_dispatch_dir / "filenames.json", "r") as inp: filenames_json = json.load(inp) listed_file_names = filenames_json["other_simd_filenames"] assert len(listed_file_names) > 0, "other_simd_filenames is empty." other_simd_files_found = find_other_simd_files(ui_dispatch_dir) total_listed_files = 0 for se, files in listed_file_names.items(): assert len(files) > 0, f"other_simd_filenames[{se}] is empty." total_listed_files += len(files) assert ( len(other_simd_files_found) == total_listed_files ), "other_simd files mismatch between filenames.json and files present in dir." for files in listed_file_names.values(): for file in files: assert ( len(file) == 3 ), "other_simd_filenames entry must be [filename, begin, end]." with open(ui_dispatch_dir / file[0], "r") as inp: other_simd_file_data = json.load(inp) assert ( file[1] == other_simd_file_data["begin_time"] ), "begin time mismatch filenames.json and other_simd_se*_*.json" assert ( file[2] == other_simd_file_data["end_time"] ), "end time mismatch filenames.json and other_simd_se*_*.json" assert ( other_simd_file_data["instructions_count"] > 0 ), "other simd instructions is empty." other_simd_instructions = other_simd_file_data["instructions"] assert ( len(other_simd_instructions) == other_simd_file_data["instructions_count"] ) last_known_time = other_simd_instructions[0][0] last_known_duration = other_simd_instructions[0][1] assert last_known_time == other_simd_file_data["begin_time"] # Start from the second element for other_simd_instruction in other_simd_instructions[1:]: assert ( other_simd_instruction[0] >= last_known_time ), "data from other_simd file is not in increasing time, corrupted data." last_known_time = other_simd_instruction[0] last_known_duration = other_simd_instruction[1] assert ( last_known_time + last_known_duration == other_simd_file_data["end_time"] ) def test_shaderdata(att_shaderdata_out_dir_path): expected_value = 3735928559 # m0 value from kernel_lds.cpp (0xDEADBEEF) def find_ui_output_dirs(att_out_dir_path): matches = [ p for p in Path(att_out_dir_path).glob("ui_output_agent_*") if p.is_dir() ] return matches def find_shaderdata_files(shaderdata_files_path): root = Path(shaderdata_files_path) if not root.is_dir(): return [] matches = [p for p in root.glob("shaderdata_*") if p.is_file()] return matches att_ui_dispatch_dirs = find_ui_output_dirs(att_shaderdata_out_dir_path) assert len(att_ui_dispatch_dirs) > 0, "ui_output_agent_* dirs not found." found_shaderdata = False for ui_dispatch_dir in att_ui_dispatch_dirs: with open(ui_dispatch_dir / "filenames.json", "r") as inp: filenames_json = json.load(inp) listed_file_names = filenames_json.get("shaderdata_filenames", {}) if not listed_file_names: continue found_shaderdata = True shaderdata_files_found = find_shaderdata_files(ui_dispatch_dir) listed_count = sum(len(files) for files in listed_file_names.values()) assert ( len(shaderdata_files_found) == listed_count ), "shaderdata files mismatch between filenames.json and files present in dir." for files in listed_file_names.values(): for file in files: with open(ui_dispatch_dir / file[0], "r") as inp: shaderdata_file_data = json.load(inp) assert ( file[1] == shaderdata_file_data["begin_time"] ), "begin time mismatch filenames.json and shaderdata_*.json" assert ( file[2] == shaderdata_file_data["end_time"] ), "end time mismatch filenames.json and shaderdata_*.json" assert ( shaderdata_file_data["records_count"] > 0 ), "shaderdata records are empty." shaderdata_records = shaderdata_file_data["records"] assert len(shaderdata_records) == shaderdata_file_data["records_count"] # Validate ordering and sentinel value in records. last_known_time = shaderdata_records[0][0] assert last_known_time == shaderdata_file_data["begin_time"] for record in shaderdata_records: assert ( record[1] == expected_value ), "shaderdata record value mismatch." for record in shaderdata_records[1:]: assert ( record[0] >= last_known_time ), "data from shaderdata file is not in increasing time." last_known_time = record[0] assert ( last_known_time == shaderdata_file_data["end_time"] ), "end time mismatch between records and shaderdata_*.json" # Require at least one ui_output_agent_* directory with shaderdata data. assert found_shaderdata, "No ui_output_agent_* directory contains shaderdata data." def test_shaderdata(att_shaderdata_out_dir_path): expected_value = 3735928559 # m0 value from kernel_lds.cpp (0xDEADBEEF) def find_ui_output_dirs(att_out_dir_path): matches = [ p for p in Path(att_out_dir_path).glob("ui_output_agent_*") if p.is_dir() ] return matches def find_shaderdata_files(shaderdata_files_path): root = Path(shaderdata_files_path) if not root.is_dir(): return [] matches = [p for p in root.glob("shaderdata_*") if p.is_file()] return matches att_ui_dispatch_dirs = find_ui_output_dirs(att_shaderdata_out_dir_path) assert len(att_ui_dispatch_dirs) > 0, "ui_output_agent_* dirs not found." found_shaderdata = False for ui_dispatch_dir in att_ui_dispatch_dirs: with open(ui_dispatch_dir / "filenames.json", "r") as inp: filenames_json = json.load(inp) listed_file_names = filenames_json.get("shaderdata_filenames", {}) if not listed_file_names: continue found_shaderdata = True shaderdata_files_found = find_shaderdata_files(ui_dispatch_dir) listed_count = sum(len(files) for files in listed_file_names.values()) assert ( len(shaderdata_files_found) == listed_count ), "shaderdata files mismatch between filenames.json and files present in dir." for files in listed_file_names.values(): for file in files: with open(ui_dispatch_dir / file[0], "r") as inp: shaderdata_file_data = json.load(inp) assert ( file[1] == shaderdata_file_data["begin_time"] ), "begin time mismatch filenames.json and shaderdata_*.json" assert ( file[2] == shaderdata_file_data["end_time"] ), "end time mismatch filenames.json and shaderdata_*.json" assert ( shaderdata_file_data["records_count"] > 0 ), "shaderdata records are empty." shaderdata_records = shaderdata_file_data["records"] assert len(shaderdata_records) == shaderdata_file_data["records_count"] # Validate ordering and sentinel value in records. last_known_time = shaderdata_records[0][0] assert last_known_time == shaderdata_file_data["begin_time"] for record in shaderdata_records: assert ( record[1] == expected_value ), "shaderdata record value mismatch." for record in shaderdata_records[1:]: assert ( record[0] >= last_known_time ), "data from shaderdata file is not in increasing time." last_known_time = record[0] assert ( last_known_time == shaderdata_file_data["end_time"] ), "end time mismatch between records and shaderdata_*.json" # Require at least one ui_output_agent_* directory with shaderdata data. assert found_shaderdata, "No ui_output_agent_* directory contains shaderdata data." def test_multi_gpu_separate_agents(att_multi_gpu_out_dir_path): """ When multiple GPUs are traced for the same dispatch, each agent must get its own ui_output_agent_* directory. Before the fix, all agents' ATT data was merged into a single directory, causing shaderdata timestamp resets (each GPU has its own independent SQTT counter) and silently dropping occupancy data from all but the first agent. """ out_dir = Path(att_multi_gpu_out_dir_path) # Use agent_info.csv to determine how many GPUs are on this machine. # The file is inside a hostname subdirectory: //_agent_info.csv agent_csvs = list(out_dir.glob("**/*_agent_info.csv")) assert agent_csvs, ( f"No *_agent_info.csv found under {out_dir}. " f"Ensure the execute test runs with --output-format csv." ) with open(agent_csvs[0], "r") as f: gpu_count = sum(1 for row in csv.DictReader(f) if row["Agent_Type"] == "GPU") if gpu_count < 2: pytest.skip(f"Only {gpu_count} GPU(s) on this machine, need >= 2") ui_dirs = sorted(p for p in out_dir.glob("ui_output_agent_*") if p.is_dir()) # With --att-gpu-index 0,1 we expect at least two output directories # (one per agent). assert len(ui_dirs) >= 2, ( f"Expected at least 2 ui_output_agent_* directories (one per GPU), " f"found {len(ui_dirs)}. ATT data from multiple agents may be merged." ) # Extract agent ids from directory names and verify they are distinct. agent_ids = set() pattern = re.compile(r"ui_output_agent_(\d+)_dispatch_(\d+)") for d in ui_dirs: m = pattern.search(d.name) assert m, f"Unexpected directory name format: {d.name}" agent_ids.add(m.group(1)) assert len(agent_ids) >= 2, ( f"Expected directories for at least 2 distinct agents, " f"found agent ids: {agent_ids}" ) # For each directory that has shaderdata, verify timestamps are # monotonically increasing across all chunks (no resets from other GPUs). for ui_dir in ui_dirs: filenames_path = ui_dir / "filenames.json" if not filenames_path.exists(): continue with open(filenames_path, "r") as f: filenames_json = json.load(f) shaderdata_filenames = filenames_json.get("shaderdata_filenames", {}) if not shaderdata_filenames: continue for se, files in shaderdata_filenames.items(): prev_end_time = -1 for file_entry in files: begin_time = file_entry[1] end_time = file_entry[2] # Each chunk's begin_time must be >= the previous chunk's # end_time. A reset (begin < prev_end) indicates data from a # different GPU was mixed in. assert begin_time >= prev_end_time, ( f"Shaderdata timestamp reset in {ui_dir.name} SE {se}: " f"chunk {file_entry[0]} begins at {begin_time} but " f"previous chunk ended at {prev_end_time}. " f"Data from multiple agents may be merged." ) prev_end_time = end_time def test_att_marker_trace(json_data, att_marker_trace_out_dir_path): """Verify marker-controlled ATT traced only kernels between Resume and Pause. Test binary launches 4 kernels: - before_trace_kernel: before roctxProfilerResume (should NOT be traced) - traced_kernel_first: after roctxProfilerResume (should be traced) - traced_kernel_second: after roctxProfilerResume (should be traced) - after_trace_kernel: after roctxProfilerPause (should NOT be traced) Validation parses the stats_*.csv files produced by the ATT decoder. Kernel names appear as instruction rows with "; " and the demangled name in the Source column. """ data = json_data["rocprofiler-sdk-tool"] strings = data["strings"] # Verify ATT data was produced assert "att_filenames" in strings.keys(), "No att_filenames in output" att_files = strings["att_filenames"] assert len(att_files) > 0, "Expected ATT data from marker-controlled thread trace" # Verify decoded ATT output directories exist att_ui_dirs = [ p for p in Path(att_marker_trace_out_dir_path).glob("ui_output_agent_*") if p.is_dir() ] assert len(att_ui_dirs) > 0, "No ui_output_agent_* directories found" traced_kernel_names = read_att_stats_kernel_names(att_marker_trace_out_dir_path) # Verify traced_kernel_first and traced_kernel_second were traced for expected in ("traced_kernel_first", "traced_kernel_second"): found = any(expected in name for name in traced_kernel_names) assert found, ( f"Expected '{expected}' to be in ATT trace but it was not. " f"Traced kernels: {traced_kernel_names}" ) # Verify before_trace_kernel and after_trace_kernel were NOT traced for not_expected in ("before_trace_kernel", "after_trace_kernel"): found = any(not_expected in name for name in traced_kernel_names) assert not found, ( f"Expected '{not_expected}' to NOT be in ATT trace but it was. " f"Traced kernels: {traced_kernel_names}" ) def test_att_no_intercept_target_kernel(att_no_intercept_out_dir_path): out_dir = Path(att_no_intercept_out_dir_path) att_ui_dirs = [p for p in out_dir.glob("ui_output_agent_*") if p.is_dir()] assert len(att_ui_dirs) > 0, "No ui_output_agent_* directories found" traced_kernel_names = read_att_stats_kernel_names(out_dir) expected = "looping_lds_kernel" assert any(expected in name for name in traced_kernel_names), ( f"Expected '{expected}' to be in ATT no-intercept trace. " f"Traced kernels: {traced_kernel_names}" ) if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) sys.exit(exit_code)