#!/usr/bin/env python3 # MIT License # # Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # Regression coverage for the hsa-runtime InterceptQueue retry/overflow hang # during counter collection. The multistream workload launches 100 streams x # 100 iterations = 10000 kernel dispatches. Counter collection rewrites every # dispatch (counter start/stop packets) via the hsa-runtime InterceptQueue, so # a regression either hangs (the execute test hits its TIMEOUT) or drops # dispatches / corrupts the ring copy. The assertions below require that all # 10000 dispatches were captured with valid SQ_WAVES data. import sys import pytest import pandas as pd import re # multistream: 100 streams x 100 iterations (see tests/bin/multistream/multistream_app.cpp) EXPECTED_DISPATCHES = 100 * 100 KERNEL_NAME = "add(int, float*, float*)" def is_helper(name): return bool(re.search(r"__amd_rocclr_.*", name)) def test_validate_counter_collection_queue_overflow(input_data: pd.DataFrame): df = input_data assert not df.empty df_agent_id = df["Agent_Id"].str.split(" ").str[-1] assert (df_agent_id.astype(int).values >= 0).all() assert (df["Queue_Id"].astype(int).values > 0).all() assert (df["Process_Id"].astype(int).values > 0).all() # only real (non-rocclr-helper) kernel dispatched by multistream is the add kernel real_kernels = sorted( x for x in df["Kernel_Name"].unique().tolist() if not is_helper(x) ) assert real_kernels == [KERNEL_NAME] # all counters requested are SQ_WAVES and every dispatch reported a positive value assert df["Counter_Name"].str.contains("SQ_WAVES").all() assert (df["Counter_Value"].astype(int).values > 0).all() # key regression check: every one of the 10000 dispatches was captured (a hang or # out-of-bounds ring copy in the overflow/retry path would strand or drop dispatches) add_rows = df[df["Kernel_Name"] == KERNEL_NAME] assert len(add_rows) == EXPECTED_DISPATCHES # dispatch ids are unique and contiguous from 1 di_list = df["Dispatch_Id"].astype(int).values.tolist() di_uniq = sorted(df["Dispatch_Id"].unique().tolist()) di_expect = [idx + 1 for idx in range(len(di_list))] assert di_expect == di_uniq def test_validate_counter_collection_queue_overflow_json(json_data): data = json_data["rocprofiler-sdk-tool"] counter_collection_data = data["callback_records"]["counter_collection"] # at present, AQLProfile has bugs when reporting the counters for below architectures skip_gfx = ("gfx1101", "gfx1102", "gfx1150", "gfx1151", "gfx1152", "gfx1153") def get_kernel_name(kernel_id): return data["kernel_symbols"][kernel_id]["formatted_kernel_name"] def get_agent(agent_id): for agent in data["agents"]: if agent["id"]["handle"] == agent_id["handle"]: return agent return None def get_counter(counter_id): for counter in data["counters"]: if counter["id"]["handle"] == counter_id["handle"]: return counter return None dispatch_ids = [] add_dispatches = 0 for counter in counter_collection_data: dispatch_data = counter["dispatch_data"]["dispatch_info"] assert dispatch_data["dispatch_id"] > 0 assert dispatch_data["agent_id"]["handle"] > 0 assert dispatch_data["queue_id"]["handle"] > 0 agent = get_agent(dispatch_data["agent_id"]) kernel_name = get_kernel_name(dispatch_data["kernel_id"]) assert agent is not None assert len(kernel_name) > 0 dispatch_ids.append(dispatch_data["dispatch_id"]) if is_helper(kernel_name): continue add_dispatches += 1 sq_waves_values = [] for record in counter["records"]: counter_meta = get_counter(record["counter_id"]) assert counter_meta is not None, f"record:\n\t{record}" assert ( counter_meta["name"] == "SQ_WAVES" ), f"record:\n\t{record}\ncounter:\n\t{counter_meta}" if agent["name"] not in skip_gfx: sq_waves_values.append(record["value"]) if agent["name"] not in skip_gfx: assert sum(sq_waves_values) > 0, "SQ_WAVES value is not > 0" # key regression check: all 10000 dispatches captured, none stranded/dropped assert add_dispatches == EXPECTED_DISPATCHES di_uniq = sorted(set(dispatch_ids)) di_expect = [idx + 1 for idx in range(len(dispatch_ids))] assert di_expect == di_uniq if __name__ == "__main__": exit_code = pytest.main(["-x", __file__] + sys.argv[1:]) sys.exit(exit_code)