#!/usr/bin/env python3 # Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. import argparse import json import os import re import stat import statistics import sys class colors: OK = "\033[92m" FAIL = "\033[91m" END_COLOR = "\033[0m" def print_results(results): # Store the length of the longest value in a column longest = { "name": max(len(result["name"]) for result in results), "noisy_specializations": max( len(result["noisy_specializations"]) for result in results ), "mean": max(len(result["mean"]) for result in results), "median": max(len(result["median"]) for result in results), "max": max(len(result["max"]) for result in results), "size": max(len(result["size"]) for result in results), } # The name of a column can be longer than its values longest = {key: max(value, len(key)) for key, value in longest.items()} printed = "name".ljust(longest["name"] + 2) printed += "noisy specializations".ljust(longest["noisy_specializations"] + 2) printed += "mean".ljust(longest["mean"] + 2) printed += "median".ljust(longest["median"] + 2) printed += "max".ljust(longest["max"] + 2) printed += "size".ljust(longest["size"] + 2) print(printed) for result in results: printed = result["name"].ljust(longest["name"]) printed += " " printed += colors.FAIL if result["noisy"] else colors.OK printed += ( f'{result["noisy_specializations"].ljust(longest["noisy_specializations"])}' ) printed += colors.END_COLOR printed += " " printed += colors.FAIL if result["bad_mean"] else colors.OK printed += result["mean"].ljust(longest["mean"]) printed += colors.END_COLOR printed += " " printed += colors.FAIL if result["bad_median"] else colors.OK printed += result["median"].ljust(longest["median"]) printed += colors.END_COLOR printed += " " printed += colors.FAIL if result["bad_max"] else colors.OK printed += result["max"].ljust(longest["max"]) printed += colors.END_COLOR printed += " " printed += colors.FAIL if result["bad_size"] else colors.OK printed += result["size"].ljust(longest["size"]) printed += colors.END_COLOR print(printed) def get_results(benchmarks, threshold): def get_humanized_size(size): for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: if size < 1024.0 or unit == "PiB": break size /= 1024.0 return f"{size:.1f} {unit}" success = True results = [] for benchmark in benchmarks: data = benchmark["data"] name = benchmark["name"] context = data["context"] settings = context["settings"] specializations = data["specializations"] noise_percents = [ specialization["noise_percent"] for specialization in specializations ] num_noisy = sum(noise_percent > threshold for noise_percent in noise_percents) noisy = num_noisy > 0 if noisy: success = False noisy_specializations = f"{num_noisy}/{len(specializations)}" mean = statistics.mean(noise_percents) median = statistics.median(noise_percents) max_ = max(noise_percents) size_ = int(settings["size"]) results.append( { "name": name, "noisy": noisy, "noisy_specializations": noisy_specializations, "bad_mean": mean > threshold, "mean": f"{mean:.1f}%", "bad_median": median > threshold, "median": f"{median:.1f}%", "bad_max": max_ > threshold, "max": f"{max_:.1f}%", "bad_size": 0 < size_ < 128 * 1024 * 1024, # 128 MiB "size": get_humanized_size(size_), } ) return results, success def load_benchmarks(json_dir): def is_benchmark_json(filename): if not re.match(r".*\.json$", filename): return False path = os.path.join(json_dir, filename) st_mode = os.stat(path).st_mode # we are not interested in permissions, just whether it is a regular file (S_IFREG) return st_mode & stat.S_IFREG benchmark_names = [name for name in os.listdir(json_dir) if is_benchmark_json(name)] success = True benchmarks = [] for benchmark_name in benchmark_names: with open(os.path.join(json_dir, benchmark_name)) as f: try: benchmarks.append({"name": benchmark_name, "data": json.load(f)}) except json.JSONDecodeError as e: print( f"{colors.FAIL}Failed to load {benchmark_name}{colors.END_COLOR}: {e}\n", file=sys.stderr, ) success = False return benchmarks, success def main(): parser = argparse.ArgumentParser() parser.add_argument( "--json-dir", help="The directory of benchmark JSON files that will be analyzed for noise", required=True, ) parser.add_argument( "--noise-threshold-percentage", help="The noise threshold percentage, past which benchmark specializations are considered to be too noisy", required=True, type=float, ) parser.add_argument( "--accept-high-noise", help="Don't call exit(1) when there is a noisy benchmark specialization", action=argparse.BooleanOptionalAction, ) args = parser.parse_args() print(f"The noise threshold is {args.noise_threshold_percentage:.1f}%\n") benchmarks, load_success = load_benchmarks(args.json_dir) results, results_success = get_results(benchmarks, args.noise_threshold_percentage) print_results(results) if not load_success: return False if args.accept_high_noise: return True return results_success if __name__ == "__main__": success = main() if success: exit(0) else: exit(1)