# Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT """Base class for benchmark tests with common functionality.""" import shutil import sys from pathlib import Path from typing import Dict, List, Any from prettytable import PrettyTable # Add parent directory to path for utils import sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from utils.logger import log from utils.exceptions import TestExecutionError from utils.extended_test_base import ExtendedTestBase, gha_append_step_summary # TODO(lajagapp): Set to True once the results API network/firewall issue is # resolved. (https://github.com/ROCm/TheRock/issues/3850) ENABLE_RESULTS_API = False class BenchmarkBase(ExtendedTestBase): """Base class providing common benchmark logic. Inherits shared infrastructure from ExtendedTestBase (execute_command, create_test_result, calculate_statistics, upload_results, etc.). Child classes must implement run_benchmarks() and parse_results(). """ def __init__(self, benchmark_name: str, display_name: str = None): """Initialize benchmark test. Args: benchmark_name: Internal benchmark name (e.g., 'rocfft') display_name: Display name for reports (e.g., 'ROCfft'), defaults to benchmark_name """ super().__init__(benchmark_name, display_name or benchmark_name.upper()) self.benchmark_name = benchmark_name self.script_dir = Path(__file__).resolve().parent def _validate_openmpi(self) -> None: """Check if OpenMPI is installed and available in the system. Raises: TestExecutionError: If OpenMPI (mpirun) is not found """ if not shutil.which("mpirun"): raise TestExecutionError( "OpenMPI not found in system\n" "Ensure OpenMPI is installed and 'mpirun' is available in PATH" ) log.info("OpenMPI validated: mpirun found in system") def create_test_result( self, test_name: str, subtest_name: str, status: str, score: float, unit: str, flag: str, **kwargs, ) -> Dict[str, Any]: """Create a standardized benchmark test result dictionary. Overrides ExtendedTestBase.create_test_result to enforce benchmark-specific required fields (score, unit, flag) and provide defaults for batch_size and ngpu. Args: test_name: Benchmark name subtest_name: Specific test identifier status: Test status ('PASS' or 'FAIL') score: Performance metric value unit: Unit of measurement (e.g., 'ms', 'GFLOPS', 'GB/s') flag: 'H' (higher is better) or 'L' (lower is better) **kwargs: Additional test-specific parameters (batch_size, ngpu, mode, etc.) Returns: Dict[str, Any]: Test result dictionary with test data and configuration """ # Extract benchmark-specific parameters with defaults batch_size = kwargs.pop("batch_size", 0) ngpu = kwargs.pop("ngpu", 1) return super().create_test_result( test_name=test_name, subtest_name=subtest_name, status=status, score=float(score), unit=unit, flag=flag, batch_size=batch_size, ngpu=ngpu, **kwargs, ) def upload_results(self, **kwargs) -> bool: """Upload results to API and save locally. Overrides ExtendedTestBase.upload_results to gate on ENABLE_RESULTS_API. """ if not ENABLE_RESULTS_API: log.warning( "Results API is disabled temporarily (ENABLE_RESULTS_API=False). Skipping upload." ) return False return super().upload_results(**kwargs) def compare_with_lkg(self, tables: Any) -> Any: """Compare results with Last Known Good baseline.""" if not ENABLE_RESULTS_API: log.warning( "Results API is disabled temporarily (ENABLE_RESULTS_API=False). Skipping LKG comparison." ) # Print raw tables so scores are still visible in the log. table_list = tables if isinstance(tables, list) else [tables] for table in table_list: if table._rows: log.info(f"\n{table}") return None log.info("Comparing results with LKG") if isinstance(tables, list): # Compare each table with LKG final_tables = [] for table in tables: if table._rows: final_table = self.client.compare_results( test_name=self.benchmark_name, table=table ) log.info(f"\n{final_table}") final_tables.append(final_table) else: log.warning(f"Table '{table.title}' has no results, skipping") return final_tables # Single table final_table = self.client.compare_results( test_name=self.benchmark_name, table=tables ) log.info(f"\n{final_table}") return final_table def write_step_summary(self, stats: Dict[str, Any], final_tables: Any) -> None: """Write results to GitHub Actions step summary.""" summary = ( f"## {self.display_name} Benchmark Results\n\n" f"**Status:** {stats['overall_status']} | " f"**Passed:** {stats['passed']}/{stats['total']} | " f"**Failed:** {stats['failed']}/{stats['total']}\n\n" ) if isinstance(final_tables, list): # Multiple tables - add each one for table in final_tables: summary += ( f"
\n" f"{table.title}\n\n" f"```\n{table}\n```\n\n" f"
\n\n" ) else: # Single table summary += ( f"
\n" f"View detailed results ({stats['total']} tests)\n\n" f"```\n{final_tables}\n```\n\n" f"
" ) gha_append_step_summary(summary) def determine_final_status(self, final_tables: Any) -> str: """Determine final test status from results table(s).""" tables = final_tables if isinstance(final_tables, list) else [final_tables] has_fail = has_unknown = False for table in tables: if "FinalResult" not in table.field_names: raise ValueError(f"Table '{table.title}' missing 'FinalResult' column") idx = table.field_names.index("FinalResult") results = [row[idx] for row in table._rows] has_fail = has_fail or "FAIL" in results has_unknown = has_unknown or "UNKNOWN" in results if has_unknown and not has_fail: log.warning("Some results have UNKNOWN status (no LKG data available)") return "FAIL" if has_fail else ("UNKNOWN" if has_unknown else "PASS") def run(self) -> int: """Execute benchmark workflow and return exit code (0=PASS, 1=FAIL).""" log.info(f"Initializing {self.display_name} Benchmark Test") # Run benchmarks (implemented by child class) self.run_benchmarks() # Parse results (implemented by child class) test_results, tables = self.parse_results() if not test_results: log.error("No test results found") return 1 # Calculate statistics stats = self.calculate_statistics(test_results) log.info(f"Test Summary: {stats['passed']} passed, {stats['failed']} failed") # Upload results self.upload_results( test_results=test_results, stats=stats, test_type="benchmark", output_dir=str(self.script_dir / "results"), extra_metadata={ "benchmark_name": self.benchmark_name, "total_subtests": stats["total"], "passed_subtests": stats["passed"], "failed_subtests": stats["failed"], }, ) # Compare with LKG (returns None when ENABLE_RESULTS_API is False) final_tables = self.compare_with_lkg(tables) if final_tables is not None: # Write to GitHub Actions step summary self.write_step_summary(stats, final_tables) # Determine final status from LKG comparison final_status = self.determine_final_status(final_tables) else: # API disabled — use pass/fail stats directly final_status = stats["overall_status"] log.info(f"Final Status: {final_status}") # Return 0 only if PASS, otherwise return 1 return 0 if final_status == "PASS" else 1 def run_benchmark_main(benchmark_instance): """Run benchmark with standard error handling. Raises: KeyboardInterrupt: If execution is interrupted by user Exception: If benchmark execution fails """ try: exit_code = benchmark_instance.run() if exit_code != 0: raise RuntimeError(f"Benchmark failed with exit code {exit_code}") except KeyboardInterrupt: log.warning("\nExecution interrupted by user") raise except Exception as e: log.error(f"Execution failed: {e}") import traceback traceback.print_exc() raise