TheRock/rocm-systems/experimental/python/perfxpert/tests/test_tools/test_sol.py

"""Tests for perfxpert.tools.sol — including anti-Sakana reward-hacking defense."""

import pytest

from perfxpert.tools import sol
from perfxpert.tools._class import ToolClass


# -- sanity_check (anti-Sakana) --------------------------------------------

def test_claimed_speedup_within_sol_passes():
    """Normal optimization — 2× speedup on fp64 kernel ~ 40 TFLOPS on MI300X (peak 81.7)."""
    r = sol.sanity_check(
        achieved_flops_per_sec=40e12,
        kernel_type="fp64",
        gfx_id="gfx942",
    )
    assert r["plausible"] is True


def test_claimed_speedup_exceeding_sol_rejected():
    """Sakana-style: claim 500 TFLOPS fp64 on MI300X whose peak is 81.7."""
    r = sol.sanity_check(
        achieved_flops_per_sec=500e12,
        kernel_type="fp64",
        gfx_id="gfx942",
    )
    assert r["plausible"] is False
    assert "exceeds" in r["reason"].lower() or "peak" in r["reason"].lower()
    assert r["sol_peak"] == 81.7e12


def test_fp32_variant():
    """MI300X FP32 peak = 163.4 TFLOPS."""
    r = sol.sanity_check(
        achieved_flops_per_sec=150e12,
        kernel_type="fp32",
        gfx_id="gfx942",
    )
    assert r["plausible"] is True


def test_bf16_variant():
    """MI300X BF16 peak = 1307 TFLOPS (matrix)."""
    r = sol.sanity_check(
        achieved_flops_per_sec=1000e12,
        kernel_type="bf16",
        gfx_id="gfx942",
    )
    assert r["plausible"] is True


def test_unknown_kernel_type_raises():
    # gpu_specs.yaml now exposes peak_fp16_tflops on every arch (Phase
    # 10 Live-Roofline extension) so "fp16" is no longer an unknown
    # kernel_type. Use a token that will stay unknown forever.
    with pytest.raises(ValueError):
        sol.sanity_check(achieved_flops_per_sec=1e12, kernel_type="fp42", gfx_id="gfx942")


# -- classify_utilization --------------------------------------------------

def test_utilization_high():
    assert sol.classify_utilization(0.85)["category"] == "high"


def test_utilization_medium():
    assert sol.classify_utilization(0.60)["category"] == "medium"


def test_utilization_low():
    assert sol.classify_utilization(0.20)["category"] == "low"


def test_utilization_out_of_range_raises():
    with pytest.raises(ValueError):
        sol.classify_utilization(1.5)


def test_is_read_only_class():
    assert sol.sanity_check.__tool_class__ == ToolClass.READ_ONLY
    assert sol.classify_utilization.__tool_class__ == ToolClass.READ_ONLY