TheRock/rocm-systems/experimental/python/perfxpert/tests/test_agents/test_analysis.py

"""Isolation tests for Analysis agent (Layer 1)."""

import sys
import types

import pytest
from unittest.mock import MagicMock

from perfxpert.agents import analysis as analysis_module
from perfxpert.agents import schemas
from perfxpert.agents.framework import FakeProviderResponse


def test_analysis_agent_builds():
    agent = analysis_module.build_analysis_agent()
    assert agent.name == "Analysis"
    assert agent.layer == 1


def test_analysis_tool_count_exactly_five():
    agent = analysis_module.build_analysis_agent()
    assert len(agent.tools) == 5


def test_analysis_no_execution_tools():
    agent = analysis_module.build_analysis_agent()
    forbidden = {"patch.apply", "patch.revert", "compile.build", "profile.run", "anchors.check"}
    declared = {t.name for t in agent.tools}
    assert not (declared & forbidden), "Analysis must have zero execution tools"


def test_analysis_has_no_allowed_handoffs():
    """Layer 1 agents return to Root — they don't fan out (only Recommendation does)."""
    agent = analysis_module.build_analysis_agent()
    assert agent.allowed_handoffs == ()


def test_analysis_classifies_compute_bound(fake_provider, monkeypatch):
    """Given a high-VALU kernel, LLM produces a compute classification."""
    fake_provider.return_value = FakeProviderResponse(
        structured_output={
            "primary_bottleneck": "compute",
            "confidence": 0.88,
            "time_breakdown": {"kernel_pct": 0.90, "memcpy_pct": 0.05, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [{"name": "[KERNEL_1]", "pct": 0.75}],
            "counter_data_available": True,
        },
    )
    monkeypatch.setattr(
        analysis_module,
        "_collect_deterministic_metrics",
        lambda db, top_n=10: {
            "time_breakdown": {"kernel_pct": 0.90, "memcpy_pct": 0.05, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [{"name": "[KERNEL_1]", "pct": 0.75}],
            "metrics_for_classifier": {"valu_util_pct": 0.85},
            "counter_data_available": True,
        },
    )
    # Tools are stubbed via monkeypatch; the agent trusts LLM's synthesis
    result = analysis_module.run_analysis(
        schemas.AnalysisInput(database_path="fake.db", top_kernels=10),
        provider="anthropic",
    )
    assert isinstance(result, schemas.AnalysisOutput)
    assert result.primary_bottleneck == "compute"
    assert result.confidence == 0.88


def test_analysis_classifies_memory_bound(fake_provider, monkeypatch):
    fake_provider.return_value = FakeProviderResponse(
        structured_output={
            "primary_bottleneck": "memory_transfer",
            "confidence": 0.80,
            "time_breakdown": {"kernel_pct": 0.55, "memcpy_pct": 0.40, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [{"name": "[KERNEL_1]", "pct": 0.40}],
            "counter_data_available": False,
        },
    )
    monkeypatch.setattr(
        analysis_module,
        "_collect_deterministic_metrics",
        lambda db, top_n=10: {
            "time_breakdown": {"kernel_pct": 0.55, "memcpy_pct": 0.40, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [{"name": "[KERNEL_1]", "pct": 0.40}],
            "metrics_for_classifier": {"memcpy_pct": 0.40},
            "counter_data_available": False,
        },
    )
    result = analysis_module.run_analysis(
        schemas.AnalysisInput(database_path="fake.db"),
        provider="anthropic",
    )
    assert result.primary_bottleneck == "memory_transfer"


def test_analysis_airgap_uses_deterministic_classifier(monkeypatch):
    """Airgap mode must still produce a classification — from
    bottleneck.classify_from_metrics (pure rule).
    """
    monkeypatch.setenv("PERFXPERT_AIRGAP", "1")
    # Stub the time breakdown + hotspots tools
    monkeypatch.setattr(
        analysis_module, "_collect_deterministic_metrics",
        lambda db, top_n=10, min_duration=0.0: {
            "time_breakdown": {"kernel_pct": 0.90, "memcpy_pct": 0.05, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [],
            "metrics_for_classifier": {"valu_util_pct": 0.85},
            "counter_data_available": True,
        },
    )
    result = analysis_module.run_analysis(
        schemas.AnalysisInput(database_path="fake.db"),
        airgap=True,
    )
    # The rule-based classifier in bottleneck.classify_from_metrics should
    # classify as compute given valu_util_pct=0.85
    assert result.primary_bottleneck in ("compute", "mixed")


def test_analysis_airgap_prefers_runtime_overhead_when_it_dominates(monkeypatch):
    monkeypatch.setenv("PERFXPERT_AIRGAP", "1")
    monkeypatch.setattr(
        analysis_module,
        "_collect_deterministic_metrics",
        lambda db, top_n=10, min_duration=0.0: {
            "time_breakdown": {
                "kernel_pct": 8.9,
                "memcpy_pct": 4.5,
                "api_pct": 86.6,
                "idle_pct": 0.0,
            },
            "hot_kernels": [],
            "metrics_for_classifier": {
                "memcpy_pct": 0.045,
                "api_overhead_pct": 0.866,
                "valu_util_pct": 1.0,
                "arithmetic_intensity_above_ridge": 1,
                "gpu_util_pct": 1.0,
            },
            "counter_data_available": True,
        },
    )

    result = analysis_module.run_analysis(
        schemas.AnalysisInput(database_path="fake.db"),
        airgap=True,
    )

    assert result.primary_bottleneck == "latency"


def test_analysis_airgap_rejects_unknown_rule_bottleneck(monkeypatch):
    monkeypatch.setenv("PERFXPERT_AIRGAP", "1")
    monkeypatch.setattr(
        analysis_module,
        "_collect_deterministic_metrics",
        lambda db, top_n=10: {
            "time_breakdown": {"kernel_pct": 0.90, "memcpy_pct": 0.05, "api_pct": 0.03, "idle_pct": 0.02},
            "hot_kernels": [],
            "metrics_for_classifier": {},
            "counter_data_available": True,
        },
    )
    monkeypatch.setattr(
        analysis_module.bottleneck,
        "classify_from_metrics",
        lambda metrics: {"type": "unknown", "confidence": 0.2},
    )

    with pytest.raises(ValueError, match="unknown"):
        analysis_module.run_analysis(
            schemas.AnalysisInput(database_path="fake.db"),
            airgap=True,
        )


def test_analysis_propagates_counter_availability(fake_provider, monkeypatch):
    fake_provider.return_value = FakeProviderResponse(
        structured_output={
            "primary_bottleneck": "mixed",
            "confidence": 0.5,
            "time_breakdown": {"kernel_pct": 0.5, "memcpy_pct": 0.2, "api_pct": 0.2, "idle_pct": 0.1},
            "hot_kernels": [],
            "counter_data_available": False,
        },
    )
    monkeypatch.setattr(
        analysis_module,
        "_collect_deterministic_metrics",
        lambda db, top_n=10: {
            "time_breakdown": {"kernel_pct": 0.5, "memcpy_pct": 0.2, "api_pct": 0.2, "idle_pct": 0.1},
            "hot_kernels": [],
            "metrics_for_classifier": {},
            "counter_data_available": False,
        },
    )
    result = analysis_module.run_analysis(
        schemas.AnalysisInput(database_path="fake.db"),
        provider="anthropic",
    )
    assert result.counter_data_available is False


def test_extract_hw_metrics_uses_detected_arch_specs(monkeypatch):
    class _Cursor:
        def __init__(self, rows):
            self._rows = rows

        def fetchone(self):
            if isinstance(self._rows, list):
                return self._rows[0] if self._rows else None
            return self._rows

        def fetchall(self):
            if isinstance(self._rows, list):
                return self._rows
            return [self._rows]

    def _fake_execute_statement(_conn, query):
        if "COUNT(*) FROM pmc_events" in query:
            return _Cursor((1,))
        if "GROUP BY counter_name" in query:
            return _Cursor(
                [
                    ("GRBM_GUI_ACTIVE", 100.0, 100.0),
                    ("GRBM_COUNT", 100.0, 100.0),
                    ("FETCH_SIZE", 600.0, 600.0),
                    ("WRITE_SIZE", 400.0, 400.0),
                    ("SQ_INSTS_VALU_MFMA", 35.0, 35.0),
                ]
            )
        if "MAX(end) - MIN(start) FROM kernels" in query:
            return _Cursor((1000,))
        if "FROM rocpd_info_agent" in query:
            return _Cursor(("gfx950",))
        raise AssertionError(f"unexpected query: {query}")

    fake_connection = types.SimpleNamespace(
        PerfxpertConnection=lambda db: object(),
        execute_statement=_fake_execute_statement,
    )
    monkeypatch.setitem(sys.modules, "perfxpert.connection", fake_connection)

    metrics = analysis_module._extract_hw_metrics("fake.db")

    assert metrics["arithmetic_intensity_above_ridge"] == 0
    assert metrics["arithmetic_intensity_below_ridge"] == 1
    assert metrics["hbm_bw_utilization"] == pytest.approx(0.000125, rel=0.01)


def test_extract_hw_metrics_leaves_arch_sensitive_fields_unknown_when_gpu_missing(monkeypatch):
    class _Cursor:
        def __init__(self, rows):
            self._rows = rows

        def fetchone(self):
            if isinstance(self._rows, list):
                return self._rows[0] if self._rows else None
            return self._rows

        def fetchall(self):
            if isinstance(self._rows, list):
                return self._rows
            return [self._rows]

    def _fake_execute_statement(_conn, query):
        if "COUNT(*) FROM pmc_events" in query:
            return _Cursor((1,))
        if "GROUP BY counter_name" in query:
            return _Cursor(
                [
                    ("FETCH_SIZE", 600.0, 600.0),
                    ("WRITE_SIZE", 400.0, 400.0),
                    ("SQ_INSTS_VALU_MFMA", 35.0, 35.0),
                ]
            )
        if "FROM rocpd_info_agent" in query:
            return _Cursor((None,))
        raise AssertionError(f"unexpected query: {query}")

    fake_connection = types.SimpleNamespace(
        PerfxpertConnection=lambda db: object(),
        execute_statement=_fake_execute_statement,
    )
    monkeypatch.setitem(sys.modules, "perfxpert.connection", fake_connection)

    metrics = analysis_module._extract_hw_metrics("fake.db")

    assert metrics["hbm_bw_utilization"] is None
    assert metrics["arithmetic_intensity_above_ridge"] is None
    assert metrics["arithmetic_intensity_below_ridge"] is None