TheRock/rocm-systems/projects/rocprofiler-compute/tests/test_roofline_calc.py

# Copyright (c) Advanced Micro Devices, Inc.
# SPDX-License-Identifier:  MIT


import numpy as np
import pandas as pd
import pytest

from utils import schema
from utils.mi_gpu_spec import mi_gpu_specs
from utils.roofline_calc import (
    GraphPoints,
    calc_ai_analyze,
    calc_ceilings,
    sanitize_ai_value,
    sanitize_mem_level,
)


def run_calc_ai_analyze_with_values(
    monkeypatch: pytest.MonkeyPatch, metric_values: dict[str, object]
) -> dict:
    """
    Build mocks and invoke calc_ai_analyze with controlled metric values.

    ``metric_values`` is a dict with keys ``ai_hbm``, ``ai_l2``, ``ai_l1``,
    ``ai_lds``, ``performance`` whose values are injected into the table-402
    DataFrame that ``eval_metric`` would normally populate.

    Returns the plot-points dict produced by ``calc_ai_analyze``.

    Note: this mock simulates MI350 AI metric values based on the architecture's cache
    levels available on the hardware. Cache levels will vary for other architectures.
    """
    kernel_name = "test_kernel"
    kernel_id = 0

    workload = schema.Workload()
    workload.dfs = {1: pd.DataFrame({"Kernel_Name": [kernel_name]}, index=[kernel_id])}
    workload.sys_info = pd.DataFrame([{"gpu_arch": "gfx90a"}])
    workload.roofline_peaks = pd.DataFrame()
    workload.filter_kernel_ids = []
    workload.path = "/mock/path"

    arch_config = schema.ArchConfig()
    arch_config.dfs = {
        401: pd.DataFrame(),
        402: pd.DataFrame({
            "Metric": pd.Series(dtype="str"),
            "Value": pd.Series(dtype="object"),
        }),
    }
    arch_config.dfs_type = {401: "metric_table", 402: "metric_table"}

    pmc_df = pd.DataFrame({"Kernel_Name": [kernel_name]})

    def mock_eval_metric(
        dfs: dict,
        dfs_type: dict,
        dfs_expressions: object,
        sys_info_row: object,
        roofline_peaks: object,
        pmc_data: object,
        debug: object,
    ) -> None:
        dfs[402] = pd.DataFrame({
            "Metric": [
                "AI HBM",
                "AI L2",
                "AI L1",
                "AI LDS",
                "Performance (GFLOPs)",
            ],
            "Value": pd.array(
                [
                    metric_values["ai_hbm"],
                    metric_values["ai_l2"],
                    metric_values["ai_l1"],
                    metric_values["ai_lds"],
                    metric_values["performance"],
                ],
                dtype=object,
            ),
        })

    monkeypatch.setattr("utils.roofline_calc.eval_metric", mock_eval_metric)

    monkeypatch.setattr("utils.roofline_calc.console_debug", lambda *a, **kw: None)
    monkeypatch.setattr("utils.roofline_calc.console_warning", lambda *a, **kw: None)

    return calc_ai_analyze(
        workload=workload,
        pmc_df=pmc_df,
        arch_config=arch_config,
    )


def test_calc_ai_analyze_replaces_inf_with_zero(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """np.inf / -np.inf metric values are replaced with 0."""
    result = run_calc_ai_analyze_with_values(
        monkeypatch,
        {
            "ai_hbm": np.inf,
            "ai_l2": -np.inf,
            "ai_l1": 1.5,
            "ai_lds": np.inf,
            "performance": 100.0,
        },
    )

    assert result["kernelNames"] == ["test_kernel"]
    assert result["ai_hbm"][0] == [0], "np.inf should be replaced with 0"
    assert result["ai_hbm"][1] == [100.0]
    assert result["ai_l2"][0] == [0], "-np.inf should be replaced with 0"
    assert result["ai_l2"][1] == [100.0]
    assert result["ai_l1"][0] == [1.5], "valid float should pass through"
    assert result["ai_l1"][1] == [100.0]
    assert result["ai_lds"][0] == [0], "np.inf should be replaced with 0"
    assert result["ai_lds"][1] == [100.0]


def test_calc_ai_analyze_replaces_none_with_zero(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """None metric values are replaced with 0 and still included in plot points."""
    result = run_calc_ai_analyze_with_values(
        monkeypatch,
        {
            "ai_hbm": None,
            "ai_l2": None,
            "ai_l1": None,
            "ai_lds": None,
            "performance": 50.0,
        },
    )

    assert result["kernelNames"] == ["test_kernel"]
    assert result["ai_hbm"][0] == [0], "None should be replaced with 0"
    assert result["ai_l2"][0] == [0], "None should be replaced with 0"
    assert result["ai_l1"][0] == [0], "None should be replaced with 0"
    assert result["ai_lds"][0] == [0], "None should be replaced with 0"


def test_calc_ai_analyze_valid_values_pass_through(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Normal positive floats pass through unchanged."""
    result = run_calc_ai_analyze_with_values(
        monkeypatch,
        {
            "ai_hbm": 2.5,
            "ai_l2": 3.0,
            "ai_l1": 1.5,
            "ai_lds": 4.0,
            "performance": 100.0,
        },
    )

    assert result["kernelNames"] == ["test_kernel"]
    assert result["ai_hbm"][0] == [2.5]
    assert result["ai_hbm"][1] == [100.0]
    assert result["ai_l2"][0] == [3.0]
    assert result["ai_l2"][1] == [100.0]
    assert result["ai_l1"][0] == [1.5]
    assert result["ai_l1"][1] == [100.0]
    assert result["ai_lds"][0] == [4.0]
    assert result["ai_lds"][1] == [100.0]


def test_calc_ai_analyze_na_and_empty_replaced(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Sentinel values 'N/A' and '' are replaced with 0."""
    result = run_calc_ai_analyze_with_values(
        monkeypatch,
        {
            "ai_hbm": "N/A",
            "ai_l2": "",
            "ai_l1": "N/A",
            "ai_lds": "",
            "performance": 75.0,
        },
    )

    assert result["kernelNames"] == ["test_kernel"]
    assert result["ai_hbm"][0] == [0], "'N/A' should be replaced with 0"
    assert result["ai_l2"][0] == [0], "'' should be replaced with 0"
    assert result["ai_l1"][0] == [0], "'N/A' should be replaced with 0"
    assert result["ai_lds"][0] == [0], "'' should be replaced with 0"


def test_sanitize_ai_value_replaces_invalid_values_with_zero() -> None:
    """Invalid values are replaced with 0."""
    assert sanitize_ai_value(np.inf) == 0
    assert sanitize_ai_value(-np.inf) == 0
    assert sanitize_ai_value("N/A") == 0
    assert sanitize_ai_value("") == 0
    assert sanitize_ai_value(None) == 0
    assert sanitize_ai_value(1.5) == 1.5


##############################################################################
# sanitize_mem_level Tests
##############################################################################


def test_sanitize_mem_level_all_falls_back_to_hierarchy() -> None:
    """'ALL' results in full memory levels for the model, minus MALL."""
    result = sanitize_mem_level("ALL", "mi210")
    # mi210 has no MALL, so result equals memory_levels directly
    assert result == mi_gpu_specs.get_memory_levels("mi210")


def test_sanitize_mem_level_all_list_falls_back_to_hierarchy() -> None:
    """['ALL'] behaves the same as 'ALL'."""
    result = sanitize_mem_level(["ALL"], "mi210")
    assert result == mi_gpu_specs.get_memory_levels("mi210")


def test_sanitize_mem_level_supported_string() -> None:
    """A supported single string level is returned as a single-item list."""
    result = sanitize_mem_level("HBM", "mi210")
    assert result == ["HBM"]


def test_sanitize_mem_level_supported_list() -> None:
    """A list of supported levels is returned unchanged."""
    result = sanitize_mem_level(["HBM", "L2"], "mi210")
    assert result == ["HBM", "L2"]


def test_sanitize_mem_level_unsupported_falls_back_to_hierarchy() -> None:
    """Fully unsupported input falls back to full memory levels, minus MALL."""
    result = sanitize_mem_level("HBM", "rdna35_halo")
    # rdna35_halo memory_levels includes MALL, which is stripped by sanitize_mem_level
    expected = [m for m in mi_gpu_specs.get_memory_levels("rdna35_halo") if m != "MALL"]
    assert result == expected


def test_sanitize_mem_level_mixed_filters_unsupported() -> None:
    """Unsupported levels in a mixed list are filtered out."""
    # rdna35_halo supports L0, L1, L2, MALL, LDS — not HBM; MALL is also stripped
    result = sanitize_mem_level(["HBM", "L2"], "rdna35_halo")
    assert result == ["L2"]


def test_sanitize_mem_level_mall_is_stripped() -> None:
    """MALL is always removed from results even when explicitly requested."""
    result = sanitize_mem_level("MALL", "rdna35_halo")
    assert "MALL" not in result


def test_sanitize_mem_level_vl1d_normalised() -> None:
    """'vL1D' is normalised to 'L1' before filtering."""
    result = sanitize_mem_level("vL1D", "mi210")
    assert result == ["L1"]


##############################################################################
# calc_ceilings Tests
##############################################################################

# gfx942-class model whose memory levels (LDS/L1/L2/HBM) match BW_COLUMNS.
MFMA_GPU_MODEL = "mi300x_a1"
MFMA_GPU_ARCH = "gfx942"
# gfx1151-class model; memory levels resolve to LDS/L0/L1/L2 (MALL skipped).
WMMA_GPU_MODEL = "rdna35_halo"
WMMA_GPU_ARCH = "gfx1151"

# Union of BW columns needed by both models (mi300x_a1 reads HBM, rdna35_halo reads
# L0); calc_ceilings only consumes the levels its model supports.
BW_COLUMNS = ["HBMBw", "L2Bw", "L1Bw", "L0Bw", "LDSBw"]
BW_VALUE = 500.0

PEAK_VALUES = {
    "FP16Flops": 1000.0,
    "BF16Flops": 1500.0,
    "FP32Flops": 2000.0,
    "FP64Flops": 3000.0,
    "I8Ops": 4000.0,
    "I32Ops": 5000.0,
    "I64Ops": 6000.0,
    "MFMAF8Flops": 9000.0,
    "MFMAF16Flops": 10000.0,
    "MFMAF32Flops": 11000.0,
    "MFMAF64Flops": 12000.0,
    "MFMAI8Ops": 13000.0,
    "WMMAF16Flops": 20000.0,
    "WMMABF16Flops": 21000.0,
    "WMMAF32Flops": 22000.0,
    "WMMAF64Flops": 23000.0,
    "WMMAI8Ops": 24000.0,
}

# MFMA (CDNA3) supports the full datatype set expect F4 and F6.
MFMA_CASES = [
    ("FP32", "FP32Flops", "MFMAF32Flops"),
    ("FP16", "FP16Flops", "MFMAF16Flops"),
    ("FP64", "FP64Flops", "MFMAF64Flops"),
    ("I8", "I8Ops", "MFMAI8Ops"),
    ("I32", "I32Ops", None),
    ("I64", "I64Ops", None),
    ("BF16", None, "MFMAF16Flops"),
    ("FP8", None, "MFMAF8Flops"),
]

# FP4/FP6/FP8 are not supported on gfx1151.
WMMA_CASES = [
    ("FP32", "FP32Flops", None),
    ("FP16", "FP16Flops", None),
    ("FP64", "FP64Flops", None),
    ("I8", "I8Ops", None),
    ("I32", "I32Ops", None),
    ("I64", "I64Ops", None),
    ("BF16", "BF16Flops", None),
]

# (matrix_ops_type, gpu_model, gpu_arch, cases) tuples driving the parametrized tests.
MATRIX_FAMILIES = [
    ("MFMA", MFMA_GPU_MODEL, MFMA_GPU_ARCH, MFMA_CASES),
    ("WMMA", WMMA_GPU_MODEL, WMMA_GPU_ARCH, WMMA_CASES),
]

# Flattened (matrix_ops_type, gpu_model, gpu_arch, dtype, valu_col, matrix_col) rows.
ROOFLINE_DATATYPE_CASES = [
    (prefix, model, arch, dtype, valu_col, matrix_col)
    for prefix, model, arch, cases in MATRIX_FAMILIES
    for (dtype, valu_col, matrix_col) in cases
]


class MockMspec:
    """Minimal stand-in for MachineSpecs; calc_ceilings reads gpu_model and gpu_arch."""

    def __init__(
        self, gpu_model: str = MFMA_GPU_MODEL, gpu_arch: str = MFMA_GPU_ARCH
    ) -> None:
        self.gpu_model = gpu_model
        self.gpu_arch = gpu_arch


def roofline_parameters(
    matrix_ops_type: str = "MFMA", gpu_arch: str = MFMA_GPU_ARCH
) -> dict[str, object]:
    # matrix_ops_type is "MFMA" for CDNA (MI-series) and "WMMA" for RDNA.
    return {
        "device_id": 0,
        "mem_level": "ALL",
        "workload_dir": "/tmp",
        "matrix_ops_type": matrix_ops_type,
        "gpu_arch": gpu_arch,
    }


def full_benchmark_data() -> dict[str, list[str]]:
    """Benchmark dict with every BW, PEAK_OPS and matrix column populated."""
    data = {col: [str(BW_VALUE)] for col in BW_COLUMNS}
    for col, value in PEAK_VALUES.items():
        data[col] = [str(value)]
    return data


@pytest.mark.parametrize(
    ("matrix_ops_type", "gpu_model", "gpu_arch", "dtype", "valu_col", "matrix_col"),
    ROOFLINE_DATATYPE_CASES,
    ids=[f"{row[0]}-{row[3]}" for row in ROOFLINE_DATATYPE_CASES],
)
def test_calc_ceilings_roofline_datatype(
    matrix_ops_type: str,
    gpu_model: str,
    gpu_arch: str,
    dtype: str,
    valu_col: str | None,
    matrix_col: str | None,
) -> None:
    """Each datatype populates exactly its expected VALU and/or matrix roof."""
    result = calc_ceilings(
        roofline_parameters(matrix_ops_type, gpu_arch),
        dtype,
        full_benchmark_data(),
        MockMspec(gpu_model, gpu_arch),
    )

    if valu_col is None:
        assert result["valu"] == [], (
            f"{dtype} should not produce a VALU (PEAK_OPS) roof"
        )
    else:
        assert result["valu"], f"{dtype} should produce a VALU (PEAK_OPS) roof"
        assert result["valu"][2] == PEAK_VALUES[valu_col], (
            f"{dtype} VALU peak should be read from {valu_col}"
        )

    if matrix_col is None:
        assert result["matrix_ops"] == [], (
            f"{dtype} should not produce a {matrix_ops_type} (matrix) roof"
        )
    else:
        assert result["matrix_ops"], (
            f"{dtype} should produce a {matrix_ops_type} (matrix) roof"
        )
        assert result["matrix_ops"][2] == PEAK_VALUES[matrix_col], (
            f"{dtype} {matrix_ops_type} peak should be read from {matrix_col}"
        )


def test_fp8_special_mfma_only() -> None:
    """FP8 is matrix-only and reads its peak from the dedicated MFMAF8 column.

    FP8 is a CDNA/MFMA datatype only.
    """
    result = calc_ceilings(
        roofline_parameters("MFMA", MFMA_GPU_ARCH),
        "FP8",
        full_benchmark_data(),
        MockMspec(),
    )

    assert result["valu"] == [], "FP8 is not a PEAK_OPS datatype; no VALU roof"
    assert result["matrix_ops"][2] == PEAK_VALUES["MFMAF8Flops"]


def test_missing_peak_ops_column_returns_empty() -> None:
    """A PEAK_OPS datatype missing its Flops column returns empty ceilings."""
    benchmark_data = full_benchmark_data()
    del benchmark_data["FP64Flops"]

    result = calc_ceilings(
        roofline_parameters(gpu_arch=MFMA_GPU_ARCH), "FP64", benchmark_data, MockMspec()
    )

    assert result == GraphPoints.empty().__dict__


@pytest.mark.parametrize(
    ("matrix_ops_type", "gpu_model", "gpu_arch", "matrix_col"),
    [("MFMA", MFMA_GPU_MODEL, MFMA_GPU_ARCH, "MFMAF16Flops")],
    ids=["MFMA"],
)
def test_missing_matrix_column_skips_matrix_roof(
    matrix_ops_type: str, gpu_model: str, gpu_arch: str, matrix_col: str
) -> None:
    """A matrix datatype missing its matrix column emits no matrix roof.

    BF16 is matrix-only.
    """
    benchmark_data = full_benchmark_data()
    del benchmark_data[matrix_col]

    result = calc_ceilings(
        roofline_parameters(matrix_ops_type, gpu_arch),
        "BF16",
        benchmark_data,
        MockMspec(gpu_model, gpu_arch),
    )

    assert result["valu"] == [], "BF16 has no VALU roof regardless of matrix data"
    assert result["matrix_ops"] == [], (
        f"missing {matrix_ops_type} column should skip the roof"
    )