TheRock/rocm-systems/experimental/python/perfxpert/scripts/test-samples.py

#!/usr/bin/env python3
"""
scripts/test-samples.py — Extract and execute bash/Python samples from Markdown docs
Part of the docs-audit tooling
"""

import os
import re
import subprocess
import sys
import json
from pathlib import Path
from typing import List, Dict, Any

# Python samples that import `perfxpert` need the app root on sys.path.
# This script already lives under `experimental/python/perfxpert/scripts/`,
# so `parent.parent` is the package root we want to prepend.
_PERFXPERT_SRC = Path(__file__).resolve().parent.parent


def _python_env() -> Dict[str, str]:
    """Return a subprocess env with PYTHONPATH pointing at perfxpert/."""
    env = os.environ.copy()
    existing = env.get("PYTHONPATH", "")
    parts = [str(_PERFXPERT_SRC)]
    if existing:
        parts.append(existing)
    env["PYTHONPATH"] = os.pathsep.join(parts)
    return env

# Regex patterns for code blocks
BASH_PATTERN = r'```bash\s*\n(.*?)```'
PYTHON_PATTERN = r'```python\s*\n(.*?)```'

# Dangerous commands to skip (destructive, network-dependent)
SKIP_PATTERNS = [
    r'rm\s+-rf',  # Recursive delete
    r'kill',      # Kill processes
    r'reboot',    # Reboot
    r'shutdown',  # Shutdown
    r'curl\s+https?://',  # Network calls
    r'wget',      # Network download
    r'apt-get.*install',  # Package install
    r'pip\s+install',     # Pip install
]


def should_skip_sample(code: str) -> bool:
    """Check if sample has SKIP-SAMPLE or PSEUDOCODE marker."""
    if '# SKIP-SAMPLE' in code or '# PSEUDOCODE' in code:
        return True
    if '#!/' in code and ('SKIP' in code or 'PSEUDOCODE' in code):
        return True
    return False


def should_skip_destructive(code: str) -> bool:
    """Check if code contains dangerous patterns."""
    for pattern in SKIP_PATTERNS:
        if re.search(pattern, code):
            return True
    return False


def extract_samples(doc_path: Path) -> List[Dict[str, Any]]:
    """Extract bash and Python samples from a Markdown file."""
    samples = []

    try:
        content = doc_path.read_text(encoding='utf-8')
    except Exception as e:
        return []

    # Extract bash samples
    for match in re.finditer(BASH_PATTERN, content, re.DOTALL):
        code = match.group(1).strip()
        samples.append({
            'file': str(doc_path),
            'type': 'bash',
            'code': code,
            'line': content[:match.start()].count('\n') + 1,
        })

    # Extract Python samples
    for match in re.finditer(PYTHON_PATTERN, content, re.DOTALL):
        code = match.group(1).strip()
        samples.append({
            'file': str(doc_path),
            'type': 'python',
            'code': code,
            'line': content[:match.start()].count('\n') + 1,
        })

    return samples


def run_bash_sample(code: str) -> Dict[str, Any]:
    """Execute a bash sample."""
    try:
        result = subprocess.run(
            code,
            shell=True,
            capture_output=True,
            text=True,
            timeout=10,
        )
        return {
            'passed': result.returncode == 0,
            'returncode': result.returncode,
            'stdout': result.stdout[:200],
            'stderr': result.stderr[:200],
        }
    except subprocess.TimeoutExpired:
        return {
            'passed': False,
            'error': 'Timeout (10s)',
        }
    except Exception as e:
        return {
            'passed': False,
            'error': str(e),
        }


def run_python_sample(code: str) -> Dict[str, Any]:
    """Execute a Python sample.

    Runs under a PYTHONPATH that includes the perfxpert source tree
    so `import perfxpert` works in a clean checkout without requiring
    `pip install -e .` first. See `_python_env()` for details.
    """
    try:
        result = subprocess.run(
            ['python3', '-c', code],
            capture_output=True,
            text=True,
            timeout=10,
            env=_python_env(),
        )
        return {
            'passed': result.returncode == 0,
            'returncode': result.returncode,
            'stdout': result.stdout[:200],
            'stderr': result.stderr[:200],
        }
    except subprocess.TimeoutExpired:
        return {
            'passed': False,
            'error': 'Timeout (10s)',
        }
    except Exception as e:
        return {
            'passed': False,
            'error': str(e),
        }


def run_sample(sample: Dict[str, Any]) -> Dict[str, Any]:
    """Execute a sample and return results."""
    code = sample['code']

    # Check for skip markers
    if should_skip_sample(code):
        sample['status'] = 'SKIPPED'
        sample['reason'] = 'Marked with SKIP-SAMPLE or PSEUDOCODE'
        return sample

    # Check for destructive patterns
    if should_skip_destructive(code):
        sample['status'] = 'SKIPPED'
        sample['reason'] = 'Dangerous pattern (rm, kill, install, etc.)'
        return sample

    # Run sample
    if sample['type'] == 'bash':
        result = run_bash_sample(code)
    elif sample['type'] == 'python':
        result = run_python_sample(code)
    else:
        result = {'passed': False, 'error': f"Unknown type: {sample['type']}"}

    sample['status'] = 'PASSED' if result['passed'] else 'FAILED'
    sample['result'] = result
    return sample


def main():
    """Main entry point."""
    # Parse args: optional directory + --strict flag
    strict = '--strict' in sys.argv
    args = [a for a in sys.argv[1:] if a != '--strict']
    search_root = Path(args[0] if args else ".")

    all_samples = []
    failed_count = 0

    for md_file in search_root.rglob("*.md"):
        # Skip hidden and cache dirs
        if any(part.startswith('.') for part in md_file.parts):
            continue
        # Skip legacy ai_analysis tree — being deleted by the agentic refactor
        if 'ai_analysis' in md_file.parts:
            continue
        # Skip the upstream opencode submodule (MIT). Its README translations
        # and bun node_modules contain non-executable bash samples (curl
        # installers, translated docs) that are out of our scope.
        if 'opencode' in md_file.parts or 'node_modules' in md_file.parts:
            continue

        samples = extract_samples(md_file)
        for sample in samples:
            sample = run_sample(sample)
            all_samples.append(sample)
            if sample['status'] == 'FAILED':
                failed_count += 1
                if not strict:
                    print(f"FAIL: {sample['file']}:{sample['line']} ({sample['type']})")
                    if 'result' in sample and 'stderr' in sample['result']:
                        print(f"  {sample['result']['stderr'][:100]}")

    # Output JSON summary
    if not strict:
        print(json.dumps({
            'total': len(all_samples),
            'passed': sum(1 for s in all_samples if s['status'] == 'PASSED'),
            'failed': failed_count,
            'skipped': sum(1 for s in all_samples if s['status'] == 'SKIPPED'),
            'samples': all_samples,
        }, indent=2))
    else:
        # Strict mode: only emit FAIL lines (none if zero failures)
        for s in all_samples:
            if s['status'] == 'FAILED':
                print(f"FAIL: {s['file']}:{s['line']} ({s['type']})")

    return 1 if failed_count > 0 else 0


if __name__ == "__main__":
    sys.exit(main())