CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1638 lines
58 KiB
Python
Executable File
1638 lines
58 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Reduce a hotswap corpus failure while preserving an external predicate.
|
|
|
|
The reducer treats the interestingness command as an argv vector. It never
|
|
passes user input through a shell. See ``docs/HotswapReduce.md`` for the
|
|
bundle schema and examples.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import copy
|
|
import fnmatch
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional, Sequence
|
|
|
|
|
|
BUNDLE_FORMAT = "comgr-hotswap-reducer-bundle"
|
|
BUNDLE_VERSION = 1
|
|
LOG_FORMAT = "comgr-hotswap-reduction-log"
|
|
LOG_VERSION = 1
|
|
CACHE_FORMAT = "comgr-hotswap-reduction-cache"
|
|
CACHE_VERSION = 1
|
|
|
|
KNOWN_PLACEHOLDERS = ("{bundle}", "{input}", "{metadata}", "{workspace}")
|
|
DEFAULT_PROTECTED_SECTIONS = (
|
|
"",
|
|
".text",
|
|
".note",
|
|
".note.*",
|
|
".symtab",
|
|
".dynsym",
|
|
".strtab",
|
|
".dynstr",
|
|
".shstrtab",
|
|
".rel",
|
|
".rel.*",
|
|
".rela",
|
|
".rela.*",
|
|
".group",
|
|
".rodata",
|
|
".data",
|
|
".bss",
|
|
".dynamic",
|
|
".AMDGPU.*",
|
|
)
|
|
CAPTURE_LIMIT = 4096
|
|
MAX_COMPONENT_CHARS = 80
|
|
|
|
|
|
class ReducerError(Exception):
|
|
"""A user-facing, fail-safe reduction error."""
|
|
|
|
|
|
def _reject_json_constant(value: str) -> None:
|
|
raise ValueError(f"non-finite JSON number {value!r} is not supported")
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
try:
|
|
with path.open("r", encoding="utf-8") as stream:
|
|
return json.load(stream, parse_constant=_reject_json_constant)
|
|
except (OSError, json.JSONDecodeError, ValueError, RecursionError) as error:
|
|
raise ReducerError(f"{path}: malformed JSON: {error}") from error
|
|
|
|
|
|
def canonical_json_bytes(value: Any) -> bytes:
|
|
try:
|
|
text = json.dumps(
|
|
value,
|
|
allow_nan=False,
|
|
ensure_ascii=True,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
)
|
|
except (TypeError, ValueError, RecursionError) as error:
|
|
raise ReducerError(f"value is not canonical JSON: {error}") from error
|
|
return text.encode("ascii")
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
try:
|
|
with path.open("rb") as stream:
|
|
while True:
|
|
chunk = stream.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
digest.update(chunk)
|
|
except OSError as error:
|
|
raise ReducerError(f"could not hash {path}: {error}") from error
|
|
return digest.hexdigest()
|
|
|
|
|
|
def atomic_write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor, temporary_name = tempfile.mkstemp(
|
|
prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent)
|
|
)
|
|
temporary_path = Path(temporary_name)
|
|
try:
|
|
with os.fdopen(descriptor, "wb") as stream:
|
|
stream.write(canonical_json_bytes(value))
|
|
stream.write(b"\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary_path, path)
|
|
except BaseException:
|
|
try:
|
|
temporary_path.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def _safe_component(value: str) -> str:
|
|
component = re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("._")
|
|
return component[:MAX_COMPONENT_CHARS] or "object"
|
|
|
|
|
|
def _object_output_name(index: int, object_id: str, original_name: str) -> str:
|
|
return f"{index:04d}-{_safe_component(object_id)}-{_safe_component(original_name)}"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CodeObject:
|
|
object_id: str
|
|
original_name: str
|
|
path: Path
|
|
original_sha256: str
|
|
content_sha256: Optional[str] = None
|
|
removed_sections: tuple[str, ...] = ()
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.content_sha256 is None:
|
|
object.__setattr__(self, "content_sha256", self.original_sha256)
|
|
|
|
@property
|
|
def digest(self) -> str:
|
|
return self.content_sha256 or self.original_sha256
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Candidate:
|
|
code_objects: tuple[CodeObject, ...]
|
|
metadata: dict[str, Any]
|
|
source_paths: tuple[Path, ...] = ()
|
|
|
|
def digest(self) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(b"comgr-hotswap-reducer-candidate-v1\0")
|
|
for code_object in self.code_objects:
|
|
digest.update(canonical_json_bytes(code_object.object_id))
|
|
digest.update(b"\0")
|
|
digest.update(canonical_json_bytes(code_object.original_name))
|
|
digest.update(b"\0")
|
|
digest.update(bytes.fromhex(code_object.digest))
|
|
digest.update(b"\0metadata\0")
|
|
digest.update(canonical_json_bytes(self.metadata))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _validate_metadata(metadata: Any, source: str) -> dict[str, Any]:
|
|
if not isinstance(metadata, dict):
|
|
raise ReducerError(f"{source}: metadata must be a JSON object")
|
|
for key in ("kernels", "cases", "arguments", "selected_tests"):
|
|
if key in metadata and not isinstance(metadata[key], list):
|
|
raise ReducerError(f"{source}: metadata.{key} must be a JSON list")
|
|
cases = metadata.get("cases", [])
|
|
for index, case in enumerate(cases):
|
|
if isinstance(case, dict) and "arguments" in case:
|
|
if not isinstance(case["arguments"], list):
|
|
raise ReducerError(
|
|
f"{source}: metadata.cases[{index}].arguments must be a JSON list"
|
|
)
|
|
canonical_json_bytes(metadata)
|
|
return copy.deepcopy(metadata)
|
|
|
|
|
|
def _resolve_input_path(
|
|
path_value: Any,
|
|
base: Path,
|
|
description: str,
|
|
require_within_base: bool = False,
|
|
) -> Path:
|
|
if not isinstance(path_value, str) or not path_value:
|
|
raise ReducerError(f"{description}: path must be a non-empty string")
|
|
path = Path(path_value)
|
|
if require_within_base and path.is_absolute():
|
|
raise ReducerError(f"{description}: bundle path must be relative")
|
|
if not path.is_absolute():
|
|
path = base / path
|
|
try:
|
|
path = path.resolve(strict=True)
|
|
except OSError as error:
|
|
raise ReducerError(
|
|
f"{description}: could not resolve {path}: {error}"
|
|
) from error
|
|
if not path.is_file():
|
|
raise ReducerError(f"{description}: {path} is not a regular file")
|
|
if require_within_base and not _path_is_within(path, base.resolve()):
|
|
raise ReducerError(f"{description}: bundle path escapes {base.resolve()}")
|
|
return path
|
|
|
|
|
|
def load_bundle(path: Path) -> Candidate:
|
|
bundle_path = path.resolve()
|
|
value = load_json(bundle_path)
|
|
if not isinstance(value, dict):
|
|
raise ReducerError(f"{bundle_path}: bundle must be a JSON object")
|
|
if value.get("format") != BUNDLE_FORMAT:
|
|
raise ReducerError(f"{bundle_path}: format must be {BUNDLE_FORMAT!r}")
|
|
if value.get("version") != BUNDLE_VERSION:
|
|
raise ReducerError(
|
|
f"{bundle_path}: unsupported bundle version "
|
|
f"{value.get('version')!r}; expected {BUNDLE_VERSION}"
|
|
)
|
|
object_values = value.get("code_objects")
|
|
if not isinstance(object_values, list) or not object_values:
|
|
raise ReducerError(f"{bundle_path}: code_objects must be a non-empty JSON list")
|
|
|
|
code_objects: list[CodeObject] = []
|
|
seen_ids: set[str] = set()
|
|
for index, object_value in enumerate(object_values):
|
|
if not isinstance(object_value, dict):
|
|
raise ReducerError(
|
|
f"{bundle_path}: code_objects[{index}] must be a JSON object"
|
|
)
|
|
object_id = object_value.get("id")
|
|
if not isinstance(object_id, str) or not object_id:
|
|
raise ReducerError(
|
|
f"{bundle_path}: code_objects[{index}].id must be a non-empty string"
|
|
)
|
|
if object_id in seen_ids:
|
|
raise ReducerError(f"{bundle_path}: duplicate code object id {object_id!r}")
|
|
seen_ids.add(object_id)
|
|
object_path = _resolve_input_path(
|
|
object_value.get("path"),
|
|
bundle_path.parent,
|
|
f"{bundle_path}: code_objects[{index}]",
|
|
require_within_base=True,
|
|
)
|
|
code_objects.append(
|
|
CodeObject(
|
|
object_id=object_id,
|
|
original_name=object_path.name,
|
|
path=object_path,
|
|
original_sha256=sha256_file(object_path),
|
|
)
|
|
)
|
|
|
|
source_paths = [bundle_path]
|
|
source_paths.extend(code_object.path for code_object in code_objects)
|
|
metadata_value = value.get("metadata", {})
|
|
if isinstance(metadata_value, str):
|
|
metadata_path = _resolve_input_path(
|
|
metadata_value,
|
|
bundle_path.parent,
|
|
f"{bundle_path}: metadata",
|
|
require_within_base=True,
|
|
)
|
|
source_paths.append(metadata_path)
|
|
metadata_value = load_json(metadata_path)
|
|
metadata_source = str(metadata_path)
|
|
else:
|
|
metadata_source = f"{bundle_path}: metadata"
|
|
metadata = _validate_metadata(metadata_value, metadata_source)
|
|
return Candidate(tuple(code_objects), metadata, tuple(source_paths))
|
|
|
|
|
|
def load_nul_worklist(path: Path) -> list[Path]:
|
|
"""Load a NUL-delimited path list without interpreting shell syntax."""
|
|
|
|
worklist_path = path.resolve()
|
|
try:
|
|
contents = worklist_path.read_bytes()
|
|
except OSError as error:
|
|
raise ReducerError(
|
|
f"could not read worklist {worklist_path}: {error}"
|
|
) from error
|
|
if not contents:
|
|
raise ReducerError(f"{worklist_path}: worklist is empty")
|
|
if not contents.endswith(b"\0"):
|
|
raise ReducerError(f"{worklist_path}: NUL-delimited worklist must end with NUL")
|
|
encoded_paths = contents[:-1].split(b"\0")
|
|
if any(not value for value in encoded_paths):
|
|
raise ReducerError(f"{worklist_path}: worklist contains an empty path")
|
|
|
|
paths: list[Path] = []
|
|
seen: set[Path] = set()
|
|
for index, encoded_path in enumerate(encoded_paths):
|
|
decoded_path = os.fsdecode(encoded_path)
|
|
resolved_path = _resolve_input_path(
|
|
decoded_path,
|
|
worklist_path.parent,
|
|
f"{worklist_path}: entry #{index + 1}",
|
|
)
|
|
if resolved_path in seen:
|
|
raise ReducerError(
|
|
f"{worklist_path}: duplicate worklist path {resolved_path}"
|
|
)
|
|
seen.add(resolved_path)
|
|
paths.append(resolved_path)
|
|
return paths
|
|
|
|
|
|
def load_inputs(
|
|
bundle: Optional[Path],
|
|
code_object_paths: Sequence[Path],
|
|
worklist: Optional[Path],
|
|
metadata_path: Optional[Path],
|
|
) -> Candidate:
|
|
if bundle is not None:
|
|
if code_object_paths or worklist is not None or metadata_path is not None:
|
|
raise ReducerError(
|
|
"--bundle cannot be combined with --code-object, --worklist, "
|
|
"or --metadata"
|
|
)
|
|
return load_bundle(bundle)
|
|
if worklist is not None:
|
|
if code_object_paths:
|
|
raise ReducerError("--worklist cannot be combined with --code-object")
|
|
input_paths = load_nul_worklist(worklist)
|
|
else:
|
|
input_paths = list(code_object_paths)
|
|
if not input_paths:
|
|
raise ReducerError("one of --bundle, --code-object, or --worklist is required")
|
|
|
|
code_objects: list[CodeObject] = []
|
|
for index, path in enumerate(input_paths):
|
|
object_path = _resolve_input_path(
|
|
str(path), Path.cwd(), f"--code-object #{index + 1}"
|
|
)
|
|
code_objects.append(
|
|
CodeObject(
|
|
object_id=f"object-{index:04d}",
|
|
original_name=object_path.name,
|
|
path=object_path,
|
|
original_sha256=sha256_file(object_path),
|
|
)
|
|
)
|
|
|
|
source_paths = [item.path for item in code_objects]
|
|
if worklist is not None:
|
|
source_paths.append(worklist.resolve())
|
|
if metadata_path is None:
|
|
metadata: dict[str, Any] = {}
|
|
else:
|
|
resolved_metadata = _resolve_input_path(
|
|
str(metadata_path), Path.cwd(), "--metadata"
|
|
)
|
|
source_paths.append(resolved_metadata)
|
|
metadata = _validate_metadata(
|
|
load_json(resolved_metadata), str(resolved_metadata)
|
|
)
|
|
return Candidate(tuple(code_objects), metadata, tuple(source_paths))
|
|
|
|
|
|
def materialize_candidate(candidate: Candidate, destination: Path) -> dict[str, Any]:
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
object_directory = destination / "objects"
|
|
object_directory.mkdir()
|
|
object_entries: list[dict[str, Any]] = []
|
|
for index, code_object in enumerate(candidate.code_objects):
|
|
output_name = _object_output_name(
|
|
index, code_object.object_id, code_object.original_name
|
|
)
|
|
relative_path = Path("objects") / output_name
|
|
output_path = destination / relative_path
|
|
shutil.copyfile(code_object.path, output_path)
|
|
copied_digest = sha256_file(output_path)
|
|
if copied_digest != code_object.digest:
|
|
raise ReducerError(
|
|
f"{code_object.path}: content changed while materializing candidate"
|
|
)
|
|
object_entries.append(
|
|
{
|
|
"id": code_object.object_id,
|
|
"path": relative_path.as_posix(),
|
|
"sha256": code_object.digest,
|
|
"size": output_path.stat().st_size,
|
|
}
|
|
)
|
|
atomic_write_json(destination / "metadata.json", candidate.metadata)
|
|
bundle = {
|
|
"format": BUNDLE_FORMAT,
|
|
"version": BUNDLE_VERSION,
|
|
"code_objects": object_entries,
|
|
"metadata": "metadata.json",
|
|
}
|
|
atomic_write_json(destination / "bundle.json", bundle)
|
|
return bundle
|
|
|
|
|
|
def snapshot_candidate(candidate: Candidate, destination: Path) -> Candidate:
|
|
"""Copy original objects once so later predicate runs see immutable bytes."""
|
|
|
|
destination.mkdir(parents=True)
|
|
snapshots: list[CodeObject] = []
|
|
for index, code_object in enumerate(candidate.code_objects):
|
|
output_path = destination / _object_output_name(
|
|
index, code_object.object_id, code_object.original_name
|
|
)
|
|
shutil.copyfile(code_object.path, output_path)
|
|
snapshot_digest = sha256_file(output_path)
|
|
if snapshot_digest != code_object.original_sha256:
|
|
raise ReducerError(
|
|
f"{code_object.path}: content changed while taking the "
|
|
"reduction snapshot"
|
|
)
|
|
snapshots.append(
|
|
replace(
|
|
code_object,
|
|
path=output_path,
|
|
content_sha256=snapshot_digest,
|
|
)
|
|
)
|
|
return replace(candidate, code_objects=tuple(snapshots))
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PredicateResult:
|
|
status: str
|
|
exit_codes: tuple[Optional[int], ...]
|
|
stdout: str
|
|
stderr: str
|
|
cached: bool = False
|
|
|
|
@property
|
|
def interesting(self) -> bool:
|
|
return self.status == "interesting"
|
|
|
|
def for_log(self) -> dict[str, Any]:
|
|
return {
|
|
"status": self.status,
|
|
"exit_codes": list(self.exit_codes),
|
|
"stdout": self.stdout,
|
|
"stderr": self.stderr,
|
|
}
|
|
|
|
|
|
class PredicateCache:
|
|
def __init__(self, path: Optional[Path]) -> None:
|
|
self.path = path
|
|
self.values: dict[str, dict[str, Any]] = {}
|
|
if path is None or not path.exists():
|
|
return
|
|
value = load_json(path)
|
|
if (
|
|
not isinstance(value, dict)
|
|
or value.get("format") != CACHE_FORMAT
|
|
or value.get("version") != CACHE_VERSION
|
|
or not isinstance(value.get("entries"), dict)
|
|
):
|
|
raise ReducerError(f"{path}: unsupported predicate cache format")
|
|
self.values = value["entries"]
|
|
|
|
def get(self, key: str) -> Optional[PredicateResult]:
|
|
value = self.values.get(key)
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, dict):
|
|
raise ReducerError(f"{self.path or 'predicate cache'}: corrupt entry {key}")
|
|
status = value.get("status")
|
|
exit_codes = value.get("exit_codes")
|
|
if status not in ("interesting", "uninteresting") or not isinstance(
|
|
exit_codes, list
|
|
):
|
|
raise ReducerError(f"{self.path or 'predicate cache'}: corrupt entry {key}")
|
|
if not exit_codes or not all(
|
|
isinstance(code, int) and not isinstance(code, bool) for code in exit_codes
|
|
):
|
|
raise ReducerError(f"{self.path or 'predicate cache'}: corrupt entry {key}")
|
|
stdout = value.get("stdout", "")
|
|
stderr = value.get("stderr", "")
|
|
if not isinstance(stdout, str) or not isinstance(stderr, str):
|
|
raise ReducerError(f"{self.path or 'predicate cache'}: corrupt entry {key}")
|
|
return PredicateResult(
|
|
status,
|
|
tuple(exit_codes),
|
|
stdout,
|
|
stderr,
|
|
cached=True,
|
|
)
|
|
|
|
def put(self, key: str, result: PredicateResult) -> None:
|
|
# Only stable terminal outcomes are safe to replay during reduction.
|
|
if result.status not in ("interesting", "uninteresting"):
|
|
return
|
|
self.values[key] = {
|
|
"status": result.status,
|
|
"exit_codes": list(result.exit_codes),
|
|
"stdout": result.stdout,
|
|
"stderr": result.stderr,
|
|
}
|
|
if self.path is not None:
|
|
atomic_write_json(
|
|
self.path,
|
|
{
|
|
"format": CACHE_FORMAT,
|
|
"version": CACHE_VERSION,
|
|
"entries": self.values,
|
|
},
|
|
)
|
|
|
|
|
|
class PredicateRunner:
|
|
def __init__(
|
|
self,
|
|
argv_template: Sequence[str],
|
|
interesting_exit_code: int,
|
|
runs: int,
|
|
timeout: float,
|
|
work_root: Path,
|
|
cache: PredicateCache,
|
|
cache_dependencies: Sequence[Path] = (),
|
|
cache_tags: Sequence[str] = (),
|
|
) -> None:
|
|
if not argv_template:
|
|
raise ReducerError("the interestingness argv must not be empty")
|
|
if runs < 1:
|
|
raise ReducerError("--predicate-runs must be at least 1")
|
|
if timeout <= 0:
|
|
raise ReducerError("--timeout must be greater than zero")
|
|
resolved_executable = shutil.which(argv_template[0])
|
|
if resolved_executable is None:
|
|
raise ReducerError(
|
|
f"could not find predicate executable {argv_template[0]!r}"
|
|
)
|
|
executable_path = Path(resolved_executable).resolve()
|
|
if not executable_path.is_file():
|
|
raise ReducerError(
|
|
f"predicate executable is not a regular file: {executable_path}"
|
|
)
|
|
normalized_arguments: list[str] = []
|
|
for argument in argv_template[1:]:
|
|
if any(placeholder in argument for placeholder in KNOWN_PLACEHOLDERS):
|
|
normalized_arguments.append(argument)
|
|
continue
|
|
argument_path = Path(argument)
|
|
normalized_arguments.append(
|
|
str(argument_path.resolve()) if argument_path.is_file() else argument
|
|
)
|
|
self.argv_template = (str(executable_path),) + tuple(normalized_arguments)
|
|
self.interesting_exit_code = interesting_exit_code
|
|
self.runs = runs
|
|
self.timeout = timeout
|
|
self.work_root = work_root
|
|
self.cache = cache
|
|
self._validate_placeholders()
|
|
self.identity_files = self._collect_identity_files(
|
|
executable_path, cache_dependencies
|
|
)
|
|
if any(not tag for tag in cache_tags):
|
|
raise ReducerError("--cache-tag values must be non-empty")
|
|
self.cache_tags = tuple(cache_tags)
|
|
self.config_digest = hashlib.sha256(
|
|
canonical_json_bytes(
|
|
{
|
|
"argv": self.argv_template,
|
|
"identity_files": self.identity_files,
|
|
"cache_tags": self.cache_tags,
|
|
"interesting_exit_code": interesting_exit_code,
|
|
"runs": runs,
|
|
"timeout": timeout,
|
|
}
|
|
)
|
|
).hexdigest()
|
|
|
|
def _collect_identity_files(
|
|
self, executable: Path, cache_dependencies: Sequence[Path]
|
|
) -> tuple[dict[str, Any], ...]:
|
|
identified_paths: list[tuple[str, Path]] = [("executable", executable)]
|
|
for index, argument in enumerate(self.argv_template[1:], start=1):
|
|
if any(placeholder in argument for placeholder in KNOWN_PLACEHOLDERS):
|
|
continue
|
|
argument_path = Path(argument)
|
|
if argument_path.is_file():
|
|
identified_paths.append((f"argv[{index}]", argument_path.resolve()))
|
|
for index, dependency in enumerate(cache_dependencies):
|
|
dependency_path = dependency.resolve()
|
|
if not dependency_path.is_file():
|
|
raise ReducerError(
|
|
f"--cache-dependency is not a regular file: {dependency_path}"
|
|
)
|
|
identified_paths.append((f"dependency[{index}]", dependency_path))
|
|
|
|
identities: list[dict[str, Any]] = []
|
|
seen_paths: set[Path] = set()
|
|
for role, path in identified_paths:
|
|
if path in seen_paths:
|
|
continue
|
|
seen_paths.add(path)
|
|
try:
|
|
file_mode = stat.S_IMODE(path.stat().st_mode)
|
|
except OSError as error:
|
|
raise ReducerError(
|
|
f"could not inspect predicate identity {path}: {error}"
|
|
)
|
|
identity: dict[str, Any] = {
|
|
"path": str(path),
|
|
"role": role,
|
|
"sha256": sha256_file(path),
|
|
"mode": file_mode,
|
|
}
|
|
if role == "executable":
|
|
identity["executable"] = os.access(path, os.X_OK)
|
|
identities.append(identity)
|
|
return tuple(identities)
|
|
|
|
def _verify_identity_files(self) -> None:
|
|
for identity in self.identity_files:
|
|
path = Path(identity["path"])
|
|
try:
|
|
current_mode = stat.S_IMODE(path.stat().st_mode)
|
|
except OSError:
|
|
current_mode = None
|
|
executable_matches = identity.get("role") != "executable" or os.access(
|
|
path, os.X_OK
|
|
) == identity.get("executable")
|
|
if (
|
|
not path.is_file()
|
|
or current_mode != identity.get("mode")
|
|
or not executable_matches
|
|
or sha256_file(path) != identity["sha256"]
|
|
):
|
|
raise ReducerError(
|
|
f"predicate identity changed during reduction: {path}"
|
|
)
|
|
|
|
def _validate_placeholders(self) -> None:
|
|
for argument in self.argv_template:
|
|
for placeholder in re.findall(r"\{[^{}]+\}", argument):
|
|
if placeholder not in KNOWN_PLACEHOLDERS:
|
|
raise ReducerError(
|
|
f"unknown interestingness placeholder {placeholder!r}"
|
|
)
|
|
|
|
def _cache_key(self, candidate_digest: str) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(bytes.fromhex(self.config_digest))
|
|
digest.update(bytes.fromhex(candidate_digest))
|
|
return digest.hexdigest()
|
|
|
|
def _expand_template(self, replacements: dict[str, str]) -> list[str]:
|
|
argv: list[str] = []
|
|
for argument in self.argv_template:
|
|
expanded = argument
|
|
for placeholder, value in replacements.items():
|
|
expanded = expanded.replace(placeholder, value)
|
|
argv.append(expanded)
|
|
return argv
|
|
|
|
def _expand_argv(self, workspace: Path, candidate: Candidate) -> list[str]:
|
|
replacements = {
|
|
"{workspace}": str(workspace),
|
|
"{bundle}": str(workspace / "bundle.json"),
|
|
"{metadata}": str(workspace / "metadata.json"),
|
|
}
|
|
if len(candidate.code_objects) == 1:
|
|
input_name = _object_output_name(
|
|
0,
|
|
candidate.code_objects[0].object_id,
|
|
candidate.code_objects[0].original_name,
|
|
)
|
|
replacements["{input}"] = str(workspace / "objects" / input_name)
|
|
elif any("{input}" in argument for argument in self.argv_template):
|
|
raise ReducerError(
|
|
"{input} requires exactly one code object; use {bundle} "
|
|
"while reducing a multi-object corpus"
|
|
)
|
|
return self._expand_template(replacements)
|
|
|
|
@staticmethod
|
|
def _normalize_capture(value: Any, workspace: Path) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, bytes):
|
|
text = value.decode("utf-8", errors="replace")
|
|
else:
|
|
text = str(value)
|
|
text = text.replace(str(workspace), "{workspace}")
|
|
if len(text) > CAPTURE_LIMIT:
|
|
text = text[:CAPTURE_LIMIT] + "\n<truncated>"
|
|
return text
|
|
|
|
@staticmethod
|
|
def _terminate_process_tree(process: subprocess.Popen[bytes]) -> None:
|
|
if os.name == "nt":
|
|
if process.poll() is None:
|
|
try:
|
|
subprocess.run(
|
|
[
|
|
"taskkill.exe",
|
|
"/PID",
|
|
str(process.pid),
|
|
"/T",
|
|
"/F",
|
|
],
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
check=False,
|
|
timeout=5.0,
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
if process.poll() is None:
|
|
process.kill()
|
|
else:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
except PermissionError:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
try:
|
|
process.wait(timeout=5.0)
|
|
except subprocess.TimeoutExpired:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
|
|
def _run_predicate(
|
|
self, argv: Sequence[str], workspace: Path
|
|
) -> tuple[Optional[int], str, str, bool]:
|
|
creation_flags = (
|
|
getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0
|
|
)
|
|
with tempfile.TemporaryFile() as stdout_stream:
|
|
with tempfile.TemporaryFile() as stderr_stream:
|
|
try:
|
|
process = subprocess.Popen(
|
|
argv,
|
|
cwd=workspace,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=stdout_stream,
|
|
stderr=stderr_stream,
|
|
start_new_session=os.name != "nt",
|
|
creationflags=creation_flags,
|
|
)
|
|
except OSError as error:
|
|
return (
|
|
None,
|
|
"",
|
|
self._normalize_capture(
|
|
f"could not launch predicate: {error}", workspace
|
|
),
|
|
False,
|
|
)
|
|
timed_out = False
|
|
try:
|
|
try:
|
|
return_code = process.wait(timeout=self.timeout)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
return_code = None
|
|
finally:
|
|
self._terminate_process_tree(process)
|
|
stdout_stream.seek(0)
|
|
stderr_stream.seek(0)
|
|
stdout = self._normalize_capture(
|
|
stdout_stream.read(CAPTURE_LIMIT + 1), workspace
|
|
)
|
|
stderr = self._normalize_capture(
|
|
stderr_stream.read(CAPTURE_LIMIT + 1), workspace
|
|
)
|
|
return return_code, stdout, stderr, timed_out
|
|
|
|
def evaluate(self, candidate: Candidate) -> PredicateResult:
|
|
self._verify_identity_files()
|
|
candidate_digest = candidate.digest()
|
|
cache_key = self._cache_key(candidate_digest)
|
|
cached = self.cache.get(cache_key)
|
|
if cached is not None:
|
|
if (
|
|
len(cached.exit_codes) != self.runs
|
|
or len(set(cached.exit_codes)) != 1
|
|
or cached.interesting
|
|
!= (cached.exit_codes[0] == self.interesting_exit_code)
|
|
):
|
|
raise ReducerError(
|
|
f"{self.cache.path or 'predicate cache'}: corrupt entry {cache_key}"
|
|
)
|
|
return cached
|
|
|
|
exit_codes: list[Optional[int]] = []
|
|
outputs: list[str] = []
|
|
errors: list[str] = []
|
|
for _ in range(self.runs):
|
|
with tempfile.TemporaryDirectory(
|
|
prefix="predicate-", dir=self.work_root
|
|
) as temporary_name:
|
|
workspace = Path(temporary_name)
|
|
materialize_candidate(candidate, workspace)
|
|
argv = self._expand_argv(workspace, candidate)
|
|
return_code, stdout, stderr, timed_out = self._run_predicate(
|
|
argv, workspace
|
|
)
|
|
exit_codes.append(return_code)
|
|
outputs.append(stdout)
|
|
errors.append(stderr)
|
|
if timed_out:
|
|
return PredicateResult(
|
|
"timeout",
|
|
tuple(exit_codes),
|
|
"\n".join(outputs),
|
|
"\n".join(errors),
|
|
)
|
|
if return_code is None:
|
|
return PredicateResult(
|
|
"launch-error",
|
|
tuple(exit_codes),
|
|
"\n".join(outputs),
|
|
"\n".join(errors),
|
|
)
|
|
|
|
self._verify_identity_files()
|
|
if len(set(exit_codes)) != 1:
|
|
status = "flaky"
|
|
elif exit_codes[0] == self.interesting_exit_code:
|
|
status = "interesting"
|
|
else:
|
|
status = "uninteresting"
|
|
result = PredicateResult(
|
|
status,
|
|
tuple(exit_codes),
|
|
"\n".join(outputs),
|
|
"\n".join(errors),
|
|
)
|
|
self.cache.put(cache_key, result)
|
|
return result
|
|
|
|
def reproduction_argv(self, candidate: Candidate) -> list[str]:
|
|
replacements = {
|
|
"{workspace}": ".",
|
|
"{bundle}": "bundle.json",
|
|
"{metadata}": "metadata.json",
|
|
}
|
|
if len(candidate.code_objects) == 1:
|
|
replacements["{input}"] = (
|
|
Path("objects")
|
|
/ _object_output_name(
|
|
0,
|
|
candidate.code_objects[0].object_id,
|
|
candidate.code_objects[0].original_name,
|
|
)
|
|
).as_posix()
|
|
return self._expand_template(replacements)
|
|
|
|
|
|
def ddmin(
|
|
items: Sequence[Any],
|
|
try_complement: Callable[[list[Any], list[Any]], bool],
|
|
minimum_size: int = 0,
|
|
) -> list[Any]:
|
|
"""Return a 1-minimal subsequence accepted by ``try_complement``."""
|
|
|
|
current = list(items)
|
|
if minimum_size < 0 or minimum_size > len(current):
|
|
raise ValueError("invalid minimum_size")
|
|
granularity = 2
|
|
while len(current) > minimum_size:
|
|
granularity = min(granularity, len(current))
|
|
reduced = False
|
|
for part in range(granularity):
|
|
start = part * len(current) // granularity
|
|
end = (part + 1) * len(current) // granularity
|
|
removed = current[start:end]
|
|
complement = current[:start] + current[end:]
|
|
if len(complement) < minimum_size:
|
|
continue
|
|
if try_complement(complement, removed):
|
|
current = complement
|
|
granularity = max(granularity - 1, 2)
|
|
reduced = True
|
|
break
|
|
if reduced:
|
|
continue
|
|
if granularity == len(current):
|
|
break
|
|
granularity = min(len(current), granularity * 2)
|
|
return current
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SectionInfo:
|
|
name: str
|
|
allocated: bool
|
|
|
|
|
|
class ElfSectionTools:
|
|
def __init__(
|
|
self,
|
|
readobj: str,
|
|
objcopy: str,
|
|
timeout: float,
|
|
allow_patterns: Sequence[str],
|
|
protect_patterns: Sequence[str],
|
|
artifact_root: Path,
|
|
) -> None:
|
|
if timeout <= 0:
|
|
raise ReducerError("--tool-timeout must be greater than zero")
|
|
self.readobj = self._resolve_tool(readobj, "--readobj")
|
|
self.objcopy = self._resolve_tool(objcopy, "--objcopy")
|
|
self.timeout = timeout
|
|
self.allow_patterns = tuple(allow_patterns)
|
|
self.protect_patterns = DEFAULT_PROTECTED_SECTIONS + tuple(protect_patterns)
|
|
self.artifact_root = artifact_root
|
|
self.artifact_digests: dict[Path, str] = {}
|
|
|
|
@staticmethod
|
|
def _resolve_tool(tool: str, option: str) -> str:
|
|
resolved = shutil.which(tool)
|
|
if resolved is None:
|
|
raise ReducerError(f"{option}: could not find executable {tool!r}")
|
|
return resolved
|
|
|
|
def list_sections(self, path: Path) -> list[SectionInfo]:
|
|
argv = [
|
|
self.readobj,
|
|
"--sections",
|
|
"--elf-output-style=JSON",
|
|
str(path),
|
|
]
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
timeout=self.timeout,
|
|
)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise ReducerError(
|
|
f"llvm-readobj timed out while inspecting {path}"
|
|
) from error
|
|
except OSError as error:
|
|
raise ReducerError(
|
|
f"could not launch llvm-readobj for {path}: {error}"
|
|
) from error
|
|
if completed.returncode != 0:
|
|
detail = completed.stderr.decode("utf-8", errors="replace").strip()
|
|
raise ReducerError(
|
|
f"llvm-readobj could not inspect {path}: "
|
|
f"{detail or f'exit {completed.returncode}'}"
|
|
)
|
|
try:
|
|
value = json.loads(
|
|
completed.stdout.decode("utf-8"),
|
|
parse_constant=_reject_json_constant,
|
|
)
|
|
file_value = value[0]
|
|
summary = file_value["FileSummary"]
|
|
section_values = file_value["Sections"]
|
|
except (
|
|
UnicodeDecodeError,
|
|
json.JSONDecodeError,
|
|
ValueError,
|
|
RecursionError,
|
|
IndexError,
|
|
KeyError,
|
|
TypeError,
|
|
) as error:
|
|
raise ReducerError(
|
|
f"llvm-readobj returned malformed section JSON for {path}: {error}"
|
|
) from error
|
|
if not isinstance(summary, dict):
|
|
raise ReducerError(f"llvm-readobj returned no file summary for {path}")
|
|
file_format = summary.get("Format")
|
|
architecture = summary.get("Arch")
|
|
if not isinstance(file_format, str) or not file_format.startswith("elf"):
|
|
raise ReducerError(f"{path}: section reduction only supports ELF objects")
|
|
if not isinstance(architecture, str) or architecture.lower() not in (
|
|
"amdgcn",
|
|
"amdgpu",
|
|
):
|
|
raise ReducerError(
|
|
f"{path}: section reduction requires an AMDGPU ELF object; "
|
|
f"llvm-readobj reported {architecture!r}"
|
|
)
|
|
if not isinstance(section_values, list):
|
|
raise ReducerError(f"llvm-readobj returned no section list for {path}")
|
|
|
|
sections: list[SectionInfo] = []
|
|
for index, wrapped_section in enumerate(section_values):
|
|
try:
|
|
section = wrapped_section["Section"]
|
|
name = section["Name"]["Name"]
|
|
flag_values = section["Flags"]["Flags"]
|
|
flag_names = [flag["Name"] for flag in flag_values]
|
|
except (KeyError, TypeError) as error:
|
|
raise ReducerError(
|
|
f"llvm-readobj returned malformed section #{index} for {path}"
|
|
) from error
|
|
if not isinstance(name, str) or not all(
|
|
isinstance(flag, str) for flag in flag_names
|
|
):
|
|
raise ReducerError(
|
|
f"llvm-readobj returned malformed section #{index} for {path}"
|
|
)
|
|
sections.append(SectionInfo(name, "SHF_ALLOC" in flag_names))
|
|
return sections
|
|
|
|
def removable_sections(self, path: Path) -> list[str]:
|
|
sections_by_name: dict[str, list[SectionInfo]] = {}
|
|
section_order: list[str] = []
|
|
for section in self.list_sections(path):
|
|
if section.name not in sections_by_name:
|
|
sections_by_name[section.name] = []
|
|
section_order.append(section.name)
|
|
sections_by_name[section.name].append(section)
|
|
|
|
removable: list[str] = []
|
|
for section_name in section_order:
|
|
same_name_sections = sections_by_name[section_name]
|
|
allowed = any(
|
|
fnmatch.fnmatchcase(section_name, pattern)
|
|
for pattern in self.allow_patterns
|
|
)
|
|
protected = any(section.allocated for section in same_name_sections) or any(
|
|
fnmatch.fnmatchcase(section_name, pattern)
|
|
for pattern in self.protect_patterns
|
|
)
|
|
if allowed and not protected:
|
|
removable.append(section_name)
|
|
return removable
|
|
|
|
def _verify_objcopy_output(
|
|
self,
|
|
input_path: Path,
|
|
output_path: Path,
|
|
requested_removals: Sequence[str],
|
|
) -> None:
|
|
input_sections = self.list_sections(input_path)
|
|
output_sections = self.list_sections(output_path)
|
|
output_counts: dict[str, int] = {}
|
|
for section in output_sections:
|
|
output_counts[section.name] = output_counts.get(section.name, 0) + 1
|
|
requested = set(requested_removals)
|
|
retained_counts: dict[str, int] = {}
|
|
for section in input_sections:
|
|
if section.name not in requested:
|
|
retained_counts[section.name] = retained_counts.get(section.name, 0) + 1
|
|
remaining_requested = sorted(
|
|
name for name in requested if output_counts.get(name, 0) != 0
|
|
)
|
|
missing_retained = sorted(
|
|
name
|
|
for name, count in retained_counts.items()
|
|
if output_counts.get(name, 0) < count
|
|
)
|
|
if remaining_requested or missing_retained:
|
|
details: list[str] = []
|
|
if remaining_requested:
|
|
details.append(
|
|
"requested sections remain: " + ", ".join(remaining_requested)
|
|
)
|
|
if missing_retained:
|
|
details.append(
|
|
"unrequested sections disappeared: " + ", ".join(missing_retained)
|
|
)
|
|
raise ReducerError(
|
|
f"llvm-objcopy output verification failed for {input_path}: "
|
|
+ "; ".join(details)
|
|
)
|
|
|
|
def remove_sections(
|
|
self, code_object: CodeObject, section_names: Sequence[str]
|
|
) -> CodeObject:
|
|
if not section_names:
|
|
return code_object
|
|
key = hashlib.sha256(
|
|
canonical_json_bytes(
|
|
{
|
|
"input": code_object.digest,
|
|
"remove": sorted(section_names),
|
|
}
|
|
)
|
|
).hexdigest()
|
|
output_path = self.artifact_root / f"{key}.elf"
|
|
if not output_path.exists():
|
|
temporary_path = output_path.with_suffix(".tmp")
|
|
argv = [self.objcopy]
|
|
argv.extend(
|
|
f"--remove-section={section_name}"
|
|
for section_name in sorted(section_names)
|
|
)
|
|
argv.extend((str(code_object.path), str(temporary_path)))
|
|
try:
|
|
completed = subprocess.run(
|
|
argv,
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
timeout=self.timeout,
|
|
)
|
|
except subprocess.TimeoutExpired as error:
|
|
try:
|
|
temporary_path.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
raise ReducerError(
|
|
f"llvm-objcopy timed out while reducing {code_object.object_id!r}"
|
|
) from error
|
|
except OSError as error:
|
|
try:
|
|
temporary_path.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
raise ReducerError(
|
|
f"could not launch llvm-objcopy for "
|
|
f"{code_object.object_id!r}: {error}"
|
|
) from error
|
|
if completed.returncode != 0 or not temporary_path.is_file():
|
|
try:
|
|
temporary_path.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
detail = completed.stderr.decode("utf-8", errors="replace").strip()
|
|
raise ReducerError(
|
|
f"llvm-objcopy could not reduce "
|
|
f"{code_object.object_id!r}: "
|
|
f"{detail or f'exit {completed.returncode}'}"
|
|
)
|
|
if temporary_path.stat().st_size == 0:
|
|
temporary_path.unlink()
|
|
raise ReducerError(
|
|
f"llvm-objcopy produced an empty object for "
|
|
f"{code_object.object_id!r}"
|
|
)
|
|
try:
|
|
self._verify_objcopy_output(
|
|
code_object.path, temporary_path, section_names
|
|
)
|
|
except ReducerError:
|
|
temporary_path.unlink()
|
|
raise
|
|
os.replace(temporary_path, output_path)
|
|
self.artifact_digests[output_path] = sha256_file(output_path)
|
|
output_digest = self.artifact_digests.get(output_path)
|
|
if output_digest is None:
|
|
output_digest = sha256_file(output_path)
|
|
self.artifact_digests[output_path] = output_digest
|
|
return replace(
|
|
code_object,
|
|
path=output_path,
|
|
content_sha256=output_digest,
|
|
removed_sections=tuple(sorted(section_names)),
|
|
)
|
|
|
|
|
|
class Reducer:
|
|
def __init__(
|
|
self,
|
|
initial: Candidate,
|
|
runner: PredicateRunner,
|
|
allow_empty_bundle: bool,
|
|
section_tools: Optional[ElfSectionTools],
|
|
) -> None:
|
|
self.initial = initial
|
|
self.current = initial
|
|
self.runner = runner
|
|
self.allow_empty_bundle = allow_empty_bundle
|
|
self.section_tools = section_tools
|
|
self.transformations: list[dict[str, Any]] = []
|
|
|
|
def _attempt(
|
|
self,
|
|
pass_name: str,
|
|
transformation: dict[str, Any],
|
|
candidate: Candidate,
|
|
) -> bool:
|
|
candidate_digest = candidate.digest()
|
|
result = self.runner.evaluate(candidate)
|
|
accepted = result.interesting
|
|
self.transformations.append(
|
|
{
|
|
"pass": pass_name,
|
|
"transformation": transformation,
|
|
"candidate_digest": candidate_digest,
|
|
"accepted": accepted,
|
|
"predicate": result.for_log(),
|
|
}
|
|
)
|
|
if accepted:
|
|
self.current = candidate
|
|
return accepted
|
|
|
|
def _record_tool_error(
|
|
self, pass_name: str, transformation: dict[str, Any], error: ReducerError
|
|
) -> None:
|
|
self.transformations.append(
|
|
{
|
|
"pass": pass_name,
|
|
"transformation": transformation,
|
|
"candidate_digest": self.current.digest(),
|
|
"accepted": False,
|
|
"predicate": {
|
|
"status": "tool-error",
|
|
"error": str(error),
|
|
},
|
|
}
|
|
)
|
|
|
|
def reduce_bundle(self) -> None:
|
|
original = list(self.current.code_objects)
|
|
|
|
def try_objects(kept: list[CodeObject], removed: list[CodeObject]) -> bool:
|
|
candidate = replace(self.current, code_objects=tuple(kept))
|
|
return self._attempt(
|
|
"code-objects",
|
|
{"remove_ids": [item.object_id for item in removed]},
|
|
candidate,
|
|
)
|
|
|
|
minimum_size = 0 if self.allow_empty_bundle else 1
|
|
kept = ddmin(original, try_objects, minimum_size)
|
|
self.current = replace(self.current, code_objects=tuple(kept))
|
|
|
|
@staticmethod
|
|
def _set_json_path(
|
|
value: dict[str, Any], path: Sequence[Any], replacement: Any
|
|
) -> None:
|
|
target: Any = value
|
|
for component in path[:-1]:
|
|
target = target[component]
|
|
target[path[-1]] = replacement
|
|
|
|
def _reduce_metadata_list(self, path: Sequence[Any]) -> None:
|
|
target: Any = self.current.metadata
|
|
for component in path:
|
|
target = target[component]
|
|
original = list(target)
|
|
path_text = ".".join(str(component) for component in path)
|
|
|
|
def try_values(kept: list[Any], removed: list[Any]) -> bool:
|
|
metadata = copy.deepcopy(self.current.metadata)
|
|
self._set_json_path(metadata, path, kept)
|
|
candidate = replace(self.current, metadata=metadata)
|
|
removed_digests = [
|
|
hashlib.sha256(canonical_json_bytes(item)).hexdigest()
|
|
for item in removed
|
|
]
|
|
return self._attempt(
|
|
"metadata",
|
|
{
|
|
"path": path_text,
|
|
"remove_value_digests": removed_digests,
|
|
},
|
|
candidate,
|
|
)
|
|
|
|
kept = ddmin(original, try_values)
|
|
metadata = copy.deepcopy(self.current.metadata)
|
|
self._set_json_path(metadata, path, kept)
|
|
self.current = replace(self.current, metadata=metadata)
|
|
|
|
def reduce_metadata(self) -> None:
|
|
for key in ("kernels", "cases", "arguments", "selected_tests"):
|
|
if key in self.current.metadata and self.current.metadata[key]:
|
|
self._reduce_metadata_list((key,))
|
|
cases = self.current.metadata.get("cases", [])
|
|
for index, case in enumerate(cases):
|
|
if (
|
|
isinstance(case, dict)
|
|
and isinstance(case.get("arguments"), list)
|
|
and case["arguments"]
|
|
):
|
|
self._reduce_metadata_list(("cases", index, "arguments"))
|
|
|
|
def reduce_sections(self) -> None:
|
|
if self.section_tools is None:
|
|
return
|
|
section_tools = self.section_tools
|
|
object_index = 0
|
|
while object_index < len(self.current.code_objects):
|
|
base_object = self.current.code_objects[object_index]
|
|
try:
|
|
removable = section_tools.removable_sections(base_object.path)
|
|
except ReducerError as error:
|
|
raise ReducerError(
|
|
f"could not enumerate removable sections for "
|
|
f"{base_object.object_id!r}: {error}"
|
|
) from error
|
|
if not removable:
|
|
object_index += 1
|
|
continue
|
|
|
|
def try_sections(kept: list[str], removed: list[str]) -> bool:
|
|
removed_sections = [
|
|
section for section in removable if section not in kept
|
|
]
|
|
transformation = {
|
|
"object_id": base_object.object_id,
|
|
"remove_sections": sorted(removed_sections),
|
|
"delta_remove_sections": sorted(removed),
|
|
}
|
|
try:
|
|
reduced_object = section_tools.remove_sections(
|
|
base_object, removed_sections
|
|
)
|
|
except ReducerError as error:
|
|
self._record_tool_error("elf-sections", transformation, error)
|
|
return False
|
|
objects = list(self.current.code_objects)
|
|
objects[object_index] = reduced_object
|
|
candidate = replace(self.current, code_objects=tuple(objects))
|
|
return self._attempt("elf-sections", transformation, candidate)
|
|
|
|
kept = ddmin(removable, try_sections)
|
|
removed_sections = [section for section in removable if section not in kept]
|
|
if removed_sections:
|
|
reduced_object = section_tools.remove_sections(
|
|
base_object, removed_sections
|
|
)
|
|
objects = list(self.current.code_objects)
|
|
objects[object_index] = reduced_object
|
|
self.current = replace(self.current, code_objects=tuple(objects))
|
|
object_index += 1
|
|
|
|
def run(self) -> Candidate:
|
|
initial_result = self.runner.evaluate(self.initial)
|
|
if not initial_result.interesting:
|
|
raise ReducerError(
|
|
"the original input is not stably interesting: "
|
|
f"predicate status is {initial_result.status!r}"
|
|
)
|
|
while True:
|
|
previous_digest = self.current.digest()
|
|
self.reduce_bundle()
|
|
self.reduce_metadata()
|
|
self.reduce_sections()
|
|
if self.current.digest() == previous_digest:
|
|
break
|
|
return self.current
|
|
|
|
def make_log(self) -> dict[str, Any]:
|
|
originals = [
|
|
{
|
|
"id": code_object.object_id,
|
|
"name": code_object.original_name,
|
|
"sha256": code_object.original_sha256,
|
|
}
|
|
for code_object in self.initial.code_objects
|
|
]
|
|
return {
|
|
"format": LOG_FORMAT,
|
|
"version": LOG_VERSION,
|
|
"initial_digest": self.initial.digest(),
|
|
"final_digest": self.current.digest(),
|
|
"originals": originals,
|
|
"predicate": {
|
|
"argv_template": list(self.runner.argv_template),
|
|
"identity_files": list(self.runner.identity_files),
|
|
"cache_tags": list(self.runner.cache_tags),
|
|
"interesting_exit_code": self.runner.interesting_exit_code,
|
|
"runs": self.runner.runs,
|
|
"timeout_seconds": self.runner.timeout,
|
|
},
|
|
"section_policy": {
|
|
"enabled": self.section_tools is not None,
|
|
"allow": (
|
|
list(self.section_tools.allow_patterns)
|
|
if self.section_tools is not None
|
|
else []
|
|
),
|
|
"protect": (
|
|
list(self.section_tools.protect_patterns)
|
|
if self.section_tools is not None
|
|
else []
|
|
),
|
|
"allocated_sections_are_protected": True,
|
|
},
|
|
"transformations": self.transformations,
|
|
"reproduction": {
|
|
"working_directory": ".",
|
|
"argv": self.runner.reproduction_argv(self.current),
|
|
"interesting_exit_code": self.runner.interesting_exit_code,
|
|
},
|
|
}
|
|
|
|
|
|
def atomic_publish(
|
|
output: Path, candidate: Candidate, reduction_log: dict[str, Any]
|
|
) -> None:
|
|
output = output.resolve()
|
|
if output.exists():
|
|
raise ReducerError(f"refusing to overwrite existing output {output}")
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary_path = Path(
|
|
tempfile.mkdtemp(prefix=f".{output.name}.", dir=output.parent)
|
|
)
|
|
try:
|
|
materialize_candidate(candidate, temporary_path)
|
|
atomic_write_json(temporary_path / "reduction-log.json", reduction_log)
|
|
os.replace(temporary_path, output)
|
|
except BaseException:
|
|
shutil.rmtree(temporary_path, ignore_errors=True)
|
|
raise
|
|
|
|
|
|
def _path_is_within(path: Path, directory: Path) -> bool:
|
|
try:
|
|
path.relative_to(directory)
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _positive_float(value: str) -> float:
|
|
parsed = float(value)
|
|
if not math.isfinite(parsed) or parsed <= 0:
|
|
raise argparse.ArgumentTypeError("must be finite and greater than zero")
|
|
return parsed
|
|
|
|
|
|
def make_argument_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
input_group = parser.add_mutually_exclusive_group(required=True)
|
|
input_group.add_argument("--bundle", type=Path, help="Versioned input bundle JSON")
|
|
input_group.add_argument(
|
|
"--code-object",
|
|
type=Path,
|
|
action="append",
|
|
help="Code object input; repeat for a corpus",
|
|
)
|
|
input_group.add_argument(
|
|
"--worklist",
|
|
type=Path,
|
|
help="NUL-delimited code object paths from inventory or selection",
|
|
)
|
|
parser.add_argument(
|
|
"--metadata",
|
|
type=Path,
|
|
help="Structured launch/test metadata for --code-object or --worklist",
|
|
)
|
|
parser.add_argument(
|
|
"--output", type=Path, required=True, help="New output directory"
|
|
)
|
|
parser.add_argument(
|
|
"--predicate",
|
|
required=True,
|
|
help="Interestingness executable (no shell is used)",
|
|
)
|
|
parser.add_argument(
|
|
"--predicate-arg",
|
|
action="append",
|
|
default=[],
|
|
help=(
|
|
"One interestingness argv item; repeat as needed. Known "
|
|
"placeholders: " + ", ".join(KNOWN_PLACEHOLDERS)
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--interesting-exit-code",
|
|
type=int,
|
|
default=0,
|
|
help="Exit code meaning interesting (default: 0)",
|
|
)
|
|
parser.add_argument(
|
|
"--predicate-runs",
|
|
type=int,
|
|
default=1,
|
|
help="Repeat every uncached predicate; disagreement is flaky",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout",
|
|
type=_positive_float,
|
|
default=60.0,
|
|
help="Per-predicate timeout in seconds (default: 60)",
|
|
)
|
|
parser.add_argument(
|
|
"--cache-file",
|
|
type=Path,
|
|
help="Optional persistent content-addressed predicate cache",
|
|
)
|
|
parser.add_argument(
|
|
"--cache-dependency",
|
|
type=Path,
|
|
action="append",
|
|
default=[],
|
|
help="File whose content participates in predicate cache identity",
|
|
)
|
|
parser.add_argument(
|
|
"--cache-tag",
|
|
action="append",
|
|
default=[],
|
|
help="Environment or configuration tag for predicate cache identity",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-empty-bundle",
|
|
action="store_true",
|
|
help="Permit reducing all code objects away",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-remove-section",
|
|
action="append",
|
|
default=[],
|
|
metavar="GLOB",
|
|
help=("Opt in a non-allocated ELF section glob for removal; repeat as needed"),
|
|
)
|
|
parser.add_argument(
|
|
"--protect-section",
|
|
action="append",
|
|
default=[],
|
|
metavar="GLOB",
|
|
help="Additional protected ELF section glob; repeat as needed",
|
|
)
|
|
parser.add_argument(
|
|
"--readobj",
|
|
default="llvm-readobj",
|
|
help="llvm-readobj executable for section reduction",
|
|
)
|
|
parser.add_argument(
|
|
"--objcopy",
|
|
default="llvm-objcopy",
|
|
help="llvm-objcopy executable for section reduction",
|
|
)
|
|
parser.add_argument(
|
|
"--tool-timeout",
|
|
type=_positive_float,
|
|
default=30.0,
|
|
help="Per LLVM tool timeout in seconds (default: 30)",
|
|
)
|
|
return parser
|
|
|
|
|
|
def run_from_arguments(args: argparse.Namespace) -> tuple[Candidate, dict[str, Any]]:
|
|
initial = load_inputs(
|
|
args.bundle,
|
|
args.code_object or [],
|
|
args.worklist,
|
|
args.metadata,
|
|
)
|
|
output = args.output.resolve()
|
|
if output.exists():
|
|
raise ReducerError(f"refusing to overwrite existing output {output}")
|
|
cache_path = args.cache_file.resolve() if args.cache_file is not None else None
|
|
if cache_path is not None:
|
|
protected_inputs = {path.resolve() for path in initial.source_paths}
|
|
protected_inputs.update(path.resolve() for path in args.cache_dependency)
|
|
if cache_path in protected_inputs:
|
|
raise ReducerError(
|
|
f"refusing to overwrite reducer input with cache {cache_path}"
|
|
)
|
|
if _path_is_within(cache_path, output):
|
|
raise ReducerError(
|
|
f"--cache-file must not be inside output directory {output}"
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory(prefix="hotswap-reduce-") as temporary_name:
|
|
work_root = Path(temporary_name)
|
|
initial = snapshot_candidate(initial, work_root / "originals")
|
|
runner = PredicateRunner(
|
|
[args.predicate] + args.predicate_arg,
|
|
args.interesting_exit_code,
|
|
args.predicate_runs,
|
|
args.timeout,
|
|
work_root,
|
|
PredicateCache(None),
|
|
args.cache_dependency,
|
|
args.cache_tag,
|
|
)
|
|
if cache_path is not None and cache_path in {
|
|
Path(identity["path"]) for identity in runner.identity_files
|
|
}:
|
|
raise ReducerError(
|
|
f"refusing to overwrite predicate identity with cache {cache_path}"
|
|
)
|
|
runner.cache = PredicateCache(cache_path)
|
|
section_tools: Optional[ElfSectionTools] = None
|
|
if args.allow_remove_section:
|
|
artifact_root = work_root / "artifacts"
|
|
artifact_root.mkdir()
|
|
section_tools = ElfSectionTools(
|
|
args.readobj,
|
|
args.objcopy,
|
|
args.tool_timeout,
|
|
args.allow_remove_section,
|
|
args.protect_section,
|
|
artifact_root,
|
|
)
|
|
reducer = Reducer(
|
|
initial,
|
|
runner,
|
|
args.allow_empty_bundle,
|
|
section_tools,
|
|
)
|
|
final = reducer.run()
|
|
reduction_log = reducer.make_log()
|
|
atomic_publish(output, final, reduction_log)
|
|
published = load_bundle(output / "bundle.json")
|
|
return published, reduction_log
|
|
|
|
|
|
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
parser = make_argument_parser()
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
final, reduction_log = run_from_arguments(args)
|
|
except KeyboardInterrupt:
|
|
print("hotswap-reduce: interrupted; no output was published", file=sys.stderr)
|
|
return 130
|
|
except ReducerError as error:
|
|
print(f"hotswap-reduce: error: {error}", file=sys.stderr)
|
|
return 1
|
|
except OSError as error:
|
|
print(f"hotswap-reduce: filesystem error: {error}", file=sys.stderr)
|
|
return 1
|
|
print(
|
|
f"wrote {args.output} with {len(final.code_objects)} code object(s); "
|
|
f"digest {reduction_log['final_digest']}"
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|