recall_guard.core.loader

recall_guard.core.loader

Generic JSONL evaluation-set loader and cutoff-date guard.

Implements the input-source-agnostic JSONL contract for the honest-model-ranking harness (Req 2.1-2.5):

• EvalRow / EvalSet frozen dataclasses describe the in-memory shape.
• load_eval_set(path) parses an optional _cutoff_date header line plus one {prompt, target_direction[, metadata]} row per line. It validates the row schema strictly (raises ValueError on bad rows) and emits logging.WARNING records for low-N (<100) and class-imbalance (>60%) conditions, rather than raising.
• load_cutoffs(path) parses data/cutoffs.yaml of shape {models: {model_id: YYYY-MM-DD}} into a dict[str, date].
• assert_cutoff_safe(eval_set, models, cutoffs) enforces the cutoff guard: every shortlisted model must appear in cutoffs, and no model's training cutoff may post-date the eval set's declared cutoff_date. Violations raise CutoffViolation.

The loader never performs a train/dev split (Req 2.4): the entire file is the evaluation set.

CutoffViolation

Bases: Exception

Raised when shortlisted models post-date the eval set's cutoff.

Source code in recall_guard/core/loader.py

class CutoffViolation(Exception):
    """Raised when shortlisted models post-date the eval set's cutoff."""

EvalRow dataclass

One evaluation row: prompt + ground-truth direction + opaque metadata.

Source code in recall_guard/core/loader.py

@dataclass(frozen=True)
class EvalRow:
    """One evaluation row: prompt + ground-truth direction + opaque metadata."""

    prompt: str
    target_direction: int  # in {-1, 0, 1}
    metadata: dict[str, str] = field(default_factory=dict)

EvalSet dataclass

A loaded JSONL eval set plus its cutoff header and content hash.

Source code in recall_guard/core/loader.py

@dataclass(frozen=True)
class EvalSet:
    """A loaded JSONL eval set plus its cutoff header and content hash."""

    rows: list[EvalRow]
    cutoff_date: date | None
    path_hash: str  # sha256 hex digest of the file bytes

load_eval_set

load_eval_set(path)

Parse a JSONL eval file into an EvalSet.

The first line may optionally be a header object containing {"_cutoff_date": "YYYY-MM-DD"}. All other lines must be row objects matching the input contract (Req 2.1). Logs WARNING records for low-N and class-imbalance conditions (Req 2.2, 2.3); never raises for those. Returns the entire set as a single list with no train/dev split (Req 2.4).

Source code in recall_guard/core/loader.py

def load_eval_set(path: Path | str) -> EvalSet:
    """Parse a JSONL eval file into an ``EvalSet``.

    The first line may optionally be a header object containing
    ``{"_cutoff_date": "YYYY-MM-DD"}``. All other lines must be row objects
    matching the input contract (Req 2.1). Logs WARNING records for low-N
    and class-imbalance conditions (Req 2.2, 2.3); never raises for those.
    Returns the entire set as a single list with no train/dev split (Req 2.4).
    """
    path = Path(path)
    cutoff: date | None = None
    rows: list[EvalRow] = []
    first_content_line = True

    with path.open("r", encoding="utf-8") as fh:
        for idx, raw_line in enumerate(fh, start=1):
            stripped = raw_line.strip()
            if not stripped:
                continue
            try:
                obj = json.loads(stripped)
            except json.JSONDecodeError as exc:
                raise ValueError(f"Row {idx}: invalid JSON ({exc.msg}).") from exc
            if not isinstance(obj, dict):
                raise ValueError(f"Row {idx}: expected JSON object, got {type(obj).__name__}.")

            # Header: only accepted as the very first non-empty line.
            is_header_slot = first_content_line
            first_content_line = False
            if is_header_slot and "_cutoff_date" in obj and "prompt" not in obj:
                raw_cutoff = obj["_cutoff_date"]
                if not isinstance(raw_cutoff, str):
                    raise ValueError(
                        f"Header '_cutoff_date' must be an ISO-8601 string, got "
                        f"{type(raw_cutoff).__name__}."
                    )
                try:
                    cutoff = date.fromisoformat(raw_cutoff)
                except ValueError as exc:
                    raise ValueError(
                        f"Header '_cutoff_date' must be ISO-8601 (YYYY-MM-DD): {exc}."
                    ) from exc
                continue

            rows.append(_parse_row(obj, idx))

    _emit_quality_warnings(path, rows)
    return EvalSet(rows=rows, cutoff_date=cutoff, path_hash=_hash_file(path))

parse_metadata_date

parse_metadata_date(raw)

Normalise an eval-row metadata.date value to a date.

The shared contract for every post-harness consumer (orchestrator, gap analyzer, backtest): accept plain YYYY-MM-DD and any ISO-8601 datetime whose first ten characters are the date (2024-06-30T00:00:00). Returns None for non-strings and unparseable values so callers can skip the row instead of crashing.

Source code in recall_guard/core/loader.py

def parse_metadata_date(raw: object) -> date | None:
    """Normalise an eval-row ``metadata.date`` value to a ``date``.

    The shared contract for every post-harness consumer (orchestrator,
    gap analyzer, backtest): accept plain ``YYYY-MM-DD`` and any ISO-8601
    datetime whose first ten characters are the date (``2024-06-30T00:00:00``).
    Returns ``None`` for non-strings and unparseable values so callers can
    skip the row instead of crashing.
    """
    if not isinstance(raw, str):
        return None
    try:
        return date.fromisoformat(raw[:10])
    except ValueError:
        return None

load_cutoffs

load_cutoffs(path)

Parse the cutoffs YAML registry into {model_id: cutoff_date}.

Source code in recall_guard/core/loader.py

def load_cutoffs(path: Path | str) -> dict[str, date]:
    """Parse the cutoffs YAML registry into ``{model_id: cutoff_date}``."""
    path = Path(path)
    with path.open("r", encoding="utf-8") as fh:
        doc = yaml.safe_load(fh)

    if not isinstance(doc, dict) or "models" not in doc:
        raise ValueError(
            f"Cutoffs file {path} must be a YAML mapping with a top-level 'models' key."
        )
    models = doc["models"]
    if not isinstance(models, dict):
        raise ValueError(
            f"Cutoffs file {path}: 'models' must be a mapping of model_id -> date."
        )

    out: dict[str, date] = {}
    for model_id, raw in models.items():
        if isinstance(raw, date):
            out[str(model_id)] = raw
        elif isinstance(raw, str):
            out[str(model_id)] = date.fromisoformat(raw)
        else:
            raise ValueError(
                f"Cutoffs file {path}: model {model_id!r} cutoff must be a date "
                f"or ISO-8601 string, got {type(raw).__name__}."
            )
    return out

assert_cutoff_safe

assert_cutoff_safe(eval_set, models, cutoffs)

Fail-fast guard: every shortlisted model has a cutoff <= eval cutoff.

Raises: CutoffViolation: if any model is missing from cutoffs or, when eval_set.cutoff_date is set, post-dates it.

Source code in recall_guard/core/loader.py

def assert_cutoff_safe(
    eval_set: EvalSet,
    models: list[str],
    cutoffs: dict[str, date],
) -> None:
    """Fail-fast guard: every shortlisted model has a cutoff <= eval cutoff.

    Raises:
        CutoffViolation: if any model is missing from ``cutoffs`` or, when
            ``eval_set.cutoff_date`` is set, post-dates it.
    """
    missing = [m for m in models if m not in cutoffs]
    if missing:
        raise CutoffViolation(
            "Shortlisted models missing from cutoffs registry: " + ", ".join(missing)
        )

    if eval_set.cutoff_date is None:
        return

    too_late: list[tuple[str, date]] = [
        (m, cutoffs[m]) for m in models if cutoffs[m] > eval_set.cutoff_date
    ]
    if too_late:
        details = ", ".join(f"{m}={c.isoformat()}" for m, c in too_late)
        raise CutoffViolation(
            f"Models with training cutoffs after eval cutoff "
            f"{eval_set.cutoff_date.isoformat()}: {details}"
        )