ProxMenux/.github/scripts/ai-models-verifier/apply.py

#!/usr/bin/env python3
"""Apply a verifier report to ``AppImage/config/verified_ai_models.json``.

Reads the machine-readable report emitted by ``verify.py --json-out`` and
merges the passing models into the on-disk catalog. Preserves the
maintainer's editorial curation across three axes:

* ``_exclude``: per-provider list of model IDs (exact match) that must
  never appear in the surfaced ``models`` list even when the verifier
  passes them. Meant for models that respond correctly to the technical
  test but are the wrong fit for notification translation — Arabic-only
  bases, Chinese-first fine-tunes, safety-classifier variants,
  agentic-only endpoints, legacy dated snapshots, etc.
* ``recommended``: if the current recommendation is still in the
  passing (and non-excluded) set, it is preserved. Only when the
  previous recommendation disappears (deprecated upstream, or newly
  excluded) is a fallback chosen — the fastest passing model.
* ``_note`` / ``_deprecated``: never touched. Those are maintainer
  annotations that outlive any single verifier run.

Fail-safe rules:
* Providers absent from the report (no API key configured in the
  Action for that run) are left untouched.
* Providers whose report carries an error are left untouched.
* If the ``_exclude`` filter drops every passing model, the block is
  left untouched — an empty models list would silently kill the
  provider in the UI; keeping the previous list is more forgiving
  than shipping "nothing works".
* ``_updated`` bumps to today's date only when the merge actually
  changed something. A no-op run leaves the file byte-identical.

Exits 0 when the file is unchanged, 10 when it was updated. The
workflow uses that exit code to decide whether to commit.
"""
from __future__ import annotations

import argparse
import datetime as dt
import fnmatch
import json
import sys
from pathlib import Path


def _load_json(path: Path) -> dict:
    with open(path, "r", encoding="utf-8") as fh:
        return json.load(fh)


def _save_json(path: Path, data: dict) -> None:
    tmp = path.with_suffix(path.suffix + ".tmp")
    with open(tmp, "w", encoding="utf-8") as fh:
        json.dump(data, fh, indent=2, ensure_ascii=False)
        fh.write("\n")
    tmp.replace(path)


def _is_excluded(model: str, patterns: list[str]) -> bool:
    """Match a model against the ``_exclude`` list. Supports exact
    matches and shell-style globs (``gpt-4o-*``, ``*-2024-*``, ...) so
    a provider that periodically publishes dated snapshots can be
    covered by a single pattern instead of one entry per date."""
    for pat in patterns:
        if pat == model or fnmatch.fnmatchcase(model, pat):
            return True
    return False


def _passing_models(provider_report: dict, exclude: list[str]) -> list[str]:
    """Passing models minus the editorial exclusion list, fastest first."""
    passing = [
        r for r in provider_report.get("results", [])
        if r.get("verdict") == "pass" and not _is_excluded(r.get("model", ""), exclude)
    ]
    passing.sort(key=lambda r: r.get("latency_s", 999))
    return [r["model"] for r in passing]


def apply_report(report_path: Path, catalog_path: Path, today: str) -> bool:
    """Rewrite the catalog from the report. Returns True if it changed."""
    report = _load_json(report_path)
    catalog = _load_json(catalog_path) if catalog_path.exists() else {}

    changed = False
    for provider_report in report:
        name = provider_report.get("provider")
        if not name:
            continue
        if provider_report.get("error"):
            print(f"[{name}] skipped — verifier reported error: {provider_report['error']}",
                  file=sys.stderr)
            continue

        block = catalog.setdefault(name, {})
        exclude = list(block.get("_exclude", []))
        passing = _passing_models(provider_report, exclude)

        if not passing:
            # Either the verifier returned no passes for this provider,
            # or every pass got filtered by _exclude. Both cases mean
            # "no signal we can trust to overwrite the curated list";
            # leaving the block alone is safer than blanking it.
            print(f"[{name}] skipped — no passing models after exclude filter",
                  file=sys.stderr)
            continue

        prev_models = list(block.get("models", []))
        prev_recommended = block.get("recommended", "")
        # Preserve the maintainer's choice of recommended when it is
        # still valid. Only fall back to fastest when the previous
        # value disappeared from the passing set.
        recommended = prev_recommended if prev_recommended in passing else passing[0]

        if sorted(prev_models) != sorted(passing) or prev_recommended != recommended:
            block["models"] = passing
            block["recommended"] = recommended
            changed = True
            print(f"[{name}] updated — {len(passing)} models, recommended={recommended}")
        else:
            print(f"[{name}] unchanged — {len(passing)} models")

    if changed:
        catalog["_updated"] = today
        _save_json(catalog_path, catalog)
    return changed


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--report", required=True, help="verify.py --json-out path")
    ap.add_argument("--catalog", required=True,
                    help="AppImage/config/verified_ai_models.json path")
    ap.add_argument("--today", default=None,
                    help="Override the date written into _updated (YYYY-MM-DD).")
    args = ap.parse_args()

    today = args.today or dt.datetime.utcnow().strftime("%Y-%m-%d")
    changed = apply_report(Path(args.report), Path(args.catalog), today)
    return 10 if changed else 0


if __name__ == "__main__":
    sys.exit(main())