diff --git a/.github/scripts/ai-models-verifier/apply.py b/.github/scripts/ai-models-verifier/apply.py index 972a53a1..c5dff76b 100644 --- a/.github/scripts/ai-models-verifier/apply.py +++ b/.github/scripts/ai-models-verifier/apply.py @@ -2,18 +2,32 @@ """Apply a verifier report to ``AppImage/config/verified_ai_models.json``. Reads the machine-readable report emitted by ``verify.py --json-out`` and -merges the passing models into the on-disk catalog: +merges the passing models into the on-disk catalog. Preserves the +maintainer's editorial curation across three axes: -* Passing models per provider replace the existing ``models`` list. -* ``recommended`` is set to the fastest passing model. -* Existing ``_note`` / ``_deprecated`` / provider metadata is preserved - when unchanged so the file's manual annotations survive the automated - refresh. -* Providers absent from the report (e.g. no API key configured in the - GitHub Action for that run) are left untouched — the goal is - additive maintenance, not silent removal. -* ``_updated`` bumps to today's date only when the model set actually - changes; a no-op run leaves the file byte-identical. +* ``_exclude``: per-provider list of model IDs (exact match) that must + never appear in the surfaced ``models`` list even when the verifier + passes them. Meant for models that respond correctly to the technical + test but are the wrong fit for notification translation — Arabic-only + bases, Chinese-first fine-tunes, safety-classifier variants, + agentic-only endpoints, legacy dated snapshots, etc. +* ``recommended``: if the current recommendation is still in the + passing (and non-excluded) set, it is preserved. Only when the + previous recommendation disappears (deprecated upstream, or newly + excluded) is a fallback chosen — the fastest passing model. +* ``_note`` / ``_deprecated``: never touched. Those are maintainer + annotations that outlive any single verifier run. + +Fail-safe rules: +* Providers absent from the report (no API key configured in the + Action for that run) are left untouched. +* Providers whose report carries an error are left untouched. +* If the ``_exclude`` filter drops every passing model, the block is + left untouched — an empty models list would silently kill the + provider in the UI; keeping the previous list is more forgiving + than shipping "nothing works". +* ``_updated`` bumps to today's date only when the merge actually + changed something. A no-op run leaves the file byte-identical. Exits 0 when the file is unchanged, 10 when it was updated. The workflow uses that exit code to decide whether to commit. @@ -22,8 +36,8 @@ from __future__ import annotations import argparse import datetime as dt +import fnmatch import json -import os import sys from pathlib import Path @@ -41,9 +55,23 @@ def _save_json(path: Path, data: dict) -> None: tmp.replace(path) -def _passing_models(provider_report: dict) -> list[str]: - """Return the passing models for one provider, fastest first.""" - passing = [r for r in provider_report.get("results", []) if r.get("verdict") == "pass"] +def _is_excluded(model: str, patterns: list[str]) -> bool: + """Match a model against the ``_exclude`` list. Supports exact + matches and shell-style globs (``gpt-4o-*``, ``*-2024-*``, ...) so + a provider that periodically publishes dated snapshots can be + covered by a single pattern instead of one entry per date.""" + for pat in patterns: + if pat == model or fnmatch.fnmatchcase(model, pat): + return True + return False + + +def _passing_models(provider_report: dict, exclude: list[str]) -> list[str]: + """Passing models minus the editorial exclusion list, fastest first.""" + passing = [ + r for r in provider_report.get("results", []) + if r.get("verdict") == "pass" and not _is_excluded(r.get("model", ""), exclude) + ] passing.sort(key=lambda r: r.get("latency_s", 999)) return [r["model"] for r in passing] @@ -62,21 +90,32 @@ def apply_report(report_path: Path, catalog_path: Path, today: str) -> bool: print(f"[{name}] skipped — verifier reported error: {provider_report['error']}", file=sys.stderr) continue - passing = _passing_models(provider_report) + + block = catalog.setdefault(name, {}) + exclude = list(block.get("_exclude", [])) + passing = _passing_models(provider_report, exclude) + if not passing: - print(f"[{name}] no passing models this run — leaving catalog untouched", + # Either the verifier returned no passes for this provider, + # or every pass got filtered by _exclude. Both cases mean + # "no signal we can trust to overwrite the curated list"; + # leaving the block alone is safer than blanking it. + print(f"[{name}] skipped — no passing models after exclude filter", file=sys.stderr) continue - block = catalog.setdefault(name, {}) prev_models = list(block.get("models", [])) prev_recommended = block.get("recommended", "") + # Preserve the maintainer's choice of recommended when it is + # still valid. Only fall back to fastest when the previous + # value disappeared from the passing set. + recommended = prev_recommended if prev_recommended in passing else passing[0] - if sorted(prev_models) != sorted(passing) or prev_recommended != passing[0]: + if sorted(prev_models) != sorted(passing) or prev_recommended != recommended: block["models"] = passing - block["recommended"] = passing[0] + block["recommended"] = recommended changed = True - print(f"[{name}] updated — {len(passing)} models, recommended={passing[0]}") + print(f"[{name}] updated — {len(passing)} models, recommended={recommended}") else: print(f"[{name}] unchanged — {len(passing)} models") diff --git a/AppImage/config/verified_ai_models.json b/AppImage/config/verified_ai_models.json index 85077cbc..f7d31695 100644 --- a/AppImage/config/verified_ai_models.json +++ b/AppImage/config/verified_ai_models.json @@ -1,7 +1,7 @@ { "_description": "Verified AI models for ProxMenux notifications. Only models listed here will be shown to users. Models are tested to work with the chat/completions API format.", "_updated": "2026-07-14", - "_verifier": "Refreshed with tools/ai-models-verifier (private). Re-run before each ProxMenux release to keep the list current. The verifier and ProxMenux share the same reasoning/thinking-model handlers so their verdicts stay aligned with runtime behaviour.", + "_verifier": "Refreshed by .github/workflows/verify-ai-models.yml (daily). The workflow runs .github/scripts/ai-models-verifier/verify.py against every provider whose API key is configured in repository Secrets, then applies the report via apply.py — which honours per-provider `_exclude` lists so editorial curation survives automated refreshes. Manually re-run from the Actions tab (any branch) when a new model needs to be picked up out of cycle.", "groq": { "models": [ @@ -12,7 +12,15 @@ "openai/gpt-oss-20b" ], "recommended": "llama-3.3-70b-versatile", - "_note": "Verified functionally 2026-07-14 with the Groq API (15 models discovered, 9 passed). Legacy llama-3.1-70b-versatile / llama3-70b-8192 / llama3-8b-8192 / mixtral-8x7b-32768 / gemma2-9b-it removed (retired upstream). llama-4-scout added (current-gen Llama 4, 0.47s). openai/gpt-oss-120b / gpt-oss-20b confirmed. Passing but excluded: allam-2-7b (Arabic-focused), qwen/qwen3-32b (Chinese-first, unreliable Spanish output), openai/gpt-oss-safeguard-20b (safety-classifier variant), groq/compound-mini (agentic system, wrong fit for notification translation)." + "_exclude": [ + "allam-2-7b", + "qwen/qwen3-32b", + "qwen/qwen3*", + "openai/gpt-oss-safeguard-*", + "groq/compound", + "groq/compound-*" + ], + "_note": "Verified functionally 2026-07-14 with the Groq API. `_exclude` covers models that pass the technical test but are the wrong fit for notification translation: allam-2-7b (Arabic-focused), qwen/* (Chinese-first, unreliable Spanish), openai/gpt-oss-safeguard-* (safety-classifier variant), groq/compound* (agentic system, not a chat model)." }, "gemini": { @@ -25,8 +33,17 @@ "gemini-3.5-flash" ], "recommended": "gemini-2.5-flash-lite", - "_note": "Verified 2026-07-13. gemini-flash-lite-latest now passes consistently (1.6s) and is fastest, but gemini-2.5-flash-lite remains recommended because 'latest' aliases can drift over time. gemini-3.1-flash-lite is the stable successor to 3-flash-preview. Pro variants continue to reject thinkingBudget=0 and are overkill for notification translation.", - "_deprecated": ["gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-flash", "gemini-1.0-pro", "gemini-pro"] + "_exclude": [ + "gemini-*-pro*", + "gemini-*-thinking*", + "gemini-embedding-*", + "gemini-2.0-*", + "gemini-1.5-*", + "gemini-1.0-*", + "gemini-pro" + ], + "_deprecated": ["gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-flash", "gemini-1.0-pro", "gemini-pro"], + "_note": "Verified 2026-07-13. gemini-flash-lite-latest now passes consistently (1.6s) and is fastest, but gemini-2.5-flash-lite remains recommended because 'latest' aliases can drift over time. gemini-3.1-flash-lite is the stable successor to 3-flash-preview. Pro variants continue to reject thinkingBudget=0 and are overkill for notification translation." }, "openai": { @@ -40,7 +57,37 @@ "gpt-5-nano" ], "recommended": "gpt-4.1-nano", - "_note": "Verified 2026-07-13. gpt-5.4-nano / gpt-5.4-mini removed (HTTP 400 — provider params rejected). gpt-5-nano added (2.0s, current-gen fast). Reasoning models (o-series, gpt-5/5.1/5.2 non-chat variants) are supported by openai_provider.py via max_completion_tokens + reasoning_effort=minimal, but not listed here: their latency is higher and they do not improve translation quality for notifications. Add specific reasoning IDs to this list only if a user explicitly wants them." + "_exclude": [ + "gpt-3.5-*", + "gpt-3-*", + "gpt-4", + "gpt-4-0613", + "gpt-4-turbo*", + "gpt-4o-audio*", + "gpt-4o-realtime*", + "gpt-4o-search*", + "gpt-4o-transcribe*", + "gpt-4o-mini-audio*", + "gpt-4o-mini-realtime*", + "gpt-4o-mini-search*", + "gpt-4o-mini-transcribe*", + "gpt-4o-mini-tts", + "gpt-4.1-nano-2*", + "gpt-4.1-mini-2*", + "gpt-4.1-2*", + "gpt-4o-2*", + "gpt-4o-mini-2*", + "gpt-5-nano-2*", + "gpt-5-mini-2*", + "gpt-5-chat-2*", + "gpt-5-2*", + "o1-*-2*", + "o3-*-2*", + "o4-*-2*", + "codex-*", + "computer-use-*" + ], + "_note": "Verified 2026-07-13. `_exclude` drops (a) dated snapshots (`gpt-4o-2024-11-20`, `gpt-4.1-nano-2025-04-14`, ...) — the stable aliases are preferred so the recommended model doesn't silently pin to a specific point-in-time build; (b) legacy families (gpt-3.5, gpt-4, gpt-4-turbo) that gpt-4.1 supersedes; (c) audio/realtime/search/transcribe/tts variants (wrong modality for notifications); (d) reasoning models (o-series, gpt-5.1/5.2 non-chat), which openai_provider.py supports via max_completion_tokens + reasoning_effort=minimal but do not improve translation quality and are slower. Add specific reasoning IDs to `models` manually if a user explicitly wants them." }, "anthropic": { @@ -53,7 +100,13 @@ "claude-fable-5" ], "recommended": "claude-haiku-4-5", - "_note": "Verified 2026-07-13 with all 10 discovered models passing after aligning the verifier with anthropic_provider.py (temperature omitted — newest generations reject it with 'temperature is deprecated for this model'). Legacy claude-3-5-haiku-latest / claude-3-5-sonnet-latest / claude-3-opus-latest removed (deprecated upstream, not in the Models API). haiku-4-5 is the sweet spot for notification translation (3.6s, $1/$5 per MTok); sonnet-5 for slightly richer output (3.1s, $3/$15); opus-4-8 / fable-5 for demanding cases." + "_exclude": [ + "claude-*-2*", + "claude-3-*", + "claude-3-5-*", + "claude-3-opus-*" + ], + "_note": "Verified 2026-07-13 with all 10 discovered models passing after aligning the verifier with anthropic_provider.py (temperature omitted — newest generations reject it with 'temperature is deprecated for this model'). `_exclude` drops dated snapshots (`claude-haiku-4-5-20251001`) and legacy generations (claude-3-*) that are deprecated upstream. haiku-4-5 is the sweet spot for notification translation (3.6s, $1/$5 per MTok); sonnet-5 for slightly richer output (3.1s, $3/$15); opus-4-8 / fable-5 for demanding cases." }, "openrouter": { @@ -77,7 +130,33 @@ "openai/gpt-oss-20b:free" ], "recommended": "meta-llama/llama-3.3-70b-instruct", - "_note": "Paid tier verified functionally 2026-07-14 with the OpenRouter API — all 10 curated candidates pass the Spanish-translation notification test. Fastest: llama-4-scout (0.51s), gemini-2.5-flash-lite (1.14s), gemini-2.5-flash (1.94s), llama-3.3-70b-instruct (2.29s), claude-haiku-4.5 (2.71s). Free tier verified 2026-08-17 — 7 :free models pass and are appended, ordered by latency: nemotron-3-super-120b-a12b (3.5s), gemma-4-26b-a4b-it (4.1s), nemotron-nano-12b-v2-vl (5.3s), nemotron-3-nano-30b-a3b (5.8s), laguna-s-2.1 (8.3s), nemotron-3-nano-omni-30b-a3b-reasoning (10.7s), gpt-oss-20b (12.2s). Free-tier rate limits (~20 req/min shared across all OpenRouter free users on that model) may cause 429 in high-traffic windows — usable for occasional notification translation, not for high-volume automation. recommended kept as llama-3.3-70b for capability/latency balance; llama-4-scout is a faster alternative worth considering as recommended after a broader release." + "_exclude": [ + "*/wizardlm-*", + "*/qwen*", + "*/yi-*", + "*/ernie-*", + "*/glm-*", + "*/deepseek*", + "*/hermes-*-405b*", + "*/dolphin*", + "*/euryale*", + "*/mythomax*", + "*/toppy*", + "*/rocinante*", + "*/nsfw*", + "*/*-uncensored*", + "*/*-vision*", + "*/*-image*", + "*/*-audio*", + "*/*-tts*", + "*/*-whisper*", + "*/*-embed*", + "openai/o1-*", + "openai/o3-*", + "openai/o4-*", + "anthropic/claude-3-*" + ], + "_note": "OpenRouter aggregates hundreds of models — `_exclude` is aggressive by design so the surfaced list stays curated. Blocked families: Chinese-first (Qwen, Yi, ERNIE, GLM, DeepSeek), role-play / uncensored, wrong-modality (vision/image/audio/tts/whisper/embed), reasoning models (o-series) and legacy Claude 3. Kept: the mainline chat/instruct models that were manually validated. Paid tier verified functionally 2026-07-14; free tier verified 2026-08-17. Free-tier rate limits (~20 req/min shared across all OpenRouter free users on that model) may cause 429 in high-traffic windows — usable for occasional notification translation, not for high-volume automation." }, "ollama": {