mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-09-14 18:56:52 +00:00
AI models catalog — _exclude list + preserve recommended
This commit is contained in:
MacRimi
@@ -2,18 +2,32 @@
|
||||
"""Apply a verifier report to ``AppImage/config/verified_ai_models.json``.
|
||||
|
||||
Reads the machine-readable report emitted by ``verify.py --json-out`` and
|
||||
merges the passing models into the on-disk catalog:
|
||||
merges the passing models into the on-disk catalog. Preserves the
|
||||
maintainer's editorial curation across three axes:
|
||||
|
||||
* Passing models per provider replace the existing ``models`` list.
|
||||
* ``recommended`` is set to the fastest passing model.
|
||||
* Existing ``_note`` / ``_deprecated`` / provider metadata is preserved
|
||||
when unchanged so the file's manual annotations survive the automated
|
||||
refresh.
|
||||
* Providers absent from the report (e.g. no API key configured in the
|
||||
GitHub Action for that run) are left untouched — the goal is
|
||||
additive maintenance, not silent removal.
|
||||
* ``_updated`` bumps to today's date only when the model set actually
|
||||
changes; a no-op run leaves the file byte-identical.
|
||||
* ``_exclude``: per-provider list of model IDs (exact match) that must
|
||||
never appear in the surfaced ``models`` list even when the verifier
|
||||
passes them. Meant for models that respond correctly to the technical
|
||||
test but are the wrong fit for notification translation — Arabic-only
|
||||
bases, Chinese-first fine-tunes, safety-classifier variants,
|
||||
agentic-only endpoints, legacy dated snapshots, etc.
|
||||
* ``recommended``: if the current recommendation is still in the
|
||||
passing (and non-excluded) set, it is preserved. Only when the
|
||||
previous recommendation disappears (deprecated upstream, or newly
|
||||
excluded) is a fallback chosen — the fastest passing model.
|
||||
* ``_note`` / ``_deprecated``: never touched. Those are maintainer
|
||||
annotations that outlive any single verifier run.
|
||||
|
||||
Fail-safe rules:
|
||||
* Providers absent from the report (no API key configured in the
|
||||
Action for that run) are left untouched.
|
||||
* Providers whose report carries an error are left untouched.
|
||||
* If the ``_exclude`` filter drops every passing model, the block is
|
||||
left untouched — an empty models list would silently kill the
|
||||
provider in the UI; keeping the previous list is more forgiving
|
||||
than shipping "nothing works".
|
||||
* ``_updated`` bumps to today's date only when the merge actually
|
||||
changed something. A no-op run leaves the file byte-identical.
|
||||
|
||||
Exits 0 when the file is unchanged, 10 when it was updated. The
|
||||
workflow uses that exit code to decide whether to commit.
|
||||
@@ -22,8 +36,8 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import fnmatch
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -41,9 +55,23 @@ def _save_json(path: Path, data: dict) -> None:
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _passing_models(provider_report: dict) -> list[str]:
|
||||
"""Return the passing models for one provider, fastest first."""
|
||||
passing = [r for r in provider_report.get("results", []) if r.get("verdict") == "pass"]
|
||||
def _is_excluded(model: str, patterns: list[str]) -> bool:
|
||||
"""Match a model against the ``_exclude`` list. Supports exact
|
||||
matches and shell-style globs (``gpt-4o-*``, ``*-2024-*``, ...) so
|
||||
a provider that periodically publishes dated snapshots can be
|
||||
covered by a single pattern instead of one entry per date."""
|
||||
for pat in patterns:
|
||||
if pat == model or fnmatch.fnmatchcase(model, pat):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _passing_models(provider_report: dict, exclude: list[str]) -> list[str]:
|
||||
"""Passing models minus the editorial exclusion list, fastest first."""
|
||||
passing = [
|
||||
r for r in provider_report.get("results", [])
|
||||
if r.get("verdict") == "pass" and not _is_excluded(r.get("model", ""), exclude)
|
||||
]
|
||||
passing.sort(key=lambda r: r.get("latency_s", 999))
|
||||
return [r["model"] for r in passing]
|
||||
|
||||
@@ -62,21 +90,32 @@ def apply_report(report_path: Path, catalog_path: Path, today: str) -> bool:
|
||||
print(f"[{name}] skipped — verifier reported error: {provider_report['error']}",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
passing = _passing_models(provider_report)
|
||||
|
||||
block = catalog.setdefault(name, {})
|
||||
exclude = list(block.get("_exclude", []))
|
||||
passing = _passing_models(provider_report, exclude)
|
||||
|
||||
if not passing:
|
||||
print(f"[{name}] no passing models this run — leaving catalog untouched",
|
||||
# Either the verifier returned no passes for this provider,
|
||||
# or every pass got filtered by _exclude. Both cases mean
|
||||
# "no signal we can trust to overwrite the curated list";
|
||||
# leaving the block alone is safer than blanking it.
|
||||
print(f"[{name}] skipped — no passing models after exclude filter",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
|
||||
block = catalog.setdefault(name, {})
|
||||
prev_models = list(block.get("models", []))
|
||||
prev_recommended = block.get("recommended", "")
|
||||
# Preserve the maintainer's choice of recommended when it is
|
||||
# still valid. Only fall back to fastest when the previous
|
||||
# value disappeared from the passing set.
|
||||
recommended = prev_recommended if prev_recommended in passing else passing[0]
|
||||
|
||||
if sorted(prev_models) != sorted(passing) or prev_recommended != passing[0]:
|
||||
if sorted(prev_models) != sorted(passing) or prev_recommended != recommended:
|
||||
block["models"] = passing
|
||||
block["recommended"] = passing[0]
|
||||
block["recommended"] = recommended
|
||||
changed = True
|
||||
print(f"[{name}] updated — {len(passing)} models, recommended={passing[0]}")
|
||||
print(f"[{name}] updated — {len(passing)} models, recommended={recommended}")
|
||||
else:
|
||||
print(f"[{name}] unchanged — {len(passing)} models")
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_description": "Verified AI models for ProxMenux notifications. Only models listed here will be shown to users. Models are tested to work with the chat/completions API format.",
|
||||
"_updated": "2026-07-14",
|
||||
"_verifier": "Refreshed with tools/ai-models-verifier (private). Re-run before each ProxMenux release to keep the list current. The verifier and ProxMenux share the same reasoning/thinking-model handlers so their verdicts stay aligned with runtime behaviour.",
|
||||
"_verifier": "Refreshed by .github/workflows/verify-ai-models.yml (daily). The workflow runs .github/scripts/ai-models-verifier/verify.py against every provider whose API key is configured in repository Secrets, then applies the report via apply.py — which honours per-provider `_exclude` lists so editorial curation survives automated refreshes. Manually re-run from the Actions tab (any branch) when a new model needs to be picked up out of cycle.",
|
||||
|
||||
"groq": {
|
||||
"models": [
|
||||
@@ -12,7 +12,15 @@
|
||||
"openai/gpt-oss-20b"
|
||||
],
|
||||
"recommended": "llama-3.3-70b-versatile",
|
||||
"_note": "Verified functionally 2026-07-14 with the Groq API (15 models discovered, 9 passed). Legacy llama-3.1-70b-versatile / llama3-70b-8192 / llama3-8b-8192 / mixtral-8x7b-32768 / gemma2-9b-it removed (retired upstream). llama-4-scout added (current-gen Llama 4, 0.47s). openai/gpt-oss-120b / gpt-oss-20b confirmed. Passing but excluded: allam-2-7b (Arabic-focused), qwen/qwen3-32b (Chinese-first, unreliable Spanish output), openai/gpt-oss-safeguard-20b (safety-classifier variant), groq/compound-mini (agentic system, wrong fit for notification translation)."
|
||||
"_exclude": [
|
||||
"allam-2-7b",
|
||||
"qwen/qwen3-32b",
|
||||
"qwen/qwen3*",
|
||||
"openai/gpt-oss-safeguard-*",
|
||||
"groq/compound",
|
||||
"groq/compound-*"
|
||||
],
|
||||
"_note": "Verified functionally 2026-07-14 with the Groq API. `_exclude` covers models that pass the technical test but are the wrong fit for notification translation: allam-2-7b (Arabic-focused), qwen/* (Chinese-first, unreliable Spanish), openai/gpt-oss-safeguard-* (safety-classifier variant), groq/compound* (agentic system, not a chat model)."
|
||||
},
|
||||
|
||||
"gemini": {
|
||||
@@ -25,8 +33,17 @@
|
||||
"gemini-3.5-flash"
|
||||
],
|
||||
"recommended": "gemini-2.5-flash-lite",
|
||||
"_note": "Verified 2026-07-13. gemini-flash-lite-latest now passes consistently (1.6s) and is fastest, but gemini-2.5-flash-lite remains recommended because 'latest' aliases can drift over time. gemini-3.1-flash-lite is the stable successor to 3-flash-preview. Pro variants continue to reject thinkingBudget=0 and are overkill for notification translation.",
|
||||
"_deprecated": ["gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-flash", "gemini-1.0-pro", "gemini-pro"]
|
||||
"_exclude": [
|
||||
"gemini-*-pro*",
|
||||
"gemini-*-thinking*",
|
||||
"gemini-embedding-*",
|
||||
"gemini-2.0-*",
|
||||
"gemini-1.5-*",
|
||||
"gemini-1.0-*",
|
||||
"gemini-pro"
|
||||
],
|
||||
"_deprecated": ["gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-flash", "gemini-1.0-pro", "gemini-pro"],
|
||||
"_note": "Verified 2026-07-13. gemini-flash-lite-latest now passes consistently (1.6s) and is fastest, but gemini-2.5-flash-lite remains recommended because 'latest' aliases can drift over time. gemini-3.1-flash-lite is the stable successor to 3-flash-preview. Pro variants continue to reject thinkingBudget=0 and are overkill for notification translation."
|
||||
},
|
||||
|
||||
"openai": {
|
||||
@@ -40,7 +57,37 @@
|
||||
"gpt-5-nano"
|
||||
],
|
||||
"recommended": "gpt-4.1-nano",
|
||||
"_note": "Verified 2026-07-13. gpt-5.4-nano / gpt-5.4-mini removed (HTTP 400 — provider params rejected). gpt-5-nano added (2.0s, current-gen fast). Reasoning models (o-series, gpt-5/5.1/5.2 non-chat variants) are supported by openai_provider.py via max_completion_tokens + reasoning_effort=minimal, but not listed here: their latency is higher and they do not improve translation quality for notifications. Add specific reasoning IDs to this list only if a user explicitly wants them."
|
||||
"_exclude": [
|
||||
"gpt-3.5-*",
|
||||
"gpt-3-*",
|
||||
"gpt-4",
|
||||
"gpt-4-0613",
|
||||
"gpt-4-turbo*",
|
||||
"gpt-4o-audio*",
|
||||
"gpt-4o-realtime*",
|
||||
"gpt-4o-search*",
|
||||
"gpt-4o-transcribe*",
|
||||
"gpt-4o-mini-audio*",
|
||||
"gpt-4o-mini-realtime*",
|
||||
"gpt-4o-mini-search*",
|
||||
"gpt-4o-mini-transcribe*",
|
||||
"gpt-4o-mini-tts",
|
||||
"gpt-4.1-nano-2*",
|
||||
"gpt-4.1-mini-2*",
|
||||
"gpt-4.1-2*",
|
||||
"gpt-4o-2*",
|
||||
"gpt-4o-mini-2*",
|
||||
"gpt-5-nano-2*",
|
||||
"gpt-5-mini-2*",
|
||||
"gpt-5-chat-2*",
|
||||
"gpt-5-2*",
|
||||
"o1-*-2*",
|
||||
"o3-*-2*",
|
||||
"o4-*-2*",
|
||||
"codex-*",
|
||||
"computer-use-*"
|
||||
],
|
||||
"_note": "Verified 2026-07-13. `_exclude` drops (a) dated snapshots (`gpt-4o-2024-11-20`, `gpt-4.1-nano-2025-04-14`, ...) — the stable aliases are preferred so the recommended model doesn't silently pin to a specific point-in-time build; (b) legacy families (gpt-3.5, gpt-4, gpt-4-turbo) that gpt-4.1 supersedes; (c) audio/realtime/search/transcribe/tts variants (wrong modality for notifications); (d) reasoning models (o-series, gpt-5.1/5.2 non-chat), which openai_provider.py supports via max_completion_tokens + reasoning_effort=minimal but do not improve translation quality and are slower. Add specific reasoning IDs to `models` manually if a user explicitly wants them."
|
||||
},
|
||||
|
||||
"anthropic": {
|
||||
@@ -53,7 +100,13 @@
|
||||
"claude-fable-5"
|
||||
],
|
||||
"recommended": "claude-haiku-4-5",
|
||||
"_note": "Verified 2026-07-13 with all 10 discovered models passing after aligning the verifier with anthropic_provider.py (temperature omitted — newest generations reject it with 'temperature is deprecated for this model'). Legacy claude-3-5-haiku-latest / claude-3-5-sonnet-latest / claude-3-opus-latest removed (deprecated upstream, not in the Models API). haiku-4-5 is the sweet spot for notification translation (3.6s, $1/$5 per MTok); sonnet-5 for slightly richer output (3.1s, $3/$15); opus-4-8 / fable-5 for demanding cases."
|
||||
"_exclude": [
|
||||
"claude-*-2*",
|
||||
"claude-3-*",
|
||||
"claude-3-5-*",
|
||||
"claude-3-opus-*"
|
||||
],
|
||||
"_note": "Verified 2026-07-13 with all 10 discovered models passing after aligning the verifier with anthropic_provider.py (temperature omitted — newest generations reject it with 'temperature is deprecated for this model'). `_exclude` drops dated snapshots (`claude-haiku-4-5-20251001`) and legacy generations (claude-3-*) that are deprecated upstream. haiku-4-5 is the sweet spot for notification translation (3.6s, $1/$5 per MTok); sonnet-5 for slightly richer output (3.1s, $3/$15); opus-4-8 / fable-5 for demanding cases."
|
||||
},
|
||||
|
||||
"openrouter": {
|
||||
@@ -77,7 +130,33 @@
|
||||
"openai/gpt-oss-20b:free"
|
||||
],
|
||||
"recommended": "meta-llama/llama-3.3-70b-instruct",
|
||||
"_note": "Paid tier verified functionally 2026-07-14 with the OpenRouter API — all 10 curated candidates pass the Spanish-translation notification test. Fastest: llama-4-scout (0.51s), gemini-2.5-flash-lite (1.14s), gemini-2.5-flash (1.94s), llama-3.3-70b-instruct (2.29s), claude-haiku-4.5 (2.71s). Free tier verified 2026-08-17 — 7 :free models pass and are appended, ordered by latency: nemotron-3-super-120b-a12b (3.5s), gemma-4-26b-a4b-it (4.1s), nemotron-nano-12b-v2-vl (5.3s), nemotron-3-nano-30b-a3b (5.8s), laguna-s-2.1 (8.3s), nemotron-3-nano-omni-30b-a3b-reasoning (10.7s), gpt-oss-20b (12.2s). Free-tier rate limits (~20 req/min shared across all OpenRouter free users on that model) may cause 429 in high-traffic windows — usable for occasional notification translation, not for high-volume automation. recommended kept as llama-3.3-70b for capability/latency balance; llama-4-scout is a faster alternative worth considering as recommended after a broader release."
|
||||
"_exclude": [
|
||||
"*/wizardlm-*",
|
||||
"*/qwen*",
|
||||
"*/yi-*",
|
||||
"*/ernie-*",
|
||||
"*/glm-*",
|
||||
"*/deepseek*",
|
||||
"*/hermes-*-405b*",
|
||||
"*/dolphin*",
|
||||
"*/euryale*",
|
||||
"*/mythomax*",
|
||||
"*/toppy*",
|
||||
"*/rocinante*",
|
||||
"*/nsfw*",
|
||||
"*/*-uncensored*",
|
||||
"*/*-vision*",
|
||||
"*/*-image*",
|
||||
"*/*-audio*",
|
||||
"*/*-tts*",
|
||||
"*/*-whisper*",
|
||||
"*/*-embed*",
|
||||
"openai/o1-*",
|
||||
"openai/o3-*",
|
||||
"openai/o4-*",
|
||||
"anthropic/claude-3-*"
|
||||
],
|
||||
"_note": "OpenRouter aggregates hundreds of models — `_exclude` is aggressive by design so the surfaced list stays curated. Blocked families: Chinese-first (Qwen, Yi, ERNIE, GLM, DeepSeek), role-play / uncensored, wrong-modality (vision/image/audio/tts/whisper/embed), reasoning models (o-series) and legacy Claude 3. Kept: the mainline chat/instruct models that were manually validated. Paid tier verified functionally 2026-07-14; free tier verified 2026-08-17. Free-tier rate limits (~20 req/min shared across all OpenRouter free users on that model) may cause 429 in high-traffic windows — usable for occasional notification translation, not for high-volume automation."
|
||||
},
|
||||
|
||||
"ollama": {
|
||||
|
||||
Reference in New Issue
Block a user