diff --git a/AppImage/config/verified_ai_models.json b/AppImage/config/verified_ai_models.json index 767e3bea..a758b925 100644 --- a/AppImage/config/verified_ai_models.json +++ b/AppImage/config/verified_ai_models.json @@ -1,69 +1,56 @@ { "_description": "Verified AI models for ProxMenux notifications. Only models listed here will be shown to users. Models are tested to work with the chat/completions API format.", "_updated": "2026-09-02", - "_verifier": "Refreshed by .github/workflows/verify-ai-models.yml (daily). The workflow runs .github/scripts/ai-models-verifier/verify.py against every provider whose API key is configured in repository Secrets, then applies the report via apply.py — which honours per-provider `_exclude` lists so editorial curation survives automated refreshes. Manually re-run from the Actions tab (any branch) when a new model needs to be picked up out of cycle.", + "_verifier": "Refreshed by .github/workflows/verify-ai-models.yml (daily). The workflow runs .github/scripts/ai-models-verifier/verify.py against every provider whose API key is configured in repository Secrets, then applies the report via apply.py — which honours per-provider `_exclude` lists. `_exclude` is intentionally minimal: it only drops models that are technically incapable of generating a chat completion for a normal prompt (safety classifiers, agentic-only endpoints, meta-routers, wrong modalities). Everything else the verifier passes is surfaced — including language-specialised models (Arabic, Chinese, ...), reasoning models, legacy families and dated snapshots — so a user with a specific need can still pick the model that fits. Manually re-run from the Actions tab (any branch) when a new model needs to be picked up out of cycle.", + "groq": { "models": [ - "openai/gpt-oss-20b", - "openai/gpt-oss-120b" + "llama-3.3-70b-versatile", + "llama-3.1-8b-instant", + "meta-llama/llama-4-scout-17b-16e-instruct", + "openai/gpt-oss-120b", + "openai/gpt-oss-20b" ], - "recommended": "openai/gpt-oss-20b", + "recommended": "llama-3.3-70b-versatile", "_exclude": [ - "allam-2-7b", - "qwen/qwen3-32b", - "qwen/qwen3*", "openai/gpt-oss-safeguard-*", "groq/compound", "groq/compound-*" ], - "_note": "Verified functionally 2026-07-14 with the Groq API. `_exclude` covers models that pass the technical test but are the wrong fit for notification translation: allam-2-7b (Arabic-focused), qwen/* (Chinese-first, unreliable Spanish), openai/gpt-oss-safeguard-* (safety-classifier variant), groq/compound* (agentic system, not a chat model)." + "_note": "`_exclude` covers models the verifier may technically pass but that do not produce a usable chat completion: openai/gpt-oss-safeguard-* is a safety classifier (returns a category, not free text); groq/compound* is an agentic system that expects multi-step tool use, not a plain prompt." }, + "gemini": { "models": [ + "gemini-flash-lite-latest", "gemini-2.5-flash-lite", "gemini-2.5-flash", "gemini-3.1-flash-lite", - "gemini-3.1-flash-lite-preview", - "gemini-flash-lite-latest", - "gemini-3.5-flash-lite", "gemini-3-flash-preview", "gemini-3.5-flash" ], "recommended": "gemini-2.5-flash-lite", "_exclude": [ - "gemini-*-pro*", - "gemini-*-thinking*", "gemini-embedding-*", - "gemini-2.0-*", - "gemini-1.5-*", - "gemini-1.0-*", - "gemini-pro" + "gemini-*-pro*", + "gemini-*-thinking*" ], - "_deprecated": [ - "gemini-2.0-flash", - "gemini-2.0-flash-lite", - "gemini-1.5-flash", - "gemini-1.0-pro", - "gemini-pro" - ], - "_note": "Verified 2026-07-13. gemini-flash-lite-latest now passes consistently (1.6s) and is fastest, but gemini-2.5-flash-lite remains recommended because 'latest' aliases can drift over time. gemini-3.1-flash-lite is the stable successor to 3-flash-preview. Pro variants continue to reject thinkingBudget=0 and are overkill for notification translation." + "_deprecated": ["gemini-2.0-flash", "gemini-2.0-flash-lite", "gemini-1.5-flash", "gemini-1.0-pro", "gemini-pro"], + "_note": "`_exclude` drops embeddings (wrong modality) and Pro / thinking variants that reject `thinkingConfig.thinkingBudget: 0` and therefore never return a visible completion within a reasonable token budget — technical failure with our current provider config." }, + "openai": { "models": [ "gpt-4.1-nano", - "gpt-4.1", - "gpt-5-nano", "gpt-4.1-mini", "gpt-4o-mini", - "gpt-4o" + "gpt-4.1", + "gpt-4o", + "gpt-5-chat-latest", + "gpt-5-nano" ], "recommended": "gpt-4.1-nano", "_exclude": [ - "gpt-3.5-*", - "gpt-3-*", - "gpt-4", - "gpt-4-0613", - "gpt-4-turbo*", "gpt-4o-audio*", "gpt-4o-realtime*", "gpt-4o-search*", @@ -73,77 +60,60 @@ "gpt-4o-mini-search*", "gpt-4o-mini-transcribe*", "gpt-4o-mini-tts", - "gpt-4.1-nano-2*", - "gpt-4.1-mini-2*", - "gpt-4.1-2*", - "gpt-4o-2*", - "gpt-4o-mini-2*", - "gpt-5-nano-2*", - "gpt-5-mini-2*", - "gpt-5-chat-2*", - "gpt-5-2*", - "o1-*-2*", - "o3-*-2*", - "o4-*-2*", - "codex-*", "computer-use-*" ], - "_note": "Verified 2026-07-13. `_exclude` drops (a) dated snapshots (`gpt-4o-2024-11-20`, `gpt-4.1-nano-2025-04-14`, ...) — the stable aliases are preferred so the recommended model doesn't silently pin to a specific point-in-time build; (b) legacy families (gpt-3.5, gpt-4, gpt-4-turbo) that gpt-4.1 supersedes; (c) audio/realtime/search/transcribe/tts variants (wrong modality for notifications); (d) reasoning models (o-series, gpt-5.1/5.2 non-chat), which openai_provider.py supports via max_completion_tokens + reasoning_effort=minimal but do not improve translation quality and are slower. Add specific reasoning IDs to `models` manually if a user explicitly wants them." + "_note": "`_exclude` covers wrong-modality variants (audio, realtime, search, transcribe, tts) that cannot handle a plain notification-translation prompt, plus computer-use which requires an agent loop. All other OpenAI chat/completion models are surfaced — including legacy families (gpt-3.5, gpt-4), reasoning models (o-series, gpt-5.x non-chat) and dated snapshots — so users can pick by their own criteria (cost, quality, reproducibility). openai_provider.py already handles reasoning models via max_completion_tokens + reasoning_effort=minimal." }, + "anthropic": { "models": [ - "claude-opus-4-7", - "claude-opus-4-8", + "claude-haiku-4-5", "claude-sonnet-5", + "claude-opus-4-8", "claude-sonnet-4-6", - "claude-fable-5", - "claude-opus-4-6" + "claude-opus-4-6", + "claude-fable-5" ], - "recommended": "claude-opus-4-7", - "_exclude": [ - "claude-*-2*", - "claude-3-*", - "claude-3-5-*", - "claude-3-opus-*" - ], - "_note": "Verified 2026-07-13 with all 10 discovered models passing after aligning the verifier with anthropic_provider.py (temperature omitted — newest generations reject it with 'temperature is deprecated for this model'). `_exclude` drops dated snapshots (`claude-haiku-4-5-20251001`) and legacy generations (claude-3-*) that are deprecated upstream. haiku-4-5 is the sweet spot for notification translation (3.6s, $1/$5 per MTok); sonnet-5 for slightly richer output (3.1s, $3/$15); opus-4-8 / fable-5 for demanding cases." + "recommended": "claude-haiku-4-5", + "_exclude": [], + "_note": "No technical exclusions — every Claude generation returns free-text completions for a plain prompt. The verifier tests each model listed under `models`; if a specific ID stops working upstream it simply drops out of the passing set. Anthropic does not expose a public models-list API, so new models must be added to `models` manually before the verifier can test them." }, + "openrouter": { "models": [ - "openrouter/free", + "meta-llama/llama-3.3-70b-instruct", + "meta-llama/llama-3.1-70b-instruct", + "meta-llama/llama-3.1-8b-instruct", + "meta-llama/llama-4-scout", + "anthropic/claude-haiku-4.5", + "anthropic/claude-sonnet-4.6", + "google/gemini-2.5-flash-lite", + "google/gemini-2.5-flash", + "openai/gpt-4o-mini", + "mistralai/mistral-small-3.2-24b-instruct", "nvidia/nemotron-3-super-120b-a12b:free", - "minimax/minimax-m3:free", - "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free" + "google/gemma-4-26b-a4b-it:free", + "nvidia/nemotron-nano-12b-v2-vl:free", + "nvidia/nemotron-3-nano-30b-a3b:free", + "poolside/laguna-s-2.1:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "openai/gpt-oss-20b:free" ], - "recommended": "openrouter/free", + "recommended": "meta-llama/llama-3.3-70b-instruct", "_exclude": [ - "*/wizardlm-*", - "*/qwen*", - "*/yi-*", - "*/ernie-*", - "*/glm-*", - "*/deepseek*", - "*/hermes-*-405b*", - "*/dolphin*", - "*/euryale*", - "*/mythomax*", - "*/toppy*", - "*/rocinante*", - "*/nsfw*", - "*/*-uncensored*", - "*/*-vision*", - "*/*-image*", + "openrouter/*", "*/*-audio*", + "*/*-audio-*", "*/*-tts*", "*/*-whisper*", "*/*-embed*", - "openai/o1-*", - "openai/o3-*", - "openai/o4-*", - "anthropic/claude-3-*" + "*/*-embedding*", + "*/*-image*", + "*/*-vision-only*" ], - "_note": "OpenRouter aggregates hundreds of models — `_exclude` is aggressive by design so the surfaced list stays curated. Blocked families: Chinese-first (Qwen, Yi, ERNIE, GLM, DeepSeek), role-play / uncensored, wrong-modality (vision/image/audio/tts/whisper/embed), reasoning models (o-series) and legacy Claude 3. Kept: the mainline chat/instruct models that were manually validated. Paid tier verified functionally 2026-07-14; free tier verified 2026-08-17. Free-tier rate limits (~20 req/min shared across all OpenRouter free users on that model) may cause 429 in high-traffic windows — usable for occasional notification translation, not for high-volume automation." + "_note": "OpenRouter aggregates hundreds of models; the free-tier variants (:free suffix) are intentionally supported per user request and never blocked. `_exclude` covers only meta-routers (`openrouter/free`, `openrouter/auto` — they route dynamically to something else, so their behaviour is not the model the user picked) and wrong-modality models (audio, tts, whisper, embeddings, image, vision-only). Everything else — chat models across every family, language and price tier — is surfaced so the user can pick the fit." }, + "ollama": { "_note": "Ollama models are local, we don't filter them. User manages their own models.", "models": [],