Merge pull request #320 from MacRimi/develop

New version 1.2.5

.github/scripts/ai-models-verifier/.env.example

@@ -0,0 +1,16 @@
+# ProxMenux AI-model verifier — API keys
+# Copy this file to .env (gitignored) and fill only the ones you have.
+
+OPENAI_API_KEY=
+# Optional: for LiteLLM/MLX/LM Studio/vLLM/LocalAI/Ollama-proxy testing,
+# point OPENAI_API_KEY to any non-empty placeholder and set OPENAI_BASE_URL
+# to the endpoint's root (without /v1 — the tool appends it).
+# OPENAI_BASE_URL=http://localhost:4000
+
+GROQ_API_KEY=
+
+GEMINI_API_KEY=
+
+ANTHROPIC_API_KEY=
+
+OPENROUTER_API_KEY=

.github/scripts/ai-models-verifier/README.md

@@ -0,0 +1,37 @@
+# AI models verifier (public copy)
+
+Standalone verifier used by the daily GitHub Action to refresh
+`AppImage/config/verified_ai_models.json`.
+
+The code lives here so the Action can execute it. API keys are read from
+GitHub Secrets at run time and never written to disk.
+
+Local dev runs (interactive verifier over your own keys) can keep using
+the private copy — `verify.py` is identical.
+
+## What the Action does
+
+Each run:
+
+1. Loads keys from Secrets into environment variables.
+2. Runs `verify.py --json-out /tmp/report.json` against every provider that
+   has a key set.
+3. Rewrites `AppImage/config/verified_ai_models.json` with the passing
+   models, sorted with the recommended one first per provider.
+4. Bumps the `_updated` field to the current date.
+5. If the file changed, commits directly to `main` as a bot commit.
+
+## Adding provider keys
+
+- Repository → Settings → Secrets and variables → Actions.
+- Add each key with the exact name expected by `verify.py`:
+  `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GROQ_API_KEY`, `GEMINI_API_KEY`,
+  `OPENROUTER_API_KEY`.
+- Any provider without a key is silently skipped — the Action logs a
+  warning and continues with the rest.
+
+## Running the Action on demand
+
+The workflow accepts `workflow_dispatch`, so you can trigger a refresh
+manually from the Actions tab. Useful when a new model has just been
+released upstream and you don't want to wait for the daily cron.

.github/scripts/ai-models-verifier/apply.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Apply a verifier report to ``AppImage/config/verified_ai_models.json``.
+
+Reads the machine-readable report emitted by ``verify.py --json-out`` and
+merges the passing models into the on-disk catalog:
+
+* Passing models per provider replace the existing ``models`` list.
+* ``recommended`` is set to the fastest passing model.
+* Existing ``_note`` / ``_deprecated`` / provider metadata is preserved
+  when unchanged so the file's manual annotations survive the automated
+  refresh.
+* Providers absent from the report (e.g. no API key configured in the
+  GitHub Action for that run) are left untouched — the goal is
+  additive maintenance, not silent removal.
+* ``_updated`` bumps to today's date only when the model set actually
+  changes; a no-op run leaves the file byte-identical.
+
+Exits 0 when the file is unchanged, 10 when it was updated. The
+workflow uses that exit code to decide whether to commit.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def _load_json(path: Path) -> dict:
+    with open(path, "r", encoding="utf-8") as fh:
+        return json.load(fh)
+
+
+def _save_json(path: Path, data: dict) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    with open(tmp, "w", encoding="utf-8") as fh:
+        json.dump(data, fh, indent=2, ensure_ascii=False)
+        fh.write("\n")
+    tmp.replace(path)
+
+
+def _passing_models(provider_report: dict) -> list[str]:
+    """Return the passing models for one provider, fastest first."""
+    passing = [r for r in provider_report.get("results", []) if r.get("verdict") == "pass"]
+    passing.sort(key=lambda r: r.get("latency_s", 999))
+    return [r["model"] for r in passing]
+
+
+def apply_report(report_path: Path, catalog_path: Path, today: str) -> bool:
+    """Rewrite the catalog from the report. Returns True if it changed."""
+    report = _load_json(report_path)
+    catalog = _load_json(catalog_path) if catalog_path.exists() else {}
+
+    changed = False
+    for provider_report in report:
+        name = provider_report.get("provider")
+        if not name:
+            continue
+        if provider_report.get("error"):
+            print(f"[{name}] skipped — verifier reported error: {provider_report['error']}",
+                  file=sys.stderr)
+            continue
+        passing = _passing_models(provider_report)
+        if not passing:
+            print(f"[{name}] no passing models this run — leaving catalog untouched",
+                  file=sys.stderr)
+            continue
+
+        block = catalog.setdefault(name, {})
+        prev_models = list(block.get("models", []))
+        prev_recommended = block.get("recommended", "")
+
+        if sorted(prev_models) != sorted(passing) or prev_recommended != passing[0]:
+            block["models"] = passing
+            block["recommended"] = passing[0]
+            changed = True
+            print(f"[{name}] updated — {len(passing)} models, recommended={passing[0]}")
+        else:
+            print(f"[{name}] unchanged — {len(passing)} models")
+
+    if changed:
+        catalog["_updated"] = today
+        _save_json(catalog_path, catalog)
+    return changed
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--report", required=True, help="verify.py --json-out path")
+    ap.add_argument("--catalog", required=True,
+                    help="AppImage/config/verified_ai_models.json path")
+    ap.add_argument("--today", default=None,
+                    help="Override the date written into _updated (YYYY-MM-DD).")
+    args = ap.parse_args()
+
+    today = args.today or dt.datetime.utcnow().strftime("%Y-%m-%d")
+    changed = apply_report(Path(args.report), Path(args.catalog), today)
+    return 10 if changed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/ai-models-verifier/prompts.py

@@ -0,0 +1,50 @@
+"""Standardized test prompt for ProxMenux AI-model verification.
+
+Mirrors the real AI-enrichment use case: take a raw Proxmox system
+notification (English, with technical identifiers), translate it into
+Spanish, explain in plain terms, and suggest one concrete action. It is
+intentionally simple — if a model can't do this, it won't do the real
+thing either. Models that pass this test are fine for inclusion in
+verified_ai_models.json.
+"""
+
+SYSTEM_PROMPT = (
+    "You are a Proxmox system-notification assistant. "
+    "When given a raw notification from a Proxmox host, you: "
+    "(1) translate it into Spanish, "
+    "(2) explain in 2-3 sentences what the user is seeing and the likely cause, "
+    "(3) suggest ONE concrete next action. "
+    "Keep technical identifiers (device paths like /dev/sdd, SMART keywords, "
+    "ata port numbers, BDFs) in their original form. "
+    "Respond only in Spanish. Stay under 200 tokens total."
+)
+
+# Realistic ProxMenux notification payload: multi-line body with
+# SMART/ATA vocabulary and a frequency hint — the exact shape the real
+# pipeline emits.
+USER_MESSAGE = (
+    "Event: disk_io_error\n"
+    "Severity: CRITICAL\n"
+    "Host: pve-constructor\n"
+    "Device: /dev/sdd\n"
+    "SMART status: PASSED\n"
+    "Summary: 3 I/O event(s) in 5 minutes, disk passed SMART short test\n"
+    "Sample kernel line: ata4.00: exception Emask 0x0 SAct 0x804000 SErr 0x0 action 0x6\n"
+    "Frequency: 3 occurrences in 24h, first seen 6h ago"
+)
+
+# Common Spanish stopwords. A response missing ALL of these is almost
+# certainly not Spanish (or empty/truncated). Cheap heuristic, good
+# enough for a coarse pass/fail.
+REQUIRED_SPANISH_HINTS = [
+    " el ", " la ", " los ", " las ", " un ", " una ",
+    " de ", " del ", " en ", " con ", " que ", " para ",
+    " es ", " se ", " ha ", " por ", " y ",
+]
+
+# Domain keywords — at least one must appear to confirm the model
+# actually engaged with the notification instead of replying generically.
+DOMAIN_HINTS = [
+    "disco", "sdd", "smart", "ata", "i/o", "e/s", "error",
+    "kernel", "proxmox",
+]

.github/scripts/ai-models-verifier/providers.py

@@ -0,0 +1,263 @@
+"""Self-contained API wrappers for AI-model verification.
+
+Kept independent from the ProxMenux AppImage's ai_providers module so
+this tool can live in a private repo with no import coupling to the
+public project. Uses only the Python standard library.
+"""
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from typing import List, Optional
+
+
+class ProviderError(Exception):
+    pass
+
+
+class Provider:
+    """Base class. Subclasses implement list_models() and generate()."""
+    name = "base"
+
+    def __init__(self, api_key: str, base_url: Optional[str] = None):
+        self.api_key = api_key
+        self.base_url = base_url
+
+    def list_models(self) -> List[str]:
+        raise NotImplementedError
+
+    def generate(self, model: str, system: str, user: str,
+                 max_tokens: int = 250, timeout: int = 30) -> str:
+        raise NotImplementedError
+
+    # ── HTTP helpers ────────────────────────────────────────────
+
+    # Cloudflare in front of api.groq.com (and probably other providers
+    # over time) returns 403 "error code: 1010" for the default
+    # `Python-urllib/3.x` User-Agent — the "browser signature ban" rule.
+    # A plain identifier is enough to get through; we're not spoofing a
+    # browser, just avoiding a naive UA fingerprint match.
+    _USER_AGENT = "ProxMenux-AI-Verifier/1.0"
+
+    def _post_json(self, url: str, payload: dict, headers: dict,
+                   timeout: int = 30) -> dict:
+        data = json.dumps(payload).encode("utf-8")
+        headers = {**headers, "User-Agent": self._USER_AGENT}
+        req = urllib.request.Request(url, data=data, headers=headers, method="POST")
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+
+    def _get_json(self, url: str, headers: dict, timeout: int = 30) -> dict:
+        headers = {**headers, "User-Agent": self._USER_AGENT}
+        req = urllib.request.Request(url, headers=headers, method="GET")
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+
+
+# ══════════════════════════════════════════════════════════════════
+# OpenAI-compatible family (OpenAI, Groq, OpenRouter, LiteLLM, LM
+# Studio, vLLM, LocalAI, etc.). They all speak the same endpoints.
+# ══════════════════════════════════════════════════════════════════
+
+class OpenAICompatProvider(Provider):
+    name = "openai-compat"
+    default_base = "https://api.openai.com"
+    models_path = "/v1/models"
+    chat_path = "/v1/chat/completions"
+
+    def _base(self) -> str:
+        return (self.base_url or self.default_base).rstrip("/")
+
+    @staticmethod
+    def _is_reasoning_model(model: str) -> bool:
+        """True for OpenAI reasoning models (o-series + non-chat gpt-5+).
+
+        Must be kept in sync with the matching helper in ProxMenux's
+        openai_provider.py — same rule, same consequence:
+          - send max_completion_tokens instead of max_tokens
+          - omit temperature (default is the only accepted value).
+        """
+        m = model.lower()
+        if len(m) >= 2 and m[0] == "o" and m[1].isdigit():
+            return True
+        if m.startswith("gpt-5") and "-chat" not in m:
+            return True
+        return False
+
+    def list_models(self) -> List[str]:
+        url = f"{self._base()}{self.models_path}"
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+        data = self._get_json(url, headers)
+        return [m.get("id", "") for m in data.get("data", []) if m.get("id")]
+
+    def generate(self, model: str, system: str, user: str,
+                 max_tokens: int = 250, timeout: int = 30) -> str:
+        url = f"{self._base()}{self.chat_path}"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        payload = {
+            "model": model,
+            "messages": [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+        }
+        if self._is_reasoning_model(model):
+            # Reasoning models spend budget on internal reasoning by
+            # default, which yields empty replies at small max_tokens.
+            # reasoning_effort=minimal keeps that overhead low so the
+            # whole budget reaches the user, aligned with the short
+            # translate+explain task ProxMenux uses. Mirror this in
+            # ProxMenux's openai_provider.py.
+            payload["max_completion_tokens"] = max_tokens
+            payload["reasoning_effort"] = "minimal"
+        else:
+            payload["max_tokens"] = max_tokens
+            payload["temperature"] = 0.3
+        data = self._post_json(url, payload, headers, timeout)
+        try:
+            return data["choices"][0]["message"]["content"].strip()
+        except (KeyError, IndexError) as exc:
+            raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
+
+
+class OpenAIProvider(OpenAICompatProvider):
+    name = "openai"
+    default_base = "https://api.openai.com"
+
+
+class GroqProvider(OpenAICompatProvider):
+    name = "groq"
+    default_base = "https://api.groq.com/openai"
+
+
+class OpenRouterProvider(OpenAICompatProvider):
+    name = "openrouter"
+    default_base = "https://openrouter.ai/api"
+
+
+# ══════════════════════════════════════════════════════════════════
+# Gemini — different endpoint shape, keyed via ?key=... query param.
+# ══════════════════════════════════════════════════════════════════
+
+class GeminiProvider(Provider):
+    name = "gemini"
+    default_base = "https://generativelanguage.googleapis.com/v1beta"
+
+    @staticmethod
+    def _has_thinking_mode(model: str) -> bool:
+        """True for Gemini variants that enable "thinking" by default.
+
+        Kept in sync with ProxMenux's gemini_provider.py. 2.5+ pro/flash
+        and 3.x pro/flash consume output tokens on reasoning, which
+        yields empty replies when max_tokens is small. We pass
+        thinkingBudget=0 to disable thinking so the short translate+
+        explain test sees actual text. Lite variants don't have thinking
+        enabled and are not flagged here.
+        """
+        m = model.lower()
+        if "lite" in m:
+            return False
+        return m.startswith("gemini-2.5") or m.startswith("gemini-3")
+
+    def list_models(self) -> List[str]:
+        url = f"{self.default_base}/models?key={self.api_key}"
+        data = self._get_json(url, {})
+        names = []
+        for m in data.get("models", []):
+            raw = m.get("name", "")
+            if raw.startswith("models/"):
+                raw = raw[len("models/"):]
+            # Only keep text-generation capable models.
+            if "generateContent" in m.get("supportedGenerationMethods", []):
+                names.append(raw)
+        return names
+
+    def generate(self, model: str, system: str, user: str,
+                 max_tokens: int = 250, timeout: int = 30) -> str:
+        url = f"{self.default_base}/models/{model}:generateContent?key={self.api_key}"
+        gen_config = {
+            "maxOutputTokens": max_tokens,
+            "temperature": 0.3,
+        }
+        if self._has_thinking_mode(model):
+            gen_config["thinkingConfig"] = {"thinkingBudget": 0}
+        payload = {
+            "system_instruction": {"parts": [{"text": system}]},
+            "contents": [{"parts": [{"text": user}]}],
+            "generationConfig": gen_config,
+        }
+        data = self._post_json(url, payload, {"Content-Type": "application/json"}, timeout)
+        try:
+            return data["candidates"][0]["content"]["parts"][0]["text"].strip()
+        except (KeyError, IndexError) as exc:
+            raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
+
+
+# ══════════════════════════════════════════════════════════════════
+# Anthropic — own schema and headers.
+# ══════════════════════════════════════════════════════════════════
+
+class AnthropicProvider(Provider):
+    name = "anthropic"
+    default_base = "https://api.anthropic.com"
+    version = "2023-06-01"
+
+    def list_models(self) -> List[str]:
+        url = f"{self.default_base}/v1/models"
+        headers = {
+            "x-api-key": self.api_key,
+            "anthropic-version": self.version,
+        }
+        try:
+            data = self._get_json(url, headers)
+            return [m.get("id", "") for m in data.get("data", []) if m.get("id")]
+        except urllib.error.HTTPError:
+            # Older keys don't have the endpoint; caller decides what to do.
+            return []
+
+    def generate(self, model: str, system: str, user: str,
+                 max_tokens: int = 250, timeout: int = 30) -> str:
+        url = f"{self.default_base}/v1/messages"
+        headers = {
+            "x-api-key": self.api_key,
+            "anthropic-version": self.version,
+            "Content-Type": "application/json",
+        }
+        # ProxMenux's real anthropic_provider.py does NOT send `temperature`,
+        # and Anthropic's newest generation (Claude Sonnet 5, Opus 4.7 /
+        # 4.8, Fable 5) now rejects it with:
+        #   invalid_request_error: `temperature` is deprecated for this model.
+        # Omitting it here aligns the verifier with production behaviour so
+        # the newest-gen models can be reached during verification.
+        payload = {
+            "model": model,
+            "max_tokens": max_tokens,
+            "system": system,
+            "messages": [{"role": "user", "content": user}],
+        }
+        data = self._post_json(url, payload, headers, timeout)
+        try:
+            return data["content"][0]["text"].strip()
+        except (KeyError, IndexError) as exc:
+            raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
+
+
+PROVIDERS = {
+    "openai": OpenAIProvider,
+    "groq": GroqProvider,
+    "gemini": GeminiProvider,
+    "anthropic": AnthropicProvider,
+    "openrouter": OpenRouterProvider,
+}
+
+
+def make_provider(name: str, api_key: str,
+                  base_url: Optional[str] = None) -> Provider:
+    cls = PROVIDERS.get(name)
+    if not cls:
+        raise ValueError(f"unknown provider: {name}")
+    return cls(api_key, base_url=base_url)

.github/scripts/ai-models-verifier/verify.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""ProxMenux AI-model verifier.
+
+Runs a standardized translate+explain test against every model each
+provider currently advertises, and emits a per-model verdict so the
+verified_ai_models.json list can be refreshed with confidence.
+
+Not packaged with the AppImage — keep this in a private repo alongside
+the API keys.
+
+Usage:
+    cp .env.example .env                       # fill in the API keys you have
+    python3 verify.py                          # test all providers with keys
+    python3 verify.py --provider groq          # just one
+    python3 verify.py --provider openai --limit 5    # only first 5 models
+    python3 verify.py --json-out report.json   # machine-readable output too
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from prompts import DOMAIN_HINTS, REQUIRED_SPANISH_HINTS, SYSTEM_PROMPT, USER_MESSAGE
+from providers import PROVIDERS, make_provider
+
+
+# Non-chat model name patterns. Skipping these saves test time and keeps
+# the report focused on models that could actually serve notifications.
+SKIP_PATTERNS = (
+    "embedding", "whisper", "tts", "dall-e", "dalle", "image",
+    "realtime", "audio", "moderation", "search",
+    "code-search", "text-similarity", "babbage", "davinci",
+    "curie", "ada", "transcribe",
+)
+
+
+def load_env(env_path: Path) -> Dict[str, str]:
+    """Minimal .env loader (avoids a python-dotenv dependency)."""
+    if not env_path.exists():
+        return {}
+    env: Dict[str, str] = {}
+    for line in env_path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        env[k.strip()] = v.strip().strip('"').strip("'")
+    return env
+
+
+def should_skip_model(model: str) -> bool:
+    m = model.lower()
+    return any(p in m for p in SKIP_PATTERNS)
+
+
+def assess_response(text: str) -> Tuple[str, List[str]]:
+    """Classify the model output. Returns (verdict, reasons).
+
+    verdict is one of:
+      - 'pass':  Spanish, on-topic, reasonable length
+      - 'warn':  responded but one heuristic failed (borderline)
+      - 'fail':  empty, wrong language, or off-topic
+    """
+    reasons: List[str] = []
+    if not text or len(text) < 30:
+        return "fail", ["empty or too short response"]
+
+    text_low = " " + text.lower() + " "
+    spanish_hits = sum(1 for h in REQUIRED_SPANISH_HINTS if h in text_low)
+    domain_hits = sum(1 for h in DOMAIN_HINTS if h.lower() in text_low)
+
+    if spanish_hits < 3:
+        reasons.append(f"not Spanish ({spanish_hits}/{len(REQUIRED_SPANISH_HINTS)} hints)")
+    if domain_hits < 1:
+        reasons.append("did not engage with the domain")
+    if len(text) > 1500:
+        reasons.append("response unusually long")
+
+    if not reasons:
+        return "pass", []
+    # Responded and engaged, but one signal missed → warn (keep in list
+    # with a caveat; don't auto-include).
+    if domain_hits >= 1 and spanish_hits >= 1:
+        return "warn", reasons
+    return "fail", reasons
+
+
+def run_model(provider, model: str, timeout: int) -> dict:
+    t0 = time.time()
+    try:
+        out = provider.generate(
+            model, SYSTEM_PROMPT, USER_MESSAGE,
+            max_tokens=250, timeout=timeout,
+        )
+        latency = time.time() - t0
+        verdict, reasons = assess_response(out)
+        return {
+            "model": model,
+            "verdict": verdict,
+            "latency_s": round(latency, 2),
+            "reasons": reasons,
+            "sample": (out[:140] if out else "").replace("\n", " "),
+            "error": None,
+        }
+    except Exception as exc:  # HTTPError, ProviderError, timeouts
+        return {
+            "model": model,
+            "verdict": "fail",
+            "latency_s": round(time.time() - t0, 2),
+            "reasons": [],
+            "sample": "",
+            "error": str(exc)[:200],
+        }
+
+
+def tag(verdict: str) -> str:
+    return {"pass": "✓", "warn": "⚠", "fail": "✗"}.get(verdict, "?")
+
+
+def run_provider(name: str, api_key: str, base_url: Optional[str],
+                 timeout: int, limit: Optional[int]) -> dict:
+    print(f"\n=== {name} ===")
+    try:
+        provider = make_provider(name, api_key, base_url=base_url)
+        models = provider.list_models()
+    except Exception as exc:
+        print(f"  list_models() failed: {exc}")
+        return {"provider": name, "error": str(exc), "results": []}
+
+    models = [m for m in models if not should_skip_model(m)]
+    if limit:
+        models = models[:limit]
+    if not models:
+        print("  (no eligible models)")
+        return {"provider": name, "error": None, "results": []}
+
+    print(f"  discovered {len(models)} model(s)")
+    results: List[dict] = []
+    for m in models:
+        r = run_model(provider, m, timeout)
+        results.append(r)
+        latency = f"{r['latency_s']}s"
+        if r["error"]:
+            suffix = f" — {r['error']}"
+        elif r["reasons"]:
+            suffix = f" — {'; '.join(r['reasons'])}"
+        else:
+            suffix = ""
+        print(f"  {tag(r['verdict'])} {m:<50} {latency:>6}{suffix}")
+
+    return {"provider": name, "error": None, "results": results}
+
+
+def summarize(all_results: List[dict]) -> None:
+    print("\n" + "=" * 64)
+    print("Suggested verified_ai_models.json entries (passing models only)")
+    print("=" * 64)
+    any_output = False
+    for pr in all_results:
+        if pr.get("error"):
+            continue
+        passed = [r for r in pr["results"] if r["verdict"] == "pass"]
+        if not passed:
+            continue
+        any_output = True
+        passed_sorted = sorted(passed, key=lambda x: x["latency_s"])
+        print(f'\n  "{pr["provider"]}": {{')
+        print('    "models": [')
+        for r in passed_sorted:
+            print(f'      "{r["model"]}",')
+        print("    ],")
+        print(f'    "recommended": "{passed_sorted[0]["model"]}"')
+        print("  },")
+
+    warn_total = sum(
+        1 for pr in all_results for r in pr["results"]
+        if r["verdict"] == "warn"
+    )
+    if warn_total:
+        print(f"\n  Note: {warn_total} model(s) came back as ⚠ (warn) — review those manually.")
+    if not any_output:
+        print("\n  (no models passed; check keys and network)")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(
+        description="Verify AI models for ProxMenux notification enrichment."
+    )
+    ap.add_argument("--env", default=".env", help=".env file path")
+    ap.add_argument("--provider", action="append", default=[],
+                    help="run a specific provider (repeat to add more)")
+    ap.add_argument("--timeout", type=int, default=30,
+                    help="seconds per request (default: 30)")
+    ap.add_argument("--limit", type=int, default=None,
+                    help="test at most N models per provider (debug)")
+    ap.add_argument("--json-out", default=None,
+                    help="write machine-readable report to this path")
+    args = ap.parse_args()
+
+    env = {**os.environ, **load_env(Path(args.env))}
+
+    provider_list = args.provider or list(PROVIDERS.keys())
+    tested: List[dict] = []
+    for name in provider_list:
+        if name not in PROVIDERS:
+            print(f"unknown provider: {name}", file=sys.stderr)
+            continue
+        key_var = f"{name.upper()}_API_KEY"
+        url_var = f"{name.upper()}_BASE_URL"
+        api_key = env.get(key_var, "")
+        base_url = env.get(url_var) or None
+        if not api_key:
+            print(f"\n=== {name} ===\n  skipped — {key_var} not set")
+            continue
+        tested.append(run_provider(name, api_key, base_url, args.timeout, args.limit))
+
+    summarize(tested)
+
+    if args.json_out:
+        Path(args.json_out).write_text(json.dumps(tested, indent=2))
+        print(f"\nReport written to {args.json_out}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/verify-ai-models.yml

@@ -0,0 +1,103 @@
+name: Verify AI models catalog
+
+# Runs the AI-model verifier and commits any changes to
+# AppImage/config/verified_ai_models.json on the same branch the run
+# was launched from.
+#
+# GitHub only fires `on: schedule` from the default branch, so the
+# daily cron always runs against main and keeps stable users fresh.
+# When a beta cycle needs its own refresh on develop, use the
+# "Run workflow" button on the Actions tab and pick develop from the
+# branch selector — the same YAML then checks out develop, runs the
+# verifier and commits back to develop. Cross-branch pushes never
+# happen: each run only touches the branch it started on.
+#
+# The verifier code lives at .github/scripts/ai-models-verifier/ and
+# reads API keys from repository Secrets. Any provider without a key
+# is skipped silently — the workflow keeps going with the rest.
+
+on:
+  schedule:
+    - cron: '0 4 * * *'   # 04:00 UTC every day — cron always fires from main
+  workflow_dispatch:       # manual trigger — branch is picked in the UI
+
+permissions:
+  contents: write
+
+concurrency:
+  # Keyed by branch so a manual develop run does not collide with the
+  # scheduled main run — each branch gets its own serialisation lane.
+  group: verify-ai-models-${{ github.ref_name }}
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out the branch this run belongs to
+        uses: actions/checkout@v4
+        with:
+          # `github.ref_name` resolves to main for the cron and to the
+          # branch selected in the dispatch UI otherwise. The same
+          # value is used again below when we push, so every run is
+          # symmetric: checkout X → refresh → push X.
+          ref: ${{ github.ref_name }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Run verifier
+        working-directory: .github/scripts/ai-models-verifier
+        env:
+          # API keys — each is optional. verify.py silently skips any
+          # provider whose *_API_KEY env var is empty, so the workflow
+          # runs even when only a subset of keys is configured.
+          OPENAI_API_KEY:     ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY:  ${{ secrets.ANTHROPIC_API_KEY }}
+          GROQ_API_KEY:       ${{ secrets.GROQ_API_KEY }}
+          GEMINI_API_KEY:     ${{ secrets.GEMINI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+          # Optional base URLs for custom-endpoint providers.
+          OPENAI_BASE_URL:    ${{ secrets.OPENAI_BASE_URL }}
+        run: |
+          python3 verify.py --json-out /tmp/report.json || true
+          if [ ! -s /tmp/report.json ]; then
+            echo "Verifier produced no report — bailing"
+            exit 1
+          fi
+
+      - name: Apply report to catalog
+        id: apply
+        working-directory: .
+        run: |
+          set +e
+          python3 .github/scripts/ai-models-verifier/apply.py \
+            --report /tmp/report.json \
+            --catalog AppImage/config/verified_ai_models.json
+          code=$?
+          set -e
+          case "$code" in
+            0)  echo "changed=false" >> "$GITHUB_OUTPUT" ;;
+            10) echo "changed=true"  >> "$GITHUB_OUTPUT" ;;
+            *)  echo "apply.py exited with $code"; exit "$code" ;;
+          esac
+
+      - name: Commit and push back to the same branch
+        if: steps.apply.outputs.changed == 'true'
+        run: |
+          git config user.name  "proxmenux-bot"
+          git config user.email "proxmenux-bot@users.noreply.github.com"
+          git add AppImage/config/verified_ai_models.json
+          git commit -m "chore(ai-models): daily catalog refresh"
+          # Push to the branch this run started on — same ref used at
+          # checkout above, so the operation is symmetric regardless of
+          # whether cron (main) or dispatch (any branch) triggered it.
+          git push origin HEAD:${{ github.ref_name }}
+
+      - name: Report unchanged
+        if: steps.apply.outputs.changed != 'true'
+        run: echo "Catalog already up to date — nothing to commit."

AppImage/components/notification-settings.tsx

@@ -341,6 +341,16 @@ export function NotificationSettings() {
   const [aiTestResult, setAiTestResult] = useState<{ success: boolean; message: string; model?: string } | null>(null)
   const [providerModels, setProviderModels] = useState<string[]>([])
   const [loadingProviderModels, setLoadingProviderModels] = useState(false)
+  // Metadata returned by the backend after the catalog refresh — used
+  // to render the "Last update" line and toast the diff.
+  const [catalogMeta, setCatalogMeta] = useState<{
+    success: boolean
+    message: string
+    changed: boolean
+    previous_updated: string | null
+    new_updated: string | null
+    diff: Record<string, { added: string[]; removed: string[] }>
+  } | null>(null)
   const [showCustomPromptInfo, setShowCustomPromptInfo] = useState(false)
   const [editingCustomPrompt, setEditingCustomPrompt] = useState(false)
   const [customPromptDraft, setCustomPromptDraft] = useState("")
@@ -993,16 +1003,38 @@ export function NotificationSettings() {
     
     setLoadingProviderModels(true)
     try {
-      const data = await fetchApi<{ success: boolean; models: string[]; recommended: string; message: string }>("/api/notifications/provider-models", {
+      const data = await fetchApi<{
+        success: boolean
+        models: string[]
+        recommended: string
+        message: string
+        catalog_meta?: {
+          success: boolean
+          message: string
+          changed: boolean
+          previous_updated: string | null
+          new_updated: string | null
+          diff: Record<string, { added: string[]; removed: string[] }>
+        } | null
+      }>("/api/notifications/provider-models", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ 
+        body: JSON.stringify({
           provider,
           api_key: apiKey,
           ollama_url: config.ai_ollama_url,
           openai_base_url: config.ai_openai_base_url,
+          // Every click pulls the latest verified catalog from GitHub
+          // before the intersection runs, so a user with an outdated
+          // AppImage still sees today's model list. The backend keeps
+          // the local file when the fetch fails, so an offline node
+          // simply doesn't refresh but still resolves models.
+          refresh_catalog: true,
         }),
       })
+      if (data.catalog_meta) {
+        setCatalogMeta(data.catalog_meta)
+      }
       if (data.success && data.models && data.models.length > 0) {
         setProviderModels(data.models)
         // Auto-select recommended model if current selection is empty or not in the list
@@ -2409,7 +2441,7 @@ export function NotificationSettings() {
                             className="h-9 px-3 shrink-0"
                             onClick={() => fetchProviderModels()}
                             disabled={
-                              loadingProviderModels || 
+                              loadingProviderModels ||
                               (config.ai_provider === 'ollama' && !config.ai_ollama_url) ||
                               (config.ai_provider !== 'ollama' && config.ai_provider !== 'anthropic' && !config.ai_api_keys?.[config.ai_provider])
                             }
@@ -2419,7 +2451,12 @@ export function NotificationSettings() {
                             ) : (
                               <>
                                 <RefreshCw className="h-4 w-4 mr-1" />
-                                {t("settings.notifications.ai.load")}
+                                {/* Label swaps once we have models loaded — "Load" reads
+                                    right on the empty state, "Update" reads right after
+                                    the first pull. Same action underneath either way. */}
+                                {t(providerModels.length === 0
+                                  ? "settings.notifications.ai.load"
+                                  : "settings.notifications.ai.updateModels")}
                               </>
                             )}
                           </Button>
@@ -2427,6 +2464,32 @@ export function NotificationSettings() {
                         {providerModels.length > 0 && (
                           <p className="text-xs text-green-500">{t("settings.notifications.ai.modelsAvailable", { count: providerModels.length })}</p>
                         )}
+                        {/* Catalog freshness line — surfaces the `_updated` date of the
+                            verified_ai_models.json currently in use, plus a compact diff
+                            summary from the most recent GitHub pull so the user knows
+                            what just happened without opening a modal. */}
+                        {catalogMeta && (
+                          <div className="text-[11px] text-muted-foreground space-y-0.5">
+                            <div>
+                              {t("settings.notifications.ai.catalogLastUpdated", {
+                                date: catalogMeta.new_updated || catalogMeta.previous_updated || "—",
+                              })}
+                            </div>
+                            {catalogMeta.changed && catalogMeta.success && Object.keys(catalogMeta.diff).length > 0 && (
+                              <div className="text-blue-400">
+                                {t("settings.notifications.ai.catalogChanged", {
+                                  added: Object.values(catalogMeta.diff).reduce((n, d) => n + d.added.length, 0),
+                                  removed: Object.values(catalogMeta.diff).reduce((n, d) => n + d.removed.length, 0),
+                                })}
+                              </div>
+                            )}
+                            {!catalogMeta.success && catalogMeta.message && (
+                              <div className="text-amber-400">
+                                {t("settings.notifications.ai.catalogFetchFailed", { reason: catalogMeta.message })}
+                              </div>
+                            )}
+                          </div>
+                        )}
                       </div>
                       
                       {/* Prompt Mode section */}

AppImage/messages/de/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "Es ist eine kostenlose Stufe mit einem guten Preis-Leistungs-Verhältnis verfügbar.",
           "ollama": "Verwendet Modelle auf Ihrem Ollama-Server. Völlig lokal, privat und kostenlos nutzbar.",
           "openrouter": "Zugriff auf mehr als 100 Modelle über einen API-Schlüssel."
-        }
+        },
+        "updateModels": "Aktualisieren",
+        "catalogLastUpdated": "Katalog zuletzt aktualisiert: {date}",
+        "catalogChanged": "Katalog aktualisiert — {added} hinzugefügt, {removed} entfernt",
+        "catalogFetchFailed": "Katalog konnte nicht aktualisiert werden: {reason}. Lokale Kopie wird verwendet."
       },
       "telegramGuide": {
         "title": "Anleitung zur Einrichtung des Telegram-Bots",

AppImage/messages/en/common.json

@@ -1921,7 +1921,11 @@
           "gemini": "A free tier is available, with a good quality-to-price ratio.",
           "ollama": "Uses models on your Ollama server. Fully local, private and free to run.",
           "openrouter": "Access to more than 100 models through one API key."
-        }
+        },
+        "updateModels": "Update",
+        "catalogLastUpdated": "Verified catalog last updated: {date}",
+        "catalogChanged": "Catalog refreshed — {added} added, {removed} removed",
+        "catalogFetchFailed": "Could not refresh catalog: {reason}. Using local copy."
       },
       "telegramGuide": {
         "title": "Telegram bot setup guide",

AppImage/messages/es/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "Hay disponible un nivel gratuito, con una buena relación calidad-precio.",
           "ollama": "Utiliza modelos en su servidor Ollama. Totalmente local, privado y gratuito.",
           "openrouter": "Acceso a más de 100 modelos a través de una clave API."
-        }
+        },
+        "updateModels": "Actualizar",
+        "catalogLastUpdated": "Última actualización del catálogo verificado: {date}",
+        "catalogChanged": "Catálogo actualizado — {added} añadidos, {removed} retirados",
+        "catalogFetchFailed": "No se pudo actualizar el catálogo: {reason}. Se usa la copia local."
       },
       "telegramGuide": {
         "title": "Guía de configuración del bot de Telegram",

AppImage/messages/fr/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "Un niveau gratuit est disponible, avec un bon rapport qualité-prix.",
           "ollama": "Utilise des modèles sur votre serveur Ollama. Entièrement local, privé et gratuit.",
           "openrouter": "Accès à plus de 100 modèles via une seule clé API."
-        }
+        },
+        "updateModels": "Actualiser",
+        "catalogLastUpdated": "Dernière mise à jour du catalogue vérifié : {date}",
+        "catalogChanged": "Catalogue mis à jour — {added} ajoutés, {removed} retirés",
+        "catalogFetchFailed": "Impossible d'actualiser le catalogue : {reason}. Copie locale utilisée."
       },
       "telegramGuide": {
         "title": "Guide de configuration du robot Telegram",

AppImage/messages/it/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "È disponibile un livello gratuito, con un buon rapporto qualità-prezzo.",
           "ollama": "Utilizza i modelli sul tuo server Ollama. Completamente locale, privato e gratuito.",
           "openrouter": "Accesso a più di 100 modelli tramite una chiave API."
-        }
+        },
+        "updateModels": "Aggiorna",
+        "catalogLastUpdated": "Ultimo aggiornamento del catalogo verificato: {date}",
+        "catalogChanged": "Catalogo aggiornato — {added} aggiunti, {removed} rimossi",
+        "catalogFetchFailed": "Impossibile aggiornare il catalogo: {reason}. Verrà usata la copia locale."
       },
       "telegramGuide": {
         "title": "Guida alla configurazione del bot di Telegram",

AppImage/messages/pt/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "Um nível gratuito está disponível, com uma boa relação qualidade/preço.",
           "ollama": "Usa modelos em seu servidor Ollama. Totalmente local, privado e de operação gratuita.",
           "openrouter": "Acesso a mais de 100 modelos através de uma chave API."
-        }
+        },
+        "updateModels": "Atualizar",
+        "catalogLastUpdated": "Última atualização do catálogo verificado: {date}",
+        "catalogChanged": "Catálogo atualizado — {added} adicionados, {removed} removidos",
+        "catalogFetchFailed": "Não foi possível atualizar o catálogo: {reason}. A usar a cópia local."
       },
       "telegramGuide": {
         "title": "Guia de configuração do bot do Telegram",

AppImage/messages/sk/common.json

@@ -1921,7 +1921,11 @@
           "gemini": "Ponúka bezplatnú úroveň a dobrý pomer kvality a ceny.",
           "ollama": "Používa modely na vašom Ollama serveri. Beží lokálne, súkromne a bez poplatkov.",
           "openrouter": "Prístup k viac než 100 modelom cez jeden API kľúč."
-        }
+        },
+        "updateModels": "Aktualizovať",
+        "catalogLastUpdated": "Posledná aktualizácia overeného katalógu: {date}",
+        "catalogChanged": "Katalóg aktualizovaný — {added} pridaných, {removed} odstránených",
+        "catalogFetchFailed": "Katalóg nebolo možné aktualizovať: {reason}. Používa sa lokálna kópia."
       },
       "telegramGuide": {
         "title": "Nastavenie Telegram bota",

AppImage/messages/sv/common.json

@@ -1922,7 +1922,11 @@
           "gemini": "En gratis nivå är tillgänglig, med ett bra förhållande mellan kvalitet och pris.",
           "ollama": "Använder modeller på din Ollama-server. Helt lokalt, privat och gratis att köra.",
           "openrouter": "Tillgång till mer än 100 modeller via en API-nyckel."
-        }
+        },
+        "updateModels": "Uppdatera",
+        "catalogLastUpdated": "Verifierad katalog senast uppdaterad: {date}",
+        "catalogChanged": "Katalog uppdaterad — {added} tillagda, {removed} borttagna",
+        "catalogFetchFailed": "Katalogen kunde inte uppdateras: {reason}. Använder lokal kopia."
       },
       "telegramGuide": {
         "title": "Installationsguide för Telegram bot",

AppImage/scripts/flask_notification_routes.py

@@ -402,32 +402,190 @@ def test_notification():
         return jsonify({'error': f'Internal error ({type(e).__name__})'}), 500
 
 
+_VERIFIED_MODELS_REMOTE_URL = (
+    "https://raw.githubusercontent.com/MacRimi/ProxMenux/main/"
+    "AppImage/config/verified_ai_models.json"
+)
+
+
+def _resolve_verified_models_path() -> Path:
+    """Locate the on-disk verified_ai_models.json used at runtime.
+
+    AppImage layout keeps scripts and config under /usr/bin/; the dev
+    tree has them one level apart. We probe both and return whichever
+    exists so `load_verified_models` and the refresh helper agree on the
+    same file — otherwise a refresh writing to one path and a read
+    hitting the other would silently do nothing.
+    """
+    script_dir = Path(__file__).parent
+    candidate = script_dir / 'config' / 'verified_ai_models.json'
+    if candidate.exists():
+        return candidate
+    dev_candidate = script_dir.parent / 'config' / 'verified_ai_models.json'
+    if dev_candidate.exists():
+        return dev_candidate
+    # No file exists yet — return the AppImage-shaped path so a fresh
+    # refresh has somewhere to write. The parent dir is created if needed.
+    candidate.parent.mkdir(parents=True, exist_ok=True)
+    return candidate
+
+
 def load_verified_models():
     """Load verified models from config file.
-    
+
-    Checks multiple paths:
+    Returns {} if the file is missing or unreadable; callers already
-    1. Same directory as script (AppImage: /usr/bin/config/)
+    handle the empty case by returning provider defaults or the API's
-    2. Parent directory config folder (dev: AppImage/config/)
+    unfiltered list.
     """
     try:
-        # Try AppImage path first (scripts and config both in /usr/bin/)
+        path = _resolve_verified_models_path()
-        script_dir = Path(__file__).parent
+        if path.exists():
-        config_path = script_dir / 'config' / 'verified_ai_models.json'
+            with open(path, 'r') as f:
-        
-        if not config_path.exists():
-            # Try development path (AppImage/scripts/ -> AppImage/config/)
-            config_path = script_dir.parent / 'config' / 'verified_ai_models.json'
-        
-        if config_path.exists():
-            with open(config_path, 'r') as f:
                 return json.load(f)
-        else:
+        print(f"[flask_notification_routes] Config not found at {path}")
-            print(f"[flask_notification_routes] Config not found at {config_path}")
     except Exception as e:
         print(f"[flask_notification_routes] Failed to load verified models: {e}")
     return {}
 
 
+def _catalog_signature(catalog: dict) -> dict:
+    """Reduce a catalog to `{provider: [sorted models]}`.
+
+    Used to diff the local vs the remote catalog without dragging the
+    unrelated `_note` / `_deprecated` / `_updated` metadata into the
+    comparison.
+    """
+    sig = {}
+    for provider, block in catalog.items():
+        if isinstance(block, dict) and isinstance(block.get('models'), list):
+            sig[provider] = sorted(str(m) for m in block['models'])
+    return sig
+
+
+def refresh_verified_models_from_github() -> dict:
+    """Fetch the canonical catalog from the main branch and overwrite the
+    local copy in-place.
+
+    Returns a dict shaped for the API response:
+      {
+        'success': True|False,
+        'message': '...',
+        'changed': bool,
+        'previous_updated': 'YYYY-MM-DD' | None,
+        'new_updated': 'YYYY-MM-DD' | None,
+        'diff': { <provider>: {'added': [...], 'removed': [...]} },
+      }
+
+    Never touches the local file when the remote fetch fails — the
+    installed catalog remains authoritative in the offline / degraded
+    case, which is the deliberate behaviour the UI relies on to show
+    'the catalog is still active' after a failed refresh.
+    """
+    import urllib.request
+    import urllib.error
+
+    result = {
+        'success': False,
+        'message': '',
+        'changed': False,
+        'previous_updated': None,
+        'new_updated': None,
+        'diff': {},
+    }
+
+    try:
+        req = urllib.request.Request(_VERIFIED_MODELS_REMOTE_URL, method='GET')
+        req.add_header('User-Agent', 'ProxMenux-Monitor/1.1')
+        req.add_header('Accept', 'application/json')
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            raw = resp.read().decode('utf-8')
+            remote = json.loads(raw)
+    except urllib.error.HTTPError as e:
+        result['message'] = f'GitHub returned HTTP {e.code}'
+        return result
+    except urllib.error.URLError as e:
+        result['message'] = f'Could not reach GitHub: {e.reason}'
+        return result
+    except (json.JSONDecodeError, ValueError) as e:
+        result['message'] = f'Remote catalog is not valid JSON: {e}'
+        return result
+    except Exception as e:
+        result['message'] = f'Refresh failed: {type(e).__name__}: {e}'
+        return result
+
+    if not isinstance(remote, dict) or not any(
+        isinstance(remote.get(k), dict) and 'models' in remote[k]
+        for k in remote.keys()
+        if not k.startswith('_')
+    ):
+        result['message'] = 'Remote catalog has an unexpected shape'
+        return result
+
+    local = load_verified_models() or {}
+    prev_sig = _catalog_signature(local)
+    new_sig = _catalog_signature(remote)
+
+    diff = {}
+    for provider in sorted(set(prev_sig) | set(new_sig)):
+        prev_models = set(prev_sig.get(provider, []))
+        new_models = set(new_sig.get(provider, []))
+        added = sorted(new_models - prev_models)
+        removed = sorted(prev_models - new_models)
+        if added or removed:
+            diff[provider] = {'added': added, 'removed': removed}
+
+    result['previous_updated'] = local.get('_updated')
+    result['new_updated'] = remote.get('_updated')
+    result['changed'] = bool(diff) or (result['previous_updated'] != result['new_updated'])
+    result['diff'] = diff
+
+    # Only touch disk when something actually changed to keep mtimes
+    # meaningful and avoid a spurious modification in backup diffs.
+    if result['changed']:
+        try:
+            path = _resolve_verified_models_path()
+            tmp = path.with_suffix(path.suffix + '.tmp')
+            with open(tmp, 'w') as f:
+                json.dump(remote, f, indent=2, ensure_ascii=False)
+                f.write('\n')
+            tmp.replace(path)
+            result['success'] = True
+            result['message'] = 'Catalog updated'
+        except Exception as e:
+            result['success'] = False
+            result['message'] = f'Downloaded but failed to save: {type(e).__name__}: {e}'
+            return result
+    else:
+        result['success'] = True
+        result['message'] = 'Catalog already up to date'
+    return result
+
+
+@notification_bp.route('/api/notifications/refresh-model-catalog', methods=['POST'])
+@require_auth
+def refresh_model_catalog():
+    """Pull the verified_ai_models.json from ProxMenux/main on GitHub and
+    overwrite the local copy. Returns the diff so the UI can toast a
+    meaningful summary (added / removed models per provider) instead of a
+    generic 'refreshed' message.
+
+    Standalone endpoint so an admin can refresh without also fetching
+    provider models; the provider-models call carries a shortcut flag
+    for the common 'refresh + load' click from the Notifications UI.
+    """
+    try:
+        return jsonify(refresh_verified_models_from_github())
+    except Exception as e:
+        return jsonify({
+            'success': False,
+            'message': f'Refresh failed: {type(e).__name__}: {e}',
+            'changed': False,
+            'previous_updated': None,
+            'new_updated': None,
+            'diff': {},
+        }), 500
+
+
 @notification_bp.route('/api/notifications/provider-models', methods=['POST'])
 @require_auth
 def get_provider_models():
@@ -459,9 +617,25 @@ def get_provider_models():
         api_key = _resolve_masked_api_key(provider, data.get('api_key', ''))
         ollama_url = data.get('ollama_url', 'http://localhost:11434')
         openai_base_url = data.get('openai_base_url', '')
+        # `refresh_catalog=true` in the request body triggers a GitHub
+        # pull of verified_ai_models.json before the intersection runs.
+        # The Notifications UI passes this on every Load / Update click
+        # so the catalog stays fresh without a separate button.
+        refresh_catalog = bool(data.get('refresh_catalog', False))
+        catalog_meta = None
+        if refresh_catalog:
+            catalog_meta = refresh_verified_models_from_github()
+
+        def _reply(payload, status=200):
+            # Every response from this endpoint carries `catalog_meta`
+            # when a refresh was requested, so the UI can render the
+            # diff toast + "last updated" line from a single roundtrip.
+            if catalog_meta is not None:
+                payload = {**payload, 'catalog_meta': catalog_meta}
+            return jsonify(payload), status
 
         if not provider:
-            return jsonify({'success': False, 'models': [], 'message': 'Provider not specified'})
+            return _reply({'success': False, 'models': [], 'message': 'Provider not specified'})
 
         # SSRF guard before we touch the URL. Ollama is local-by-design so
         # loopback is allowed there; OpenAI base URL must be a real external
@@ -469,11 +643,11 @@ def get_provider_models():
         if provider == 'ollama':
             ok, err = validate_external_url(ollama_url, allow_loopback=True)
             if not ok:
-                return jsonify({'success': False, 'models': [], 'message': f'Invalid ollama_url: {err}'}), 400
+                return _reply({'success': False, 'models': [], 'message': f'Invalid ollama_url: {err}'}, 400)
         if provider == 'openai' and openai_base_url:
             ok, err = validate_external_url(openai_base_url, allow_loopback=False)
             if not ok:
-                return jsonify({'success': False, 'models': [], 'message': f'Invalid openai_base_url: {err}'}), 400
+                return _reply({'success': False, 'models': [], 'message': f'Invalid openai_base_url: {err}'}, 400)
         
         # Load verified models config
         verified_config = load_verified_models()
@@ -494,13 +668,13 @@ def get_provider_models():
                 result = json.loads(resp.read().decode('utf-8'))
                 models = [m.get('name', '') for m in result.get('models', []) if m.get('name')]
                 models = sorted(models)
-                return jsonify({
+                return _reply({
                     'success': True,
                     'models': models,
                     'recommended': models[0] if models else '',
                     'message': f'Found {len(models)} local models'
                 })
-        
+
         # Handle Anthropic - no models list API, return verified models directly
         if provider == 'anthropic':
             models = list(verified_models) if verified_models else [
@@ -508,31 +682,31 @@ def get_provider_models():
                 'claude-3-5-sonnet-latest',
                 'claude-3-opus-latest',
             ]
-            return jsonify({
+            return _reply({
                 'success': True,
                 'models': sorted(models),
                 'recommended': recommended or models[0],
                 'message': f'{len(models)} verified models'
             })
-        
+
         # For other providers, fetch from API and filter by verified list.
         # Custom OpenAI-compatible endpoints (LiteLLM, opencode.ai, vLLM,
         # LocalAI…) often expose `/v1/models` without authentication, so
         # we only require an api_key when there's no custom base URL to
         # consult. Issue #11.5 — OpenCode provider Custom Base URL fetch.
         if not api_key and not (provider == 'openai' and openai_base_url):
-            return jsonify({'success': False, 'models': [], 'message': 'API key required'})
+            return _reply({'success': False, 'models': [], 'message': 'API key required'})
-        
+
         from ai_providers import get_provider
         ai_provider = get_provider(
-            provider, 
+            provider,
-            api_key=api_key, 
+            api_key=api_key,
-            model='', 
+            model='',
             base_url=openai_base_url if provider == 'openai' else None
         )
-        
+
         if not ai_provider:
-            return jsonify({'success': False, 'models': [], 'message': f'Unknown provider: {provider}'})
+            return _reply({'success': False, 'models': [], 'message': f'Unknown provider: {provider}'})
         
         # Get all models from provider API
         api_models = ai_provider.list_models()
@@ -551,13 +725,13 @@ def get_provider_models():
             # so "gpt-4o-mini" as a fallback would be misleading).
             if verified_models and not is_openai_compat:
                 models = sorted(verified_models)
-                return jsonify({
+                return _reply({
                     'success': True,
                     'models': models,
                     'recommended': recommended or models[0],
                     'message': f'{len(models)} verified models (API unavailable)'
                 })
-            return jsonify({
+            return _reply({
                 'success': False,
                 'models': [],
                 'message': 'Could not retrieve models. Check your API key and endpoint URL.'
@@ -567,7 +741,7 @@ def get_provider_models():
             # Custom OpenAI-compatible endpoint: surface every model the
             # endpoint reports. No verified-list intersection.
             models = sorted(api_models)
-            return jsonify({
+            return _reply({
                 'success': True,
                 'models': models,
                 'recommended': models[0] if models else '',
@@ -594,20 +768,29 @@ def get_provider_models():
         else:
             # No verified list for this provider, return all from API
             models = sorted(api_models)
-        
+
-        return jsonify({
+        return _reply({
             'success': True,
             'models': models,
             'recommended': recommended if recommended in models else (models[0] if models else ''),
             'message': f'{len(models)} verified models available'
         })
-        
+
     except Exception as e:
-        return jsonify({
+        # Outside the _reply closure — build the payload manually so the
+        # catch-all still carries catalog_meta when a refresh was
+        # attempted before the crash.
+        payload = {
             'success': False,
             'models': [],
-            'message': f'Error: {str(e)}'
+            'message': f'Error: {str(e)}',
-        })
+        }
+        try:
+            if catalog_meta is not None:
+                payload['catalog_meta'] = catalog_meta
+        except UnboundLocalError:
+            pass
+        return jsonify(payload)
 
 
 @notification_bp.route('/api/notifications/test-ai', methods=['POST'])