mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-09-14 10:47:36 +00:00
MacRimi
@@ -0,0 +1,16 @@
|
||||
# ProxMenux AI-model verifier — API keys
|
||||
# Copy this file to .env (gitignored) and fill only the ones you have.
|
||||
|
||||
OPENAI_API_KEY=
|
||||
# Optional: for LiteLLM/MLX/LM Studio/vLLM/LocalAI/Ollama-proxy testing,
|
||||
# point OPENAI_API_KEY to any non-empty placeholder and set OPENAI_BASE_URL
|
||||
# to the endpoint's root (without /v1 — the tool appends it).
|
||||
# OPENAI_BASE_URL=http://localhost:4000
|
||||
|
||||
GROQ_API_KEY=
|
||||
|
||||
GEMINI_API_KEY=
|
||||
|
||||
ANTHROPIC_API_KEY=
|
||||
|
||||
OPENROUTER_API_KEY=
|
||||
@@ -0,0 +1,37 @@
|
||||
# AI models verifier (public copy)
|
||||
|
||||
Standalone verifier used by the daily GitHub Action to refresh
|
||||
`AppImage/config/verified_ai_models.json`.
|
||||
|
||||
The code lives here so the Action can execute it. API keys are read from
|
||||
GitHub Secrets at run time and never written to disk.
|
||||
|
||||
Local dev runs (interactive verifier over your own keys) can keep using
|
||||
the private copy — `verify.py` is identical.
|
||||
|
||||
## What the Action does
|
||||
|
||||
Each run:
|
||||
|
||||
1. Loads keys from Secrets into environment variables.
|
||||
2. Runs `verify.py --json-out /tmp/report.json` against every provider that
|
||||
has a key set.
|
||||
3. Rewrites `AppImage/config/verified_ai_models.json` with the passing
|
||||
models, sorted with the recommended one first per provider.
|
||||
4. Bumps the `_updated` field to the current date.
|
||||
5. If the file changed, commits directly to `main` as a bot commit.
|
||||
|
||||
## Adding provider keys
|
||||
|
||||
- Repository → Settings → Secrets and variables → Actions.
|
||||
- Add each key with the exact name expected by `verify.py`:
|
||||
`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GROQ_API_KEY`, `GEMINI_API_KEY`,
|
||||
`OPENROUTER_API_KEY`.
|
||||
- Any provider without a key is silently skipped — the Action logs a
|
||||
warning and continues with the rest.
|
||||
|
||||
## Running the Action on demand
|
||||
|
||||
The workflow accepts `workflow_dispatch`, so you can trigger a refresh
|
||||
manually from the Actions tab. Useful when a new model has just been
|
||||
released upstream and you don't want to wait for the daily cron.
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply a verifier report to ``AppImage/config/verified_ai_models.json``.
|
||||
|
||||
Reads the machine-readable report emitted by ``verify.py --json-out`` and
|
||||
merges the passing models into the on-disk catalog:
|
||||
|
||||
* Passing models per provider replace the existing ``models`` list.
|
||||
* ``recommended`` is set to the fastest passing model.
|
||||
* Existing ``_note`` / ``_deprecated`` / provider metadata is preserved
|
||||
when unchanged so the file's manual annotations survive the automated
|
||||
refresh.
|
||||
* Providers absent from the report (e.g. no API key configured in the
|
||||
GitHub Action for that run) are left untouched — the goal is
|
||||
additive maintenance, not silent removal.
|
||||
* ``_updated`` bumps to today's date only when the model set actually
|
||||
changes; a no-op run leaves the file byte-identical.
|
||||
|
||||
Exits 0 when the file is unchanged, 10 when it was updated. The
|
||||
workflow uses that exit code to decide whether to commit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
return json.load(fh)
|
||||
|
||||
|
||||
def _save_json(path: Path, data: dict) -> None:
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with open(tmp, "w", encoding="utf-8") as fh:
|
||||
json.dump(data, fh, indent=2, ensure_ascii=False)
|
||||
fh.write("\n")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _passing_models(provider_report: dict) -> list[str]:
|
||||
"""Return the passing models for one provider, fastest first."""
|
||||
passing = [r for r in provider_report.get("results", []) if r.get("verdict") == "pass"]
|
||||
passing.sort(key=lambda r: r.get("latency_s", 999))
|
||||
return [r["model"] for r in passing]
|
||||
|
||||
|
||||
def apply_report(report_path: Path, catalog_path: Path, today: str) -> bool:
|
||||
"""Rewrite the catalog from the report. Returns True if it changed."""
|
||||
report = _load_json(report_path)
|
||||
catalog = _load_json(catalog_path) if catalog_path.exists() else {}
|
||||
|
||||
changed = False
|
||||
for provider_report in report:
|
||||
name = provider_report.get("provider")
|
||||
if not name:
|
||||
continue
|
||||
if provider_report.get("error"):
|
||||
print(f"[{name}] skipped — verifier reported error: {provider_report['error']}",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
passing = _passing_models(provider_report)
|
||||
if not passing:
|
||||
print(f"[{name}] no passing models this run — leaving catalog untouched",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
|
||||
block = catalog.setdefault(name, {})
|
||||
prev_models = list(block.get("models", []))
|
||||
prev_recommended = block.get("recommended", "")
|
||||
|
||||
if sorted(prev_models) != sorted(passing) or prev_recommended != passing[0]:
|
||||
block["models"] = passing
|
||||
block["recommended"] = passing[0]
|
||||
changed = True
|
||||
print(f"[{name}] updated — {len(passing)} models, recommended={passing[0]}")
|
||||
else:
|
||||
print(f"[{name}] unchanged — {len(passing)} models")
|
||||
|
||||
if changed:
|
||||
catalog["_updated"] = today
|
||||
_save_json(catalog_path, catalog)
|
||||
return changed
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--report", required=True, help="verify.py --json-out path")
|
||||
ap.add_argument("--catalog", required=True,
|
||||
help="AppImage/config/verified_ai_models.json path")
|
||||
ap.add_argument("--today", default=None,
|
||||
help="Override the date written into _updated (YYYY-MM-DD).")
|
||||
args = ap.parse_args()
|
||||
|
||||
today = args.today or dt.datetime.utcnow().strftime("%Y-%m-%d")
|
||||
changed = apply_report(Path(args.report), Path(args.catalog), today)
|
||||
return 10 if changed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Standardized test prompt for ProxMenux AI-model verification.
|
||||
|
||||
Mirrors the real AI-enrichment use case: take a raw Proxmox system
|
||||
notification (English, with technical identifiers), translate it into
|
||||
Spanish, explain in plain terms, and suggest one concrete action. It is
|
||||
intentionally simple — if a model can't do this, it won't do the real
|
||||
thing either. Models that pass this test are fine for inclusion in
|
||||
verified_ai_models.json.
|
||||
"""
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"You are a Proxmox system-notification assistant. "
|
||||
"When given a raw notification from a Proxmox host, you: "
|
||||
"(1) translate it into Spanish, "
|
||||
"(2) explain in 2-3 sentences what the user is seeing and the likely cause, "
|
||||
"(3) suggest ONE concrete next action. "
|
||||
"Keep technical identifiers (device paths like /dev/sdd, SMART keywords, "
|
||||
"ata port numbers, BDFs) in their original form. "
|
||||
"Respond only in Spanish. Stay under 200 tokens total."
|
||||
)
|
||||
|
||||
# Realistic ProxMenux notification payload: multi-line body with
|
||||
# SMART/ATA vocabulary and a frequency hint — the exact shape the real
|
||||
# pipeline emits.
|
||||
USER_MESSAGE = (
|
||||
"Event: disk_io_error\n"
|
||||
"Severity: CRITICAL\n"
|
||||
"Host: pve-constructor\n"
|
||||
"Device: /dev/sdd\n"
|
||||
"SMART status: PASSED\n"
|
||||
"Summary: 3 I/O event(s) in 5 minutes, disk passed SMART short test\n"
|
||||
"Sample kernel line: ata4.00: exception Emask 0x0 SAct 0x804000 SErr 0x0 action 0x6\n"
|
||||
"Frequency: 3 occurrences in 24h, first seen 6h ago"
|
||||
)
|
||||
|
||||
# Common Spanish stopwords. A response missing ALL of these is almost
|
||||
# certainly not Spanish (or empty/truncated). Cheap heuristic, good
|
||||
# enough for a coarse pass/fail.
|
||||
REQUIRED_SPANISH_HINTS = [
|
||||
" el ", " la ", " los ", " las ", " un ", " una ",
|
||||
" de ", " del ", " en ", " con ", " que ", " para ",
|
||||
" es ", " se ", " ha ", " por ", " y ",
|
||||
]
|
||||
|
||||
# Domain keywords — at least one must appear to confirm the model
|
||||
# actually engaged with the notification instead of replying generically.
|
||||
DOMAIN_HINTS = [
|
||||
"disco", "sdd", "smart", "ata", "i/o", "e/s", "error",
|
||||
"kernel", "proxmox",
|
||||
]
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Self-contained API wrappers for AI-model verification.
|
||||
|
||||
Kept independent from the ProxMenux AppImage's ai_providers module so
|
||||
this tool can live in a private repo with no import coupling to the
|
||||
public project. Uses only the Python standard library.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class ProviderError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Provider:
|
||||
"""Base class. Subclasses implement list_models() and generate()."""
|
||||
name = "base"
|
||||
|
||||
def __init__(self, api_key: str, base_url: Optional[str] = None):
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def generate(self, model: str, system: str, user: str,
|
||||
max_tokens: int = 250, timeout: int = 30) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
# ── HTTP helpers ────────────────────────────────────────────
|
||||
|
||||
# Cloudflare in front of api.groq.com (and probably other providers
|
||||
# over time) returns 403 "error code: 1010" for the default
|
||||
# `Python-urllib/3.x` User-Agent — the "browser signature ban" rule.
|
||||
# A plain identifier is enough to get through; we're not spoofing a
|
||||
# browser, just avoiding a naive UA fingerprint match.
|
||||
_USER_AGENT = "ProxMenux-AI-Verifier/1.0"
|
||||
|
||||
def _post_json(self, url: str, payload: dict, headers: dict,
|
||||
timeout: int = 30) -> dict:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
headers = {**headers, "User-Agent": self._USER_AGENT}
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
def _get_json(self, url: str, headers: dict, timeout: int = 30) -> dict:
|
||||
headers = {**headers, "User-Agent": self._USER_AGENT}
|
||||
req = urllib.request.Request(url, headers=headers, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# OpenAI-compatible family (OpenAI, Groq, OpenRouter, LiteLLM, LM
|
||||
# Studio, vLLM, LocalAI, etc.). They all speak the same endpoints.
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
class OpenAICompatProvider(Provider):
|
||||
name = "openai-compat"
|
||||
default_base = "https://api.openai.com"
|
||||
models_path = "/v1/models"
|
||||
chat_path = "/v1/chat/completions"
|
||||
|
||||
def _base(self) -> str:
|
||||
return (self.base_url or self.default_base).rstrip("/")
|
||||
|
||||
@staticmethod
|
||||
def _is_reasoning_model(model: str) -> bool:
|
||||
"""True for OpenAI reasoning models (o-series + non-chat gpt-5+).
|
||||
|
||||
Must be kept in sync with the matching helper in ProxMenux's
|
||||
openai_provider.py — same rule, same consequence:
|
||||
- send max_completion_tokens instead of max_tokens
|
||||
- omit temperature (default is the only accepted value).
|
||||
"""
|
||||
m = model.lower()
|
||||
if len(m) >= 2 and m[0] == "o" and m[1].isdigit():
|
||||
return True
|
||||
if m.startswith("gpt-5") and "-chat" not in m:
|
||||
return True
|
||||
return False
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
url = f"{self._base()}{self.models_path}"
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
data = self._get_json(url, headers)
|
||||
return [m.get("id", "") for m in data.get("data", []) if m.get("id")]
|
||||
|
||||
def generate(self, model: str, system: str, user: str,
|
||||
max_tokens: int = 250, timeout: int = 30) -> str:
|
||||
url = f"{self._base()}{self.chat_path}"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
if self._is_reasoning_model(model):
|
||||
# Reasoning models spend budget on internal reasoning by
|
||||
# default, which yields empty replies at small max_tokens.
|
||||
# reasoning_effort=minimal keeps that overhead low so the
|
||||
# whole budget reaches the user, aligned with the short
|
||||
# translate+explain task ProxMenux uses. Mirror this in
|
||||
# ProxMenux's openai_provider.py.
|
||||
payload["max_completion_tokens"] = max_tokens
|
||||
payload["reasoning_effort"] = "minimal"
|
||||
else:
|
||||
payload["max_tokens"] = max_tokens
|
||||
payload["temperature"] = 0.3
|
||||
data = self._post_json(url, payload, headers, timeout)
|
||||
try:
|
||||
return data["choices"][0]["message"]["content"].strip()
|
||||
except (KeyError, IndexError) as exc:
|
||||
raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
|
||||
|
||||
|
||||
class OpenAIProvider(OpenAICompatProvider):
|
||||
name = "openai"
|
||||
default_base = "https://api.openai.com"
|
||||
|
||||
|
||||
class GroqProvider(OpenAICompatProvider):
|
||||
name = "groq"
|
||||
default_base = "https://api.groq.com/openai"
|
||||
|
||||
|
||||
class OpenRouterProvider(OpenAICompatProvider):
|
||||
name = "openrouter"
|
||||
default_base = "https://openrouter.ai/api"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Gemini — different endpoint shape, keyed via ?key=... query param.
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
class GeminiProvider(Provider):
|
||||
name = "gemini"
|
||||
default_base = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
@staticmethod
|
||||
def _has_thinking_mode(model: str) -> bool:
|
||||
"""True for Gemini variants that enable "thinking" by default.
|
||||
|
||||
Kept in sync with ProxMenux's gemini_provider.py. 2.5+ pro/flash
|
||||
and 3.x pro/flash consume output tokens on reasoning, which
|
||||
yields empty replies when max_tokens is small. We pass
|
||||
thinkingBudget=0 to disable thinking so the short translate+
|
||||
explain test sees actual text. Lite variants don't have thinking
|
||||
enabled and are not flagged here.
|
||||
"""
|
||||
m = model.lower()
|
||||
if "lite" in m:
|
||||
return False
|
||||
return m.startswith("gemini-2.5") or m.startswith("gemini-3")
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
url = f"{self.default_base}/models?key={self.api_key}"
|
||||
data = self._get_json(url, {})
|
||||
names = []
|
||||
for m in data.get("models", []):
|
||||
raw = m.get("name", "")
|
||||
if raw.startswith("models/"):
|
||||
raw = raw[len("models/"):]
|
||||
# Only keep text-generation capable models.
|
||||
if "generateContent" in m.get("supportedGenerationMethods", []):
|
||||
names.append(raw)
|
||||
return names
|
||||
|
||||
def generate(self, model: str, system: str, user: str,
|
||||
max_tokens: int = 250, timeout: int = 30) -> str:
|
||||
url = f"{self.default_base}/models/{model}:generateContent?key={self.api_key}"
|
||||
gen_config = {
|
||||
"maxOutputTokens": max_tokens,
|
||||
"temperature": 0.3,
|
||||
}
|
||||
if self._has_thinking_mode(model):
|
||||
gen_config["thinkingConfig"] = {"thinkingBudget": 0}
|
||||
payload = {
|
||||
"system_instruction": {"parts": [{"text": system}]},
|
||||
"contents": [{"parts": [{"text": user}]}],
|
||||
"generationConfig": gen_config,
|
||||
}
|
||||
data = self._post_json(url, payload, {"Content-Type": "application/json"}, timeout)
|
||||
try:
|
||||
return data["candidates"][0]["content"]["parts"][0]["text"].strip()
|
||||
except (KeyError, IndexError) as exc:
|
||||
raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# Anthropic — own schema and headers.
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
|
||||
class AnthropicProvider(Provider):
|
||||
name = "anthropic"
|
||||
default_base = "https://api.anthropic.com"
|
||||
version = "2023-06-01"
|
||||
|
||||
def list_models(self) -> List[str]:
|
||||
url = f"{self.default_base}/v1/models"
|
||||
headers = {
|
||||
"x-api-key": self.api_key,
|
||||
"anthropic-version": self.version,
|
||||
}
|
||||
try:
|
||||
data = self._get_json(url, headers)
|
||||
return [m.get("id", "") for m in data.get("data", []) if m.get("id")]
|
||||
except urllib.error.HTTPError:
|
||||
# Older keys don't have the endpoint; caller decides what to do.
|
||||
return []
|
||||
|
||||
def generate(self, model: str, system: str, user: str,
|
||||
max_tokens: int = 250, timeout: int = 30) -> str:
|
||||
url = f"{self.default_base}/v1/messages"
|
||||
headers = {
|
||||
"x-api-key": self.api_key,
|
||||
"anthropic-version": self.version,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
# ProxMenux's real anthropic_provider.py does NOT send `temperature`,
|
||||
# and Anthropic's newest generation (Claude Sonnet 5, Opus 4.7 /
|
||||
# 4.8, Fable 5) now rejects it with:
|
||||
# invalid_request_error: `temperature` is deprecated for this model.
|
||||
# Omitting it here aligns the verifier with production behaviour so
|
||||
# the newest-gen models can be reached during verification.
|
||||
payload = {
|
||||
"model": model,
|
||||
"max_tokens": max_tokens,
|
||||
"system": system,
|
||||
"messages": [{"role": "user", "content": user}],
|
||||
}
|
||||
data = self._post_json(url, payload, headers, timeout)
|
||||
try:
|
||||
return data["content"][0]["text"].strip()
|
||||
except (KeyError, IndexError) as exc:
|
||||
raise ProviderError(f"unexpected response: {exc} // {str(data)[:200]}")
|
||||
|
||||
|
||||
PROVIDERS = {
|
||||
"openai": OpenAIProvider,
|
||||
"groq": GroqProvider,
|
||||
"gemini": GeminiProvider,
|
||||
"anthropic": AnthropicProvider,
|
||||
"openrouter": OpenRouterProvider,
|
||||
}
|
||||
|
||||
|
||||
def make_provider(name: str, api_key: str,
|
||||
base_url: Optional[str] = None) -> Provider:
|
||||
cls = PROVIDERS.get(name)
|
||||
if not cls:
|
||||
raise ValueError(f"unknown provider: {name}")
|
||||
return cls(api_key, base_url=base_url)
|
||||
Executable
+232
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ProxMenux AI-model verifier.
|
||||
|
||||
Runs a standardized translate+explain test against every model each
|
||||
provider currently advertises, and emits a per-model verdict so the
|
||||
verified_ai_models.json list can be refreshed with confidence.
|
||||
|
||||
Not packaged with the AppImage — keep this in a private repo alongside
|
||||
the API keys.
|
||||
|
||||
Usage:
|
||||
cp .env.example .env # fill in the API keys you have
|
||||
python3 verify.py # test all providers with keys
|
||||
python3 verify.py --provider groq # just one
|
||||
python3 verify.py --provider openai --limit 5 # only first 5 models
|
||||
python3 verify.py --json-out report.json # machine-readable output too
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from prompts import DOMAIN_HINTS, REQUIRED_SPANISH_HINTS, SYSTEM_PROMPT, USER_MESSAGE
|
||||
from providers import PROVIDERS, make_provider
|
||||
|
||||
|
||||
# Non-chat model name patterns. Skipping these saves test time and keeps
|
||||
# the report focused on models that could actually serve notifications.
|
||||
SKIP_PATTERNS = (
|
||||
"embedding", "whisper", "tts", "dall-e", "dalle", "image",
|
||||
"realtime", "audio", "moderation", "search",
|
||||
"code-search", "text-similarity", "babbage", "davinci",
|
||||
"curie", "ada", "transcribe",
|
||||
)
|
||||
|
||||
|
||||
def load_env(env_path: Path) -> Dict[str, str]:
|
||||
"""Minimal .env loader (avoids a python-dotenv dependency)."""
|
||||
if not env_path.exists():
|
||||
return {}
|
||||
env: Dict[str, str] = {}
|
||||
for line in env_path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
env[k.strip()] = v.strip().strip('"').strip("'")
|
||||
return env
|
||||
|
||||
|
||||
def should_skip_model(model: str) -> bool:
|
||||
m = model.lower()
|
||||
return any(p in m for p in SKIP_PATTERNS)
|
||||
|
||||
|
||||
def assess_response(text: str) -> Tuple[str, List[str]]:
|
||||
"""Classify the model output. Returns (verdict, reasons).
|
||||
|
||||
verdict is one of:
|
||||
- 'pass': Spanish, on-topic, reasonable length
|
||||
- 'warn': responded but one heuristic failed (borderline)
|
||||
- 'fail': empty, wrong language, or off-topic
|
||||
"""
|
||||
reasons: List[str] = []
|
||||
if not text or len(text) < 30:
|
||||
return "fail", ["empty or too short response"]
|
||||
|
||||
text_low = " " + text.lower() + " "
|
||||
spanish_hits = sum(1 for h in REQUIRED_SPANISH_HINTS if h in text_low)
|
||||
domain_hits = sum(1 for h in DOMAIN_HINTS if h.lower() in text_low)
|
||||
|
||||
if spanish_hits < 3:
|
||||
reasons.append(f"not Spanish ({spanish_hits}/{len(REQUIRED_SPANISH_HINTS)} hints)")
|
||||
if domain_hits < 1:
|
||||
reasons.append("did not engage with the domain")
|
||||
if len(text) > 1500:
|
||||
reasons.append("response unusually long")
|
||||
|
||||
if not reasons:
|
||||
return "pass", []
|
||||
# Responded and engaged, but one signal missed → warn (keep in list
|
||||
# with a caveat; don't auto-include).
|
||||
if domain_hits >= 1 and spanish_hits >= 1:
|
||||
return "warn", reasons
|
||||
return "fail", reasons
|
||||
|
||||
|
||||
def run_model(provider, model: str, timeout: int) -> dict:
|
||||
t0 = time.time()
|
||||
try:
|
||||
out = provider.generate(
|
||||
model, SYSTEM_PROMPT, USER_MESSAGE,
|
||||
max_tokens=250, timeout=timeout,
|
||||
)
|
||||
latency = time.time() - t0
|
||||
verdict, reasons = assess_response(out)
|
||||
return {
|
||||
"model": model,
|
||||
"verdict": verdict,
|
||||
"latency_s": round(latency, 2),
|
||||
"reasons": reasons,
|
||||
"sample": (out[:140] if out else "").replace("\n", " "),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as exc: # HTTPError, ProviderError, timeouts
|
||||
return {
|
||||
"model": model,
|
||||
"verdict": "fail",
|
||||
"latency_s": round(time.time() - t0, 2),
|
||||
"reasons": [],
|
||||
"sample": "",
|
||||
"error": str(exc)[:200],
|
||||
}
|
||||
|
||||
|
||||
def tag(verdict: str) -> str:
|
||||
return {"pass": "✓", "warn": "⚠", "fail": "✗"}.get(verdict, "?")
|
||||
|
||||
|
||||
def run_provider(name: str, api_key: str, base_url: Optional[str],
|
||||
timeout: int, limit: Optional[int]) -> dict:
|
||||
print(f"\n=== {name} ===")
|
||||
try:
|
||||
provider = make_provider(name, api_key, base_url=base_url)
|
||||
models = provider.list_models()
|
||||
except Exception as exc:
|
||||
print(f" list_models() failed: {exc}")
|
||||
return {"provider": name, "error": str(exc), "results": []}
|
||||
|
||||
models = [m for m in models if not should_skip_model(m)]
|
||||
if limit:
|
||||
models = models[:limit]
|
||||
if not models:
|
||||
print(" (no eligible models)")
|
||||
return {"provider": name, "error": None, "results": []}
|
||||
|
||||
print(f" discovered {len(models)} model(s)")
|
||||
results: List[dict] = []
|
||||
for m in models:
|
||||
r = run_model(provider, m, timeout)
|
||||
results.append(r)
|
||||
latency = f"{r['latency_s']}s"
|
||||
if r["error"]:
|
||||
suffix = f" — {r['error']}"
|
||||
elif r["reasons"]:
|
||||
suffix = f" — {'; '.join(r['reasons'])}"
|
||||
else:
|
||||
suffix = ""
|
||||
print(f" {tag(r['verdict'])} {m:<50} {latency:>6}{suffix}")
|
||||
|
||||
return {"provider": name, "error": None, "results": results}
|
||||
|
||||
|
||||
def summarize(all_results: List[dict]) -> None:
|
||||
print("\n" + "=" * 64)
|
||||
print("Suggested verified_ai_models.json entries (passing models only)")
|
||||
print("=" * 64)
|
||||
any_output = False
|
||||
for pr in all_results:
|
||||
if pr.get("error"):
|
||||
continue
|
||||
passed = [r for r in pr["results"] if r["verdict"] == "pass"]
|
||||
if not passed:
|
||||
continue
|
||||
any_output = True
|
||||
passed_sorted = sorted(passed, key=lambda x: x["latency_s"])
|
||||
print(f'\n "{pr["provider"]}": {{')
|
||||
print(' "models": [')
|
||||
for r in passed_sorted:
|
||||
print(f' "{r["model"]}",')
|
||||
print(" ],")
|
||||
print(f' "recommended": "{passed_sorted[0]["model"]}"')
|
||||
print(" },")
|
||||
|
||||
warn_total = sum(
|
||||
1 for pr in all_results for r in pr["results"]
|
||||
if r["verdict"] == "warn"
|
||||
)
|
||||
if warn_total:
|
||||
print(f"\n Note: {warn_total} model(s) came back as ⚠ (warn) — review those manually.")
|
||||
if not any_output:
|
||||
print("\n (no models passed; check keys and network)")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Verify AI models for ProxMenux notification enrichment."
|
||||
)
|
||||
ap.add_argument("--env", default=".env", help=".env file path")
|
||||
ap.add_argument("--provider", action="append", default=[],
|
||||
help="run a specific provider (repeat to add more)")
|
||||
ap.add_argument("--timeout", type=int, default=30,
|
||||
help="seconds per request (default: 30)")
|
||||
ap.add_argument("--limit", type=int, default=None,
|
||||
help="test at most N models per provider (debug)")
|
||||
ap.add_argument("--json-out", default=None,
|
||||
help="write machine-readable report to this path")
|
||||
args = ap.parse_args()
|
||||
|
||||
env = {**os.environ, **load_env(Path(args.env))}
|
||||
|
||||
provider_list = args.provider or list(PROVIDERS.keys())
|
||||
tested: List[dict] = []
|
||||
for name in provider_list:
|
||||
if name not in PROVIDERS:
|
||||
print(f"unknown provider: {name}", file=sys.stderr)
|
||||
continue
|
||||
key_var = f"{name.upper()}_API_KEY"
|
||||
url_var = f"{name.upper()}_BASE_URL"
|
||||
api_key = env.get(key_var, "")
|
||||
base_url = env.get(url_var) or None
|
||||
if not api_key:
|
||||
print(f"\n=== {name} ===\n skipped — {key_var} not set")
|
||||
continue
|
||||
tested.append(run_provider(name, api_key, base_url, args.timeout, args.limit))
|
||||
|
||||
summarize(tested)
|
||||
|
||||
if args.json_out:
|
||||
Path(args.json_out).write_text(json.dumps(tested, indent=2))
|
||||
print(f"\nReport written to {args.json_out}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
name: Verify AI models catalog
|
||||
|
||||
# Runs the AI-model verifier and commits any changes to
|
||||
# AppImage/config/verified_ai_models.json on the same branch the run
|
||||
# was launched from.
|
||||
#
|
||||
# GitHub only fires `on: schedule` from the default branch, so the
|
||||
# daily cron always runs against main and keeps stable users fresh.
|
||||
# When a beta cycle needs its own refresh on develop, use the
|
||||
# "Run workflow" button on the Actions tab and pick develop from the
|
||||
# branch selector — the same YAML then checks out develop, runs the
|
||||
# verifier and commits back to develop. Cross-branch pushes never
|
||||
# happen: each run only touches the branch it started on.
|
||||
#
|
||||
# The verifier code lives at .github/scripts/ai-models-verifier/ and
|
||||
# reads API keys from repository Secrets. Any provider without a key
|
||||
# is skipped silently — the workflow keeps going with the rest.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * *' # 04:00 UTC every day — cron always fires from main
|
||||
workflow_dispatch: # manual trigger — branch is picked in the UI
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
concurrency:
|
||||
# Keyed by branch so a manual develop run does not collide with the
|
||||
# scheduled main run — each branch gets its own serialisation lane.
|
||||
group: verify-ai-models-${{ github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Check out the branch this run belongs to
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# `github.ref_name` resolves to main for the cron and to the
|
||||
# branch selected in the dispatch UI otherwise. The same
|
||||
# value is used again below when we push, so every run is
|
||||
# symmetric: checkout X → refresh → push X.
|
||||
ref: ${{ github.ref_name }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Run verifier
|
||||
working-directory: .github/scripts/ai-models-verifier
|
||||
env:
|
||||
# API keys — each is optional. verify.py silently skips any
|
||||
# provider whose *_API_KEY env var is empty, so the workflow
|
||||
# runs even when only a subset of keys is configured.
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
# Optional base URLs for custom-endpoint providers.
|
||||
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
||||
run: |
|
||||
python3 verify.py --json-out /tmp/report.json || true
|
||||
if [ ! -s /tmp/report.json ]; then
|
||||
echo "Verifier produced no report — bailing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Apply report to catalog
|
||||
id: apply
|
||||
working-directory: .
|
||||
run: |
|
||||
set +e
|
||||
python3 .github/scripts/ai-models-verifier/apply.py \
|
||||
--report /tmp/report.json \
|
||||
--catalog AppImage/config/verified_ai_models.json
|
||||
code=$?
|
||||
set -e
|
||||
case "$code" in
|
||||
0) echo "changed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "changed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "apply.py exited with $code"; exit "$code" ;;
|
||||
esac
|
||||
|
||||
- name: Commit and push back to the same branch
|
||||
if: steps.apply.outputs.changed == 'true'
|
||||
run: |
|
||||
git config user.name "proxmenux-bot"
|
||||
git config user.email "proxmenux-bot@users.noreply.github.com"
|
||||
git add AppImage/config/verified_ai_models.json
|
||||
git commit -m "chore(ai-models): daily catalog refresh"
|
||||
# Push to the branch this run started on — same ref used at
|
||||
# checkout above, so the operation is symmetric regardless of
|
||||
# whether cron (main) or dispatch (any branch) triggered it.
|
||||
git push origin HEAD:${{ github.ref_name }}
|
||||
|
||||
- name: Report unchanged
|
||||
if: steps.apply.outputs.changed != 'true'
|
||||
run: echo "Catalog already up to date — nothing to commit."
|
||||
@@ -341,6 +341,16 @@ export function NotificationSettings() {
|
||||
const [aiTestResult, setAiTestResult] = useState<{ success: boolean; message: string; model?: string } | null>(null)
|
||||
const [providerModels, setProviderModels] = useState<string[]>([])
|
||||
const [loadingProviderModels, setLoadingProviderModels] = useState(false)
|
||||
// Metadata returned by the backend after the catalog refresh — used
|
||||
// to render the "Last update" line and toast the diff.
|
||||
const [catalogMeta, setCatalogMeta] = useState<{
|
||||
success: boolean
|
||||
message: string
|
||||
changed: boolean
|
||||
previous_updated: string | null
|
||||
new_updated: string | null
|
||||
diff: Record<string, { added: string[]; removed: string[] }>
|
||||
} | null>(null)
|
||||
const [showCustomPromptInfo, setShowCustomPromptInfo] = useState(false)
|
||||
const [editingCustomPrompt, setEditingCustomPrompt] = useState(false)
|
||||
const [customPromptDraft, setCustomPromptDraft] = useState("")
|
||||
@@ -993,16 +1003,38 @@ export function NotificationSettings() {
|
||||
|
||||
setLoadingProviderModels(true)
|
||||
try {
|
||||
const data = await fetchApi<{ success: boolean; models: string[]; recommended: string; message: string }>("/api/notifications/provider-models", {
|
||||
const data = await fetchApi<{
|
||||
success: boolean
|
||||
models: string[]
|
||||
recommended: string
|
||||
message: string
|
||||
catalog_meta?: {
|
||||
success: boolean
|
||||
message: string
|
||||
changed: boolean
|
||||
previous_updated: string | null
|
||||
new_updated: string | null
|
||||
diff: Record<string, { added: string[]; removed: string[] }>
|
||||
} | null
|
||||
}>("/api/notifications/provider-models", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
body: JSON.stringify({
|
||||
provider,
|
||||
api_key: apiKey,
|
||||
ollama_url: config.ai_ollama_url,
|
||||
openai_base_url: config.ai_openai_base_url,
|
||||
// Every click pulls the latest verified catalog from GitHub
|
||||
// before the intersection runs, so a user with an outdated
|
||||
// AppImage still sees today's model list. The backend keeps
|
||||
// the local file when the fetch fails, so an offline node
|
||||
// simply doesn't refresh but still resolves models.
|
||||
refresh_catalog: true,
|
||||
}),
|
||||
})
|
||||
if (data.catalog_meta) {
|
||||
setCatalogMeta(data.catalog_meta)
|
||||
}
|
||||
if (data.success && data.models && data.models.length > 0) {
|
||||
setProviderModels(data.models)
|
||||
// Auto-select recommended model if current selection is empty or not in the list
|
||||
@@ -2409,7 +2441,7 @@ export function NotificationSettings() {
|
||||
className="h-9 px-3 shrink-0"
|
||||
onClick={() => fetchProviderModels()}
|
||||
disabled={
|
||||
loadingProviderModels ||
|
||||
loadingProviderModels ||
|
||||
(config.ai_provider === 'ollama' && !config.ai_ollama_url) ||
|
||||
(config.ai_provider !== 'ollama' && config.ai_provider !== 'anthropic' && !config.ai_api_keys?.[config.ai_provider])
|
||||
}
|
||||
@@ -2419,7 +2451,12 @@ export function NotificationSettings() {
|
||||
) : (
|
||||
<>
|
||||
<RefreshCw className="h-4 w-4 mr-1" />
|
||||
{t("settings.notifications.ai.load")}
|
||||
{/* Label swaps once we have models loaded — "Load" reads
|
||||
right on the empty state, "Update" reads right after
|
||||
the first pull. Same action underneath either way. */}
|
||||
{t(providerModels.length === 0
|
||||
? "settings.notifications.ai.load"
|
||||
: "settings.notifications.ai.updateModels")}
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
@@ -2427,6 +2464,32 @@ export function NotificationSettings() {
|
||||
{providerModels.length > 0 && (
|
||||
<p className="text-xs text-green-500">{t("settings.notifications.ai.modelsAvailable", { count: providerModels.length })}</p>
|
||||
)}
|
||||
{/* Catalog freshness line — surfaces the `_updated` date of the
|
||||
verified_ai_models.json currently in use, plus a compact diff
|
||||
summary from the most recent GitHub pull so the user knows
|
||||
what just happened without opening a modal. */}
|
||||
{catalogMeta && (
|
||||
<div className="text-[11px] text-muted-foreground space-y-0.5">
|
||||
<div>
|
||||
{t("settings.notifications.ai.catalogLastUpdated", {
|
||||
date: catalogMeta.new_updated || catalogMeta.previous_updated || "—",
|
||||
})}
|
||||
</div>
|
||||
{catalogMeta.changed && catalogMeta.success && Object.keys(catalogMeta.diff).length > 0 && (
|
||||
<div className="text-blue-400">
|
||||
{t("settings.notifications.ai.catalogChanged", {
|
||||
added: Object.values(catalogMeta.diff).reduce((n, d) => n + d.added.length, 0),
|
||||
removed: Object.values(catalogMeta.diff).reduce((n, d) => n + d.removed.length, 0),
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{!catalogMeta.success && catalogMeta.message && (
|
||||
<div className="text-amber-400">
|
||||
{t("settings.notifications.ai.catalogFetchFailed", { reason: catalogMeta.message })}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Prompt Mode section */}
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "Es ist eine kostenlose Stufe mit einem guten Preis-Leistungs-Verhältnis verfügbar.",
|
||||
"ollama": "Verwendet Modelle auf Ihrem Ollama-Server. Völlig lokal, privat und kostenlos nutzbar.",
|
||||
"openrouter": "Zugriff auf mehr als 100 Modelle über einen API-Schlüssel."
|
||||
}
|
||||
},
|
||||
"updateModels": "Aktualisieren",
|
||||
"catalogLastUpdated": "Katalog zuletzt aktualisiert: {date}",
|
||||
"catalogChanged": "Katalog aktualisiert — {added} hinzugefügt, {removed} entfernt",
|
||||
"catalogFetchFailed": "Katalog konnte nicht aktualisiert werden: {reason}. Lokale Kopie wird verwendet."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Anleitung zur Einrichtung des Telegram-Bots",
|
||||
|
||||
@@ -1921,7 +1921,11 @@
|
||||
"gemini": "A free tier is available, with a good quality-to-price ratio.",
|
||||
"ollama": "Uses models on your Ollama server. Fully local, private and free to run.",
|
||||
"openrouter": "Access to more than 100 models through one API key."
|
||||
}
|
||||
},
|
||||
"updateModels": "Update",
|
||||
"catalogLastUpdated": "Verified catalog last updated: {date}",
|
||||
"catalogChanged": "Catalog refreshed — {added} added, {removed} removed",
|
||||
"catalogFetchFailed": "Could not refresh catalog: {reason}. Using local copy."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Telegram bot setup guide",
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "Hay disponible un nivel gratuito, con una buena relación calidad-precio.",
|
||||
"ollama": "Utiliza modelos en su servidor Ollama. Totalmente local, privado y gratuito.",
|
||||
"openrouter": "Acceso a más de 100 modelos a través de una clave API."
|
||||
}
|
||||
},
|
||||
"updateModels": "Actualizar",
|
||||
"catalogLastUpdated": "Última actualización del catálogo verificado: {date}",
|
||||
"catalogChanged": "Catálogo actualizado — {added} añadidos, {removed} retirados",
|
||||
"catalogFetchFailed": "No se pudo actualizar el catálogo: {reason}. Se usa la copia local."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Guía de configuración del bot de Telegram",
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "Un niveau gratuit est disponible, avec un bon rapport qualité-prix.",
|
||||
"ollama": "Utilise des modèles sur votre serveur Ollama. Entièrement local, privé et gratuit.",
|
||||
"openrouter": "Accès à plus de 100 modèles via une seule clé API."
|
||||
}
|
||||
},
|
||||
"updateModels": "Actualiser",
|
||||
"catalogLastUpdated": "Dernière mise à jour du catalogue vérifié : {date}",
|
||||
"catalogChanged": "Catalogue mis à jour — {added} ajoutés, {removed} retirés",
|
||||
"catalogFetchFailed": "Impossible d'actualiser le catalogue : {reason}. Copie locale utilisée."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Guide de configuration du robot Telegram",
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "È disponibile un livello gratuito, con un buon rapporto qualità-prezzo.",
|
||||
"ollama": "Utilizza i modelli sul tuo server Ollama. Completamente locale, privato e gratuito.",
|
||||
"openrouter": "Accesso a più di 100 modelli tramite una chiave API."
|
||||
}
|
||||
},
|
||||
"updateModels": "Aggiorna",
|
||||
"catalogLastUpdated": "Ultimo aggiornamento del catalogo verificato: {date}",
|
||||
"catalogChanged": "Catalogo aggiornato — {added} aggiunti, {removed} rimossi",
|
||||
"catalogFetchFailed": "Impossibile aggiornare il catalogo: {reason}. Verrà usata la copia locale."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Guida alla configurazione del bot di Telegram",
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "Um nível gratuito está disponível, com uma boa relação qualidade/preço.",
|
||||
"ollama": "Usa modelos em seu servidor Ollama. Totalmente local, privado e de operação gratuita.",
|
||||
"openrouter": "Acesso a mais de 100 modelos através de uma chave API."
|
||||
}
|
||||
},
|
||||
"updateModels": "Atualizar",
|
||||
"catalogLastUpdated": "Última atualização do catálogo verificado: {date}",
|
||||
"catalogChanged": "Catálogo atualizado — {added} adicionados, {removed} removidos",
|
||||
"catalogFetchFailed": "Não foi possível atualizar o catálogo: {reason}. A usar a cópia local."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Guia de configuração do bot do Telegram",
|
||||
|
||||
@@ -1921,7 +1921,11 @@
|
||||
"gemini": "Ponúka bezplatnú úroveň a dobrý pomer kvality a ceny.",
|
||||
"ollama": "Používa modely na vašom Ollama serveri. Beží lokálne, súkromne a bez poplatkov.",
|
||||
"openrouter": "Prístup k viac než 100 modelom cez jeden API kľúč."
|
||||
}
|
||||
},
|
||||
"updateModels": "Aktualizovať",
|
||||
"catalogLastUpdated": "Posledná aktualizácia overeného katalógu: {date}",
|
||||
"catalogChanged": "Katalóg aktualizovaný — {added} pridaných, {removed} odstránených",
|
||||
"catalogFetchFailed": "Katalóg nebolo možné aktualizovať: {reason}. Používa sa lokálna kópia."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Nastavenie Telegram bota",
|
||||
|
||||
@@ -1922,7 +1922,11 @@
|
||||
"gemini": "En gratis nivå är tillgänglig, med ett bra förhållande mellan kvalitet och pris.",
|
||||
"ollama": "Använder modeller på din Ollama-server. Helt lokalt, privat och gratis att köra.",
|
||||
"openrouter": "Tillgång till mer än 100 modeller via en API-nyckel."
|
||||
}
|
||||
},
|
||||
"updateModels": "Uppdatera",
|
||||
"catalogLastUpdated": "Verifierad katalog senast uppdaterad: {date}",
|
||||
"catalogChanged": "Katalog uppdaterad — {added} tillagda, {removed} borttagna",
|
||||
"catalogFetchFailed": "Katalogen kunde inte uppdateras: {reason}. Använder lokal kopia."
|
||||
},
|
||||
"telegramGuide": {
|
||||
"title": "Installationsguide för Telegram bot",
|
||||
|
||||
@@ -402,32 +402,190 @@ def test_notification():
|
||||
return jsonify({'error': f'Internal error ({type(e).__name__})'}), 500
|
||||
|
||||
|
||||
_VERIFIED_MODELS_REMOTE_URL = (
|
||||
"https://raw.githubusercontent.com/MacRimi/ProxMenux/main/"
|
||||
"AppImage/config/verified_ai_models.json"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_verified_models_path() -> Path:
|
||||
"""Locate the on-disk verified_ai_models.json used at runtime.
|
||||
|
||||
AppImage layout keeps scripts and config under /usr/bin/; the dev
|
||||
tree has them one level apart. We probe both and return whichever
|
||||
exists so `load_verified_models` and the refresh helper agree on the
|
||||
same file — otherwise a refresh writing to one path and a read
|
||||
hitting the other would silently do nothing.
|
||||
"""
|
||||
script_dir = Path(__file__).parent
|
||||
candidate = script_dir / 'config' / 'verified_ai_models.json'
|
||||
if candidate.exists():
|
||||
return candidate
|
||||
dev_candidate = script_dir.parent / 'config' / 'verified_ai_models.json'
|
||||
if dev_candidate.exists():
|
||||
return dev_candidate
|
||||
# No file exists yet — return the AppImage-shaped path so a fresh
|
||||
# refresh has somewhere to write. The parent dir is created if needed.
|
||||
candidate.parent.mkdir(parents=True, exist_ok=True)
|
||||
return candidate
|
||||
|
||||
|
||||
def load_verified_models():
|
||||
"""Load verified models from config file.
|
||||
|
||||
Checks multiple paths:
|
||||
1. Same directory as script (AppImage: /usr/bin/config/)
|
||||
2. Parent directory config folder (dev: AppImage/config/)
|
||||
|
||||
Returns {} if the file is missing or unreadable; callers already
|
||||
handle the empty case by returning provider defaults or the API's
|
||||
unfiltered list.
|
||||
"""
|
||||
try:
|
||||
# Try AppImage path first (scripts and config both in /usr/bin/)
|
||||
script_dir = Path(__file__).parent
|
||||
config_path = script_dir / 'config' / 'verified_ai_models.json'
|
||||
|
||||
if not config_path.exists():
|
||||
# Try development path (AppImage/scripts/ -> AppImage/config/)
|
||||
config_path = script_dir.parent / 'config' / 'verified_ai_models.json'
|
||||
|
||||
if config_path.exists():
|
||||
with open(config_path, 'r') as f:
|
||||
path = _resolve_verified_models_path()
|
||||
if path.exists():
|
||||
with open(path, 'r') as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
print(f"[flask_notification_routes] Config not found at {config_path}")
|
||||
print(f"[flask_notification_routes] Config not found at {path}")
|
||||
except Exception as e:
|
||||
print(f"[flask_notification_routes] Failed to load verified models: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def _catalog_signature(catalog: dict) -> dict:
|
||||
"""Reduce a catalog to `{provider: [sorted models]}`.
|
||||
|
||||
Used to diff the local vs the remote catalog without dragging the
|
||||
unrelated `_note` / `_deprecated` / `_updated` metadata into the
|
||||
comparison.
|
||||
"""
|
||||
sig = {}
|
||||
for provider, block in catalog.items():
|
||||
if isinstance(block, dict) and isinstance(block.get('models'), list):
|
||||
sig[provider] = sorted(str(m) for m in block['models'])
|
||||
return sig
|
||||
|
||||
|
||||
def refresh_verified_models_from_github() -> dict:
|
||||
"""Fetch the canonical catalog from the main branch and overwrite the
|
||||
local copy in-place.
|
||||
|
||||
Returns a dict shaped for the API response:
|
||||
{
|
||||
'success': True|False,
|
||||
'message': '...',
|
||||
'changed': bool,
|
||||
'previous_updated': 'YYYY-MM-DD' | None,
|
||||
'new_updated': 'YYYY-MM-DD' | None,
|
||||
'diff': { <provider>: {'added': [...], 'removed': [...]} },
|
||||
}
|
||||
|
||||
Never touches the local file when the remote fetch fails — the
|
||||
installed catalog remains authoritative in the offline / degraded
|
||||
case, which is the deliberate behaviour the UI relies on to show
|
||||
'the catalog is still active' after a failed refresh.
|
||||
"""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
result = {
|
||||
'success': False,
|
||||
'message': '',
|
||||
'changed': False,
|
||||
'previous_updated': None,
|
||||
'new_updated': None,
|
||||
'diff': {},
|
||||
}
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(_VERIFIED_MODELS_REMOTE_URL, method='GET')
|
||||
req.add_header('User-Agent', 'ProxMenux-Monitor/1.1')
|
||||
req.add_header('Accept', 'application/json')
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
raw = resp.read().decode('utf-8')
|
||||
remote = json.loads(raw)
|
||||
except urllib.error.HTTPError as e:
|
||||
result['message'] = f'GitHub returned HTTP {e.code}'
|
||||
return result
|
||||
except urllib.error.URLError as e:
|
||||
result['message'] = f'Could not reach GitHub: {e.reason}'
|
||||
return result
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
result['message'] = f'Remote catalog is not valid JSON: {e}'
|
||||
return result
|
||||
except Exception as e:
|
||||
result['message'] = f'Refresh failed: {type(e).__name__}: {e}'
|
||||
return result
|
||||
|
||||
if not isinstance(remote, dict) or not any(
|
||||
isinstance(remote.get(k), dict) and 'models' in remote[k]
|
||||
for k in remote.keys()
|
||||
if not k.startswith('_')
|
||||
):
|
||||
result['message'] = 'Remote catalog has an unexpected shape'
|
||||
return result
|
||||
|
||||
local = load_verified_models() or {}
|
||||
prev_sig = _catalog_signature(local)
|
||||
new_sig = _catalog_signature(remote)
|
||||
|
||||
diff = {}
|
||||
for provider in sorted(set(prev_sig) | set(new_sig)):
|
||||
prev_models = set(prev_sig.get(provider, []))
|
||||
new_models = set(new_sig.get(provider, []))
|
||||
added = sorted(new_models - prev_models)
|
||||
removed = sorted(prev_models - new_models)
|
||||
if added or removed:
|
||||
diff[provider] = {'added': added, 'removed': removed}
|
||||
|
||||
result['previous_updated'] = local.get('_updated')
|
||||
result['new_updated'] = remote.get('_updated')
|
||||
result['changed'] = bool(diff) or (result['previous_updated'] != result['new_updated'])
|
||||
result['diff'] = diff
|
||||
|
||||
# Only touch disk when something actually changed to keep mtimes
|
||||
# meaningful and avoid a spurious modification in backup diffs.
|
||||
if result['changed']:
|
||||
try:
|
||||
path = _resolve_verified_models_path()
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
with open(tmp, 'w') as f:
|
||||
json.dump(remote, f, indent=2, ensure_ascii=False)
|
||||
f.write('\n')
|
||||
tmp.replace(path)
|
||||
result['success'] = True
|
||||
result['message'] = 'Catalog updated'
|
||||
except Exception as e:
|
||||
result['success'] = False
|
||||
result['message'] = f'Downloaded but failed to save: {type(e).__name__}: {e}'
|
||||
return result
|
||||
else:
|
||||
result['success'] = True
|
||||
result['message'] = 'Catalog already up to date'
|
||||
return result
|
||||
|
||||
|
||||
@notification_bp.route('/api/notifications/refresh-model-catalog', methods=['POST'])
|
||||
@require_auth
|
||||
def refresh_model_catalog():
|
||||
"""Pull the verified_ai_models.json from ProxMenux/main on GitHub and
|
||||
overwrite the local copy. Returns the diff so the UI can toast a
|
||||
meaningful summary (added / removed models per provider) instead of a
|
||||
generic 'refreshed' message.
|
||||
|
||||
Standalone endpoint so an admin can refresh without also fetching
|
||||
provider models; the provider-models call carries a shortcut flag
|
||||
for the common 'refresh + load' click from the Notifications UI.
|
||||
"""
|
||||
try:
|
||||
return jsonify(refresh_verified_models_from_github())
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'message': f'Refresh failed: {type(e).__name__}: {e}',
|
||||
'changed': False,
|
||||
'previous_updated': None,
|
||||
'new_updated': None,
|
||||
'diff': {},
|
||||
}), 500
|
||||
|
||||
|
||||
@notification_bp.route('/api/notifications/provider-models', methods=['POST'])
|
||||
@require_auth
|
||||
def get_provider_models():
|
||||
@@ -459,9 +617,25 @@ def get_provider_models():
|
||||
api_key = _resolve_masked_api_key(provider, data.get('api_key', ''))
|
||||
ollama_url = data.get('ollama_url', 'http://localhost:11434')
|
||||
openai_base_url = data.get('openai_base_url', '')
|
||||
# `refresh_catalog=true` in the request body triggers a GitHub
|
||||
# pull of verified_ai_models.json before the intersection runs.
|
||||
# The Notifications UI passes this on every Load / Update click
|
||||
# so the catalog stays fresh without a separate button.
|
||||
refresh_catalog = bool(data.get('refresh_catalog', False))
|
||||
catalog_meta = None
|
||||
if refresh_catalog:
|
||||
catalog_meta = refresh_verified_models_from_github()
|
||||
|
||||
def _reply(payload, status=200):
|
||||
# Every response from this endpoint carries `catalog_meta`
|
||||
# when a refresh was requested, so the UI can render the
|
||||
# diff toast + "last updated" line from a single roundtrip.
|
||||
if catalog_meta is not None:
|
||||
payload = {**payload, 'catalog_meta': catalog_meta}
|
||||
return jsonify(payload), status
|
||||
|
||||
if not provider:
|
||||
return jsonify({'success': False, 'models': [], 'message': 'Provider not specified'})
|
||||
return _reply({'success': False, 'models': [], 'message': 'Provider not specified'})
|
||||
|
||||
# SSRF guard before we touch the URL. Ollama is local-by-design so
|
||||
# loopback is allowed there; OpenAI base URL must be a real external
|
||||
@@ -469,11 +643,11 @@ def get_provider_models():
|
||||
if provider == 'ollama':
|
||||
ok, err = validate_external_url(ollama_url, allow_loopback=True)
|
||||
if not ok:
|
||||
return jsonify({'success': False, 'models': [], 'message': f'Invalid ollama_url: {err}'}), 400
|
||||
return _reply({'success': False, 'models': [], 'message': f'Invalid ollama_url: {err}'}, 400)
|
||||
if provider == 'openai' and openai_base_url:
|
||||
ok, err = validate_external_url(openai_base_url, allow_loopback=False)
|
||||
if not ok:
|
||||
return jsonify({'success': False, 'models': [], 'message': f'Invalid openai_base_url: {err}'}), 400
|
||||
return _reply({'success': False, 'models': [], 'message': f'Invalid openai_base_url: {err}'}, 400)
|
||||
|
||||
# Load verified models config
|
||||
verified_config = load_verified_models()
|
||||
@@ -494,13 +668,13 @@ def get_provider_models():
|
||||
result = json.loads(resp.read().decode('utf-8'))
|
||||
models = [m.get('name', '') for m in result.get('models', []) if m.get('name')]
|
||||
models = sorted(models)
|
||||
return jsonify({
|
||||
return _reply({
|
||||
'success': True,
|
||||
'models': models,
|
||||
'recommended': models[0] if models else '',
|
||||
'message': f'Found {len(models)} local models'
|
||||
})
|
||||
|
||||
|
||||
# Handle Anthropic - no models list API, return verified models directly
|
||||
if provider == 'anthropic':
|
||||
models = list(verified_models) if verified_models else [
|
||||
@@ -508,31 +682,31 @@ def get_provider_models():
|
||||
'claude-3-5-sonnet-latest',
|
||||
'claude-3-opus-latest',
|
||||
]
|
||||
return jsonify({
|
||||
return _reply({
|
||||
'success': True,
|
||||
'models': sorted(models),
|
||||
'recommended': recommended or models[0],
|
||||
'message': f'{len(models)} verified models'
|
||||
})
|
||||
|
||||
|
||||
# For other providers, fetch from API and filter by verified list.
|
||||
# Custom OpenAI-compatible endpoints (LiteLLM, opencode.ai, vLLM,
|
||||
# LocalAI…) often expose `/v1/models` without authentication, so
|
||||
# we only require an api_key when there's no custom base URL to
|
||||
# consult. Issue #11.5 — OpenCode provider Custom Base URL fetch.
|
||||
if not api_key and not (provider == 'openai' and openai_base_url):
|
||||
return jsonify({'success': False, 'models': [], 'message': 'API key required'})
|
||||
|
||||
return _reply({'success': False, 'models': [], 'message': 'API key required'})
|
||||
|
||||
from ai_providers import get_provider
|
||||
ai_provider = get_provider(
|
||||
provider,
|
||||
api_key=api_key,
|
||||
model='',
|
||||
provider,
|
||||
api_key=api_key,
|
||||
model='',
|
||||
base_url=openai_base_url if provider == 'openai' else None
|
||||
)
|
||||
|
||||
|
||||
if not ai_provider:
|
||||
return jsonify({'success': False, 'models': [], 'message': f'Unknown provider: {provider}'})
|
||||
return _reply({'success': False, 'models': [], 'message': f'Unknown provider: {provider}'})
|
||||
|
||||
# Get all models from provider API
|
||||
api_models = ai_provider.list_models()
|
||||
@@ -551,13 +725,13 @@ def get_provider_models():
|
||||
# so "gpt-4o-mini" as a fallback would be misleading).
|
||||
if verified_models and not is_openai_compat:
|
||||
models = sorted(verified_models)
|
||||
return jsonify({
|
||||
return _reply({
|
||||
'success': True,
|
||||
'models': models,
|
||||
'recommended': recommended or models[0],
|
||||
'message': f'{len(models)} verified models (API unavailable)'
|
||||
})
|
||||
return jsonify({
|
||||
return _reply({
|
||||
'success': False,
|
||||
'models': [],
|
||||
'message': 'Could not retrieve models. Check your API key and endpoint URL.'
|
||||
@@ -567,7 +741,7 @@ def get_provider_models():
|
||||
# Custom OpenAI-compatible endpoint: surface every model the
|
||||
# endpoint reports. No verified-list intersection.
|
||||
models = sorted(api_models)
|
||||
return jsonify({
|
||||
return _reply({
|
||||
'success': True,
|
||||
'models': models,
|
||||
'recommended': models[0] if models else '',
|
||||
@@ -594,20 +768,29 @@ def get_provider_models():
|
||||
else:
|
||||
# No verified list for this provider, return all from API
|
||||
models = sorted(api_models)
|
||||
|
||||
return jsonify({
|
||||
|
||||
return _reply({
|
||||
'success': True,
|
||||
'models': models,
|
||||
'recommended': recommended if recommended in models else (models[0] if models else ''),
|
||||
'message': f'{len(models)} verified models available'
|
||||
})
|
||||
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
# Outside the _reply closure — build the payload manually so the
|
||||
# catch-all still carries catalog_meta when a refresh was
|
||||
# attempted before the crash.
|
||||
payload = {
|
||||
'success': False,
|
||||
'models': [],
|
||||
'message': f'Error: {str(e)}'
|
||||
})
|
||||
'message': f'Error: {str(e)}',
|
||||
}
|
||||
try:
|
||||
if catalog_meta is not None:
|
||||
payload['catalog_meta'] = catalog_meta
|
||||
except UnboundLocalError:
|
||||
pass
|
||||
return jsonify(payload)
|
||||
|
||||
|
||||
@notification_bp.route('/api/notifications/test-ai', methods=['POST'])
|
||||
|
||||
Reference in New Issue
Block a user