Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-09-14 18:56:52 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
9a1642b53c40272bd5a02718c93d507fae6ec7f5
ProxMenux/AppImage/scripts/lxc_apps.py
T
MacRimi ea8c29ecbd New version 1.2.6 
Restores AI Assistant support for OpenAI-compatible endpoints on private IPs, loopback
and Docker networks (LiteLLM, LM Studio, LocalAI, vLLM, OmniRoute, self-hosted proxies)
and surfaces the server's error under the Load button (#325). Aligns the Secure Gateway
wizard's Alpine template selection and pct create with the host's real architecture on
x86_64 and arm64 (#324). Consolidates changes landing on develop: atomic notification
delivery, custom SSH port for Borg remote targets (#236), optional GitHub API token for
app version tracking (#306), and richer replication failure notifications.
2026-09-02 22:18:52 +02:00

4464 lines
187 KiB
Python
Raw Blame History

# ==========================================================
# ProxMenux — LXC App Watch
# ==========================================================
# Per-CT user-registered application metadata + upstream version
# tracking. Sidecar-per-CT under /etc/proxmenux/apps/<vmid>.json,
# mode 0600. Each sidecar carries a LIST of apps because a single
# CT may host several services (e.g. Frigate on 5000 + go2rtc on
# 1984, or a media server that also runs a metrics agent).
#
# The four ``installed_via`` methods (dpkg / apk / file / binary /
# docker) all use ``pct exec`` argv-style — NEVER through ``sh -c``,
# so a user-typed package name or image tag can't inject a shell.
#
# Public surface (called by flask_server.py):
# load_sidecar(vmid) -> dict|None {vmid, apps[], …}
# add_app(vmid, config) -> (bool, saved|error) appends to list
# update_app(vmid, app_id, config) -> (bool, …)
# delete_app(vmid, app_id) -> bool
# delete_all(vmid) -> bool
# check_app(vmid, app_id, force=False) -> dict|None
# check_all(vmid, force=False) -> dict|None
# get_active_apps() -> {str(vmid): [summary, …]}
# get_suggestions(vmid) -> {name, port_suggestions[], web_path_hint}
# ==========================================================
from __future__ import annotations
import datetime
import concurrent.futures
import hashlib
import json
import os
import re
import signal
import shlex
import socket
import subprocess
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from typing import Any, Optional
_APPS_DIR = "/etc/proxmenux/apps"
_PCT_BIN = "/usr/sbin/pct"
_PROBE_TIMEOUT_SEC = 15
_GITHUB_TIMEOUT_SEC = 15
# Aligned with the master LXC update cycle in
# notification_events.PollingCollector (UPDATE_CHECK_INTERVAL = 24 h).
# Previously this was 6 h — half a day out of sync with the apt/apk
# scan — so `refresh_all_apps` inside the 24 h collector would still
# hit GitHub for apps whose upstream TTL had elapsed, doubling
# checks. Unifying both to 24 h means one poll per day drives every
# update flavour (OS packages + community-scripts app upstream).
# Manual "Check" button + post-apply hook still pass force=True and
# ignore this TTL, so the user never has to wait for the timer to
# see a fresh result they explicitly asked for.
_UPSTREAM_CACHE_TTL_SEC = 24 * 3600
_VALID_METHODS = ("dpkg", "apk", "file", "binary",
"python_dist", "docker_label", "docker_exec",
"command", "manual")
_DETECTOR_FIELDS = (
"package", "file_path", "file_regex", "binary_path", "binary_args",
"python_path", "distribution", "container_name", "label",
"command_argv", "installed_version",
)
_VALID_SOURCES = ("releases", "tags")
# Max args for binary / docker_exec / command — bounded so a malformed
# hint can't blow up pct exec with megabytes of argv.
_MAX_BINARY_ARGS = 8
_MAX_BINARY_ARG_LEN = 128
# `command` method is more permissive on arg count than binary_args
# (users may need slightly longer pipelines through subcommands).
_MAX_COMMAND_ARGV = 12
_MAX_COMMAND_ARGV_LEN = 256
# `manual` method holds a user-typed version string. Kept small so a
# broken paste can't blow up the sidecar or downstream renderers.
_MAX_MANUAL_VERSION_LEN = 64
_MAX_UPDATE_COMMAND_LEN = 4096
_MAX_UPSTREAM_URL_LEN = 512
_MAX_UPSTREAM_JSON_PATH_LEN = 128
_MAX_DOCKER_IMAGE_LEN = 255
_VALID_UPSTREAM_TYPES = ("github", "http_json", "docker_hub")
# Scheduled updates: cron-driven runs of apply_updates.sh. Config
# lives at the sidecar top level (per-CT, not per-app). Cron parser
# below supports the standard 5-field syntax with `*`, exact numbers,
# `*/N` step, and comma lists — that covers every preset the UI
# exposes and the freeform "custom" text field.
_VALID_SCHEDULE_TARGETS = ("os", "app", "both")
_SCHEDULE_TARGET_ID_RE = re.compile(
r"^(?:os|apps|app:[A-Za-z0-9_-]{1,64}|docker-engine|docker-(?:compose|container):[A-Za-z0-9][A-Za-z0-9_.-]{0,127}|docker-unit:[a-f0-9]{20})$"
)
_BULK_TARGET_ID_RE = re.compile(
r"^(?:os|app:[A-Za-z0-9_-]{1,64}|docker-engine|docker-unit:[a-f0-9]{20})$"
)
_MAX_CRON_FIELD_LEN = 64
# JSONPath (simplified): letters/digits/dots/underscores/hyphens + [N]
# array indices. Rejects wildcards, filters, .. recursion — we don't
# need JSONPath's full grammar and refusing them keeps parsing tight.
_JSON_PATH_RE = re.compile(r"^[A-Za-z0-9._\-\[\]]+$")
# Docker Hub image: `owner/name` or `name` (defaults to library/name).
# Lowercase per Docker's registry rules; underscore/dash/period allowed.
_DOCKER_IMAGE_RE = re.compile(
r"^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)?$"
)
_DOCKER_COMPOSE_PROJECT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
# Curated tracking hints, keyed by the slug we can recognise for the
# CT (typically the community-scripts slug extracted from
# /usr/bin/update, but any stable identifier works). Each hint carries
# the exact installed_via method + package / binary_path / file
# metadata + GitHub repo + tag_regex we've verified in a real
# container, so the App tab can auto-fill every advanced field.
#
# The map is NOT embedded in this module — it lives in
# json/app_tracking_hints.json in the repo and is fetched at runtime
# with a 7-day cache. Adding a new hint (or fixing a broken one) is
# a commit to that JSON — no AppImage rebuild required, every Monitor
# picks the update up on its next refresh. See _fetch_tracking_hints
# for the fetch pipeline (network → disk cache → bundled fallback).
_TRACKING_HINTS_URL = (
"https://raw.githubusercontent.com/MacRimi/ProxMenux/"
"refs/heads/main/json/app_tracking_hints.json"
)
_TRACKING_HINTS_DISK = "/var/lib/proxmenux/app_tracking_hints.json"
_TRACKING_HINTS_TTL = 7 * 24 * 3600
_TRACKING_HINTS_HTTP_TIMEOUT = 10
# Bundled fallback: build_appimage.sh copies the JSON next to this
# module so the very first Monitor startup works even offline / before
# the JSON has been merged to main.
_TRACKING_HINTS_BUNDLED = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"app_tracking_hints.json",
)
_tracking_hints_lock = threading.RLock()
_tracking_hints_cache: Optional[dict] = None
_tracking_hints_ts: float = 0.0
# Docker Hub tag previews are requested while the user edits a form.
# Cache the raw repository tag list (not the regex result) so changing
# filters does not create another external request. Sixty seconds is
# enough to absorb typing bursts while still feeling live.
_DOCKER_HUB_TAG_CACHE_TTL_SEC = 60
_DOCKER_HUB_TAG_PREVIEW_LIMIT = 5
_DEFAULT_DOCKER_HUB_TAG_REGEX = (
r"(?i)^v?(\d+\.\d+\.\d+(?:[-+._][0-9A-Za-z.-]+)?)$"
)
_MOVING_DOCKER_TAGS = {
"latest", "main", "master", "edge", "stable", "nightly",
"develop", "dev", "rolling", "lts",
}
_docker_hub_tag_cache_lock = threading.RLock()
_docker_hub_tag_cache: dict[str, dict[str, Any]] = {}
# Docker image inventory is deliberately separate from App Watch. The
# Docker engine version and the applications delivered by its images have
# different lifecycles: updating docker-ce does not update a Portainer or
# LinuxServer image. Inventory checks are read-only (docker image ls +
# registry manifest HEAD), cached only in process memory, and never
# pull/recreate anything. Do not create runtime files outside ProxMenux's
# owned application directory just to preserve this derived inventory.
# Docker registry drift follows the same daily rolling check as registered
# application releases. Opening the Updates tab reads this in-memory value;
# only the daily collector, an explicit user check or a completed update forces
# a new registry comparison.
_DOCKER_INVENTORY_TTL_SEC = 24 * 3600
_DOCKER_REGISTRY_TIMEOUT_SEC = 8
_DOCKER_MAX_IMAGES = 50
_DOCKER_MANIFEST_ACCEPT = ", ".join((
"application/vnd.oci.image.index.v1+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v2+json",
))
_docker_inventory_lock = threading.RLock()
_docker_inventory_cache: dict[str, dict] = {}
def _load_bundled_hints() -> dict:
try:
with open(_TRACKING_HINTS_BUNDLED) as f:
data = json.load(f)
return data if isinstance(data, dict) else {}
except (OSError, json.JSONDecodeError):
return {}
def _fetch_tracking_hints() -> dict:
"""Return the curated tracking-hint map (slug → hint dict).
Fetch order: memory cache (fresh) → GitHub raw merged with the bundled
catalog → on-disk cache → bundled JSON. A newly built AppImage can contain
a larger catalog than ``main`` while a phase is being validated; in that
case the smaller remote file must not erase bundled detectors. When the
remote catalog has equal or greater coverage it wins per entry, preserving
the no-rebuild update path. Never raises — a total failure returns an empty
dict so callers can just ``.get(slug)``.
"""
global _tracking_hints_cache, _tracking_hints_ts
with _tracking_hints_lock:
now = time.time()
if _tracking_hints_cache is not None and (now - _tracking_hints_ts) < _TRACKING_HINTS_TTL:
return _tracking_hints_cache
bundled = _load_bundled_hints()
try:
req = urllib.request.Request(
_TRACKING_HINTS_URL,
headers={"User-Agent": "ProxMenux-Monitor"},
)
with urllib.request.urlopen(req, timeout=_TRACKING_HINTS_HTTP_TIMEOUT) as r:
raw = json.loads(r.read().decode("utf-8"))
remote = raw if isinstance(raw, dict) else {}
if len(remote) >= len(bundled):
hints = dict(bundled)
hints.update(remote)
else:
hints = dict(remote)
hints.update(bundled)
_tracking_hints_cache = hints
_tracking_hints_ts = now
try:
os.makedirs(os.path.dirname(_TRACKING_HINTS_DISK), exist_ok=True)
tmp = f"{_TRACKING_HINTS_DISK}.tmp.{os.getpid()}"
with open(tmp, "w") as f:
json.dump({"ts": now, "hints": hints}, f)
os.replace(tmp, _TRACKING_HINTS_DISK)
except OSError:
pass
return hints
except Exception:
if _tracking_hints_cache is not None:
return _tracking_hints_cache
try:
with open(_TRACKING_HINTS_DISK) as f:
disk = json.load(f)
_tracking_hints_cache = disk.get("hints") or {}
_tracking_hints_ts = float(disk.get("ts") or 0)
return _tracking_hints_cache
except (OSError, json.JSONDecodeError):
_tracking_hints_cache = bundled
_tracking_hints_ts = now # avoid re-hammering
return _tracking_hints_cache
# Cheap guardrails on user input. Not exhaustive — the point is to
# reject obvious footguns (shell metachars) before the value ends up
# as a pct-exec argv entry. Real safety comes from never using sh -c.
_PACKAGE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.+@:/\-]{0,127}$")
_PATH_RE = re.compile(r"^/[A-Za-z0-9._/\-+@]{1,255}$")
_REPO_RE = re.compile(r"^[A-Za-z0-9._\-]+/[A-Za-z0-9._\-]+$")
_NAME_RE = re.compile(r"^[\w\s._+\-()/]{1,64}$", re.UNICODE)
# Docker container name / id: lowercase letters/digits/underscore/./-
_DOCKER_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.\-]{0,63}$")
_DESC_RE = re.compile(r"^[\w\s._+\-()/:,]{0,64}$", re.UNICODE)
_WEB_PATH_RE = re.compile(r"^/[\w\-._~:/?#\[\]@!$&'()*+,;=%]{0,254}$")
# http(s) URL for the app logo — restrictive scheme allow-list prevents
# javascript:/data:/file: sneak-ins through the App card's <img src>.
_LOGO_URL_RE = re.compile(r"^https?://[\w\-._~:/?#\[\]@!$&'()*+,;=%]{1,510}$")
# Community-scripts slug — lowercase letters/digits/dashes/underscores/dots.
# Same shape helpers_cache uses for its own slug field.
_HELPER_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
# Web Link category — free-text label the user picks from the presets
# built from helpers_cache.category_names, or types freely. Keep the
# charset permissive enough for community-scripts labels ("Media &
# Streaming", "*Arr Suite", "AI / Coding & Dev-Tools").
_CATEGORY_RE = re.compile(r"^[\w\s&/,.\-*+()]{1,60}$", re.UNICODE)
# OCI label key (e.g. org.opencontainers.image.version) — reverse-DNS
# style dot-separated identifiers.
_OCI_LABEL_RE = re.compile(r"^[a-zA-Z][a-zA-Z0-9._\-]{0,127}$")
# PEP 503 Python distribution name — flexible enough for `open-webui`,
# `python_dotenv`, `Werkzeug`, etc. Case is preserved but comparison
# is case-insensitive at pip level.
_PYDIST_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._\-]{0,127}$")
_cache_lock = threading.RLock()
# ── Storage ────────────────────────────────────────────────────────
def _ensure_dir() -> None:
try:
os.makedirs(_APPS_DIR, mode=0o700, exist_ok=True)
except OSError:
pass
def _sidecar_path(vmid) -> str:
return f"{_APPS_DIR}/{int(vmid)}.json"
def _now_iso() -> str:
return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
def _read_sidecar(vmid) -> Optional[dict]:
path = _sidecar_path(vmid)
try:
with open(path) as f:
data = json.load(f)
if isinstance(data, dict):
return _migrate_legacy(data)
except (FileNotFoundError, json.JSONDecodeError, OSError):
pass
return None
def _migrate_legacy(data: dict) -> dict:
"""The Phase 2c.0 shape stored a single {config, state}. Convert
those files on-read into the new {apps: [...]} shape so upgrades
don't lose the user's registration."""
if "apps" in data and isinstance(data["apps"], list):
return data
if "config" in data and isinstance(data["config"], dict):
legacy_cfg = data["config"]
legacy_state = data.get("state") or {}
# Move the single port + web_path onto the ports[] array
port = legacy_cfg.pop("port", None)
web_path = legacy_cfg.pop("web_path", None)
ports = []
if port:
ports.append({
"port": int(port),
"description": "",
"web_path": web_path or "/",
})
migrated = {
"vmid": data.get("vmid"),
"apps": [{
"id": data.get("app_id") or _new_app_id(),
**legacy_cfg,
"ports": ports,
"state": legacy_state,
}],
"created_at": data.get("created_at") or _now_iso(),
"updated_at": data.get("updated_at") or _now_iso(),
}
return migrated
return {"vmid": data.get("vmid"), "apps": [],
"created_at": data.get("created_at") or _now_iso(),
"updated_at": data.get("updated_at") or _now_iso()}
def _write_sidecar(vmid, data: dict) -> bool:
_ensure_dir()
path = _sidecar_path(vmid)
tmp = f"{path}.tmp.{os.getpid()}"
try:
with open(tmp, "w") as f:
json.dump(data, f, indent=2, sort_keys=True)
os.chmod(tmp, 0o600)
os.replace(tmp, path)
return True
except OSError as e:
print(f"[ProxMenux] lxc_apps: could not write sidecar {path}: {e}")
try:
os.unlink(tmp)
except OSError:
pass
return False
def _new_app_id() -> str:
return uuid.uuid4().hex[:12]
# ── Validation ─────────────────────────────────────────────────────
def _err(msg: str) -> tuple[bool, str]:
return False, msg
def _validate_command_argv(raw: Any) -> tuple[bool, Any]:
"""Validate ``command`` method's argv list. Same shape/rules as
``_validate_binary_args`` but with looser count/length limits and
a REQUIRED non-empty first arg (the command to run). Every arg is
passed argv-style through ``pct exec`` — no shell interpretation,
never — so the only guardrails are size and control characters.
Reference: security policy is "the user typed the command; user
is responsible for what it does". We reject only what would break
the pct-exec argv wire format.
"""
if raw is None or raw == "":
return _err("command_argv is required (non-empty list)")
if not isinstance(raw, list) or not raw:
return _err("command_argv must be a non-empty list of strings")
if len(raw) > _MAX_COMMAND_ARGV:
return _err(f"command_argv accepts at most {_MAX_COMMAND_ARGV} entries")
out: list = []
for i, item in enumerate(raw):
if not isinstance(item, str) or not item:
return _err(f"command_argv[{i}] must be a non-empty string")
if len(item) > _MAX_COMMAND_ARGV_LEN:
return _err(f"command_argv[{i}] exceeds {_MAX_COMMAND_ARGV_LEN} chars")
if "\x00" in item or "\n" in item or "\r" in item:
return _err(f"command_argv[{i}] contains a forbidden control character")
out.append(item)
return True, out
def _validate_binary_args(raw: Any) -> tuple[bool, Any]:
"""Return (True, [args…]) or (False, error). Optional field: an
empty/None input returns ``(True, [])``. Args are passed through
``pct exec`` argv-style — no shell interpretation ever — so the
guardrails are just count/length + reject null bytes and newlines
which would confuse the pct-exec argv wire format.
"""
if raw in (None, ""):
return True, []
if not isinstance(raw, list):
return _err("binary_args must be a list of strings")
if len(raw) > _MAX_BINARY_ARGS:
return _err(f"binary_args accepts at most {_MAX_BINARY_ARGS} entries")
out: list = []
for i, item in enumerate(raw):
if not isinstance(item, str) or not item:
return _err(f"binary_args[{i}] must be a non-empty string")
if len(item) > _MAX_BINARY_ARG_LEN:
return _err(f"binary_args[{i}] exceeds {_MAX_BINARY_ARG_LEN} chars")
if "\x00" in item or "\n" in item or "\r" in item:
return _err(f"binary_args[{i}] contains a forbidden control character")
out.append(item)
return True, out
def _validate_ports(ports_in: Any) -> tuple[bool, Any]:
"""Validate ports[] array: each entry {port: int, description: str,
scheme: "http"|"https", web_path: str}. Empty list = no port
assignment (fine)."""
if ports_in in (None, ""):
return True, []
if not isinstance(ports_in, list):
return _err("ports must be a list of {port, description, scheme}")
out: list = []
seen_ports: set = set()
for i, item in enumerate(ports_in):
if not isinstance(item, dict):
return _err(f"ports[{i}] must be an object")
raw_port = item.get("port")
if raw_port in (None, "", 0):
return _err(f"ports[{i}].port is required")
try:
p = int(raw_port)
except (TypeError, ValueError):
return _err(f"ports[{i}].port must be an integer")
if not (1 <= p <= 65535):
return _err(f"ports[{i}].port must be 1-65535")
if p in seen_ports:
return _err(f"port {p} appears more than once for this app")
seen_ports.add(p)
desc = (item.get("description") or "").strip()
if desc and not _DESC_RE.match(desc):
return _err(f"ports[{i}].description has invalid characters")
scheme = (item.get("scheme") or "http").strip().lower()
if scheme not in ("http", "https"):
return _err(f"ports[{i}].scheme must be 'http' or 'https'")
web = (item.get("web_path") or "/").strip()
if not _WEB_PATH_RE.match(web):
return _err(f"ports[{i}].web_path must be a valid URL path (max 255 chars)")
entry = {"port": p, "description": desc, "scheme": scheme, "web_path": web}
# Per-link logo — optional. Same http(s) allow-list as the
# app-level logo. Used to render each Web Link with its own
# icon (e.g. Portainer on 9000, MakeMKV on 5800).
link_logo = (item.get("logo_url") or "").strip()
if link_logo:
if not _LOGO_URL_RE.match(link_logo):
return _err(f"ports[{i}].logo_url must be an http(s) URL (max 512 chars)")
entry["logo_url"] = link_logo
# Per-link category — optional free-text label the user picks
# from the presets sourced from helpers_cache.category_names.
# Powers the Apps dashboard (filter/group by category).
category = (item.get("category") or "").strip()
if category:
if not _CATEGORY_RE.match(category):
return _err(f"ports[{i}].category has invalid characters or is too long")
entry["category"] = category
# Optional custom URL — takes precedence over the ip:port
# composition when present. Used for apps reached through a
# reverse-proxy domain (e.g. https://vault.example.com) so the
# Apps dashboard opens the public URL instead of the internal
# ip:port. Same http(s) allow-list as the app-level logo.
custom_url = (item.get("custom_url") or "").strip()
if custom_url:
if not _LOGO_URL_RE.match(custom_url):
return _err(f"ports[{i}].custom_url must be an http(s) URL (max 512 chars)")
entry["custom_url"] = custom_url
out.append(entry)
return True, out
def _parse_cron_field(field: str, min_v: int, max_v: int) -> Optional[set]:
"""Expand a single cron field into the set of integers it covers.
Supports: ``*`` (all), ``N`` (exact), ``*/N`` (step), and
comma-separated combinations of those. Returns None on any parse
failure. Ranges (``1-5``) are deliberately unsupported for now —
every UI preset boils down to *, N, or */N.
"""
if not isinstance(field, str) or not field or len(field) > _MAX_CRON_FIELD_LEN:
return None
field = field.strip()
out: set = set()
for part in field.split(","):
part = part.strip()
if not part:
return None
if part == "*":
out.update(range(min_v, max_v + 1))
continue
if part.startswith("*/"):
try:
step = int(part[2:])
except ValueError:
return None
if step <= 0:
return None
out.update(range(min_v, max_v + 1, step))
continue
try:
n = int(part)
except ValueError:
return None
if n < min_v or n > max_v:
return None
out.add(n)
return out if out else None
def _validate_cron(expr: str) -> Optional[str]:
"""Return None if `expr` is a valid 5-field cron the internal
scheduler can honour, else a short error string. Mirrors the
fields expected by `cron_matches` below."""
if not isinstance(expr, str):
return "cron must be a string"
parts = expr.strip().split()
if len(parts) != 5:
return "cron must have exactly 5 space-separated fields (minute hour day month weekday)"
bounds = ((0, 59), (0, 23), (1, 31), (1, 12), (0, 6))
for p, (lo, hi) in zip(parts, bounds):
if _parse_cron_field(p, lo, hi) is None:
return f"cron field '{p}' is not valid"
return None
def cron_matches(expr: str, dt: datetime.datetime) -> bool:
"""True when the cron expression matches the given datetime at
minute granularity. Called every 60s by the scheduler thread; a
False from the parser (invalid expr) matches nothing so a
malformed schedule silently no-ops instead of firing anything
unexpected."""
parts = expr.strip().split()
if len(parts) != 5:
return False
m_set = _parse_cron_field(parts[0], 0, 59)
h_set = _parse_cron_field(parts[1], 0, 23)
d_set = _parse_cron_field(parts[2], 1, 31)
mon_set = _parse_cron_field(parts[3], 1, 12)
dow_set = _parse_cron_field(parts[4], 0, 6)
if not (m_set and h_set and d_set and mon_set and dow_set):
return False
# Python weekday(): Monday=0..Sunday=6. Cron: Sunday=0..Saturday=6.
# Convert Python weekday to cron weekday.
cron_dow = (dt.weekday() + 1) % 7
return (dt.minute in m_set
and dt.hour in h_set
and dt.day in d_set
and dt.month in mon_set
and cron_dow in dow_set)
def validate_schedule(payload: Any) -> tuple[bool, Any]:
"""Validate a schedule config block. Returns
``(True, normalised_schedule)`` or ``(False, error)``. Called by
both the endpoint handler and by config migration so the same
shape check applies everywhere. When `enabled` is false only the
minimum fields are required; the rest are kept so re-enabling
doesn't wipe the operator's cron + toggles."""
if not isinstance(payload, dict):
return _err("schedule must be a JSON object")
enabled = bool(payload.get("enabled"))
cron = (payload.get("cron") or "").strip()
if enabled and not cron:
return _err("cron is required when schedule is enabled")
if cron:
err = _validate_cron(cron)
if err:
return _err(err)
target = (payload.get("target") or "both").strip().lower()
if target not in _VALID_SCHEDULE_TARGETS:
return _err(f"target must be one of: {', '.join(_VALID_SCHEDULE_TARGETS)}")
targets_raw = payload.get("targets")
if targets_raw is None:
# Backward-compatible migration for schedules saved before the
# per-app selector existed. `apps` means every eligible registered
# app and is expanded by the runner at execution time.
targets = (["os"] if target in ("os", "both") else []) + (["apps"] if target in ("app", "both") else [])
else:
if not isinstance(targets_raw, list):
return _err("targets must be a JSON array")
targets = []
for value in targets_raw:
item = str(value or "").strip()
if not _SCHEDULE_TARGET_ID_RE.match(item):
return _err(f"invalid schedule target: {item[:80]}")
if item not in targets:
targets.append(item)
if len(targets) > 128:
return _err("targets may contain at most 128 items")
if enabled and not targets:
return _err("at least one schedule target is required when enabled")
target = "both" if "os" in targets and any(item != "os" for item in targets) else ("os" if targets == ["os"] else "app")
backup = bool(payload.get("backup"))
restart = bool(payload.get("restart"))
release_delay_raw = payload.get("release_delay_days", 0)
try:
release_delay_days = int(release_delay_raw)
except (TypeError, ValueError):
return _err("release_delay_days must be an integer from 0 to 365")
if release_delay_days < 0 or release_delay_days > 365:
return _err("release_delay_days must be an integer from 0 to 365")
backup_storage = (payload.get("backup_storage") or "").strip()
if backup and not backup_storage:
# Not fatal — the runner falls back to the first vzdump-capable
# storage the frontend passes at run time. Persist as empty so
# the UI knows the user relied on the default.
backup_storage = ""
if backup_storage and (len(backup_storage) > 64 or not re.match(r"^[A-Za-z0-9._\-]+$", backup_storage)):
return _err("backup_storage must be a valid PVE storage name")
out: dict = {
"enabled": enabled,
"cron": cron,
"target": target,
"targets": targets,
"backup": backup,
"backup_storage": backup_storage,
"restart": restart,
"release_delay_days": release_delay_days,
}
# Preserve `last_run_at` / `last_run_status` when the caller sent
# them (typical when the scheduler writes back after firing);
# otherwise leave the field unset so persisted values survive.
for k in ("last_run_at", "last_run_status", "last_run_target", "last_run_reason"):
v = payload.get(k)
if v is not None:
out[k] = v
return True, out
def validate_bulk_update(payload: Any) -> tuple[bool, Any]:
"""Validate the reusable manual bulk-update selection.
This configuration is deliberately independent from ``schedule``:
changing what a manual bulk run does must never rewrite the operator's
cron automation. ``os`` is mandatory and at least one additional,
explicit update method must be selected.
"""
if not isinstance(payload, dict):
return _err("bulk_update must be a JSON object")
raw_targets = payload.get("targets")
if not isinstance(raw_targets, list):
return _err("targets must be a JSON array")
targets: list[str] = []
for value in raw_targets:
item = str(value or "").strip()
if not _BULK_TARGET_ID_RE.match(item):
return _err(f"invalid bulk update target: {item[:80]}")
if item not in targets:
targets.append(item)
if len(targets) > 128:
return _err("targets may contain at most 128 items")
if "os" not in targets:
return _err("the OS target is required for a bulk update")
if not any(item != "os" for item in targets):
return _err("at least one application target is required for a bulk update")
return True, {"targets": ["os", *sorted(item for item in targets if item != "os")]}
def validate_config(payload: dict) -> tuple[bool, Any]:
"""Return (True, normalised_config_without_state_id) or
(False, error). Rejects anything that would give shell-injection
at check-time. Only the five fixed installed_via methods are
accepted; each has its own required field set."""
if not isinstance(payload, dict):
return _err("payload must be a JSON object")
name = (payload.get("name") or "").strip()
if not name or not _NAME_RE.match(name):
return _err("name is required and must be 1-64 chars of letters/digits/spaces/._+-()/")
# `installed_via` is OPTIONAL now. When empty, the app is
# "register-only" — we produce clickable web links but never try
# to detect a version, never fetch upstream, never emit warnings.
# This is the default for casual users who just want a link, and
# for docker apps (whose version lifecycle Docker owns).
method = (payload.get("installed_via") or "").strip().lower()
if method and method not in _VALID_METHODS:
return _err(f"installed_via must be one of: {', '.join(_VALID_METHODS)} or empty")
conf: dict = {"name": name}
if method:
conf["installed_via"] = method
if method in ("dpkg", "apk"):
pkg = (payload.get("package") or "").strip()
if not pkg or not _PACKAGE_RE.match(pkg):
return _err("package is required (letters/digits/._+@:/ up to 127 chars)")
conf["package"] = pkg
elif method == "file":
fp = (payload.get("file_path") or "").strip()
if not fp or not _PATH_RE.match(fp):
return _err("file_path is required and must be an absolute path")
fr = payload.get("file_regex") or ""
if not isinstance(fr, str) or not fr.strip():
return _err("file_regex is required")
try:
re.compile(fr)
except re.error as e:
return _err(f"file_regex is not a valid regex: {e}")
conf["file_path"] = fp
conf["file_regex"] = fr.strip()
elif method == "binary":
bp = (payload.get("binary_path") or "").strip()
if not bp or not _PATH_RE.match(bp):
return _err("binary_path is required and must be an absolute path")
conf["binary_path"] = bp
ok, args = _validate_binary_args(payload.get("binary_args"))
if not ok:
return _err(args)
if args:
conf["binary_args"] = args
elif method == "python_dist":
# importlib.metadata.version(<distribution>) run through the
# configured venv's python interpreter. Zero shell, argv-only.
pp = (payload.get("python_path") or "").strip()
if not pp or not _PATH_RE.match(pp):
return _err("python_path is required and must be an absolute path")
dist = (payload.get("distribution") or "").strip()
if not dist or not _PYDIST_RE.match(dist):
return _err("distribution is required (PEP 503 name)")
conf["python_path"] = pp
conf["distribution"] = dist
elif method == "docker_label":
# docker inspect --format '{{index .Config.Labels "<label>"}}' <container>
cn = (payload.get("container_name") or "").strip()
if not cn or not _DOCKER_NAME_RE.match(cn):
return _err("container_name is required (docker naming rules)")
lbl = (payload.get("label") or "").strip()
if not lbl or not _OCI_LABEL_RE.match(lbl):
return _err("label is required (OCI label naming rules)")
conf["container_name"] = cn
conf["label"] = lbl
elif method == "docker_exec":
# docker exec <container> <binary> [args...]
cn = (payload.get("container_name") or "").strip()
if not cn or not _DOCKER_NAME_RE.match(cn):
return _err("container_name is required (docker naming rules)")
bp = (payload.get("binary_path") or "").strip()
# docker_exec binary may be relative (docker resolves PATH inside
# the container) — validate as either an absolute path OR a bare
# binary name (letters/digits/dash/underscore/dot).
if not bp or not (_PATH_RE.match(bp) or re.match(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", bp)):
return _err("binary_path is required (absolute path or bare command)")
conf["container_name"] = cn
conf["binary_path"] = bp
ok, args = _validate_binary_args(payload.get("binary_args"))
if not ok:
return _err(args)
if args:
conf["binary_args"] = args
elif method == "command":
# Advanced-user escape hatch: an arbitrary argv passed to
# `pct exec` inside the CT. User-supplied and user-responsible;
# we validate only size/control-char sanity (see
# _validate_command_argv). Never sh -c, never string; always
# argv, always as-typed.
ok, argv = _validate_command_argv(payload.get("command_argv"))
if not ok:
return _err(argv)
conf["command_argv"] = argv
elif method == "manual":
# No probing ever. The user tells us the installed version, we
# store it verbatim. Version-check flow still fires against
# `repo` if set — the "update available" notification depends
# only on comparing this string to the upstream tag. After
# updating the app, the user edits and updates the string.
v = (payload.get("installed_version") or "").strip()
if not v:
return _err("installed_version is required for the manual method")
if len(v) > _MAX_MANUAL_VERSION_LEN:
return _err(f"installed_version exceeds {_MAX_MANUAL_VERSION_LEN} chars")
# Reject control chars but ALLOW almost anything else — user
# may have version strings like `1.2.3-beta.4+build.5`.
if any(ch in v for ch in "\x00\n\r"):
return _err("installed_version contains a forbidden control character")
conf["installed_version"] = v
# Optional installed_regex — separate from tag_regex for cases where
# local output format differs from the upstream tag (Squid reports
# "7.6-1" via dpkg while upstream tag is "SQUID_7_6"). Falls back to
# tag_regex during detection when unset.
if method:
ir = (payload.get("installed_regex") or "").strip()
if ir:
try:
re.compile(ir)
except re.error as e:
return _err(f"installed_regex is not a valid regex: {e}")
conf["installed_regex"] = ir
# Upstream source — optional, and only meaningful when we have a
# detection method (otherwise there's nothing to compare against).
# Three types supported (`upstream_type` discriminator):
# github — repo + github_source + tag_regex (original path,
# default when `repo` is set and upstream_type is
# omitted, so pre-existing sidecars keep working)
# http_json — GET url, walk upstream_json_path in the JSON reply,
# optionally squeeze the raw value through tag_regex
# docker_hub — list tags for docker_image, filter by tag_regex,
# pick the semver-highest match
if method:
upstream_type = (payload.get("upstream_type") or "").strip().lower()
# Backward-compat: legacy sidecars set `repo` without
# `upstream_type`; treat that as github implicitly.
if not upstream_type and (payload.get("repo") or "").strip():
upstream_type = "github"
if upstream_type:
if upstream_type not in _VALID_UPSTREAM_TYPES:
return _err(f"upstream_type must be one of: {', '.join(_VALID_UPSTREAM_TYPES)}")
# tag_regex is shared across all three types but has
# different roles: mandatory for github (tag → version),
# optional post-processing for http_json (extract a version
# substring from the raw endpoint value), and mandatory
# filter for docker_hub (pick which tags qualify).
tag_regex_raw = (payload.get("tag_regex") or "").strip()
default_tag_regex = r"v?(\d+\.\d+\.\d+)"
tag_regex = tag_regex_raw or default_tag_regex
try:
re.compile(tag_regex)
except re.error as e:
return _err(f"tag_regex is not a valid regex: {e}")
if upstream_type == "github":
repo = (payload.get("repo") or "").strip()
if not repo:
return _err("repo is required for upstream_type=github")
if not _REPO_RE.match(repo):
return _err("repo must be 'owner/name'")
source = (payload.get("github_source") or "releases").strip().lower()
if source not in _VALID_SOURCES:
return _err(f"github_source must be one of: {', '.join(_VALID_SOURCES)}")
conf["upstream_type"] = "github"
conf["repo"] = repo
conf["github_source"] = source
conf["tag_regex"] = tag_regex
elif upstream_type == "http_json":
url = (payload.get("upstream_url") or "").strip()
if not url:
return _err("upstream_url is required for upstream_type=http_json")
if not url.startswith(("http://", "https://")):
return _err("upstream_url must be an http(s) URL")
if len(url) > _MAX_UPSTREAM_URL_LEN:
return _err(f"upstream_url exceeds {_MAX_UPSTREAM_URL_LEN} chars")
path = (payload.get("upstream_json_path") or "").strip()
if not path:
return _err("upstream_json_path is required for upstream_type=http_json")
if len(path) > _MAX_UPSTREAM_JSON_PATH_LEN:
return _err(f"upstream_json_path exceeds {_MAX_UPSTREAM_JSON_PATH_LEN} chars")
if not _JSON_PATH_RE.match(path):
return _err("upstream_json_path uses forbidden characters")
conf["upstream_type"] = "http_json"
conf["upstream_url"] = url
conf["upstream_json_path"] = path
# tag_regex is optional post-processing here; only
# persist when the user explicitly set one so the
# default isn't spuriously applied to raw JSON values
# that already look like clean versions.
if tag_regex_raw:
conf["tag_regex"] = tag_regex
elif upstream_type == "docker_hub":
image = (payload.get("docker_image") or "").strip().lower()
if not image:
return _err("docker_image is required for upstream_type=docker_hub")
if len(image) > _MAX_DOCKER_IMAGE_LEN:
return _err(f"docker_image exceeds {_MAX_DOCKER_IMAGE_LEN} chars")
if not _DOCKER_IMAGE_RE.match(image):
return _err("docker_image must match Docker Hub naming (owner/name or name)")
conf["upstream_type"] = "docker_hub"
conf["docker_image"] = image
if tag_regex_raw:
conf["tag_regex"] = tag_regex
# Ports (list of {port, description, web_path})
ok, ports = _validate_ports(payload.get("ports"))
if not ok:
return _err(ports)
conf["ports"] = ports
# Optional health path (single, applied to the first port if any)
health = (payload.get("health_path") or "").strip()
if health:
if not _WEB_PATH_RE.match(health):
return _err("health_path must be a valid URL path (max 255 chars)")
conf["health_path"] = health
# Optional logo URL — either the auto-fill from the catalog/hint
# or a user-provided URL for a custom app. Restricted to http(s)
# so the browser can't be tricked into loading javascript: / data:
# payloads through the App card's <img src>.
logo = (payload.get("logo_url") or "").strip()
if logo:
if not _LOGO_URL_RE.match(logo):
return _err("logo_url must be an http(s) URL (max 512 chars)")
conf["logo_url"] = logo
# Optional helper_slug — set by the frontend when the user picks
# an auto-detected app (primary or extra). Persisted so we can
# filter the "also detected" chip list against apps already
# registered on this CT.
hs = (payload.get("helper_slug") or "").strip().lower()
if hs:
if not _HELPER_SLUG_RE.match(hs):
return _err("helper_slug must be a lowercase slug (letters/digits/._-)")
conf["helper_slug"] = hs
# Optional user-defined update command. Freeform bash that runs
# under `pct exec vmid -- sh -c "$command"` when the user hits
# "Apply {app} update" from the Updates tab. This is deliberately
# NOT sanitised beyond size/null-byte checks — the threat model
# is "same as if the user typed it via pct exec themselves". The
# user owns the command; ProxMenux only executes it.
uc = payload.get("update_command")
if uc is not None:
if not isinstance(uc, str):
return _err("update_command must be a string")
uc = uc.strip()
if uc:
if len(uc) > _MAX_UPDATE_COMMAND_LEN:
return _err(f"update_command exceeds {_MAX_UPDATE_COMMAND_LEN} chars")
if "\x00" in uc:
return _err("update_command contains a null byte")
conf["update_command"] = uc
# A custom command is the application updater. It always
# replaces /usr/bin/update for that same app. Ignore legacy
# two-step strategy payloads and normalize them on write.
conf["update_strategy"] = "custom_override"
# Optional per-app dismiss flag for the "no update method defined"
# notice shown in the Updates tab. Only affects the notice card;
# the App tab keeps its purple update signal regardless.
hn = payload.get("hide_no_updater_notice")
if hn is not None:
conf["hide_no_updater_notice"] = bool(hn)
# Optional per-app switch for `app_update_available` notifications.
# Default is True (opt-out). Set to False from the App tab when the
# user knows an app can't be updated on their box (compat, forked
# setup, etc.) and wants to keep the "you have updates" badge but
# silence the outbound notification for THIS app only, without
# touching the global toggle.
ne = payload.get("notifications_enabled")
if ne is not None:
conf["notifications_enabled"] = bool(ne)
# Optional per-app switch for the CT's aggregate updates badge.
# Default is False (include). Set to True when the user knowingly
# keeps a specific version (e.g. qBittorrent pinned to the version
# their private tracker requires) and doesn't want the LXC list
# badge blinking about an "available" update that doesn't apply to
# them. Independent from `notifications_enabled` on purpose — a
# user may still want the outbound notification and just hide the
# counter, or the reverse. The App tab itself always shows the
# real state (purple update signal, editor version fields).
efb = payload.get("exclude_from_badge")
if efb is not None:
conf["exclude_from_badge"] = bool(efb)
return True, conf
# ── Version detection: installed side ──────────────────────────────
def _pct_exec(vmid, argv: list[str], timeout: int = _PROBE_TIMEOUT_SEC) -> tuple[int, str, str]:
"""Wrapper around ``pct exec`` argv-style — NEVER through sh -c.
``subprocess.run(timeout=...)`` only kills the immediate ``pct`` process.
A command inside the CT can retain the captured stdout/stderr pipes, which
makes Python wait for that command long after the advertised timeout. This
is especially visible while a restored CT is waiting for
``network-online.target``: ``docker version`` used to keep a request open
for five minutes despite an eight-second timeout. Run ``pct`` in its own
process group and terminate the complete local execution tree so the
deadline is real.
"""
cmd = [_PCT_BIN, "exec", str(vmid), "--"] + argv
process: Optional[subprocess.Popen[str]] = None
try:
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
start_new_session=True,
)
stdout, stderr = process.communicate(timeout=timeout)
return process.returncode, stdout or "", stderr or ""
except subprocess.TimeoutExpired:
if process is not None:
try:
os.killpg(process.pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError, OSError):
try:
process.kill()
except (ProcessLookupError, OSError):
pass
try:
process.communicate(timeout=2)
except (subprocess.TimeoutExpired, OSError):
pass
return 124, "", f"timed out after {timeout}s"
except (FileNotFoundError, OSError) as e:
return 127, "", str(e)
def _extract_version(text: str, pattern: str) -> Optional[str]:
"""Extract a version string from ``text`` using ``pattern``.
- Zero capture groups → return the full match
- One capture group → return that group's text (typical case)
- Multiple capture groups → join with "." — handy for formats like
Paperless-ngx `__version__ = (2, 9, 0)` where each digit lives
in its own group. Empty/None groups are dropped from the join.
"""
try:
m = re.search(pattern, text)
except re.error:
return None
if not m:
return None
groups = [g for g in m.groups() if g]
if len(groups) > 1:
return ".".join(groups)
if len(groups) == 1:
return groups[0]
return m.group(0)
def detect_installed_version(vmid, config: dict) -> tuple[Optional[str], Optional[str]]:
"""Run the configured install-check inside the CT and return
(version, error). Version None + error set on failure.
Version None + error None is reserved for configurations without
an installed-version method. A configured probe that runs but
does not match its regex returns an explicit error; otherwise the
editor cannot distinguish a bad regex from an unconfigured app."""
method = config.get("installed_via")
# installed_regex is applied to the LOCAL command output; tag_regex
# is for the upstream tag string. When installed_regex isn't set,
# tag_regex is reused (backward-compat with older hints).
pattern = config.get("installed_regex") or config.get("tag_regex") or r"(\d+[.\d]+)"
if method == "dpkg":
rc, out, err = _pct_exec(vmid, ["dpkg-query", "-W", "-f=${Version}", config["package"]])
if rc != 0:
low = (err or out).lower()
if "no packages found" in low or "not installed" in low:
return None, f"{config['package']} is not installed via dpkg"
return None, (err or out).strip()[:200] or "dpkg-query failed"
version = _extract_version(out, pattern)
if not version:
return None, "no version matched in dpkg output"
return version, None
if method == "apk":
rc, out, err = _pct_exec(vmid, ["apk", "info", "-v", config["package"]])
if rc != 0:
return None, (err or out).strip()[:200] or "apk info failed"
version = _extract_version(out, pattern)
if not version:
return None, "no version matched in apk output"
return version, None
if method == "file":
rc, out, err = _pct_exec(vmid, ["cat", config["file_path"]])
if rc != 0:
return None, (err or "").strip()[:200] or f"could not read {config['file_path']}"
version = _extract_version(out, config["file_regex"])
if not version:
return None, f"file_regex did not match {config['file_path']}"
return version, None
if method == "binary":
# binary_args defaults to ["--version"] but can be overridden
# for tools like `grafana-cli`, `myapp version`, etc.
args = config.get("binary_args") or ["--version"]
rc, out, err = _pct_exec(vmid, [config["binary_path"], *args])
combined = out + "\n" + err
v = _extract_version(combined, pattern)
if v:
return v, None
if rc != 0:
return None, (err or out).strip()[:200] or "binary invocation failed"
return None, "no version matched in binary output"
if method == "python_dist":
# Runs the venv's python: `python -c 'import importlib.metadata as m;
# print(m.version("<dist>"))'`. distribution name is a literal
# arg, no format string, no eval — safe from injection because
# pct exec never invokes a shell.
dist = config["distribution"]
snippet = (
"import importlib.metadata as m, sys\n"
f"try:\n sys.stdout.write(m.version({dist!r}))\n"
"except Exception as e:\n sys.stderr.write(str(e))\n sys.exit(2)\n"
)
rc, out, err = _pct_exec(vmid, [config["python_path"], "-c", snippet])
if rc != 0:
return None, (err or out).strip()[:200] or "python -c importlib.metadata failed"
version = _extract_version(out, pattern)
if not version:
return None, "no version matched in Python distribution output"
return version, None
if method == "docker_label":
# docker inspect --format '{{index .Config.Labels "<label>"}}' <container>
fmt = '{{index .Config.Labels "' + config["label"] + '"}}'
rc, out, err = _pct_exec(vmid, ["docker", "inspect", "--format", fmt, config["container_name"]])
if rc != 0:
return None, (err or out).strip()[:200] or "docker inspect failed"
text = (out or "").strip()
if not text or text == "<no value>":
return None, f"container has no {config['label']!r} label"
# Reject mutable tags disguised as versions.
if text.lower() in ("latest", "stable", "main", "master", "edge"):
return None, f"docker label reports {text!r} (mutable tag, not a version)"
version = _extract_version(text, pattern)
if not version:
return None, "no version matched in docker label output"
return version, None
if method == "docker_exec":
# docker exec <container> <binary> [args…]
args = config.get("binary_args") or ["--version"]
rc, out, err = _pct_exec(vmid, ["docker", "exec", config["container_name"],
config["binary_path"], *args])
combined = out + "\n" + err
v = _extract_version(combined, pattern)
if v:
return v, None
if rc != 0:
return None, (err or out).strip()[:200] or "docker exec failed"
return None, "no version matched in docker exec output"
if method == "command":
# User-supplied argv, run through pct exec. Zero shell, so
# metachar-injection isn't a class of attack — the user gets
# exactly the argv they typed. installed_regex extracts the
# version from combined stdout + stderr; falls back to
# tag_regex.
argv = list(config.get("command_argv") or [])
if not argv:
return None, "command_argv is empty"
rc, out, err = _pct_exec(vmid, argv)
combined = out + "\n" + err
v = _extract_version(combined, pattern)
if v:
return v, None
if rc != 0:
return None, (err or out).strip()[:200] or "command failed"
return None, "no version matched in command output"
if method == "manual":
# User-typed installed version, no probe. Returned as-is.
v = (config.get("installed_version") or "").strip()
return (v or None), None
# No method configured → register-only, no detection, no errors.
if not method:
return None, None
return None, f"unsupported method: {method}"
def _detector_probe_config(hint: dict, detector: dict) -> dict:
"""Build a complete probe from one detector in a catalog hint.
Alternative detectors intentionally contain only method-specific fields.
Local/upstream regexes and upstream metadata live on the primary hint, so
they must be inherited before calling ``detect_installed_version``.
"""
probe = {"installed_via": detector.get("installed_via")}
for key in _DETECTOR_FIELDS:
if key in detector:
probe[key] = detector[key]
for key in ("installed_regex", "tag_regex"):
if key in detector:
probe[key] = detector[key]
elif key in hint:
probe[key] = hint[key]
return probe
def _ordered_hint_detectors(hint: dict):
"""Yield primary, cross-method alternatives and file fallbacks.
``/root/.<slug>`` is the maintained version contract for modern
community-scripts installs. It is reported as ``helper_marker`` so the UI
can distinguish it from canonical application/package probes; legacy and
manual installs continue through the other detectors.
"""
if not isinstance(hint, dict):
return
primary = {"installed_via": hint.get("installed_via")}
for key in _DETECTOR_FIELDS:
if key in hint:
primary[key] = hint[key]
if primary.get("installed_via"):
primary_source = (
"helper_marker"
if primary.get("installed_via") == "file"
and re.fullmatch(r"/root/\.[A-Za-z0-9_.-]+", str(primary.get("file_path") or ""))
else "primary"
)
yield primary, primary_source
for alt in hint.get("alt_detectors") or []:
if isinstance(alt, dict) and alt.get("installed_via"):
yield alt, "alternative"
for fallback in hint.get("file_fallbacks") or []:
if not isinstance(fallback, dict) or not fallback.get("path"):
continue
detector = {
"installed_via": "file",
"file_path": fallback["path"],
"file_regex": fallback.get("regex") or hint.get("file_regex", ""),
}
fallback_source = (
"helper_marker"
if fallback.get("source") == "helper_marker"
or re.fullmatch(r"/root/\.[A-Za-z0-9_.-]+", str(fallback.get("path") or ""))
else "legacy_fallback"
)
yield detector, fallback_source
def _select_working_hint_detector(vmid, hint: dict) -> tuple[dict, Optional[str], str, Optional[str]]:
"""Return the first detector that produces a parseable real version.
Presence alone is insufficient: a stale file, similarly named package or
helper marker can exist without representing the running app. The return
tuple is ``(tracking_config, version, source, last_error)``. If no probe
succeeds the unchanged primary hint is returned as a catalog candidate,
with source ``candidate`` so the UI can stay transparent.
"""
original = dict(hint) if isinstance(hint, dict) else {}
last_error: Optional[str] = None
for detector, source in _ordered_hint_detectors(original):
probe = _detector_probe_config(original, detector)
try:
version, error = detect_installed_version(vmid, probe)
except (KeyError, TypeError, ValueError) as exc:
version, error = None, str(exc)
if version:
selected = dict(original)
for key in _DETECTOR_FIELDS + ("installed_via",):
selected.pop(key, None)
selected.update(detector)
selected["detected_version"] = version
selected["detector_source"] = source
selected["detector_verified"] = True
return selected, version, source, None
if error:
last_error = error
original["detector_source"] = "candidate"
original["detector_verified"] = False
if last_error:
original["detector_error"] = last_error[:200]
return original, None, "candidate", last_error
# ── Version detection: upstream side ───────────────────────────────
def _github_pat() -> Optional[str]:
try:
from notification_manager import notification_manager
# The notification manager owns the shared encrypted settings store.
# During very early calls its runtime cache may not have been loaded
# yet, so initialise it before reading the optional GitHub token.
if not notification_manager._config:
notification_manager._load_config()
pat = notification_manager._config.get("github_pat")
return pat.strip() if isinstance(pat, str) and pat.strip() else None
except Exception:
return None
def fetch_latest_upstream(config: dict) -> tuple[Optional[str], Optional[str]]:
"""Dispatch to the appropriate upstream fetcher based on
``upstream_type``. Falls back to github for legacy sidecars that
only set ``repo``. Returns (version, error) — version None + error
None means the app has no upstream configured (skip the check)."""
version, error, _published_at = fetch_latest_upstream_details(config)
return version, error
def fetch_latest_upstream_details(config: dict) -> tuple[Optional[str], Optional[str], Optional[str]]:
"""Like :func:`fetch_latest_upstream`, plus the upstream publish date.
The date is intentionally optional. GitHub releases expose a reliable
``published_at`` value; tag lists and arbitrary JSON endpoints often do
not. Scheduled release holds treat an unavailable date conservatively.
"""
upstream_type = config.get("upstream_type")
# Legacy sidecars: repo set, upstream_type not.
if not upstream_type and config.get("repo"):
upstream_type = "github"
if not upstream_type:
return None, None, None
if upstream_type == "github":
return _fetch_github_latest_details(config)
if upstream_type == "http_json":
version, error = _fetch_http_json_latest(config)
return version, error, None
if upstream_type == "docker_hub":
version, error = _fetch_docker_hub_latest(config)
return version, error, None
return None, f"unknown upstream_type: {upstream_type}", None
def _fetch_github_latest(config: dict) -> tuple[Optional[str], Optional[str]]:
version, error, _published_at = _fetch_github_latest_details(config)
return version, error
def _fetch_github_latest_details(config: dict) -> tuple[Optional[str], Optional[str], Optional[str]]:
repo = config.get("repo")
if not repo:
return None, None, None
source = config.get("github_source") or "releases"
if source == "releases":
url = f"https://api.github.com/repos/{urllib.parse.quote(repo, safe='/')}/releases/latest"
else:
url = f"https://api.github.com/repos/{urllib.parse.quote(repo, safe='/')}/tags?per_page=30"
headers = {
"User-Agent": "ProxMenux-Monitor",
"Accept": "application/vnd.github+json",
}
pat = _github_pat()
if pat:
headers["Authorization"] = f"Bearer {pat}"
try:
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=_GITHUB_TIMEOUT_SEC) as r:
payload = json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
if e.code == 404:
return None, f"{repo}: not found", None
if e.code == 403:
remaining = e.headers.get("X-RateLimit-Remaining", "1")
if remaining == "0":
return None, "github rate limited — configure a PAT in Settings → GitHub API", None
return None, "github rejected the request (403)", None
return None, f"github error {e.code}", None
except (urllib.error.URLError, TimeoutError, OSError) as e:
return None, f"network error: {e}", None
pattern = config.get("tag_regex") or r"v?(\d+\.\d+\.\d+)"
tag = None
published_at = None
if source == "releases" and isinstance(payload, dict):
tag = payload.get("tag_name") or payload.get("name")
published_at = payload.get("published_at") or payload.get("created_at")
elif source == "tags" and isinstance(payload, list):
for entry in payload:
candidate = entry.get("name") if isinstance(entry, dict) else None
if candidate:
v = _extract_version(candidate, pattern)
if v:
tag = candidate
break
if not tag:
return None, "no tag / release name in response", None
v = _extract_version(tag, pattern)
if not v:
return None, f"tag_regex did not match '{tag}'", None
return v, None, published_at if isinstance(published_at, str) else None
def _resolve_json_path(data: Any, path: str) -> Any:
"""Simple JSONPath walker — supports dotted keys and ``[N]`` array
indices, e.g. ``computer.Linux.version`` or ``results[0].name``.
Returns None if any step doesn't resolve. Keeps parsing tight (no
wildcards, filters, or recursive descent) so the field is safe to
accept from users."""
if not path:
return None
# Split into path tokens: bare identifiers OR bracketed indices.
parts = re.findall(r"[^.\[\]]+|\[-?\d+\]", path)
if not parts:
return None
node = data
for p in parts:
if p.startswith("["):
try:
idx = int(p[1:-1])
except ValueError:
return None
if not isinstance(node, list):
return None
try:
node = node[idx]
except IndexError:
return None
else:
if not isinstance(node, dict):
return None
if p not in node:
return None
node = node[p]
return node
def _fetch_http_json_latest(config: dict) -> tuple[Optional[str], Optional[str]]:
url = config.get("upstream_url")
json_path = config.get("upstream_json_path")
if not url or not json_path:
return None, None
req = urllib.request.Request(url, headers={"User-Agent": "ProxMenux-Monitor", "Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=_GITHUB_TIMEOUT_SEC) as r:
raw = r.read().decode("utf-8", errors="replace")
except urllib.error.HTTPError as e:
return None, f"http error {e.code}"
except (urllib.error.URLError, TimeoutError, OSError) as e:
return None, f"network error: {e}"
try:
payload = json.loads(raw)
except json.JSONDecodeError as e:
return None, f"invalid JSON response: {e}"
val = _resolve_json_path(payload, json_path)
if val is None:
return None, f"json_path '{json_path}' did not resolve"
val_str = str(val).strip()
if not val_str:
return None, "empty value at json_path"
# Optional tag_regex extraction — only when the user set one. When
# unset we trust the endpoint's value as-is (many vendor APIs
# already publish a clean semver at the target path).
pattern = config.get("tag_regex")
if pattern:
extracted = _extract_version(val_str, pattern)
if not extracted:
return None, f"tag_regex did not match '{val_str}'"
return extracted, None
return val_str, None
def _docker_hub_tag_records(image: str) -> tuple[list[dict], Optional[str]]:
"""Fetch recent Docker Hub tag names and their immutable digests.
Keeping the digest lets the Docker inventory resolve a moving tag such as
``latest`` to a real version tag only when both point at the exact same
manifest. This is deliberately stricter than selecting the numerically
greatest tag, which could silently cross release channels or variants.
"""
if "/" not in image:
image = f"library/{image}"
now = time.time()
with _docker_hub_tag_cache_lock:
cached = _docker_hub_tag_cache.get(image)
if (
cached
and now - float(cached.get("ts") or 0) < _DOCKER_HUB_TAG_CACHE_TTL_SEC
and isinstance(cached.get("records"), list)
):
return [dict(item) for item in cached["records"] if isinstance(item, dict)], None
url = (
f"https://hub.docker.com/v2/repositories/{urllib.parse.quote(image, safe='/')}"
"/tags/?page_size=100&ordering=last_updated"
)
req = urllib.request.Request(url, headers={"User-Agent": "ProxMenux-Monitor", "Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=_GITHUB_TIMEOUT_SEC) as r:
payload = json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
if e.code == 404:
return [], f"docker image '{image}' not found"
if e.code == 429:
return [], "docker hub rate limited the request"
return [], f"docker hub error {e.code}"
except (urllib.error.URLError, TimeoutError, OSError) as e:
return [], f"network error: {e}"
except json.JSONDecodeError as e:
return [], f"invalid docker hub JSON: {e}"
results = payload.get("results") if isinstance(payload, dict) else None
if not isinstance(results, list) or not results:
return [], "no tags returned by docker hub"
records = [
{
"name": str(item.get("name")),
"digest": str(item.get("digest")) if item.get("digest") else None,
}
for item in results
if isinstance(item, dict) and item.get("name")
]
with _docker_hub_tag_cache_lock:
_docker_hub_tag_cache[image] = {"ts": now, "records": records}
return records, None
def _docker_hub_tag_names(image: str) -> tuple[list[str], Optional[str]]:
"""Fetch at most 100 recent Docker Hub tag names, cached per repository."""
records, error = _docker_hub_tag_records(image)
if error:
return [], error
return [str(item["name"]) for item in records if item.get("name")], None
def _docker_tag_semver_key(tag: str) -> tuple:
match = re.search(r"(\d+)\.(\d+)\.(\d+)", tag)
if match:
return (1, int(match.group(1)), int(match.group(2)), int(match.group(3)), tag)
return (0, 0, 0, 0, tag)
_DOCKER_DISPLAY_VERSION_RE = re.compile(
r"(?i)^v?((?:\d+\.){1,3}\d+(?:[-+._][0-9A-Za-z.-]+)?)$"
)
def _normalise_docker_display_version(value: Any) -> Optional[str]:
"""Return a display-safe version only when the whole value is versioned."""
if not isinstance(value, (str, int, float)):
return None
candidate = str(value).strip()
if not candidate or len(candidate) > 128 or candidate.lower() in _MOVING_DOCKER_TAGS:
return None
match = _DOCKER_DISPLAY_VERSION_RE.fullmatch(candidate)
return match.group(1) if match else None
def _docker_version_from_image_inspect(parsed: dict, inspected: dict) -> tuple[Optional[str], Optional[str]]:
"""Resolve an installed image version from local, immutable evidence."""
direct_tag = _normalise_docker_display_version(parsed.get("tag"))
if direct_tag:
return direct_tag, "reference_tag"
labels = ((inspected.get("Config") or {}).get("Labels") or {})
# Application-specific labels take precedence over OCI labels because a
# few publishers put the base distribution version in the latter.
for key in ("Version", "version"):
version = _normalise_docker_display_version(labels.get(key))
if version:
return version, f"image_label:{key}"
# A moving tag often shares an image ID with an explicit release tag
# already present locally (for example frigate:stable + frigate:0.17.2).
alternate_versions: list[str] = []
for repo_tag in inspected.get("RepoTags") or []:
if not isinstance(repo_tag, str) or ":" not in repo_tag:
continue
alt_repo, alt_tag = repo_tag.rsplit(":", 1)
alt = _parse_docker_reference(alt_repo, alt_tag)
if not alt:
continue
if alt.get("registry") != parsed.get("registry") or alt.get("repository") != parsed.get("repository"):
continue
version = _normalise_docker_display_version(alt_tag)
if version:
alternate_versions.append(version)
if alternate_versions:
alternate_versions.sort(key=_docker_tag_semver_key, reverse=True)
return alternate_versions[0], "local_equivalent_tag"
version = _normalise_docker_display_version(labels.get("org.opencontainers.image.version"))
if version:
return version, "image_label:org.opencontainers.image.version"
return None, None
def _docker_hub_version_for_digest(records: list[dict], digest: Optional[str]) -> Optional[str]:
"""Find a version tag whose Docker Hub digest exactly matches ``digest``."""
if not digest:
return None
versions = []
for item in records:
if item.get("digest") != digest:
continue
version = _normalise_docker_display_version(item.get("name"))
if version:
versions.append(version)
if not versions:
return None
versions.sort(key=_docker_tag_semver_key, reverse=True)
return versions[0]
def preview_docker_hub_tags(image: str, pattern: str | None, limit: int = _DOCKER_HUB_TAG_PREVIEW_LIMIT) -> tuple[bool, Any]:
"""Return real tags matching an editor draft without tracking a CT.
Moving tags remain visible in the preview but are flagged because their
update state must be determined by image digest, not version comparison.
"""
image = (image or "").strip().lower()
if not image or len(image) > _MAX_DOCKER_IMAGE_LEN or not _DOCKER_IMAGE_RE.match(image):
return False, "docker_image must match Docker Hub naming (owner/name or name)"
pattern = (pattern or "").strip() or _DEFAULT_DOCKER_HUB_TAG_REGEX
if len(pattern) > 512:
return False, "tag_regex exceeds 512 chars"
try:
regex = re.compile(pattern)
except re.error as exc:
return False, f"tag_regex is not a valid regex: {exc}"
tag_names, error = _docker_hub_tag_names(image)
if error:
return False, error
matched = [tag for tag in tag_names if regex.search(tag)]
matched.sort(key=_docker_tag_semver_key, reverse=True)
entries = []
for tag in matched[:max(1, min(int(limit), 10))]:
entries.append({
"tag": tag,
"version": _extract_version(tag, pattern),
"moving": tag.lower() in _MOVING_DOCKER_TAGS,
})
return True, {
"image": image if "/" in image else f"library/{image}",
"regex": pattern,
"tags": entries,
"matched_count": len(matched),
"scanned_count": len(tag_names),
"cached_for_seconds": _DOCKER_HUB_TAG_CACHE_TTL_SEC,
}
def _fetch_docker_hub_latest(config: dict) -> tuple[Optional[str], Optional[str]]:
image = config.get("docker_image")
if not image:
return None, None
tag_names, error = _docker_hub_tag_names(image)
if error:
return None, error
pattern = config.get("tag_regex") or _DEFAULT_DOCKER_HUB_TAG_REGEX
try:
rx = re.compile(pattern)
except re.error as e:
return None, f"tag_regex is not a valid regex: {e}"
tag_names = [tag for tag in tag_names if rx.search(tag)]
if not tag_names:
return None, "no tags matched (docker_hub)"
versioned_tags = [tag for tag in tag_names if tag.lower() not in _MOVING_DOCKER_TAGS]
if not versioned_tags:
return None, "tag_regex matched only moving tags; use Docker image digest tracking"
versioned_tags.sort(key=_docker_tag_semver_key, reverse=True)
winner = versioned_tags[0]
# If user set a tag_regex with capture groups, run the extraction to
# normalise "v1.2.3" → "1.2.3" and similar.
if pattern:
extracted = _extract_version(winner, pattern)
if extracted:
return extracted, None
return winner, None
# ── Docker image inventory ────────────────────────────────────────
def _parse_docker_reference(repository: str, tag: str) -> Optional[dict]:
"""Normalise a Docker CLI repository/tag into registry API parts."""
repository = (repository or "").strip()
tag = (tag or "").strip()
if not repository or repository == "<none>" or not tag or tag == "<none>":
return None
first, sep, rest = repository.partition("/")
if sep and ("." in first or ":" in first or first == "localhost"):
registry, path = first.lower(), rest
else:
registry = "docker.io"
path = repository
if registry in ("docker.io", "index.docker.io", "registry-1.docker.io"):
registry = "docker.io"
if "/" not in path:
path = f"library/{path}"
api_host = "registry-1.docker.io"
else:
api_host = registry
if not path or any(ch in path for ch in "\x00\r\n"):
return None
return {
"registry": registry,
"api_host": api_host,
"repository": path,
"tag": tag,
"reference": f"{repository}:{tag}",
}
def _normalise_docker_container_reference(reference: str) -> str:
"""Match Docker's implicit ``latest`` tag to ``docker image ls`` rows."""
reference = str(reference or "").strip()
if not reference or "@" in reference:
return reference
final_component = reference.rsplit("/", 1)[-1]
if ":" not in final_component:
return f"{reference}:latest"
return reference
def _parse_bearer_challenge(value: str) -> Optional[dict]:
if not isinstance(value, str) or not value.lower().startswith("bearer "):
return None
params = {}
for key, quoted, bare in re.findall(r'(\w+)=(?:"([^"]*)"|([^,\s]+))', value[7:]):
params[key.lower()] = quoted or bare
realm = params.get("realm")
if not realm or not realm.startswith("https://"):
return None
return params
def _registry_digest_request(url: str, headers: dict) -> tuple[Optional[str], Optional[str]]:
req = urllib.request.Request(url, headers=headers, method="HEAD")
try:
with urllib.request.urlopen(req, timeout=_DOCKER_REGISTRY_TIMEOUT_SEC) as response:
return response.headers.get("Docker-Content-Digest"), None
except urllib.error.HTTPError as exc:
if exc.code != 401:
return None, f"registry HTTP {exc.code}"
challenge = _parse_bearer_challenge(exc.headers.get("WWW-Authenticate", ""))
if not challenge:
return None, "registry authentication required"
query = {
key: challenge[key]
for key in ("service", "scope") if challenge.get(key)
}
token_url = challenge["realm"]
if query:
token_url += ("&" if "?" in token_url else "?") + urllib.parse.urlencode(query)
token_req = urllib.request.Request(
token_url,
headers={"User-Agent": "ProxMenux-Monitor", "Accept": "application/json"},
)
try:
with urllib.request.urlopen(token_req, timeout=_DOCKER_REGISTRY_TIMEOUT_SEC) as response:
token_payload = json.loads(response.read().decode("utf-8"))
token = token_payload.get("token") or token_payload.get("access_token")
if not token:
return None, "registry token response was empty"
auth_headers = dict(headers)
auth_headers["Authorization"] = f"Bearer {token}"
auth_req = urllib.request.Request(url, headers=auth_headers, method="HEAD")
with urllib.request.urlopen(auth_req, timeout=_DOCKER_REGISTRY_TIMEOUT_SEC) as response:
return response.headers.get("Docker-Content-Digest"), None
except urllib.error.HTTPError as auth_exc:
return None, f"registry HTTP {auth_exc.code}"
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as auth_exc:
return None, f"registry network error: {auth_exc}"
except (urllib.error.URLError, TimeoutError, OSError) as exc:
return None, f"registry network error: {exc}"
def _fetch_registry_manifest_digest(parsed: dict) -> tuple[Optional[str], Optional[str]]:
repo = urllib.parse.quote(parsed["repository"], safe="/")
tag = urllib.parse.quote(parsed["tag"], safe="._-")
url = f"https://{parsed['api_host']}/v2/{repo}/manifests/{tag}"
return _registry_digest_request(url, {
"User-Agent": "ProxMenux-Monitor",
"Accept": _DOCKER_MANIFEST_ACCEPT,
})
def _parse_compose_depends_on(value: Any) -> list[str]:
"""Return service names from Compose's ``depends_on`` label.
Compose serialises entries as comma-separated
``service:condition:restart`` tuples. Older releases may emit just the
service name. Invalid names are ignored because they must never reach a
generated shell command.
"""
dependencies: list[str] = []
for raw in str(value or "").split(","):
service = raw.strip().split(":", 1)[0].strip()
if not service or not _DOCKER_COMPOSE_PROJECT_RE.match(service):
continue
if service not in dependencies:
dependencies.append(service)
return sorted(dependencies)
def _docker_compose_update_command(
working_dir: str,
config_files: list[str],
services: list[str],
) -> str:
prefix = ["docker", "compose", "--project-directory", working_dir]
for config_file in config_files:
prefix.extend(["-f", config_file])
pull = " ".join(shlex.quote(arg) for arg in [*prefix, "pull", *services])
recreate = " ".join(shlex.quote(arg) for arg in [
*prefix, "up", "-d", "--no-deps", *services,
])
return f"{pull} && {recreate}"
def _docker_update_unit_id(kind: str, *parts: str) -> str:
identity = "\0".join([kind, *parts]).encode("utf-8")
return f"docker-unit:{hashlib.sha256(identity).hexdigest()[:20]}"
def _build_docker_update_units(images: list[dict]) -> list[dict]:
"""Build reusable Docker selections for manual bulk updates.
Independent services remain independent even when they share a Compose
project. A root service includes only the transitive services named by
its real ``depends_on`` labels. This mirrors the user's mental model of
selecting an app while still recreating its declared database/cache
dependencies exactly once.
"""
projects: dict[str, dict] = {}
for image in images:
for target in image.get("update_targets") or []:
project = str(target.get("project") or "").strip()
working_dir = str(target.get("working_dir") or "").strip()
config_files = [str(value) for value in target.get("config_files") or []]
if not project or not working_dir or not config_files:
continue
project_entry = projects.get(project)
if project_entry and (
project_entry["working_dir"] != working_dir
or project_entry["config_files"] != config_files
):
continue
project_entry = projects.setdefault(project, {
"working_dir": working_dir,
"config_files": config_files,
"services": {},
})
dependencies = target.get("dependencies") or {}
for service in target.get("services") or []:
service = str(service or "").strip()
if not _DOCKER_COMPOSE_PROJECT_RE.match(service):
continue
project_entry["services"].setdefault(service, {
"reference": image.get("reference"),
"display_name": image.get("display_name"),
"logo_url": image.get("logo_url"),
"update_available": image.get("update_available"),
"depends_on": [],
})
service_entry = project_entry["services"][service]
service_entry["depends_on"] = sorted(set(
service_entry.get("depends_on") or []
) | {
str(dep) for dep in dependencies.get(service) or []
if _DOCKER_COMPOSE_PROJECT_RE.match(str(dep))
})
units: list[dict] = []
for project in sorted(projects):
project_entry = projects[project]
service_map: dict[str, dict] = project_entry["services"]
if not service_map:
continue
known_services = set(service_map)
for info in service_map.values():
info["depends_on"] = [
dep for dep in info.get("depends_on") or [] if dep in known_services
]
depended_on = {
dep for info in service_map.values() for dep in info.get("depends_on") or []
}
roots = sorted(known_services - depended_on)
def closure(root: str) -> list[str]:
found: set[str] = set()
pending = [root]
while pending:
service = pending.pop()
if service in found or service not in service_map:
continue
found.add(service)
pending.extend(service_map[service].get("depends_on") or [])
return sorted(found)
# A cycle has no root. Represent each disconnected cyclic component
# once instead of silently dropping it or emitting duplicate units.
if not roots:
roots = [min(known_services)]
covered: set[str] = set()
root_closures: list[tuple[str, list[str]]] = []
for root in roots:
services = closure(root)
root_closures.append((root, services))
covered.update(services)
for service in sorted(known_services - covered):
services = closure(service)
root_closures.append((service, services))
covered.update(services)
for root, services in root_closures:
primary = service_map[root]
references = sorted({
str(service_map[service].get("reference") or "")
for service in services if service_map[service].get("reference")
})
states = [service_map[service].get("update_available") for service in services]
update_available: Optional[bool]
if any(state is True for state in states):
update_available = True
elif states and all(state is False for state in states):
update_available = False
else:
update_available = None
units.append({
"id": _docker_update_unit_id("compose", project, root),
"kind": "compose",
"project": project,
"primary_service": root,
"services": services,
"dependent_services": [service for service in services if service != root],
"references": references,
"primary_reference": primary.get("reference"),
"display_name": primary.get("display_name") or root,
"logo_url": primary.get("logo_url"),
"working_dir": project_entry["working_dir"],
"config_files": project_entry["config_files"],
"update_command": _docker_compose_update_command(
project_entry["working_dir"], project_entry["config_files"], services,
),
"standalone_containers": [],
"update_available": update_available,
})
for image in images:
containers = sorted(set(image.get("standalone_containers") or []))
reference = str(image.get("reference") or "").strip()
if not containers or not reference:
continue
units.append({
"id": _docker_update_unit_id("standalone", reference),
"kind": "standalone",
"project": None,
"primary_service": containers[0],
"services": [],
"dependent_services": [],
"references": [reference],
"primary_reference": reference,
"display_name": image.get("display_name") or reference,
"logo_url": image.get("logo_url"),
"working_dir": None,
"config_files": [],
"update_command": "",
"standalone_containers": containers,
"update_available": image.get("update_available"),
})
return sorted(units, key=lambda unit: (
str(unit.get("display_name") or "").lower(), unit["id"],
))
def _aggregate_docker_compose_projects(images: list[dict]) -> list[dict]:
"""Merge per-image Compose targets into one safe action per project.
One Compose project can contain several services whose images all moved.
The image rows intentionally retain their narrow, service-specific action;
scheduled and bulk updates use this aggregate so the project is pulled and
recreated once with the union of affected services.
"""
projects: dict[str, dict] = {}
for image in images:
for target in image.get("update_targets") or []:
project = str(target.get("project") or "").strip()
working_dir = str(target.get("working_dir") or "").strip()
config_files = [str(value) for value in target.get("config_files") or []]
if not project or not working_dir or not config_files:
continue
existing = projects.get(project)
if existing and (
existing["working_dir"] != working_dir
or existing["config_files"] != config_files
):
# Compose project names should be unique on one engine. If
# provenance conflicts, keep the first verified project rather
# than combining commands from unrelated working directories.
continue
entry = projects.setdefault(project, {
"kind": "compose",
"project": project,
"working_dir": working_dir,
"config_files": config_files,
"services": [],
})
entry["services"] = sorted(set(entry["services"]) | {
str(service) for service in target.get("services") or [] if service
})
result = []
for project in sorted(projects):
target = projects[project]
if not target["services"]:
continue
target["update_command"] = _docker_compose_update_command(
target["working_dir"], target["config_files"], target["services"],
)
result.append(target)
return result
def _docker_inventory_from_ct(vmid) -> dict:
checked_at = _now_iso()
rc, out, err = _pct_exec(
vmid, ["docker", "version", "--format", "{{.Server.Version}}"], timeout=10
)
if rc != 0:
return {
"vmid": int(vmid), "available": False, "engine_version": None,
"images": [], "update_count": 0, "checked_at": checked_at,
"error": (err or out).strip()[:200] or "Docker is not available",
}
engine_version = out.strip()
rc_cont, containers_out, _ = _pct_exec(
vmid,
["docker", "ps", "-a", "--no-trunc", "--format",
"{{.Names}}\t{{.Image}}\t{{.Status}}"],
timeout=10,
)
if rc_cont != 0:
return {
"vmid": int(vmid), "available": False,
"refreshing": False, "engine_version": engine_version,
"images": [], "update_count": 0, "checked_at": checked_at,
"error": "docker container inventory is not ready",
}
containers: list[dict] = []
for line in containers_out.splitlines():
parts = line.split("\t", 2)
if len(parts) == 3:
containers.append({"name": parts[0], "image": parts[1], "status": parts[2]})
# Compose provenance is the safe bridge from read-only detection to
# reproducible updates. Docker Compose writes these labels on every
# service container; unlike guessing a docker-run command, they point
# back to the declarative project, service and config file(s).
if containers:
inspect_argv = [
"docker", "inspect", "--format", "{{json .}}",
*[item["name"] for item in containers],
]
rc_inspect, inspect_out, _ = _pct_exec(vmid, inspect_argv, timeout=20)
inspected: dict[str, dict] = {}
if rc_inspect == 0:
for raw in inspect_out.splitlines():
try:
obj = json.loads(raw)
except json.JSONDecodeError:
continue
name = str(obj.get("Name") or "").lstrip("/")
if name:
inspected[name] = obj
for item in containers:
obj = inspected.get(item["name"]) or {}
config = obj.get("Config") or {}
labels = config.get("Labels") or {}
item["image_reference"] = _normalise_docker_container_reference(
str(config.get("Image") or item.get("image") or "")
)
item["image_id"] = str(obj.get("Image") or "").strip()
project = str(labels.get("com.docker.compose.project") or "").strip()
service = str(labels.get("com.docker.compose.service") or "").strip()
working_dir = str(labels.get("com.docker.compose.project.working_dir") or "").strip()
raw_files = str(labels.get("com.docker.compose.project.config_files") or "").strip()
config_files = [p.strip() for p in raw_files.split(",") if p.strip()]
depends_on = _parse_compose_depends_on(
labels.get("com.docker.compose.depends_on")
)
compose_valid = (
bool(project and service and working_dir and config_files)
and bool(_DOCKER_COMPOSE_PROJECT_RE.match(project))
and bool(_DOCKER_COMPOSE_PROJECT_RE.match(service))
and working_dir.startswith("/")
and all(path.startswith("/") for path in config_files)
and all("\x00" not in value and "\n" not in value for value in [working_dir, *config_files])
)
item["compose"] = ({
"project": project,
"service": service,
"working_dir": working_dir,
"config_files": config_files,
"depends_on": depends_on,
} if compose_valid else None)
rc_img, images_out, images_err = _pct_exec(
vmid,
["docker", "image", "ls", "--no-trunc", "--digests", "--format",
"{{.Repository}}\t{{.Tag}}\t{{.Digest}}\t{{.ID}}"],
timeout=15,
)
if rc_img != 0:
return {
"vmid": int(vmid), "available": False,
"refreshing": False, "engine_version": engine_version,
"images": [], "update_count": 0, "checked_at": checked_at,
"error": (images_err or images_out).strip()[:200] or "docker image ls failed",
}
raw_image_rows: list[tuple[str, str, str, str]] = []
for line in images_out.splitlines():
parts = line.split("\t", 3)
if len(parts) == 4:
raw_image_rows.append(tuple(part.strip() for part in parts))
inspected_images: dict[str, dict] = {}
unique_image_ids = list(dict.fromkeys(row[3] for row in raw_image_rows if row[3]))
if unique_image_ids:
rc_image_inspect, image_inspect_out, _ = _pct_exec(
vmid,
["docker", "image", "inspect", "--format", "{{json .}}", *unique_image_ids],
timeout=30,
)
if rc_image_inspect == 0:
for raw in image_inspect_out.splitlines():
try:
obj = json.loads(raw)
except json.JSONDecodeError:
continue
image_key = str(obj.get("Id") or "").strip()
if image_key:
inspected_images[image_key] = obj
inspected_images[image_key.removeprefix("sha256:")] = obj
images: list[dict] = []
seen: set[str] = set()
for repository, tag, digest, image_id in raw_image_rows:
parsed = _parse_docker_reference(repository, tag)
if not parsed or parsed["reference"] in seen:
continue
seen.add(parsed["reference"])
used_by = sorted({
item["name"] for item in containers
if (
_normalise_docker_container_reference(str(item.get("image_reference") or item.get("image") or ""))
== parsed["reference"]
or str(item.get("image_id") or item.get("image") or "")
in (image_id, image_id.removeprefix("sha256:"))
or str(item.get("image_id") or "").removeprefix("sha256:")
== image_id.removeprefix("sha256:")
)
})
if not used_by:
# Skip orphan images (no container — running or stopped —
# references them). They are residual `docker pull` artifacts
# that would report bogus "update available" entries for tags
# no live workload uses. The user manages orphan cleanup with
# `docker image prune` / `docker rmi` outside of ProxMenux.
continue
used_containers = [item for item in containers if item.get("name") in used_by]
compose_targets: dict[str, dict] = {}
standalone_containers: list[str] = []
for container in used_containers:
compose = container.get("compose")
if not compose:
standalone_containers.append(container["name"])
continue
project = compose["project"]
target = compose_targets.setdefault(project, {
"kind": "compose",
"project": project,
"working_dir": compose["working_dir"],
"config_files": compose["config_files"],
"services": [],
"dependencies": {},
})
if compose["service"] not in target["services"]:
target["services"].append(compose["service"])
target["dependencies"][compose["service"]] = compose.get("depends_on") or []
for target in compose_targets.values():
target["services"].sort()
target["update_command"] = _docker_compose_update_command(
target["working_dir"], target["config_files"], target["services"],
)
inspected_image = (
inspected_images.get(image_id)
or inspected_images.get(image_id.removeprefix("sha256:"))
or {}
)
installed_version, installed_version_source = _docker_version_from_image_inspect(
parsed, inspected_image,
)
primary_container = used_containers[0] if used_containers else {}
primary_compose = primary_container.get("compose") or {}
display_meta = _docker_service_catalog_meta(
str(primary_compose.get("service") or ""),
str(primary_container.get("name") or ""),
parsed["reference"],
)
images.append({
**parsed,
"local_digest": digest if digest.startswith("sha256:") else None,
"remote_digest": None,
"image_id": image_id,
"used_by": used_by,
"update_targets": sorted(compose_targets.values(), key=lambda target: target["project"]),
"standalone_containers": sorted(standalone_containers),
"display_name": display_meta.get("name"),
"logo_url": display_meta.get("logo_url"),
"installed_version": installed_version,
"installed_version_source": installed_version_source,
"available_version": None,
"update_available": None,
"error": None,
})
if len(images) >= _DOCKER_MAX_IMAGES:
break
def _check(item: dict) -> tuple[str, Optional[str], Optional[str]]:
remote, error = _fetch_registry_manifest_digest(item)
return item["reference"], remote, error
if images:
with concurrent.futures.ThreadPoolExecutor(max_workers=min(4, len(images))) as pool:
results = list(pool.map(_check, images))
by_ref = {ref: (digest, error) for ref, digest, error in results}
for item in images:
remote, remote_error = by_ref.get(item["reference"], (None, None))
item["remote_digest"] = remote
item["error"] = remote_error
if item["local_digest"] and remote:
item["update_available"] = item["local_digest"] != remote
# Docker Hub exposes the digest for each tag. Resolve versions only
# when an explicit version tag has the exact local/remote digest;
# otherwise leave the value empty and let the UI report a new image
# without claiming a version number.
hub_repositories = sorted({
item["repository"]
for item in images
if item.get("registry") == "docker.io"
and (not item.get("installed_version") or item.get("update_available") is True)
})
hub_records: dict[str, list[dict]] = {}
if hub_repositories:
with concurrent.futures.ThreadPoolExecutor(max_workers=min(4, len(hub_repositories))) as pool:
fetched = list(pool.map(_docker_hub_tag_records, hub_repositories))
for repository, (records, _error) in zip(hub_repositories, fetched):
hub_records[repository] = records
for item in images:
records = hub_records.get(item.get("repository")) or []
if not records:
continue
if not item.get("installed_version"):
local_version = _docker_hub_version_for_digest(records, item.get("local_digest"))
if local_version:
item["installed_version"] = local_version
item["installed_version_source"] = "docker_hub_digest_tag"
if item.get("update_available") is True:
available_version = _docker_hub_version_for_digest(records, item.get("remote_digest"))
if available_version and available_version != item.get("installed_version"):
item["available_version"] = available_version
return {
"vmid": int(vmid),
"available": True,
"refreshing": False,
"engine_version": engine_version,
"containers": containers,
"images": images,
"compose_projects": _aggregate_docker_compose_projects(images),
"update_units": _build_docker_update_units(images),
"update_count": sum(1 for item in images if item.get("update_available") is True),
"checked_at": checked_at,
"truncated": len(seen) >= _DOCKER_MAX_IMAGES,
"error": None,
}
def get_docker_inventory(vmid, force: bool = False) -> dict:
"""Return cached/read-only Docker image update state for one CT."""
key = str(int(vmid))
with _docker_inventory_lock:
cached = _docker_inventory_cache.get(key)
if cached and not force:
age = time.time() - float(cached.get("checked_at_unix") or 0)
# Retry unavailable/stopped CTs quickly. A full 24-hour
# negative cache made Docker stay unavailable after the user
# started the container from the VM page.
cache_ttl = _DOCKER_INVENTORY_TTL_SEC if cached.get("available") else 30
if age < cache_ttl:
return dict(cached)
result = _docker_inventory_from_ct(vmid)
result["checked_at_unix"] = time.time()
# A restored CT can expose the Docker binary and socket before the daemon
# has finished waiting for network-online.target and loading its image
# store. Never replace a useful inventory (and the stable docker-unit IDs
# referenced by bulk updates) with that transient failure. The snapshot is
# marked as refreshing and uses the short negative TTL, so the next normal
# request retries instead of keeping stale data for 24 hours.
if (
not result.get("available")
and cached
and (
cached.get("refreshing")
or cached.get("images")
or cached.get("update_units")
)
):
pending = dict(cached)
pending.update({
"available": False,
"refreshing": True,
"error": result.get("error") or "Docker is not ready",
"checked_at_unix": result["checked_at_unix"],
})
result = pending
with _docker_inventory_lock:
_docker_inventory_cache[key] = result
return dict(result)
def mark_docker_inventory_refreshing(vmid) -> dict:
"""Publish a non-destructive lifecycle transition for one Docker CT.
Restores and starts invalidate the meaning of the previous digest result,
but its update-unit IDs are still required to render the user's saved bulk
selection. Keep that structural metadata while explicitly marking every
update state as pending until the daemon becomes usable.
"""
key = str(int(vmid))
now = time.time()
with _docker_inventory_lock:
previous = dict(_docker_inventory_cache.get(key) or {})
previous_images = [
{**item, "update_available": None}
for item in (previous.get("images") or [])
if isinstance(item, dict)
]
previous_units = [
{**item, "update_available": None}
for item in (previous.get("update_units") or [])
if isinstance(item, dict)
]
pending = {
"vmid": int(vmid),
"available": False,
"refreshing": True,
"engine_version": previous.get("engine_version"),
"containers": previous.get("containers") or [],
"images": previous_images,
"compose_projects": previous.get("compose_projects") or [],
"update_units": previous_units,
"update_count": 0,
"checked_at": previous.get("checked_at"),
"checked_at_unix": now,
"truncated": bool(previous.get("truncated")),
"error": None,
}
_docker_inventory_cache[key] = pending
return dict(pending)
def get_cached_docker_inventories() -> dict:
"""Return in-memory inventories without probing CTs or registries."""
with _docker_inventory_lock:
return {key: dict(value) for key, value in _docker_inventory_cache.items()}
def invalidate_docker_inventory(vmid) -> None:
"""Drop one CT's Docker snapshot before a lifecycle refresh.
Restoring a container backup can roll Docker Engine, images and Compose
state backwards. Keeping the pre-restore digest snapshot visible while
that CT boots would falsely report the restored workloads as current.
"""
key = str(int(vmid))
with _docker_inventory_lock:
_docker_inventory_cache.pop(key, None)
def refresh_docker_inventories(force: bool = False) -> int:
"""Daily best-effort scan of running LXC containers that host Docker."""
try:
import managed_installs
items = managed_installs.get_active_items() or []
except Exception:
items = []
vmids: set[int] = set()
for item in items:
if item.get("type") != "lxc" or item.get("_vmid") is None:
continue
vmid = int(item["_vmid"])
# Cheap local signature also covers manual Docker installs whose CT
# has no community-scripts slug.
rc, _, _ = _pct_exec(vmid, ["test", "-x", "/usr/bin/docker"], timeout=5)
if rc == 0:
vmids.add(vmid)
count = 0
for vmid in sorted(vmids):
try:
get_docker_inventory(vmid, force=force)
count += 1
except Exception as exc:
print(f"[ProxMenux] Docker inventory CT {vmid} failed: {exc}")
return count
# ── Version comparison ────────────────────────────────────────────
def _version_tuple(v: str) -> tuple:
return tuple(int(x) for x in re.findall(r"\d+", v or ""))
def compare(installed: Optional[str], latest: Optional[str]) -> Optional[bool]:
if not installed or not latest:
return None
ti, tl = _version_tuple(installed), _version_tuple(latest)
if not ti or not tl:
return installed != latest
return tl > ti
# ── Public API ────────────────────────────────────────────────────
def _empty_state() -> dict:
return {
"installed_version": None,
"latest_version": None,
"latest_published_at": None,
"update_available": None,
"error": None,
"checked_at": None,
}
def test_config(vmid, payload: dict) -> tuple[bool, Any]:
"""Validate and execute an app-tracking draft without persisting it.
This is the editor's diagnostic contract. It deliberately returns
the installed and upstream outcomes separately so a user can tell a
bad local detector from a bad repository/tag filter. Probe output is
never returned: a user-selected file or command could contain secrets.
"""
ok, config = validate_config(payload)
if not ok:
return False, config
method = config.get("installed_via")
upstream_type = config.get("upstream_type")
if not upstream_type and config.get("repo"):
upstream_type = "github"
installed_version, installed_error = detect_installed_version(vmid, config)
latest_version, upstream_error, latest_published_at = fetch_latest_upstream_details(config)
effective_regex = None
if method:
effective_regex = (
config.get("file_regex") if method == "file"
else config.get("installed_regex")
or config.get("tag_regex")
or r"(\d+[.\d]+)"
)
return True, {
"valid": True,
"persisted": False,
"checked_at": _now_iso(),
"installed": {
"configured": bool(method),
"method": method,
"effective_regex": effective_regex,
"version": installed_version,
"error": installed_error,
},
"upstream": {
"configured": bool(upstream_type),
"type": upstream_type,
"version": latest_version,
"published_at": latest_published_at,
"error": upstream_error,
},
"update_available": compare(installed_version, latest_version),
}
def load_sidecar(vmid) -> Optional[dict]:
return _read_sidecar(vmid)
def set_dismissed_slug(vmid, slug: str, dismissed: bool) -> tuple[bool, Any]:
"""Add/remove a slug from the per-CT ``dismissed_slugs`` list.
Dismissed slugs are auto-detected apps the user chose to hide from
the "Detected on this container" chip list. Persisted so the
detection doesn't come back on every page reload. Registering an
app for the same slug afterwards implicitly un-dismisses (the
filter also excludes registered slugs).
"""
slug = (slug or "").strip().lower()
if not slug or not _HELPER_SLUG_RE.match(slug):
return False, "invalid slug"
with _cache_lock:
sidecar = _read_sidecar(vmid) or {
"vmid": int(vmid), "apps": [],
"created_at": _now_iso(), "updated_at": _now_iso(),
}
current = list(sidecar.get("dismissed_slugs") or [])
if dismissed:
if slug not in current:
current.append(slug)
else:
current = [s for s in current if s != slug]
sidecar["dismissed_slugs"] = current
sidecar["updated_at"] = _now_iso()
if not _write_sidecar(vmid, sidecar):
return False, "could not persist sidecar (permission?)"
return True, _read_sidecar(vmid)
def _find_app(sidecar: dict, app_id: str) -> Optional[dict]:
for app in sidecar.get("apps") or []:
if app.get("id") == app_id:
return app
return None
def add_app(vmid, payload: dict) -> tuple[bool, Any]:
ok, cfg = validate_config(payload)
if not ok:
return False, cfg
with _cache_lock:
sidecar = _read_sidecar(vmid) or {
"vmid": int(vmid), "apps": [],
"created_at": _now_iso(), "updated_at": _now_iso(),
}
sidecar.setdefault("apps", [])
new_id = _new_app_id()
sidecar["apps"].append({
"id": new_id,
**cfg,
"state": _empty_state(),
"created_at": _now_iso(),
})
sidecar["updated_at"] = _now_iso()
if not _write_sidecar(vmid, sidecar):
return False, "could not persist sidecar (permission?)"
# Kick a first check so the UI shows real numbers immediately
check_app(vmid, new_id, force=True)
return True, _read_sidecar(vmid)
def update_app(vmid, app_id: str, payload: dict) -> tuple[bool, Any]:
ok, cfg = validate_config(payload)
if not ok:
return False, cfg
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar:
return False, "no apps registered for this vmid"
app = _find_app(sidecar, app_id)
if not app:
return False, f"app_id '{app_id}' not found"
# Preserve id + created_at + state; replace the rest
state = app.get("state") or _empty_state()
created = app.get("created_at") or _now_iso()
idx = sidecar["apps"].index(app)
sidecar["apps"][idx] = {
"id": app_id, **cfg, "state": state, "created_at": created,
}
sidecar["updated_at"] = _now_iso()
if not _write_sidecar(vmid, sidecar):
return False, "could not persist sidecar"
check_app(vmid, app_id, force=True)
return True, _read_sidecar(vmid)
def delete_app(vmid, app_id: str) -> bool:
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar:
return True
before = len(sidecar.get("apps") or [])
sidecar["apps"] = [a for a in sidecar.get("apps") or [] if a.get("id") != app_id]
sidecar["updated_at"] = _now_iso()
# If the CT has no apps left, remove the sidecar entirely so
# the empty state shows correctly.
if not sidecar["apps"]:
try:
os.unlink(_sidecar_path(vmid))
return True
except OSError:
pass
if before != len(sidecar["apps"]):
_write_sidecar(vmid, sidecar)
return True
def delete_all(vmid) -> bool:
try:
os.unlink(_sidecar_path(vmid))
return True
except FileNotFoundError:
return True
except OSError as e:
print(f"[ProxMenux] lxc_apps: delete_all failed: {e}")
return False
# ── External update-cron detection ─────────────────────────────────
#
# Some users already run the community-scripts host-wide cron
# (`cron-update-lxcs.sh`) to auto-update every LXC on the node. We
# don't try to compete with that or ask them to remove it — the UX
# just reflects "already covered by an external cron" so ProxMenux's
# own per-CT scheduler is offered as an addition, not a replacement.
#
# Scope is DELIBERATELY strict: only patterns tied to community-scripts
# specifically (their published script name + the well-known repo
# path) so we never flag a random user cron that touches `pct` — false
# positives here would just add noise. When community-scripts publishes
# new update scripts, add their identifiers to this list.
# Each entry: (pattern, variant, scope). `scope` describes what the
# cron actually touches — verified by reading each script's source.
# Both known variants only run `apt-get dist-upgrade` / `apk upgrade`
# inside every CT; neither invokes `/usr/bin/update`, so per-app
# helper updates are NOT covered. `scope="os"` reflects that.
_EXTERNAL_CRON_MATCHERS = (
("update-lxcs-cron.sh", "community-scripts", "os"),
("cron-update-lxcs.sh", "community-scripts", "os"),
("tteck/Proxmox", "tteck-legacy", "os"),
("update-apps.sh", "unknown", "unknown"),
("community-scripts/ProxmoxVE", "community-scripts", "os"),
)
_EXTERNAL_CRON_LOCATIONS = (
"/etc/cron.d",
"/etc/cron.hourly",
"/etc/cron.daily",
"/etc/cron.weekly",
"/etc/cron.monthly",
"/var/spool/cron/crontabs",
)
def _humanise_cron(cron_5field: str) -> str:
"""Turn a 5-field cron expression into a plain-English label for
the UI. Falls back to the raw expression when the shape doesn't
match one of the presets the picker exposes."""
if not isinstance(cron_5field, str):
return ""
parts = cron_5field.strip().split()
if len(parts) != 5:
return cron_5field
m, h, d, mo, w = parts
def _hhmm() -> str:
try:
return f"{int(h):02d}:{int(m):02d}"
except ValueError:
return f"{h}:{m}"
if d == "*" and mo == "*" and w == "*" and m.isdigit() and h.isdigit():
return f"Daily at {_hhmm()}"
if d == "*" and mo == "*" and w.isdigit() and m.isdigit() and h.isdigit():
wdays = ["Sunday", "Monday", "Tuesday", "Wednesday",
"Thursday", "Friday", "Saturday"]
wname = wdays[int(w)] if 0 <= int(w) <= 6 else w
return f"Weekly ({wname} {_hhmm()})"
if mo == "*" and w == "*" and d.isdigit() and m.isdigit() and h.isdigit():
return f"Monthly (day {int(d)} at {_hhmm()})"
if h == "*" and d == "*" and mo == "*" and w == "*" and m == "0":
return "Hourly"
return cron_5field
def _scan_cron_line(line: str) -> Optional[dict]:
"""Try to interpret ``line`` as a cron entry that references one
of the known external update patterns. Returns
``{cron, cron_line, human_schedule, variant, scope}`` or None
when the line isn't a match. ``variant`` identifies which known
updater the cron drives (tteck-legacy, community-scripts,
unknown); ``scope`` is what that variant actually touches (os,
unknown). Silently skips comments and blank lines."""
line = line.strip()
if not line or line.startswith("#"):
return None
matched = None
for pat, variant, scope in _EXTERNAL_CRON_MATCHERS:
if pat in line:
matched = (variant, scope)
break
if not matched:
return None
tokens = line.split(None, 6)
if len(tokens) < 6:
return None
cron_5 = " ".join(tokens[:5])
if _validate_cron(cron_5) is not None:
return None
return {
"cron": cron_5,
"cron_line": line,
"human_schedule": _humanise_cron(cron_5),
"variant": matched[0],
"scope": matched[1],
}
def detect_external_update_cron() -> Optional[dict]:
"""Walk the well-known cron locations looking for a community-
scripts update entry. First hit wins and is returned as
``{source, cron_line, cron, human_schedule, type}``; None when
nothing recognised is present. Errors reading a file are ignored
silently — a permissions issue on one entry shouldn't blow up
the whole probe."""
for loc in _EXTERNAL_CRON_LOCATIONS:
if not os.path.isdir(loc):
# Might be a single file (cron.hourly is a dir, but
# `/etc/crontab` — added below — is a file).
if os.path.isfile(loc):
try:
with open(loc) as f:
for raw in f:
parsed = _scan_cron_line(raw)
if parsed:
return {**parsed, "source": loc, "type": parsed["variant"]}
except OSError:
pass
continue
try:
names = sorted(os.listdir(loc))
except OSError:
continue
for name in names:
path = os.path.join(loc, name)
if not os.path.isfile(path):
continue
try:
with open(path) as f:
for raw in f:
parsed = _scan_cron_line(raw)
if parsed:
return {**parsed, "source": path, "type": parsed["variant"]}
except OSError:
continue
# /etc/crontab (single file at root)
try:
with open("/etc/crontab") as f:
for raw in f:
parsed = _scan_cron_line(raw)
if parsed:
return {**parsed, "source": "/etc/crontab", "type": parsed["variant"]}
except OSError:
pass
return None
# ── Scheduled updates CRUD ──────────────────────────────────────────
def get_schedule(vmid) -> Optional[dict]:
"""Return the persisted schedule config for this vmid, or None if
the sidecar has no schedule set. Safe on missing sidecar."""
sidecar = _read_sidecar(vmid)
if not sidecar:
return None
sched = sidecar.get("schedule")
return sched if isinstance(sched, dict) else None
def update_schedule(vmid, payload: dict) -> tuple[bool, Any]:
"""Persist a new/updated schedule config. Creates the sidecar if
the CT hasn't registered any apps yet — a bare CT can still be
scheduled for OS updates (target=os). Returns (True, sidecar) or
(False, err_msg)."""
ok, sched = validate_schedule(payload)
if not ok:
return False, sched
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar:
sidecar = {
"vmid": vmid,
"apps": [],
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
# Merge over previous schedule so last_run_at etc. survive an
# edit that doesn't re-send them.
prev = sidecar.get("schedule") or {}
merged = dict(prev)
merged.update(sched)
sidecar["schedule"] = merged
sidecar["updated_at"] = _now_iso()
if not _write_sidecar(vmid, sidecar):
return False, "could not persist sidecar"
return True, sidecar
def delete_schedule(vmid) -> bool:
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar or "schedule" not in sidecar:
return True
sidecar.pop("schedule", None)
sidecar["updated_at"] = _now_iso()
return _write_sidecar(vmid, sidecar)
# ── Manual bulk updates CRUD ───────────────────────────────────────
def get_bulk_update(vmid) -> Optional[dict]:
"""Return the persisted manual bulk selection, if configured."""
sidecar = _read_sidecar(vmid)
if not sidecar:
return None
bulk = sidecar.get("bulk_update")
return bulk if isinstance(bulk, dict) else None
def update_bulk_update(vmid, payload: dict) -> tuple[bool, Any]:
"""Persist a reusable bulk selection independently from cron options."""
ok, bulk = validate_bulk_update(payload)
if not ok:
return False, bulk
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar:
sidecar = {
"vmid": vmid,
"apps": [],
"created_at": _now_iso(),
"updated_at": _now_iso(),
}
previous = sidecar.get("bulk_update") or {}
now = _now_iso()
bulk["configured_at"] = previous.get("configured_at") or now
bulk["updated_at"] = now
sidecar["bulk_update"] = bulk
sidecar["updated_at"] = now
if not _write_sidecar(vmid, sidecar):
return False, "could not persist sidecar"
return True, sidecar
def delete_bulk_update(vmid) -> bool:
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar or "bulk_update" not in sidecar:
return True
sidecar.pop("bulk_update", None)
sidecar["updated_at"] = _now_iso()
return _write_sidecar(vmid, sidecar)
def get_all_schedules() -> list:
"""Enumerate every sidecar with a schedule set. Used by the
scheduler thread every minute to know which CTs to check.
Returns a list of ``{vmid: int, schedule: dict}`` — one entry per
CT with a non-empty schedule (enabled OR disabled; the scheduler
decides whether to fire)."""
out: list = []
try:
entries = os.listdir(_APPS_DIR)
except (FileNotFoundError, OSError):
return out
for name in entries:
if not name.endswith(".json"):
continue
try:
vmid = int(name[:-5])
except ValueError:
continue
sidecar = _read_sidecar(vmid)
if not sidecar:
continue
sched = sidecar.get("schedule")
if isinstance(sched, dict) and sched.get("cron"):
out.append({"vmid": vmid, "schedule": sched})
return out
def partition_scheduled_release_targets(
targets: list[str],
apps: list[dict],
) -> tuple[set[str], list[str]]:
"""Split scheduled targets by whether the release-age hold applies.
Only registered applications with an installed-version detector are
release-gated. Web-Link-only apps with a custom updater, Docker targets
and OS packages remain executable on every scheduled occurrence. The
second return value is the target list to use when the gated applications
must be deferred. Legacy ``apps`` wildcards are expanded to explicit
untracked registrations so those commands are not accidentally blocked.
"""
raw_targets = [str(value) for value in (targets or [])]
select_all_apps = "apps" in raw_targets
selected_app_ids = {
value.split(":", 1)[1]
for value in raw_targets
if value.startswith("app:")
}
gated_ids: set[str] = set()
eligible_apps: list[dict] = []
for app in apps or []:
app_id = str(app.get("id") or "").strip()
if not app_id or app.get("managed_oci_app_id") or app.get("helper_slug") == "docker":
continue
if not select_all_apps and app_id not in selected_app_ids:
continue
eligible_apps.append(app)
if app.get("installed_via"):
gated_ids.add(app_id)
remaining: list[str] = []
def add(target_id: str) -> None:
if target_id and target_id not in remaining:
remaining.append(target_id)
for target_id in raw_targets:
if target_id == "apps":
for app in eligible_apps:
app_id = str(app.get("id") or "").strip()
if app_id and app_id not in gated_ids:
add(f"app:{app_id}")
continue
if target_id.startswith("app:") and target_id.split(":", 1)[1] in gated_ids:
continue
add(target_id)
return gated_ids, remaining
def scheduled_app_release_gate(
vmid,
delay_days: int,
now: Optional[datetime.datetime] = None,
app_ids: Optional[set[str]] = None,
) -> dict:
"""Decide whether a scheduled app updater may run after a release hold.
Manual updates never call this function. A non-zero hold first refreshes
every registered detector, then permits automation only when at least one
update is known and every pending release has a trustworthy publish date
old enough for the configured delay. This is deliberately conservative:
a CT-wide helper updater cannot skip one young application safely.
"""
try:
days = int(delay_days)
except (TypeError, ValueError):
days = 0
if days <= 0:
return {"allowed": True, "status": "ready", "reason": None}
try:
check_all(vmid, force=True)
except Exception as exc:
return {"allowed": False, "status": "deferred", "reason": f"version refresh failed: {exc}"}
sidecar = _read_sidecar(vmid) or {}
tracked = [
app for app in (sidecar.get("apps") or [])
if app.get("installed_via") and not app.get("managed_oci_app_id")
and (app_ids is None or app.get("id") in app_ids)
]
if not tracked:
return {
"allowed": False,
"status": "deferred",
"reason": "release hold requires at least one version-tracked app",
}
unknown = [app for app in tracked if (app.get("state") or {}).get("update_available") is None]
if unknown:
names = ", ".join(str(app.get("name") or "app") for app in unknown[:3])
return {
"allowed": False,
"status": "deferred",
"reason": f"release date/update state unavailable for: {names}",
}
pending = [app for app in tracked if (app.get("state") or {}).get("update_available") is True]
if not pending:
return {"allowed": False, "status": "skipped", "reason": "no pending app updates"}
current = now or datetime.datetime.now(datetime.timezone.utc)
if current.tzinfo is None:
current = current.replace(tzinfo=datetime.timezone.utc)
for app in pending:
state = app.get("state") or {}
raw_date = state.get("latest_published_at")
if not isinstance(raw_date, str) or not raw_date.strip():
return {
"allowed": False,
"status": "deferred",
"reason": f"upstream publish date unavailable for {app.get('name') or 'app'}",
}
try:
published = datetime.datetime.fromisoformat(raw_date.replace("Z", "+00:00"))
if published.tzinfo is None:
published = published.replace(tzinfo=datetime.timezone.utc)
except ValueError:
return {
"allowed": False,
"status": "deferred",
"reason": f"invalid upstream publish date for {app.get('name') or 'app'}",
}
eligible_at = published + datetime.timedelta(days=days)
if current < eligible_at:
return {
"allowed": False,
"status": "deferred",
"reason": f"release hold active until {eligible_at.isoformat()}",
}
return {"allowed": True, "status": "ready", "reason": None}
def record_schedule_run(
vmid,
status: str,
target: str,
reason: Optional[str] = None,
*,
log_name: Optional[str] = None,
reboot_required: Optional[bool] = None,
reboot_packages: Optional[list[str]] = None,
) -> bool:
"""Called by the scheduler after a fired run completes. Updates
the schedule with last_run_at + last_run_status so the UI can show
the outcome. `status` is one of "success" | "failure" |
"partial" | "deferred" | "skipped"."""
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar or not isinstance(sidecar.get("schedule"), dict):
return False
sidecar["schedule"]["last_run_at"] = _now_iso()
sidecar["schedule"]["last_run_status"] = status
sidecar["schedule"]["last_run_target"] = target
if reason:
sidecar["schedule"]["last_run_reason"] = str(reason)[:300]
else:
sidecar["schedule"].pop("last_run_reason", None)
if log_name:
sidecar["schedule"]["last_run_log"] = os.path.basename(str(log_name))[:220]
else:
sidecar["schedule"].pop("last_run_log", None)
if reboot_required is None:
sidecar["schedule"].pop("last_run_reboot_required", None)
else:
sidecar["schedule"]["last_run_reboot_required"] = bool(reboot_required)
packages = [
str(package).strip()[:160]
for package in (reboot_packages or [])[:32]
if str(package).strip()
]
if reboot_required and packages:
sidecar["schedule"]["last_run_reboot_packages"] = packages
else:
sidecar["schedule"].pop("last_run_reboot_packages", None)
sidecar["updated_at"] = _now_iso()
return _write_sidecar(vmid, sidecar)
def clear_schedule_reboot_required(vmid) -> bool:
"""Clear a persisted reboot warning after the CT starts or reboots."""
with _cache_lock:
sidecar = _read_sidecar(vmid)
schedule = (sidecar or {}).get("schedule")
if not isinstance(schedule, dict):
return False
if schedule.get("last_run_reboot_required") is not True:
return True
schedule["last_run_reboot_required"] = False
schedule.pop("last_run_reboot_packages", None)
sidecar["updated_at"] = _now_iso()
return _write_sidecar(vmid, sidecar)
def _fire_update_notification(vmid, app: dict) -> None:
# Per-app opt-out: user flipped the bell icon off for this specific
# app (because they know it can't be updated on their box or they
# just don't care). Field defaults to True — an app registered
# before this feature landed keeps receiving notifications.
if app.get("notifications_enabled", True) is False:
return
if app.get("helper_slug") == "docker":
return
try:
from notification_manager import notification_manager
import socket
state = app.get("state") or {}
notification_manager.emit_event(
event_type='app_update_available',
severity='INFO',
data={
'hostname': socket.gethostname(),
'vmid': int(vmid),
'ct_name': app.get('name') or f'CT-{vmid}',
'app_name': app.get('name') or 'app',
'installed': state.get('installed_version') or 'unknown',
'latest': state.get('latest_version') or 'unknown',
},
source='app_watch',
entity='ct',
# vmid + app_id + latest so multi-app CTs don't dedup and
# subsequent upstream releases still fire.
entity_id=f"{vmid}:{app.get('id')}:{state.get('latest_version') or ''}",
)
except Exception as e:
print(f"[ProxMenux] lxc_apps: notif emit failed for CT {vmid}: {e}")
def _docker_stack_notification_payload(
vmid: int, docker_app: dict, inventory: dict, ct_name: str,
) -> Optional[dict]:
state = docker_app.get('state') or {}
engine_pending = bool(
state.get('update_available')
and state.get('latest_version')
)
pending_images = sorted(
(
image for image in (inventory.get('images') or [])
if image.get('update_available') is True
),
key=lambda image: str(image.get('reference') or ''),
)
if not engine_pending and not pending_images:
return None
identity = {
'engine': ({
'installed': state.get('installed_version'),
'latest': state.get('latest_version'),
} if engine_pending else None),
'images': [
{
'reference': image.get('reference'),
'remote_digest': image.get('remote_digest'),
'available_version': image.get('available_version'),
}
for image in pending_images
],
}
signature = hashlib.sha256(
json.dumps(identity, sort_keys=True, separators=(',', ':')).encode('utf-8')
).hexdigest()[:20]
lines = []
if engine_pending:
installed = state.get('installed_version') or 'unknown'
latest = state.get('latest_version') or 'unknown'
lines.append(f'• Docker Engine: {installed} → {latest}')
for image in pending_images[:12]:
reference = image.get('reference') or 'Docker image'
installed = image.get('installed_version')
available = image.get('available_version')
if installed and available and installed != available:
lines.append(f'• {reference}: {installed} → {available}')
else:
lines.append(f'• {reference}: new registry digest')
if len(pending_images) > 12:
lines.append(f'• +{len(pending_images) - 12} additional image update(s)')
return {
'hostname': socket.gethostname(),
'vmid': int(vmid),
'ct_name': ct_name or f'CT-{vmid}',
'count': len(pending_images) + (1 if engine_pending else 0),
'details': '\n'.join(lines),
'signature': signature,
}
def emit_all_pending_docker_stacks() -> int:
"""Emit one cached Docker update event per registered container."""
inventories = get_cached_docker_inventories()
if not inventories:
return 0
names: dict[int, str] = {}
try:
import managed_installs
names = {
int(item['_vmid']): str(item.get('name') or f"CT-{item['_vmid']}")
for item in (managed_installs.get_active_items() or [])
if item.get('type') == 'lxc' and item.get('_vmid') is not None
}
except Exception:
names = {}
emitted = 0
for raw_vmid, inventory in sorted(inventories.items(), key=lambda item: int(item[0])):
try:
vmid = int(raw_vmid)
except (TypeError, ValueError):
continue
sidecar = _read_sidecar(vmid) or {}
docker_app = next(
(
app for app in (sidecar.get('apps') or [])
if app.get('helper_slug') == 'docker'
),
None,
)
if not docker_app or docker_app.get('notifications_enabled', True) is False:
continue
payload = _docker_stack_notification_payload(
vmid, docker_app, inventory, names.get(vmid) or f'CT-{vmid}',
)
if not payload:
continue
try:
from notification_manager import notification_manager
signature = payload.pop('signature')
notification_manager.emit_event(
event_type='docker_stack_update_available',
severity='INFO',
data=payload,
source='polling',
entity='ct',
entity_id=f'{vmid}:{signature}',
)
emitted += 1
except Exception as exc:
print(f'[ProxMenux] Docker update notification for CT {vmid} failed: {exc}')
return emitted
def _detect_with_alt_healing(vmid, app: dict) -> tuple:
"""Detect the installed version for an app, falling back to
``alt_detectors`` from the hint when the primary detector's target
isn't present on this CT. On a successful fallback the app dict
is MUTATED in place to reflect the working detector — the sidecar
write happens by the caller — so subsequent checks go straight to
the resolved detector without paying the fallback cost again.
Returns ``(installed_version, error, healed_bool)`` where
``healed_bool`` is True when the working detector was an alt and
the app dict was rewritten.
"""
installed, err = detect_installed_version(vmid, app)
if installed or not err:
return installed, err, False
slug = app.get("helper_slug")
if not slug:
return installed, err, False
hint = (_fetch_tracking_hints() or {}).get(slug) or {}
# Build a unified fallback list from both:
# • alt_detectors — cross-method (file→binary, file→dpkg, …)
# • file_fallbacks — same-method secondary file paths (legacy
# layouts of the same install). Same semantics for auto-heal,
# different JSON shape for historical reasons.
fallbacks: list = []
for alt in hint.get("alt_detectors") or []:
if isinstance(alt, dict):
fallbacks.append(alt)
for fb in hint.get("file_fallbacks") or []:
if isinstance(fb, dict) and fb.get("path"):
fallbacks.append({
"installed_via": "file",
"file_path": fb["path"],
"file_regex": fb.get("regex") or hint.get("file_regex"),
})
if not fallbacks:
return installed, err, False
# Try each fallback in order; first that produces a parseable
# version wins. We copy its fields into a probe dict so
# detect_installed_version can run unchanged.
for alt in fallbacks:
method = alt.get("installed_via")
if method not in _VALID_METHODS:
continue
probe = {"installed_via": method}
for k in _DETECTOR_FIELDS:
if k in alt:
probe[k] = alt[k]
# Inherit the app's tag_regex + installed_regex for output
# parsing when the alt hasn't overridden them.
for k in ("tag_regex", "installed_regex"):
if k in app and k not in probe:
probe[k] = app[k]
alt_installed, alt_err = detect_installed_version(vmid, probe)
if alt_installed:
# Heal: mutate app with the winning detector's fields.
# Clear stale fields from the previous method so the
# sidecar reflects exactly what's being used.
for k in _DETECTOR_FIELDS:
app.pop(k, None)
for k, v in probe.items():
if k not in ("tag_regex", "installed_regex"):
app[k] = v
return alt_installed, None, True
return installed, err, False
def check_app(vmid, app_id: str, force: bool = False) -> Optional[dict]:
with _cache_lock:
sidecar = _read_sidecar(vmid)
if not sidecar:
return None
app = _find_app(sidecar, app_id)
if not app:
return None
# Docker apps are register-only — no version detection, no
# upstream check, no error emission. Just stamp checked_at so
# the UI can show "we know about you, we're not tracking you".
if app.get("installed_via") == "docker":
app["state"] = {**_empty_state(), "checked_at": _now_iso()}
sidecar["updated_at"] = _now_iso()
_write_sidecar(vmid, sidecar)
return sidecar
state = app.get("state") or _empty_state()
checked_at = state.get("checked_at")
if not force and checked_at:
try:
t = datetime.datetime.strptime(checked_at.rstrip("Z"), "%Y-%m-%dT%H:%M:%S")
age = (datetime.datetime.utcnow() - t).total_seconds()
if age < _UPSTREAM_CACHE_TTL_SEC:
return sidecar
except (ValueError, TypeError):
pass
installed, inst_err, _healed = _detect_with_alt_healing(vmid, app)
# Trigger the upstream fetch when ANY upstream source is
# configured. The dispatcher inside `fetch_latest_upstream`
# returns (None, None) cleanly when nothing is set, so the
# cheap gate below only skips the no-upstream case and lets
# http_json / docker_hub (which don't set `repo`) through.
latest, up_err, latest_published_at = (None, None, None)
if app.get("repo") or app.get("upstream_type") in ("http_json", "docker_hub"):
latest, up_err, latest_published_at = fetch_latest_upstream_details(app)
err = inst_err or up_err
update_available = compare(installed, latest) if (installed and latest) else None
app["state"] = {
"installed_version": installed,
"latest_version": latest,
"latest_published_at": latest_published_at,
"update_available": update_available,
"error": err,
"checked_at": _now_iso(),
}
sidecar["updated_at"] = _now_iso()
_write_sidecar(vmid, sidecar)
# Emit every time an update is pending. The old `latest !=
# prev_latest` guard tried to prevent spam by only firing on
# the first observation of each new upstream version, but it
# also swallowed the emit whenever the notification setting
# was toggled off → on after the first observation (the
# sidecar already had `latest_version` recorded, so subsequent
# checks looked like "same latest, nothing to do"). Anti-spam
# is the notification manager's job: it dedups by `entity_id`
# (vmid + app_id + latest_version) with its cooldown, and only
# a genuinely new upstream release changes the entity_id and
# triggers a fresh delivery.
if update_available and latest:
_fire_update_notification(vmid, app)
return sidecar
def emit_all_pending_updates() -> int:
"""Walk every sidecar and emit `app_update_available` for each
app currently marked with a pending upstream release. Safe to
call repeatedly — `notification_manager` dedups by entity_id
(vmid + app_id + latest_version), so a given release only sends
once until a newer version appears.
Needed because `check_app(force=False)` short-circuits on a fresh
`checked_at` and never reaches the emit path. The 24 h
PollingCollector runs `refresh_all_apps(force=False)`, so without
this helper the notification only ever fired on the exact tick
where a new upstream version was FIRST observed — and even that
was silenced when the user's setting was OFF at the time.
Returns the number of emits attempted (delivery still depends on
channel enablement + cooldown + rate limit)."""
try:
entries = sorted(os.listdir(_APPS_DIR))
except (FileNotFoundError, OSError):
print("[ProxMenux] emit_all_pending_updates: _APPS_DIR missing", flush=True)
return 0
n = 0
print(f"[ProxMenux] emit_all_pending_updates: scanning {len(entries)} sidecar file(s)", flush=True)
for name in entries:
if not name.endswith(".json"):
continue
try:
vmid = int(name[:-5])
except ValueError:
continue
try:
sidecar = _read_sidecar(vmid)
if not sidecar:
print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} sidecar empty", flush=True)
continue
apps = sidecar.get("apps") or []
pending = [a for a in apps
if (a.get("state") or {}).get("update_available")
and (a.get("state") or {}).get("latest_version")]
print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} apps={len(apps)} pending={len(pending)}", flush=True)
for app in pending:
try:
_fire_update_notification(vmid, app)
n += 1
print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} emit '{app.get('name')}'", flush=True)
except Exception as inner:
print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} emit '{app.get('name')}' FAILED: {inner}", flush=True)
except Exception as e:
print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} outer failure: {e}", flush=True)
print(f"[ProxMenux] emit_all_pending_updates: {n} emit(s) attempted total", flush=True)
return n
def check_all(vmid, force: bool = False) -> Optional[dict]:
sidecar = _read_sidecar(vmid)
if not sidecar:
return None
for app in (sidecar.get("apps") or []):
try:
check_app(vmid, app.get("id"), force=force)
except Exception as e:
print(f"[ProxMenux] lxc_apps.check_all: CT {vmid} app {app.get('id')} failed: {e}")
return _read_sidecar(vmid)
def refresh_all_apps(force: bool = False) -> int:
"""Called from the polling collector's daily cycle so header
badges stay fresh without needing to open every modal."""
try:
entries = os.listdir(_APPS_DIR)
except (FileNotFoundError, OSError):
return 0
n = 0
for name in entries:
if not name.endswith(".json"):
continue
try:
vmid = int(name[:-5])
except ValueError:
continue
try:
check_all(vmid, force=force)
n += 1
except Exception as e:
print(f"[ProxMenux] lxc_apps refresh_all: CT {vmid} failed: {e}")
return n
def _summarise_app(app: dict) -> dict:
"""Compact summary used by /api/vms to decorate LXC rows without
forcing the frontend to fetch the full sidecar. Includes ports
so the modal header can render clickable web links inline."""
state = app.get("state") or {}
return {
"id": app.get("id"),
"name": app.get("name"),
"installed_via": app.get("installed_via"),
"ports": app.get("ports") or [],
# Keep the application-level logo in the compact /api/vms
# projection. Consumers can prefer a per-link logo and fall
# back to this one when the Web Link intentionally leaves its
# own logo empty.
"logo_url": app.get("logo_url") or None,
"health_path": app.get("health_path"),
"installed_version": state.get("installed_version"),
"latest_version": state.get("latest_version"),
"update_available": state.get("update_available"),
"error": state.get("error"),
"checked_at": state.get("checked_at"),
"has_repo": bool(app.get("repo")),
# Updates tab surfaces: whether the user has a custom bash
# command wired up ("Apply {app}" runs `pct exec sh -c` on
# it) and whether the "no method" notice is suppressed for
# this app.
"update_command": app.get("update_command") or "",
# Compatibility field for older clients. The only supported
# strategy is now replacement; legacy sidecars are normalized
# in the API even before their next write.
"update_strategy": "custom_override",
"hide_no_updater_notice": bool(app.get("hide_no_updater_notice")),
# Whether this app should be counted in the CT's aggregate
# updates badge (default: yes). See validator for full context.
"exclude_from_badge": bool(app.get("exclude_from_badge")),
"notifications_enabled": app.get("notifications_enabled", True) is not False,
# Community-scripts slug that the Register-chip flow attaches
# to the app. Surfaced so the Updates tab helper section can
# match this registered app against the CT's helper_slug and
# display its installed/upstream versions.
"helper_slug": app.get("helper_slug") or "",
}
def get_category_presets() -> list:
"""Return the sorted list of unique category names sourced from
helpers_cache. Powers the "Categoría" dropdown in the Web Link
editor and the Apps dashboard filter. If the cache is missing or
empty, returns a short built-in fallback so the UI never shows an
empty preset list.
"""
fallback = [
"Adblock & DNS", "Authentication & Security", "Automation & Scheduling",
"Backup & Recovery", "Containers & Docker", "Databases",
"Documents & Notes", "Files & Downloads", "Media & Streaming",
"Miscellaneous", "Monitoring & Analytics", "Network & Firewall",
]
try:
import managed_installs
cache = managed_installs._fetch_helpers_cache() or {}
except Exception:
return fallback
seen: set = set()
for entry in cache.values():
if not isinstance(entry, dict):
continue
for name in entry.get("category_names") or []:
if isinstance(name, str) and name.strip():
seen.add(name.strip())
return sorted(seen) if seen else fallback
def suggest_category_for(name_or_slug: str) -> Optional[str]:
"""Look up a category preset by app name/slug against helpers_cache.
Powers the auto-fill in the Web Link editor — when the user types
a name that matches a catalog entry, the category dropdown
pre-selects the first category_names value. Returns None when the
name has no match or the cache is unavailable.
"""
if not name_or_slug:
return None
needle = name_or_slug.strip().lower()
if not needle:
return None
try:
import managed_installs
cache = managed_installs._fetch_helpers_cache() or {}
except Exception:
return None
for slug, entry in cache.items():
if not isinstance(entry, dict):
continue
if slug == needle or (entry.get("name") or "").lower() == needle:
cats = entry.get("category_names") or []
if cats and isinstance(cats[0], str) and cats[0].strip():
return cats[0].strip()
return None
return None
def get_catalog() -> list:
"""Return a compact catalog of registerable apps for the frontend
picker. Sourced from helpers_cache.json (community-scripts, ~700
apps with name/logo/port/website) enriched with a `has_tracking`
flag that tells the frontend whether we have a curated tracking
hint for this slug (→ Register button pre-fills the advanced
form). Response is small enough (~40-50 KB) to cache client-side.
"""
try:
import managed_installs
cache = managed_installs._fetch_helpers_cache() or {}
except Exception:
cache = {}
hints = _fetch_tracking_hints() or {}
out: list = []
for slug, entry in cache.items():
if not isinstance(entry, dict):
continue
out.append({
"slug": slug,
"name": entry.get("name") or slug,
"logo": entry.get("logo") or "",
"default_port": entry.get("default_port") or 0,
"has_tracking": slug in hints,
})
# Also surface tracking-hint slugs that aren't in helpers_cache
# (our user-only fallback entries like docker/pihole/wireguard).
seen = {e["slug"] for e in out}
for slug, hint in hints.items():
if slug in seen:
continue
out.append({
"slug": slug,
"name": (hint.get("name") if isinstance(hint, dict) else None) or slug,
"logo": (hint.get("logo") if isinstance(hint, dict) else "") or "",
"default_port": ((hint.get("default_ports") or [0])[0] if isinstance(hint, dict) else 0),
"has_tracking": True,
})
out.sort(key=lambda e: e["name"].lower())
return out
def get_catalog_entry(slug: str, vmid=None) -> Optional[dict]:
"""Detail for a single catalog slug, including any curated hint
fields. Called by the frontend when the user picks an app from the
Combobox — the response seeds the editor with detector metadata
when we have it."""
if not slug:
return None
slug = slug.strip().lower()
try:
import managed_installs
cache = managed_installs._fetch_helpers_cache() or {}
except Exception:
cache = {}
hints = _fetch_tracking_hints() or {}
catalog = cache.get(slug) or {}
hint = hints.get(slug) or {}
if not catalog and not hint:
return None
# When the caller supplies a CT, return the detector that actually
# extracts a version there. This makes catalog search work for manual
# and legacy layouts too, not only for the helper-derived primary slug.
tracking = None
if hint:
resolved = dict(hint)
if vmid is not None:
resolved, _version, _source, _error = _select_working_hint_detector(vmid, resolved)
tracking = {k: v for k, v in resolved.items()
if k not in ("logo", "website", "default_ports",
"file_fallbacks", "alt_detectors")}
# Enrich port from catalog when hint doesn't specify one.
default_ports: list = []
raw_ports = hint.get("default_ports") if isinstance(hint, dict) else None
if isinstance(raw_ports, list):
for p in raw_ports:
try:
n = int(p)
if 1 <= n <= 65535:
default_ports.append(n)
except (TypeError, ValueError):
continue
if not default_ports and catalog.get("default_port"):
try:
n = int(catalog["default_port"])
if 1 <= n <= 65535:
default_ports.append(n)
except (TypeError, ValueError):
pass
# First category name from helpers_cache — auto-fills the port's
# Categoría field when the user picks this app from the catalog.
category = None
cat_names = catalog.get("category_names") or []
if cat_names and isinstance(cat_names[0], str) and cat_names[0].strip():
category = cat_names[0].strip()
return {
"slug": slug,
"name": catalog.get("name") or (hint.get("name") if isinstance(hint, dict) else None) or slug,
"logo_url": (
(hint.get("logo") if isinstance(hint, dict) else "")
or catalog.get("logo") or ""
) or None,
"website": catalog.get("website") or "",
"default_ports": default_ports,
"category": category,
"tracking_suggestion": tracking,
}
def get_active_apps() -> dict:
"""``{vmid_str: [summary, …]}``. Never triggers a re-check —
reads persisted state only."""
out: dict = {}
try:
entries = os.listdir(_APPS_DIR)
except (FileNotFoundError, OSError):
return out
for name in entries:
if not name.endswith(".json"):
continue
try:
vmid = int(name[:-5])
except ValueError:
continue
sidecar = _read_sidecar(vmid)
if not sidecar:
continue
apps = sidecar.get("apps") or []
if not apps:
continue
out[str(vmid)] = [_summarise_app(a) for a in apps]
return out
# ── Suggestions endpoint helpers ──────────────────────────────────
_KNOWN_WEB_PORTS = {80, 443, 3000, 3001, 4444, 5000, 5001, 5432, 6379,
7000, 7878, 8000, 8080, 8081, 8096, 8123, 8181,
8384, 8443, 8686, 8787, 8989, 9000, 9090, 9091, 9117}
# Port probing is a `pct exec ss -tlnH` per CT — ~500-800ms wall time
# on a warm host. Memoized per-vmid with a 60s TTL so opening the App
# tab multiple times in quick succession stays snappy; the first open
# still pays the probe cost.
_PORT_PROBE_TTL_SEC = 60
_port_probe_cache: dict = {}
_port_probe_lock = threading.RLock()
# Docker-published endpoints are a different concept from applications
# installed natively in the LXC. Keep a short-lived, read-only cache of
# container → host-port mappings so the App editor can offer web links under
# the Docker entry without registering every container as an independent LXC
# application.
_docker_web_links_cache: dict = {}
_docker_web_links_lock = threading.RLock()
# Same TTL discipline for file-existence probes used to resolve
# legacy install layouts (see _resolve_file_candidate).
_file_probe_cache: dict = {}
_file_probe_lock = threading.RLock()
def _first_existing_file(vmid, paths: list) -> Optional[str]:
"""Return the first path from ``paths`` that exists as a regular
file inside the CT, or None if none exist. Single ``pct exec find``
(busybox-compatible) so probing a 3-candidate list is one round-
trip. Memoized per (vmid, tuple(paths)) with a 60 s TTL to keep
repeated App-tab opens snappy.
"""
if not paths:
return None
key = (str(vmid), tuple(paths))
now = time.time()
with _file_probe_lock:
cached = _file_probe_cache.get(key)
if cached and (now - cached[0]) < _PORT_PROBE_TTL_SEC:
return cached[1]
# `find <paths> -maxdepth 0 -type f -print` — busybox-safe. Missing
# paths are silently skipped; existing regular files land on stdout.
rc, out, _ = _pct_exec(vmid, ["find"] + list(paths) + ["-maxdepth", "0", "-type", "f", "-print"])
found = {l.strip() for l in out.splitlines() if l.strip()} if rc in (0, 1) else set()
resolved = next((p for p in paths if p in found), None)
with _file_probe_lock:
_file_probe_cache[key] = (now, resolved)
return resolved
# Multi-app detection: probe every hint we have against the CT and
# return the slugs whose install signature is present. The point is
# CTs that host more than one app (helper-scripts install + a manual
# Docker on top, or several apps side by side): the primary detection
# via community-scripts marker + hostname fuzzy only surfaces ONE
# slug, but here we surface every hint whose install is real. All
# probes are batched by method → 3 pct-exec calls per CT max.
#
# Memoized per-vmid with the same 60 s TTL as the port probe.
_detected_apps_cache: dict = {}
_detected_apps_lock = threading.RLock()
def _iter_hint_detectors(h: dict):
"""Yield every detector (primary + alt_detectors) for a hint as
``{installed_via, ...method-specific fields}`` dicts. Used by
multi-detect probes and by the version-check auto-heal so an app
that has moved from its canonical layout (manual install, legacy)
is still detected via whatever secondary target does exist.
"""
if not isinstance(h, dict):
return
primary = {"installed_via": h.get("installed_via")}
for k in _DETECTOR_FIELDS:
if k in h:
primary[k] = h[k]
if primary["installed_via"]:
yield primary
for alt in h.get("alt_detectors") or []:
if isinstance(alt, dict) and alt.get("installed_via"):
yield alt
def _probe_detected_apps_map(vmid) -> dict:
"""Return ``{slug: [detector_dicts_that_matched]}`` — the full
working-detector map for every hint whose install signature is
present on the CT. Detectors are kept in the SAME order they
appear in the hint (primary → alt_detectors → file_fallbacks) so
callers can just take ``[0]`` as the preferred one for this CT.
Same batched probes as before; the extra bookkeeping is a dict
holding the detector dict alongside the slug at each mapping key.
"""
key = str(vmid)
now = time.time()
with _detected_apps_lock:
cached = _detected_apps_cache.get(key)
if cached and (now - cached[0]) < _PORT_PROBE_TTL_SEC:
return {slug: [dict(d) for d in dets]
for slug, dets in (cached[1] or {}).items()}
hints = _fetch_tracking_hints() or {}
# Each mapping key is (slug, detector_dict) — same target may map
# to multiple slugs in theory (unlikely) so we store a list.
executable_paths: dict = {} # path → [(slug, det), …]
file_paths: dict = {}
dpkg_pkgs: dict = {}
apk_pkgs: dict = {}
docker_containers: dict = {}
def _add(bucket, target, slug, det):
bucket.setdefault(target, []).append((slug, det))
for slug, h in hints.items():
if not isinstance(h, dict):
continue
# file_fallbacks: same-method secondary paths — synthesize
# per-fallback detector dicts that inherit the primary's file
# method so downstream code has full detector context.
for fb in h.get("file_fallbacks") or []:
if not isinstance(fb, dict):
continue
p = fb.get("path")
r = fb.get("regex")
if isinstance(p, str) and p:
fb_det = {
"installed_via": "file",
"file_path": p,
"file_regex": r or h.get("file_regex", ""),
}
_add(file_paths, p, slug, fb_det)
# Primary + every alt_detector share the same batching logic.
for det in _iter_hint_detectors(h):
method = det.get("installed_via")
if method == "binary":
bp = det.get("binary_path")
if isinstance(bp, str) and bp:
_add(executable_paths, bp, slug, det)
elif method == "file":
fp = det.get("file_path")
if isinstance(fp, str) and fp:
_add(file_paths, fp, slug, det)
elif method == "dpkg":
pkg = det.get("package")
if isinstance(pkg, str) and pkg:
_add(dpkg_pkgs, pkg, slug, det)
elif method == "apk":
pkg = det.get("package")
if isinstance(pkg, str) and pkg:
_add(apk_pkgs, pkg, slug, det)
elif method == "python_dist":
pp = det.get("python_path")
if isinstance(pp, str) and pp:
_add(executable_paths, pp, slug, det)
elif method in ("docker_label", "docker_exec"):
cn = det.get("container_name")
if isinstance(cn, str) and cn:
_add(docker_containers, cn, slug, det)
# slug → ordered list of matched detectors (primary preference
# preserved by natural insertion order from _iter_hint_detectors).
matched: dict = {}
def _record(slug, det):
matched.setdefault(slug, []).append(det)
def _probe_paths(paths: dict) -> None:
if not paths:
return
# No ``-type f``: venv/bin/python and /usr/bin tools are commonly
# symlinks. Actual version execution below rejects broken/wrong
# targets, so existence is only a cheap batching pre-filter.
rc, out, _ = _pct_exec(vmid, ["find"] + list(paths) + ["-maxdepth", "0", "-print"])
if rc not in (0, 1):
return
for line in out.splitlines():
p = line.strip()
if p in paths:
for slug, det in paths[p]:
_record(slug, det)
_probe_paths(executable_paths)
_probe_paths(file_paths)
if dpkg_pkgs:
rc, out, _ = _pct_exec(vmid, ["dpkg-query", "-W", "-f", "${Package}\\t${Status}\\n"] + list(dpkg_pkgs))
if rc in (0, 1):
for line in out.splitlines():
parts = line.split("\t")
if len(parts) >= 2 and "install ok installed" in parts[1]:
pkg = parts[0].strip()
if pkg in dpkg_pkgs:
for slug, det in dpkg_pkgs[pkg]:
_record(slug, det)
if apk_pkgs:
rc, out, _ = _pct_exec(vmid, ["apk", "info", "-e"] + list(apk_pkgs))
if rc == 0:
for line in out.splitlines():
pkg = line.strip()
if pkg in apk_pkgs:
for slug, det in apk_pkgs[pkg]:
_record(slug, det)
docker_runtime_available = False
if docker_containers:
rc, out, _ = _pct_exec(vmid, ["docker", "ps", "-a", "--format", "{{.Names}}"])
if rc == 0:
docker_runtime_available = True
present = {line.strip() for line in out.splitlines() if line.strip()}
for cn, entries in docker_containers.items():
if cn in present:
for slug, det in entries:
_record(slug, det)
# A path/package/container-name match is only a candidate. Prove it by
# extracting a real version with that detector before surfacing an app to
# the user. This eliminates the generic /root/.slug false positives and
# stale files left behind by migrations.
verified: dict = {}
for slug, detectors in matched.items():
hint = hints.get(slug) or {}
for detector in detectors:
probe = _detector_probe_config(hint, detector)
try:
version, _error = detect_installed_version(vmid, probe)
except (KeyError, TypeError, ValueError):
version = None
if not version:
continue
enriched = dict(detector)
enriched["detected_version"] = version
enriched["detector_verified"] = True
verified.setdefault(slug, []).append(enriched)
# A just-started CT can answer ``docker ps`` a moment after the initial
# batched ``find /usr/bin/docker`` probe failed. In that narrow window a
# Docker child such as Portainer used to be cached as a native LXC app for
# 60 seconds. A successful Docker API call proves the parent runtime is
# available, so retry Docker's normal curated detector and only publish it
# when it also yields a real version.
if docker_runtime_available and "docker" not in verified:
docker_hint = hints.get("docker") or {}
for detector, _source in _ordered_hint_detectors(docker_hint):
probe = _detector_probe_config(docker_hint, detector)
try:
version, _error = detect_installed_version(vmid, probe)
except (KeyError, TypeError, ValueError):
version = None
if not version:
continue
enriched = dict(detector)
enriched["detected_version"] = version
enriched["detector_verified"] = True
verified["docker"] = [enriched]
break
with _detected_apps_lock:
_detected_apps_cache[key] = (now, {slug: list(dets) for slug, dets in verified.items()})
return verified
def _probe_detected_apps(vmid) -> set:
"""Legacy set-returning wrapper — kept for callers that only need
presence, not detector context."""
return set(_probe_detected_apps_map(vmid).keys())
def _resolve_file_candidate(vmid, tracking: dict) -> None:
"""Rewrite ``tracking``'s ``file_path`` / ``file_regex`` to the
first candidate that actually exists on the CT, so the sidecar the
user saves points at the layout their install produced.
Enables curated hints to declare legacy fallbacks (e.g. NPM's
modern `/root/.nginxproxymanager` + legacy `/app/package.json`).
Each fallback carries its own regex, since legacy layouts often
stored the version in a very different format (a JSON blob vs a
single line). If none of the candidates exist, the primary path
is preserved so the user still gets the auto-fill and can adjust
manually. Silently strips ``file_fallbacks`` from the returned
hint — the frontend never sees the candidate list.
"""
if tracking.get("installed_via") != "file":
return
primary = tracking.get("file_path")
if not primary:
return
fallbacks_raw = tracking.pop("file_fallbacks", None)
if not isinstance(fallbacks_raw, list) or not fallbacks_raw:
return
# Build ordered probe list (primary first)
candidates: list = []
seen: set = {primary}
candidates.append({"path": primary, "regex": tracking.get("file_regex", "")})
for f in fallbacks_raw:
if not isinstance(f, dict):
continue
p = f.get("path")
r = f.get("regex")
if not isinstance(p, str) or not p or not isinstance(r, str) or not r:
continue
if p in seen:
continue
seen.add(p)
candidates.append({"path": p, "regex": r})
paths_only = [c["path"] for c in candidates]
found = _first_existing_file(vmid, paths_only)
if found and found != primary:
for c in candidates:
if c["path"] == found:
tracking["file_path"] = c["path"]
tracking["file_regex"] = c["regex"]
break
def _probe_listening_ports(vmid) -> list[int]:
key = str(vmid)
now = time.time()
with _port_probe_lock:
cached = _port_probe_cache.get(key)
if cached and (now - cached[0]) < _PORT_PROBE_TTL_SEC:
return list(cached[1])
rc, out, _ = _pct_exec(vmid, ["ss", "-tlnH"], timeout=5)
if rc != 0:
rc, out, _ = _pct_exec(vmid, ["netstat", "-tln"], timeout=5)
if rc != 0:
with _port_probe_lock:
_port_probe_cache[key] = (now, [])
return []
ports: set = set()
for line in out.splitlines():
for token in line.split():
if ":" not in token:
continue
candidate = token.rsplit(":", 1)[-1]
if candidate.isdigit():
p = int(candidate)
if 1 <= p <= 65535:
ports.add(p)
result = sorted(p for p in ports if p not in (22, 53, 5353))
with _port_probe_lock:
_port_probe_cache[key] = (now, result)
return result
def _docker_service_catalog_meta(service: str, container: str, image: str) -> dict:
"""Best-effort display metadata for a Docker workload.
Identity remains the real container/image reported by Docker. Catalog
matching is used only for a friendly label/logo; a failed match never
hides a published endpoint or turns the workload into an App Watch.
"""
image_base = image.split("@", 1)[0].rsplit("/", 1)[-1].split(":", 1)[0]
raw_candidates = [service, container, image_base]
candidates: list[str] = []
for raw in raw_candidates:
candidate = re.sub(r"[^a-z0-9._-]+", "-", str(raw or "").strip().lower()).strip("-._")
if candidate and candidate not in candidates:
candidates.append(candidate)
for suffix in ("-ce", "-ee"):
if candidate.endswith(suffix):
base = candidate[:-len(suffix)]
if base and base not in candidates:
candidates.append(base)
hints = _fetch_tracking_hints() or {}
for candidate in candidates:
hint = hints.get(candidate) or {}
catalog = _catalog_lookup(candidate) or {}
if hint or catalog:
logo = hint.get("logo") or catalog.get("logo")
return {
"slug": candidate,
"name": catalog.get("name") or hint.get("name") or service or container,
"logo_url": logo if isinstance(logo, str) and logo.startswith(("http://", "https://")) else None,
}
return {"slug": None, "name": service or container, "logo_url": None}
def _probe_docker_web_links(vmid) -> list[dict]:
"""Return running Docker workloads that publish TCP ports on the LXC.
The result is suggestion-only. No sidecar entry is written and no port is
assumed to be HTTP until the user explicitly adds it in the editor. IPv4
and IPv6 bindings of the same host port are deduplicated; loopback-only
bindings are omitted because they cannot form a usable remote LXC link.
"""
key = str(vmid)
now = time.time()
with _docker_web_links_lock:
cached = _docker_web_links_cache.get(key)
if cached and (now - cached[0]) < _PORT_PROBE_TTL_SEC:
return [dict(item) for item in cached[1]]
rc, out, _ = _pct_exec(vmid, ["docker", "ps", "-q"], timeout=10)
if rc != 0:
result: list[dict] = []
else:
container_ids = [line.strip() for line in out.splitlines() if line.strip()][:_DOCKER_MAX_IMAGES]
result = []
if container_ids:
rc, inspect_out, _ = _pct_exec(
vmid,
["docker", "inspect", "--format", "{{json .}}", *container_ids],
timeout=20,
)
if rc == 0:
for raw in inspect_out.splitlines():
try:
obj = json.loads(raw)
except json.JSONDecodeError:
continue
container = str(obj.get("Name") or "").lstrip("/")
config = obj.get("Config") or {}
image = str(config.get("Image") or "").strip()
labels = config.get("Labels") or {}
service = str(labels.get("com.docker.compose.service") or container).strip()
meta = _docker_service_catalog_meta(service, container, image)
ports = (obj.get("NetworkSettings") or {}).get("Ports") or {}
seen_host_ports: set[int] = set()
for container_endpoint, bindings in ports.items():
if not str(container_endpoint).endswith("/tcp") or not isinstance(bindings, list):
continue
try:
container_port = int(str(container_endpoint).split("/", 1)[0])
except (TypeError, ValueError):
continue
for binding in bindings:
if not isinstance(binding, dict):
continue
host_ip = str(binding.get("HostIp") or "").strip()
if host_ip in {"127.0.0.1", "::1"}:
continue
try:
host_port = int(binding.get("HostPort"))
except (TypeError, ValueError):
continue
if not (1 <= host_port <= 65535) or host_port in seen_host_ports:
continue
seen_host_ports.add(host_port)
scheme = "https" if host_port in {443, 4443, 8443, 9443} or container_port in {443, 4443, 8443, 9443} else "http"
result.append({
"container_name": container,
"service_name": meta["name"],
"service_slug": meta["slug"],
"image": image,
"host_port": host_port,
"container_port": container_port,
"scheme": scheme,
"web_path": "/",
"logo_url": meta["logo_url"],
})
result.sort(key=lambda item: (str(item.get("service_name") or "").lower(), int(item.get("host_port") or 0)))
with _docker_web_links_lock:
_docker_web_links_cache[key] = (now, result)
return [dict(item) for item in result]
def _helper_slug_meta(vmid) -> Optional[dict]:
try:
import managed_installs
except Exception:
return None
try:
items = managed_installs.get_active_items() or []
except Exception:
return None
for it in items:
if it.get("type") == "lxc" and str(it.get("_vmid")) == str(vmid):
slug = it.get("_helper_slug")
name = it.get("_helper_app_name")
if slug or name:
return {"slug": slug, "name": name}
return None
def _catalog_lookup(slug: str) -> Optional[dict]:
"""Fetch the community-scripts catalog entry for a slug.
Returns {name, updateable, default_port, logo} or None.
Cached inside managed_installs (7 day TTL, disk-backed)."""
if not slug:
return None
try:
import managed_installs
cache = managed_installs._fetch_helpers_cache() or {}
except Exception:
return None
return cache.get(slug)
def _merge_tracking_hints(slug: str) -> Optional[dict]:
"""Return the curated tracking suggestion for a slug.
Reads from json/app_tracking_hints.json (built in CI by merging
the audit generator's verified entries with our manual
overrides). Returns None if the slug has no hint — the frontend's
"Register with version tracking" flow requires `installed_via`
to pre-fill the advanced form, so a missing hint means only the
"Just register a link" path is offered.
"""
hint = _fetch_tracking_hints().get(slug)
if not hint:
return None
return dict(hint)
def invalidate_suggestion_probes(vmid) -> None:
"""Discard the short-lived discovery probes for one container."""
key = str(vmid)
with _detected_apps_lock:
_detected_apps_cache.pop(key, None)
with _cache_lock:
_port_probe_cache.pop(key, None)
with _docker_web_links_lock:
_docker_web_links_cache.pop(key, None)
with _file_probe_lock:
stale = [cache_key for cache_key in _file_probe_cache if cache_key[0] == key]
for cache_key in stale:
_file_probe_cache.pop(cache_key, None)
def get_suggestions(vmid, force: bool = False) -> dict:
if force:
invalidate_suggestion_probes(vmid)
ports = _probe_listening_ports(vmid)
web_hint = None
for p in ports:
if p in _KNOWN_WEB_PORTS:
web_hint = "/"
break
meta = _helper_slug_meta(vmid) or {}
slug = meta.get("slug")
# Suppress base-OS helper slugs from the suggestion pipeline.
# community-scripts publishes bare-OS templates (alpine, debian,
# ubuntu, fedora, archlinux, gentoo, opensuse) under the same
# helpers_cache the App tab uses to seed detection, so a CT that
# only has the OS installed was showing up as "detected app:
# Alpine Linux" and inviting the user to register the OS as if
# it were an application. These are not trackable apps — treat
# the slug as absent for suggestion purposes so the panel goes
# straight to the empty state instead.
if slug in {"alpine", "archlinux", "archlinux-vm", "debian", "fedora", "gentoo", "opensuse", "ubuntu"}:
slug = None
meta = {}
# Resolve all verified runtime detectors before building the primary
# suggestion. On a cold Docker CT, legacy helper identity can briefly
# name a child container (for example Portainer). If that child is proven
# to be Docker-hosted and Docker Engine is also proven, Docker is the LXC
# application; the child remains a workload/link beneath it.
hints_map = _fetch_tracking_hints() or {}
detected_map = _probe_detected_apps_map(vmid) if hints_map else {}
primary_matches = detected_map.get(slug) or []
primary_is_docker_workload = bool(primary_matches) and all(
detector.get("installed_via") in ("docker_label", "docker_exec")
for detector in primary_matches
)
if "docker" in detected_map and primary_is_docker_workload:
slug = "docker"
meta = {"slug": "docker", "name": "Docker"}
# Tracking hint pipeline: catalog + curated hints merged.
# • catalog (community-scripts helpers_cache.json) covers ~430
# apps with name+repo+port+upstream_version, zero curation
# from us — refreshed by generate_helpers_cache.py in CI.
# • curated hints (json/app_tracking_hints.json) add
# `installed_via` + method data for the apps where we've
# hand-verified how to detect the installed version.
# Together the frontend can pre-fill the full advanced form when
# both are present, so the user's manual burden shrinks.
tracking = _merge_tracking_hints(slug) if slug else None
# Legacy-layout resolution: when a curated hint declares
# `file_fallbacks`, probe the CT and switch to whichever candidate
# actually exists. Sidecar entry the user saves therefore points at
# the file this specific install produced, not the "modern" path
# the audit assumed. Fallback list is stripped from the returned
# suggestion so the frontend never sees candidate arrays.
if tracking:
tracking, _detected_version, _detector_source, _detector_error = (
_select_working_hint_detector(vmid, tracking)
)
# Candidate arrays remain in the server-side catalog for auto-heal;
# the editor only needs the detector selected for this CT.
tracking.pop("file_fallbacks", None)
tracking.pop("alt_detectors", None)
# Name suggestion: prefer the catalog's `name` (nicer display) but
# keep the raw slug metadata as fallback for older entries.
catalog = _catalog_lookup(slug) if slug else None
name_sug = (catalog or {}).get("name") or meta.get("name")
# Logo URL priority: curated hint > catalog. The hint's logo may
# be an override we set for a mis-detected slug; the catalog is
# the broad fallback (~735 apps in helpers_cache).
hint_dict_for_logo = _fetch_tracking_hints().get(slug) if slug else None
logo_url = ""
if isinstance(hint_dict_for_logo, dict):
raw_logo = hint_dict_for_logo.get("logo")
if isinstance(raw_logo, str) and raw_logo.startswith(("http://", "https://")):
logo_url = raw_logo
if not logo_url and catalog:
raw_logo = catalog.get("logo")
if isinstance(raw_logo, str) and raw_logo.startswith(("http://", "https://")):
logo_url = raw_logo
# Default ports for the editor pre-fill. Priority is:
# 1. Curated hint's `default_ports` (list, may hold several) —
# lets us encode multi-port apps like AdGuard (setup+DNS)
# or NPM (81 admin, 80/443 proxy).
# 2. Catalog's `port` (single) — fallback for slugs we haven't
# curated but community-scripts has a port for.
hint_dict = _fetch_tracking_hints().get(slug) if slug else None
default_ports: list = []
if isinstance(hint_dict, dict):
raw = hint_dict.get("default_ports")
if isinstance(raw, list):
for p in raw:
try:
n = int(p)
if 1 <= n <= 65535:
default_ports.append(n)
except (TypeError, ValueError):
continue
if not default_ports and catalog:
raw = catalog.get("default_port")
try:
n = int(raw)
if 1 <= n <= 65535:
default_ports.append(n)
except (TypeError, ValueError):
pass
# Multi-app detection: probe every hint slug against the CT and
# surface each installed app the primary detection didn't already
# cover. This lets an "AgentDVR + Docker" CT show both apps as
# detected so the user just clicks Register per app instead of
# typing name/logo/repo by hand.
#
# Primary slug is excluded from extras so we don't offer it twice.
# Docker-child suppression: when the user has already registered
# Docker on this CT, they've chosen to manage every containerised
# app under that single Docker entry (web links + notes). Any hint
# whose only matching detector is docker_label/docker_exec would
# therefore appear as a duplicate — the paperless container that
# already runs inside Docker gets offered again as "Paperless-ngx
# detected". Skip those extras so the panel stays honest about
# what lives natively on the CT vs. inside Docker.
sidecar_apps = (_read_sidecar(vmid) or {}).get("apps") or []
docker_registered = any(
(a.get("helper_slug") == "docker") or (a.get("installed_via") == "binary" and (a.get("binary_path") or "").endswith("/docker"))
for a in sidecar_apps
)
# Docker workloads are not native LXC applications. Once Docker is
# detected (or already registered), docker_label/docker_exec matches are
# represented as published service links under Docker rather than generic
# application chips. Native matches remain visible and existing
# independently registered apps are never modified.
docker_workload_detected = any(
det_slug != "docker" and detectors and all(
detector.get("installed_via") in ("docker_label", "docker_exec")
for detector in detectors
)
for det_slug, detectors in detected_map.items()
)
docker_host_detected = (
docker_registered or slug == "docker" or "docker" in detected_map
or docker_workload_detected
)
docker_web_links = _probe_docker_web_links(vmid) if docker_host_detected else []
extras: list = []
for det_slug in sorted(detected_map):
if slug and det_slug == slug:
continue
det_hint = hints_map.get(det_slug) or {}
det_catalog = _catalog_lookup(det_slug) or {}
# Use the DETECTOR THAT ACTUALLY MATCHED on this CT, not the
# hint's primary. Ex: paperless-ngx hint has primary
# docker_label + alt file; on a native install the file
# matched → we build tracking_suggestion around that file
# detector so the form pre-fills the RIGHT method.
matched_detectors = detected_map.get(det_slug) or []
working = matched_detectors[0] if matched_detectors else None
# Skip docker-hosted extras when Docker is registered on the CT.
# We check ALL matched detectors — if EVERY match is a docker
# method, this app is exclusively running inside Docker and
# doesn't warrant a separate registration. If ANY non-docker
# detector also matched (native install alongside a container),
# keep the extra so the user can register the native side.
if docker_host_detected and matched_detectors:
all_docker = all(
d.get("installed_via") in ("docker_label", "docker_exec")
for d in matched_detectors
)
if all_docker:
continue
det_tracking = dict(det_hint)
if working:
# Overwrite the primary-detector fields with what actually
# works here, so the user sees the correct method + target
# in the form. Fields common to all methods (repo,
# tag_regex, github_source, installed_regex) come from the
# hint's primary and stay put.
for k in _DETECTOR_FIELDS + ("installed_via",):
det_tracking.pop(k, None)
for k, v in working.items():
det_tracking[k] = v
det_tracking["detected_version"] = working.get("detected_version")
det_tracking["detector_verified"] = True
det_tracking["detector_source"] = "runtime_probe"
_resolve_file_candidate(vmid, det_tracking)
# Strip fields the frontend doesn't need in the compact chip
det_tracking.pop("file_fallbacks", None)
det_tracking.pop("alt_detectors", None)
# Name: catalog display first, then slug titlecased fallback
det_name = det_catalog.get("name") or det_hint.get("name") or det_slug
# Logo: hint > catalog
det_logo = ""
if isinstance(det_hint.get("logo"), str) and det_hint["logo"].startswith(("http://", "https://")):
det_logo = det_hint["logo"]
elif isinstance(det_catalog.get("logo"), str) and det_catalog["logo"].startswith(("http://", "https://")):
det_logo = det_catalog["logo"]
# default_ports: hint list > catalog single
det_ports: list = []
raw_ports = det_hint.get("default_ports")
if isinstance(raw_ports, list):
for p in raw_ports:
try:
n = int(p)
if 1 <= n <= 65535:
det_ports.append(n)
except (TypeError, ValueError):
continue
if not det_ports and det_catalog.get("default_port"):
try:
n = int(det_catalog["default_port"])
if 1 <= n <= 65535:
det_ports.append(n)
except (TypeError, ValueError):
pass
# Auto-fill Categoría preset from helpers_cache for extras too
# so the Register button pre-selects the category on the port.
det_category = None
det_cat_names = det_catalog.get("category_names") or []
if det_cat_names and isinstance(det_cat_names[0], str) and det_cat_names[0].strip():
det_category = det_cat_names[0].strip()
extras.append({
"slug": det_slug,
"name": det_name,
"logo_url": det_logo or None,
"default_ports": det_ports,
"category": det_category,
"tracking_suggestion": det_tracking,
})
return {
"name_suggestion": name_sug,
"helper_slug": slug,
"port_suggestions": ports,
"web_path_hint": web_hint,
"tracking_suggestion": tracking,
"default_ports": default_ports,
"logo_url": logo_url or None,
# Categoría preset for the primary detection — same lookup as
# get_catalog_entry so the Register button pre-selects it.
"category": suggest_category_for(slug),
"extras": extras,
"docker_web_links": docker_web_links,
}
Reference in New Issue View Git Blame Copy Permalink