mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 10:52:22 +00:00
docs: review Monitor documentation in all seven translations
This commit is contained in:
Jonatan
@@ -18,8 +18,8 @@
|
||||
},
|
||||
"requestFlow": {
|
||||
"heading": "Request flow",
|
||||
"intro": "From the browser to the kernel, every dashboard view follows the same path:",
|
||||
"diagramCaption": "Each request is authenticated by JWT (when auth is enabled), dispatched to a blueprint, and answered with data collected on demand from host tooling. If Fail2Ban is installed and the proxmenux jail is active, the middleware also checks the request against the jail's banned IP list. The optional reverse proxy is transparent to Flask — it forwards X-Forwarded-* headers and the app recovers the real client IP from them. State that needs to outlive a request lives in SQLite.",
|
||||
"intro": "A typical dashboard data request follows this path; browser-only preferences can be handled locally:",
|
||||
"diagramCaption": "Protected routes validate JWT when authentication is enabled. Core routes and blueprints return cached or newly collected host data. The optional Fail2Ban hook checks its cached jail ban list. Forwarded client-IP headers are trusted only with PROXMENUX_TRUST_PROXY=1; otherwise the peer address is used. Persistent state includes SQLite databases and JSON files, including auth.json.",
|
||||
"diagramArrowLabel": "HTTP / WS",
|
||||
"nodes": {
|
||||
"clientLabel": "Client",
|
||||
@@ -31,7 +31,7 @@
|
||||
"stateLabel": "Local state",
|
||||
"stateDetail": "SQLite DB\n+ auth.json"
|
||||
},
|
||||
"threadsIntro": "The same process also runs four <strong>background threads</strong> started at boot — they don't serve HTTP, they push state into SQLite or into the notification queue while the host is up:",
|
||||
"threadsIntro": "The process also runs <strong>background threads</strong>. The examples below sample host data, update caches and SQLite, and dispatch notifications. Other workers also exist, and notification delivery depends on its enabled setting:",
|
||||
"headerThread": "Thread",
|
||||
"headerCadence": "Cadence",
|
||||
"headerJob": "Job",
|
||||
@@ -48,8 +48,8 @@
|
||||
},
|
||||
{
|
||||
"thread": "_vital_signs_sampler",
|
||||
"cadence": "~1 s",
|
||||
"job": "High-frequency CPU + temperature sampler used for live widgets in the Overview panel."
|
||||
"cadence": "CPU 5 s; temperature 15 s; memory 30 s",
|
||||
"job": "Samples CPU usage, memory and CPU temperature for live metrics and sustained-condition detection, independently of the five-minute health collector."
|
||||
},
|
||||
{
|
||||
"thread": "notification_manager.start()",
|
||||
@@ -62,9 +62,9 @@
|
||||
"heading": "systemd unit",
|
||||
"intro": "The installer drops a unit at <code>/etc/systemd/system/proxmenux-monitor.service</code>. Default content:",
|
||||
"items": [
|
||||
"<strong><code>User=root</code></strong> — required: SMART, <code>pvesh</code>, journal scopes, ZFS commands and the web terminal all need root.",
|
||||
"<strong><code>User=root</code></strong> — the supported service runs with root privileges for host operations such as SMART access, <code>pvesh</code> and ZFS management. Its web-terminal processes inherit those privileges.",
|
||||
"<strong><code>Restart=on-failure</code></strong> with a 10-second back-off — non-zero exits relaunch automatically.",
|
||||
"<strong><code>After=network.target</code></strong> — waits for the host network stack to be online."
|
||||
"<strong><code>After=network.target</code></strong> — orders startup after that target; it does not guarantee an assigned IP, DNS or external connectivity."
|
||||
],
|
||||
"inspectTitle": "Inspect the live unit"
|
||||
},
|
||||
@@ -73,13 +73,13 @@
|
||||
"intro": "The AppImage is a self-mounting filesystem. <code>AppRun</code> at the root sets up the environment and execs <code>flask_server.py</code>:",
|
||||
"consequencesIntro": "Two consequences of this layout:",
|
||||
"consequences": [
|
||||
"<strong>No host Python pollution.</strong> The vendored interpreter and packages are isolated inside the AppImage — upgrading the host's system Python doesn't affect the Monitor and vice-versa.",
|
||||
"<strong>Hardware tools are bundled too.</strong> <code>ipmitool</code>, <code>lm-sensors</code> and <code>upsc</code> ship inside the AppImage so the dashboard can read out-of-band sensors and UPS state without forcing the user to install Debian packages."
|
||||
"<strong>Bundled Python environment.</strong> The interpreter and Python packages are shipped inside the AppImage, avoiding installation into the host’s system Python environment. The application still uses host kernel interfaces and external host tools.",
|
||||
"<strong>Hardware clients are bundled too.</strong> <code>ipmitool</code>, <code>lm-sensors</code> and <code>upsc</code> are included. Readings still depend on supported hardware, access to sensors and, for UPS data, a configured NUT server."
|
||||
]
|
||||
},
|
||||
"flask": {
|
||||
"heading": "Flask app structure",
|
||||
"intro": "<code>flask_server.py</code> creates a single <code>Flask(__name__)</code> instance, enables CORS, and registers six blueprints plus a WebSocket initializer:",
|
||||
"intro": "<code>flask_server.py</code> creates one <code>Flask(__name__)</code> instance with restricted CORS origins, registers nine blueprints directly and initializes terminal routes. The table highlights the main modules; audit, VM-backup and PBS blueprints are also registered. PVE-specific prewarmers run only on PVE hosts.",
|
||||
"headerBlueprint": "Blueprint / module",
|
||||
"headerPrefix": "Routes prefix",
|
||||
"headerOwns": "Owns",
|
||||
@@ -109,7 +109,7 @@
|
||||
"prefix": [
|
||||
"/api/health/*"
|
||||
],
|
||||
"owns": "Public health probe, detailed status, active / dismissed errors, suppression settings."
|
||||
"owns": "Health probes, detailed status, active and dismissed errors, and suppression settings. Protected routes require authentication when it is enabled."
|
||||
},
|
||||
{
|
||||
"blueprint": "flask_terminal_routes.py",
|
||||
@@ -137,7 +137,7 @@
|
||||
"prefix": [
|
||||
"/api/proxmenux/*"
|
||||
],
|
||||
"owns": "Reads which ProxMenux post-install optimizations are installed on the host."
|
||||
"owns": "Reads the ProxMenux post-install optimizations recorded in the host registry."
|
||||
},
|
||||
{
|
||||
"blueprint": "flask_oci_routes.py",
|
||||
@@ -147,11 +147,11 @@
|
||||
"owns": "OCI / container app deployment helpers (Proxmox VE 9.1+)."
|
||||
}
|
||||
],
|
||||
"endpointsLink": "The full endpoint list with request / response shapes is in <link>API Reference</link>."
|
||||
"endpointsLink": "Integration endpoints and their request/response formats are described in <link>API Reference</link>."
|
||||
},
|
||||
"dataSources": {
|
||||
"heading": "Data sources",
|
||||
"intro": "Nothing is collected from a custom agent — the Monitor reads the same files and runs the same commands a human admin would:",
|
||||
"intro": "Host data comes mainly from standard libraries, files and administrative commands. Some guest information also depends on the QEMU Guest Agent or commands executed inside a CT:",
|
||||
"headerSource": "Source",
|
||||
"headerUsedFor": "Used for",
|
||||
"rows": [
|
||||
@@ -181,7 +181,7 @@
|
||||
},
|
||||
{
|
||||
"source": "nvidia-smi · intel_gpu_top",
|
||||
"usedFor": "GPU utilisation, VRAM, temperature, encoder / decoder load."
|
||||
"usedFor": "Available GPU utilisation, VRAM, temperature and encoder/decoder measurements; fields depend on the vendor, tool and driver."
|
||||
},
|
||||
{
|
||||
"source": "lspci · lscpu · dmidecode",
|
||||
@@ -201,7 +201,7 @@
|
||||
},
|
||||
"persistence": {
|
||||
"heading": "Persistence",
|
||||
"intro": "Two filesystem locations split state by sensitivity:",
|
||||
"intro": "Several state files have different contents and access requirements:",
|
||||
"headerPath": "Path",
|
||||
"headerOwner": "Owner",
|
||||
"headerContents": "Contents",
|
||||
@@ -214,25 +214,25 @@
|
||||
{
|
||||
"path": "/usr/local/share/proxmenux/.notification_key",
|
||||
"owner": "root <code>0600</code>",
|
||||
"contents": "32-byte XOR key used to encrypt sensitive notification settings before storing them in the DB (Telegram tokens, AI API keys, etc.)."
|
||||
"contents": "32-byte master key for the ENC2 notification-secret format. The implementation uses a per-value random salt, derived keys and a MAC; old fixed-key XOR values are supported for migration."
|
||||
},
|
||||
{
|
||||
"path": "/root/.config/proxmenux-monitor/auth.json",
|
||||
"owner": "root:root",
|
||||
"contents": "Authentication state: enabled flag, username, SHA-256 password hash, TOTP secret, backup codes, list of issued API tokens, list of revoked token hashes."
|
||||
"contents": "Authentication configuration: enabled flag, username, salted PBKDF2-HMAC-SHA256 password hash, TOTP secret, backup codes, per-install JWT signing secret, API-token metadata and revocation hashes. Legacy SHA-256 password hashes are migrated on successful login."
|
||||
},
|
||||
{
|
||||
"path": "/var/log/proxmenux-auth.log",
|
||||
"owner": "root:root",
|
||||
"contents": "Plain-text auth event log. Always written. If Fail2Ban is installed with the <code>[proxmenux]</code> jail, the jail reads this file to ban brute-force attempts; if not, the file simply accumulates the log entries."
|
||||
"contents": "Plain-text authentication event log, written independently of Fail2Ban. When the <code>[proxmenux]</code> jail is installed and enabled, it reads this log to detect repeated failed logins. Without that jail, the log remains available for inspection."
|
||||
}
|
||||
],
|
||||
"backupTitle": "Back up auth.json before reinstalling",
|
||||
"backupBody": "Reinstalling the AppImage replaces the binary but leaves <code>/root/.config/proxmenux-monitor/auth.json</code> and <code>/usr/local/share/proxmenux/health_monitor.db</code> intact. If you restore from a host backup, keep both files together — the API tokens stored in <code>auth.json</code> are validated against <code>JWT_SECRET</code>; if the DB and auth.json get out of sync, dismissed errors and stored tokens may misbehave."
|
||||
"backupBody": "Replacing the AppImage normally preserves <code>/root/.config/proxmenux-monitor/auth.json</code> and <code>/usr/local/share/proxmenux/health_monitor.db</code>. Preserve <code>auth.json</code> with its per-install jwt_secret to retain token validity and metadata. The database stores health and notification state independently. Back up <code>.notification_key</code> with the database to restore encrypted channel settings; restoring the DB alone does not restore API authentication."
|
||||
},
|
||||
"health": {
|
||||
"heading": "Health Monitor cycle",
|
||||
"intro": "Every 5 minutes <code>health_monitor.py</code> runs a deterministic cycle across the ten categories shown on the dashboard:",
|
||||
"intro": "About every five minutes, the background collector invokes <code>health_monitor.py</code> to run the checks behind the dashboard’s ten categories:",
|
||||
"items": [
|
||||
"Critical PVE services (<code>pveproxy</code>, <code>pvedaemon</code>, <code>pvestatd</code>, <code>pve-cluster</code>).",
|
||||
"Proxmox storage pools (<code>pvesh get /storage</code> + per-storage availability).",
|
||||
@@ -242,11 +242,11 @@
|
||||
"Updates: pending package upgrades and security patches.",
|
||||
"Logs: persistent / spike / cascade pattern detection in the system journal.",
|
||||
"Memory: OOM killer activity, sustained high pressure.",
|
||||
"Temperature: CPU / chassis sensors against vendor thresholds.",
|
||||
"Temperature: CPU and disk readings checked against monitoring thresholds; available sensors depend on the hardware.",
|
||||
"Security: authentication failures, ban events, fail2ban jail status."
|
||||
],
|
||||
"afterIntro": "Each finding is normalised into a stable <code>error_key</code> + category + severity. The persistence layer deduplicates against the existing <code>errors</code> table — repeated events update <code>last_seen</code> and the occurrence counter without spamming notifications.",
|
||||
"cycleEnd": "The cycle also auto-resolves stale errors using the per-category <em>Suppression Duration</em> setting, cleans up errors for resources that no longer exist (deleted VMs / removed disks / unmounted storages), and prunes the <code>events</code> log older than 30 days. The full catalogue of categories and the dashboard view that surfaces them is documented in <link>Dashboard → Health Monitor</link>."
|
||||
"afterIntro": "Findings use a stable <code>error_key</code>, category and severity. Existing records in <code>errors</code> are updated with <code>last_seen</code> and occurrence counts rather than duplicated. Notification delivery also has its own fingerprint and cooldown rules.",
|
||||
"cycleEnd": "Cleanup can auto-resolve stale errors using category <em>Suppression Duration</em> values and state-based rules, remove orphan records for resources that were removed, and prune <code>events</code> older than 30 days. The categories and dashboard are described in <link>Dashboard → Health Monitor</link>."
|
||||
},
|
||||
"notifications": {
|
||||
"heading": "Notification engine",
|
||||
@@ -254,7 +254,7 @@
|
||||
"items": [
|
||||
"<strong>Watchers</strong> push events: <code>JournalWatcher</code> tails the system journal, <code>TaskWatcher</code> polls the Proxmox task list, <code>ProxmoxHookWatcher</code> reacts to backup / replication / snapshot hooks, and <code>PollingCollector</code> handles slow data sources.",
|
||||
"<strong>Templates</strong> turn an event into a (title, body) pair. The same template can run through the configured AI provider (OpenAI / Anthropic / Gemini / Groq / Ollama / OpenRouter) to produce a plain-language rewrite; both versions are stored in <code>notification_history</code>.",
|
||||
"<strong>Channels</strong> deliver messages: Telegram, Discord, Email, Gotify, Pushover and Apprise (multi-channel). Each is implemented in <code>notification_channels.py</code> behind the same <code>create_channel()</code> / <code>send()</code> interface, so adding a new channel is a single class.",
|
||||
"<strong>Channels</strong> deliver messages through services including Telegram, Discord, Email, Gotify, Pushover and Apprise. Implementations in <code>notification_channels.py</code> use the shared <code>create_channel()</code> / <code>send()</code> interface.",
|
||||
"<strong>Encryption.</strong> Sensitive settings (<code>telegram.bot_token</code>, <code>discord.webhook_url</code>, <code>pushover.user_key</code>, <code>pushover.api_token</code>, <code>ai_api_key_*</code>, <code>email.password</code>) are encrypted with the key in <code>.notification_key</code> before being written to the DB and are masked in the interface."
|
||||
],
|
||||
"linksFooter": "Per-event toggles, channel overrides and AI configuration are surfaced in <notifLink>Settings → Notifications</notifLink> and <aiLink>Settings → AI Assistant</aiLink>."
|
||||
@@ -263,21 +263,21 @@
|
||||
"heading": "WebSocket terminal",
|
||||
"intro": "The <em>Terminal</em> tab in the dashboard is a thin <code>xterm.js</code> client wired to a server-side PTY through a WebSocket. Two transport modes:",
|
||||
"items": [
|
||||
"<strong>HTTP mode (default):</strong> Flask's development server with <code>flask-sock</code> handles upgrade requests. Good enough for LAN / direct access.",
|
||||
"<strong>HTTPS / WSS mode:</strong> when an SSL certificate is configured, the process switches to <code>gevent.pywsgi.WSGIServer</code> with <code>geventwebsocket.handler.WebSocketHandler</code>, so WebSockets work over TLS without polyfills."
|
||||
"<strong>HTTP mode (default):</strong> the built-in Flask server with <code>flask-sock</code> handles WebSocket upgrade requests.",
|
||||
"<strong>HTTPS / WSS mode:</strong> the TLS server uses <code>gevent.pywsgi.WSGIServer</code> with <code>geventwebsocket.handler.WebSocketHandler</code> for WebSocket connections over TLS."
|
||||
],
|
||||
"outro": "The PTY is a child of the Flask process, so it inherits <code>User=root</code> from the unit. Every terminal request goes through JWT auth; the user must already be logged in to the dashboard before a PTY is allocated.",
|
||||
"outro": "Terminal child processes inherit <code>User=root</code> from the service. With authentication enabled, opening a WebSocket requires a short-lived, single-use ticket obtained from the authenticated terminal-ticket endpoint with administrative scope. When authentication is disabled or declined, the handshake follows that unauthenticated mode.",
|
||||
"proxyNote": "If you access the Monitor through a reverse proxy, make sure WebSocket forwarding is enabled (the <code>Upgrade</code> and <code>Connection</code> headers). Without it the terminal won't work."
|
||||
},
|
||||
"proxy": {
|
||||
"heading": "Reverse proxy & Fail2Ban",
|
||||
"intro": "Two safeguards make sure security works the same way whether the dashboard is hit directly or through a reverse proxy:",
|
||||
"intro": "Client-IP selection and optional application-level blocking depend on these settings and components:",
|
||||
"items": [
|
||||
"<strong>Real client IP recovery.</strong> A <code>before_request</code> hook reads <code>X-Forwarded-For</code> and <code>X-Real-IP</code> in that order, falling back to <code>request.remote_addr</code>. The recovered address is what auth logging and rate limiting see. This is always on.",
|
||||
"<strong>Application-level Fail2Ban check (optional).</strong> When the dashboard sits behind a proxy, the kernel firewall can't block the real attacker IP — the connection always comes from the proxy. To plug that gap, the same hook above queries the <code>proxmenux</code> Fail2Ban jail every 30 seconds, caches the banned IP set, and short-circuits requests from those IPs with HTTP 403 inside Flask."
|
||||
"<strong>Client IP selection.</strong> The <code>before_request</code> ban check and auth logging trust <code>X-Forwarded-For</code>, then <code>X-Real-IP</code>, only when PROXMENUX_TRUST_PROXY=1. Otherwise they use <code>request.remote_addr</code>. Forwarded headers are not accepted automatically.",
|
||||
"<strong>Optional application-level Fail2Ban.</strong> A proxy connection can present the proxy as the TCP peer. When the trusted client address is available and the <code>proxmenux</code> jail is active, Flask rejects listed IPs with HTTP 403. The ban list is cached for 30 seconds and refreshed by request processing after expiry, rather than polled by a dedicated timer."
|
||||
],
|
||||
"calloutTitle": "Fail2Ban is not bundled",
|
||||
"calloutBody": "Fail2Ban is <strong>not</strong> installed by ProxMenux Monitor itself. The application-level check is a no-op until you install Fail2Ban on the host (e.g. via <link>Security → Fail2Ban</link> in the ProxMenux menu). When the <code>fail2ban-client</code> binary or the <code>proxmenux</code> jail is absent, the call fails silently and requests are not gated — auth still applies, but no IP-level banning.",
|
||||
"calloutBody": "Fail2Ban is <strong>not</strong> bundled or installed automatically. It can be installed through the optional Monitor Security workflow or <link>Security → Fail2Ban</link> in ProxMenux Scripts. If <code>fail2ban-client</code> or the <code>proxmenux</code> jail is absent, this application hook does not block requests. The configured authentication policy still applies.",
|
||||
"outro": "Reverse-proxy snippets (Nginx / Caddy / Traefik) and the Fail2Ban jail walkthrough are in <accessLink>Access & Authentication</accessLink> and <fail2banLink>Security → Fail2Ban</fail2banLink>."
|
||||
},
|
||||
"whereNext": {
|
||||
@@ -291,7 +291,7 @@
|
||||
{
|
||||
"label": "API Reference",
|
||||
"href": "/docs/monitor/api",
|
||||
"tail": " — every endpoint, token management, security best-practices."
|
||||
"tail": " — integration endpoints, token management and authentication conventions."
|
||||
},
|
||||
{
|
||||
"label": "Settings → ProxMenux Monitor",
|
||||
|
||||
Reference in New Issue
Block a user