Instant VM/LXC modals via bulk hydration + server prewarmer

Backend keeps every guest's modal payload (details, backups, apps, schedule, mount points) warm in-memory and exposes them through a single `/api/vms/modal-cache-all` endpoint. The dashboard hydrates its entire modal cache from that one request on page load, so opening any guest — first click or after coming back later — renders instantly. Replaces ~84 per-guest fetches with 1.

AppImage/scripts/flask_notification_routes.py

@@ -1212,9 +1212,13 @@ def setup_pve_webhook_core() -> dict:
         # `could not decode UTF8 string from base64, key 'X-Webhook-Secret' (500)`
         # whenever `token_urlsafe` produced `-` or `_` chars (GH #198).
         secret_b64 = base64.b64encode(secret.encode()).decode()
+        # PVE parses /etc/pve/*.cfg TAB-strict. The endpoint_block above
+        # indents with `\t`; the priv_block MUST too, or PVE silently
+        # ignores the `secret` line and never sends `X-Webhook-Secret`,
+        # so every remote delivery lands as 401 invalid_secret. GH #294.
         priv_block = (
             f"webhook: {_PVE_ENDPOINT_ID}\n"
-            f"        secret name=X-Webhook-Secret,value={secret_b64}\n"
+            f"\tsecret name=X-Webhook-Secret,value={secret_b64}\n"
         )
         
         if priv_text is not None:
@@ -1234,9 +1238,14 @@ def setup_pve_webhook_core() -> dict:
             result['error'] = f'Permission denied writing {_PVE_PRIV_CFG}'
             result['fallback_commands'] = _build_webhook_fallback()
             return result
-        except Exception:
-            pass
-        
+        except Exception as e:
+            # Silently swallowing this here would report configured:True
+            # while PVE has no valid secret — exactly the failure mode of
+            # GH #294. Surface it so the caller can flag the setup.
+            result['error'] = f'Failed writing {_PVE_PRIV_CFG}: {e}'
+            result['fallback_commands'] = _build_webhook_fallback()
+            return result
+
         result['configured'] = True
         result['secret'] = secret
         return result
@@ -1489,19 +1498,37 @@ def proxmox_webhook():
             if not hmac.compare_digest(configured_secret, request_secret):
                 return _reject(401, 'invalid_secret', 401)
         
-        # Layer 3: Anti-replay timestamp
+        # Layer 3: Anti-replay timestamp.
+        # PVE's webhook notification target can only send a static secret
+        # header + a Handlebars-templated body; it cannot inject a custom
+        # dynamic header, so `X-ProxMenux-Timestamp` never arrives from a
+        # PVE-origin delivery (GH #294). Our own PVE endpoint template
+        # already embeds `"timestamp":"{{ timestamp }}"` (Unix epoch), so
+        # accept the body value as a fallback when the header is absent.
+        # The replay cache in Layer 4 still binds every accepted request
+        # to (timestamp, raw_body), so this widens the source of the
+        # timestamp without weakening the anti-replay guarantee.
+        raw_body = request.get_data(as_text=True) or ''
         ts_header = request.headers.get('X-ProxMenux-Timestamp', '')
-        if not ts_header:
+        ts_value = None
+        if ts_header:
+            try:
+                ts_value = int(ts_header)
+            except (ValueError, TypeError):
+                return _reject(401, 'invalid_timestamp', 401)
+        elif raw_body:
+            try:
+                body_ts = json.loads(raw_body).get('timestamp')
+                if body_ts is not None:
+                    ts_value = int(str(body_ts).strip())
+            except (ValueError, TypeError, json.JSONDecodeError, AttributeError):
+                ts_value = None
+        if ts_value is None:
             return _reject(401, 'missing_timestamp', 401)
-        try:
-            ts_value = int(ts_header)
-        except (ValueError, TypeError):
-            return _reject(401, 'invalid_timestamp', 401)
         if abs(time.time() - ts_value) > _TIMESTAMP_MAX_DRIFT:
             return _reject(401, 'timestamp_expired', 401)
-        
+
         # Layer 4: Replay cache
-        raw_body = request.get_data(as_text=True) or ''
         signature = hashlib.sha256(f"{ts_value}:{raw_body}".encode(errors='replace')).hexdigest()
         if _replay_cache.check_and_record(signature):
             return _reject(409, 'replay_detected', 409)

AppImage/scripts/health_monitor.py

@@ -2591,7 +2591,13 @@ class HealthMonitor:
             # the `removable` flag, since USB-NVMe and USB-HDD both report
             # `removable=0` even though they ARE USB.
             attempts = []
-            if _is_disk_usb(disk_name) or _is_disk_removable(disk_name):
+            # SNT drivers are NVMe-Storage-Namespace-Transport — only
+            # meaningful when the underlying device is NVMe. Restricting
+            # to `nvme*` kernel nodes stops `sd*` USB-SATA (TerraMaster
+            # DAS, USB HDD/SSD enclosures) from paying 3×5 s of dead
+            # smartctl timeouts before the plain probe runs (GH #293).
+            is_nvme_class = disk_name.startswith('nvme')
+            if is_nvme_class and (_is_disk_usb(disk_name) or _is_disk_removable(disk_name)):
                 for drv in _USB_NVME_DRIVERS:
                     attempts.append(['smartctl', '-i', '-j', '-d', drv, dev_path])
             attempts.append(['smartctl', '-i', '-j', dev_path])
@@ -2656,7 +2662,11 @@ class HealthMonitor:
             # USB detection uses the sysfs path so USB-NVMe bridges (which
             # report removable=0) are caught too.
             attempts = []
-            if _is_disk_usb(disk_name) or _is_disk_removable(disk_name):
+            # Same NVMe-class guard as `_get_disk_identity` — see the
+            # comment there. Prevents USB-SATA drives from wasting
+            # timeouts on drivers that will never respond (GH #293).
+            is_nvme_class = disk_name.startswith('nvme')
+            if is_nvme_class and (_is_disk_usb(disk_name) or _is_disk_removable(disk_name)):
                 for drv in _USB_NVME_DRIVERS:
                     attempts.append(['smartctl', '-n', 'standby', '--health', '-j', '-d', drv, dev_path])
             attempts.append(['smartctl', '-n', 'standby', '--health', '-j', dev_path])

AppImage/scripts/jwt_middleware.py

@@ -4,7 +4,7 @@ Provides decorator to protect Flask routes with JWT authentication
 Automatically checks auth status and validates tokens
 """
 
-from flask import request, jsonify
+from flask import request, jsonify, g
 from functools import wraps
 from auth_manager import load_auth_config, verify_token, verify_token_full
 
@@ -26,9 +26,20 @@ def require_auth(f):
     """
     @wraps(f)
     def decorated_function(*args, **kwargs):
+        # Internal calls (background prewarmers, in-process cache
+        # refresh) bypass auth. Set `g._internal_call = True` inside
+        # an `app.test_request_context()` block before invoking a
+        # decorated handler — the flag lives only for that context so
+        # a real HTTP request can never accidentally inherit it.
+        try:
+            if getattr(g, '_internal_call', False):
+                return f(*args, **kwargs)
+        except RuntimeError:
+            pass  # No request context yet — treat as normal auth flow.
+
         # Check if authentication is enabled
         config = load_auth_config()
-        
+
         # If auth is disabled or declined, allow access
         if not config.get("enabled", False) or config.get("declined", False):
             return f(*args, **kwargs)

AppImage/scripts/lxc_apps.py

@@ -796,6 +796,16 @@ def validate_config(payload: dict) -> tuple[bool, Any]:
     if hn is not None:
         conf["hide_no_updater_notice"] = bool(hn)
 
+    # Optional per-app switch for `app_update_available` notifications.
+    # Default is True (opt-out). Set to False from the App tab when the
+    # user knows an app can't be updated on their box (compat, forked
+    # setup, etc.) and wants to keep the "you have updates" badge but
+    # silence the outbound notification for THIS app only, without
+    # touching the global toggle.
+    ne = payload.get("notifications_enabled")
+    if ne is not None:
+        conf["notifications_enabled"] = bool(ne)
+
     return True, conf
 
 
@@ -1575,6 +1585,12 @@ def record_schedule_run(vmid, status: str, target: str) -> bool:
 
 
 def _fire_update_notification(vmid, app: dict) -> None:
+    # Per-app opt-out: user flipped the bell icon off for this specific
+    # app (because they know it can't be updated on their box or they
+    # just don't care). Field defaults to True — an app registered
+    # before this feature landed keeps receiving notifications.
+    if app.get("notifications_enabled", True) is False:
+        return
     try:
         from notification_manager import notification_manager
         import socket
@@ -1708,7 +1724,6 @@ def check_app(vmid, app_id: str, force: bool = False) -> Optional[dict]:
 
         err = inst_err or up_err
         update_available = compare(installed, latest) if (installed and latest) else None
-        prev_latest = state.get("latest_version")
 
         app["state"] = {
             "installed_version": installed,
@@ -1720,12 +1735,75 @@ def check_app(vmid, app_id: str, force: bool = False) -> Optional[dict]:
         sidecar["updated_at"] = _now_iso()
         _write_sidecar(vmid, sidecar)
 
-        if update_available and latest and latest != prev_latest:
+        # Emit every time an update is pending. The old `latest !=
+        # prev_latest` guard tried to prevent spam by only firing on
+        # the first observation of each new upstream version, but it
+        # also swallowed the emit whenever the notification setting
+        # was toggled off → on after the first observation (the
+        # sidecar already had `latest_version` recorded, so subsequent
+        # checks looked like "same latest, nothing to do"). Anti-spam
+        # is the notification manager's job: it dedups by `entity_id`
+        # (vmid + app_id + latest_version) with its cooldown, and only
+        # a genuinely new upstream release changes the entity_id and
+        # triggers a fresh delivery.
+        if update_available and latest:
             _fire_update_notification(vmid, app)
 
         return sidecar
 
 
+def emit_all_pending_updates() -> int:
+    """Walk every sidecar and emit `app_update_available` for each
+    app currently marked with a pending upstream release. Safe to
+    call repeatedly — `notification_manager` dedups by entity_id
+    (vmid + app_id + latest_version), so a given release only sends
+    once until a newer version appears.
+
+    Needed because `check_app(force=False)` short-circuits on a fresh
+    `checked_at` and never reaches the emit path. The 24 h
+    PollingCollector runs `refresh_all_apps(force=False)`, so without
+    this helper the notification only ever fired on the exact tick
+    where a new upstream version was FIRST observed — and even that
+    was silenced when the user's setting was OFF at the time.
+    Returns the number of emits attempted (delivery still depends on
+    channel enablement + cooldown + rate limit)."""
+    try:
+        entries = sorted(os.listdir(_APPS_DIR))
+    except (FileNotFoundError, OSError):
+        print("[ProxMenux] emit_all_pending_updates: _APPS_DIR missing", flush=True)
+        return 0
+    n = 0
+    print(f"[ProxMenux] emit_all_pending_updates: scanning {len(entries)} sidecar file(s)", flush=True)
+    for name in entries:
+        if not name.endswith(".json"):
+            continue
+        try:
+            vmid = int(name[:-5])
+        except ValueError:
+            continue
+        try:
+            sidecar = _read_sidecar(vmid)
+            if not sidecar:
+                print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} sidecar empty", flush=True)
+                continue
+            apps = sidecar.get("apps") or []
+            pending = [a for a in apps
+                       if (a.get("state") or {}).get("update_available")
+                       and (a.get("state") or {}).get("latest_version")]
+            print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} apps={len(apps)} pending={len(pending)}", flush=True)
+            for app in pending:
+                try:
+                    _fire_update_notification(vmid, app)
+                    n += 1
+                    print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} emit '{app.get('name')}'", flush=True)
+                except Exception as inner:
+                    print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} emit '{app.get('name')}' FAILED: {inner}", flush=True)
+        except Exception as e:
+            print(f"[ProxMenux] emit_all_pending_updates: CT {vmid} outer failure: {e}", flush=True)
+    print(f"[ProxMenux] emit_all_pending_updates: {n} emit(s) attempted total", flush=True)
+    return n
+
+
 def check_all(vmid, force: bool = False) -> Optional[dict]:
     sidecar = _read_sidecar(vmid)
     if not sidecar:

AppImage/scripts/notification_events.py

@@ -3516,6 +3516,17 @@ class PollingCollector:
         try:
             import lxc_apps
             lxc_apps.refresh_all_apps(force=False)
+            # After the refresh, emit `app_update_available` for every
+            # sidecar entry currently flagged with a pending upstream
+            # release. `check_app(force=False)` short-circuits on a
+            # fresh `checked_at` and never reaches the emit path, so
+            # without this call the notification only ever fired on
+            # the exact tick where a new version was FIRST observed —
+            # missed forever if the user had the toggle off at that
+            # moment. `notification_manager` dedups by entity_id
+            # (vmid + app_id + latest_version) so repeated calls only
+            # deliver one notification per release.
+            lxc_apps.emit_all_pending_updates()
         except Exception as e:
             print(f"[PollingCollector] lxc_apps refresh failed: {e}")
 

AppImage/scripts/notification_manager.py

@@ -398,7 +398,13 @@ GROUP_RATE_LIMITS = {
     'backup':    {'max_per_minute': 5,  'max_per_hour': 30},
     'services':  {'max_per_minute': 5,  'max_per_hour': 30},
     'health':    {'max_per_minute': 3,  'max_per_hour': 20},
-    'updates':   {'max_per_minute': 3,  'max_per_hour': 15},
+    # Bumped from 3/min-15/hour: startup reset re-fires every update
+    # event (app_update x N + nvidia + secure_gateway + post_install
+    # + summary…) in a single burst; a 3/min ceiling silently dropped
+    # everything past the third. Steady-state update noise is very
+    # low (one event per upstream release), so a wider window costs
+    # nothing.
+    'updates':   {'max_per_minute': 15, 'max_per_hour': 60},
     'other':     {'max_per_minute': 5,  'max_per_hour': 30},
 }
 
@@ -507,6 +513,13 @@ _DEFAULT_AGGREGATION = {'window': 60, 'min_count': 2, 'burst_type': 'burst_gener
 # recovery is per-event; collapsing them adds zero information.
 _AGGREGATION_EXEMPT_EVENTS = frozenset({
     'error_resolved',
+    # Per-app upstream update. Each event carries a distinct app name,
+    # version and CT id — collapsing "5 app updates burst" into a
+    # summary hides exactly the information the user wants (which
+    # apps, which versions). Startup emit fires all pending updates
+    # at once, so without this exemption only the first 1-2 land and
+    # the rest get buffered into a useless summary.
+    'app_update_available',
 })
 
 
@@ -1940,14 +1953,19 @@ class NotificationManager:
     # (log_critical_*, disk errors, smart_*, …) — preserves the
     # anti-flood guarantee for sources that can burst.
     _EVENT_TYPES_RESET_ON_START = (
-        # Update-status reports
+        # Update-status reports — re-fire on Monitor restart so the
+        # user gets a fresh "here's what's pending" as a health check
+        # that the notification pipeline is alive. Steady-state 24 h
+        # cooldown resumes after that first post-restart send.
         'update_summary',
         'proxmenux_update',
         'post_install_update',
         'pve_update',
         'update_available',
         'nvidia_driver_update_available',
+        'coral_driver_update_available',
         'secure_gateway_update_available',
+        'app_update_available',
         # Security events that must not be silenced by stale cooldowns
         # following a Monitor reinstall (Pedro Rico, 19/05).
         'auth_fail',

AppImage/scripts/notification_templates.py

@@ -524,12 +524,21 @@ TEMPLATES = {
         'title': '{hostname}: {app_name} update available on CT {vmid}',
         'body': (
             '{app_name} on CT {vmid} ({ct_name}) has a new version:\n'
-            '    {installed} → {latest}\n'
-            'Registered via ProxMenux App Watch.'
+            '    {installed} → {latest}'
         ),
-        'label': 'App update available (App Watch)',
-        'group': 'vm_ct',
-        'default_enabled': False,
+        'label': 'App update available',
+        # Grouped under `updates` (not `vm_ct`) so the user can toggle
+        # per-app upstream notifications independently from VM/CT
+        # lifecycle events (start/stop/reboot). Sitting alongside the
+        # other update templates keeps the Settings UI consistent and
+        # leaves the group ready for future OCI-image notifications
+        # that share the same "an upstream release is available"
+        # semantics.
+        'group': 'updates',
+        # Every other update template ships enabled by default; leaving
+        # this one off meant users who registered apps in the App tab
+        # never received the notification they explicitly asked for.
+        'default_enabled': True,
     },
     'vm_start': {
         'title': '{hostname}: VM {vmname} ({vmid}) started',
@@ -1076,7 +1085,7 @@ TEMPLATES = {
             'Kernel updates: {kernel_count}\n'
             'Important packages:\n{important_list}'
         ),
-        'label': 'Updates available',
+        'label': 'Host package updates',
         'group': 'updates',
         'default_enabled': True,
     },
@@ -1090,7 +1099,7 @@ TEMPLATES = {
     'update_complete': {
         'title': '{hostname}: System update completed',
         'body': 'System packages have been successfully updated.\n{details}',
-        'label': 'Update completed',
+        'label': 'Host update completed',
         'group': 'updates',
         'default_enabled': False,
     },