diff --git a/AppImage/scripts/flask_notification_routes.py b/AppImage/scripts/flask_notification_routes.py index 8749424c..3f045f7b 100644 --- a/AppImage/scripts/flask_notification_routes.py +++ b/AppImage/scripts/flask_notification_routes.py @@ -229,16 +229,32 @@ def _pve_remove_our_blocks(text, headers_to_remove): return ''.join(cleaned) -@notification_bp.route('/api/notifications/proxmox/setup-webhook', methods=['POST']) -def setup_proxmox_webhook(): - """Automatically configure PVE notifications to call our webhook. - - Strategy: parse existing config into discrete blocks, only add or - replace blocks whose name matches our IDs, preserve everything else - byte-for-byte. Creates timestamped backups before any modification. +def _build_webhook_fallback(): + """Build fallback manual commands for webhook setup.""" + return [ + "# Append to END of /etc/pve/notifications.cfg", + "# (do NOT delete existing content):", + "", + f"webhook: {_PVE_ENDPOINT_ID}", + f"\tmethod post", + f"\turl {_PVE_WEBHOOK_URL}", + '\theader Content-Type=application/json', + '\tbody {"title":"{{ escape title }}","message":"{{ escape message }}","severity":"{{ severity }}","timestamp":{{ timestamp }},"fields":{{ json fields }}}', + "", + f"matcher: {_PVE_MATCHER_ID}", + f"\ttarget {_PVE_ENDPOINT_ID}", + "\tmode all", + "", + "# ALSO append to /etc/pve/priv/notifications.cfg :", + f"webhook: {_PVE_ENDPOINT_ID}", + ] + + +def setup_pve_webhook_core() -> dict: + """Core logic to configure PVE webhook. Callable from anywhere. + Returns dict with 'configured', 'error', 'fallback_commands' keys. Idempotent: safe to call multiple times. - Only touches blocks named 'proxmenux-webhook' / 'proxmenux-default'. """ import secrets as secrets_mod @@ -251,20 +267,6 @@ def setup_proxmox_webhook(): 'error': None, } - def _build_fallback(): - return [ - "# Append to END of /etc/pve/notifications.cfg", - "# (do NOT delete existing content):", - "", - f"webhook: {_PVE_ENDPOINT_ID}", - f"\tmethod post", - f"\turl {_PVE_WEBHOOK_URL}", - "", - f"matcher: {_PVE_MATCHER_ID}", - f"\ttarget {_PVE_ENDPOINT_ID}", - "\tmode all", - ] - try: # ── Step 1: Ensure webhook secret exists (for our own internal use) ── secret = notification_manager.get_webhook_secret() @@ -276,8 +278,8 @@ def setup_proxmox_webhook(): cfg_text, err = _pve_read_file(_PVE_NOTIFICATIONS_CFG) if err: result['error'] = err - result['fallback_commands'] = _build_fallback() - return jsonify(result), 200 + result['fallback_commands'] = _build_webhook_fallback() + return result # ── Step 3: Read priv config (to clean up any broken blocks we wrote before) ── priv_text, err = _pve_read_file(_PVE_PRIV_CFG) @@ -306,10 +308,18 @@ def setup_proxmox_webhook(): # PVE secret format is: secret name=key,value= # Neither is needed for localhost calls. + # PVE webhook format requires: + # - body: Handlebars template for the HTTP body + # - header: Content-Type header for JSON + # - A matching entry in priv/notifications.cfg (even if empty) + # The body template uses PVE's Handlebars syntax to pass notification + # metadata to our webhook handler as structured JSON. endpoint_block = ( f"webhook: {_PVE_ENDPOINT_ID}\n" f"\tmethod post\n" f"\turl {_PVE_WEBHOOK_URL}\n" + f'\theader Content-Type=application/json\n' + f'\tbody {{"title":"{{{{ escape title }}}}","message":"{{{{ escape message }}}}","severity":"{{{{ severity }}}}","timestamp":{{{{ timestamp }}}},"fields":{{{{ json fields }}}}}}\n' ) matcher_block = ( @@ -332,8 +342,8 @@ def setup_proxmox_webhook(): f.write(new_cfg) except PermissionError: result['error'] = f'Permission denied writing {_PVE_NOTIFICATIONS_CFG}' - result['fallback_commands'] = _build_fallback() - return jsonify(result), 200 + result['fallback_commands'] = _build_webhook_fallback() + return result except Exception as e: try: with open(_PVE_NOTIFICATIONS_CFG, 'w') as f: @@ -341,35 +351,58 @@ def setup_proxmox_webhook(): except Exception: pass result['error'] = str(e) - result['fallback_commands'] = _build_fallback() - return jsonify(result), 200 + result['fallback_commands'] = _build_webhook_fallback() + return result - # ── Step 9: Clean priv config (remove our broken blocks, write nothing new) ── - if priv_text is not None and cleaned_priv != priv_text: - try: - with open(_PVE_PRIV_CFG, 'w') as f: - f.write(cleaned_priv) - except Exception: - pass + # ── Step 9: Write priv config with our webhook entry ── + # PVE REQUIRES a matching block in priv/notifications.cfg for every + # webhook endpoint, even if it has no secrets. Without it PVE throws: + # "Could not instantiate endpoint: private config does not exist" + priv_block = ( + f"webhook: {_PVE_ENDPOINT_ID}\n" + ) + + if priv_text is not None: + # Start from cleaned priv (our old blocks removed) + if cleaned_priv and not cleaned_priv.endswith('\n'): + cleaned_priv += '\n' + if cleaned_priv and not cleaned_priv.endswith('\n\n'): + cleaned_priv += '\n' + new_priv = cleaned_priv + priv_block + else: + new_priv = priv_block + + try: + with open(_PVE_PRIV_CFG, 'w') as f: + f.write(new_priv) + except PermissionError: + result['error'] = f'Permission denied writing {_PVE_PRIV_CFG}' + result['fallback_commands'] = _build_webhook_fallback() + return result + except Exception: + pass result['configured'] = True result['secret'] = secret - return jsonify(result), 200 + return result except Exception as e: result['error'] = str(e) - result['fallback_commands'] = _build_fallback() - return jsonify(result), 200 + result['fallback_commands'] = _build_webhook_fallback() + return result -@notification_bp.route('/api/notifications/proxmox/cleanup-webhook', methods=['POST']) -def cleanup_proxmox_webhook(): - """Remove ProxMenux webhook blocks from PVE notification config. +@notification_bp.route('/api/notifications/proxmox/setup-webhook', methods=['POST']) +def setup_proxmox_webhook(): + """HTTP endpoint wrapper for webhook setup.""" + return jsonify(setup_pve_webhook_core()), 200 + + +def cleanup_pve_webhook_core() -> dict: + """Core logic to remove PVE webhook blocks. Callable from anywhere. - Called when the notification service is disabled. + Returns dict with 'cleaned', 'error' keys. Only removes blocks named 'proxmenux-webhook' / 'proxmenux-default'. - All other blocks are preserved byte-for-byte. - Creates backups before modification. """ result = {'cleaned': False, 'error': None} @@ -378,7 +411,7 @@ def cleanup_proxmox_webhook(): cfg_text, err = _pve_read_file(_PVE_NOTIFICATIONS_CFG) if err: result['error'] = err - return jsonify(result), 200 + return result priv_text, err = _pve_read_file(_PVE_PRIV_CFG) if err: @@ -392,7 +425,7 @@ def cleanup_proxmox_webhook(): if not has_our_blocks and not has_priv_blocks: result['cleaned'] = True - return jsonify(result), 200 + return result # Backup before modification _pve_backup_file(_PVE_NOTIFICATIONS_CFG) @@ -407,7 +440,7 @@ def cleanup_proxmox_webhook(): f.write(cleaned_cfg) except PermissionError: result['error'] = f'Permission denied writing {_PVE_NOTIFICATIONS_CFG}' - return jsonify(result), 200 + return result except Exception as e: # Rollback try: @@ -416,7 +449,7 @@ def cleanup_proxmox_webhook(): except Exception: pass result['error'] = str(e) - return jsonify(result), 200 + return result if has_priv_blocks and priv_text is not None: cleaned_priv = _pve_remove_our_blocks(priv_text, _PVE_OUR_HEADERS) @@ -427,11 +460,17 @@ def cleanup_proxmox_webhook(): pass # Best-effort result['cleaned'] = True - return jsonify(result), 200 + return result except Exception as e: result['error'] = str(e) - return jsonify(result), 200 + return result + + +@notification_bp.route('/api/notifications/proxmox/cleanup-webhook', methods=['POST']) +def cleanup_proxmox_webhook(): + """HTTP endpoint wrapper for webhook cleanup.""" + return jsonify(cleanup_pve_webhook_core()), 200 @notification_bp.route('/api/notifications/proxmox/read-cfg', methods=['GET']) diff --git a/AppImage/scripts/notification_events.py b/AppImage/scripts/notification_events.py index 129127fe..a829eff7 100644 --- a/AppImage/scripts/notification_events.py +++ b/AppImage/scripts/notification_events.py @@ -935,12 +935,34 @@ class ProxmoxHookWatcher: if not payload: return {'accepted': False, 'error': 'Empty payload'} + # ── Normalise PVE native webhook format ── + # PVE sends: {title, message, severity, timestamp, fields: {type, hostname, job-id}} + # Our code expects: {type, severity, title, body, component, ...} + # Flatten `fields` into the top-level payload so _classify sees them. + if 'fields' in payload and isinstance(payload['fields'], dict): + fields = payload['fields'] + # Map PVE field names to our expected names + if 'type' in fields and 'type' not in payload: + payload['type'] = fields['type'] # vzdump, fencing, replication, etc. + if 'hostname' in fields: + payload['hostname'] = fields['hostname'] + if 'job-id' in fields: + payload['job_id'] = fields['job-id'] + # Merge remaining fields + for k, v in fields.items(): + if k not in payload: + payload[k] = v + + # PVE uses 'message' for the body text + if 'message' in payload and 'body' not in payload: + payload['body'] = payload['message'] + # Extract common fields from PVE notification payload notification_type = payload.get('type', payload.get('notification-type', '')) severity_raw = payload.get('severity', payload.get('priority', 'info')) title = payload.get('title', payload.get('subject', '')) body = payload.get('body', payload.get('message', '')) - source_component = payload.get('component', payload.get('source', '')) + source_component = payload.get('component', payload.get('source', payload.get('type', ''))) # If 'type' is already a known template key, use it directly. # This allows tests and internal callers to inject events by exact type @@ -1030,6 +1052,37 @@ class ProxmoxHookWatcher: if 'updates' in title_lower and ('changed' in title_lower or 'status' in title_lower): return '_skip', '', '' + # ── PVE native notification types ── + # When PVE sends via our webhook body template, fields.type is one of: + # vzdump, fencing, replication, package-updates, system-mail + pve_type = payload.get('type', '').lower() + + if pve_type == 'vzdump': + # Backup notification -- determine success or failure from severity + pve_sev = payload.get('severity', 'info').lower() + vmid = '' + # Try to extract VMID from title like "Backup of VM 100 (qemu)" + import re + m = re.search(r'VM\s+(\d+)|CT\s+(\d+)', title, re.IGNORECASE) + if m: + vmid = m.group(1) or m.group(2) or '' + if pve_sev == 'error': + return 'backup_fail', 'vm', vmid + return 'backup_complete', 'vm', vmid + + if pve_type == 'fencing': + return 'split_brain', 'node', payload.get('hostname', '') + + if pve_type == 'replication': + return 'replication_fail', 'vm', '' + + if pve_type == 'package-updates': + return 'update_available', 'node', '' + + if pve_type == 'system-mail': + # Forwarded system mail (e.g. from smartd) -- treat as system_problem + return 'system_problem', 'node', '' + # VM / CT lifecycle events (if sent via webhook) vmid = str(payload.get('vmid', '')) if any(k in component_lower for k in ('qemu', 'lxc', 'vm', 'ct', 'container')): diff --git a/AppImage/scripts/notification_manager.py b/AppImage/scripts/notification_manager.py index 7cb689fe..681cc0a3 100644 --- a/AppImage/scripts/notification_manager.py +++ b/AppImage/scripts/notification_manager.py @@ -401,6 +401,19 @@ class NotificationManager: self._running = True self._stats['started_at'] = datetime.now().isoformat() + # Ensure PVE webhook is configured (repairs priv config if missing) + try: + from flask_notification_routes import setup_pve_webhook_core + wh_result = setup_pve_webhook_core() + if wh_result.get('configured'): + print("[NotificationManager] PVE webhook configured OK.") + elif wh_result.get('error'): + print(f"[NotificationManager] PVE webhook warning: {wh_result['error']}") + except ImportError: + pass # flask_notification_routes not loaded yet (early startup) + except Exception as e: + print(f"[NotificationManager] PVE webhook setup error: {e}") + # Start event watchers self._journal_watcher = JournalWatcher(self._event_queue) self._task_watcher = TaskWatcher(self._event_queue) @@ -1095,13 +1108,35 @@ class NotificationManager: self._enabled = self._config.get('enabled', 'false') == 'true' self._rebuild_channels() - # Start/stop service if enabled state changed - if self._enabled and not self._running: - self.start() - elif not self._enabled and self._running: - self.stop() + # Start/stop service and auto-configure PVE webhook + pve_webhook_result = None + if self._enabled and not was_enabled: + # Notifications just got ENABLED -> start service + setup PVE webhook + if not self._running: + self.start() + try: + from flask_notification_routes import setup_pve_webhook_core + pve_webhook_result = setup_pve_webhook_core() + except ImportError: + pass # flask_notification_routes not available (CLI mode) + except Exception as e: + pve_webhook_result = {'configured': False, 'error': str(e)} + elif not self._enabled and was_enabled: + # Notifications just got DISABLED -> stop service + cleanup PVE webhook + if self._running: + self.stop() + try: + from flask_notification_routes import cleanup_pve_webhook_core + cleanup_pve_webhook_core() + except ImportError: + pass + except Exception: + pass - return {'success': True, 'channels_active': list(self._channels.keys())} + result = {'success': True, 'channels_active': list(self._channels.keys())} + if pve_webhook_result: + result['pve_webhook'] = pve_webhook_result + return result except Exception as e: return {'success': False, 'error': str(e)}