""" Flask routes for health monitoring with persistence support """ from flask import Blueprint, jsonify, request from health_monitor import health_monitor from health_persistence import health_persistence # Sprint 13: remote-mount monitor (NFS/CIFS/SMB) — separate module so a # missing helper doesn't crash the health blueprint. try: import mount_monitor MOUNT_MONITOR_AVAILABLE = True except ImportError: MOUNT_MONITOR_AVAILABLE = False health_bp = Blueprint('health', __name__) @health_bp.route('/api/health/status', methods=['GET']) def get_health_status(): """Get overall health status summary""" try: status = health_monitor.get_overall_status() return jsonify(status) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/details', methods=['GET']) def get_health_details(): """Get detailed health status with all checks""" try: details = health_monitor.get_detailed_status() return jsonify(details) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/system-info', methods=['GET']) def get_system_info(): """ Get lightweight system info for header display. Returns: hostname, uptime, and health status with proper structure. """ try: info = health_monitor.get_system_info() if 'health' in info: status_map = { 'OK': 'healthy', 'WARNING': 'warning', 'CRITICAL': 'critical', 'UNKNOWN': 'warning' } current_status = info['health'].get('status', 'OK').upper() info['health']['status'] = status_map.get(current_status, 'healthy') return jsonify(info) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/acknowledge', methods=['POST']) def acknowledge_error(): """ Acknowledge/dismiss an error manually. Returns details about the acknowledged error including original severity and suppression period info. Body accepts an optional ``suppression_hours`` field — if omitted the server uses the user-configured value for the error's category (current behavior). When provided, the value overrides the category default for this specific dismiss: - positive integer N → silence for N hours - ``-1`` → silence permanently (only revertible from Settings → Active Suppressions) """ try: data = request.get_json() if not data or 'error_key' not in data: return jsonify({'error': 'error_key is required'}), 400 error_key = data['error_key'] sup_override = None if 'suppression_hours' in data and data['suppression_hours'] is not None: try: sup_override = int(data['suppression_hours']) # Accept positive durations and the permanent sentinel (-1) # only. Zero / other negatives would be nonsensical here. if sup_override < -1 or sup_override == 0: return jsonify({'error': 'suppression_hours must be a positive integer or -1 (permanent)'}), 400 except (ValueError, TypeError): return jsonify({'error': 'suppression_hours must be an integer'}), 400 result = health_persistence.acknowledge_error(error_key, suppression_hours=sup_override) if result.get('success'): # Invalidate cached health results so next fetch reflects the dismiss # Use the error's category to clear the correct cache category = result.get('category', '') cache_key_map = { 'logs': 'logs_analysis', 'pve_services': 'pve_services', 'updates': 'updates_check', 'security': 'security_check', 'temperature': 'cpu_check', 'network': 'network_check', # Both 'disks' (kept for compat) and 'storage' land on the # same cache — the two categories share `storage_check` in # health_monitor. Without the 'storage' entry, dismissing # a storage_unavailable / mount_stale / lxc_mount_low # error persisted but never invalidated the cache, so the # error stayed visible in the next fetch. 'disks': 'storage_check', 'storage': 'storage_check', 'vms': 'vms_check', } cache_key = cache_key_map.get(category) if cache_key: health_monitor.last_check_times.pop(cache_key, None) health_monitor.cached_results.pop(cache_key, None) # Also invalidate ALL background/overall caches so next fetch reflects dismiss for ck in ['_bg_overall', '_bg_detailed', 'overall_health']: health_monitor.last_check_times.pop(ck, None) health_monitor.cached_results.pop(ck, None) # Use the per-record suppression hours from acknowledge_error() sup_hours = result.get('suppression_hours', 24) if sup_hours == -1: suppression_label = 'permanently' elif sup_hours >= 8760: suppression_label = f'{sup_hours // 8760} year(s)' elif sup_hours >= 720: suppression_label = f'{sup_hours // 720} month(s)' elif sup_hours >= 168: suppression_label = f'{sup_hours // 168} week(s)' elif sup_hours >= 72: suppression_label = f'{sup_hours // 24} day(s)' else: suppression_label = f'{sup_hours} hours' return jsonify({ 'success': True, 'message': f'Error dismissed for {suppression_label}', 'error_key': error_key, 'original_severity': result.get('original_severity', 'WARNING'), 'category': category, 'suppression_hours': sup_hours, 'suppression_label': suppression_label, 'acknowledged_at': result.get('acknowledged_at') }) else: return jsonify({ 'success': False, 'message': 'Error not found or already dismissed', 'error_key': error_key }), 404 except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/un-acknowledge', methods=['POST']) def unacknowledge_error(): """ Re-enable a previously dismissed error. Used by Settings → Active Suppressions when the user explicitly removes a suppression (time-limited or permanent). After this call the error becomes eligible to re-emit and re-notify on the next health scan if the underlying condition is still present. Body: ``{"error_key": ""}`` """ try: data = request.get_json() if not data or 'error_key' not in data: return jsonify({'error': 'error_key is required'}), 400 error_key = data['error_key'] result = health_persistence.unacknowledge_error(error_key) # Invalidate caches so the next health fetch reflects the new state # (the alert may re-appear immediately if the condition still holds). category = result.get('category', '') cache_key_map = { 'logs': 'logs_analysis', 'pve_services': 'pve_services', 'updates': 'updates_check', 'security': 'security_check', 'temperature': 'cpu_check', 'network': 'network_check', 'disks': 'storage_check', 'vms': 'vms_check', } cache_key = cache_key_map.get(category) if cache_key: health_monitor.last_check_times.pop(cache_key, None) health_monitor.cached_results.pop(cache_key, None) for ck in ['_bg_overall', '_bg_detailed', 'overall_health']: health_monitor.last_check_times.pop(ck, None) health_monitor.cached_results.pop(ck, None) if not result.get('success'): return jsonify(result), 404 return jsonify(result) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/active-errors', methods=['GET']) def get_active_errors(): """Get all active persistent errors""" try: category = request.args.get('category') errors = health_persistence.get_active_errors(category) return jsonify({'errors': errors}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/dismissed', methods=['GET']) def get_dismissed_errors(): """ Get dismissed errors that are still within their suppression period. These are shown as INFO items with a 'Dismissed' badge in the frontend. """ try: dismissed = health_persistence.get_dismissed_errors() return jsonify({'dismissed': dismissed}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/full', methods=['GET']) def get_full_health(): """ Get complete health data in a single request: detailed status + active errors + dismissed. Uses background-cached results if fresh (< 6 min) for instant response, otherwise runs a fresh check. ?refresh=1 busts the background + per-check caches for updates/services/ security before returning, so an event that just changed underlying state (Update Now finished, dismiss action) sees the new value immediately instead of waiting for the next polling tick. """ import time as _time try: if request.args.get('refresh') == '1': for ck in ('updates_check', 'pve_services', 'security_check', '_bg_detailed', '_bg_overall', 'overall_health'): health_monitor.last_check_times.pop(ck, None) health_monitor.cached_results.pop(ck, None) # Try to use the background-cached detailed result for instant response bg_key = '_bg_detailed' bg_last = health_monitor.last_check_times.get(bg_key, 0) bg_age = _time.time() - bg_last if bg_age < 360 and bg_key in health_monitor.cached_results: # Use cached result (at most ~5 min old) details = health_monitor.cached_results[bg_key] else: # No fresh cache, run live (first load or cache expired) details = health_monitor.get_detailed_status() active_errors = health_persistence.get_active_errors() dismissed = health_persistence.get_dismissed_errors() custom_suppressions = health_persistence.get_custom_suppressions() return jsonify({ 'health': details, 'active_errors': active_errors, 'dismissed': dismissed, 'custom_suppressions': custom_suppressions, 'timestamp': details.get('timestamp') }) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/cleanup-orphans', methods=['POST']) def cleanup_orphan_errors(): """ Clean up errors for devices that no longer exist in the system. Useful when USB drives or temporary devices are disconnected. """ import os import re try: cleaned = [] # Get all active disk errors disk_errors = health_persistence.get_active_errors(category='disks') for err in disk_errors: err_key = err.get('error_key', '') details = err.get('details', {}) if isinstance(details, str): try: import json as _json details = _json.loads(details) except Exception: details = {} device = details.get('device', '') base_disk = details.get('disk', '') # Try to determine the device path dev_path = None if base_disk: dev_path = f'/dev/{base_disk}' elif device: dev_path = device if device.startswith('/dev/') else f'/dev/{device}' elif err_key.startswith('disk_'): # Extract device from error_key dev_name = err_key.replace('disk_fs_', '').replace('disk_', '') dev_name = re.sub(r'_.*$', '', dev_name) # Remove suffix if dev_name: dev_path = f'/dev/{dev_name}' if dev_path: # Also check base disk (remove partition number) base_path = re.sub(r'\d+$', '', dev_path) if not os.path.exists(dev_path) and not os.path.exists(base_path): health_persistence.resolve_error(err_key, 'Device no longer present (manual cleanup)') cleaned.append({'error_key': err_key, 'device': dev_path}) # Also cleanup disk_observations for non-existent devices try: health_persistence.cleanup_orphan_observations() except Exception: pass return jsonify({ 'success': True, 'cleaned_count': len(cleaned), 'cleaned_errors': cleaned }) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/pending-notifications', methods=['GET']) def get_pending_notifications(): """ Get events pending notification (for future Telegram/Gotify/Discord integration). This endpoint will be consumed by the Notification Service (Bloque A). """ try: pending = health_persistence.get_pending_notifications() return jsonify({'pending': pending, 'count': len(pending)}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/mark-notified', methods=['POST']) def mark_events_notified(): """ Mark events as notified after notification was sent successfully. Used by the Notification Service (Bloque A) after sending alerts. """ try: data = request.get_json() if not data or 'event_ids' not in data: return jsonify({'error': 'event_ids array is required'}), 400 event_ids = data['event_ids'] if not isinstance(event_ids, list): return jsonify({'error': 'event_ids must be an array'}), 400 health_persistence.mark_events_notified(event_ids) return jsonify({'success': True, 'marked_count': len(event_ids)}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/settings', methods=['GET']) def get_health_settings(): """ Get per-category suppression duration settings. Returns all health categories with their current configured hours. """ try: categories = health_persistence.get_suppression_categories() return jsonify({'categories': categories}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/settings', methods=['POST']) def save_health_settings(): """ Save per-category suppression duration settings. Expects JSON body with key-value pairs like: {"suppress_cpu": "168", "suppress_memory": "-1"} Valid values: 24, 72, 168, 720, 8760, -1 (permanent), or any positive integer for custom. """ try: data = request.get_json() if not data: return jsonify({'error': 'No settings provided'}), 400 valid_keys = set(health_persistence.CATEGORY_SETTING_MAP.values()) updated = [] for key, value in data.items(): if key not in valid_keys: continue try: hours = int(value) # Validate: must be -1 (permanent) or positive if hours != -1 and hours < 1: continue health_persistence.set_setting(key, str(hours)) updated.append(key) except (ValueError, TypeError): continue # Retroactively sync all existing dismissed errors # so changes are effective immediately, not just on next dismiss synced_count = health_persistence.sync_dismissed_suppression() return jsonify({ 'success': True, 'updated': updated, 'count': len(updated), 'synced_dismissed': synced_count }) except Exception as e: return jsonify({'error': str(e)}), 500 # ── Remote Storage Exclusions Endpoints ── @health_bp.route('/api/health/remote-storages', methods=['GET']) def get_remote_storages(): """ Get list of all remote storages with their exclusion status. Remote storages are those that can be offline (PBS, NFS, CIFS, etc.) """ try: from proxmox_storage_monitor import proxmox_storage_monitor # Get current storage status storage_status = proxmox_storage_monitor.get_storage_status() all_storages = storage_status.get('available', []) + storage_status.get('unavailable', []) # Filter to only remote types remote_types = health_persistence.REMOTE_STORAGE_TYPES remote_storages = [s for s in all_storages if s.get('type', '').lower() in remote_types] # Get current exclusions exclusions = {e['storage_name']: e for e in health_persistence.get_excluded_storages()} # Combine info result = [] for storage in remote_storages: name = storage.get('name', '') exclusion = exclusions.get(name, {}) result.append({ 'name': name, 'type': storage.get('type', 'unknown'), 'status': storage.get('status', 'unknown'), 'total': storage.get('total', 0), 'used': storage.get('used', 0), 'available': storage.get('available', 0), 'percent': storage.get('percent', 0), 'exclude_health': exclusion.get('exclude_health', 0) == 1, 'exclude_notifications': exclusion.get('exclude_notifications', 0) == 1, 'excluded_at': exclusion.get('excluded_at'), 'reason': exclusion.get('reason') }) return jsonify({ 'storages': result, 'remote_types': list(remote_types) }) except ImportError: return jsonify({'error': 'Storage monitor not available', 'storages': []}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/storage-exclusions', methods=['GET']) def get_storage_exclusions(): """Get all storage exclusions.""" try: exclusions = health_persistence.get_excluded_storages() return jsonify({'exclusions': exclusions}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/storage-exclusions', methods=['POST']) def save_storage_exclusion(): """ Add or update a storage exclusion. Request body: { "storage_name": "pbs-backup", "storage_type": "pbs", "exclude_health": true, "exclude_notifications": true, "reason": "PBS server is offline daily" } """ try: data = request.get_json() if not data or 'storage_name' not in data: return jsonify({'error': 'storage_name is required'}), 400 storage_name = data['storage_name'] storage_type = data.get('storage_type', 'unknown') exclude_health = data.get('exclude_health', True) exclude_notifications = data.get('exclude_notifications', True) reason = data.get('reason') # Check if already excluded existing = health_persistence.get_excluded_storages() exists = any(e['storage_name'] == storage_name for e in existing) if exists: # Update existing success = health_persistence.update_storage_exclusion( storage_name, exclude_health, exclude_notifications ) else: # Add new success = health_persistence.exclude_storage( storage_name, storage_type, exclude_health, exclude_notifications, reason ) if success: return jsonify({ 'success': True, 'message': f'Storage {storage_name} exclusion saved', 'storage_name': storage_name }) else: return jsonify({'error': 'Failed to save exclusion'}), 500 except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/storage-exclusions/', methods=['DELETE']) def delete_storage_exclusion(storage_name): """Remove a storage from the exclusion list.""" try: success = health_persistence.remove_storage_exclusion(storage_name) if success: return jsonify({ 'success': True, 'message': f'Storage {storage_name} removed from exclusions' }) else: return jsonify({'error': 'Storage not found in exclusions'}), 404 except Exception as e: return jsonify({'error': str(e)}), 500 # ═══════════════════════════════════════════════════════════════════════════ # NETWORK INTERFACE EXCLUSION ROUTES # ═══════════════════════════════════════════════════════════════════════════ @health_bp.route('/api/health/interfaces', methods=['GET']) def get_network_interfaces(): """Get all network interfaces with their exclusion status.""" try: import psutil # Get all interfaces net_if_stats = psutil.net_if_stats() net_if_addrs = psutil.net_if_addrs() # Get current exclusions exclusions = {e['interface_name']: e for e in health_persistence.get_excluded_interfaces()} result = [] for iface, stats in net_if_stats.items(): if iface == 'lo': continue # Determine interface type if iface.startswith('vmbr'): iface_type = 'bridge' elif iface.startswith('bond'): iface_type = 'bond' elif iface.startswith(('vlan', 'veth')): iface_type = 'vlan' elif iface.startswith(('eth', 'ens', 'enp', 'eno')): iface_type = 'physical' else: iface_type = 'other' # Get IP address if any ip_addr = None if iface in net_if_addrs: for addr in net_if_addrs[iface]: if addr.family == 2: # IPv4 ip_addr = addr.address break exclusion = exclusions.get(iface, {}) result.append({ 'name': iface, 'type': iface_type, 'is_up': stats.isup, 'speed': stats.speed, 'ip_address': ip_addr, 'exclude_health': exclusion.get('exclude_health', 0) == 1, 'exclude_notifications': exclusion.get('exclude_notifications', 0) == 1, 'excluded_at': exclusion.get('excluded_at'), 'reason': exclusion.get('reason') }) # Sort: bridges first, then physical, then others type_order = {'bridge': 0, 'bond': 1, 'physical': 2, 'vlan': 3, 'other': 4} result.sort(key=lambda x: (type_order.get(x['type'], 5), x['name'])) return jsonify({'interfaces': result}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/interface-exclusions', methods=['GET']) def get_interface_exclusions(): """Get all interface exclusions.""" try: exclusions = health_persistence.get_excluded_interfaces() return jsonify({'exclusions': exclusions}) except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/interface-exclusions', methods=['POST']) def save_interface_exclusion(): """ Add or update an interface exclusion. Request body: { "interface_name": "vmbr0", "interface_type": "bridge", "exclude_health": true, "exclude_notifications": true, "reason": "Intentionally disabled bridge" } """ try: data = request.get_json() if not data or 'interface_name' not in data: return jsonify({'error': 'interface_name is required'}), 400 interface_name = data['interface_name'] interface_type = data.get('interface_type', 'unknown') exclude_health = data.get('exclude_health', True) exclude_notifications = data.get('exclude_notifications', True) reason = data.get('reason') # Check if already excluded existing = health_persistence.get_excluded_interfaces() exists = any(e['interface_name'] == interface_name for e in existing) if exists: # Update existing success = health_persistence.update_interface_exclusion( interface_name, exclude_health, exclude_notifications ) else: # Add new success = health_persistence.exclude_interface( interface_name, interface_type, exclude_health, exclude_notifications, reason ) if success: return jsonify({ 'success': True, 'message': f'Interface {interface_name} exclusion saved', 'interface_name': interface_name }) else: return jsonify({'error': 'Failed to save exclusion'}), 500 except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/health/interface-exclusions/', methods=['DELETE']) def delete_interface_exclusion(interface_name): """Remove an interface from the exclusion list.""" try: success = health_persistence.remove_interface_exclusion(interface_name) if success: return jsonify({ 'success': True, 'message': f'Interface {interface_name} removed from exclusions' }) else: return jsonify({'error': 'Interface not found in exclusions'}), 404 except Exception as e: return jsonify({'error': str(e)}), 500 @health_bp.route('/api/mounts', methods=['GET']) def get_remote_mounts(): """Sprint 13: list NFS/CIFS/SMB mounts on the host AND inside every running LXC, with per-mount health (reachable / stale / read-only). Returns: ``mounts`` — host-level remote mounts (Sprint 13.11) ``lxc_mounts`` — mounts inside running LXCs (Sprint 13.24) Both lists share the same per-row shape; LXC entries add three extra fields (lxc_id, lxc_name, lxc_pid). The frontend renders them in two separate cards so the user immediately knows whether the mount lives on the host or inside a container. """ if not MOUNT_MONITOR_AVAILABLE: return jsonify({ 'mounts': [], 'lxc_mounts': [], 'available': False, }) try: mounts = mount_monitor.scan_remote_mounts() # LXC scan is wrapped separately so a flaky `pct exec` doesn't # blank the host list. The host scan is cheap and reliable; # LXC scan can hit timeouts on stuck containers. try: lxc_mounts = mount_monitor.scan_lxc_mounts() except Exception as lxc_err: print(f"[flask_health_routes] LXC mount scan failed: {lxc_err}") lxc_mounts = [] return jsonify({ 'mounts': mounts, 'lxc_mounts': lxc_mounts, 'available': True, }) except Exception as e: return jsonify({ 'mounts': [], 'lxc_mounts': [], 'available': True, 'error': str(e), }), 500