Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-02-20 01:16:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update health monitor

Browse Source
This commit is contained in:
MacRimi
2026-02-17 11:35:11 +01:00
parent 8004ee48c9
commit 31c5eeb6c3
5 changed files with 659 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
AppImage/scripts/flask_health_routes.py
View File

@@ -89,14 +89,20 @@ def acknowledge_error():
health_monitor.last_check_times.pop('overall_health', None)
health_monitor.cached_results.pop('overall_health', None)
# Determine suppression period for the response
category = result.get('category', '')
if category == 'updates':
suppression_hours = 180 * 24 # 180 days in hours
suppression_label = '6 months'
# Use the per-record suppression hours from acknowledge_error()
sup_hours = result.get('suppression_hours', 24)
if sup_hours == -1:
suppression_label = 'permanently'
elif sup_hours >= 8760:
suppression_label = f'{sup_hours // 8760} year(s)'
elif sup_hours >= 720:
suppression_label = f'{sup_hours // 720} month(s)'
elif sup_hours >= 168:
suppression_label = f'{sup_hours // 168} week(s)'
elif sup_hours >= 72:
suppression_label = f'{sup_hours // 24} day(s)'
else:
suppression_hours = 24
suppression_label = '24 hours'
suppression_label = f'{sup_hours} hours'
return jsonify({
'success': True,
@@ -104,7 +110,7 @@ def acknowledge_error():
'error_key': error_key,
'original_severity': result.get('original_severity', 'WARNING'),
'category': category,
'suppression_hours': suppression_hours,
'suppression_hours': sup_hours,
'suppression_label': suppression_label,
'acknowledged_at': result.get('acknowledged_at')
})
@@ -190,3 +196,54 @@ def mark_events_notified():
return jsonify({'success': True, 'marked_count': len(event_ids)})
except Exception as e:
return jsonify({'error': str(e)}), 500
@health_bp.route('/api/health/settings', methods=['GET'])
def get_health_settings():
"""
Get per-category suppression duration settings.
Returns all health categories with their current configured hours.
"""
try:
categories = health_persistence.get_suppression_categories()
return jsonify({'categories': categories})
except Exception as e:
return jsonify({'error': str(e)}), 500
@health_bp.route('/api/health/settings', methods=['POST'])
def save_health_settings():
"""
Save per-category suppression duration settings.
Expects JSON body with key-value pairs like: {"suppress_cpu": "168", "suppress_memory": "-1"}
Valid values: 24, 72, 168, 720, 8760, -1 (permanent), or any positive integer for custom.
"""
try:
data = request.get_json()
if not data:
return jsonify({'error': 'No settings provided'}), 400
valid_keys = set(health_persistence.CATEGORY_SETTING_MAP.values())
updated = []
for key, value in data.items():
if key not in valid_keys:
continue
try:
hours = int(value)
# Validate: must be -1 (permanent) or positive
if hours != -1 and hours < 1:
continue
health_persistence.set_setting(key, str(hours))
updated.append(key)
except (ValueError, TypeError):
continue
return jsonify({
'success': True,
'updated': updated,
'count': len(updated)
})
except Exception as e:
return jsonify({'error': str(e)}), 500

33
AppImage/scripts/health_monitor.py
View File

@@ -2258,10 +2258,9 @@ class HealthMonitor:
try:
issues = []
checks = {
'uptime': {'status': 'OK', 'detail': ''},
'certificates': {'status': 'OK', 'detail': ''},
'login_attempts': {'status': 'OK', 'detail': ''},
'fail2ban': {'status': 'OK', 'detail': 'Not installed'}
'uptime': {'status': 'OK', 'detail': ''},
'certificates': {'status': 'OK', 'detail': ''},
'login_attempts': {'status': 'OK', 'detail': ''},
}
# Sub-check 1: Uptime for potential kernel vulnerabilities
@@ -2322,21 +2321,23 @@ class HealthMonitor:
except Exception:
checks['login_attempts'] = {'status': 'OK', 'detail': 'Unable to check login attempts'}
# Sub-check 4: Fail2Ban ban detection
# Sub-check 4: Fail2Ban ban detection (only show if installed)
try:
f2b = self._check_fail2ban_bans()
f2b_status = f2b.get('status', 'OK')
checks['fail2ban'] = {
'status': f2b_status,
'dismissable': True if f2b_status not in ['OK'] else False,
'detail': f2b.get('detail', ''),
'installed': f2b.get('installed', False),
'banned_count': f2b.get('banned_count', 0)
}
if f2b.get('status') == 'WARNING':
issues.append(f2b.get('detail', 'Fail2Ban bans detected'))
if f2b.get('installed', False):
f2b_status = f2b.get('status', 'OK')
checks['fail2ban'] = {
'status': f2b_status,
'dismissable': True if f2b_status not in ['OK'] else False,
'detail': f2b.get('detail', ''),
'installed': True,
'banned_count': f2b.get('banned_count', 0)
}
if f2b.get('status') == 'WARNING':
issues.append(f2b.get('detail', 'Fail2Ban bans detected'))
# If not installed, simply don't add it to checks
except Exception:
checks['fail2ban'] = {'status': 'OK', 'detail': 'Unable to check Fail2Ban'}
pass
# Determine overall security status
if issues:

298
AppImage/scripts/health_persistence.py
View File

@@ -28,7 +28,23 @@ class HealthPersistence:
VM_ERROR_RETENTION = 48 * 3600 # 48 hours
LOG_ERROR_RETENTION = 24 * 3600 # 24 hours
DISK_ERROR_RETENTION = 48 * 3600 # 48 hours
UPDATES_SUPPRESSION = 180 * 24 * 3600 # 180 days (6 months)
# Default suppression: 24 hours (user can change per-category in settings)
DEFAULT_SUPPRESSION_HOURS = 24
# Mapping from error categories to settings keys
CATEGORY_SETTING_MAP = {
'temperature': 'suppress_cpu',
'memory': 'suppress_memory',
'storage': 'suppress_storage',
'disks': 'suppress_disks',
'network': 'suppress_network',
'vms': 'suppress_vms',
'pve_services': 'suppress_pve_services',
'logs': 'suppress_logs',
'updates': 'suppress_updates',
'security': 'suppress_security',
}
def __init__(self):
"""Initialize persistence with database in shared ProxMenux data directory"""
@@ -80,6 +96,21 @@ class HealthPersistence:
)
''')
# User settings table (per-category suppression durations, etc.)
cursor.execute('''
CREATE TABLE IF NOT EXISTS user_settings (
setting_key TEXT PRIMARY KEY,
setting_value TEXT NOT NULL,
updated_at TEXT NOT NULL
)
''')
# Migration: add suppression_hours column to errors if not present
cursor.execute("PRAGMA table_info(errors)")
columns = [col[1] for col in cursor.fetchall()]
if 'suppression_hours' not in columns:
cursor.execute('ALTER TABLE errors ADD COLUMN suppression_hours INTEGER DEFAULT 24')
# Indexes for performance
cursor.execute('CREATE INDEX IF NOT EXISTS idx_error_key ON errors(error_key)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_category ON errors(category)')
@@ -102,33 +133,8 @@ class HealthPersistence:
details_json = json.dumps(details) if details else None
cursor.execute('''
SELECT acknowledged, resolved_at
FROM errors
WHERE error_key = ? AND acknowledged = 1
''', (error_key,))
ack_check = cursor.fetchone()
if ack_check and ack_check[1]: # Has resolved_at timestamp
try:
resolved_dt = datetime.fromisoformat(ack_check[1])
hours_since_ack = (datetime.now() - resolved_dt).total_seconds() / 3600
if category == 'updates':
# Updates: suppress for 180 days (6 months)
suppression_hours = self.UPDATES_SUPPRESSION / 3600
else:
# Other errors: suppress for 24 hours
suppression_hours = 24
if hours_since_ack < suppression_hours:
# Skip re-adding recently acknowledged errors
conn.close()
return {'type': 'skipped_acknowledged', 'needs_notification': False}
except Exception:
pass
cursor.execute('''
SELECT id, first_seen, notification_sent, acknowledged, resolved_at
SELECT id, acknowledged, resolved_at, category, severity, first_seen,
notification_sent, suppression_hours
FROM errors WHERE error_key = ?
''', (error_key,))
existing = cursor.fetchone()
@@ -136,13 +142,64 @@ class HealthPersistence:
event_info = {'type': 'updated', 'needs_notification': False}
if existing:
error_id, first_seen, notif_sent, acknowledged, resolved_at = existing
err_id, ack, resolved_at, old_cat, old_severity, first_seen, notif_sent, stored_suppression = existing
if acknowledged == 1:
conn.close()
return {'type': 'skipped_acknowledged', 'needs_notification': False}
if ack == 1:
# SAFETY OVERRIDE: Critical CPU temperature ALWAYS re-triggers
# regardless of any dismiss/permanent setting (hardware protection)
if error_key == 'cpu_temperature' and severity == 'CRITICAL':
cursor.execute('DELETE FROM errors WHERE error_key = ?', (error_key,))
cursor.execute('''
INSERT INTO errors
(error_key, category, severity, reason, details, first_seen, last_seen)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (error_key, category, severity, reason, details_json, now, now))
event_info = {'type': 'new', 'needs_notification': True}
self._record_event(cursor, 'new', error_key,
{'severity': severity, 'reason': reason,
'note': 'CRITICAL temperature override - safety alert'})
conn.commit()
conn.close()
return event_info
# Check suppression: use per-record stored hours (set at dismiss time)
sup_hours = stored_suppression if stored_suppression is not None else self.DEFAULT_SUPPRESSION_HOURS
# Permanent dismiss (sup_hours == -1): always suppress
if sup_hours == -1:
conn.close()
return {'type': 'skipped_acknowledged', 'needs_notification': False}
# Time-limited suppression
still_suppressed = False
if resolved_at:
try:
resolved_dt = datetime.fromisoformat(resolved_at)
elapsed_hours = (datetime.now() - resolved_dt).total_seconds() / 3600
still_suppressed = elapsed_hours < sup_hours
except Exception:
pass
if still_suppressed:
conn.close()
return {'type': 'skipped_acknowledged', 'needs_notification': False}
else:
# Suppression expired - reset as a NEW event
cursor.execute('DELETE FROM errors WHERE error_key = ?', (error_key,))
cursor.execute('''
INSERT INTO errors
(error_key, category, severity, reason, details, first_seen, last_seen)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (error_key, category, severity, reason, details_json, now, now))
event_info = {'type': 'new', 'needs_notification': True}
self._record_event(cursor, 'new', error_key,
{'severity': severity, 'reason': reason,
'note': 'Re-triggered after suppression expired'})
conn.commit()
conn.close()
return event_info
# Update existing error (only if NOT acknowledged)
# Not acknowledged - update existing active error
cursor.execute('''
UPDATE errors
SET last_seen = ?, severity = ?, reason = ?, details = ?
@@ -150,13 +207,9 @@ class HealthPersistence:
''', (now, severity, reason, details_json, error_key))
# Check if severity escalated
cursor.execute('SELECT severity FROM errors WHERE error_key = ?', (error_key,))
old_severity_row = cursor.fetchone()
if old_severity_row:
old_severity = old_severity_row[0]
if old_severity == 'WARNING' and severity == 'CRITICAL':
event_info['type'] = 'escalated'
event_info['needs_notification'] = True
if old_severity == 'WARNING' and severity == 'CRITICAL':
event_info['type'] = 'escalated'
event_info['needs_notification'] = True
else:
# Insert new error
cursor.execute('''
@@ -225,21 +278,40 @@ class HealthPersistence:
"""
Remove/resolve a specific error immediately.
Used when the condition that caused the error no longer exists
(e.g., storage became available again).
(e.g., storage became available again, CPU temp recovered).
For acknowledged errors: if the condition resolved on its own,
we delete the record entirely so it can re-trigger as a fresh
event if the condition returns later.
"""
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
now = datetime.now().isoformat()
# Check if this error was acknowledged (dismissed)
cursor.execute('''
UPDATE errors
SET resolved_at = ?
WHERE error_key = ? AND resolved_at IS NULL
''', (now, error_key))
SELECT acknowledged FROM errors WHERE error_key = ?
''', (error_key,))
row = cursor.fetchone()
if cursor.rowcount > 0:
self._record_event(cursor, 'cleared', error_key, {'reason': 'condition_resolved'})
if row and row[0] == 1:
# Dismissed error that naturally resolved - delete entirely
# so it can re-trigger as a new event if it happens again
cursor.execute('DELETE FROM errors WHERE error_key = ?', (error_key,))
if cursor.rowcount > 0:
self._record_event(cursor, 'cleared', error_key,
{'reason': 'condition_resolved_after_dismiss'})
else:
# Normal active error - mark as resolved
cursor.execute('''
UPDATE errors
SET resolved_at = ?
WHERE error_key = ? AND resolved_at IS NULL
''', (now, error_key))
if cursor.rowcount > 0:
self._record_event(cursor, 'cleared', error_key, {'reason': 'condition_resolved'})
conn.commit()
conn.close()
@@ -247,13 +319,9 @@ class HealthPersistence:
def acknowledge_error(self, error_key: str) -> Dict[str, Any]:
"""
Manually acknowledge an error (dismiss).
- Looks up the category's configured suppression duration from user settings
- Stores suppression_hours on the error record (snapshot at dismiss time)
- Marks as acknowledged so it won't re-appear during the suppression period
- Stores the original severity for reference
- Returns info about the acknowledged error
Suppression periods:
- updates category: 180 days (6 months)
- other categories: 24 hours
"""
conn = sqlite3.connect(str(self.db_path))
conn.row_factory = sqlite3.Row
@@ -272,15 +340,27 @@ class HealthPersistence:
original_severity = error_dict.get('severity', 'WARNING')
category = error_dict.get('category', '')
# Look up the user's configured suppression for this category
setting_key = self.CATEGORY_SETTING_MAP.get(category, '')
sup_hours = self.DEFAULT_SUPPRESSION_HOURS
if setting_key:
stored = self.get_setting(setting_key)
if stored is not None:
try:
sup_hours = int(stored)
except (ValueError, TypeError):
pass
cursor.execute('''
UPDATE errors
SET acknowledged = 1, resolved_at = ?
SET acknowledged = 1, resolved_at = ?, suppression_hours = ?
WHERE error_key = ?
''', (now, error_key))
''', (now, sup_hours, error_key))
self._record_event(cursor, 'acknowledged', error_key, {
'original_severity': original_severity,
'category': category
'category': category,
'suppression_hours': sup_hours
})
result = {
@@ -288,7 +368,8 @@ class HealthPersistence:
'error_key': error_key,
'original_severity': original_severity,
'category': category,
'acknowledged_at': now
'acknowledged_at': now,
'suppression_hours': sup_hours
}
conn.commit()
@@ -432,22 +513,30 @@ class HealthPersistence:
except (json.JSONDecodeError, TypeError):
pass
# Check if still within suppression period
# Check if still within suppression period using per-record hours
try:
resolved_dt = datetime.fromisoformat(error_dict['resolved_at'])
elapsed_seconds = (now - resolved_dt).total_seconds()
sup_hours = error_dict.get('suppression_hours')
if sup_hours is None:
sup_hours = self.DEFAULT_SUPPRESSION_HOURS
if error_dict.get('category') == 'updates':
suppression = self.UPDATES_SUPPRESSION
else:
suppression = 24 * 3600 # 24 hours
error_dict['dismissed'] = True
if elapsed_seconds < suppression:
error_dict['dismissed'] = True
error_dict['suppression_remaining_hours'] = round(
(suppression - elapsed_seconds) / 3600, 1
)
if sup_hours == -1:
# Permanent dismiss
error_dict['suppression_remaining_hours'] = -1
error_dict['permanent'] = True
dismissed.append(error_dict)
else:
elapsed_seconds = (now - resolved_dt).total_seconds()
suppression_seconds = sup_hours * 3600
if elapsed_seconds < suppression_seconds:
error_dict['suppression_remaining_hours'] = round(
(suppression_seconds - elapsed_seconds) / 3600, 1
)
error_dict['permanent'] = False
dismissed.append(error_dict)
except (ValueError, TypeError):
pass
@@ -623,6 +712,79 @@ class HealthPersistence:
# from Proxmox storage types in health_monitor.get_detailed_status()
# This avoids redundant subprocess calls and ensures immediate detection
# when the user adds new ZFS/LVM storage via Proxmox.
# ─── User Settings ──────────────────────────────────────────
def get_setting(self, key: str, default: Optional[str] = None) -> Optional[str]:
"""Get a user setting value by key."""
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute(
'SELECT setting_value FROM user_settings WHERE setting_key = ?', (key,)
)
row = cursor.fetchone()
conn.close()
return row[0] if row else default
def set_setting(self, key: str, value: str):
"""Store a user setting value."""
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
cursor.execute('''
INSERT OR REPLACE INTO user_settings (setting_key, setting_value, updated_at)
VALUES (?, ?, ?)
''', (key, value, datetime.now().isoformat()))
conn.commit()
conn.close()
def get_all_settings(self, prefix: Optional[str] = None) -> Dict[str, str]:
"""Get all user settings, optionally filtered by key prefix."""
conn = sqlite3.connect(str(self.db_path))
cursor = conn.cursor()
if prefix:
cursor.execute(
'SELECT setting_key, setting_value FROM user_settings WHERE setting_key LIKE ?',
(f'{prefix}%',)
)
else:
cursor.execute('SELECT setting_key, setting_value FROM user_settings')
rows = cursor.fetchall()
conn.close()
return {row[0]: row[1] for row in rows}
def get_suppression_categories(self) -> List[Dict[str, Any]]:
"""
Get all health categories with their current suppression settings.
Used by the settings page to render the per-category configuration.
"""
category_labels = {
'suppress_cpu': {'label': 'CPU Usage & Temperature', 'category': 'temperature', 'icon': 'cpu'},
'suppress_memory': {'label': 'Memory & Swap', 'category': 'memory', 'icon': 'memory'},
'suppress_storage': {'label': 'Storage Mounts & Space', 'category': 'storage', 'icon': 'storage'},
'suppress_disks': {'label': 'Disk I/O & Errors', 'category': 'disks', 'icon': 'disk'},
'suppress_network': {'label': 'Network Interfaces', 'category': 'network', 'icon': 'network'},
'suppress_vms': {'label': 'VMs & Containers', 'category': 'vms', 'icon': 'vms'},
'suppress_pve_services': {'label': 'PVE Services', 'category': 'pve_services', 'icon': 'services'},
'suppress_logs': {'label': 'System Logs', 'category': 'logs', 'icon': 'logs'},
'suppress_updates': {'label': 'System Updates', 'category': 'updates', 'icon': 'updates'},
'suppress_security': {'label': 'Security & Certificates', 'category': 'security', 'icon': 'security'},
}
current_settings = self.get_all_settings('suppress_')
result = []
for key, meta in category_labels.items():
stored = current_settings.get(key)
hours = int(stored) if stored else self.DEFAULT_SUPPRESSION_HOURS
result.append({
'key': key,
'label': meta['label'],
'category': meta['category'],
'icon': meta['icon'],
'hours': hours,
})
return result
# Global instance