Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-05-01 20:06:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update health_persistence.py

Browse Source
This commit is contained in:
MacRimi
2026-04-16 19:18:42 +02:00
parent 1ef4bc4fed
commit 6660122e69
1 changed files with 251 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
AppImage/scripts/health_persistence.py
View File

@@ -716,12 +716,11 @@ class HealthPersistence:
if 'disk' in error_key or 'smart' in error_key or 'sda' in error_key or 'sdb' in error_key or 'nvme' in error_key: if 'disk' in error_key or 'smart' in error_key or 'sda' in error_key or 'sdb' in error_key or 'nvme' in error_key:
category = 'disks' category = 'disks'
else: else:
category = 'general' # Use 'general' as ultimate fallback instead of empty string category = 'general'
setting_key = self.CATEGORY_SETTING_MAP.get(category, '') setting_key = self.CATEGORY_SETTING_MAP.get(category, '')
sup_hours = self.DEFAULT_SUPPRESSION_HOURS sup_hours = self.DEFAULT_SUPPRESSION_HOURS
if setting_key: if setting_key:
# P4 fix: use _get_setting_impl with existing connection
stored = self._get_setting_impl(conn, setting_key) stored = self._get_setting_impl(conn, setting_key)
if stored is not None: if stored is not None:
try: try:
@@ -762,7 +761,6 @@ class HealthPersistence:
setting_key = self.CATEGORY_SETTING_MAP.get(category, '') setting_key = self.CATEGORY_SETTING_MAP.get(category, '')
sup_hours = self.DEFAULT_SUPPRESSION_HOURS sup_hours = self.DEFAULT_SUPPRESSION_HOURS
if setting_key: if setting_key:
# P4 fix: use _get_setting_impl with existing connection
stored = self._get_setting_impl(conn, setting_key) stored = self._get_setting_impl(conn, setting_key)
if stored is not None: if stored is not None:
try: try:
@@ -771,8 +769,6 @@ class HealthPersistence:
pass pass
# Mark as acknowledged but DO NOT set resolved_at # Mark as acknowledged but DO NOT set resolved_at
# The error remains active until it actually disappears from the system
# resolved_at should only be set when the error is truly resolved
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors
SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ? SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ?
@@ -786,18 +782,11 @@ class HealthPersistence:
}) })
# Cascade acknowledge: when dismissing a group check # Cascade acknowledge: when dismissing a group check
# (e.g. log_persistent_errors), also dismiss all individual
# sub-errors that share the same prefix in the DB.
# Currently only persistent errors have per-pattern sub-records
# (e.g. log_persistent_a1b2c3d4).
CASCADE_PREFIXES = { CASCADE_PREFIXES = {
'log_persistent_errors': 'log_persistent_', 'log_persistent_errors': 'log_persistent_',
} }
child_prefix = CASCADE_PREFIXES.get(error_key) child_prefix = CASCADE_PREFIXES.get(error_key)
if child_prefix: if child_prefix:
# Only cascade to active (unresolved) child errors.
# Already-resolved/expired entries must NOT be re-surfaced.
# Mark as acknowledged but DO NOT set resolved_at
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors
SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ? SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ?
@@ -939,9 +928,6 @@ class HealthPersistence:
cursor.execute('DELETE FROM errors WHERE resolved_at < ?', (cutoff_resolved,)) cursor.execute('DELETE FROM errors WHERE resolved_at < ?', (cutoff_resolved,))
# ── Auto-resolve stale errors using Suppression Duration settings ── # ── Auto-resolve stale errors using Suppression Duration settings ──
# Read per-category suppression hours from user_settings.
# If the user hasn't configured a value, use DEFAULT_SUPPRESSION_HOURS.
# This is the SINGLE source of truth for auto-resolution timing.
user_settings = {} user_settings = {}
try: try:
cursor.execute( cursor.execute(
@@ -960,7 +946,6 @@ class HealthPersistence:
except (ValueError, TypeError): except (ValueError, TypeError):
hours = self.DEFAULT_SUPPRESSION_HOURS hours = self.DEFAULT_SUPPRESSION_HOURS
# -1 means permanently suppressed -- skip auto-resolve
if hours < 0: if hours < 0:
continue continue
@@ -975,7 +960,6 @@ class HealthPersistence:
''', (now_iso, category, cutoff)) ''', (now_iso, category, cutoff))
# Catch-all: auto-resolve any error from an unmapped category # Catch-all: auto-resolve any error from an unmapped category
# whose last_seen exceeds DEFAULT_SUPPRESSION_HOURS.
fallback_cutoff = (now - timedelta(hours=self.DEFAULT_SUPPRESSION_HOURS)).isoformat() fallback_cutoff = (now - timedelta(hours=self.DEFAULT_SUPPRESSION_HOURS)).isoformat()
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors
@@ -989,138 +973,105 @@ class HealthPersistence:
cutoff_events = (now - timedelta(days=30)).isoformat() cutoff_events = (now - timedelta(days=30)).isoformat()
cursor.execute('DELETE FROM events WHERE timestamp < ?', (cutoff_events,)) cursor.execute('DELETE FROM events WHERE timestamp < ?', (cutoff_events,))
# ══════════════════════════════════════════════════════════════════════ # ── SMART AUTO-RESOLVE: Based on system state ──
# SMART AUTO-RESOLVE: Based on system state, not hardcoded patterns
# ══════════════════════════════════════════════════════════════════════
# Logic: If an error hasn't been seen recently AND the system is healthy,
# the error is stale and should be auto-resolved.
# This works for ANY error pattern, not just predefined ones.
try: try:
import psutil import psutil
# Get system uptime
with open('/proc/uptime', 'r') as f: with open('/proc/uptime', 'r') as f:
uptime_seconds = float(f.read().split()[0]) uptime_seconds = float(f.read().split()[0])
# Only auto-resolve if system has been stable for at least 10 minutes if uptime_seconds > 600:
if uptime_seconds > 600: # 10 minutes
current_cpu = psutil.cpu_percent(interval=0.1) current_cpu = psutil.cpu_percent(interval=0.1)
current_mem = psutil.virtual_memory().percent current_mem = psutil.virtual_memory().percent
# ── 1. LOGS category: Auto-resolve if not seen in 15 minutes ── # 1. LOGS: Auto-resolve if not seen in 15 minutes
# Log errors are transient - if journalctl hasn't reported them recently,
# they are from a previous state and should be resolved.
stale_logs_cutoff = (now - timedelta(minutes=15)).isoformat() stale_logs_cutoff = (now - timedelta(minutes=15)).isoformat()
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors SET resolved_at = ?
SET resolved_at = ? WHERE category = 'logs' AND resolved_at IS NULL
WHERE category = 'logs' AND acknowledged = 0 AND last_seen < ?
AND resolved_at IS NULL
AND acknowledged = 0
AND last_seen < ?
''', (now_iso, stale_logs_cutoff)) ''', (now_iso, stale_logs_cutoff))
# ── 2. CPU category: Auto-resolve if CPU is normal (<75%) ── # 2. CPU: Auto-resolve if CPU is normal (<75%)
if current_cpu < 75: if current_cpu < 75:
stale_cpu_cutoff = (now - timedelta(minutes=5)).isoformat() stale_cpu_cutoff = (now - timedelta(minutes=5)).isoformat()
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors SET resolved_at = ?
SET resolved_at = ?
WHERE (category = 'cpu' OR category = 'temperature') WHERE (category = 'cpu' OR category = 'temperature')
AND resolved_at IS NULL AND resolved_at IS NULL AND acknowledged = 0
AND acknowledged = 0
AND last_seen < ? AND last_seen < ?
AND (error_key LIKE 'cpu_%' OR reason LIKE '%CPU%') AND (error_key LIKE 'cpu_%' OR reason LIKE '%CPU%')
''', (now_iso, stale_cpu_cutoff)) ''', (now_iso, stale_cpu_cutoff))
# ── 3. MEMORY category: Auto-resolve if memory is normal (<80%) ── # 3. MEMORY: Auto-resolve if memory is normal (<80%)
if current_mem < 80: if current_mem < 80:
stale_mem_cutoff = (now - timedelta(minutes=5)).isoformat() stale_mem_cutoff = (now - timedelta(minutes=5)).isoformat()
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors SET resolved_at = ?
SET resolved_at = ?
WHERE (category = 'memory' OR category = 'logs') WHERE (category = 'memory' OR category = 'logs')
AND resolved_at IS NULL AND resolved_at IS NULL AND acknowledged = 0
AND acknowledged = 0
AND last_seen < ? AND last_seen < ?
AND (error_key LIKE '%oom%' AND (error_key LIKE '%oom%' OR error_key LIKE '%memory%'
OR error_key LIKE '%memory%' OR reason LIKE '%memory%' OR reason LIKE '%OOM%'
OR reason LIKE '%memory%'
OR reason LIKE '%OOM%'
OR reason LIKE '%killed%process%') OR reason LIKE '%killed%process%')
''', (now_iso, stale_mem_cutoff)) ''', (now_iso, stale_mem_cutoff))
# ── 4. VMS category: Auto-resolve if VM/CT is now running or deleted ── # 4. VMS: Auto-resolve if VM/CT is now running or deleted
# Check all active VM/CT errors and resolve if the VM/CT is now running
# NOTE: We do this inline to avoid deadlock (check_vm_running uses _db_lock)
cursor.execute(''' cursor.execute('''
SELECT error_key, category, reason FROM errors SELECT error_key, category, reason FROM errors
WHERE (category IN ('vms', 'vmct') OR error_key LIKE 'vm_%' OR error_key LIKE 'ct_%' OR error_key LIKE 'vmct_%') WHERE (category IN ('vms', 'vmct') OR error_key LIKE 'vm_%'
AND resolved_at IS NULL OR error_key LIKE 'ct_%' OR error_key LIKE 'vmct_%')
AND acknowledged = 0 AND resolved_at IS NULL AND acknowledged = 0
''') ''')
vm_errors = cursor.fetchall() vm_errors = cursor.fetchall()
for error_key, cat, reason in vm_errors: for vm_ek, cat, vm_reason in vm_errors:
# Extract VM/CT ID from error_key vmid_match = re.search(r'(?:vm_|ct_|vmct_)(\d+)', vm_ek)
vmid_match = re.search(r'(?:vm_|ct_|vmct_)(\d+)', error_key)
if vmid_match: if vmid_match:
vmid = vmid_match.group(1) vmid = vmid_match.group(1)
try: try:
# Check if VM/CT exists and is running
vm_running = False vm_running = False
ct_running = False ct_running = False
vm_exists = False vm_exists = False
ct_exists = False ct_exists = False
# Check VM
result_vm = subprocess.run( result_vm = subprocess.run(
['qm', 'status', vmid], ['qm', 'status', vmid],
capture_output=True, text=True, timeout=2 capture_output=True, text=True, timeout=2)
)
if result_vm.returncode == 0: if result_vm.returncode == 0:
vm_exists = True vm_exists = True
vm_running = 'running' in result_vm.stdout.lower() vm_running = 'running' in result_vm.stdout.lower()
# Check CT
if not vm_exists: if not vm_exists:
result_ct = subprocess.run( result_ct = subprocess.run(
['pct', 'status', vmid], ['pct', 'status', vmid],
capture_output=True, text=True, timeout=2 capture_output=True, text=True, timeout=2)
)
if result_ct.returncode == 0: if result_ct.returncode == 0:
ct_exists = True ct_exists = True
ct_running = 'running' in result_ct.stdout.lower() ct_running = 'running' in result_ct.stdout.lower()
# Resolve if deleted
if not vm_exists and not ct_exists: if not vm_exists and not ct_exists:
cursor.execute(''' cursor.execute('''
UPDATE errors SET resolved_at = ? UPDATE errors SET resolved_at = ?
WHERE error_key = ? AND resolved_at IS NULL WHERE error_key = ? AND resolved_at IS NULL
''', (now_iso, error_key)) ''', (now_iso, vm_ek))
# Resolve transient errors if running (not persistent config errors)
elif vm_running or ct_running: elif vm_running or ct_running:
reason_lower = (reason or '').lower() reason_lower = (vm_reason or '').lower()
is_persistent = any(x in reason_lower for x in [ is_persistent = any(x in reason_lower for x in [
'device', 'missing', 'does not exist', 'permission', 'device', 'missing', 'does not exist', 'permission',
'not found', 'no such', 'invalid' 'not found', 'no such', 'invalid'])
])
if not is_persistent: if not is_persistent:
cursor.execute(''' cursor.execute('''
UPDATE errors SET resolved_at = ? UPDATE errors SET resolved_at = ?
WHERE error_key = ? AND resolved_at IS NULL WHERE error_key = ? AND resolved_at IS NULL
''', (now_iso, error_key)) ''', (now_iso, vm_ek))
except Exception: except Exception:
pass # Skip this VM/CT if check fails pass
# ── 5. GENERIC: Any error not seen in 30 minutes while system is healthy ── # 5. GENERIC: Any error not seen in 30 min while system is healthy
# If CPU < 80% and Memory < 85% and error hasn't been seen in 30 min,
# the system has recovered and the error is stale.
if current_cpu < 80 and current_mem < 85: if current_cpu < 80 and current_mem < 85:
stale_generic_cutoff = (now - timedelta(minutes=30)).isoformat() stale_generic_cutoff = (now - timedelta(minutes=30)).isoformat()
cursor.execute(''' cursor.execute('''
UPDATE errors UPDATE errors SET resolved_at = ?
SET resolved_at = ? WHERE resolved_at IS NULL AND acknowledged = 0
WHERE resolved_at IS NULL
AND acknowledged = 0
AND last_seen < ? AND last_seen < ?
AND category NOT IN ('disks', 'storage') AND category NOT IN ('disks', 'storage')
''', (now_iso, stale_generic_cutoff)) ''', (now_iso, stale_generic_cutoff))