mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-05-02 12:26:23 +00:00
Update health_persistence.py
This commit is contained in:
MacRimi
@@ -716,12 +716,11 @@ class HealthPersistence:
|
||||
if 'disk' in error_key or 'smart' in error_key or 'sda' in error_key or 'sdb' in error_key or 'nvme' in error_key:
|
||||
category = 'disks'
|
||||
else:
|
||||
category = 'general' # Use 'general' as ultimate fallback instead of empty string
|
||||
category = 'general'
|
||||
|
||||
setting_key = self.CATEGORY_SETTING_MAP.get(category, '')
|
||||
sup_hours = self.DEFAULT_SUPPRESSION_HOURS
|
||||
if setting_key:
|
||||
# P4 fix: use _get_setting_impl with existing connection
|
||||
stored = self._get_setting_impl(conn, setting_key)
|
||||
if stored is not None:
|
||||
try:
|
||||
@@ -762,7 +761,6 @@ class HealthPersistence:
|
||||
setting_key = self.CATEGORY_SETTING_MAP.get(category, '')
|
||||
sup_hours = self.DEFAULT_SUPPRESSION_HOURS
|
||||
if setting_key:
|
||||
# P4 fix: use _get_setting_impl with existing connection
|
||||
stored = self._get_setting_impl(conn, setting_key)
|
||||
if stored is not None:
|
||||
try:
|
||||
@@ -771,8 +769,6 @@ class HealthPersistence:
|
||||
pass
|
||||
|
||||
# Mark as acknowledged but DO NOT set resolved_at
|
||||
# The error remains active until it actually disappears from the system
|
||||
# resolved_at should only be set when the error is truly resolved
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ?
|
||||
@@ -786,18 +782,11 @@ class HealthPersistence:
|
||||
})
|
||||
|
||||
# Cascade acknowledge: when dismissing a group check
|
||||
# (e.g. log_persistent_errors), also dismiss all individual
|
||||
# sub-errors that share the same prefix in the DB.
|
||||
# Currently only persistent errors have per-pattern sub-records
|
||||
# (e.g. log_persistent_a1b2c3d4).
|
||||
CASCADE_PREFIXES = {
|
||||
'log_persistent_errors': 'log_persistent_',
|
||||
}
|
||||
child_prefix = CASCADE_PREFIXES.get(error_key)
|
||||
if child_prefix:
|
||||
# Only cascade to active (unresolved) child errors.
|
||||
# Already-resolved/expired entries must NOT be re-surfaced.
|
||||
# Mark as acknowledged but DO NOT set resolved_at
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET acknowledged = 1, acknowledged_at = ?, suppression_hours = ?
|
||||
@@ -939,9 +928,6 @@ class HealthPersistence:
|
||||
cursor.execute('DELETE FROM errors WHERE resolved_at < ?', (cutoff_resolved,))
|
||||
|
||||
# ── Auto-resolve stale errors using Suppression Duration settings ──
|
||||
# Read per-category suppression hours from user_settings.
|
||||
# If the user hasn't configured a value, use DEFAULT_SUPPRESSION_HOURS.
|
||||
# This is the SINGLE source of truth for auto-resolution timing.
|
||||
user_settings = {}
|
||||
try:
|
||||
cursor.execute(
|
||||
@@ -960,7 +946,6 @@ class HealthPersistence:
|
||||
except (ValueError, TypeError):
|
||||
hours = self.DEFAULT_SUPPRESSION_HOURS
|
||||
|
||||
# -1 means permanently suppressed -- skip auto-resolve
|
||||
if hours < 0:
|
||||
continue
|
||||
|
||||
@@ -975,7 +960,6 @@ class HealthPersistence:
|
||||
''', (now_iso, category, cutoff))
|
||||
|
||||
# Catch-all: auto-resolve any error from an unmapped category
|
||||
# whose last_seen exceeds DEFAULT_SUPPRESSION_HOURS.
|
||||
fallback_cutoff = (now - timedelta(hours=self.DEFAULT_SUPPRESSION_HOURS)).isoformat()
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
@@ -989,138 +973,105 @@ class HealthPersistence:
|
||||
cutoff_events = (now - timedelta(days=30)).isoformat()
|
||||
cursor.execute('DELETE FROM events WHERE timestamp < ?', (cutoff_events,))
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# SMART AUTO-RESOLVE: Based on system state, not hardcoded patterns
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# Logic: If an error hasn't been seen recently AND the system is healthy,
|
||||
# the error is stale and should be auto-resolved.
|
||||
# This works for ANY error pattern, not just predefined ones.
|
||||
# ── SMART AUTO-RESOLVE: Based on system state ──
|
||||
try:
|
||||
import psutil
|
||||
# Get system uptime
|
||||
with open('/proc/uptime', 'r') as f:
|
||||
uptime_seconds = float(f.read().split()[0])
|
||||
|
||||
# Only auto-resolve if system has been stable for at least 10 minutes
|
||||
if uptime_seconds > 600: # 10 minutes
|
||||
if uptime_seconds > 600:
|
||||
current_cpu = psutil.cpu_percent(interval=0.1)
|
||||
current_mem = psutil.virtual_memory().percent
|
||||
|
||||
# ── 1. LOGS category: Auto-resolve if not seen in 15 minutes ──
|
||||
# Log errors are transient - if journalctl hasn't reported them recently,
|
||||
# they are from a previous state and should be resolved.
|
||||
# 1. LOGS: Auto-resolve if not seen in 15 minutes
|
||||
stale_logs_cutoff = (now - timedelta(minutes=15)).isoformat()
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET resolved_at = ?
|
||||
WHERE category = 'logs'
|
||||
AND resolved_at IS NULL
|
||||
AND acknowledged = 0
|
||||
AND last_seen < ?
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE category = 'logs' AND resolved_at IS NULL
|
||||
AND acknowledged = 0 AND last_seen < ?
|
||||
''', (now_iso, stale_logs_cutoff))
|
||||
|
||||
# ── 2. CPU category: Auto-resolve if CPU is normal (<75%) ──
|
||||
# 2. CPU: Auto-resolve if CPU is normal (<75%)
|
||||
if current_cpu < 75:
|
||||
stale_cpu_cutoff = (now - timedelta(minutes=5)).isoformat()
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET resolved_at = ?
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE (category = 'cpu' OR category = 'temperature')
|
||||
AND resolved_at IS NULL
|
||||
AND acknowledged = 0
|
||||
AND resolved_at IS NULL AND acknowledged = 0
|
||||
AND last_seen < ?
|
||||
AND (error_key LIKE 'cpu_%' OR reason LIKE '%CPU%')
|
||||
''', (now_iso, stale_cpu_cutoff))
|
||||
|
||||
# ── 3. MEMORY category: Auto-resolve if memory is normal (<80%) ──
|
||||
# 3. MEMORY: Auto-resolve if memory is normal (<80%)
|
||||
if current_mem < 80:
|
||||
stale_mem_cutoff = (now - timedelta(minutes=5)).isoformat()
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET resolved_at = ?
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE (category = 'memory' OR category = 'logs')
|
||||
AND resolved_at IS NULL
|
||||
AND acknowledged = 0
|
||||
AND resolved_at IS NULL AND acknowledged = 0
|
||||
AND last_seen < ?
|
||||
AND (error_key LIKE '%oom%'
|
||||
OR error_key LIKE '%memory%'
|
||||
OR reason LIKE '%memory%'
|
||||
OR reason LIKE '%OOM%'
|
||||
AND (error_key LIKE '%oom%' OR error_key LIKE '%memory%'
|
||||
OR reason LIKE '%memory%' OR reason LIKE '%OOM%'
|
||||
OR reason LIKE '%killed%process%')
|
||||
''', (now_iso, stale_mem_cutoff))
|
||||
|
||||
# ── 4. VMS category: Auto-resolve if VM/CT is now running or deleted ──
|
||||
# Check all active VM/CT errors and resolve if the VM/CT is now running
|
||||
# NOTE: We do this inline to avoid deadlock (check_vm_running uses _db_lock)
|
||||
# 4. VMS: Auto-resolve if VM/CT is now running or deleted
|
||||
cursor.execute('''
|
||||
SELECT error_key, category, reason FROM errors
|
||||
WHERE (category IN ('vms', 'vmct') OR error_key LIKE 'vm_%' OR error_key LIKE 'ct_%' OR error_key LIKE 'vmct_%')
|
||||
AND resolved_at IS NULL
|
||||
AND acknowledged = 0
|
||||
WHERE (category IN ('vms', 'vmct') OR error_key LIKE 'vm_%'
|
||||
OR error_key LIKE 'ct_%' OR error_key LIKE 'vmct_%')
|
||||
AND resolved_at IS NULL AND acknowledged = 0
|
||||
''')
|
||||
vm_errors = cursor.fetchall()
|
||||
for error_key, cat, reason in vm_errors:
|
||||
# Extract VM/CT ID from error_key
|
||||
vmid_match = re.search(r'(?:vm_|ct_|vmct_)(\d+)', error_key)
|
||||
for vm_ek, cat, vm_reason in vm_errors:
|
||||
vmid_match = re.search(r'(?:vm_|ct_|vmct_)(\d+)', vm_ek)
|
||||
if vmid_match:
|
||||
vmid = vmid_match.group(1)
|
||||
try:
|
||||
# Check if VM/CT exists and is running
|
||||
vm_running = False
|
||||
ct_running = False
|
||||
vm_exists = False
|
||||
ct_exists = False
|
||||
|
||||
# Check VM
|
||||
result_vm = subprocess.run(
|
||||
['qm', 'status', vmid],
|
||||
capture_output=True, text=True, timeout=2
|
||||
)
|
||||
capture_output=True, text=True, timeout=2)
|
||||
if result_vm.returncode == 0:
|
||||
vm_exists = True
|
||||
vm_running = 'running' in result_vm.stdout.lower()
|
||||
|
||||
# Check CT
|
||||
if not vm_exists:
|
||||
result_ct = subprocess.run(
|
||||
['pct', 'status', vmid],
|
||||
capture_output=True, text=True, timeout=2
|
||||
)
|
||||
capture_output=True, text=True, timeout=2)
|
||||
if result_ct.returncode == 0:
|
||||
ct_exists = True
|
||||
ct_running = 'running' in result_ct.stdout.lower()
|
||||
|
||||
# Resolve if deleted
|
||||
if not vm_exists and not ct_exists:
|
||||
cursor.execute('''
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE error_key = ? AND resolved_at IS NULL
|
||||
''', (now_iso, error_key))
|
||||
# Resolve transient errors if running (not persistent config errors)
|
||||
''', (now_iso, vm_ek))
|
||||
elif vm_running or ct_running:
|
||||
reason_lower = (reason or '').lower()
|
||||
reason_lower = (vm_reason or '').lower()
|
||||
is_persistent = any(x in reason_lower for x in [
|
||||
'device', 'missing', 'does not exist', 'permission',
|
||||
'not found', 'no such', 'invalid'
|
||||
])
|
||||
'not found', 'no such', 'invalid'])
|
||||
if not is_persistent:
|
||||
cursor.execute('''
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE error_key = ? AND resolved_at IS NULL
|
||||
''', (now_iso, error_key))
|
||||
''', (now_iso, vm_ek))
|
||||
except Exception:
|
||||
pass # Skip this VM/CT if check fails
|
||||
pass
|
||||
|
||||
# ── 5. GENERIC: Any error not seen in 30 minutes while system is healthy ──
|
||||
# If CPU < 80% and Memory < 85% and error hasn't been seen in 30 min,
|
||||
# the system has recovered and the error is stale.
|
||||
# 5. GENERIC: Any error not seen in 30 min while system is healthy
|
||||
if current_cpu < 80 and current_mem < 85:
|
||||
stale_generic_cutoff = (now - timedelta(minutes=30)).isoformat()
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
SET resolved_at = ?
|
||||
WHERE resolved_at IS NULL
|
||||
AND acknowledged = 0
|
||||
UPDATE errors SET resolved_at = ?
|
||||
WHERE resolved_at IS NULL AND acknowledged = 0
|
||||
AND last_seen < ?
|
||||
AND category NOT IN ('disks', 'storage')
|
||||
''', (now_iso, stale_generic_cutoff))
|
||||
|
||||
Reference in New Issue
Block a user