Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-10-15 10:01:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix principal cause caps and bind measured recovery to incident policy

Browse Source
This commit is contained in:
martino
2026-10-01 08:38:32 +02:00
parent 8e396f957f
commit 3be15211b1
9 changed files with 537 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AppImage/scripts/health_monitor.py
+25 -3
View File
@@ -1427,6 +1427,8 @@ class HealthMonitor:
'details': f'Sustained for {actual_duration}s above {self.CPU_CRITICAL}%.',
'cpu_percent': cpu_percent,
'duration': actual_duration,
'cpu_policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
'recovery': self.CPU_RECOVERY},
},
)
elif len(warning_samples) >= WARNING_MIN_SAMPLES and len(recovery_samples) < RECOVERY_MIN_SAMPLES:
@@ -1446,6 +1448,8 @@ class HealthMonitor:
'details': f'Sustained for {actual_duration}s above {self.CPU_WARNING}%.',
'cpu_percent': cpu_percent,
'duration': actual_duration,
'cpu_policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
'recovery': self.CPU_RECOVERY},
},
)
else:
@@ -1453,8 +1457,21 @@ class HealthMonitor:
reason = None
# CPU is normal - auto-resolve any existing CPU errors
evidence = None
if cpu_percent < self.CPU_RECOVERY and len(recovery_samples) >= RECOVERY_MIN_SAMPLES:
evidence = {'check': 'cpu_usage', 'checked_at': current_time}
# Presentation proof is stricter than operational hysteresis:
# a supported warning can be below the fixed recovery cutoff.
from health_recovery import _finite_number
criterion = min(self.CPU_WARNING, self.CPU_RECOVERY)
normal_samples = [entry for entry in self.state_history[state_key]
if _finite_number(entry['value']) and 0 <= entry['value'] < criterion
and _finite_number(entry['time'])
and 0 <= current_time - entry['time'] <= self.CPU_RECOVERY_DURATION]
if (_finite_number(cpu_percent) and 0 <= cpu_percent < criterion
and len(normal_samples) >= RECOVERY_MIN_SAMPLES):
evidence = {'check': 'cpu_usage', 'checked_at': current_time,
'value': cpu_percent, 'normal_samples': len(normal_samples),
'max_sample': max(entry['value'] for entry in normal_samples),
'policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
'recovery': self.CPU_RECOVERY}}
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal',
check_evidence=evidence)
@@ -3904,6 +3921,7 @@ class HealthMonitor:
failed_services = []
service_details = {}
active_evidence = {}
for service in services_to_check:
try:
@@ -3918,6 +3936,10 @@ class HealthMonitor:
if result.returncode != 0 or status != 'active':
failed_services.append(service)
service_details[service] = status or 'inactive'
else:
active_evidence[service] = {'check': f'pve_service_{service}',
'checked_at': time.time(), 'service': service, 'state': status,
'returncode': result.returncode}
except Exception:
failed_services.append(service)
service_details[service] = 'error'
@@ -3932,7 +3954,7 @@ class HealthMonitor:
if svc not in failed_services:
error_key = f'pve_service_{svc}'
if health_persistence.is_error_active(error_key):
health_persistence.clear_error(error_key)
health_persistence.clear_error(error_key, check_evidence=active_evidence.get(svc))
# Build checks dict with status per service
checks = {}