feat: support PBS hosts and remote PBS monitoring

This commit is contained in:
Jonatan
2026-10-16 21:46:26 +02:00
parent 3000c1cd66
commit 511bfe52ea
26 changed files with 1355 additions and 15 deletions
+36 -8
View File
@@ -51,6 +51,7 @@ DEBUG_PERF = False
# ─── Startup Grace Period ────────────────────────────────────────────────────
# Import centralized startup grace management for consistent behavior
import startup_grace
from pbs_manager import get_platform, PBS_SERVICES
def _is_startup_health_grace() -> bool:
"""Check if we're within the startup health grace period (5 min).
@@ -466,6 +467,7 @@ class HealthMonitor:
def __init__(self):
"""Initialize health monitor with state tracking"""
self.platform_type = get_platform().get("product")
self.state_history = defaultdict(list)
self.last_check_times = {}
self.cached_results = {}
@@ -919,9 +921,9 @@ class HealthMonitor:
_perf_log("services", (time.time() - _t) * 1000)
details['services'] = services_status
if services_status['status'] == 'CRITICAL':
critical_issues.append(f"PVE Services: {services_status.get('reason', 'Service failure')}")
critical_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service failure')}")
elif services_status['status'] == 'WARNING':
warning_issues.append(f"PVE Services: {services_status.get('reason', 'Service issue')}")
warning_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service issue')}")
# Priority 1.5: Proxmox Storage Check (External Module)
_t = time.time()
@@ -2347,6 +2349,8 @@ class HealthMonitor:
# The new _check_proxmox_storage function handles this logic better.
def _check_proxmox_storages(self) -> Dict[str, Any]:
"""Check Proxmox-specific storages (only report problems)"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
storages = {}
try:
@@ -3515,6 +3519,8 @@ class HealthMonitor:
Optimized VM/CT check - detects qmp failures and startup errors from logs.
Improved detection of container and VM errors from journalctl.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
try:
# First: auto-resolve any persisted VM/CT errors where the guest
# is now running. This clears stale "Failed to start" / QMP
@@ -3678,6 +3684,8 @@ class HealthMonitor:
Check VMs/CTs with persistent error tracking.
Errors persist until VM starts or 48h elapsed.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
try:
issues = []
vm_details = {}
@@ -3919,8 +3927,18 @@ class HealthMonitor:
"""
try:
# Build service list: base PVE services + corosync if clustered
services_to_check = list(self.PVE_SERVICES)
is_cluster = os.path.exists('/etc/corosync/corosync.conf')
pbs_host = getattr(self, "platform_type", "pve") == "pbs"
service_prefix = "pbs" if pbs_host else "pve"
product_name = "PBS" if pbs_host else "PVE"
services_to_check = list(PBS_SERVICES) if pbs_host else list(self.PVE_SERVICES)
if pbs_host:
# Old Monitor versions incorrectly treated absent PVE services
# as PBS failures; remove only those known inapplicable errors.
for service in self.PVE_SERVICES:
old_key = f'pve_service_{service}'
if health_persistence.is_error_active(old_key):
health_persistence.clear_error(old_key)
is_cluster = not pbs_host and os.path.exists('/etc/corosync/corosync.conf')
if is_cluster and 'corosync' not in services_to_check:
services_to_check.append('corosync')
@@ -3952,14 +3970,14 @@ class HealthMonitor:
# cleanup loop). Audit Tier 5 — Health stack.
for svc in services_to_check:
if svc not in failed_services:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
if health_persistence.is_error_active(error_key):
health_persistence.clear_error(error_key)
# Build checks dict with status per service
checks = {}
for svc in services_to_check:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
if svc in failed_services:
state = service_details.get(svc, 'inactive')
checks[svc] = {
@@ -3987,12 +4005,12 @@ class HealthMonitor:
# Record each failed service in persistence, respecting dismiss
active_failed = []
for svc in failed_services:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
rec_result = health_persistence.record_error(
error_key=error_key,
category='pve_services',
severity='CRITICAL',
reason=f'PVE service {svc} is {service_details.get(svc, "inactive")}',
reason=f'{product_name} service {svc} is {service_details.get(svc, "inactive")}',
details={'service': svc, 'state': service_details.get(svc, 'inactive')}
)
if rec_result and rec_result.get('type') == 'skipped_acknowledged':
@@ -5816,6 +5834,8 @@ class HealthMonitor:
- No persistent errors are recorded
This prevents false positives when NFS/PBS/remote storage is still mounting.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not PROXMOX_STORAGE_AVAILABLE:
return None
@@ -6205,6 +6225,8 @@ class HealthMonitor:
Notification cooldown is the standard 1h via the storage
category, so a CT teetering at 96% won't spam the channel.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
# Cheap pre-check: bail out when there are no running LXCs. On
# nodes without CTs, the previous version still paid ~860ms per
# cycle for a `pvesh /cluster/resources` call that produced no
@@ -6365,6 +6387,8 @@ class HealthMonitor:
path, so a slow / dead guest agent can't stretch the health
cycle.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
try:
import flask_server # deferred — avoids circular import
resources = flask_server.get_cached_pvesh_cluster_resources_vm() or []
@@ -6524,6 +6548,8 @@ class HealthMonitor:
check (already covered) because those are usually large data
mounts whose fill rate is driven by user content, not log spam.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not MOUNT_MONITOR_AVAILABLE:
return None
try:
@@ -6627,6 +6653,8 @@ class HealthMonitor:
check via _check_filesystem. Re-alerting on the same condition
through two channels just clutters the inbox.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not PROXMOX_STORAGE_AVAILABLE:
return None
try: