mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-01 10:22:08 +00:00
feat: support PBS hosts and remote PBS monitoring
This commit is contained in:
@@ -51,6 +51,7 @@ DEBUG_PERF = False
|
||||
# ─── Startup Grace Period ────────────────────────────────────────────────────
|
||||
# Import centralized startup grace management for consistent behavior
|
||||
import startup_grace
|
||||
from pbs_manager import get_platform, PBS_SERVICES
|
||||
|
||||
def _is_startup_health_grace() -> bool:
|
||||
"""Check if we're within the startup health grace period (5 min).
|
||||
@@ -466,6 +467,7 @@ class HealthMonitor:
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize health monitor with state tracking"""
|
||||
self.platform_type = get_platform().get("product")
|
||||
self.state_history = defaultdict(list)
|
||||
self.last_check_times = {}
|
||||
self.cached_results = {}
|
||||
@@ -919,9 +921,9 @@ class HealthMonitor:
|
||||
_perf_log("services", (time.time() - _t) * 1000)
|
||||
details['services'] = services_status
|
||||
if services_status['status'] == 'CRITICAL':
|
||||
critical_issues.append(f"PVE Services: {services_status.get('reason', 'Service failure')}")
|
||||
critical_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service failure')}")
|
||||
elif services_status['status'] == 'WARNING':
|
||||
warning_issues.append(f"PVE Services: {services_status.get('reason', 'Service issue')}")
|
||||
warning_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service issue')}")
|
||||
|
||||
# Priority 1.5: Proxmox Storage Check (External Module)
|
||||
_t = time.time()
|
||||
@@ -2347,6 +2349,8 @@ class HealthMonitor:
|
||||
# The new _check_proxmox_storage function handles this logic better.
|
||||
def _check_proxmox_storages(self) -> Dict[str, Any]:
|
||||
"""Check Proxmox-specific storages (only report problems)"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
storages = {}
|
||||
|
||||
try:
|
||||
@@ -3515,6 +3519,8 @@ class HealthMonitor:
|
||||
Optimized VM/CT check - detects qmp failures and startup errors from logs.
|
||||
Improved detection of container and VM errors from journalctl.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
try:
|
||||
# First: auto-resolve any persisted VM/CT errors where the guest
|
||||
# is now running. This clears stale "Failed to start" / QMP
|
||||
@@ -3678,6 +3684,8 @@ class HealthMonitor:
|
||||
Check VMs/CTs with persistent error tracking.
|
||||
Errors persist until VM starts or 48h elapsed.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
try:
|
||||
issues = []
|
||||
vm_details = {}
|
||||
@@ -3919,8 +3927,18 @@ class HealthMonitor:
|
||||
"""
|
||||
try:
|
||||
# Build service list: base PVE services + corosync if clustered
|
||||
services_to_check = list(self.PVE_SERVICES)
|
||||
is_cluster = os.path.exists('/etc/corosync/corosync.conf')
|
||||
pbs_host = getattr(self, "platform_type", "pve") == "pbs"
|
||||
service_prefix = "pbs" if pbs_host else "pve"
|
||||
product_name = "PBS" if pbs_host else "PVE"
|
||||
services_to_check = list(PBS_SERVICES) if pbs_host else list(self.PVE_SERVICES)
|
||||
if pbs_host:
|
||||
# Old Monitor versions incorrectly treated absent PVE services
|
||||
# as PBS failures; remove only those known inapplicable errors.
|
||||
for service in self.PVE_SERVICES:
|
||||
old_key = f'pve_service_{service}'
|
||||
if health_persistence.is_error_active(old_key):
|
||||
health_persistence.clear_error(old_key)
|
||||
is_cluster = not pbs_host and os.path.exists('/etc/corosync/corosync.conf')
|
||||
if is_cluster and 'corosync' not in services_to_check:
|
||||
services_to_check.append('corosync')
|
||||
|
||||
@@ -3952,14 +3970,14 @@ class HealthMonitor:
|
||||
# cleanup loop). Audit Tier 5 — Health stack.
|
||||
for svc in services_to_check:
|
||||
if svc not in failed_services:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
if health_persistence.is_error_active(error_key):
|
||||
health_persistence.clear_error(error_key)
|
||||
|
||||
# Build checks dict with status per service
|
||||
checks = {}
|
||||
for svc in services_to_check:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
if svc in failed_services:
|
||||
state = service_details.get(svc, 'inactive')
|
||||
checks[svc] = {
|
||||
@@ -3987,12 +4005,12 @@ class HealthMonitor:
|
||||
# Record each failed service in persistence, respecting dismiss
|
||||
active_failed = []
|
||||
for svc in failed_services:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
rec_result = health_persistence.record_error(
|
||||
error_key=error_key,
|
||||
category='pve_services',
|
||||
severity='CRITICAL',
|
||||
reason=f'PVE service {svc} is {service_details.get(svc, "inactive")}',
|
||||
reason=f'{product_name} service {svc} is {service_details.get(svc, "inactive")}',
|
||||
details={'service': svc, 'state': service_details.get(svc, 'inactive')}
|
||||
)
|
||||
if rec_result and rec_result.get('type') == 'skipped_acknowledged':
|
||||
@@ -5816,6 +5834,8 @@ class HealthMonitor:
|
||||
- No persistent errors are recorded
|
||||
This prevents false positives when NFS/PBS/remote storage is still mounting.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not PROXMOX_STORAGE_AVAILABLE:
|
||||
return None
|
||||
|
||||
@@ -6205,6 +6225,8 @@ class HealthMonitor:
|
||||
Notification cooldown is the standard 1h via the storage
|
||||
category, so a CT teetering at 96% won't spam the channel.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
# Cheap pre-check: bail out when there are no running LXCs. On
|
||||
# nodes without CTs, the previous version still paid ~860ms per
|
||||
# cycle for a `pvesh /cluster/resources` call that produced no
|
||||
@@ -6365,6 +6387,8 @@ class HealthMonitor:
|
||||
path, so a slow / dead guest agent can't stretch the health
|
||||
cycle.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
try:
|
||||
import flask_server # deferred — avoids circular import
|
||||
resources = flask_server.get_cached_pvesh_cluster_resources_vm() or []
|
||||
@@ -6524,6 +6548,8 @@ class HealthMonitor:
|
||||
check (already covered) because those are usually large data
|
||||
mounts whose fill rate is driven by user content, not log spam.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not MOUNT_MONITOR_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
@@ -6627,6 +6653,8 @@ class HealthMonitor:
|
||||
check via _check_filesystem. Re-alerting on the same condition
|
||||
through two channels just clutters the inbox.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not PROXMOX_STORAGE_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user