Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-09-16 19:56:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add audit and reports page, and a change journal

Browse Source 
ProxMenux modifies the host: it rewrites configuration files, installs packages, enables services. Until now nobody could say afterwards what had changed, and showing the script does not answer that question — a four-hundred-line function may alter two values, and the reader has no way to know which two. This adds the two halves of an answer.

The change journal records what ProxMenux does as it does it. Eleven bash primitives capture the previous state, apply the change and record it in the same step, writing to a spool that the Monitor reads back. One hundred and thirteen functions across twenty-five scripts are instrumented, covering post-install, shared storage, security tooling, container conversions, disk operations and the PVE 8 to 9 upgrade path. The page shows the difference — rotate 7 becoming rotate 14 — and never the script. Restore and backup scripts are deliberately left out: a restore puts the host back to a state some other script already recorded.

The Audit and reports page answers the other half: what state is this host in, regardless of who put it there. Forty-three checks across seven areas read the host and classify each result as critical, warning, observation, conformant, unverified or not applicable, with the evidence they read attached to each one. A declared policy lets the reader say what this particular host is expected to do — which guests must have a backup, which storages are essential — so the report judges the host against its own intent rather than a generic template. An inventory records the hardware, network and guest topology behind those readings, a comparison shows what moved between two runs, and six report profiles produce a printable document scoped to what the reader needs. Everything is available in the eight supported languages.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
MacRimi
2026-09-08 21:06:04 +02:00
co-authored by Claude Opus 5
parent ae75508eff
commit da8a480eff
102 changed files with 24118 additions and 1403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AppImage/scripts/notification_events.py
+169 -15
View File
@@ -513,6 +513,16 @@ class JournalWatcher:
self._oom_lines = []
self._oom_started_at = 0.0
# Keep the small amount of journal history that precedes a kernel
# diagnostic. `Call Trace:` is only a structural marker inside that
# diagnostic, never the cause itself. The old detector promoted the
# marker to an event and therefore sent an unactionable "Kernel call
# trace" every 24 h, sometimes followed by a second burst message for
# another line from the same incident.
from collections import deque as _deque
self._kernel_context = _deque(maxlen=40)
self._KERNEL_CONTEXT_WINDOW_SECS = 15
# 24h anti-cascade for disk I/O + filesystem errors. The dict
# key includes a tier suffix (`sdh:warning`, `sdh:critical`)
# so a disk in WARNING cooldown can still escalate to CRITICAL
@@ -526,7 +536,6 @@ class JournalWatcher:
# paper showed ~36% of failed drives gave no SMART warning.
# Rate-based escalation catches the dying drives that SMART
# would never flag until they were already bricked.
from collections import deque as _deque
self._disk_error_window: Dict[str, "_deque[float]"] = {}
self._DISK_ERROR_WINDOW_SECS = 86400 # 24h
# Tiers calibrated for homelab/SMB Proxmox usage:
@@ -767,7 +776,7 @@ class JournalWatcher:
self._check_auth_failure(msg, syslog_id, entry)
self._check_fail2ban(msg, syslog_id)
self._check_kernel_critical(msg, syslog_id, priority)
self._check_kernel_critical(msg, syslog_id, priority, entry)
self._check_service_failure(msg, unit)
self._check_disk_io(msg, syslog_id, priority)
self._check_cluster_events(msg, syslog_id)
@@ -849,13 +858,69 @@ class JournalWatcher:
'hostname': self._hostname,
}, entity='user', entity_id=ip)
def _check_kernel_critical(self, msg: str, syslog_id: str, priority: int):
def _remember_kernel_context(self, msg: str, now: float) -> str:
"""Record and return the recent journal excerpt for a kernel event."""
self._kernel_context.append((now, msg))
cutoff = now - self._KERNEL_CONTEXT_WINDOW_SECS
while self._kernel_context and self._kernel_context[0][0] < cutoff:
self._kernel_context.popleft()
return '\n'.join(line for _, line in self._kernel_context)[-4000:]
@staticmethod
def _kernel_diagnostic(msg: str) -> Optional[Tuple[str, str, str]]:
"""Return (kind, process, component) for an attributable kernel event.
A bare ``Call Trace:`` intentionally has no match. It is analogous to
a heading in a diagnostic block and cannot establish that a new fault
occurred. The patterns below identify the line that explains why the
kernel printed the trace.
"""
patterns = (
(r'\bWARNING:\s+CPU:', 'Kernel warning'),
(r'\bINFO:\s+task\s+.+?\s+blocked for more than\s+\d+', 'Blocked kernel task'),
(r'\btask\s+.+?\s+blocked for more than\s+\d+', 'Blocked kernel task'),
(r'\brcu(?:_preempt|_sched|):.*detected stalls?', 'RCU stall'),
(r'\bsoft lockup\b', 'CPU soft lockup'),
(r'\bhard LOCKUP\b', 'CPU hard lockup'),
(r'\bgeneral protection fault\b', 'General protection fault'),
(r'\bunable to handle kernel (?:NULL pointer dereference|paging request)', 'Kernel memory access fault'),
(r'\bOops:', 'Kernel oops'),
(r'\bUBSAN:', 'Undefined behaviour detected'),
(r'\bKASAN:', 'Kernel memory safety violation'),
)
kind = ''
for pattern, label in patterns:
if re.search(pattern, msg, re.IGNORECASE):
kind = label
break
if not kind:
return None
process = ''
process_match = re.search(r'\bPID:\s*(\d+)\s+Comm:\s*([^\s]+)', msg)
if process_match:
process = f'{process_match.group(2)} (PID {process_match.group(1)})'
else:
blocked_match = re.search(r'\btask\s+([^:\s]+)(?::\d+)?\s+blocked for more than', msg, re.IGNORECASE)
if blocked_match:
process = blocked_match.group(1)
component = ''
component_match = re.search(r'\bat\s+([^\s+]+)(?:\+0x[0-9a-f]+/0x[0-9a-f]+)?', msg, re.IGNORECASE)
if component_match:
component = component_match.group(1)
return kind, process, component
def _check_kernel_critical(self, msg: str, syslog_id: str, priority: int,
entry: Optional[Dict] = None):
"""Detect kernel panics, OOM, segfaults, hardware errors."""
# Only process messages from kernel or systemd (not app-level logs)
if syslog_id and syslog_id not in ('kernel', 'systemd', 'systemd-coredump', ''):
return
now = time.time()
journal_context = self._remember_kernel_context(msg, now)
if self._oom_lines and now - self._oom_started_at > 15:
self._oom_lines = []
self._oom_started_at = 0.0
@@ -918,6 +983,43 @@ class JournalWatcher:
for noise in _KERNEL_NOISE:
if re.search(noise, msg, re.IGNORECASE):
return
# A JSON journal entry lets us prove that the diagnostic came from the
# kernel transport. Plain-mode input remains supported for older
# journalctl fallbacks, but a systemd/application entry containing the
# words "WARNING: CPU" cannot masquerade as a kernel event.
transport = str((entry or {}).get('_TRANSPORT', '') or '')
is_kernel_source = entry is None or syslog_id == 'kernel' or transport == 'kernel'
diagnostic = self._kernel_diagnostic(msg) if is_kernel_source and not self._oom_lines else None
if diagnostic:
kind, process, component = diagnostic
observed_us = str((entry or {}).get('__REALTIME_TIMESTAMP', '') or '')
try:
observed_ts = int(observed_us) / 1_000_000 if observed_us else now
except (TypeError, ValueError):
observed_ts = now
observed_at = time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime(observed_ts))
details = [f'Type: {kind}']
if process:
details.append(f'Process: {process}')
if component:
details.append(f'Component: {component}')
details.extend((f'Message: {msg[:500]}', f'Recorded: {observed_at}'))
identity = f'{kind}\x1f{component}\x1f{process}\x1f{msg[:300]}'
entity_id = f'kernel_{hashlib.sha256(identity.encode(errors="replace")).hexdigest()[:16]}'
self._emit(
'kernel_warning',
'WARNING',
{
'hostname': self._hostname,
'reason': f'{kind}\n{msg[:500]}',
'kernel_details': '\n'.join(details),
'_journal_context': journal_context,
},
entity='node',
entity_id=entity_id,
)
return
# NOTE: Disk I/O errors (ATA, SCSI, blk_update_request) are NOT handled
# here. They are detected exclusively by HealthMonitor._check_disks_optimized
@@ -932,7 +1034,6 @@ class JournalWatcher:
r'Out of memory': ('system_problem', 'CRITICAL', 'Out of memory killer activated'),
r'segfault': ('system_problem', 'WARNING', 'Segmentation fault detected'),
r'BUG:': ('system_problem', 'CRITICAL', 'Kernel BUG detected'),
r'Call Trace:': ('system_problem', 'WARNING', 'Kernel call trace'),
r'EXT4-fs error': ('system_problem', 'CRITICAL', 'Filesystem error'),
r'BTRFS error': ('system_problem', 'CRITICAL', 'Filesystem error'),
r'XFS.*error': ('system_problem', 'CRITICAL', 'Filesystem error'),
@@ -2634,6 +2735,53 @@ class PollingCollector:
def _hostname(self) -> str:
return _hostname()
@staticmethod
def _guest_storage_error_is_now_foreign(error_key: str, old_meta: dict) -> bool:
"""Return True when a disappearing guest-capacity error moved nodes.
Older versions recorded `lxc_disk_<vmid>` and `vm_disk_<vmid>` on
every cluster member because the health check consumed the unfiltered
cluster resource list. A normal `resolved_keys` transition would make
those foreign records produce one final, false recovery after the
ownership filter is installed. The same distinction matters during a
real migration: leaving the old node is not recovery.
Prefer the current cluster owner over the historical details, because
a legitimate local alert can subsequently migrate. The stored node is
only a fallback for a guest no longer present in the resource list.
"""
match = re.fullmatch(r'(?:lxc|vm)_disk_(\d+)', str(error_key or ''))
if not match:
return False
try:
import flask_server # deferred: flask_server imports this module
local_node = str(flask_server.get_proxmox_node_name() or '')
resources = flask_server.get_cached_pvesh_cluster_resources_vm() or []
vmid = match.group(1)
for resource in resources:
if str(resource.get('vmid', '')) != vmid:
continue
if resource.get('type') not in ('lxc', 'qemu', 'vm'):
continue
owner = str(resource.get('node') or '')
if owner and local_node:
return owner != local_node
except Exception:
local_node = ''
details = old_meta.get('details') if isinstance(old_meta, dict) else None
if isinstance(details, str):
try:
details = json.loads(details)
except (json.JSONDecodeError, TypeError):
details = None
if isinstance(details, dict):
owner = str(details.get('node') or '')
if owner and local_node:
return owner != local_node
return False
def start(self):
if self._running:
return
@@ -2988,6 +3136,15 @@ class PollingCollector:
reason = old_meta.get('reason', '')
first_seen = old_meta.get('first_seen', '')
# A guest moving to another cluster node — or a legacy foreign
# record created by the old cluster-wide capacity scan — has not
# recovered. Drop only this node's tracking state and let the
# current owner report the condition if it is still present.
if self._guest_storage_error_is_now_foreign(key, old_meta):
self._last_notified.pop(key, None)
self._notified_severity.pop(key, None)
continue
# Skip recovery for INFO/OK - they never triggered an alert
if old_meta.get('severity', '') in ('INFO', 'OK'):
self._last_notified.pop(key, None)
@@ -3642,7 +3799,11 @@ class PollingCollector:
# blocks the others.
try:
import lxc_apps
lxc_apps.refresh_all_apps(force=False)
# The automatic sweep builds one detailed summary after every app
# has been refreshed. Suppress the per-app emit here so the user
# does not receive the individual messages before that summary.
# Explicit UI checks keep the default notify=True behaviour.
lxc_apps.refresh_all_apps(force=False, notify=False)
# Docker images have an independent lifecycle from both the OS
# packages and the Docker engine. Refresh their read-only
# registry digest inventory on the same daily cadence; this never
@@ -3652,16 +3813,9 @@ class PollingCollector:
# yesterday's cycle cannot postpone the next automatic scan by an
# additional day. Normal UI reads remain cache-only for 24 hours.
lxc_apps.refresh_docker_inventories(force=True)
# After the refresh, emit `app_update_available` for every
# sidecar entry currently flagged with a pending upstream
# release. `check_app(force=False)` short-circuits on a
# fresh `checked_at` and never reaches the emit path, so
# without this call the notification only ever fired on
# the exact tick where a new version was FIRST observed —
# missed forever if the user had the toggle off at that
# moment. `notification_manager` dedups by entity_id
# (vmid + app_id + latest_version) so repeated calls only
# deliver one notification per release.
# Emit one detailed registered-app summary for this sweep. A
# single pending app retains the existing individual wording;
# several apps are grouped by CT with every version pair intact.
lxc_apps.emit_all_pending_docker_stacks()
lxc_apps.emit_all_pending_updates()
except Exception as e: