Update notification service

This commit is contained in:
MacRimi
2026-02-26 20:58:40 +01:00
parent 17eecfca9d
commit 0693acc07b
3 changed files with 61 additions and 57 deletions

View File

@@ -686,16 +686,6 @@ def proxmox_webhook():
else: else:
return _reject(400, 'empty_payload', 400) return _reject(400, 'empty_payload', 400)
# DEBUG: Capture raw webhook payload for parser analysis
import json as _json
try:
with open('/tmp/proxmenux_webhook_payload.log', 'a') as _f:
_f.write(f"\n{'='*80}\n{time.strftime('%Y-%m-%d %H:%M:%S')}\n")
_f.write(_json.dumps(payload, indent=2, default=str, ensure_ascii=False))
_f.write('\n')
except Exception:
pass
result = notification_manager.process_webhook(payload) result = notification_manager.process_webhook(payload)
# Always return 200 to PVE -- a non-200 makes PVE report the webhook as broken. # Always return 200 to PVE -- a non-200 makes PVE report the webhook as broken.
# The 'accepted' field in the JSON body indicates actual processing status. # The 'accepted' field in the JSON body indicates actual processing status.

View File

@@ -571,11 +571,9 @@ class TaskWatcher:
self._thread: Optional[threading.Thread] = None self._thread: Optional[threading.Thread] = None
self._hostname = _hostname() self._hostname = _hostname()
self._last_position = 0 self._last_position = 0
# Track active vzdump jobs. While a vzdump is running, VM/CT # Cache for active vzdump detection
# start/stop/shutdown events are backup-induced (mode=stop/snapshot) self._vzdump_active_cache: float = 0 # timestamp of last positive check
# and should NOT generate notifications. self._vzdump_cache_ttl = 5 # cache result for 5s
self._active_vzdump_ts: float = 0 # timestamp of last vzdump start
self._VZDUMP_WINDOW = 14400 # 4h max backup window
def start(self): def start(self):
if self._running: if self._running:
@@ -596,6 +594,29 @@ class TaskWatcher:
def stop(self): def stop(self):
self._running = False self._running = False
def _is_vzdump_active(self) -> bool:
"""Check if a vzdump (backup) job is currently running.
Reads /var/log/pve/tasks/active which lists all running PVE tasks.
Result is cached for a few seconds to avoid excessive file reads.
"""
now = time.time()
if now - self._vzdump_active_cache < self._vzdump_cache_ttl:
return True # Recently confirmed active
active_file = '/var/log/pve/tasks/active'
try:
with open(active_file, 'r') as f:
for line in f:
# UPID format: UPID:node:pid:pstart:starttime:type:id:user:
if ':vzdump:' in line:
self._vzdump_active_cache = now
return True
except (OSError, IOError):
pass
return False
def _watch_loop(self): def _watch_loop(self):
"""Poll the task index file for new entries.""" """Poll the task index file for new entries."""
while self._running: while self._running:
@@ -656,22 +677,7 @@ class TaskWatcher:
event_type, default_severity = event_info event_type, default_severity = event_info
# ── Track active vzdump jobs ──
# When a vzdump starts, record its timestamp. While active, we
# suppress start/stop/shutdown of individual VMs -- those are just
# the backup stopping and restarting guests (mode=stop).
if task_type == 'vzdump':
if not status:
# vzdump just started
self._active_vzdump_ts = time.time()
else:
# vzdump finished -- clear after a small grace period
# (VMs may still be restarting)
def _clear_vzdump():
time.sleep(30)
self._active_vzdump_ts = 0
threading.Thread(target=_clear_vzdump, daemon=True,
name='clear-vzdump').start()
# Check if task failed # Check if task failed
is_error = status and status != 'OK' and status != '' is_error = status and status != 'OK' and status != ''
@@ -721,10 +727,8 @@ class TaskWatcher:
# Exception: if a VM/CT FAILS to start after backup, that IS important. # Exception: if a VM/CT FAILS to start after backup, that IS important.
_BACKUP_NOISE = {'vm_start', 'vm_stop', 'vm_shutdown', 'vm_restart', _BACKUP_NOISE = {'vm_start', 'vm_stop', 'vm_shutdown', 'vm_restart',
'ct_start', 'ct_stop'} 'ct_start', 'ct_stop'}
vzdump_age = time.time() - self._active_vzdump_ts if self._active_vzdump_ts else float('inf') if event_type in _BACKUP_NOISE and not is_error:
if event_type in _BACKUP_NOISE and vzdump_age < self._VZDUMP_WINDOW: if self._is_vzdump_active():
# Allow through only if it's a FAILURE (e.g. VM failed to start)
if not is_error:
return return
self._queue.put(NotificationEvent( self._queue.put(NotificationEvent(

View File

@@ -49,28 +49,38 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
break break
if header_idx >= 0: if header_idx >= 0:
for line in lines[header_idx + 1:]: # Use column positions from the header to slice each row.
stripped = line.strip() # Header: "VMID Name Status Time Size Filename"
if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='): header = lines[header_idx]
break col_starts = []
m = re.match( for col_name in ['VMID', 'Name', 'Status', 'Time', 'Size', 'Filename']:
r'\s*(\d+)\s+' # VMID idx = header.find(col_name)
r'(\S+)\s+' # Name if idx >= 0:
r'(\S+)\s+' # Status (ok/error) col_starts.append(idx)
r'(\S+)\s+' # Time
r'([\d.]+\s+\S+)\s+' # Size (e.g. "1.423 GiB") if len(col_starts) == 6:
r'(\S+)', # Filename for line in lines[header_idx + 1:]:
line stripped = line.strip()
) if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='):
if m: break
vms.append({ # Pad line to avoid index errors
'vmid': m.group(1), padded = line.ljust(col_starts[-1] + 50)
'name': m.group(2), vmid = padded[col_starts[0]:col_starts[1]].strip()
'status': m.group(3), name = padded[col_starts[1]:col_starts[2]].strip()
'time': m.group(4), status = padded[col_starts[2]:col_starts[3]].strip()
'size': m.group(5), time_val = padded[col_starts[3]:col_starts[4]].strip()
'filename': m.group(6).split('/')[-1], size = padded[col_starts[4]:col_starts[5]].strip()
}) filename = padded[col_starts[5]:].strip()
if vmid and vmid.isdigit():
vms.append({
'vmid': vmid,
'name': name,
'status': status,
'time': time_val,
'size': size,
'filename': filename,
})
# ── Strategy 2: log-style (PBS / Proxmox Backup Server) ── # ── Strategy 2: log-style (PBS / Proxmox Backup Server) ──
# Parse from the full vzdump log lines. # Parse from the full vzdump log lines.