mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-04-25 08:56:21 +00:00
Update notification service
This commit is contained in:
@@ -686,16 +686,6 @@ def proxmox_webhook():
|
|||||||
else:
|
else:
|
||||||
return _reject(400, 'empty_payload', 400)
|
return _reject(400, 'empty_payload', 400)
|
||||||
|
|
||||||
# DEBUG: Capture raw webhook payload for parser analysis
|
|
||||||
import json as _json
|
|
||||||
try:
|
|
||||||
with open('/tmp/proxmenux_webhook_payload.log', 'a') as _f:
|
|
||||||
_f.write(f"\n{'='*80}\n{time.strftime('%Y-%m-%d %H:%M:%S')}\n")
|
|
||||||
_f.write(_json.dumps(payload, indent=2, default=str, ensure_ascii=False))
|
|
||||||
_f.write('\n')
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
result = notification_manager.process_webhook(payload)
|
result = notification_manager.process_webhook(payload)
|
||||||
# Always return 200 to PVE -- a non-200 makes PVE report the webhook as broken.
|
# Always return 200 to PVE -- a non-200 makes PVE report the webhook as broken.
|
||||||
# The 'accepted' field in the JSON body indicates actual processing status.
|
# The 'accepted' field in the JSON body indicates actual processing status.
|
||||||
|
|||||||
@@ -571,11 +571,9 @@ class TaskWatcher:
|
|||||||
self._thread: Optional[threading.Thread] = None
|
self._thread: Optional[threading.Thread] = None
|
||||||
self._hostname = _hostname()
|
self._hostname = _hostname()
|
||||||
self._last_position = 0
|
self._last_position = 0
|
||||||
# Track active vzdump jobs. While a vzdump is running, VM/CT
|
# Cache for active vzdump detection
|
||||||
# start/stop/shutdown events are backup-induced (mode=stop/snapshot)
|
self._vzdump_active_cache: float = 0 # timestamp of last positive check
|
||||||
# and should NOT generate notifications.
|
self._vzdump_cache_ttl = 5 # cache result for 5s
|
||||||
self._active_vzdump_ts: float = 0 # timestamp of last vzdump start
|
|
||||||
self._VZDUMP_WINDOW = 14400 # 4h max backup window
|
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
if self._running:
|
if self._running:
|
||||||
@@ -596,6 +594,29 @@ class TaskWatcher:
|
|||||||
def stop(self):
|
def stop(self):
|
||||||
self._running = False
|
self._running = False
|
||||||
|
|
||||||
|
def _is_vzdump_active(self) -> bool:
|
||||||
|
"""Check if a vzdump (backup) job is currently running.
|
||||||
|
|
||||||
|
Reads /var/log/pve/tasks/active which lists all running PVE tasks.
|
||||||
|
Result is cached for a few seconds to avoid excessive file reads.
|
||||||
|
"""
|
||||||
|
now = time.time()
|
||||||
|
if now - self._vzdump_active_cache < self._vzdump_cache_ttl:
|
||||||
|
return True # Recently confirmed active
|
||||||
|
|
||||||
|
active_file = '/var/log/pve/tasks/active'
|
||||||
|
try:
|
||||||
|
with open(active_file, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
# UPID format: UPID:node:pid:pstart:starttime:type:id:user:
|
||||||
|
if ':vzdump:' in line:
|
||||||
|
self._vzdump_active_cache = now
|
||||||
|
return True
|
||||||
|
except (OSError, IOError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def _watch_loop(self):
|
def _watch_loop(self):
|
||||||
"""Poll the task index file for new entries."""
|
"""Poll the task index file for new entries."""
|
||||||
while self._running:
|
while self._running:
|
||||||
@@ -656,22 +677,7 @@ class TaskWatcher:
|
|||||||
|
|
||||||
event_type, default_severity = event_info
|
event_type, default_severity = event_info
|
||||||
|
|
||||||
# ── Track active vzdump jobs ──
|
|
||||||
# When a vzdump starts, record its timestamp. While active, we
|
|
||||||
# suppress start/stop/shutdown of individual VMs -- those are just
|
|
||||||
# the backup stopping and restarting guests (mode=stop).
|
|
||||||
if task_type == 'vzdump':
|
|
||||||
if not status:
|
|
||||||
# vzdump just started
|
|
||||||
self._active_vzdump_ts = time.time()
|
|
||||||
else:
|
|
||||||
# vzdump finished -- clear after a small grace period
|
|
||||||
# (VMs may still be restarting)
|
|
||||||
def _clear_vzdump():
|
|
||||||
time.sleep(30)
|
|
||||||
self._active_vzdump_ts = 0
|
|
||||||
threading.Thread(target=_clear_vzdump, daemon=True,
|
|
||||||
name='clear-vzdump').start()
|
|
||||||
|
|
||||||
# Check if task failed
|
# Check if task failed
|
||||||
is_error = status and status != 'OK' and status != ''
|
is_error = status and status != 'OK' and status != ''
|
||||||
@@ -721,10 +727,8 @@ class TaskWatcher:
|
|||||||
# Exception: if a VM/CT FAILS to start after backup, that IS important.
|
# Exception: if a VM/CT FAILS to start after backup, that IS important.
|
||||||
_BACKUP_NOISE = {'vm_start', 'vm_stop', 'vm_shutdown', 'vm_restart',
|
_BACKUP_NOISE = {'vm_start', 'vm_stop', 'vm_shutdown', 'vm_restart',
|
||||||
'ct_start', 'ct_stop'}
|
'ct_start', 'ct_stop'}
|
||||||
vzdump_age = time.time() - self._active_vzdump_ts if self._active_vzdump_ts else float('inf')
|
if event_type in _BACKUP_NOISE and not is_error:
|
||||||
if event_type in _BACKUP_NOISE and vzdump_age < self._VZDUMP_WINDOW:
|
if self._is_vzdump_active():
|
||||||
# Allow through only if it's a FAILURE (e.g. VM failed to start)
|
|
||||||
if not is_error:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
self._queue.put(NotificationEvent(
|
self._queue.put(NotificationEvent(
|
||||||
|
|||||||
@@ -49,28 +49,38 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
|||||||
break
|
break
|
||||||
|
|
||||||
if header_idx >= 0:
|
if header_idx >= 0:
|
||||||
for line in lines[header_idx + 1:]:
|
# Use column positions from the header to slice each row.
|
||||||
stripped = line.strip()
|
# Header: "VMID Name Status Time Size Filename"
|
||||||
if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='):
|
header = lines[header_idx]
|
||||||
break
|
col_starts = []
|
||||||
m = re.match(
|
for col_name in ['VMID', 'Name', 'Status', 'Time', 'Size', 'Filename']:
|
||||||
r'\s*(\d+)\s+' # VMID
|
idx = header.find(col_name)
|
||||||
r'(\S+)\s+' # Name
|
if idx >= 0:
|
||||||
r'(\S+)\s+' # Status (ok/error)
|
col_starts.append(idx)
|
||||||
r'(\S+)\s+' # Time
|
|
||||||
r'([\d.]+\s+\S+)\s+' # Size (e.g. "1.423 GiB")
|
if len(col_starts) == 6:
|
||||||
r'(\S+)', # Filename
|
for line in lines[header_idx + 1:]:
|
||||||
line
|
stripped = line.strip()
|
||||||
)
|
if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='):
|
||||||
if m:
|
break
|
||||||
vms.append({
|
# Pad line to avoid index errors
|
||||||
'vmid': m.group(1),
|
padded = line.ljust(col_starts[-1] + 50)
|
||||||
'name': m.group(2),
|
vmid = padded[col_starts[0]:col_starts[1]].strip()
|
||||||
'status': m.group(3),
|
name = padded[col_starts[1]:col_starts[2]].strip()
|
||||||
'time': m.group(4),
|
status = padded[col_starts[2]:col_starts[3]].strip()
|
||||||
'size': m.group(5),
|
time_val = padded[col_starts[3]:col_starts[4]].strip()
|
||||||
'filename': m.group(6).split('/')[-1],
|
size = padded[col_starts[4]:col_starts[5]].strip()
|
||||||
})
|
filename = padded[col_starts[5]:].strip()
|
||||||
|
|
||||||
|
if vmid and vmid.isdigit():
|
||||||
|
vms.append({
|
||||||
|
'vmid': vmid,
|
||||||
|
'name': name,
|
||||||
|
'status': status,
|
||||||
|
'time': time_val,
|
||||||
|
'size': size,
|
||||||
|
'filename': filename,
|
||||||
|
})
|
||||||
|
|
||||||
# ── Strategy 2: log-style (PBS / Proxmox Backup Server) ──
|
# ── Strategy 2: log-style (PBS / Proxmox Backup Server) ──
|
||||||
# Parse from the full vzdump log lines.
|
# Parse from the full vzdump log lines.
|
||||||
|
|||||||
Reference in New Issue
Block a user