fix(monitor): reconcile backup outcome reports with maintainer review

This commit is contained in:
martino
2026-10-01 08:38:32 +02:00
parent dcf00cc6b9
commit 5faf55a746
6 changed files with 180 additions and 24 deletions
+12 -6
View File
@@ -4296,9 +4296,6 @@ class ProxmoxHookWatcher:
if severity in ('error', 'err', 'critical') or re.search(
r'(?im)^\s*(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
return 'failed'
if severity not in ('info', 'ok', 'success') or re.search(
r'(?im)(?:^\s*WARNING:|\bWARNINGS\s*:\s*\d+)', text):
return 'unconfirmed'
starts = re.findall(r'(?im)\bStarting Backup of VM (\d+)\s*\(', text)
finished = re.findall(r'(?im)\bFinished Backup of VM (\d+)\s*\(', text)
lines = text.splitlines()
@@ -4321,15 +4318,24 @@ class ProxmoxHookWatcher:
if not re.match(r'\s*\d+\s+', line):
break
status = line[status_start:status_end].strip().upper()
if status == 'ERROR':
if status in ('ERROR', 'ERR'):
return 'failed'
rows.append(status)
break
if table_outcome == 'unconfirmed':
if severity not in ('info', 'ok', 'success') or re.search(
r'(?im)(?:^\s*WARNING:|\bWARNINGS\s*:\s*\d+)', text):
return 'unconfirmed'
# A present table is authoritative: do not certify an incomplete table
# from a finished guest log, or reject a complete OK table merely
# because the extra diagnostic log was truncated before its finishes.
if table_outcome is not None:
return table_outcome
if any(re.match(r'\s*VMID\s+Name\s+Status\b', line, re.IGNORECASE)
for line in lines):
return 'unconfirmed'
if starts:
return 'confirmed' if sorted(starts) == sorted(finished) else 'unconfirmed'
if table_outcome == 'confirmed' or re.search(
if re.search(
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
return 'confirmed'
return 'unconfirmed'
+27 -2
View File
@@ -252,6 +252,8 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
vmid = padded[col_starts[0]:col_starts[1]].strip()
name = padded[col_starts[1]:col_starts[2]].strip()
status = padded[col_starts[2]:col_starts[3]].strip()
if status.lower() in ('err', 'error'):
status = 'error'
time_val = padded[col_starts[3]:col_starts[4]].strip()
size = padded[col_starts[4]:col_starts[5]].strip()
filename = padded[col_starts[5]:].strip()
@@ -785,7 +787,7 @@ TEMPLATES = {
# without a trailing dash.
'title': '{hostname}: No longer reported - {category}{entity_suffix}',
'body': 'The {category} issue is no longer in active health records.\n{reason}\n\U0001F6A6 Previous severity: {original_severity}\n\u23F1\uFE0F Time since first observation: {duration}',
'label': 'Health issue no longer reported',
'label': 'Recovery notification',
'group': 'health',
'default_enabled': True,
},
@@ -1003,7 +1005,7 @@ TEMPLATES = {
'backup_complete': {
'title': '{hostname}: Backup outcome unconfirmed',
'body': 'The backup outcome could not be confirmed from this notice.',
'label': 'Backup report',
'label': 'Backup complete',
'group': 'backup',
'default_enabled': True,
},
@@ -1854,13 +1856,35 @@ def render_template(event_type: str, data: Dict[str, Any],
_catalog_value(requested_catalog, key)
or _catalog_value(english_catalog, key)
)
# A catalog without the outcome keys predates this report contract.
# Keep its Settings labels, but do not render old recovery, restore
# or backup success claims (e.g. the exact upstream Slovak catalog).
if (event_type in ('backup_complete', 'error_resolved', 'system_restore_completed')
and field in ('title', 'body')
and not _catalog_value(requested_catalog, 'backup.unconfirmedBody')):
localized = _catalog_value(english_catalog, key)
if localized:
template[field] = localized
backup_title_target = ''
if event_type == 'backup_complete':
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
template['title'] = runtime_message('backup.confirmedTitle', language,
hostname=data.get('hostname') or _get_hostname())
parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
guests = (parsed_backup or {}).get('vms') or []
target = []
if storage:
target.append(storage)
if len(guests) == 1:
guest = guests[0]
kind = 'VM' if guest.get('type') == 'qemu' else 'CT' if guest.get('type') == 'lxc' else 'VM/CT'
name = guest.get('name') or kind
target.append(f"{kind} {name} ({guest['vmid']})" if name != kind
else f"{kind} {guest['vmid']}")
if target:
backup_title_target = ' — ' + ' · '.join(target)
template['body'] = runtime_message('backup.confirmedBody', language)
elif outcome == 'failed':
template['title'] = runtime_message('backup.errorTitle', language,
@@ -2003,6 +2027,7 @@ def render_template(event_type: str, data: Dict[str, Any],
title = template['title'].format_map(safe_vars)
except (ValueError, IndexError):
title = template['title']
title += backup_title_target
# ── PVE vzdump special formatting ──
# When the event came from PVE webhook with a full vzdump message,
@@ -50,6 +50,10 @@ class RuntimeCatalogTests(unittest.TestCase):
self.assertIsInstance(templates[event_type][field], str)
self.assertTrue(templates[event_type][field])
if field in source:
# Upstream Slovak backup title/body still belong to
# the pre-outcome schema; runtime falls back to EN.
if language == 'sk' and event_type == 'backup_complete' and field != 'label':
continue
self.assertEqual(
_placeholders(templates[event_type][field]),
_placeholders(source[field]),
@@ -68,10 +72,17 @@ class RuntimeCatalogTests(unittest.TestCase):
return result
en = flatten(self.catalogs["en"])
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
"channels.email.severity.observation", "channels.email.status.unconfirmed"}
for language, catalog in self.catalogs.items():
translated = flatten(catalog)
self.assertEqual(set(translated), set(en), language)
for key in en:
expected = set(en) - pending_slovak if language == 'sk' else set(en)
self.assertEqual(set(translated), expected, language)
for key in expected:
if language == 'sk' and key in ('templates.backup_complete.title',
'templates.backup_complete.body'):
continue # exact upstream SK, superseded only at render time
self.assertEqual(_placeholders(translated[key]), _placeholders(en[key]), f"{language}:{key}")
def test_notification_language_ui_keys_exist_in_both_catalogs(self):
@@ -323,7 +334,9 @@ class RuntimeCatalogTests(unittest.TestCase):
},
language="sk",
)
self.assertIn("záloha dokončená", backup["title"])
self.assertIn("Backup complete", backup["title"])
self.assertIn("pbs-main", backup["title"])
self.assertIn("VM alpha (100)", backup["title"])
self.assertNotIn("Backup job finished", backup["title"])
self.assertIn("Veľkosť: 1.5 GiB", backup["body"])
self.assertIn("Trvanie: 00:00:10", backup["body"])