fix(monitor): distinguish notification recovery and backup outcomes

This commit is contained in:
martino
2026-10-01 08:38:32 +02:00
parent d137bb3d31
commit dcf00cc6b9
13 changed files with 616 additions and 77 deletions
@@ -0,0 +1,391 @@
"""Inert producer-to-renderer checks for notification outcome claims."""
import ast
import copy
import json
import re
import time
import sys
import types
import unittest
from pathlib import Path
from typing import Any
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[3]
SCRIPTS = ROOT / 'AppImage/scripts'
CATALOG = ROOT / 'AppImage/messages/en/common.json'
EXPECTED = {
'error_resolved': {
'title': '{hostname}: No longer reported - {category}{entity_suffix}',
'body': 'The {category} issue is no longer in active health records.\n{reason}\n🚦 Previous severity: {original_severity}\n⏱️ Time since first observation: {duration}',
'label': 'Health issue no longer reported',
},
'system_restore_completed': {
'body': 'Post-restore tasks completed in background.\n\nGuests applied: {guests}\nBind-mount stubs: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}',
},
}
def extract(path, name, owner=None, namespace=None):
tree = ast.parse(path.read_text())
nodes = tree.body
if owner:
nodes = next(n.body for n in nodes if isinstance(n, ast.ClassDef) and n.name == owner)
node = next(n for n in nodes if isinstance(n, ast.FunctionDef) and n.name == name)
node.decorator_list = []
ns = namespace if namespace is not None else {}
exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), ns)
return ns[name]
def renderer(catalog, translated=None):
path = SCRIPTS / 'notification_templates.py'
tree = ast.parse(path.read_text())
templates = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'TEMPLATES' for t in n.targets)))
def lookup(obj, key):
for part in key.split('.'):
obj = obj.get(part) if isinstance(obj, dict) else None
return obj
def message(key, language='en', **values):
source = translated if language == 'it' and translated is not None else catalog
namespace = source['runtime']['notifications']
value = lookup(namespace, key) or lookup(catalog['runtime']['notifications'], key) or ''
return value.format_map(type('Safe', (dict,), {'__missing__': lambda self, k: ''})(values))
ns = {'TEMPLATES': templates, 'Dict': dict, 'Any': Any, 'time': time, 're': re,
'_get_hostname': lambda: 'node-a',
'_load_runtime_catalog': lambda lang: (translated if lang == 'it' and translated is not None
else catalog)['runtime']['notifications'],
'_catalog_value': lookup, 'runtime_message': message}
from typing import Optional
ns['Optional'] = Optional
extract(path, '_parse_vzdump_message', namespace=ns)
extract(path, '_format_vzdump_body', namespace=ns)
return templates, extract(path, 'render_template', namespace=ns)
class OutcomeWording(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.catalog = json.loads(CATALOG.read_text())
cls.templates, render = renderer(cls.catalog)
cls.render = staticmethod(render)
def test_exact_four_english_leaves_match_source_and_catalog(self):
for event, fields in EXPECTED.items():
for field, value in fields.items():
with self.subTest(event=event, field=field):
self.assertEqual(self.templates[event][field], value)
self.assertEqual(self.catalog['runtime']['notifications']['templates'][event][field], value)
def test_webhook_backup_outcome_is_evidence_based_without_rerouting(self):
path = SCRIPTS / 'notification_events.py'
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
ns['NotificationEvent'] = Event
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
class Receiver:
_hostname = 'node-a'
_classify_pve = classify
_map_severity = staticmethod(severity_map)
_backup_outcome = staticmethod(backup_outcome)
def __init__(self): self._queue = Queue()
header = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('VMID','Name','Status','Time','Size','Filename')
row_ok = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('104','alpha','OK','00:01:00','1.5 GiB','archive')
row_warning = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','WARNINGS','00:01:00','1.5 GiB','archive')
row_error = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','ERROR','00:01:00','1.5 GiB','archive')
cases = [
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
('vzdump', 'info', header+'\n'+row_error, 'failed'),
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'confirmed'),
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00', 'unconfirmed'),
('vzdump', 'info', header+'\n'+row_ok[:30], 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'confirmed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nWARNING: skipped file', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: TASK OK\n104 alpha WARNINGS: 1', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Finished Backup of VM 105 (00:01:00)', 'confirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed for VM 104', 'failed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'failed'),
('', 'warning', 'Backup scheduled', 'unconfirmed'),
('', 'info', 'Backup complete', 'unconfirmed'),
]
for kind, severity, message, expected in cases:
with self.subTest(message=message, severity=severity):
receiver = Receiver()
receive(receiver, {'fields': {'type': kind}, 'severity': severity,
'title': 'Backup', 'message': message})
event = receiver._queue.items[0]
self.assertEqual(event.event_type, 'backup_complete')
self.assertEqual(event.data['backup_outcome'], expected)
def test_backup_classifier_preserves_confirmed_and_unverified_paths(self):
path = SCRIPTS / 'notification_events.py'
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
ns['NotificationEvent'] = Event
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
class Receiver:
_hostname = 'node-a'
_classify_pve = classify
_map_severity = staticmethod(severity_map)
_backup_outcome = staticmethod(backup_outcome)
def __init__(self): self._queue = Queue()
samples = [('vzdump', 'info', 'Backup', 'job finished'),
('vzdump', 'warning', 'Backup', 'job incomplete'),
('', 'warning', 'backup job', 'Backup scheduled')]
for kind, severity, title, message in samples:
with self.subTest(kind=kind, message=message):
event, entity, _ = classify(None, kind, severity, title, message)
self.assertEqual((event, entity), ('backup_complete', 'vm'))
receiver = Receiver()
reply = receive(receiver, {'fields': {'type': kind}, 'severity': severity,
'title': title, 'message': message})
self.assertEqual(reply['event_type'], event)
self.assertEqual(len(receiver._queue.items), 1)
emitted = receiver._queue.items[0]
self.assertEqual(emitted.severity, 'WARNING' if severity == 'warning' else 'INFO')
output = self.render(emitted.event_type, emitted.data, 'en')
self.assertIn('Backup outcome unconfirmed', output['title'])
self.assertEqual(output['body'].splitlines()[-1], message) # raw PVE body retained
self.assertEqual(classify(None, 'vzdump', 'error', 'Backup', 'failed')[0], 'backup_fail')
def test_incomplete_vzdump_log_does_not_certify_a_guest(self):
from typing import Dict, Optional
ns = {'re': re, 'Dict': Dict, 'Optional': Optional, 'Any': Any,
'runtime_message': lambda key, lang, **kw: key}
parser = extract(SCRIPTS / 'notification_templates.py', '_parse_vzdump_message', namespace=ns)
formatter = extract(SCRIPTS / 'notification_templates.py', '_format_vzdump_body', namespace=ns)
incomplete = parser('INFO: Starting Backup of VM 104 (qemu)')
self.assertEqual(incomplete['vms'][0]['status'], 'unknown')
self.assertNotIn('✅', formatter(incomplete, False, 'en'))
mixed = parser('INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Starting Backup of VM 105 (lxc)')
self.assertEqual([vm['status'] for vm in mixed['vms']], ['ok', 'unknown'])
formatted = formatter(mixed, False, 'en')
self.assertEqual(formatted.count('✅'), 1)
self.assertNotIn('❌', formatted)
conflicting = parser('INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed\nINFO: Finished Backup of VM 104 (00:01:00)')
self.assertEqual(conflicting['vms'][0]['status'], 'error')
def test_backup_render_preserves_success_and_marks_unconfirmed_and_failure(self):
for outcome, message, title_part, body_part in (
('confirmed', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'Backup complete', '✅'),
('unconfirmed', 'INFO: Starting Backup of VM 104 (qemu)', 'Backup outcome unconfirmed', '❔'),
('failed', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'Backup error reported', '❌'),
):
with self.subTest(outcome=outcome):
output = self.render('backup_complete', {'hostname': 'node-a', 'pve_type': 'vzdump',
'pve_message': message, 'pve_title': 'Backup complete', 'backup_outcome': outcome}, 'en')
self.assertIn(title_part, output['title'])
self.assertIn(body_part, output['body'])
if outcome != 'confirmed': self.assertNotIn('Backup complete', output['title'])
output = self.render('backup_complete', {'hostname': 'node-a', 'vmname': 'vm', 'vmid': '104'}, 'en')
self.assertIn('Backup outcome unconfirmed', output['title'])
self.assertNotIn('successfully', output['body'])
conflict = self.render('backup_complete', {'hostname': 'node-a','backup_outcome':'failed',
'pve_message':'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nERROR: archive write failed'}, 'en')
self.assertIn('ERROR: archive write failed', conflict['body'])
self.assertNotIn('Backup complete', conflict['title'])
def test_html_email_badge_and_backup_status_are_context_specific(self):
import html
path = SCRIPTS / 'notification_channels.py'
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
with self.subTest(lang=lang):
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
def text(key, data=None, **values):
value = catalog['channels']
for part in key.split('.'): value = value[part]
return value.format(**values)
ns = {'Dict': dict, 'Optional': __import__('typing').Optional,
'_runtime_text': text, '_runtime_notification_text': lambda key, data=None: ''}
build = extract(path, '_build_detail_rows', 'EmailChannel', ns)
fmt = extract(path, '_format_html', 'EmailChannel', ns)
class Email:
_SEV_STYLE = {'OK': {'color':'#16a34a','bg':'#f0fdf4','border':'#bbf7d0'},
'CRITICAL': {'color':'#dc2626','bg':'#fef2f2','border':'#fecaca'},
'INFO': {'color':'blue','bg':'white','border':'gray'}}
_SEV_DEFAULT = {'color':'#6b7280','bg':'#f9fafb','border':'#e5e7eb'}
subject_prefix = 'ProxMenux'
_build_detail_rows = staticmethod(build)
badge = catalog['channels']['email']['severity']['observation']
recovery = fmt(Email(), 'No longer reported', 'Body', 'OK', {'_event_type': 'error_resolved',
'_notification_language': lang, '_group': 'health'})
self.assertIn('>' + badge.upper() + '</span>', recovery)
self.assertIn('color:#6b7280;', recovery)
unrelated = fmt(Email(), 'Reconnected', 'Body', 'OK', {'_event_type': 'node_reconnect',
'_notification_language': lang, '_group': 'cluster'})
self.assertIn('>' + catalog['channels']['email']['severity']['ok'].upper() + '</span>', unrelated)
for outcome, status in [('confirmed', 'completed'), ('unconfirmed','unconfirmed'), ('failed','failed')]:
email = fmt(Email(), 'Backup', 'Details', 'INFO', {'_event_type': 'backup_complete',
'backup_outcome': outcome, '_notification_language': lang, '_group': 'backup'})
self.assertIn(catalog['channels']['email']['status'][status], html.unescape(email))
badge_label = catalog['channels']['email']['status'][status].upper()
self.assertIn('>' + badge_label + '</span>', html.unescape(email))
if outcome == 'failed':
self.assertIn('color:#dc2626;font-weight:600;', email)
self.assertIn('background:#fef2f2;', email)
elif outcome == 'unconfirmed':
self.assertIn('background:#f9fafb;', email)
else:
self.assertIn('background:#f0fdf4;', email)
def test_actual_all_locale_rendering_and_fallback_for_backup_outcomes(self):
import importlib.util
module_path = SCRIPTS / 'notification_templates.py'
spec = importlib.util.spec_from_file_location('isolated_notification_templates', module_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # templates and catalogs only; no manager or sends
samples = {
'confirmed': 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)',
'unconfirmed': 'INFO: Starting Backup of VM 104 (qemu)',
'failed': 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed',
}
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
for state, message in samples.items():
with self.subTest(lang=lang, state=state):
data = {'hostname':'node-with-a-long-name','backup_outcome':state,'pve_type':'vzdump',
'pve_message':message, 'pve_title':'Backup complete', '_notification_language':lang}
result = module.render_template('backup_complete', data, lang)
if state != 'unconfirmed':
key = 'confirmedTitle' if state == 'confirmed' else 'errorTitle'
self.assertEqual(result['title'], catalog['backup'][key].format(hostname=data['hostname']))
else: self.assertEqual(result['title'], catalog['templates']['backup_complete']['title'].format(hostname=data['hostname']))
self.assertNotIn('{hostname}', result['title'])
if state == 'unconfirmed': self.assertIn(catalog['backup']['unconfirmedBody'], result['body'])
if state == 'failed': self.assertIn(catalog['backup']['errorBody'], result['body'])
enriched, _ = module.enrich_with_emojis('backup_complete', result['title'], result['body'], data)
self.assertTrue(enriched.startswith({'confirmed':'💾✅','unconfirmed':'💾❔','failed':'💾❌'}[state]))
recovery = module.render_template('error_resolved', {'hostname':'node','category':'temperature',
'reason':'Old observation','duration':'3d','original_severity':'WARNING'}, lang)
self.assertEqual(recovery['title'],catalog['templates']['error_resolved']['title'].format(hostname='node',category='temperature',entity_suffix=''))
self.assertNotIn('resolved', recovery['title'].lower()) if lang == 'en' else None
restore = module.render_template('system_restore_completed', {'hostname':'node', 'guests':4,
'stubs':1,'stale_nodes':2,'components':1,'duration':'2m','warnings_block':'Missing module'},lang)
self.assertIn('Missing module',restore['body'])
self.assertNotIn('fully ready',restore['body'].lower())
def test_stale_record_disappearance_is_not_claimed_recovery(self):
data = {'hostname': 'node-a', 'category': 'temperature', 'reason': 'Temperature observation (no longer reported)',
'original_severity': 'WARNING', 'duration': '2d 0h', 'severity': 'OK'}
output = self.render('error_resolved', data, 'en')
self.assertIn('no longer in active health records', output['body'])
self.assertNotIn('resolved', (output['title'] + output['body']).lower())
self.assertIn('Time since first observation', output['body'])
def test_actual_poller_stale_disappearance_keeps_reason_factual(self):
class Store:
def get_active_errors(self): return []
def is_error_acknowledged(self, key): return False
class Event:
def __init__(self, *args, **kwargs): self.kind, self.severity, self.data = args[:3]
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
ns = {'time': time, 'json': json, 'NotificationEvent': Event, 'Dict': dict}
poll = extract(SCRIPTS / 'notification_events.py', '_check_persistent_health', 'PollingCollector', ns)
class Collector:
_hostname = 'node-a'
_ENTITY_MAP = {'temperature': ('node', '')}
_first_poll_done = True
_known_errors = {'temp': {'category': 'temperature', 'reason': 'Temperature high',
'severity': 'WARNING', 'first_seen': '2026-09-25T00:00:00'}}
_notified_severity = {'temp': 'WARNING'}
_last_notified = {'temp': 1}
_queue = Queue()
def _guest_storage_error_is_now_foreign(self, *a): return False
def _save_known_errors_meta(self): pass
with patch.dict(sys.modules, {'health_persistence': types.SimpleNamespace(health_persistence=Store())}):
poll(Collector())
events = Collector._queue.items
self.assertEqual(len(events), 1)
self.assertEqual((events[0].kind, events[0].severity), ('error_resolved', 'OK'))
self.assertEqual(events[0].data['reason'], 'Temperature high (no longer reported)')
rendered = self.render(events[0].kind, events[0].data, 'en')
self.assertNotIn('recovered', rendered['body'].lower())
def test_warning_and_clean_restore_keep_only_reported_outcome(self):
for warnings in ('', '⚠️ Boot sanity: missing modules\n'):
with self.subTest(warnings=warnings):
events = []
ns = {'request': types.SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
'hostname': 'node-a', 'guests': '2', 'stubs': '0', 'stale_nodes': '0',
'components': 'none', 'duration': '2m',
'warnings': 'missing modules' if warnings else ''}),
'notification_manager': types.SimpleNamespace(emit_event=lambda **kw: events.append(kw)),
'jsonify': lambda obj: obj}
handler = extract(SCRIPTS / 'flask_notification_routes.py', 'internal_restore_event', namespace=ns)
response, status = handler()
self.assertEqual((status, response['event_type']), (200, 'system_restore_completed'))
event = events[0]
self.assertEqual(event['severity'], 'WARNING' if warnings else 'INFO')
result = self.render(event['event_type'], event['data'], 'en')
self.assertIn('Post-restore tasks completed', result['body'])
self.assertNotIn('fully ready', result['body'])
if warnings: self.assertIn('missing modules', result['body'])
def test_missing_key_fallback_and_synthetic_translation(self):
catalog = copy.deepcopy(self.catalog)
translated = copy.deepcopy(self.catalog)
for event, fields in EXPECTED.items():
for field in fields: translated['runtime']['notifications']['templates'][event].pop(field)
_, render = renderer(catalog, translated)
for event, fields in EXPECTED.items():
for field in fields:
if field not in ('title', 'body'):
continue
self.assertEqual(render(event, {'category': 'disk'}, 'it')[field],
render(event, {'category': 'disk'}, 'en')[field])
translated['runtime']['notifications']['templates']['error_resolved']['title'] = 'Synthetic observation: {category}'
_, render = renderer(catalog, translated)
self.assertEqual(render('error_resolved', {'category': 'disk'}, 'it')['title'], 'Synthetic observation: disk')
def test_rich_backup_icon_tracks_outcome_and_digest_default_is_neutral(self):
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
def assign(name):
return ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == name for t in n.targets)))
icons = assign('EVENT_EMOJI')
self.assertNotIn('✅', icons['backup_complete'])
ns = {'TEMPLATES': assign('TEMPLATES'), 'EVENT_EMOJI': icons,
'Dict': dict, 'Any': Any,
'CATEGORY_EMOJI': assign('CATEGORY_EMOJI'), 'SEVERITY_ICONS': assign('SEVERITY_ICONS'),
'FIELD_EMOJI': assign('FIELD_EMOJI'), '_localized_template_labels': lambda *a: {},
'_lxc_update_label_icons': lambda *a: {}}
enrich = extract(SCRIPTS / 'notification_templates.py', 'enrich_with_emojis', namespace=ns)
for state, icon in [('confirmed', '💾✅'), ('unconfirmed', '💾❔'), ('failed', '💾❌')]:
with self.subTest(state=state):
title, body = enrich('backup_complete', 'node-a: Backup', 'Backup report',
{'backup_outcome': state, '_notification_language': 'en'})
self.assertTrue(title.startswith(icon), title)
def test_uncertain_outcomes_have_no_blanket_success_icon(self):
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
icon_map = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == 'EVENT_EMOJI' for t in n.targets)))
for event in ('error_resolved', 'system_restore_completed'):
self.assertNotIn('✅', icon_map[event], event)
self.assertNotIn('✅', icon_map['backup_complete']) # buffered digest has no outcome metadata
if __name__ == '__main__': unittest.main()
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Neues Gesundheitsproblem" "label": "Neues Gesundheitsproblem"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname}: Gelöst – {category}{entity_suffix}", "title": "{hostname}: Nicht mehr gemeldet – {category}{entity_suffix}",
"body": "Das Problem {category} wurde behoben.\n{reason}\n🚦 Vorheriger Schweregrad: {original_severity}\n⏱️ Dauer: {duration}", "body": "Das Problem in der Kategorie {category} ist nicht mehr in den aktiven Zustandsmeldungen enthalten.\n{reason}\n🚦 Vorheriger Schweregrad: {original_severity}\n⏱️ Zeit seit der ersten Meldung: {duration}",
"label": "Wiederherstellungsbenachrichtigung" "label": "Wiederherstellungsbenachrichtigung"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Sicherung gestartet" "label": "Sicherung gestartet"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage}: Sicherung abgeschlossen – {vmname} ({vmid})", "title": "{hostname}: Backup-Ergebnis nicht bestätigt",
"body": "Die Sicherung von {vmname} (ID: {vmid}) wurde am {storage} erfolgreich abgeschlossen.\nGröße: {size}", "body": "Das Backup-Ergebnis lässt sich anhand dieser Meldung nicht bestätigen.",
"label": "Sicherung abgeschlossen" "label": "Sicherung abgeschlossen"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname}: Host-Wiederherstellung abgeschlossen", "title": "{hostname}: Host-Wiederherstellung abgeschlossen",
"body": "Aufgaben nach der Wiederherstellung im Hintergrund ausgeführt.\n\nGäste haben sich beworben: {guests}\nBind-Mount-Stubs: {stubs}\nVeraltete Knotenverzeichnisse entfernt: {stale_nodes}\nKomponenten neu installiert: {components}\nDauer: {duration}\n{warnings_block}\nDer Knoten ist nun vollständig einsatzbereit.", "body": "Aufgaben nach der Wiederherstellung im Hintergrund abgeschlossen.\n\nÜbernommene Gäste: {guests}\nBind-Mount-Platzhalter: {stubs}\nEntfernte veraltete Knotenverzeichnisse: {stale_nodes}\nNeu installierte Komponenten: {components}\nDauer: {duration}\n{warnings_block}",
"label": "Host-Wiederherstellung abgeschlossen" "label": "Host-Wiederherstellung abgeschlossen"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Warnung", "warning": "Warnung",
"info": "Informationen", "info": "Informationen",
"ok": "Gelöst", "ok": "Gelöst",
"default": "Hinweis" "default": "Hinweis",
"observation": "Nicht mehr gemeldet"
}, },
"groups": { "groups": {
"vm_ct": "Virtuelle Maschine / Container", "vm_ct": "Virtuelle Maschine / Container",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Fehlgeschlagen", "failed": "Fehlgeschlagen",
"completed": "Abgeschlossen", "completed": "Abgeschlossen",
"started": "Gestartet" "started": "Gestartet",
"unconfirmed": "Nicht bestätigt"
}, },
"report": "{group} Bericht", "report": "{group} Bericht",
"details": "Details", "details": "Details",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}." "sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}."
},
"backup": {
"confirmedTitle": "{hostname}: Backup abgeschlossen",
"confirmedBody": "Backup erfolgreich abgeschlossen.",
"errorTitle": "{hostname}: Backup-Fehler gemeldet",
"errorBody": "Der Backup-Bericht enthält einen Fehler.",
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt."
} }
} }
} }
File diff suppressed because one or more lines are too long
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Nuevo problema de salud" "label": "Nuevo problema de salud"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname}: Resuelto - {category}{entity_suffix}", "title": "{hostname}: Ya no se informa de {category}{entity_suffix}",
"body": "El problema {category} se ha resuelto.\n{reason}\n🚦 Gravedad anterior: {original_severity}\n⏱️ Duración: {duration}", "body": "El problema de {category} ya no figura entre las incidencias de salud activas.\n{reason}\n🚦 Gravedad anterior: {original_severity}\n⏱️ Tiempo desde la primera observación: {duration}",
"label": "Notificación de recuperación" "label": "Notificación de recuperación"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Backup iniciado" "label": "Backup iniciado"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage}: Backup completado — {vmname} ({vmid})", "title": "{hostname}: resultado del backup sin confirmar",
"body": "El backup de {vmname} (ID: {vmid}) se ha completado correctamente en {storage}.\nTamaño: {size}", "body": "Esta notificación no permite confirmar el resultado del backup.",
"label": "Backup completado" "label": "Backup completado"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname}: restauración del host finalizada", "title": "{hostname}: restauración del host finalizada",
"body": "Tareas posteriores a la restauración completadas en segundo plano.\n\nGuests aplicados: {guests}\nStubs de bind mount: {stubs}\nDirectorios de nodos obsoletos eliminados: {stale_nodes}\nComponentes reinstalados: {components}\nDuración: {duration}\n{warnings_block}\nEl nodo está listo para usarse.", "body": "Tareas posteriores a la restauración completadas en segundo plano.\n\nConfiguraciones de invitados aplicadas: {guests}\nDirectorios auxiliares de montajes bind: {stubs}\nDirectorios de nodos obsoletos eliminados: {stale_nodes}\nComponentes reinstalados: {components}\nDuración: {duration}\n{warnings_block}",
"label": "Restauración del host completada" "label": "Restauración del host completada"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Advertencia", "warning": "Advertencia",
"info": "Información", "info": "Información",
"ok": "Resuelto", "ok": "Resuelto",
"default": "Aviso" "default": "Aviso",
"observation": "Ya no se informa"
}, },
"groups": { "groups": {
"vm_ct": "Máquina virtual / Contenedor", "vm_ct": "Máquina virtual / Contenedor",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Fallido", "failed": "Fallido",
"completed": "Completado", "completed": "Completado",
"started": "Iniciado" "started": "Iniciado",
"unconfirmed": "Sin confirmar"
}, },
"report": "Informe de {group}", "report": "Informe de {group}",
"details": "Detalles", "details": "Detalles",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "Lecturas altas registradas a lo largo de {duration}." "sampleSpan": "Lecturas altas registradas a lo largo de {duration}."
},
"backup": {
"confirmedTitle": "{hostname}: backup completado",
"confirmedBody": "Backup completado correctamente.",
"errorTitle": "{hostname}: error notificado en el backup",
"errorBody": "El informe del backup contiene un error.",
"unconfirmedBody": "El resultado del backup no está confirmado."
} }
} }
} }
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Nouveau problème de santé" "label": "Nouveau problème de santé"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname} : Résolu - {category}{entity_suffix}", "title": "{hostname} : Plus signalé – {category}{entity_suffix}",
"body": "Le problème {category} a été résolu.\n{reason}\n🚦 Gravité précédente : {original_severity}\n⏱️ Durée : {duration}", "body": "Le problème {category} ne figure plus parmi les alertes de santé actives.\n{reason}\n🚦 Gravité précédente : {original_severity}\n⏱️ Temps depuis la première observation : {duration}",
"label": "Notification de récupération" "label": "Notification de récupération"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Sauvegarde démarrée" "label": "Sauvegarde démarrée"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage} : Sauvegarde terminée — {vmname} ({vmid})", "title": "{hostname} : résultat de la sauvegarde non confirmé",
"body": "La sauvegarde de {vmname} (ID : {vmid}) s'est terminée avec succès le {storage}.\nTaille : {size}", "body": "Cette notification ne permet pas de confirmer le résultat de la sauvegarde.",
"label": "Sauvegarde terminée" "label": "Sauvegarde terminée"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname} : restauration de l'hôte terminée", "title": "{hostname} : restauration de l'hôte terminée",
"body": "Tâches post-restauration effectuées en arrière-plan.\n\nInvités postulés : {guests}\nTalons de montage liés : {stubs}\nRépertoires de nœuds obsolètes supprimés : {stale_nodes}\nComposants réinstallés : {components}\nDurée : {duration}\n{warnings_block}\nLe nœud est maintenant entièrement prêt à être utilisé.", "body": "Tâches après restauration terminées en arrière-plan.\n\nInvités appliqués : {guests}\nRépertoires de support des montages bind : {stubs}\nRépertoires de nœuds obsolètes supprimés : {stale_nodes}\nComposants réinstallés : {components}\nDurée : {duration}\n{warnings_block}",
"label": "Restauration de l'hôte terminée" "label": "Restauration de l'hôte terminée"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Avertissement", "warning": "Avertissement",
"info": "Informations", "info": "Informations",
"ok": "Résolu", "ok": "Résolu",
"default": "Avis" "default": "Avis",
"observation": "Plus signalé"
}, },
"groups": { "groups": {
"vm_ct": "Machine Virtuelle / Conteneur", "vm_ct": "Machine Virtuelle / Conteneur",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Échec", "failed": "Échec",
"completed": "Terminé", "completed": "Terminé",
"started": "Commencé" "started": "Commencé",
"unconfirmed": "Non confirmé"
}, },
"report": "Rapport {group}", "report": "Rapport {group}",
"details": "Détails", "details": "Détails",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "Les relevés élevés s'étendent sur {duration}." "sampleSpan": "Les relevés élevés s'étendent sur {duration}."
},
"backup": {
"confirmedTitle": "{hostname} : sauvegarde terminée",
"confirmedBody": "Sauvegarde terminée avec succès.",
"errorTitle": "{hostname} : erreur signalée lors de la sauvegarde",
"errorBody": "Le rapport de sauvegarde contient une erreur.",
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé."
} }
} }
} }
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Nuovo problema sanitario" "label": "Nuovo problema sanitario"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname}: risolto - {category}{entity_suffix}", "title": "{hostname}: segnalazione non più attiva - {category}{entity_suffix}",
"body": "Il problema {category} è stato risolto.\n{reason}\n🚦 Gravità precedente: {original_severity}\n⏱️ Durata: {duration}", "body": "Il problema relativo a {category} non figura più tra le segnalazioni di salute attive.\n{reason}\n🚦 Gravità precedente: {original_severity}\n⏱️ Tempo dalla prima segnalazione: {duration}",
"label": "Notifica di recupero" "label": "Notifica di recupero"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Backup avviato" "label": "Backup avviato"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage}: Backup completato — {vmname} ({vmid})", "title": "{hostname}: esito del backup non confermato",
"body": "Backup di {vmname} (ID: {vmid}) completato con successo su {storage}.\nTaglia: {size}", "body": "Questa notifica non consente di confermare l’esito del backup.",
"label": "Backup completato" "label": "Backup completato"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname}: ripristino dell'host terminato", "title": "{hostname}: ripristino dell'host terminato",
"body": "Attività post-ripristino completate in background.\n\nGli ospiti hanno presentato domanda: {guests}\nStub con montaggio tramite collegamento: {stubs}\nDirectory dei nodi obsolete rimosse: {stale_nodes}\nComponenti reinstallati: {components}\nDurata: {duration}\n{warnings_block}\nIl nodo è ora completamente pronto per l'uso.", "body": "Attività post-ripristino completate in background.\n\nConfigurazioni guest copiate: {guests}\nDirectory di supporto per montaggi bind create: {stubs}\nDirectory obsolete dei nodi rimosse: {stale_nodes}\nComponenti reinstallati: {components}\nDurata: {duration}\n{warnings_block}",
"label": "Ripristino dell'host completato" "label": "Ripristino dell'host completato"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Avvertimento", "warning": "Avvertimento",
"info": "Informazioni", "info": "Informazioni",
"ok": "Risolto", "ok": "Risolto",
"default": "Avviso" "default": "Avviso",
"observation": "Non più segnalato"
}, },
"groups": { "groups": {
"vm_ct": "Macchina virtuale/Contenitore", "vm_ct": "Macchina virtuale/Contenitore",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Fallito", "failed": "Fallito",
"completed": "Completato", "completed": "Completato",
"started": "Iniziato" "started": "Iniziato",
"unconfirmed": "Non confermato"
}, },
"report": "{group} Rapporto", "report": "{group} Rapporto",
"details": "Dettagli", "details": "Dettagli",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "Intervallo dei campioni sopra soglia: {duration}." "sampleSpan": "Intervallo dei campioni sopra soglia: {duration}."
},
"backup": {
"confirmedTitle": "{hostname}: backup completato",
"confirmedBody": "Backup completato correttamente.",
"errorTitle": "{hostname}: errore segnalato nel backup",
"errorBody": "Il rapporto del backup contiene un errore.",
"unconfirmedBody": "L’esito del backup non è confermato."
} }
} }
} }
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Novo problema de saúde" "label": "Novo problema de saúde"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname}: Resolvido - {category}{entity_suffix}", "title": "{hostname}: Já não comunicado – {category}{entity_suffix}",
"body": "O problema {category} foi resolvido.\n{reason}\n🚦 Gravidade anterior: {original_severity}\n⏱️ Duração: {duration}", "body": "O problema de {category} já não consta dos registos de saúde ativos.\n{reason}\n🚦 Gravidade anterior: {original_severity}\n⏱️ Tempo desde a primeira observação: {duration}",
"label": "Notificação de recuperação" "label": "Notificação de recuperação"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Backup iniciado" "label": "Backup iniciado"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage}: Backup concluído — {vmname} ({vmid})", "title": "{hostname}: resultado do backup não confirmado",
"body": "Backup de {vmname} (ID: {vmid}) concluído com sucesso em {storage}.\nTamanho: {size}", "body": "Esta notificação não permite confirmar o resultado do backup.",
"label": "Backup concluído" "label": "Backup concluído"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname}: restauração do host concluída", "title": "{hostname}: restauração do host concluída",
"body": "Tarefas pós-restauração concluídas em segundo plano.\n\nConvidados inscritos: {guests}\nStubs de montagem de ligação: {stubs}\nDiretórios de nó obsoletos removidos: {stale_nodes}\nComponentes reinstalados: {components}\nDuração: {duration}\n{warnings_block}\nO nó agora está totalmente pronto para uso.", "body": "Tarefas pós-restauro concluídas em segundo plano.\n\nConvidados aplicados: {guests}\nDiretórios auxiliares de montagens bind: {stubs}\nDiretórios de nós obsoletos removidos: {stale_nodes}\nComponentes reinstalados: {components}\nDuração: {duration}\n{warnings_block}",
"label": "Restauração do host concluída" "label": "Restauração do host concluída"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Aviso", "warning": "Aviso",
"info": "Informação", "info": "Informação",
"ok": "Resolvido", "ok": "Resolvido",
"default": "Aviso" "default": "Aviso",
"observation": "Já não comunicado"
}, },
"groups": { "groups": {
"vm_ct": "Máquina Virtual/Contêiner", "vm_ct": "Máquina Virtual/Contêiner",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Falhou", "failed": "Falhou",
"completed": "Concluído", "completed": "Concluído",
"started": "Iniciado" "started": "Iniciado",
"unconfirmed": "Não confirmado"
}, },
"report": "Relatório {group}", "report": "Relatório {group}",
"details": "Detalhes", "details": "Detalhes",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "As amostras elevadas abrangem {duration}." "sampleSpan": "As amostras elevadas abrangem {duration}."
},
"backup": {
"confirmedTitle": "{hostname}: backup concluído",
"confirmedBody": "Backup concluído com sucesso.",
"errorTitle": "{hostname}: erro comunicado no backup",
"errorBody": "O relatório do backup contém um erro.",
"unconfirmedBody": "O resultado do backup não está confirmado."
} }
} }
} }
+16 -7
View File
@@ -6266,8 +6266,8 @@
"label": "Nytt hälsoproblem" "label": "Nytt hälsoproblem"
}, },
"error_resolved": { "error_resolved": {
"title": "{hostname}: Löst - {category}{entity_suffix}", "title": "{hostname}: Rapporteras inte längre – {category}{entity_suffix}",
"body": "{category}-problemet har lösts.\n{reason}\n🚦 Tidigare svårighetsgrad: {original_severity}\n⏱️ Varaktighet: {duration}", "body": "Problemet i kategorin {category} finns inte längre bland aktiva hälsoposter.\n{reason}\n🚦 Tidigare allvarlighetsgrad: {original_severity}\n⏱️ Tid sedan första observationen: {duration}",
"label": "Återställningsmeddelande" "label": "Återställningsmeddelande"
}, },
"error_escalated": { "error_escalated": {
@@ -6396,8 +6396,8 @@
"label": "Säkerhetskopiering startade" "label": "Säkerhetskopiering startade"
}, },
"backup_complete": { "backup_complete": {
"title": "{hostname} → {storage}: Säkerhetskopiering klar — {vmname} ({vmid})", "title": "{hostname}: säkerhetskopians resultat obekräftat",
"body": "Säkerhetskopiering av {vmname} (ID: {vmid}) slutfördes framgångsrikt på {storage}.\nStorlek: {size}", "body": "Det går inte att bekräfta säkerhetskopians resultat utifrån denna avisering.",
"label": "Säkerhetskopieringen är klar" "label": "Säkerhetskopieringen är klar"
}, },
"backup_warning": { "backup_warning": {
@@ -6557,7 +6557,7 @@
}, },
"system_restore_completed": { "system_restore_completed": {
"title": "{hostname}: Värdåterställning avslutad", "title": "{hostname}: Värdåterställning avslutad",
"body": "Uppgifter efter återställning slutförda i bakgrunden.\n\nGäster ansökte: {guests}\nBind-monterade stubbar: {stubs}\nInaktuella nodkataloger har tagits bort: {stale_nodes}\nKomponenter installerade om: {components}\nVaraktighet: {duration}\n{warnings_block}\nNoden är nu helt redo att användas.", "body": "Åtgärder efter återställning slutfördes i bakgrunden.\n\nGäster tillämpade: {guests}\nHjälpkataloger för bind-monteringar: {stubs}\nFöråldrade nodkataloger borttagna: {stale_nodes}\nKomponenter ominstallerade: {components}\nVaraktighet: {duration}\n{warnings_block}",
"label": "Värdåterställning slutförd" "label": "Värdåterställning slutförd"
}, },
"system_problem": { "system_problem": {
@@ -6910,7 +6910,8 @@
"warning": "Varning", "warning": "Varning",
"info": "Information", "info": "Information",
"ok": "Löst", "ok": "Löst",
"default": "Observera" "default": "Observera",
"observation": "Rapporteras inte längre"
}, },
"groups": { "groups": {
"vm_ct": "Virtuell maskin / behållare", "vm_ct": "Virtuell maskin / behållare",
@@ -6978,7 +6979,8 @@
"status": { "status": {
"failed": "Misslyckades", "failed": "Misslyckades",
"completed": "Klar", "completed": "Klar",
"started": "Startat" "started": "Startat",
"unconfirmed": "Obekräftat"
}, },
"report": "{group} Rapportera", "report": "{group} Rapportera",
"details": "Detaljer", "details": "Detaljer",
@@ -6988,6 +6990,13 @@
}, },
"temperature": { "temperature": {
"sampleSpan": "De höga mätvärdena sträcker sig över {duration}." "sampleSpan": "De höga mätvärdena sträcker sig över {duration}."
},
"backup": {
"confirmedTitle": "{hostname}: säkerhetskopiering klar",
"confirmedBody": "Säkerhetskopieringen slutfördes utan fel.",
"errorTitle": "{hostname}: fel rapporterat vid säkerhetskopiering",
"errorBody": "Rapporten om säkerhetskopieringen innehåller ett fel.",
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat."
} }
} }
} }
+24 -3
View File
@@ -1037,6 +1037,21 @@ class EmailChannel(NotificationChannel):
# Determine group for section header # Determine group for section header
event_type = data.get('_event_type', '') event_type = data.get('_event_type', '')
if event_type == 'error_resolved':
sev.update(self._SEV_DEFAULT)
sev['label'] = _runtime_text('email.severity.observation', data)
elif event_type == 'backup_complete':
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
sev.update(self._SEV_STYLE['OK'])
status = 'completed'
elif outcome == 'failed':
sev.update(self._SEV_STYLE['CRITICAL'])
status = 'failed'
else:
sev.update(self._SEV_DEFAULT)
status = 'unconfirmed'
sev['label'] = _runtime_text(f'email.status.{status}', data)
group = data.get('_group', 'other') group = data.get('_group', 'other')
# Keep unbroken recorded text inside the temperature email's table. # Keep unbroken recorded text inside the temperature email's table.
# Both properties are inline for mail clients; other events retain # Both properties are inline for mail clients; other events retain
@@ -1221,7 +1236,9 @@ class EmailChannel(NotificationChannel):
v = str(value).strip() if value else '' v = str(value).strip() if value else ''
if not v or v == '0' and original_label not in ('Failures',): if not v or v == '0' and original_label not in ('Failures',):
return return
if fmt == 'severity': if fmt == 'backup_error':
rows.append((esc(label), f'<span style="color:#dc2626;font-weight:600;">{esc(v)}</span>'))
elif fmt == 'severity':
sev_colors = { sev_colors = {
'CRITICAL': '#dc2626', 'WARNING': '#d97706', 'CRITICAL': '#dc2626', 'WARNING': '#d97706',
'INFO': '#2563eb', 'OK': '#16a34a', 'INFO': '#2563eb', 'OK': '#16a34a',
@@ -1254,9 +1271,13 @@ class EmailChannel(NotificationChannel):
# tell which target the backup ran against. Reported gap: emails # tell which target the backup ran against. Reported gap: emails
# showed no way to distinguish which PBS failed with 2+ configured. # showed no way to distinguish which PBS failed with 2+ configured.
_add('Storage', data.get('storage') or data.get('storage_name'), 'code') _add('Storage', data.get('storage') or data.get('storage_name'), 'code')
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started' if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
status_key = ('failed' if data.get('backup_outcome') == 'failed'
else 'unconfirmed')
else:
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
_add('Status', _runtime_text(f'email.status.{status_key}', language_data), _add('Status', _runtime_text(f'email.status.{status_key}', language_data),
'severity' if 'fail' in event_type else '') 'backup_error' if status_key == 'failed' else '')
_add('Size', data.get('size')) _add('Size', data.get('size'))
_add('Duration', data.get('duration')) _add('Duration', data.get('duration'))
_add('Snapshot', data.get('snapshot_name'), 'code') _add('Snapshot', data.get('snapshot_name'), 'code')
+56 -4
View File
@@ -3254,7 +3254,7 @@ class PollingCollector:
reason_lines = (reason or '').split('\n') reason_lines = (reason or '').split('\n')
reason_summary = reason_lines[0] if reason_lines else '' reason_summary = reason_lines[0] if reason_lines else ''
# Try to extract device info for a clean "Device: xxx (recovered)" line # Keep the earlier device context without asserting recovery.
device_line = '' device_line = ''
for line in reason_lines: for line in reason_lines:
if 'Device:' in line or 'Device not currently' in line or '/dev/' in line: if 'Device:' in line or 'Device not currently' in line or '/dev/' in line:
@@ -3267,11 +3267,11 @@ class PollingCollector:
break break
if reason_summary and device_line: if reason_summary and device_line:
clean_reason = f'{reason_summary}\n{device_line} (recovered)' clean_reason = f'{reason_summary}\n{device_line} (no longer reported)'
elif reason_summary: elif reason_summary:
clean_reason = f'{reason_summary} (recovered)' clean_reason = f'{reason_summary} (no longer reported)'
else: else:
clean_reason = 'Condition resolved' clean_reason = 'Condition no longer reported'
# `original_severity` must match what the user actually saw # `original_severity` must match what the user actually saw
# in the most-recent notification for this error, not the # in the most-recent notification for this error, not the
@@ -4289,6 +4289,51 @@ class ProxmoxHookWatcher:
def _hostname(self) -> str: def _hostname(self) -> str:
return _hostname() return _hostname()
@staticmethod
def _backup_outcome(severity: str, message: str) -> str:
"""Distinguish explicit failure, complete guest logs and unknown results."""
text = str(message or '')
if severity in ('error', 'err', 'critical') or re.search(
r'(?im)^\s*(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
return 'failed'
if severity not in ('info', 'ok', 'success') or re.search(
r'(?im)(?:^\s*WARNING:|\bWARNINGS\s*:\s*\d+)', text):
return 'unconfirmed'
starts = re.findall(r'(?im)\bStarting Backup of VM (\d+)\s*\(', text)
finished = re.findall(r'(?im)\bFinished Backup of VM (\d+)\s*\(', text)
lines = text.splitlines()
table_outcome = None
for index, header in enumerate(lines):
if not re.match(r'\s*VMID\s+Name\s+Status\b', header, re.IGNORECASE):
continue
status_start = header.find('Status')
status_end = header.find('Time', status_start)
if status_start < 0 or status_end < 0:
break
rows = []
for line in lines[index + 1:]:
if re.match(r'\s*Total\b', line, re.IGNORECASE):
table_outcome = ('confirmed' if rows and all(status == 'OK' for status in rows)
else 'unconfirmed')
break
if not line.strip():
break
if not re.match(r'\s*\d+\s+', line):
break
status = line[status_start:status_end].strip().upper()
if status == 'ERROR':
return 'failed'
rows.append(status)
break
if table_outcome == 'unconfirmed':
return 'unconfirmed'
if starts:
return 'confirmed' if sorted(starts) == sorted(finished) else 'unconfirmed'
if table_outcome == 'confirmed' or re.search(
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
return 'confirmed'
return 'unconfirmed'
def process_webhook(self, payload: dict) -> dict: def process_webhook(self, payload: dict) -> dict:
"""Process an incoming Proxmox webhook payload. """Process an incoming Proxmox webhook payload.
@@ -4347,6 +4392,13 @@ class ProxmoxHookWatcher:
'title': title or event_type, 'title': title or event_type,
'job_id': pve_job_id, 'job_id': pve_job_id,
} }
if event_type in ('backup_complete', 'backup_fail'):
# This is presentation metadata, not a new event/toggle/delivery path.
data['backup_outcome'] = (
'failed' if event_type == 'backup_fail' else
self._backup_outcome(severity_raw, message) if pve_type == 'vzdump'
else 'unconfirmed'
)
if pve_type == 'replication': if pve_type == 'replication':
replication = self._extract_replication_context( replication = self._extract_replication_context(
+42 -23
View File
@@ -299,7 +299,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
current_vm = { current_vm = {
'vmid': m_start.group(1), 'vmid': m_start.group(1),
'name': '', 'name': '',
'status': 'ok', 'status': 'unknown',
'time': '', 'time': '',
'size': '', 'size': '',
'filename': '', 'filename': '',
@@ -338,15 +338,16 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
# Finished -> duration # Finished -> duration
m_finish = re.match( m_finish = re.match(
r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean) r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
if m_finish: if m_finish and m_finish.group(1) == current_vm['vmid']:
current_vm['time'] = m_finish.group(2) current_vm['time'] = m_finish.group(2)
current_vm['status'] = 'ok' if current_vm['status'] != 'error':
current_vm['status'] = 'ok'
vms.append(current_vm) vms.append(current_vm)
current_vm = None current_vm = None
continue continue
# Error # Error
if clean.startswith('ERROR:') or clean.startswith('TASK ERROR'): if re.match(r'^\s*(?:ERROR:|TASK ERROR)', line, re.IGNORECASE):
if current_vm: if current_vm:
current_vm['status'] = 'error' current_vm['status'] = 'error'
@@ -439,7 +440,7 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
for vm in parsed.get('vms', []): for vm in parsed.get('vms', []):
status = vm.get('status', '').lower() status = vm.get('status', '').lower()
icon = '\u2705' if status == 'ok' else '\u274C' icon = '\u2705' if status == 'ok' else '\u274C' if status == 'error' else '\u2754'
# Determine VM/CT type prefix # Determine VM/CT type prefix
vm_type = vm.get('type', '') vm_type = vm.get('type', '')
@@ -501,7 +502,8 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
if vm_count > 0 or parsed.get('total_size'): if vm_count > 0 or parsed.get('total_size'):
ok_count = sum(1 for v in parsed.get('vms', []) ok_count = sum(1 for v in parsed.get('vms', [])
if v.get('status', '').lower() == 'ok') if v.get('status', '').lower() == 'ok')
fail_count = vm_count - ok_count fail_count = sum(1 for v in parsed.get('vms', [])
if v.get('status', '').lower() == 'error')
summary_parts = [] summary_parts = []
if vm_count: if vm_count:
@@ -779,11 +781,11 @@ TEMPLATES = {
# `{entity}` is populated by health_persistence.resolve_error() # `{entity}` is populated by health_persistence.resolve_error()
# (via _entity_from_details) and by PollingCollector's spread of # (via _entity_from_details) and by PollingCollector's spread of
# the original details blob. When absent, _SafeDict elides the # the original details blob. When absent, _SafeDict elides the
# placeholder and the title collapses back to "Resolved - <cat>" # placeholder and the title collapses back to "No longer reported - <cat>"
# without a trailing dash. # without a trailing dash.
'title': '{hostname}: Resolved - {category}{entity_suffix}', 'title': '{hostname}: No longer reported - {category}{entity_suffix}',
'body': 'The {category} issue has been resolved.\n{reason}\n\U0001F6A6 Previous severity: {original_severity}\n\u23F1\uFE0F Duration: {duration}', 'body': 'The {category} issue is no longer in active health records.\n{reason}\n\U0001F6A6 Previous severity: {original_severity}\n\u23F1\uFE0F Time since first observation: {duration}',
'label': 'Recovery notification', 'label': 'Health issue no longer reported',
'group': 'health', 'group': 'health',
'default_enabled': True, 'default_enabled': True,
}, },
@@ -999,9 +1001,9 @@ TEMPLATES = {
'default_enabled': False, 'default_enabled': False,
}, },
'backup_complete': { 'backup_complete': {
'title': '{hostname} → {storage}: Backup complete — {vmname} ({vmid})', 'title': '{hostname}: Backup outcome unconfirmed',
'body': 'Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}', 'body': 'The backup outcome could not be confirmed from this notice.',
'label': 'Backup complete', 'label': 'Backup report',
'group': 'backup', 'group': 'backup',
'default_enabled': True, 'default_enabled': True,
}, },
@@ -1270,8 +1272,7 @@ TEMPLATES = {
'Stale node dirs removed: {stale_nodes}\n' 'Stale node dirs removed: {stale_nodes}\n'
'Components reinstalled: {components}\n' 'Components reinstalled: {components}\n'
'Duration: {duration}\n' 'Duration: {duration}\n'
'{warnings_block}\n' '{warnings_block}'
'The node is now fully ready to use.'
), ),
'label': 'Host restore completed', 'label': 'Host restore completed',
'group': 'services', 'group': 'services',
@@ -1855,6 +1856,16 @@ def render_template(event_type: str, data: Dict[str, Any],
) )
if localized: if localized:
template[field] = localized template[field] = localized
if event_type == 'backup_complete':
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
template['title'] = runtime_message('backup.confirmedTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.confirmedBody', language)
elif outcome == 'failed':
template['title'] = runtime_message('backup.errorTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.errorBody', language)
# Ensure hostname is always available # Ensure hostname is always available
variables = { variables = {
@@ -1997,7 +2008,6 @@ def render_template(event_type: str, data: Dict[str, Any],
# When the event came from PVE webhook with a full vzdump message, # When the event came from PVE webhook with a full vzdump message,
# parse the table/logs and format a rich body instead of the sparse template. # parse the table/logs and format a rich body instead of the sparse template.
pve_message = data.get('pve_message', '') pve_message = data.get('pve_message', '')
pve_title = data.get('pve_title', '')
# Check for custom formatter function # Check for custom formatter function
formatter_name = template.get('formatter') formatter_name = template.get('formatter')
@@ -2016,13 +2026,18 @@ def render_template(event_type: str, data: Dict[str, Any],
if parsed: if parsed:
is_success = (event_type == 'backup_complete') is_success = (event_type == 'backup_complete')
body_text = _format_vzdump_body(parsed, is_success, language=language) body_text = _format_vzdump_body(parsed, is_success, language=language)
# Preserve PVE's source title for English, but never leak it into a if event_type == 'backup_complete' and data.get('backup_outcome') == 'failed':
# deterministic localized notification. error_lines = [line.strip() for line in pve_message.splitlines()
if pve_title and requested_language == 'en': if re.match(r'^\s*(?:ERROR:|TASK ERROR)', line, re.IGNORECASE)]
title = pve_title if error_lines:
body_text += '\n' + '\n'.join(error_lines)
else: else:
# Couldn't parse -- use PVE raw message as body # Couldn't parse -- use PVE raw message as body
body_text = pve_message.strip() body_text = pve_message.strip()
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
else 'backup.unconfirmedBody')
body_text = runtime_message(key, language) + '\n' + body_text
elif event_type == 'system_mail' and pve_message: elif event_type == 'system_mail' and pve_message:
# System mail -- use PVE message directly (mail bounce, cron, smartd) # System mail -- use PVE message directly (mail bounce, cron, smartd)
body_text = pve_message.strip()[:1000] body_text = pve_message.strip()[:1000]
@@ -2166,7 +2181,7 @@ EVENT_EMOJI = {
'host_backup_start': '\U0001F5C4️\U0001F680', # 🗄️🚀 cabinet + rocket 'host_backup_start': '\U0001F5C4️\U0001F680', # 🗄️🚀 cabinet + rocket
'host_backup_complete': '\U0001F5C4️✅', # 🗄️✅ cabinet + check 'host_backup_complete': '\U0001F5C4️✅', # 🗄️✅ cabinet + check
'host_backup_fail': '\U0001F5C4️❌', # 🗄️❌ cabinet + cross 'host_backup_fail': '\U0001F5C4️❌', # 🗄️❌ cabinet + cross
'backup_complete': '\U0001F4BE\u2705', # 💾✅ floppy + check 'backup_complete': '\U0001F4BE', # 💾 neutral for digests without outcome metadata
'backup_warning': '\U0001F4BE\u26A0\uFE0F', # 💾⚠️ floppy + warning 'backup_warning': '\U0001F4BE\u26A0\uFE0F', # 💾⚠️ floppy + warning
'backup_fail': '\U0001F4BE\u274C', # 💾❌ floppy + cross 'backup_fail': '\U0001F4BE\u274C', # 💾❌ floppy + cross
'snapshot_complete': '\U0001F4F8', # camera with flash 'snapshot_complete': '\U0001F4F8', # camera with flash
@@ -2204,14 +2219,14 @@ EVENT_EMOJI = {
'system_startup': '\U0001F680', # rocket (startup) 'system_startup': '\U0001F680', # rocket (startup)
'system_shutdown': '\u23FB\uFE0F', # power symbol (Unicode) 'system_shutdown': '\u23FB\uFE0F', # power symbol (Unicode)
'system_reboot': '\U0001F504', 'system_reboot': '\U0001F504',
'system_restore_completed': '✅', # check mark 'system_restore_completed': '\U0001F4CB', # post-restore task report (boot may have warnings)
'system_problem': '\u26A0\uFE0F', 'system_problem': '\u26A0\uFE0F',
'kernel_warning': '\u26A0\uFE0F', 'kernel_warning': '\u26A0\uFE0F',
'service_fail': '\u274C', 'service_fail': '\u274C',
'oom_kill': '\U0001F4A3', # bomb 'oom_kill': '\U0001F4A3', # bomb
# Health # Health
'new_error': '\U0001F198', # SOS 'new_error': '\U0001F198', # SOS
'error_resolved': '\u2705', 'error_resolved': '\U0001F4CB', # no longer active in health records, not proven recovery
'error_escalated': '\U0001F53A', # red triangle up 'error_escalated': '\U0001F53A', # red triangle up
'health_degraded': '\u26A0\uFE0F', 'health_degraded': '\u26A0\uFE0F',
'health_persistent': '\U0001F4CB', # clipboard 'health_persistent': '\U0001F4CB', # clipboard
@@ -2363,6 +2378,10 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
severity = data.get('severity', 'INFO') severity = data.get('severity', 'INFO')
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '') icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
if event_type == 'backup_complete':
icon = {
'confirmed': '💾✅', 'failed': '💾❌',
}.get(str(data.get('backup_outcome') or ''), '💾❔')
# Build enriched title: replace severity circle with event-specific icon # Build enriched title: replace severity circle with event-specific icon
# Current format: "hostname: Something" -> "ICON hostname: Something" # Current format: "hostname: Something" -> "ICON hostname: Something"
@@ -312,6 +312,7 @@ class RuntimeCatalogTests(unittest.TestCase):
"backup_complete", "backup_complete",
{ {
"hostname": "pve01", "storage": "pbs-main", "vmname": "alpha", "vmid": "100", "hostname": "pve01", "storage": "pbs-main", "vmname": "alpha", "vmid": "100",
"backup_outcome": "confirmed",
"pve_title": "Backup job finished", "pve_title": "Backup job finished",
"pve_message": ( "pve_message": (
"INFO: Starting Backup of VM 100 (qemu)\n" "INFO: Starting Backup of VM 100 (qemu)\n"
@@ -322,7 +323,7 @@ class RuntimeCatalogTests(unittest.TestCase):
}, },
language="sk", language="sk",
) )
self.assertIn("Záloha dokončená", backup["title"]) self.assertIn("záloha dokončená", backup["title"])
self.assertNotIn("Backup job finished", backup["title"]) self.assertNotIn("Backup job finished", backup["title"])
self.assertIn("Veľkosť: 1.5 GiB", backup["body"]) self.assertIn("Veľkosť: 1.5 GiB", backup["body"])
self.assertIn("Trvanie: 00:00:10", backup["body"]) self.assertIn("Trvanie: 00:00:10", backup["body"])
@@ -627,6 +628,7 @@ class RuntimeCatalogTests(unittest.TestCase):
"_notification_language": "sk", "_event_type": event_type, "_notification_language": "sk", "_event_type": event_type,
"_group": "backup", "hostname": "pve01", "vmid": "100", "_group": "backup", "hostname": "pve01", "vmid": "100",
"vmname": "alpha", "storage": "pbs-main", "vmname": "alpha", "storage": "pbs-main",
"backup_outcome": "confirmed" if event_type == "backup_complete" else "unconfirmed",
}, },
) )
self.assertIn(f">{localized_status}<", backup_html) self.assertIn(f">{localized_status}<", backup_html)
@@ -60,7 +60,7 @@ def _make_long_vzdump_report():
class VzdumpWebhookTruncationTests(unittest.TestCase): class VzdumpWebhookTruncationTests(unittest.TestCase):
def test_truncating_vzdump_report_at_4096_can_create_false_failed_backup(self): def test_truncating_vzdump_report_at_4096_leaves_guest_unconfirmed(self):
full_message = _make_long_vzdump_report() full_message = _make_long_vzdump_report()
truncated_message = full_message[:4096] truncated_message = full_message[:4096]
@@ -82,8 +82,8 @@ class VzdumpWebhookTruncationTests(unittest.TestCase):
self.assertEqual(truncated_dockflare["name"], "dockflare") self.assertEqual(truncated_dockflare["name"], "dockflare")
self.assertEqual(truncated_dockflare["status"], "") self.assertEqual(truncated_dockflare["status"], "")
self.assertIn("❌ dockflare (129)", truncated_body) self.assertIn("❔ dockflare (129)", truncated_body)
self.assertIn("❌ 1 failed", truncated_body) self.assertNotIn("❌ 1 failed", truncated_body)
def test_webhook_handler_does_not_truncate_message_before_parsing(self): def test_webhook_handler_does_not_truncate_message_before_parsing(self):
source = (SCRIPTS_DIR / "flask_notification_routes.py").read_text() source = (SCRIPTS_DIR / "flask_notification_routes.py").read_text()