Merge pull request #406 from f3rs3n/fix/notification-outcome-reporting

Clarify backup outcomes and diagnostic presentation
This commit is contained in:
MacRimi
2026-10-01 21:23:01 +02:00
committed by GitHub
25 changed files with 1955 additions and 105 deletions
@@ -0,0 +1,88 @@
"""Inert infrastructure for actual endpoint/manual/queued/SQLite release seams."""
import datetime
import sqlite3
import tempfile
import threading
import time
import types
import typing
from html.parser import HTMLParser
from pathlib import Path
from notification_fixture import templates, SCRIPTS, EmailChannel, extract
def visible(markup):
class Text(HTMLParser):
def __init__(self): super().__init__(); self.parts = []; self.tags = []
def handle_data(self, data): self.parts.append(data)
def handle_starttag(self, tag, attrs): self.tags.append(tag)
parser = Text(); parser.feed(markup)
return '\n'.join(p.strip() for p in parser.parts if p.strip()), parser.tags
def restore_event(warnings):
events = []
ns = {'request': types.SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
'hostname':'node-a', 'guests':'3', 'stubs':'1', 'stale_nodes':'0',
'components':'2', 'duration':'2m', 'warnings':warnings}),
'notification_manager':types.SimpleNamespace(emit_event=lambda **kw:events.append(kw)),
'jsonify':lambda value:value}
handler = extract(SCRIPTS/'flask_notification_routes.py', 'internal_restore_event', None, ns)
response, status = handler()
assert status == 200 and len(events) == 1
return events[0]
def deliver(event_type, data, severity='INFO', language='en', manual=False, quiet=False, quiet_before=()):
"""Execute real dispatch/manual and optional real SQLite buffer+flush."""
captured = []
channel = object.__new__(EmailChannel); channel.subject_prefix = '[ProxMenux]'
def sink(title, body, severity, data):
markup = channel._format_html(title, body, severity, data)
text, tags = visible(markup)
captured.append(dict(title=title, body=body, severity=severity, data=dict(data), html=markup, text=text, tags=tags))
return {'success':True}
manager = types.SimpleNamespace(_config={'email.rich_format':'true'}, _lock=threading.RLock(),
_channels={'email':types.SimpleNamespace(send=sink)},
_group_limiter=types.SimpleNamespace(allow=lambda group:True),
_claim_delivery=lambda event:'inert', _finish_delivery_claim=lambda *a,**kw:None,
_notification_language=lambda:language, _build_ai_config=lambda:{'ai_enabled':'false'},
_in_quiet_hours=lambda channel:quiet, _should_buffer_for_digest=lambda *a:False,
_record_history=lambda *a:None, _stats={'total_sent':0,'total_errors':0},
is_event_enabled=lambda event:True)
ns = dict(vars(typing), NotificationEvent=types.SimpleNamespace, TEMPLATES=templates.TEMPLATES, render_template=templates.render_template,
resolve_notification_hostname=lambda host, config:host or 'node-a',
enrich_with_emojis=templates.enrich_with_emojis, datetime=datetime.datetime,
_should_bypass_ai=lambda event:True, _AI_BYPASS_EVENTS=frozenset({'backup_complete','backup_fail'}))
for name in ('_dispatch_to_channels', '_dispatch_event', 'send_notification'):
setattr(manager, name, types.MethodType(extract(SCRIPTS/'notification_manager.py', name, 'NotificationManager', ns), manager))
with tempfile.TemporaryDirectory(prefix='notification-final-') as scratch:
db = Path(scratch)/'pending.sqlite'
rows = []
if quiet:
conn = sqlite3.connect(db)
conn.execute('CREATE TABLE quiet_pending (id INTEGER PRIMARY KEY, channel TEXT, event_type TEXT, event_group TEXT, severity TEXT, ts INTEGER, title TEXT, body TEXT)')
conn.commit(); conn.close()
qns = dict(vars(typing), sqlite3=sqlite3, DB_PATH=db, time=time, datetime=datetime.datetime,
_resolve_display_hostname=lambda config:'node-a', runtime_message=templates.runtime_message,
EVENT_EMOJI=templates.EVENT_EMOJI, CATEGORY_EMOJI=templates.CATEGORY_EMOJI)
for name in ('_buffer_quiet_event', '_flush_quiet_for_channel', '_compose_digest_body'):
setattr(manager,name,types.MethodType(extract(SCRIPTS/'notification_manager.py',name,'NotificationManager',qns),manager))
for earlier in quiet_before:
manager._dispatch_event(types.SimpleNamespace(**earlier,source='inert',entity_type='node',entity_id='',event_id='inert',fingerprint='inert'))
if manual:
assert not quiet
result = manager.send_notification(event_type, severity, '', '', dict(data))
assert result['success']
else:
event = types.SimpleNamespace(event_type=event_type, severity=severity, data=dict(data), source='inert', entity_type='node', entity_id='', event_id='inert', fingerprint='inert')
manager._dispatch_event(event)
if quiet:
conn = sqlite3.connect(db); rows = conn.execute('SELECT event_type,severity,title,body FROM quiet_pending').fetchall(); conn.close()
assert len(rows)==1+len(quiet_before) and not captured
manager._flush_quiet_for_channel('email', manager._channels['email'])
conn=sqlite3.connect(db); remaining=conn.execute('SELECT count(*) FROM quiet_pending').fetchone()[0]; conn.close()
assert remaining==0
assert len(captured)==1
if quiet: captured[0]['buffered']=rows
return captured[0]
@@ -0,0 +1,51 @@
"""Assertion-free inert actual consumers; no operational host imports."""
import ast
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPTS = ROOT / 'AppImage/scripts'
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
import notification_templates as templates
# Display-name resolution is an infrastructure boundary, never load manager.
templates._get_hostname = lambda: 'node-a'
from notification_channels import EmailChannel
LANGUAGES = ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv')
def extract(path, name, owner, ns):
tree = ast.parse(path.read_text())
nodes = tree.body if owner is None else next(n.body for n in tree.body if isinstance(n, ast.ClassDef) and n.name == owner)
node = next(n for n in nodes if isinstance(n, ast.FunctionDef) and n.name == name)
node.decorator_list = []
exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), ns)
return ns[name]
def receive(message, severity='info', title='Backup', kind='vzdump'):
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
ns['NotificationEvent'] = Event
methods = {name: extract(SCRIPTS / 'notification_events.py', name, 'ProxmoxHookWatcher', ns)
for name in ('_classify_pve', '_map_severity', '_backup_outcome', 'process_webhook')}
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
class Receiver:
_hostname = 'node-a'
_classify_pve = methods['_classify_pve']
_map_severity = staticmethod(methods['_map_severity'])
_backup_outcome = staticmethod(methods['_backup_outcome'])
process_webhook = methods['process_webhook']
def __init__(self): self._queue = Queue()
target = Receiver()
result = target.process_webhook({'fields': {'type': kind}, 'severity': severity, 'title': title, 'message': message})
assert result['accepted'] and len(target._queue.items) == 1
return target._queue.items[0]
def email(event_type, data, severity='INFO', language='en'):
result = templates.render_template(event_type, data, language)
channel = object.__new__(EmailChannel)
channel.subject_prefix = '[ProxMenux]'
context = {**data, 'severity': severity, '_event_type': event_type, '_group': result['group'], '_notification_language': language}
return result, channel._format_html(result['title'], result['body'], severity, context)
@@ -130,6 +130,18 @@ class CommandDescriptionsTests(unittest.TestCase):
for key in ("temperatureAlertTitle", "temperatureAlertBody",
"recordedReason", "recordedDetails"):
fallback.setdefault(key, source_fallback[key])
# Slovak remains the exact upstream catalog; model the
# pending outcome-key generator additions in disposable
# copies rather than modifying its curated values.
if lang == 'sk':
local = temporary['runtime']['notifications']
source = catalog('en')['runtime']['notifications']
for key, value in source['backup'].items():
local.setdefault('backup', {}).setdefault(key, value)
local['channels']['email']['status'].setdefault(
'unconfirmed', source['channels']['email']['status']['unconfirmed'])
local['channels']['email']['status'].setdefault(
'completed_with_warnings', source['channels']['email']['status']['completed_with_warnings'])
path.write_text(json.dumps(temporary, ensure_ascii=False))
# Model steady state after the bot fills these intentional new
# messages; keep repository locales and all other leaves intact.
@@ -0,0 +1,116 @@
"""Backup-only maintainer contract at actual render/dispatch/email seams.
Scope-approved seams: receiver, template lookup, rich enrichment, inert queued
and manual delivery with an email capture sink. No host operations or sends.
"""
import unittest
from notification_fixture import templates, LANGUAGES
from notification_final_fixture import deliver
class BackupSplitTests(unittest.TestCase):
def test_recovery_default_is_upstream_resolved_without_proof(self):
for language in LANGUAGES:
data = {'hostname': 'alias', 'category': 'temperature',
'reason': 'Temperature high (recovered)', 'duration': '2m',
'original_severity': 'WARNING'}
rendered = templates.render_template('error_resolved', data, language)
expected = templates.runtime_message('templates.error_resolved.title', language,
hostname='alias', category='temperature', entity_suffix='')
self.assertEqual(rendered['title'], expected)
rich, _ = templates.enrich_with_emojis('error_resolved', rendered['title'], rendered['body'], data)
self.assertTrue(rich.startswith('✅ '))
result = deliver('error_resolved', data, 'OK', language)
self.assertIn('background:#f0fdf4;', result['html'])
def test_pre_guest_native_subject_keeps_only_cause_once_with_display_alias(self):
from notification_fixture import receive
raw_host = 'pve-production.internal.example'
message = ('Details\n=======\nVMID Name Status Time Size Filename\n'
'\nTotal running time: 0s\nTotal size: 0 B')
for raw in (message, 'ERROR: unable to activate storage PBS\n' + message):
event = receive(raw, 'error', 'vzdump backup status (' + raw_host +
'): backup failed: unable to activate storage PBS')
event.data['hostname'] = 'display-alias {rack.location}'
for language in LANGUAGES:
for manual in (False, True):
result = deliver(event.event_type, event.data, event.severity, language, manual=manual)
self.assertNotIn(raw_host, result['text'])
self.assertNotIn('vzdump backup status', result['text'])
self.assertEqual(result['text'].count('unable to activate storage PBS'), 1)
self.assertIn('display-alias {rack.location}', result['title'])
self.assertEqual(result['data']['pve_title'], event.data['pve_title'])
self.assertEqual(result['data']['pve_message'], raw)
# No global hostname or guest-name substitution: actual diagnostics are
# authoritative, even when they happen to contain the original hostname.
event = receive(message, 'error', 'vzdump backup status (' + raw_host +
'): backup failed: cannot connect to ' + raw_host)
result = deliver(event.event_type, {**event.data, 'hostname': 'alias'}, event.severity)
self.assertEqual(result['body'].count('cannot connect to ' + raw_host), 1)
self.assertNotIn('vzdump backup status', result['body'])
def test_backup_legacy_keys_keep_meaning_and_unknown_uses_new_keys(self):
# Frozen upstream contract; no Git history required in shipped tests.
legacy = {'en': {'title': '{hostname} → {storage}: Backup complete — {vmname} ({vmid})', 'body': 'Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}', 'label': 'Backup complete'}, 'de': {'title': '{hostname} → {storage}: Sicherung abgeschlossen – {vmname} ({vmid})', 'body': 'Die Sicherung von {vmname} (ID: {vmid}) wurde am {storage} erfolgreich abgeschlossen.\nGröße: {size}', 'label': 'Sicherung abgeschlossen'}, 'es': {'title': '{hostname} → {storage}: Backup completado — {vmname} ({vmid})', 'body': 'El backup de {vmname} (ID: {vmid}) se ha completado correctamente en {storage}.\nTamaño: {size}', 'label': 'Backup completado'}, 'fr': {'title': '{hostname} → {storage}\xa0: Sauvegarde terminée — {vmname} ({vmid})', 'body': "La sauvegarde de {vmname} (ID\xa0: {vmid}) s'est terminée avec succès le {storage}.\nTaille\xa0: {size}", 'label': 'Sauvegarde terminée'}, 'it': {'title': '{hostname} → {storage}: Backup completato — {vmname} ({vmid})', 'body': 'Backup di {vmname} (ID: {vmid}) completato con successo su {storage}.\nTaglia: {size}', 'label': 'Backup completato'}, 'pt': {'title': '{hostname} → {storage}: Backup concluído — {vmname} ({vmid})', 'body': 'Backup de {vmname} (ID: {vmid}) concluído com sucesso em {storage}.\nTamanho: {size}', 'label': 'Backup concluído'}, 'sk': {'title': '{hostname} → {storage}: Záloha dokončená — {vmname} ({vmid})', 'body': 'Záloha {vmname} (ID: {vmid}) na úložisku {storage} bola úspešne dokončená.\nVeľkosť: {size}', 'label': 'Záloha bola dokončená'}, 'sv': {'title': '{hostname} → {storage}: Säkerhetskopiering klar — {vmname} ({vmid})', 'body': 'Säkerhetskopiering av {vmname} (ID: {vmid}) slutfördes framgångsrikt på {storage}.\nStorlek: {size}', 'label': 'Säkerhetskopieringen är klar'}}
for language, expected in legacy.items():
self.assertEqual(templates._load_runtime_catalog(language)['templates']['backup_complete'], expected)
result = templates.render_template('backup_complete', {'hostname': 'alias {rack.location}'}, language)
self.assertIn('alias {rack.location}', result['title'])
self.assertNotIn('()', result['title'])
new_title = templates.runtime_message('backup.unconfirmedTitle', language, hostname='alias {rack.location}')
self.assertTrue(new_title)
self.assertEqual(result['title'], new_title)
self.assertIn(templates.runtime_message('backup.unconfirmedBody', language), result['body'])
# Absent/blank/non-string translation uses English per-key fallback.
from unittest.mock import patch
english = templates._load_runtime_catalog('en')
for value in (None, '', {}, []):
missing = {'backup': {'unconfirmedTitle': value, 'unconfirmedBody': value}}
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: english if lang == 'en' else missing):
result = templates.render_template('backup_complete', {'hostname': 'alias'}, 'it')
self.assertEqual(result['title'], 'alias: Backup outcome unconfirmed')
self.assertEqual(result['body'], 'The backup outcome is not confirmed.')
# Both catalogs missing: new outcome text still has explicit EN defaults.
with patch.object(templates, '_load_runtime_catalog', return_value={}):
result = templates.render_template('backup_complete', {'hostname': 'alias'}, 'it')
self.assertEqual(result['title'], 'alias: Backup outcome unconfirmed')
self.assertEqual(result['body'], 'The backup outcome is not confirmed.')
def test_restore_keeps_original_ready_line_and_success_icon_with_warnings(self):
from notification_final_fixture import restore_event
ready = {'en': 'The node is now fully ready to use.', 'de': 'Der Knoten ist nun vollständig einsatzbereit.', 'es': 'El nodo está listo para usarse.', 'fr': 'Le nœud est maintenant entièrement prêt à être utilisé.', 'it': "Il nodo è ora completamente pronto per l'uso.", 'pt': 'O nó agora está totalmente pronto para uso.', 'sk': 'Uzol je teraz úplne pripravený na použitie.', 'sv': 'Noden är nu helt redo att användas.'}
for warning in ('', 'missing module zfs'):
event = restore_event(warning)
for language in LANGUAGES:
result = deliver(event['event_type'], event['data'], event['severity'], language)
self.assertTrue(result['title'].startswith('✅ '))
self.assertIn(ready[language], result['body'])
self.assertIn(ready[language], result['text'])
self.assertIn('2m', result['text'])
if warning:
self.assertIn(warning, result['text'])
quiet = deliver(event['event_type'], event['data'], event['severity'], language, quiet=True)
self.assertIn('✅', quiet['body'])
self.assertIn(' ' + ready[language], quiet['body'])
self.assertIn('white-space:pre-wrap;', quiet['html'])
def test_spanish_outcome_and_restore_titles_are_capitalized_and_failure_is_exact(self):
expected = {'confirmed': 'Backup completado', 'completed_with_warnings': 'Backup completado con advertencias',
'unconfirmed': 'Resultado del backup sin confirmar', 'failed': 'Backup fallido'}
for outcome, title in expected.items():
result = templates.render_template('backup_complete', {'hostname': 'alias', 'backup_outcome': outcome}, 'es')
self.assertEqual(result['title'], 'alias: ' + title)
failure = templates.render_template('backup_fail', {'hostname': 'alias'}, 'es')
self.assertEqual(failure['title'], 'alias: Backup fallido')
restore = templates.render_template('system_restore_completed', {'hostname': 'alias'}, 'es')
self.assertEqual(restore['title'], 'alias: Restauración del host finalizada')
def test_recovery_only_quiet_release_keeps_upstream_summary_contract(self):
result = deliver('error_resolved', {'hostname': 'alias', 'category': 'temperature',
'reason': 'old (recovered)', 'duration': '2m'}, 'OK', quiet=True)
self.assertIn(templates.runtime_message('digest.lead', 'en', count=1).strip(), result['body'])
self.assertIn(templates.runtime_message('digest.footer', 'en'), result['body'])
self.assertIn('✅', result['body'])
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,204 @@
"""Whole-PR outcome corrections, inert producer/actual email consumers."""
import html
import unittest
from notification_fixture import templates, receive, email, LANGUAGES
REPORT = """Details
=======
VMID Name Status Time Size Filename
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
Total running time: 1m 1s
Total size: 1 GiB
"""
class CorrectionTests(unittest.TestCase):
def test_reversed_finish_is_not_completion_evidence(self):
event = receive('INFO: Finished Backup of VM 100 (00:01:01)\nINFO: Starting Backup of VM 100 (qemu)')
self.assertEqual(event.data['backup_outcome'], 'unconfirmed')
def test_interleaved_complete_logs_keep_both_finished_guests(self):
message = ('INFO: Starting Backup of VM 100 (qemu)\nINFO: VM Name: web\n'
'INFO: Starting Backup of VM 101 (lxc)\nINFO: CT Name: db\n'
'INFO: Finished Backup of VM 100 (00:01:01)\nINFO: Finished Backup of VM 101 (00:01:02)')
event = receive(message)
self.assertEqual(event.data['backup_outcome'], 'confirmed')
result, markup = email(event.event_type, event.data, event.severity)
self.assertIn('✅ VM web (100)', result['body'])
self.assertIn('✅ CT db (101)', result['body'])
self.assertNotIn('❔', result['body'])
self.assertIn('00:01:01', result['body'])
def test_backup_identity_has_event_scoped_mail_compatible_wrapping(self):
result, markup = email('backup_complete', {'hostname': 'n' * 64,
'backup_outcome': 'confirmed', 'pve_message': 'INFO: Starting Backup of VM 100 (qemu)\nINFO: VM Name: customerproductionpostgresqlreplicaeuropewestdatacenter01\nINFO: Finished Backup of VM 100 (00:01:01)'})
self.assertIn('table-layout:fixed;', markup)
title_tag = markup.split('<h2 style="', 1)[1].split('"', 1)[0]
self.assertIn('overflow-wrap:break-word;', title_tag)
self.assertIn('word-wrap:break-word;', title_tag)
_, unrelated = email('node_reconnect', {'hostname': 'node-a'}, 'OK')
self.assertNotIn('table-layout:fixed;', unrelated)
def test_catalog_parity_accepts_eventual_valid_slovak_keys(self):
from notification_fixture import extract, SCRIPTS
import copy
import string
path = SCRIPTS / 'tests/test_notification_runtime_i18n.py'
ns = {'string': string}
ns['_placeholders'] = extract(path, '_placeholders', None, ns)
parity = extract(path, 'test_runtime_catalog_keys_and_placeholders_match', 'RuntimeCatalogTests', ns)
probe = unittest.TestCase()
probe.catalogs = {lang: copy.deepcopy(templates._load_runtime_catalog(lang)) for lang in LANGUAGES}
parity(probe) # shipped missing keys remain allowed
probe.catalogs['sk']['backup'] = copy.deepcopy(probe.catalogs['en']['backup'])
probe.catalogs['sk']['channels']['email']['status']['unconfirmed'] = probe.catalogs['en']['channels']['email']['status']['unconfirmed']
parity(probe) # generation of exactly the pending keys is legal
probe.catalogs['sk']['backup']['confirmedTitle'] = 'Missing hostname token'
with self.assertRaises(AssertionError): parity(probe)
def test_actual_neutral_style_is_not_success_green(self):
for event, severity, data in (('backup_complete', 'INFO', {'backup_outcome': 'unconfirmed'}),):
result, markup = email(event, data, severity)
self.assertIn('background:#f9fafb;', markup)
self.assertNotIn('background:#f0fdf4;', markup)
def test_actual_restore_endpoint_warnings_and_counts_reach_email(self):
from notification_fixture import extract, SCRIPTS
from types import SimpleNamespace
for warnings in ('', 'missing module zfs'):
events = []
ns = {'request': SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
'hostname': 'node-a', 'guests': '3', 'stubs': '1', 'stale_nodes': '0',
'components': '2', 'duration': '2m', 'warnings': warnings}),
'notification_manager': SimpleNamespace(emit_event=lambda **kw: events.append(kw)),
'jsonify': lambda value: value}
handler = extract(SCRIPTS / 'flask_notification_routes.py', 'internal_restore_event', None, ns)
response, status = handler()
self.assertEqual(status, 200)
event = events[0]
for language in LANGUAGES:
result, markup = email(event['event_type'], event['data'], event['severity'], language)
self.assertIn('2m', markup)
for line in result['body'].splitlines():
if line.strip(): self.assertIn(line.strip(), html.unescape(markup))
if warnings: self.assertIn(warnings, markup)
def test_raw_display_hostname_is_substituted_exactly_once(self):
for event_type, outcome in (('backup_complete', 'confirmed'), ('backup_complete', 'failed'), ('backup_fail', 'failed')):
for hostname in ('Sala {rack} – Zürich', 'node-{vmid}', 'Sala {rack.location}'):
for language in LANGUAGES:
result, markup = email(event_type, {'hostname': hostname,
'backup_outcome': outcome, 'pve_message': REPORT}, language=language)
self.assertIn(hostname, result['title'])
self.assertIn(hostname, html.unescape(markup))
def test_malformed_numeric_report_is_queued_uncertain_and_renderable(self):
for size in ('1..5 GiB', '..5 GiB'):
message = REPORT.replace('1 GiB ', size.ljust(9)).split('Total size:')[0]
event = receive(message)
self.assertEqual(event.data['backup_outcome'], 'unconfirmed')
result, markup = email(event.event_type, event.data, event.severity)
self.assertIn(size, result['body'])
def test_unknown_backup_type_keeps_explicit_err_but_cannot_certify_ok(self):
event = receive(REPORT.replace('ok ', 'err '), kind='')
self.assertEqual(event.event_type, 'backup_complete')
self.assertEqual(event.severity, 'INFO')
self.assertEqual(event.data['backup_outcome'], 'failed')
result, markup = email(event.event_type, event.data, event.severity)
self.assertIn('FAILED', markup)
self.assertEqual(receive(REPORT, kind='').data['backup_outcome'], 'unconfirmed')
def test_confirmed_metadata_only_context_is_retained(self):
data = {'hostname': 'node-a', 'backup_outcome': 'confirmed',
'vmid': '100', 'vmname': 'web {literal}', 'storage': 'PBS', 'size': '1 GiB'}
for language in LANGUAGES:
result, markup = email('backup_complete', data, language=language)
self.assertIn('web {literal} (100)', result['title'])
self.assertIn('1 GiB', result['body'])
self.assertIn('1 GiB', markup)
def test_explicit_guest_failure_overrides_only_the_linked_ok_row(self):
other = '101 db ok 1m 1s 1 GiB ct/101/2026-09-29T17:00:00Z'
message = REPORT.replace('\n\nTotal', '\n' + other + '\n\nTotal')
diagnostic = '100: 2026-09-29 17:00:00 ERROR: Backup of VM 100 failed - archive write failed'
for severity in ('info', 'error'):
event = receive(message + '\n' + diagnostic, severity)
self.assertEqual(event.data.get('vmid'), '100')
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertIn('❌ VM web (100)', result['body'])
self.assertNotIn('✅ VM web (100)', markup)
self.assertIn('✅ CT db (101)', result['body'])
self.assertIn('web (100)', result['title'])
def test_official_week_month_year_durations_stay_confirmed(self):
for duration in ('1w', '1w 1m 1s', '1M', '1y'):
# Fixed-column widths are unchanged for these bounded values.
message = REPORT.replace('1m 1s ', duration.ljust(9))
event = receive(message)
self.assertEqual(event.data['backup_outcome'], 'confirmed', duration)
result, markup = email(event.event_type, event.data, event.severity)
self.assertIn(duration, result['body'])
def test_prefixed_error_diagnostics_survive_both_source_severities(self):
diagnostic = '100: 2026-09-29 17:00:00 ERROR: archive write failed: permission denied'
for severity in ('info', 'error'):
event = receive(REPORT + '\n' + diagnostic, severity)
self.assertEqual(event.data['backup_outcome'], 'failed')
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertIn(diagnostic, result['body'])
self.assertIn(diagnostic, html.unescape(markup))
# A job-level error must not invent a failed guest.
self.assertIn('✅ VM web (100)', result['body'])
def test_official_completed_report_warning_is_distinct_and_retained(self):
warning = '100: 2026-09-29 17:00:00 WARN: unable to add notes - permission denied'
event = receive(REPORT + '\n' + warning)
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertIn(warning, result['body'])
self.assertIn(warning, html.unescape(markup))
def test_batch_and_abort_failure_titles_have_no_empty_guest_slot(self):
failed = REPORT.replace('ok ', 'err ')
failed = failed.replace('\n\nTotal', '\n101 db err 1m 1s 0 B null\n\nTotal')
for message in (failed, REPORT.replace('ok ', 'todo '),
REPORT.split('100 web')[0] + '\nTotal running time: 0s'):
event = receive(message + '\nINFO: vzdump --storage PBS', 'error')
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertNotIn('()', result['title'])
self.assertIn('PBS', result['title'])
self.assertNotIn('web (100)', result['title'])
def test_subject_only_setup_failure_survives_actual_email(self):
reason = 'unable to activate storage PBS'
title = 'vzdump backup status (node-a): backup failed: ' + reason
message = REPORT.split('100 web')[0] + '\nTotal running time: 0s\nTotal size: 0 B'
event = receive(message, 'error', title)
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertIn(reason, result['body'])
self.assertIn(reason, html.unescape(markup))
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,136 @@
"""Final review contracts at actual locale/manual/queued/quiet email seams.
All operational dependencies are inert; no manager or route module import.
"""
import ast
import copy
import html
import unittest
from unittest.mock import patch
from notification_fixture import templates, SCRIPTS, LANGUAGES, receive
from notification_final_fixture import deliver, restore_event
# Literal native send_notification body, produced by pinned PVE Perl helpers.
NATIVE_REPORT = 'Details\n=======\n' + '''VMID Name Status Time Size Filename
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
Total running time: 1m 1s
Total size: 1 GiB
Logs
====
vzdump --all 1 --storage PBS --mode snapshot
100: no log available
'''
class FinalCorrectionsTests(unittest.TestCase):
def test_runtime_backup_assertions_accept_generated_and_missing_slovak_title(self):
path = SCRIPTS / 'tests/test_notification_runtime_i18n.py'
tree = ast.parse(path.read_text())
owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'RuntimeCatalogTests')
method = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == 'test_special_formatters_digest_and_test_message_are_slovak')
start = next(i for i,n in enumerate(method.body) if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='backup' for t in n.targets))
stop = next(i for i,n in enumerate(method.body) if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='manager' for t in n.targets))
block = ast.Module(body=method.body[start:stop], type_ignores=[])
english = templates._load_runtime_catalog('en')
for title in (None, '{hostname}: GENERATED_SK potvrdené'):
sk = copy.deepcopy(templates._load_runtime_catalog('sk'))
if title is not None: sk.setdefault('backup', {})['confirmedTitle'] = title
else: sk.get('backup', {}).pop('confirmedTitle', None)
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: english if lang == 'en' else sk):
exec(compile(block,str(path),'exec'), {'self':self, 'notification_templates':templates})
def test_native_multiline_failure_diagnostics_survive_receiver_dispatch_once(self):
subject = 'vzdump backup status (node-a): backup failed: multiple problems'
for diagnostic, report in (
('external provider job cleanup failed\njob-abort hook permission denied', NATIVE_REPORT),
('job interrupted\njob-abort hook permission denied', NATIVE_REPORT.replace('ok ', 'todo ')),
('unable to initialize external provider\njob-abort hook permission denied', NATIVE_REPORT.replace('100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z\n', '')),
):
event = receive(diagnostic + '\n' + report, 'error', subject)
self.assertEqual((event.event_type, event.severity), ('backup_fail', 'CRITICAL'))
for language in LANGUAGES:
result = deliver(event.event_type, event.data, event.severity, language)
for line in diagnostic.splitlines():
self.assertEqual(result['body'].count(line), 1)
self.assertEqual(result['text'].count(line), 1)
if 'ok ' in report:
self.assertIn('✅ VM web (100)', result['body'])
def test_restore_endpoint_quiet_release_keeps_warning_counts_and_truthful_footer(self):
reason = 'missing module zfs <raw> & {literal}'
event = restore_event(reason)
self.assertEqual(event['severity'], 'WARNING')
for language in LANGUAGES:
result = deliver(event['event_type'], event['data'], event['severity'], language, quiet=True)
self.assertEqual(result['severity'], 'INFO')
buffered_body = result['buffered'][0][3]
self.assertEqual(result['text'].count(reason), 1)
for line in buffered_body.splitlines():
if line.strip(): self.assertIn(line.strip(), result['text'])
self.assertIn('2m', result['text'])
self.assertNotIn(templates.runtime_message('digest.footer', language), result['body'])
self.assertNotIn('script', result['tags'])
def test_raw_restore_cells_use_event_scoped_mail_wrapping(self):
token = 'b' * 64
event = restore_event('Boot check: recorded token ' + token + '; verification pending')
for language in LANGUAGES:
results = [deliver(event['event_type'],event['data'],event['severity'],language,quiet=quiet) for quiet in (False,True)]
for result in results:
self.assertIn('table-layout:fixed;', result['html'])
self.assertIn('word-wrap:break-word;', result['html'])
self.assertIn('overflow-wrap:break-word;', result['html'])
self.assertEqual(result['text'].count(token), 1)
unrelated = deliver('node_reconnect', {'hostname':'node-a'}, 'OK')
self.assertNotIn('table-layout:fixed;', unrelated['html'])
def test_manual_failed_and_unconfirmed_backup_keep_short_actionable_reason(self):
for outcome, reason in (('failed','PBS permission denied for datastore remote'),
('unconfirmed','Task status unavailable: upstream API timed out')):
for language in LANGUAGES:
result = deliver('backup_complete',{'hostname':'node-a','vmid':'100','vmname':'web',
'storage':'PBS','backup_outcome':outcome,'reason':reason},'INFO',language,manual=True)
self.assertEqual(result['text'].count(reason), 1)
self.assertNotIn('>COMPLETED</span>', result['html'])
def test_backup_reason_threshold_and_raw_body_deduplication(self):
for event in ('backup_complete', 'backup_fail'):
for length in (79, 80, 81, 120):
prefix = '<raw> & {rack.location} '
reason = prefix + 'b' * (length - len(prefix))
self.assertEqual(len(reason), length)
for raw in ('', reason):
data = {'hostname':'node-a','backup_outcome':'failed','reason':reason,'pve_message':raw}
for manual in (False, True):
result = deliver(event,data,'INFO','en',manual=manual)
self.assertEqual(result['text'].count(reason), 1, (event,length,raw,manual))
self.assertNotIn('raw', result['tags'])
def test_quiet_restore_after_preview_limit_is_not_omitted_or_called_info(self):
event = restore_event('missing module zfs')
earlier = [dict(event_type='service_fail',severity='WARNING',data={'hostname':'node-a','service_name':f'unit-{i}','reason':'recorded'}) for i in range(8)]
result = deliver(event['event_type'],event['data'],event['severity'],quiet=True,quiet_before=earlier)
self.assertEqual(result['text'].count('missing module zfs'), 1)
self.assertNotIn(templates.runtime_message('digest.lead','en',count=9), result['body'])
self.assertEqual(result['data']['_count'], 9)
def test_native_error_block_does_not_deduplicate_a_substring_of_inventory(self):
event = receive('web\nexternal provider job cleanup failed\n' + NATIVE_REPORT,
'error','vzdump backup status (node-a): backup failed: multiple problems')
result = deliver(event.event_type,event.data,event.severity)
self.assertEqual(result['body'].splitlines().count('web'), 1)
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,159 @@
"""Maintainer acceptance at inert actual notification consumers."""
import unittest
from notification_fixture import templates, receive, LANGUAGES, SCRIPTS, extract
from notification_final_fixture import deliver
NATIVE_REPORT = '''Details
=======
VMID Name Status Time Size Filename
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
Total running time: 1m 1s
Total size: 1 GiB
Logs
====
vzdump --all 1 --storage PBS --mode snapshot
100: 2026-09-29 17:00:00 INFO: Starting Backup of VM 100 (qemu)
100: 2026-09-29 17:01:01 INFO: Finished Backup of VM 100 (00:01:01)
'''
class MaintainerFollowupTests(unittest.TestCase):
def test_completed_with_warnings_requires_independent_completion(self):
warning = '\n100: 2026-09-29 17:00:01 WARN: file changed during backup'
event = receive(NATIVE_REPORT + warning)
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
self.assertEqual((event.event_type,event.severity), ('backup_complete','INFO'))
for manual in (False,True):
for lang in LANGUAGES:
result = deliver(event.event_type,event.data,event.severity,lang,manual=manual)
label = templates.runtime_message('backup.warningTitle',lang,hostname='node-a')
self.assertIn(label,result['title'])
status = templates.runtime_message('channels.email.status.completed_with_warnings',lang)
self.assertIn(status,result['text'])
self.assertIn('file changed during backup',result['text'])
# Manual sends intentionally skip channel emoji enrichment.
if not manual: self.assertTrue(result['title'].startswith('💾⚠️'))
rich, _ = templates.enrich_with_emojis(event.event_type,result['title'],result['body'],event.data)
self.assertTrue(rich.startswith('💾⚠️'))
self.assertEqual(receive('WARN: file changed during backup').data['backup_outcome'],'unconfirmed')
self.assertEqual(receive(NATIVE_REPORT.split('Total running time:')[0]+warning).data['backup_outcome'],'unconfirmed')
self.assertEqual(receive(NATIVE_REPORT+warning+'\nERROR: cleanup failed').data['backup_outcome'],'failed')
self.assertEqual(receive(NATIVE_REPORT+warning,'warning').data['backup_outcome'],'completed_with_warnings')
self.assertEqual(receive('INFO: Starting Backup of VM 100 (qemu)\nINFO: Finished Backup of VM 100 (00:01:01)'+warning).data['backup_outcome'],'completed_with_warnings')
self.assertEqual(receive(NATIVE_REPORT+warning,kind='').data['backup_outcome'],'unconfirmed')
def test_null_filename_failed_guest_uses_own_start_identity(self):
report = NATIVE_REPORT.replace('100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z',
'100 web err 1m 1s 0 B null')
for kind, prefix in (('qemu','VM'),('lxc','CT')):
own = report.replace('VM 100 (qemu)',f'VM 100 ({kind})')
# Unrelated guest appears first and must never supply the failed type.
message = f'INFO: Starting Backup of VM 999 ({"lxc" if kind == "qemu" else "qemu"})\n' + own
event = receive(message,'error','vzdump backup status (raw-node): backup failed')
parsed = templates._parse_vzdump_message(message)
self.assertEqual(parsed['vms'][0]['type'],kind)
result = deliver(event.event_type,event.data,event.severity)
self.assertIn(f'{prefix} web (100)',result['title'])
self.assertIn(f'❌ {prefix} web (100)',result['body'])
for message in (report.replace('Starting Backup of VM 100','Starting Backup of VM 999'),
report+'\nINFO: Starting Backup of VM 100 (lxc)'):
self.assertEqual(templates._parse_vzdump_message(message)['vms'][0]['type'],'')
def test_native_subject_keeps_cause_without_host_envelope(self):
subject = 'vzdump backup status (raw-host): backup failed: multiple problems'
event = receive('ERROR: archive write failed\n'+NATIVE_REPORT,'error',subject)
event.data['hostname']='configured-alias'
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertNotIn(subject,result['text'])
self.assertEqual(result['text'].count('ERROR: archive write failed'),1)
self.assertIn('configured-alias',result['title'])
setup=receive('Details\n=======\nVMID Name Status Time Size Filename\n\nTotal running time: 0s\nTotal size: 0 B','error',subject.replace('multiple problems','unable to open storage'))
result=deliver(setup.event_type,setup.data,setup.severity)
self.assertEqual(result['text'].count('unable to open storage'),1)
self.assertNotIn(setup.data['pve_title'],result['text'])
unique=receive(NATIVE_REPORT,'error',subject.replace('multiple problems','job-end hook denied'))
result=deliver(unique.event_type,unique.data,unique.severity)
self.assertIn('job-end hook denied',result['body'])
self.assertNotIn('vzdump backup status',result['body'])
def test_backup_diagnostics_are_bounded_with_principal_cause_and_notice(self):
raw = NATIVE_REPORT + '\n' + '\n'.join(f'WARN: repeated warning {i}' for i in range(80)) + '\nERROR: principal archive write failure'
event=receive(raw)
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertIn('ERROR: principal archive write failure',result['body'])
self.assertLessEqual(len('\n'.join(line for line in result['body'].splitlines() if line.startswith(('WARN:','ERROR:')))),1024)
self.assertLessEqual(sum(line.startswith(('WARN:','ERROR:')) for line in result['body'].splitlines()),8)
notice=templates.runtime_message('backup.diagnosticsOmitted',lang,count=73)
self.assertIn(notice,result['body'])
self.assertEqual(event.data['pve_message'],raw)
long=receive('ERROR: '+ 'b'*5000,'error','vzdump backup status (node): backup failed')
result=deliver(long.event_type,long.data,long.severity)
self.assertLess(len(result['body']),1400)
self.assertIn('ERROR: '+ 'b'*100,result['body'])
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted','en',count=1),result['body'])
def test_real_quiet_digest_retains_each_backup_outcome_icon(self):
samples=[(NATIVE_REPORT,'confirmed','💾✅'),(NATIVE_REPORT+'\nWARN: changed file','completed_with_warnings','💾⚠️'),(NATIVE_REPORT+'\nERROR: write failed','failed','💾❌'),('INFO: Starting Backup of VM 100 (qemu)','unconfirmed','💾❔')]
for raw,outcome,icon in samples:
event=receive(raw)
self.assertEqual(event.data['backup_outcome'],outcome)
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang,quiet=True)
self.assertIn(icon,result['body'])
self.assertNotIn('💾❔',result['body']) if outcome!='unconfirmed' else None
import types,datetime
ns={'datetime':datetime.datetime,'runtime_message':templates.runtime_message,'EVENT_EMOJI':templates.EVENT_EMOJI,'CATEGORY_EMOJI':templates.CATEGORY_EMOJI}
compose=extract(SCRIPTS/'notification_manager.py','_compose_digest_body','NotificationManager',ns)
target=types.SimpleNamespace(_notification_language=lambda:'en')
rows=[(i,'backup_complete','backup',1,icon+' node: Backup','') for i,(_,_,icon) in enumerate(samples)]
body=compose(target,rows,use_icons=True)
for _,_,icon in samples:self.assertIn(icon,body)
plain=compose(target,rows,use_icons=False)
for _,_,icon in samples:self.assertNotIn(icon,plain)
should=extract(SCRIPTS/'notification_manager.py','_should_buffer_for_digest','NotificationManager',{})
self.assertFalse(should(types.SimpleNamespace(_DIGEST_EXEMPT_EVENTS={'backup_complete'},_config={'email.digest_enabled':'true'}),'email','INFO','backup_complete'))
def test_quiet_restore_details_are_subordinate_in_text_and_email(self):
from notification_final_fixture import restore_event
event=restore_event('missing module zfs')
for lang in LANGUAGES:
result=deliver(event['event_type'],event['data'],event['severity'],lang,quiet=True)
body_lines=result['buffered'][0][3].splitlines()
for line in body_lines:
if line.strip():
self.assertIn(' '+line.strip(),result['body'])
self.assertIn(' '+__import__('html').escape(line.strip()),result['html'])
self.assertIn('white-space:pre-wrap;',result['html'])
self.assertNotIn(templates.runtime_message('digest.footer',lang),result['body'])
def test_concrete_subject_cause_already_in_error_log_is_not_repeated(self):
event=receive(NATIVE_REPORT+'\n100: ERROR: job-end hook denied','error',
'vzdump backup status (raw-host): backup failed: job-end hook denied')
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertEqual(result['text'].count('job-end hook denied'),1)
self.assertIn('100: ERROR: job-end hook denied',result['body'])
def test_job_level_subject_cause_survives_warning_cap(self):
raw=NATIVE_REPORT+'\n'+'\n'.join('WARN: repeated diagnostic '+str(i) for i in range(80))
event=receive(raw,'error','vzdump backup status (raw-host): backup failed: job-end hook denied')
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertEqual(result['body'].count('job-end hook denied'),1)
self.assertNotIn('vzdump backup status',result['body'])
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted',lang,count=73),result['body'])
def test_backup_quiet_email_uses_event_scoped_wrapping(self):
event=receive(NATIVE_REPORT+'\nWARN: changed file')
result=deliver(event.event_type,event.data,event.severity,'sv',quiet=True)
self.assertTrue(result['data'].get('_backup_summary'))
self.assertIn('table-layout:fixed;',result['html'])
self.assertIn('overflow-wrap:break-word;',result['html'])
unrelated=deliver('node_reconnect',{'hostname':'node-a'},'OK')
self.assertNotIn('table-layout:fixed;',unrelated['html'])
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,399 @@
"""Inert producer-to-renderer checks for notification outcome claims."""
import ast
import copy
import json
import re
import time
import sys
import types
import unittest
from pathlib import Path
from typing import Any
from unittest.mock import patch
from notification_fixture import templates as actual_templates
ROOT = Path(__file__).resolve().parents[3]
SCRIPTS = ROOT / 'AppImage/scripts'
CATALOG = ROOT / 'AppImage/messages/en/common.json'
EXPECTED = {'system_restore_completed': {'body': 'Post-restore tasks completed in background.\n\nGuests applied: {guests}\nBind-mount stubs: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}\nThe node is now fully ready to use.'}}
def extract(path, name, owner=None, namespace=None):
tree = ast.parse(path.read_text())
nodes = tree.body
if owner:
nodes = next(n.body for n in nodes if isinstance(n, ast.ClassDef) and n.name == owner)
node = next(n for n in nodes if isinstance(n, ast.FunctionDef) and n.name == name)
node.decorator_list = []
ns = namespace if namespace is not None else {}
exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), ns)
return ns[name]
def renderer(catalog, translated=None):
path = SCRIPTS / 'notification_templates.py'
tree = ast.parse(path.read_text())
templates = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'TEMPLATES' for t in n.targets)))
def lookup(obj, key):
for part in key.split('.'):
obj = obj.get(part) if isinstance(obj, dict) else None
return obj
def message(key, language='en', **values):
source = translated if language == 'it' and translated is not None else catalog
namespace = source['runtime']['notifications']
value = lookup(namespace, key) or lookup(catalog['runtime']['notifications'], key) or ''
return value.format_map(type('Safe', (dict,), {'__missing__': lambda self, k: ''})(values))
ns = {'TEMPLATES': templates, 'Dict': dict, 'Any': Any, 'time': time, 're': re,
'_get_hostname': lambda: 'node-a',
'_load_runtime_catalog': lambda lang: (translated if lang == 'it' and translated is not None
else catalog)['runtime']['notifications'],
'_catalog_value': lookup, 'runtime_message': message}
from typing import Optional
ns['Optional'] = Optional
extract(path, '_parse_vzdump_table', namespace=ns)
extract(path, '_parse_vzdump_message', namespace=ns)
extract(path, '_format_vzdump_body', namespace=ns)
return templates, extract(path, 'render_template', namespace=ns)
class OutcomeWording(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.catalog = json.loads(CATALOG.read_text())
cls.templates, render = renderer(cls.catalog)
cls.render = staticmethod(render)
def test_spanish_restore_uses_maintainer_guests_terminology(self):
import importlib.util
spec = importlib.util.spec_from_file_location('isolated_spanish_restore', SCRIPTS / 'notification_templates.py')
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
result = module.render_template('system_restore_completed', {
'hostname':'node-a','guests':3,'stubs':0,'stale_nodes':0,
'components':1,'duration':'2m','warnings_block':''}, 'es')
self.assertIn('Guests aplicados: 3', result['body'])
self.assertNotIn('invitados', result['body'].lower())
def test_settings_labels_stay_at_upstream_values_in_all_locales(self):
# Frozen from develop eb7cc548; CI shallow checkouts have no base history.
labels = {'en': {'backup_complete': 'Backup complete', 'error_resolved': 'Recovery notification'}, 'de': {'backup_complete': 'Sicherung abgeschlossen', 'error_resolved': 'Wiederherstellungsbenachrichtigung'}, 'es': {'backup_complete': 'Backup completado', 'error_resolved': 'Notificación de recuperación'}, 'fr': {'backup_complete': 'Sauvegarde terminée', 'error_resolved': 'Notification de récupération'}, 'it': {'backup_complete': 'Backup completato', 'error_resolved': 'Notifica di recupero'}, 'pt': {'backup_complete': 'Backup concluído', 'error_resolved': 'Notificação de recuperação'}, 'sk': {'backup_complete': 'Záloha bola dokončená', 'error_resolved': 'Problém bol vyriešený'}, 'sv': {'backup_complete': 'Säkerhetskopieringen är klar', 'error_resolved': 'Återställningsmeddelande'}}
for lang in labels:
path = f'AppImage/messages/{lang}/common.json'
current = json.loads((ROOT / path).read_text())
for event in ('backup_complete', 'error_resolved'):
with self.subTest(lang=lang, event=event):
expected = labels[lang][event]
self.assertEqual(current['runtime']['notifications']['templates'][event]['label'], expected)
if lang == 'en': self.assertEqual(self.templates[event]['label'], expected)
def test_exact_four_english_leaves_match_source_and_catalog(self):
for event, fields in EXPECTED.items():
for field, value in fields.items():
with self.subTest(event=event, field=field):
self.assertEqual(self.templates[event][field], value)
self.assertEqual(self.catalog['runtime']['notifications']['templates'][event][field], value)
def test_webhook_backup_outcome_is_evidence_based_without_rerouting(self):
path = SCRIPTS / 'notification_events.py'
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
ns['NotificationEvent'] = Event
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
class Receiver:
_hostname = 'node-a'
_classify_pve = classify
_map_severity = staticmethod(severity_map)
_backup_outcome = staticmethod(backup_outcome)
def __init__(self): self._queue = Queue()
header = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('VMID','Name','Status','Time','Size','Filename')
row_ok = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('104','alpha','OK','00:01:00','1.5 GiB','archive')
row_warning = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','WARNINGS','00:01:00','1.5 GiB','archive')
row_error = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','ERROR','00:01:00','1.5 GiB','archive')
row_err = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','err','00:01:00','1.5 GiB','archive')
truncated = 'INFO: Log output was too long to be displayed. Please see task log for details.'
cases = [
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_err+'\nTotal running time: 00:02:00', 'failed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\n'+header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
('vzdump', 'info', header+'\n'+row_err+'\nTotal running time: 00:01:00\n'+truncated, 'failed'),
('vzdump', 'warning', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'completed_with_warnings'),
('vzdump', 'info', header+'\n'+row_ok, 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_ok, 'unconfirmed'),
('vzdump', 'warning', header+'\n'+row_err+'\nTotal running time: 00:01:00', 'failed'),
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00\n'+truncated, 'unconfirmed'),
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00\nERROR: archive write failed', 'failed'),
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
('vzdump', 'info', header+'\n'+row_error, 'failed'),
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'confirmed'),
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00', 'unconfirmed'),
('vzdump', 'info', header+'\n'+row_ok[:30], 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'confirmed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nWARNING: skipped file', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: TASK OK\n104 alpha WARNINGS: 1', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Finished Backup of VM 105 (00:01:00)', 'confirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed for VM 104', 'failed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'failed'),
('', 'warning', 'Backup scheduled', 'unconfirmed'),
('', 'info', 'Backup complete', 'unconfirmed'),
]
for kind, severity, message, expected in cases:
with self.subTest(message=message, severity=severity):
receiver = Receiver()
receive(receiver, {'fields': {'type': kind}, 'severity': severity,
'title': 'Backup', 'message': message})
event = receiver._queue.items[0]
self.assertEqual(event.event_type, 'backup_complete')
self.assertEqual(event.data['backup_outcome'], expected)
def test_backup_classifier_preserves_confirmed_and_unverified_paths(self):
path = SCRIPTS / 'notification_events.py'
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
ns['NotificationEvent'] = Event
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
class Receiver:
_hostname = 'node-a'
_classify_pve = classify
_map_severity = staticmethod(severity_map)
_backup_outcome = staticmethod(backup_outcome)
def __init__(self): self._queue = Queue()
samples = [('vzdump', 'info', 'Backup', 'job finished'),
('vzdump', 'warning', 'Backup', 'job incomplete'),
('', 'warning', 'backup job', 'Backup scheduled')]
for kind, severity, title, message in samples:
with self.subTest(kind=kind, message=message):
event, entity, _ = classify(None, kind, severity, title, message)
self.assertEqual((event, entity), ('backup_complete', 'vm'))
receiver = Receiver()
reply = receive(receiver, {'fields': {'type': kind}, 'severity': severity,
'title': title, 'message': message})
self.assertEqual(reply['event_type'], event)
self.assertEqual(len(receiver._queue.items), 1)
emitted = receiver._queue.items[0]
self.assertEqual(emitted.severity, 'WARNING' if severity == 'warning' else 'INFO')
output = self.render(emitted.event_type, emitted.data, 'en')
self.assertIn('Backup outcome unconfirmed', output['title'])
self.assertEqual(output['body'].splitlines()[-1], message) # raw PVE body retained
self.assertEqual(classify(None, 'vzdump', 'error', 'Backup', 'failed')[0], 'backup_fail')
def test_incomplete_vzdump_log_does_not_certify_a_guest(self):
from typing import Dict, Optional
ns = {'re': re, 'Dict': Dict, 'Optional': Optional, 'Any': Any,
'runtime_message': lambda key, lang, **kw: key}
extract(SCRIPTS / 'notification_templates.py', '_parse_vzdump_table', namespace=ns)
parser = extract(SCRIPTS / 'notification_templates.py', '_parse_vzdump_message', namespace=ns)
formatter = extract(SCRIPTS / 'notification_templates.py', '_format_vzdump_body', namespace=ns)
incomplete = parser('INFO: Starting Backup of VM 104 (qemu)')
table_header = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('VMID','Name','Status','Time','Size','Filename')
table_err = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('104','alpha','err','00:01:00','1.5 GiB','archive')
failed_table = parser(table_header+'\n'+table_err+'\nTotal running time: 00:01:00')
self.assertEqual(failed_table['vms'][0]['status'].lower(), 'error')
self.assertIn('❌', formatter(failed_table, False, 'en'))
self.assertEqual(incomplete['vms'][0]['status'], 'unknown')
self.assertNotIn('✅', formatter(incomplete, False, 'en'))
mixed = parser('INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Starting Backup of VM 105 (lxc)')
self.assertEqual([vm['status'] for vm in mixed['vms']], ['ok', 'unknown'])
formatted = formatter(mixed, False, 'en')
self.assertEqual(formatted.count('✅'), 1)
self.assertNotIn('❌', formatted)
conflicting = parser('INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed\nINFO: Finished Backup of VM 104 (00:01:00)')
self.assertEqual(conflicting['vms'][0]['status'], 'error')
def test_backup_render_preserves_success_and_marks_unconfirmed_and_failure(self):
for outcome, message, title_part, body_part in (
('confirmed', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'Backup complete', '✅'),
('unconfirmed', 'INFO: Starting Backup of VM 104 (qemu)', 'Backup outcome unconfirmed', '❔'),
('failed', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'Backup error reported', '❌'),
):
with self.subTest(outcome=outcome):
output = self.render('backup_complete', {'hostname': 'node-a', 'pve_type': 'vzdump',
'pve_message': message, 'pve_title': 'Backup complete', 'backup_outcome': outcome}, 'en')
self.assertIn(title_part, output['title'])
self.assertIn(body_part, output['body'])
if outcome != 'confirmed': self.assertNotIn('Backup complete', output['title'])
output = self.render('backup_complete', {'hostname': 'node-a', 'vmname': 'vm', 'vmid': '104'}, 'en')
self.assertIn('Backup outcome unconfirmed', output['title'])
self.assertNotIn('successfully', output['body'])
conflict = self.render('backup_complete', {'hostname': 'node-a','backup_outcome':'failed',
'pve_message':'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nERROR: archive write failed'}, 'en')
self.assertIn('ERROR: archive write failed', conflict['body'])
self.assertNotIn('Backup complete', conflict['title'])
def test_confirmed_title_keeps_single_guest_and_destination_without_misnaming_batches(self):
import importlib.util
spec = importlib.util.spec_from_file_location('isolated_backup_title', SCRIPTS / 'notification_templates.py')
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
actual_render = module.render_template
log = ('INFO: starting new backup job: vzdump 104 --storage PBS-Cloud --mode snapshot\n'
'INFO: Starting Backup of VM 104 (qemu)\nINFO: VM Name: Alpha\n'
'INFO: Finished Backup of VM 104 (00:01:00)')
single = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
'pve_message':log}, 'en')
self.assertIn('PBS-Cloud',single['title'])
self.assertIn('VM Alpha (104)',single['title'])
batch = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
'pve_message':log+'\nINFO: Starting Backup of VM 105 (lxc)\n'
'INFO: Finished Backup of VM 105 (00:01:00)'}, 'en')
self.assertIn('PBS-Cloud',batch['title'])
self.assertNotIn('Alpha (104)',batch['title'])
no_context = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed'}, 'en')
self.assertEqual(no_context['title'], 'node-a: Backup complete')
named = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
'pve_message':log.replace('Alpha','Alpha {literal}')}, 'en')
self.assertIn('Alpha {literal} (104)', named['title'])
def test_html_email_badge_and_backup_status_are_context_specific(self):
import html
path = SCRIPTS / 'notification_channels.py'
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
with self.subTest(lang=lang):
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
english = self.catalog['runtime']['notifications']
def text(key, data=None, **values):
def lookup(source):
value = source['channels']
for part in key.split('.'):
value = value.get(part) if isinstance(value, dict) else None
return value
return (lookup(catalog) or lookup(english) or '').format(**values)
ns = {'Dict': dict, 'Optional': __import__('typing').Optional,
'_runtime_text': text, '_runtime_notification_text': lambda key, data=None: ''}
build = extract(path, '_build_detail_rows', 'EmailChannel', ns)
fmt = extract(path, '_format_html', 'EmailChannel', ns)
class Email:
from notification_channels import EmailChannel
_SEV_STYLE = EmailChannel._SEV_STYLE
_SEV_DEFAULT = EmailChannel._SEV_DEFAULT
subject_prefix = 'ProxMenux'
_build_detail_rows = staticmethod(build)
unrelated = fmt(Email(), 'Reconnected', 'Body', 'OK', {'_event_type': 'node_reconnect',
'_notification_language': lang, '_group': 'cluster'})
self.assertIn('>' + catalog['channels']['email']['severity']['ok'].upper() + '</span>', unrelated)
for outcome, status in [('confirmed', 'completed'), ('unconfirmed','unconfirmed'), ('failed','failed')]:
email = fmt(Email(), 'Backup', 'Details', 'INFO', {'_event_type': 'backup_complete',
'backup_outcome': outcome, '_notification_language': lang, '_group': 'backup'})
label = catalog['channels']['email']['status'].get(status) or english['channels']['email']['status'][status]
self.assertIn(label, html.unescape(email))
badge_label = label.upper()
self.assertIn('>' + badge_label + '</span>', html.unescape(email))
if outcome == 'failed':
self.assertIn('color:#dc2626;font-weight:600;', email)
self.assertIn('background:#fef2f2;', email)
elif outcome == 'unconfirmed':
self.assertIn('background:#f9fafb;', email)
else:
self.assertIn('background:#f0fdf4;', email)
def test_actual_all_locale_rendering_and_fallback_for_backup_outcomes(self):
import importlib.util
module_path = SCRIPTS / 'notification_templates.py'
spec = importlib.util.spec_from_file_location('isolated_notification_templates', module_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # templates and catalogs only; no manager or sends
samples = {
'confirmed': 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)',
'unconfirmed': 'INFO: Starting Backup of VM 104 (qemu)',
'failed': 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed',
}
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
for state, message in samples.items():
with self.subTest(lang=lang, state=state):
data = {'hostname':'node-with-a-long-name','backup_outcome':state,'pve_type':'vzdump',
'pve_message':message, 'pve_title':'Backup complete', '_notification_language':lang}
result = module.render_template('backup_complete', data, lang)
if state != 'unconfirmed':
key = 'confirmedTitle' if state == 'confirmed' else 'errorTitle'
expected_title = (catalog.get('backup', {}).get(key) or
self.catalog['runtime']['notifications']['backup'][key]).format(hostname=data['hostname'])
self.assertTrue(result['title'].startswith(expected_title), result['title'])
else:
self.assertEqual(result['title'], module.runtime_message('backup.unconfirmedTitle', lang, hostname=data['hostname']))
self.assertNotIn('{hostname}', result['title'])
if state == 'unconfirmed':
source = (catalog if catalog.get('backup', {}).get('unconfirmedBody')
else self.catalog['runtime']['notifications'])
self.assertIn(source['backup']['unconfirmedBody'], result['body'])
if state == 'failed':
self.assertIn(catalog.get('backup', {}).get('errorBody') or
self.catalog['runtime']['notifications']['backup']['errorBody'], result['body'])
enriched, _ = module.enrich_with_emojis('backup_complete', result['title'], result['body'], data)
self.assertTrue(enriched.startswith({'confirmed':'💾✅','unconfirmed':'💾❔','failed':'💾❌'}[state]))
restore = module.render_template('system_restore_completed', {'hostname':'node', 'guests':4,
'stubs':1,'stale_nodes':2,'components':1,'duration':'2m','warnings_block':'Missing module'},lang)
self.assertIn('Missing module',restore['body'])
if lang == 'en': self.assertIn('fully ready',restore['body'].lower())
def test_warning_and_clean_restore_keep_only_reported_outcome(self):
for warnings in ('', '⚠️ Boot sanity: missing modules\n'):
with self.subTest(warnings=warnings):
events = []
ns = {'request': types.SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
'hostname': 'node-a', 'guests': '2', 'stubs': '0', 'stale_nodes': '0',
'components': 'none', 'duration': '2m',
'warnings': 'missing modules' if warnings else ''}),
'notification_manager': types.SimpleNamespace(emit_event=lambda **kw: events.append(kw)),
'jsonify': lambda obj: obj}
handler = extract(SCRIPTS / 'flask_notification_routes.py', 'internal_restore_event', namespace=ns)
response, status = handler()
self.assertEqual((status, response['event_type']), (200, 'system_restore_completed'))
event = events[0]
self.assertEqual(event['severity'], 'WARNING' if warnings else 'INFO')
result = self.render(event['event_type'], event['data'], 'en')
self.assertIn('Post-restore tasks completed', result['body'])
self.assertIn('fully ready', result['body'])
if warnings: self.assertIn('missing modules', result['body'])
def test_rich_backup_icon_tracks_outcome_and_digest_default_is_neutral(self):
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
def assign(name):
return ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == name for t in n.targets)))
icons = assign('EVENT_EMOJI')
self.assertNotIn('✅', icons['backup_complete'])
ns = {'TEMPLATES': assign('TEMPLATES'), 'EVENT_EMOJI': icons,
'Dict': dict, 'Any': Any,
'CATEGORY_EMOJI': assign('CATEGORY_EMOJI'), 'SEVERITY_ICONS': assign('SEVERITY_ICONS'),
'FIELD_EMOJI': assign('FIELD_EMOJI'), '_localized_template_labels': lambda *a: {},
'_lxc_update_label_icons': lambda *a: {}}
enrich = extract(SCRIPTS / 'notification_templates.py', 'enrich_with_emojis', namespace=ns)
for state, icon in [('confirmed', '💾✅'), ('unconfirmed', '💾❔'), ('failed', '💾❌')]:
with self.subTest(state=state):
title, body = enrich('backup_complete', 'node-a: Backup', 'Backup report',
{'backup_outcome': state, '_notification_language': 'en'})
self.assertTrue(title.startswith(icon), title)
def test_uncertain_outcomes_have_no_blanket_success_icon(self):
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
icon_map = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == 'EVENT_EMOJI' for t in n.targets)))
self.assertNotIn('✅', icon_map['backup_complete']) # buffered digest has no outcome metadata
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,50 @@
"""Principal diagnostics must not be deduplicated against inventory substrings."""
import unittest
from notification_fixture import receive, LANGUAGES
from notification_final_fixture import deliver
def noisy_report(name='ordinary-web', storage='PBS', filename='null', cause='denied'):
# Frozen native Perl notifier/log-reader shape; fixed-column table is
# source-modeled from the official plaintext renderer, not a live PVE run.
return ('\nDetails\n=======\nVMID Name Status Time Size Filename\n'
f'100 {name} err 1m 1s 0 B {filename}\n\n'
'Total running time: 1m 1s\nTotal size: 0 B\n\nLogs\n====\n'
f'vzdump --all 1 --storage {storage} --mode snapshot\n\n' +
'\n'.join(f'100: 2026-09-29 17:00:00 ERROR: earlier diagnostic {i}' for i in range(12)) +
f'\n100: 2026-09-29 17:00:00 ERROR: {cause}\n\n')
class PrincipalInventoryTests(unittest.TestCase):
def assert_principal(self, raw, cause):
event = receive(raw, 'error', 'vzdump backup status (raw-host): backup failed: ' + cause)
for language in LANGUAGES:
for manual in (False, True):
with self.subTest(language=language, manual=manual):
result = deliver(event.event_type, {**event.data, 'hostname':'alias {rack.location}'},
event.severity, language, manual=manual)
exact = [line for line in result['body'].splitlines()
if line.strip() == cause or line.rstrip().endswith('ERROR: ' + cause)]
self.assertEqual(len(exact), 1)
diagnostics = [line for line in result['body'].splitlines() if 'ERROR:' in line]
self.assertLessEqual(len(diagnostics), 8)
self.assertLessEqual(len('\n'.join(diagnostics)), 1024)
self.assertTrue(all(len(line) <= 512 for line in diagnostics))
self.assertEqual(result['data']['pve_message'], raw)
self.assertNotIn('raw-host', result['text'])
self.assertIn('alias {rack.location}', result['text'])
self.assertEqual(result['text'].count('ERROR: ' + cause), 1)
def test_late_principal_survives_incidental_guest_storage_and_archive(self):
for name, storage, filename in (('ordinary-web','PBS','null'), ('denied','PBS','null'),
('ordinary-web','denied','null'), ('ordinary-web','PBS','denied.tar')):
with self.subTest(name=name, storage=storage, filename=filename):
self.assert_principal(noisy_report(name,storage,filename), 'denied')
def test_raw_principal_braces_and_markup_survive_once(self):
cause = 'denied {rack.location} <native>'
self.assert_principal(noisy_report('denied', cause=cause), cause)
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,192 @@
"""Frozen PVE 9.2 report through inert actual receiver and renderers.
No git history, host-management import, notification send or generated fixture.
"""
import ast
import importlib.util
import re
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPTS = ROOT / 'AppImage/scripts'
sys.path.insert(0, str(SCRIPTS))
import notification_templates as templates
PVE92 = '''Details
=======
VMID Name Status Time Size Filename
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
Total running time: 1m 1s
Total size: 1 GiB
'''
def receiver():
tree = ast.parse((SCRIPTS / 'notification_events.py').read_text())
owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'ProxmoxHookWatcher')
names = ('_classify_pve', '_map_severity', '_backup_outcome', 'process_webhook')
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
class Event:
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
ns['NotificationEvent'] = Event
for name in names:
node = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == name)
node.decorator_list = []
exec(compile(ast.Module(body=[node], type_ignores=[]), '<inert-webhook>', 'exec'), ns)
class Queue:
def __init__(self): self.items = []
def put(self, event): self.items.append(event)
class Receiver:
_hostname = 'node-a'
_classify_pve = ns['_classify_pve']
_map_severity = staticmethod(ns['_map_severity'])
_backup_outcome = staticmethod(ns['_backup_outcome'])
process_webhook = ns['process_webhook']
def __init__(self): self._queue = Queue()
return Receiver()
def event_for(message, severity='info'):
target = receiver()
target.process_webhook({'fields': {'type': 'vzdump'}, 'severity': severity,
'title': 'Backup', 'message': message})
return target._queue.items[0]
class PVE92Tests(unittest.TestCase):
def test_exact_maintainer_report_is_confirmed(self):
event = event_for(PVE92)
self.assertEqual(event.data['backup_outcome'], 'confirmed')
result = templates.render_template(event.event_type, event.data, 'en')
self.assertIn('Backup complete', result['title'])
self.assertIn('web (100)', result['title'])
self.assertIn('✅', result['body'])
def test_failed_guest_identity_does_not_come_from_timestamp_or_ok_guest(self):
failed = '101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z'
ok = PVE92.splitlines()[3]
for rows in (ok + '\n' + failed, failed + '\n' + ok):
for severity in ('info', 'error'):
with self.subTest(rows=rows, severity=severity):
message = ('INFO: 100 01:01:06 OK\nINFO: Starting Backup of VM 100 (qemu)\n'
+ PVE92.replace(ok, rows))
event = event_for(message, severity)
self.assertEqual(event.data['backup_outcome'], 'failed')
self.assertEqual(event.data.get('vmname'), 'db')
self.assertEqual(event.data.get('vmid'), '101')
result = templates.render_template(event.event_type, event.data, 'en')
self.assertIn('db (101)', result['title'])
self.assertNotIn('web', result['title'])
self.assertNotIn('01:01:06', result['title'])
self.assertIn('❌ CT db (101)', result['body'])
def test_pbs_prefix_is_used_in_parsed_type_and_title(self):
for prefix, kind, label in (('vm', 'qemu', 'VM'), ('ct', 'lxc', 'CT'),
('other', '', 'VM/CT')):
with self.subTest(prefix=prefix):
message = PVE92.replace('vm/100/', prefix + '/100/')
parsed = templates._parse_vzdump_message(message)
self.assertEqual(parsed['vms'][0]['type'], kind)
event = event_for(message)
result = templates.render_template(event.event_type, event.data, 'en')
self.assertIn(label + ' web (100)', result['title'])
def test_slovak_uses_normal_locale_resolution_without_source_sentence_overrides(self):
from unittest.mock import patch
data = {'hostname': 'node-a', 'category': 'temperature', 'entity_suffix': '',
'reason': 'old', 'duration': '3d', 'original_severity': 'WARNING',
'vmname': 'web', 'vmid': '100', 'storage': 'PBS', 'size': '1 GiB',
'guests': 3, 'stubs': 0, 'stale_nodes': 0, 'components': 1,
'warnings_block': ''}
slovak = templates._load_runtime_catalog('sk')
english = templates._load_runtime_catalog('en')
for event, field in (('system_restore_completed', 'body'),):
with self.subTest(event=event, field=field):
value = slovak['templates'][event][field]
result = templates.render_template(event, data, 'sk')
self.assertEqual(result[field], value.format(**data))
self.assertNotIn(value, (SCRIPTS / 'notification_templates.py').read_text())
# Independently updated and absent leaves use the usual provider.
import copy
future = copy.deepcopy(slovak)
future['templates'][event][field] = 'REVIEWED {hostname}'
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: future if lang == 'sk' else english):
self.assertEqual(templates.render_template(event, data, 'sk')[field], 'REVIEWED node-a')
future['templates'][event].pop(field)
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: future if lang == 'sk' else english):
self.assertEqual(templates.render_template(event, data, 'sk')[field],
templates.render_template(event, data, 'en')[field])
def test_complete_table_beats_only_truncated_supplemental_log(self):
message = ('INFO: Starting Backup of VM 100 (qemu)\n' + PVE92 +
'\nLogs\n====\nINFO: Log output was too long to be displayed. Please see task log for details.')
self.assertEqual(event_for(message).data['backup_outcome'], 'confirmed')
for message, severity, expected in (
(PVE92.replace('ok ', 'OK '), 'info', 'confirmed'),
(message, 'warning', 'completed_with_warnings'),
(message, 'error', 'failed'),
(message + '\nERROR: archive write failed', 'info', 'failed'),
(message + '\nWARNING: skipped file', 'info', 'completed_with_warnings'),
(PVE92.replace('ok ', 'WARNINGS '), 'info', 'unconfirmed'),
):
with self.subTest(message=message, severity=severity):
self.assertEqual(event_for(message, severity).data['backup_outcome'], expected)
def test_incomplete_and_unrelated_sections_cannot_certify_table(self):
for message in (
PVE92.split('Total running time:')[0],
PVE92.replace('vm/100/2026-09-29T17:00:00Z', ''),
PVE92.replace('\n\nTotal', '\n\nLogs\n======\nTotal'),
PVE92.replace('\n\nTotal', '\n\nUnrelated section\n100 web ok\nTotal'),
PVE92.replace('1m 1s 1 GiB', 'nonsense 1 GiB'),
PVE92.replace('1 GiB vm/', 'garbage vm/'),
PVE92.replace('1m 1s 1 GiB vm/', '1m 1s'),
):
with self.subTest(message=message):
# Complete guest logs cannot rescue a genuinely incomplete table.
message += '\nINFO: Starting Backup of VM 100 (qemu)\nINFO: Finished Backup of VM 100 (00:01:01)'
self.assertEqual(event_for(message).data['backup_outcome'], 'unconfirmed')
def test_blank_line_between_rows_does_not_hide_a_failure(self):
message = PVE92.replace('\n\nTotal',
'\n\n101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z\n\nTotal')
event = event_for(message)
self.assertEqual(event.data['backup_outcome'], 'failed')
result = templates.render_template(event.event_type, event.data, 'en')
self.assertIn('db (101)', result['title'])
self.assertIn('❌ CT db (101)', result['body'])
def test_changed_titles_and_rows_reach_actual_html_email_in_all_locales(self):
import html
from notification_channels import EmailChannel
email = object.__new__(EmailChannel)
email.subject_prefix = '[ProxMenux]'
failed = PVE92.replace('\n\nTotal',
'\n101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z\n\nTotal')
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
for message, severity in ((PVE92, 'info'), (failed, 'info'), (failed, 'error')):
with self.subTest(lang=lang, severity=severity, message=message):
event = event_for(message, severity)
result = templates.render_template(event.event_type, event.data, lang)
context = {**event.data, '_event_type': event.event_type,
'_notification_language': lang, '_group': result['group']}
markup = html.unescape(email._format_html(result['title'], result['body'], event.severity, context))
self.assertIn(result['title'], markup)
if event.data['backup_outcome'] == 'failed':
self.assertIn('db (101)', result['title'])
self.assertNotIn('web', result['title'])
self.assertIn('❌ CT db (101)', markup)
status = templates.runtime_message('channels.email.status.failed', lang)
else:
self.assertIn('VM web (100)', result['title'])
self.assertIn('✅ VM web (100)', markup)
status = templates.runtime_message('channels.email.status.completed', lang)
badge = (templates.runtime_message('channels.email.severity.critical', lang)
if event.event_type == 'backup_fail' else status)
self.assertIn('>' + badge.upper() + '</span>', markup)
if __name__ == '__main__': unittest.main()
@@ -0,0 +1,24 @@
"""Review regressions at actual consumer seams; no operational imports."""
import unittest
from notification_fixture import receive, LANGUAGES
from notification_final_fixture import deliver
class ReviewCorrectionTests(unittest.TestCase):
def test_subject_equivalent_late_error_is_reserved_before_cap(self):
cause = 'job-end hook denied'
# Frozen native Perl notifier/log-reader output; Rust table source-modeled.
raw = '\nDetails\n=======\nVMID Name Status Time Size Filename \n100 web err 1m 1s 0 B null \n\nTotal running time: 1m 1s\nTotal size: 0 B\n\nLogs\n====\nvzdump --all 1 --storage PBS --mode snapshot\n\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 0\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 1\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 2\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 3\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 4\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 5\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 6\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 7\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 8\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 9\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 10\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 11\n100: 2026-09-29 17:00:00 ERROR: job-end hook denied\n\n\n'
event = receive(raw, 'error', 'vzdump backup status (raw-host): backup failed: ' + cause)
for language in LANGUAGES:
for manual in (False, True):
with self.subTest(language=language, manual=manual):
result = deliver(event.event_type, {**event.data, 'hostname':'alias {rack.location}'}, event.severity, language, manual=manual)
self.assertEqual(result['text'].count(cause), 1)
self.assertNotIn('raw-host', result['text'])
diagnostics = [line for line in result['body'].splitlines() if 'ERROR:' in line]
self.assertLessEqual(len(diagnostics), 8)
self.assertLessEqual(len('\n'.join(diagnostics)), 1024)
self.assertTrue(all(len(line) <= 512 for line in diagnostics))
self.assertEqual(result['data']['pve_message'], raw)
if __name__ == '__main__': unittest.main()
+14 -1
View File
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Fehlgeschlagen",
"completed": "Abgeschlossen",
"started": "Gestartet"
"started": "Gestartet",
"unconfirmed": "Nicht bestätigt",
"completed_with_warnings": "Mit Warnungen abgeschlossen"
},
"report": "{group} Bericht",
"details": "Details",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname}: Backup-Ergebnis nicht bestätigt",
"confirmedTitle": "{hostname}: Backup abgeschlossen",
"confirmedBody": "Backup erfolgreich abgeschlossen.",
"errorTitle": "{hostname}: Backup-Fehler gemeldet",
"errorBody": "Der Backup-Bericht enthält einen Fehler.",
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt.",
"warningTitle": "{hostname}: Sicherung mit Warnungen abgeschlossen",
"warningBody": "Sicherung mit Warnungen abgeschlossen.",
"diagnosticsOmitted": "Weitere Diagnosezeilen oder Text ausgelassen: {count}. Originalbericht bleibt erhalten."
}
}
}
File diff suppressed because one or more lines are too long
+15 -2
View File
@@ -6556,7 +6556,7 @@
"label": "Reinicio del sistema"
},
"system_restore_completed": {
"title": "{hostname}: restauración del host finalizada",
"title": "{hostname}: Restauración del host finalizada",
"body": "Tareas posteriores a la restauración completadas en segundo plano.\n\nGuests aplicados: {guests}\nStubs de bind mount: {stubs}\nDirectorios de nodos obsoletos eliminados: {stale_nodes}\nComponentes reinstalados: {components}\nDuración: {duration}\n{warnings_block}\nEl nodo está listo para usarse.",
"label": "Restauración del host completada"
},
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Fallido",
"completed": "Completado",
"started": "Iniciado"
"started": "Iniciado",
"unconfirmed": "Sin confirmar",
"completed_with_warnings": "Completado con advertencias"
},
"report": "Informe de {group}",
"details": "Detalles",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "Lecturas altas registradas a lo largo de {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname}: Resultado del backup sin confirmar",
"confirmedTitle": "{hostname}: Backup completado",
"confirmedBody": "Backup completado correctamente.",
"errorTitle": "{hostname}: Backup fallido",
"errorBody": "El informe del backup contiene un error.",
"unconfirmedBody": "El resultado del backup no está confirmado.",
"warningTitle": "{hostname}: Backup completado con advertencias",
"warningBody": "Backup completado con advertencias.",
"diagnosticsOmitted": "Líneas o texto de diagnóstico omitidos: {count}. Se conserva el informe original."
}
}
}
+14 -1
View File
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Échec",
"completed": "Terminé",
"started": "Commencé"
"started": "Commencé",
"unconfirmed": "Non confirmé",
"completed_with_warnings": "Terminée avec avertissements"
},
"report": "Rapport {group}",
"details": "Détails",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "Les relevés élevés s'étendent sur {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname} : résultat de la sauvegarde non confirmé",
"confirmedTitle": "{hostname} : sauvegarde terminée",
"confirmedBody": "Sauvegarde terminée avec succès.",
"errorTitle": "{hostname} : erreur signalée lors de la sauvegarde",
"errorBody": "Le rapport de sauvegarde contient une erreur.",
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé.",
"warningTitle": "{hostname}: Sauvegarde terminée avec avertissements",
"warningBody": "Sauvegarde terminée avec avertissements.",
"diagnosticsOmitted": "Lignes ou texte de diagnostic omis : {count}. Rapport original conservé."
}
}
}
+14 -1
View File
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Fallito",
"completed": "Completato",
"started": "Iniziato"
"started": "Iniziato",
"unconfirmed": "Non confermato",
"completed_with_warnings": "Completato con avvisi"
},
"report": "{group} Rapporto",
"details": "Dettagli",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "Intervallo dei campioni sopra soglia: {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname}: esito del backup non confermato",
"confirmedTitle": "{hostname}: backup completato",
"confirmedBody": "Backup completato correttamente.",
"errorTitle": "{hostname}: errore segnalato nel backup",
"errorBody": "Il rapporto del backup contiene un errore.",
"unconfirmedBody": "L’esito del backup non è confermato.",
"warningTitle": "{hostname}: Backup completato con avvisi",
"warningBody": "Backup completato con avvisi.",
"diagnosticsOmitted": "Righe o testo diagnostico omessi: {count}. Il report originale è conservato."
}
}
}
+14 -1
View File
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Falhou",
"completed": "Concluído",
"started": "Iniciado"
"started": "Iniciado",
"unconfirmed": "Não confirmado",
"completed_with_warnings": "Concluído com avisos"
},
"report": "Relatório {group}",
"details": "Detalhes",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "As amostras elevadas abrangem {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname}: resultado do backup não confirmado",
"confirmedTitle": "{hostname}: backup concluído",
"confirmedBody": "Backup concluído com sucesso.",
"errorTitle": "{hostname}: erro comunicado no backup",
"errorBody": "O relatório do backup contém um erro.",
"unconfirmedBody": "O resultado do backup não está confirmado.",
"warningTitle": "{hostname}: Backup concluído com avisos",
"warningBody": "Backup concluído com avisos.",
"diagnosticsOmitted": "Linhas ou texto de diagnóstico omitidos: {count}. Relatório original preservado."
}
}
}
+14 -1
View File
@@ -6978,7 +6978,9 @@
"status": {
"failed": "Misslyckades",
"completed": "Klar",
"started": "Startat"
"started": "Startat",
"unconfirmed": "Obekräftat",
"completed_with_warnings": "Klar med varningar"
},
"report": "{group} Rapportera",
"details": "Detaljer",
@@ -6988,6 +6990,17 @@
},
"temperature": {
"sampleSpan": "De höga mätvärdena sträcker sig över {duration}."
},
"backup": {
"unconfirmedTitle": "{hostname}: säkerhetskopians resultat obekräftat",
"confirmedTitle": "{hostname}: säkerhetskopiering klar",
"confirmedBody": "Säkerhetskopieringen slutfördes utan fel.",
"errorTitle": "{hostname}: fel rapporterat vid säkerhetskopiering",
"errorBody": "Rapporten om säkerhetskopieringen innehåller ett fel.",
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat.",
"warningTitle": "{hostname}: Säkerhetskopiering klar med varningar",
"warningBody": "Säkerhetskopiering klar med varningar.",
"diagnosticsOmitted": "Utelämnade diagnosrader eller text: {count}. Originalrapporten bevaras."
}
}
}
+55 -13
View File
@@ -1037,12 +1037,33 @@ class EmailChannel(NotificationChannel):
# Determine group for section header
event_type = data.get('_event_type', '')
if event_type == 'backup_complete':
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
sev.update(self._SEV_STYLE['OK'])
status = 'completed'
elif outcome == 'completed_with_warnings':
sev.update(self._SEV_STYLE['WARNING'])
status = 'completed_with_warnings'
elif outcome == 'failed':
sev.update(self._SEV_STYLE['CRITICAL'])
status = 'failed'
else:
sev.update(self._SEV_DEFAULT)
status = 'unconfirmed'
sev['label'] = _runtime_text(f'email.status.{status}', data)
group = data.get('_group', 'other')
# Keep unbroken recorded text inside the temperature email's table.
# Both properties are inline for mail clients; other events retain
# their original markup and layout.
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if event_type == 'temp_high' else ''
temp_table_layout = 'table-layout:fixed;' if event_type == 'temp_high' else ''
# Scoped inline mail-compatible wrapping for authoritative raw-context
# bodies, including restore bodies released from quiet hours.
backup_email = event_type in {'backup_complete', 'backup_fail'}
wrap_body = (event_type in {'temp_high', 'system_restore_completed'}
or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
# Keep wrapping event-scoped; unrelated mail remains byte-identical.
context_email = backup_email
backup_title_wrap = temp_cell_wrap if context_email else ''
backup_metadata_layout = 'table-layout:fixed;' if context_email else ''
section_label = _runtime_text(f'email.groups.{group}', data)
report_label = _runtime_text('email.report', data, group=section_label)
host_label = _runtime_text('email.host', data)
@@ -1073,6 +1094,18 @@ class EmailChannel(NotificationChannel):
('', html_mod.escape(line.strip()))
for line in body.split('\n') if line.strip()
)
# A metadata-only/manual body may be generic. Keep actionable raw
# context once, without restoring duplicated inventory metadata.
reason = data.get('reason', '')
if backup_email and reason and len(reason) <= 80 and reason not in body:
detail_rows.append((html_mod.escape(_runtime_text('email.fields.reason', data)),
html_mod.escape(reason)))
if event_type == 'system_restore_completed' or data.get('_restore_summary'):
# The endpoint's warnings_block and task counts live in the
# localized body, not the generic services Event row.
detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
for line in body.split('\n') if line.strip()]
# ── Fallback: if no structured rows, render body text lines ──
if not detail_rows:
@@ -1090,6 +1123,7 @@ class EmailChannel(NotificationChannel):
# ── Render detail rows as HTML table ──
rows_html = ''
summary_whitespace = 'white-space:pre-wrap;' if data.get('_quiet_hours_summary') and data.get('_restore_summary') else ''
for label, value in detail_rows:
if label:
rows_html += f'''<tr>
@@ -1099,13 +1133,14 @@ class EmailChannel(NotificationChannel):
else:
# Full-width row (no label, just description text)
rows_html += f'''<tr>
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}">{value}</td>
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}{summary_whitespace}">{value}</td>
</tr>'''
# ── Reason / details block (long text, displayed separately) ──
reason = data.get('reason', '')
reason_html = ''
if reason and len(reason) > 80 and not (event_type == 'temp_high' and reason in body):
if reason and len(reason) > 80 and not (
(event_type == 'temp_high' or backup_email) and reason in body):
reason_html = f'''
<div style="margin:16px 0 0;padding:12px 16px;border:1px solid #d1d5db;border-radius:6px;">
<p style="margin:0 0 4px;font-size:11px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:0.05em;">{_runtime_text('email.details', data)}</p>
@@ -1140,15 +1175,15 @@ class EmailChannel(NotificationChannel):
<!-- Title bar -->
<div style="padding:16px 28px;background:{sev['bg']};border-bottom:1px solid {sev['border']};">
<h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};">{html_mod.escape(display_title)}</h2>
<h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};{backup_title_wrap}">{html_mod.escape(display_title)}</h2>
</div>
<!-- Body -->
<div style="padding:24px 28px;">
<!-- Metadata -->
<table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;">
<table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;{backup_metadata_layout}">
<tr>
<td style="font-size:12px;color:#4b5563;">
<td style="font-size:12px;color:#4b5563;{backup_title_wrap}">
{html_mod.escape(host_label)}: <strong style="color:#111827;">{html_mod.escape(data.get('hostname', ''))}</strong>
</td>
<td style="font-size:12px;color:#4b5563;text-align:right;">
@@ -1221,7 +1256,9 @@ class EmailChannel(NotificationChannel):
v = str(value).strip() if value else ''
if not v or v == '0' and original_label not in ('Failures',):
return
if fmt == 'severity':
if fmt == 'backup_error':
rows.append((esc(label), f'<span style="color:#dc2626;font-weight:600;">{esc(v)}</span>'))
elif fmt == 'severity':
sev_colors = {
'CRITICAL': '#dc2626', 'WARNING': '#d97706',
'INFO': '#2563eb', 'OK': '#16a34a',
@@ -1254,9 +1291,14 @@ class EmailChannel(NotificationChannel):
# tell which target the backup ran against. Reported gap: emails
# showed no way to distinguish which PBS failed with 2+ configured.
_add('Storage', data.get('storage') or data.get('storage_name'), 'code')
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
status_key = ('failed' if data.get('backup_outcome') == 'failed'
else 'completed_with_warnings' if data.get('backup_outcome') == 'completed_with_warnings'
else 'unconfirmed')
else:
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
_add('Status', _runtime_text(f'email.status.{status_key}', language_data),
'severity' if 'fail' in event_type else '')
'backup_error' if status_key == 'failed' else '')
_add('Size', data.get('size'))
_add('Duration', data.get('duration'))
_add('Snapshot', data.get('snapshot_name'), 'code')
+59 -4
View File
@@ -4289,6 +4289,47 @@ class ProxmoxHookWatcher:
def _hostname(self) -> str:
return _hostname()
@staticmethod
def _backup_outcome(severity: str, message: str) -> str:
"""Distinguish explicit failure, complete guest logs and unknown results."""
text = str(message or '')
if severity in ('error', 'err', 'critical') or re.search(
r'(?im)^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
return 'failed'
starts = re.findall(r'(?im)\bStarting Backup of VM (\d+)\s*\(', text)
finished = re.findall(r'(?im)\bFinished Backup of VM (\d+)\s*\(', text)
from notification_templates import _parse_vzdump_table
table = _parse_vzdump_table(text)
if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
return 'failed'
warnings = severity in ('warning', 'warn') or bool(re.search(
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*[1-9]\d*)', text))
if severity not in ('info', 'ok', 'success', 'warning', 'warn'):
return 'unconfirmed'
completed = 'completed_with_warnings' if warnings else 'confirmed'
# A present table is authoritative: do not certify an incomplete table
# from a finished guest log, or reject a complete OK table merely
# because the extra diagnostic log was truncated before its finishes.
if table is not None:
return (completed if table['complete'] and
all(guest['status'].lower() == 'ok' for guest in table['vms'])
else 'unconfirmed')
if starts:
pending = {}
for match in re.finditer(r'(?im)\b(Starting|Finished) Backup of VM (\d+)\s*\(', text):
action, vmid = match.groups()
if action.lower() == 'starting':
pending[vmid] = pending.get(vmid, 0) + 1
elif not pending.get(vmid):
return 'unconfirmed' # A finish before its start is not evidence.
else:
pending[vmid] -= 1
return completed if not any(pending.values()) else 'unconfirmed'
if re.search(
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
return completed
return 'unconfirmed'
def process_webhook(self, payload: dict) -> dict:
"""Process an incoming Proxmox webhook payload.
@@ -4347,6 +4388,13 @@ class ProxmoxHookWatcher:
'title': title or event_type,
'job_id': pve_job_id,
}
if event_type in ('backup_complete', 'backup_fail'):
# This is presentation metadata, not a new event/toggle/delivery path.
outcome = self._backup_outcome(severity_raw, message)
data['backup_outcome'] = (
'failed' if event_type == 'backup_fail' or outcome == 'failed' else
outcome if pve_type == 'vzdump' else 'unconfirmed'
)
if pve_type == 'replication':
replication = self._extract_replication_context(
@@ -4440,10 +4488,17 @@ class ProxmoxHookWatcher:
if vmids:
data['vmid'] = vmids[0]
entity_id = vmids[0]
# Try to extract VM name from the table line
name_m = re.search(r'(\d+)\s+(\S+)\s+(?:OK|ERROR|WARNINGS)', message)
if name_m:
data['vmname'] = name_m.group(2)
from notification_templates import _parse_vzdump_message
parsed = _parse_vzdump_message(message) or {}
guests = parsed.get('vms', [])
if data.get('backup_outcome') == 'failed':
guests = [guest for guest in guests if guest.get('status', '').lower() == 'error']
if len(guests) == 1:
data['vmid'] = guests[0]['vmid']
data['vmname'] = guests[0]['name']
else:
# Do not make one successful guest the subject of a batch failure.
data.pop('vmid', None)
# Extract size from "Total size: X"
size_m = re.search(r'Total size:\s*(.+?)(?:\n|$)', message)
if size_m:
+47 -11
View File
@@ -1364,6 +1364,12 @@ class NotificationManager:
# Get journal context if available (will be enriched per-channel based on detail_level)
raw_journal_context = data.get('_journal_context', '')
# Persist a presentation token in the existing title column: buffers
# otherwise discard outcome metadata before composition. Old rows
# without a token remain neutral; routing and schema are unchanged.
buffer_title = title
if event_type in ('backup_complete', 'backup_fail'):
buffer_title, _ = enrich_with_emojis(event_type, title, '', data)
for ch_name, channel in channels.items():
# ── Per-channel category check ──
@@ -1393,7 +1399,7 @@ class NotificationManager:
# delivered after Quiet Hours + Daily Digest were merged.
if severity != 'CRITICAL' and self._in_quiet_hours(ch_name):
self._buffer_quiet_event(ch_name, event_type, event_group,
severity, title, body)
severity, buffer_title, body)
continue
# ── Per-channel daily digest ──
@@ -1406,7 +1412,7 @@ class NotificationManager:
# excluded from the digest by `_DIGEST_EXEMPT_EVENTS`.
if self._should_buffer_for_digest(ch_name, severity, event_type):
self._buffer_digest_event(ch_name, event_type, event_group,
severity, title, body)
severity, buffer_title, body)
continue
try:
@@ -1821,7 +1827,8 @@ class NotificationManager:
print(f"[NotificationManager] digest cleanup failed for "
f"{ch_name}: {e}")
def _compose_digest_body(self, rows: list, use_icons: bool = False) -> str:
def _compose_digest_body(self, rows: list, use_icons: bool = False,
quiet_release: bool = False) -> str:
"""Render a grouped summary body. rows is a list of
(id, event_type, event_group, ts, title, body) tuples ordered
by timestamp ASC.
@@ -1830,27 +1837,47 @@ class NotificationManager:
groups: OrderedDict[str, list] = OrderedDict()
for _id, ev_type, group, ts, title, body in rows:
label = group or 'other'
groups.setdefault(label, []).append((ts, ev_type, title))
groups.setdefault(label, []).append((ts, ev_type, title, body))
language = self._notification_language()
lines = [runtime_message('digest.lead', language, count=len(rows))]
# The quiet summary title already carries the total; the daily lead
# incorrectly calls every buffered WARNING an INFO event.
lines = [] if quiet_release else [runtime_message('digest.lead', language, count=len(rows))]
for group, items in groups.items():
group_label = runtime_message(f'digest.groups.{group}', language) or group.title()
group_icon = CATEGORY_EMOJI.get(group, '') if use_icons else ''
group_prefix = f'{group_icon} ' if group_icon else ''
lines.append(f"{group_prefix}{group_label}: {len(items)}")
for ts, ev_type, title in items[:8]:
# Quiet hours can buffer restore warnings, unlike the daily INFO
# digest. Keep their complete recorded body, even past the usual
# title preview limit, without changing either delivery policy.
visible_items = [item for index, item in enumerate(items)
if index < 8 or (quiet_release and item[1] == 'system_restore_completed')]
for ts, ev_type, title, body in visible_items:
hhmm = datetime.fromtimestamp(ts).strftime('%H:%M')
backup_icon = ''
if ev_type in ('backup_complete', 'backup_fail'):
for token in ('💾✅', '💾⚠️', '💾❌', '💾❔', '💾'):
if title.startswith(token + ' '):
backup_icon = token
title = title[len(token) + 1:]
break
backup_icon = backup_icon or ('💾❌' if ev_type == 'backup_fail' else '💾❔')
short_title = title.split(': ', 1)[-1] if ': ' in title else title
event_icon = (
EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
backup_icon or EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
) if use_icons else ''
event_prefix = f'{event_icon} ' if event_icon else ''
lines.append(f" • {event_prefix}{hhmm} {short_title}")
if len(items) > 8:
lines.append(runtime_message('digest.more', language, count=len(items) - 8))
if quiet_release and ev_type == 'system_restore_completed' and body:
lines.extend(' ' + line.strip() for line in body.splitlines() if line.strip())
if len(items) > len(visible_items):
lines.append(runtime_message('digest.more', language, count=len(items) - len(visible_items)))
lines.append('')
lines.append(runtime_message('digest.footer', language))
# The daily footer describes live warning delivery, which is not true
# for warnings buffered during quiet hours. Do not repeat that claim.
if not quiet_release:
lines.append(runtime_message('digest.footer', language))
return '\n'.join(lines).rstrip() + '\n'
# ─── Quiet Hours buffer + flush ────────────────────────────
@@ -1985,13 +2012,17 @@ class NotificationManager:
'digest.quietTitle', language, hostname=host, count=len(rows),
)
use_icons = self._config.get(f'{ch_name}.rich_format', 'false') == 'true'
summary_body = self._compose_digest_body(rows, use_icons=use_icons)
quiet_details = any(row[1] in ('backup_complete', 'backup_fail', 'system_restore_completed')
for row in rows)
summary_body = self._compose_digest_body(rows, use_icons=use_icons, quiet_release=quiet_details)
result: dict = {'success': False, 'error': ''}
try:
result = channel.send(
summary_title, summary_body, severity='INFO',
data={'_quiet_hours_summary': True, '_count': len(rows),
'_restore_summary': any(row[1] == 'system_restore_completed' for row in rows),
'_backup_summary': any(row[1] in ('backup_complete', 'backup_fail') for row in rows),
'_notification_language': language},
) or result
except Exception as e:
@@ -2467,6 +2498,11 @@ class NotificationManager:
runtime_data.get('hostname'), self._config,
)
runtime_data.setdefault('_notification_language', self._notification_language())
# Match queued dispatch's presentation context for these outcome
# notices; this does not alter event/severity or direct-send policy.
if event_type in ('backup_complete', 'backup_fail', 'system_restore_completed'):
runtime_data['_event_type'] = event_type
runtime_data['_group'] = TEMPLATES[event_type].get('group', 'other')
# Render template if available
if event_type in TEMPLATES and not message:
+239 -62
View File
@@ -206,6 +206,58 @@ def _format_lxc_update_details(data: Dict[str, Any], language: str) -> str:
# ─── vzdump message parser ───────────────────────────────────────
def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
"""Read the bounded fixed-column summary for both outcomes and guest details."""
lines = message.splitlines()
for index, header in enumerate(lines):
if not re.match(r'\s*VMID\s+Name\s+Status\b', header, re.IGNORECASE):
continue
columns = [re.search(r'\b' + name + r'\b', header, re.IGNORECASE)
for name in ('VMID', 'Name', 'Status', 'Time', 'Size', 'Filename')]
if not all(columns):
return {'vms': [], 'complete': False}
starts = [column.start() for column in columns if column is not None]
if starts != sorted(starts):
return {'vms': [], 'complete': False}
rows = []
valid = True
complete = False
for line in lines[index + 1:]:
if not line.strip():
continue
if re.match(r'\s*Total running time:\s*\S', line, re.IGNORECASE):
complete = valid and bool(rows)
break
# Blanks are allowed, but no unrelated section can extend the table.
if not re.match(r'\s*\d+\s+', line):
break
values = [line[a:b].strip() for a, b in
zip(starts, starts[1:] + [len(line)])]
vmid, name, status, duration, size, filename = values
if not vmid.isdigit():
valid = False
break
valid = bool(valid and all(values)
and re.fullmatch(r'(?:\d+:\d{2}:\d{2}|(?:\d+[yMwdhms]\s*)+)', duration)
and re.fullmatch(r'\d+(?:\.\d+)?\s*(?:[KMGTPE]i?B?|B)', size, re.IGNORECASE))
if status.lower() in ('err', 'error'):
status = 'error'
kind = ('lxc' if 'lxc' in filename or filename.startswith('ct/') else
'qemu' if 'qemu' in filename or filename.startswith('vm/') else '')
if filename.lower() == 'null':
# Native failed rows lack archives. Match only this row's VMID;
# multiple inconsistent starts are not authoritative identity.
kinds = set(re.findall(
r'(?im)\bStarting Backup of VM ' + re.escape(vmid) + r'\s+\((lxc|qemu)\)',
message))
kind = kinds.pop() if len(kinds) == 1 else ''
rows.append({'vmid': vmid, 'name': name, 'status': status,
'time': duration, 'size': size, 'filename': filename,
'type': kind})
return {'vms': rows, 'complete': bool(complete)}
return None
def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
"""Parse a PVE vzdump notification message into structured data.
@@ -225,53 +277,10 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
lines = message.split('\n')
# ── Strategy 1: classic table (local/NFS/CIFS storage) ──
header_idx = -1
for i, line in enumerate(lines):
if re.match(r'\s*VMID\s+Name\s+Status', line, re.IGNORECASE):
header_idx = i
break
if header_idx >= 0:
# Use column positions from the header to slice each row.
# Header: "VMID Name Status Time Size Filename"
header = lines[header_idx]
col_starts = []
for col_name in ['VMID', 'Name', 'Status', 'Time', 'Size', 'Filename']:
idx = header.find(col_name)
if idx >= 0:
col_starts.append(idx)
if len(col_starts) == 6:
for line in lines[header_idx + 1:]:
stripped = line.strip()
if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='):
break
# Pad line to avoid index errors
padded = line.ljust(col_starts[-1] + 50)
vmid = padded[col_starts[0]:col_starts[1]].strip()
name = padded[col_starts[1]:col_starts[2]].strip()
status = padded[col_starts[2]:col_starts[3]].strip()
time_val = padded[col_starts[3]:col_starts[4]].strip()
size = padded[col_starts[4]:col_starts[5]].strip()
filename = padded[col_starts[5]:].strip()
if vmid and vmid.isdigit():
# Infer type from filename (vzdump-lxc-NNN or vzdump-qemu-NNN)
vm_type = ''
if 'lxc' in filename:
vm_type = 'lxc'
elif 'qemu' in filename:
vm_type = 'qemu'
vms.append({
'vmid': vmid,
'name': name,
'status': status,
'time': time_val,
'size': size,
'filename': filename,
'type': vm_type,
})
# The same summary rows drive classification, rich bodies and identities.
table = _parse_vzdump_table(message)
if table is not None:
vms = table['vms']
# ── Strategy 2: log-style (PBS / Proxmox Backup Server) ──
# Parse from the full vzdump log lines.
@@ -299,7 +308,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
current_vm = {
'vmid': m_start.group(1),
'name': '',
'status': 'ok',
'status': 'unknown',
'time': '',
'size': '',
'filename': '',
@@ -307,6 +316,18 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
}
continue
# A finish can belong to a guest already stored when another
# start arrived. Preserve that guest's actual completion too.
prior_finish = re.match(r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
if prior_finish:
prior = next((vm for vm in reversed(vms)
if vm['vmid'] == prior_finish.group(1)), None)
if prior is not None:
prior['time'] = prior_finish.group(2)
if prior['status'] != 'error':
prior['status'] = 'ok'
continue
if current_vm:
# Guest name
m_name = re.match(r'(?:CT|VM) Name:\s*(.+)', clean)
@@ -338,15 +359,16 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
# Finished -> duration
m_finish = re.match(
r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
if m_finish:
if m_finish and m_finish.group(1) == current_vm['vmid']:
current_vm['time'] = m_finish.group(2)
current_vm['status'] = 'ok'
if current_vm['status'] != 'error':
current_vm['status'] = 'ok'
vms.append(current_vm)
current_vm = None
continue
# Error
if clean.startswith('ERROR:') or clean.startswith('TASK ERROR'):
if re.match(r'^\s*(?:ERROR:|TASK ERROR)', line, re.IGNORECASE):
if current_vm:
current_vm['status'] = 'error'
@@ -354,6 +376,21 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
if current_vm:
vms.append(current_vm)
# Explicit guest-linked failures outrank a contradictory summary OK row.
# Job-level/prune errors do not invalidate unrelated successfully saved guests.
for line in lines:
error = re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:)\s*(.*)',
line, re.IGNORECASE)
if not error:
continue
failed_guest = re.search(r'\bBackup of (?:VM|CT) (\d+) failed\b|\bbackup failed for (?:VM|CT) (\d+)\b',
error.group(1), re.IGNORECASE)
if failed_guest:
vmid = failed_guest.group(1) or failed_guest.group(2)
for vm in vms:
if vm['vmid'] == vmid:
vm['status'] = 'error'
# ── Extract totals ──
for line in lines:
m_time = re.search(r'Total running time:\s*(.+)', line)
@@ -369,7 +406,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
sizes_gib = 0.0
for vm in vms:
s = vm.get('size', '')
m = re.match(r'([\d.]+)\s+(.*)', s)
m = re.fullmatch(r'(\d+(?:\.\d+)?)\s+([KMGTPE]i?B|B)', s, re.IGNORECASE)
if m:
val = float(m.group(1))
unit = m.group(2).strip().upper()
@@ -439,7 +476,7 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
for vm in parsed.get('vms', []):
status = vm.get('status', '').lower()
icon = '\u2705' if status == 'ok' else '\u274C'
icon = '\u2705' if status == 'ok' else '\u274C' if status == 'error' else '\u2754'
# Determine VM/CT type prefix
vm_type = vm.get('type', '')
@@ -501,7 +538,8 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
if vm_count > 0 or parsed.get('total_size'):
ok_count = sum(1 for v in parsed.get('vms', [])
if v.get('status', '').lower() == 'ok')
fail_count = vm_count - ok_count
fail_count = sum(1 for v in parsed.get('vms', [])
if v.get('status', '').lower() == 'error')
summary_parts = []
if vm_count:
@@ -1855,6 +1893,50 @@ def render_template(event_type: str, data: Dict[str, Any],
)
if localized:
template[field] = localized
backup_title_target = ''
if event_type == 'backup_complete':
template['title'] = runtime_message('backup.unconfirmedTitle', language,
hostname=data.get('hostname') or _get_hostname()) or (
str(data.get('hostname') or _get_hostname()) + ': Backup outcome unconfirmed')
template['body'] = runtime_message('backup.unconfirmedBody', language) or 'The backup outcome is not confirmed.'
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
template['title'] = runtime_message('backup.confirmedTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.confirmedBody', language)
elif outcome == 'completed_with_warnings':
template['title'] = runtime_message('backup.warningTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.warningBody', language)
elif outcome == 'failed':
template['title'] = runtime_message('backup.errorTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.errorBody', language)
if event_type == 'backup_fail':
template['title'] = runtime_message('backup.errorTitle', language,
hostname=data.get('hostname') or _get_hostname())
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
guests = (parsed_backup or {}).get('vms') or []
# Explicit confirmed manual metadata is useful context, not evidence
# about an unparsed batch. Only use it when there is no raw report.
if not data.get('pve_message') and data.get('backup_outcome') == 'confirmed' and data.get('vmid'):
guests = [{'vmid': str(data['vmid']), 'name': str(data.get('vmname') or ''),
'type': str(data.get('vm_type') or ''), 'status': 'ok'}]
if event_type == 'backup_fail' or data.get('backup_outcome') == 'failed':
guests = [guest for guest in guests if guest.get('status', '').lower() == 'error']
target = []
if storage:
target.append(storage)
if len(guests) == 1:
guest = guests[0]
kind = 'VM' if guest.get('type') == 'qemu' else 'CT' if guest.get('type') == 'lxc' else 'VM/CT'
name = guest.get('name') or kind
target.append(f"{kind} {name} ({guest['vmid']})" if name != kind
else f"{kind} {guest['vmid']}")
if target:
backup_title_target = ' — ' + ' · '.join(target)
# Ensure hostname is always available
variables = {
@@ -1884,6 +1966,11 @@ def render_template(event_type: str, data: Dict[str, Any],
'log_file': '',
}
variables.update(data)
if event_type in ('backup_fail', 'backup_complete'):
# The provider has already substituted raw Display Names. Insert the
# resolved title as a value, never reinterpret its literal braces.
variables['_backup_title'] = template['title']
template['title'] = '{_backup_title}'
# Old persisted errors and manual events may lack a complete reading.
# Accept plain numeric strings, but never interpret booleans or objects as
@@ -1992,12 +2079,37 @@ def render_template(event_type: str, data: Dict[str, Any],
title = template['title'].format_map(safe_vars)
except (ValueError, IndexError):
title = template['title']
title += backup_title_target
# ── PVE vzdump special formatting ──
# When the event came from PVE webhook with a full vzdump message,
# parse the table/logs and format a rich body instead of the sparse template.
pve_message = data.get('pve_message', '')
pve_title = data.get('pve_title', '')
backup_diagnostics = []
principal_cause = None
def bounded_backup_diagnostics(lines, principal_cause=None):
# 1024 chars matches the repository's small-channel message convention;
# 8 lines keeps repeated producer warnings readable. Inventory/title
# size is separate: this is not a one-Telegram-message guarantee.
unique = list(dict.fromkeys(line for line in lines if line.strip()))
principal = principal_cause or next((line for line in unique if re.search(r'\b(?:ERROR:|TASK ERROR:)', line, re.IGNORECASE)), None)
if principal:
unique.remove(principal)
unique.insert(0, principal)
shown, budget, omitted = [], 1024, 0
for line in unique:
if len(shown) >= 8 or budget < 2:
omitted += 1
continue
line_budget = min(budget, 512)
rendered = line if len(line) <= line_budget else line[:line_budget - 1] + '…'
omitted += int(rendered != line)
shown.append(rendered)
budget -= len(rendered) + 1
if omitted:
shown.append(runtime_message('backup.diagnosticsOmitted', language, count=omitted))
return '\n'.join(shown)
# Check for custom formatter function
formatter_name = template.get('formatter')
@@ -2016,13 +2128,29 @@ def render_template(event_type: str, data: Dict[str, Any],
if parsed:
is_success = (event_type == 'backup_complete')
body_text = _format_vzdump_body(parsed, is_success, language=language)
# Preserve PVE's source title for English, but never leak it into a
# deterministic localized notification.
if pve_title and requested_language == 'en':
title = pve_title
diagnostic_lines = [line.strip() for line in pve_message.splitlines()
if re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:WARN(?:ING)?:|ERROR:|TASK ERROR)',
line, re.IGNORECASE)]
if event_type == 'backup_fail' or data.get('backup_outcome') == 'failed':
# Native send_notification puts multiline job/setup errors
# before Details, while its subject says only "multiple problems".
# Keep that raw block when inventory replaces the producer body;
# it is job context, not evidence that every guest failed.
error_block = re.match(r'\A(.*?)^Details\r?\n=+\s*$',
pve_message, re.MULTILINE | re.DOTALL)
if error_block:
diagnostic_lines = error_block.group(1).rstrip('\r\n').splitlines() + diagnostic_lines
backup_diagnostics = [line for line in diagnostic_lines
if line.strip() and line not in body_text.splitlines()]
else:
# Couldn't parse -- use PVE raw message as body
body_text = pve_message.strip()
# Unparsed diagnostic-only reports remain visible but bounded.
body_text = ''
backup_diagnostics = pve_message.strip().splitlines()
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
else 'backup.unconfirmedBody')
body_text = (runtime_message(key, language) or template['body']) + '\n' + body_text
elif event_type == 'system_mail' and pve_message:
# System mail -- use PVE message directly (mail bounce, cron, smartd)
body_text = pve_message.strip()[:1000]
@@ -2032,6 +2160,51 @@ def render_template(event_type: str, data: Dict[str, Any],
except (ValueError, IndexError):
body_text = template['body']
if event_type == 'backup_complete' and data.get('backup_outcome') == 'confirmed' and not pve_message:
context = []
if data.get('vmid'):
name = str(data.get('vmname') or '')
context.append(f"{name} ({data['vmid']})" if name else str(data['vmid']))
if data.get('storage'):
context.append(str(data['storage']))
if data.get('size'):
context.append(runtime_message('vzdump.size', language, value=data['size']))
if data.get('duration'):
context.append(runtime_message('vzdump.duration', language, value=data['duration']))
if context:
body_text += '\n' + '\n'.join(context)
# PVE can move a one-line setup/abort reason exclusively into its subject.
# Preserve that raw failure context, without using it as a localized title.
if event_type in ('backup_complete', 'backup_fail') and (
event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
source_subject = str(data.get('pve_title') or '').strip()
native_failure = re.fullmatch(
r'vzdump backup status \([^\r\n]*\): backup failed(?::\s*(.*))?',
source_subject, re.IGNORECASE)
guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
if native_failure:
# Before the first guest, too, only the native cause is diagnostic;
# the original host/job envelope is not display-name context.
source_subject = (native_failure.group(1) or '').strip()
elif guest_context:
cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
source_subject = cause.group(1).strip() if cause else ''
if source_subject.lower() == 'multiple problems':
source_subject = ''
if source_subject and source_subject not in {line.strip() for line in body_text.splitlines()}:
# Reserve the subject-equivalent diagnostic BEFORE the cap. Finding
# it in uncapped logs is not enough: that late line could be omitted.
principal_cause = next((line for line in backup_diagnostics
if line.strip() == source_subject or
re.split(r'\b(?:TASK ERROR:|ERROR:)\s*', line, maxsplit=1, flags=re.IGNORECASE)[-1].strip() == source_subject), None)
if not principal_cause:
principal_cause = source_subject
backup_diagnostics.insert(0, source_subject)
if backup_diagnostics:
body_text += '\n' + bounded_backup_diagnostics(backup_diagnostics, principal_cause)
# Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
import re as _re
body_text = _re.sub(r'\n{3,}', '\n\n', body_text.strip())
@@ -2166,7 +2339,7 @@ EVENT_EMOJI = {
'host_backup_start': '\U0001F5C4️\U0001F680', # 🗄️🚀 cabinet + rocket
'host_backup_complete': '\U0001F5C4️✅', # 🗄️✅ cabinet + check
'host_backup_fail': '\U0001F5C4️❌', # 🗄️❌ cabinet + cross
'backup_complete': '\U0001F4BE\u2705', # 💾✅ floppy + check
'backup_complete': '\U0001F4BE', # 💾 neutral for digests without outcome metadata
'backup_warning': '\U0001F4BE\u26A0\uFE0F', # 💾⚠️ floppy + warning
'backup_fail': '\U0001F4BE\u274C', # 💾❌ floppy + cross
'snapshot_complete': '\U0001F4F8', # camera with flash
@@ -2363,6 +2536,10 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
severity = data.get('severity', 'INFO')
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
if event_type == 'backup_complete':
icon = {
'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
}.get(str(data.get('backup_outcome') or ''), '💾❔')
# Build enriched title: replace severity circle with event-specific icon
# Current format: "hostname: Something" -> "ICON hostname: Something"
@@ -68,10 +68,23 @@ class RuntimeCatalogTests(unittest.TestCase):
return result
en = flatten(self.catalogs["en"])
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
"backup.unconfirmedTitle", "channels.email.status.unconfirmed",
"backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
"channels.email.status.completed_with_warnings"}
for language, catalog in self.catalogs.items():
translated = flatten(catalog)
self.assertEqual(set(translated), set(en), language)
for key in en:
if language == 'sk':
# Missing maintainer-owned leaves may be generated later.
# Accept only this bounded gap, and validate every present leaf.
self.assertTrue(set(en) - pending_slovak <= set(translated), language)
self.assertTrue(set(translated) <= set(en), language)
else:
self.assertEqual(set(translated), set(en), language)
for key in translated:
self.assertIsInstance(translated[key], str, f"{language}:{key}")
self.assertTrue(translated[key].strip(), f"{language}:{key}")
self.assertEqual(_placeholders(translated[key]), _placeholders(en[key]), f"{language}:{key}")
def test_notification_language_ui_keys_exist_in_both_catalogs(self):
@@ -111,6 +124,11 @@ class RuntimeCatalogTests(unittest.TestCase):
with mock.patch.object(notification_templates, "_get_hostname", return_value="HOST-ŽILINA"):
for event_type in notification_templates.TEMPLATES:
event_values = dict(values)
if event_type == "backup_complete":
# Legacy catalog fields keep their completion meaning;
# the new unknown-outcome keys intentionally omit guest data.
# Exercise retained metadata on the explicit confirmed path.
event_values["backup_outcome"] = "confirmed"
if event_type == "temp_high":
# Temperature is a measured numeric contract; arbitrary
# DYNAMIC_VALUE is correctly rejected by its fallback.
@@ -312,6 +330,7 @@ class RuntimeCatalogTests(unittest.TestCase):
"backup_complete",
{
"hostname": "pve01", "storage": "pbs-main", "vmname": "alpha", "vmid": "100",
"backup_outcome": "confirmed",
"pve_title": "Backup job finished",
"pve_message": (
"INFO: Starting Backup of VM 100 (qemu)\n"
@@ -322,7 +341,12 @@ class RuntimeCatalogTests(unittest.TestCase):
},
language="sk",
)
self.assertIn("Záloha dokončená", backup["title"])
expected_title = notification_templates.runtime_message(
"backup.confirmedTitle", "sk", hostname="pve01",
)
self.assertTrue(backup["title"].startswith(expected_title + " — "))
self.assertIn("pbs-main", backup["title"])
self.assertIn("VM alpha (100)", backup["title"])
self.assertNotIn("Backup job finished", backup["title"])
self.assertIn("Veľkosť: 1.5 GiB", backup["body"])
self.assertIn("Trvanie: 00:00:10", backup["body"])
@@ -627,6 +651,7 @@ class RuntimeCatalogTests(unittest.TestCase):
"_notification_language": "sk", "_event_type": event_type,
"_group": "backup", "hostname": "pve01", "vmid": "100",
"vmname": "alpha", "storage": "pbs-main",
"backup_outcome": "confirmed" if event_type == "backup_complete" else "unconfirmed",
},
)
self.assertIn(f">{localized_status}<", backup_html)
@@ -275,8 +275,14 @@ class VzdumpAIIntegrityTests(unittest.TestCase):
rendered["body"], "CRITICAL", data,
)
# The failure title now identifies the unique failed guest; inventory
# remains exactly once in the detail table, not suppressed from body.
inventory = html.split('<!-- Detail table -->', 1)[1].split('</table>', 1)[0]
for vmid in range(100, 149):
self.assertEqual(html.count(f"guest-{vmid} ({vmid})"), 1, vmid)
self.assertEqual(inventory.count(f"guest-{vmid} ({vmid})"), 1, vmid)
self.assertIn('guest-148 (148)', rendered['title'])
self.assertNotIn('guest-100 (100)', rendered['title'])
self.assertEqual(html.count('guest-148 (148)'), 2) # title + inventory
self.assertEqual(html.count("49 backups"), 1)
self.assertEqual(html.count("1 failed"), 1)
self.assertEqual(html.count(">Zlyhalo<"), 1)
@@ -60,7 +60,7 @@ def _make_long_vzdump_report():
class VzdumpWebhookTruncationTests(unittest.TestCase):
def test_truncating_vzdump_report_at_4096_can_create_false_failed_backup(self):
def test_truncating_vzdump_report_at_4096_leaves_guest_unconfirmed(self):
full_message = _make_long_vzdump_report()
truncated_message = full_message[:4096]
@@ -82,8 +82,8 @@ class VzdumpWebhookTruncationTests(unittest.TestCase):
self.assertEqual(truncated_dockflare["name"], "dockflare")
self.assertEqual(truncated_dockflare["status"], "")
self.assertIn("❌ dockflare (129)", truncated_body)
self.assertIn("❌ 1 failed", truncated_body)
self.assertIn("❔ dockflare (129)", truncated_body)
self.assertNotIn("❌ 1 failed", truncated_body)
def test_webhook_handler_does_not_truncate_message_before_parsing(self):
source = (SCRIPTS_DIR / "flask_notification_routes.py").read_text()