mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-08 14:36:38 +00:00
Merge pull request #406 from f3rs3n/fix/notification-outcome-reporting
Clarify backup outcomes and diagnostic presentation
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""Inert infrastructure for actual endpoint/manual/queued/SQLite release seams."""
|
||||
import datetime
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import typing
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from notification_fixture import templates, SCRIPTS, EmailChannel, extract
|
||||
|
||||
|
||||
def visible(markup):
|
||||
class Text(HTMLParser):
|
||||
def __init__(self): super().__init__(); self.parts = []; self.tags = []
|
||||
def handle_data(self, data): self.parts.append(data)
|
||||
def handle_starttag(self, tag, attrs): self.tags.append(tag)
|
||||
parser = Text(); parser.feed(markup)
|
||||
return '\n'.join(p.strip() for p in parser.parts if p.strip()), parser.tags
|
||||
|
||||
|
||||
def restore_event(warnings):
|
||||
events = []
|
||||
ns = {'request': types.SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
|
||||
'hostname':'node-a', 'guests':'3', 'stubs':'1', 'stale_nodes':'0',
|
||||
'components':'2', 'duration':'2m', 'warnings':warnings}),
|
||||
'notification_manager':types.SimpleNamespace(emit_event=lambda **kw:events.append(kw)),
|
||||
'jsonify':lambda value:value}
|
||||
handler = extract(SCRIPTS/'flask_notification_routes.py', 'internal_restore_event', None, ns)
|
||||
response, status = handler()
|
||||
assert status == 200 and len(events) == 1
|
||||
return events[0]
|
||||
|
||||
|
||||
def deliver(event_type, data, severity='INFO', language='en', manual=False, quiet=False, quiet_before=()):
|
||||
"""Execute real dispatch/manual and optional real SQLite buffer+flush."""
|
||||
captured = []
|
||||
channel = object.__new__(EmailChannel); channel.subject_prefix = '[ProxMenux]'
|
||||
def sink(title, body, severity, data):
|
||||
markup = channel._format_html(title, body, severity, data)
|
||||
text, tags = visible(markup)
|
||||
captured.append(dict(title=title, body=body, severity=severity, data=dict(data), html=markup, text=text, tags=tags))
|
||||
return {'success':True}
|
||||
manager = types.SimpleNamespace(_config={'email.rich_format':'true'}, _lock=threading.RLock(),
|
||||
_channels={'email':types.SimpleNamespace(send=sink)},
|
||||
_group_limiter=types.SimpleNamespace(allow=lambda group:True),
|
||||
_claim_delivery=lambda event:'inert', _finish_delivery_claim=lambda *a,**kw:None,
|
||||
_notification_language=lambda:language, _build_ai_config=lambda:{'ai_enabled':'false'},
|
||||
_in_quiet_hours=lambda channel:quiet, _should_buffer_for_digest=lambda *a:False,
|
||||
_record_history=lambda *a:None, _stats={'total_sent':0,'total_errors':0},
|
||||
is_event_enabled=lambda event:True)
|
||||
ns = dict(vars(typing), NotificationEvent=types.SimpleNamespace, TEMPLATES=templates.TEMPLATES, render_template=templates.render_template,
|
||||
resolve_notification_hostname=lambda host, config:host or 'node-a',
|
||||
enrich_with_emojis=templates.enrich_with_emojis, datetime=datetime.datetime,
|
||||
_should_bypass_ai=lambda event:True, _AI_BYPASS_EVENTS=frozenset({'backup_complete','backup_fail'}))
|
||||
for name in ('_dispatch_to_channels', '_dispatch_event', 'send_notification'):
|
||||
setattr(manager, name, types.MethodType(extract(SCRIPTS/'notification_manager.py', name, 'NotificationManager', ns), manager))
|
||||
with tempfile.TemporaryDirectory(prefix='notification-final-') as scratch:
|
||||
db = Path(scratch)/'pending.sqlite'
|
||||
rows = []
|
||||
if quiet:
|
||||
conn = sqlite3.connect(db)
|
||||
conn.execute('CREATE TABLE quiet_pending (id INTEGER PRIMARY KEY, channel TEXT, event_type TEXT, event_group TEXT, severity TEXT, ts INTEGER, title TEXT, body TEXT)')
|
||||
conn.commit(); conn.close()
|
||||
qns = dict(vars(typing), sqlite3=sqlite3, DB_PATH=db, time=time, datetime=datetime.datetime,
|
||||
_resolve_display_hostname=lambda config:'node-a', runtime_message=templates.runtime_message,
|
||||
EVENT_EMOJI=templates.EVENT_EMOJI, CATEGORY_EMOJI=templates.CATEGORY_EMOJI)
|
||||
for name in ('_buffer_quiet_event', '_flush_quiet_for_channel', '_compose_digest_body'):
|
||||
setattr(manager,name,types.MethodType(extract(SCRIPTS/'notification_manager.py',name,'NotificationManager',qns),manager))
|
||||
for earlier in quiet_before:
|
||||
manager._dispatch_event(types.SimpleNamespace(**earlier,source='inert',entity_type='node',entity_id='',event_id='inert',fingerprint='inert'))
|
||||
if manual:
|
||||
assert not quiet
|
||||
result = manager.send_notification(event_type, severity, '', '', dict(data))
|
||||
assert result['success']
|
||||
else:
|
||||
event = types.SimpleNamespace(event_type=event_type, severity=severity, data=dict(data), source='inert', entity_type='node', entity_id='', event_id='inert', fingerprint='inert')
|
||||
manager._dispatch_event(event)
|
||||
if quiet:
|
||||
conn = sqlite3.connect(db); rows = conn.execute('SELECT event_type,severity,title,body FROM quiet_pending').fetchall(); conn.close()
|
||||
assert len(rows)==1+len(quiet_before) and not captured
|
||||
manager._flush_quiet_for_channel('email', manager._channels['email'])
|
||||
conn=sqlite3.connect(db); remaining=conn.execute('SELECT count(*) FROM quiet_pending').fetchone()[0]; conn.close()
|
||||
assert remaining==0
|
||||
assert len(captured)==1
|
||||
if quiet: captured[0]['buffered']=rows
|
||||
return captured[0]
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Assertion-free inert actual consumers; no operational host imports."""
|
||||
import ast
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPTS = ROOT / 'AppImage/scripts'
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import notification_templates as templates
|
||||
# Display-name resolution is an infrastructure boundary, never load manager.
|
||||
templates._get_hostname = lambda: 'node-a'
|
||||
from notification_channels import EmailChannel
|
||||
LANGUAGES = ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv')
|
||||
|
||||
def extract(path, name, owner, ns):
|
||||
tree = ast.parse(path.read_text())
|
||||
nodes = tree.body if owner is None else next(n.body for n in tree.body if isinstance(n, ast.ClassDef) and n.name == owner)
|
||||
node = next(n for n in nodes if isinstance(n, ast.FunctionDef) and n.name == name)
|
||||
node.decorator_list = []
|
||||
exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), ns)
|
||||
return ns[name]
|
||||
|
||||
def receive(message, severity='info', title='Backup', kind='vzdump'):
|
||||
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
|
||||
class Event:
|
||||
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
|
||||
ns['NotificationEvent'] = Event
|
||||
methods = {name: extract(SCRIPTS / 'notification_events.py', name, 'ProxmoxHookWatcher', ns)
|
||||
for name in ('_classify_pve', '_map_severity', '_backup_outcome', 'process_webhook')}
|
||||
class Queue:
|
||||
def __init__(self): self.items = []
|
||||
def put(self, event): self.items.append(event)
|
||||
class Receiver:
|
||||
_hostname = 'node-a'
|
||||
_classify_pve = methods['_classify_pve']
|
||||
_map_severity = staticmethod(methods['_map_severity'])
|
||||
_backup_outcome = staticmethod(methods['_backup_outcome'])
|
||||
process_webhook = methods['process_webhook']
|
||||
def __init__(self): self._queue = Queue()
|
||||
target = Receiver()
|
||||
result = target.process_webhook({'fields': {'type': kind}, 'severity': severity, 'title': title, 'message': message})
|
||||
assert result['accepted'] and len(target._queue.items) == 1
|
||||
return target._queue.items[0]
|
||||
|
||||
def email(event_type, data, severity='INFO', language='en'):
|
||||
result = templates.render_template(event_type, data, language)
|
||||
channel = object.__new__(EmailChannel)
|
||||
channel.subject_prefix = '[ProxMenux]'
|
||||
context = {**data, 'severity': severity, '_event_type': event_type, '_group': result['group'], '_notification_language': language}
|
||||
return result, channel._format_html(result['title'], result['body'], severity, context)
|
||||
@@ -130,6 +130,18 @@ class CommandDescriptionsTests(unittest.TestCase):
|
||||
for key in ("temperatureAlertTitle", "temperatureAlertBody",
|
||||
"recordedReason", "recordedDetails"):
|
||||
fallback.setdefault(key, source_fallback[key])
|
||||
# Slovak remains the exact upstream catalog; model the
|
||||
# pending outcome-key generator additions in disposable
|
||||
# copies rather than modifying its curated values.
|
||||
if lang == 'sk':
|
||||
local = temporary['runtime']['notifications']
|
||||
source = catalog('en')['runtime']['notifications']
|
||||
for key, value in source['backup'].items():
|
||||
local.setdefault('backup', {}).setdefault(key, value)
|
||||
local['channels']['email']['status'].setdefault(
|
||||
'unconfirmed', source['channels']['email']['status']['unconfirmed'])
|
||||
local['channels']['email']['status'].setdefault(
|
||||
'completed_with_warnings', source['channels']['email']['status']['completed_with_warnings'])
|
||||
path.write_text(json.dumps(temporary, ensure_ascii=False))
|
||||
# Model steady state after the bot fills these intentional new
|
||||
# messages; keep repository locales and all other leaves intact.
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Backup-only maintainer contract at actual render/dispatch/email seams.
|
||||
|
||||
Scope-approved seams: receiver, template lookup, rich enrichment, inert queued
|
||||
and manual delivery with an email capture sink. No host operations or sends.
|
||||
"""
|
||||
import unittest
|
||||
from notification_fixture import templates, LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
|
||||
|
||||
class BackupSplitTests(unittest.TestCase):
|
||||
def test_recovery_default_is_upstream_resolved_without_proof(self):
|
||||
for language in LANGUAGES:
|
||||
data = {'hostname': 'alias', 'category': 'temperature',
|
||||
'reason': 'Temperature high (recovered)', 'duration': '2m',
|
||||
'original_severity': 'WARNING'}
|
||||
rendered = templates.render_template('error_resolved', data, language)
|
||||
expected = templates.runtime_message('templates.error_resolved.title', language,
|
||||
hostname='alias', category='temperature', entity_suffix='')
|
||||
self.assertEqual(rendered['title'], expected)
|
||||
rich, _ = templates.enrich_with_emojis('error_resolved', rendered['title'], rendered['body'], data)
|
||||
self.assertTrue(rich.startswith('✅ '))
|
||||
result = deliver('error_resolved', data, 'OK', language)
|
||||
self.assertIn('background:#f0fdf4;', result['html'])
|
||||
|
||||
def test_pre_guest_native_subject_keeps_only_cause_once_with_display_alias(self):
|
||||
from notification_fixture import receive
|
||||
raw_host = 'pve-production.internal.example'
|
||||
message = ('Details\n=======\nVMID Name Status Time Size Filename\n'
|
||||
'\nTotal running time: 0s\nTotal size: 0 B')
|
||||
for raw in (message, 'ERROR: unable to activate storage PBS\n' + message):
|
||||
event = receive(raw, 'error', 'vzdump backup status (' + raw_host +
|
||||
'): backup failed: unable to activate storage PBS')
|
||||
event.data['hostname'] = 'display-alias {rack.location}'
|
||||
for language in LANGUAGES:
|
||||
for manual in (False, True):
|
||||
result = deliver(event.event_type, event.data, event.severity, language, manual=manual)
|
||||
self.assertNotIn(raw_host, result['text'])
|
||||
self.assertNotIn('vzdump backup status', result['text'])
|
||||
self.assertEqual(result['text'].count('unable to activate storage PBS'), 1)
|
||||
self.assertIn('display-alias {rack.location}', result['title'])
|
||||
self.assertEqual(result['data']['pve_title'], event.data['pve_title'])
|
||||
self.assertEqual(result['data']['pve_message'], raw)
|
||||
# No global hostname or guest-name substitution: actual diagnostics are
|
||||
# authoritative, even when they happen to contain the original hostname.
|
||||
event = receive(message, 'error', 'vzdump backup status (' + raw_host +
|
||||
'): backup failed: cannot connect to ' + raw_host)
|
||||
result = deliver(event.event_type, {**event.data, 'hostname': 'alias'}, event.severity)
|
||||
self.assertEqual(result['body'].count('cannot connect to ' + raw_host), 1)
|
||||
self.assertNotIn('vzdump backup status', result['body'])
|
||||
|
||||
def test_backup_legacy_keys_keep_meaning_and_unknown_uses_new_keys(self):
|
||||
# Frozen upstream contract; no Git history required in shipped tests.
|
||||
legacy = {'en': {'title': '{hostname} → {storage}: Backup complete — {vmname} ({vmid})', 'body': 'Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}', 'label': 'Backup complete'}, 'de': {'title': '{hostname} → {storage}: Sicherung abgeschlossen – {vmname} ({vmid})', 'body': 'Die Sicherung von {vmname} (ID: {vmid}) wurde am {storage} erfolgreich abgeschlossen.\nGröße: {size}', 'label': 'Sicherung abgeschlossen'}, 'es': {'title': '{hostname} → {storage}: Backup completado — {vmname} ({vmid})', 'body': 'El backup de {vmname} (ID: {vmid}) se ha completado correctamente en {storage}.\nTamaño: {size}', 'label': 'Backup completado'}, 'fr': {'title': '{hostname} → {storage}\xa0: Sauvegarde terminée — {vmname} ({vmid})', 'body': "La sauvegarde de {vmname} (ID\xa0: {vmid}) s'est terminée avec succès le {storage}.\nTaille\xa0: {size}", 'label': 'Sauvegarde terminée'}, 'it': {'title': '{hostname} → {storage}: Backup completato — {vmname} ({vmid})', 'body': 'Backup di {vmname} (ID: {vmid}) completato con successo su {storage}.\nTaglia: {size}', 'label': 'Backup completato'}, 'pt': {'title': '{hostname} → {storage}: Backup concluído — {vmname} ({vmid})', 'body': 'Backup de {vmname} (ID: {vmid}) concluído com sucesso em {storage}.\nTamanho: {size}', 'label': 'Backup concluído'}, 'sk': {'title': '{hostname} → {storage}: Záloha dokončená — {vmname} ({vmid})', 'body': 'Záloha {vmname} (ID: {vmid}) na úložisku {storage} bola úspešne dokončená.\nVeľkosť: {size}', 'label': 'Záloha bola dokončená'}, 'sv': {'title': '{hostname} → {storage}: Säkerhetskopiering klar — {vmname} ({vmid})', 'body': 'Säkerhetskopiering av {vmname} (ID: {vmid}) slutfördes framgångsrikt på {storage}.\nStorlek: {size}', 'label': 'Säkerhetskopieringen är klar'}}
|
||||
for language, expected in legacy.items():
|
||||
self.assertEqual(templates._load_runtime_catalog(language)['templates']['backup_complete'], expected)
|
||||
result = templates.render_template('backup_complete', {'hostname': 'alias {rack.location}'}, language)
|
||||
self.assertIn('alias {rack.location}', result['title'])
|
||||
self.assertNotIn('()', result['title'])
|
||||
new_title = templates.runtime_message('backup.unconfirmedTitle', language, hostname='alias {rack.location}')
|
||||
self.assertTrue(new_title)
|
||||
self.assertEqual(result['title'], new_title)
|
||||
self.assertIn(templates.runtime_message('backup.unconfirmedBody', language), result['body'])
|
||||
# Absent/blank/non-string translation uses English per-key fallback.
|
||||
from unittest.mock import patch
|
||||
english = templates._load_runtime_catalog('en')
|
||||
for value in (None, '', {}, []):
|
||||
missing = {'backup': {'unconfirmedTitle': value, 'unconfirmedBody': value}}
|
||||
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: english if lang == 'en' else missing):
|
||||
result = templates.render_template('backup_complete', {'hostname': 'alias'}, 'it')
|
||||
self.assertEqual(result['title'], 'alias: Backup outcome unconfirmed')
|
||||
self.assertEqual(result['body'], 'The backup outcome is not confirmed.')
|
||||
# Both catalogs missing: new outcome text still has explicit EN defaults.
|
||||
with patch.object(templates, '_load_runtime_catalog', return_value={}):
|
||||
result = templates.render_template('backup_complete', {'hostname': 'alias'}, 'it')
|
||||
self.assertEqual(result['title'], 'alias: Backup outcome unconfirmed')
|
||||
self.assertEqual(result['body'], 'The backup outcome is not confirmed.')
|
||||
|
||||
def test_restore_keeps_original_ready_line_and_success_icon_with_warnings(self):
|
||||
from notification_final_fixture import restore_event
|
||||
ready = {'en': 'The node is now fully ready to use.', 'de': 'Der Knoten ist nun vollständig einsatzbereit.', 'es': 'El nodo está listo para usarse.', 'fr': 'Le nœud est maintenant entièrement prêt à être utilisé.', 'it': "Il nodo è ora completamente pronto per l'uso.", 'pt': 'O nó agora está totalmente pronto para uso.', 'sk': 'Uzol je teraz úplne pripravený na použitie.', 'sv': 'Noden är nu helt redo att användas.'}
|
||||
for warning in ('', 'missing module zfs'):
|
||||
event = restore_event(warning)
|
||||
for language in LANGUAGES:
|
||||
result = deliver(event['event_type'], event['data'], event['severity'], language)
|
||||
self.assertTrue(result['title'].startswith('✅ '))
|
||||
self.assertIn(ready[language], result['body'])
|
||||
self.assertIn(ready[language], result['text'])
|
||||
self.assertIn('2m', result['text'])
|
||||
if warning:
|
||||
self.assertIn(warning, result['text'])
|
||||
quiet = deliver(event['event_type'], event['data'], event['severity'], language, quiet=True)
|
||||
self.assertIn('✅', quiet['body'])
|
||||
self.assertIn(' ' + ready[language], quiet['body'])
|
||||
self.assertIn('white-space:pre-wrap;', quiet['html'])
|
||||
|
||||
def test_spanish_outcome_and_restore_titles_are_capitalized_and_failure_is_exact(self):
|
||||
expected = {'confirmed': 'Backup completado', 'completed_with_warnings': 'Backup completado con advertencias',
|
||||
'unconfirmed': 'Resultado del backup sin confirmar', 'failed': 'Backup fallido'}
|
||||
for outcome, title in expected.items():
|
||||
result = templates.render_template('backup_complete', {'hostname': 'alias', 'backup_outcome': outcome}, 'es')
|
||||
self.assertEqual(result['title'], 'alias: ' + title)
|
||||
failure = templates.render_template('backup_fail', {'hostname': 'alias'}, 'es')
|
||||
self.assertEqual(failure['title'], 'alias: Backup fallido')
|
||||
restore = templates.render_template('system_restore_completed', {'hostname': 'alias'}, 'es')
|
||||
self.assertEqual(restore['title'], 'alias: Restauración del host finalizada')
|
||||
|
||||
def test_recovery_only_quiet_release_keeps_upstream_summary_contract(self):
|
||||
result = deliver('error_resolved', {'hostname': 'alias', 'category': 'temperature',
|
||||
'reason': 'old (recovered)', 'duration': '2m'}, 'OK', quiet=True)
|
||||
self.assertIn(templates.runtime_message('digest.lead', 'en', count=1).strip(), result['body'])
|
||||
self.assertIn(templates.runtime_message('digest.footer', 'en'), result['body'])
|
||||
self.assertIn('✅', result['body'])
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Whole-PR outcome corrections, inert producer/actual email consumers."""
|
||||
import html
|
||||
import unittest
|
||||
from notification_fixture import templates, receive, email, LANGUAGES
|
||||
|
||||
REPORT = """Details
|
||||
=======
|
||||
VMID Name Status Time Size Filename
|
||||
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
|
||||
|
||||
Total running time: 1m 1s
|
||||
Total size: 1 GiB
|
||||
"""
|
||||
|
||||
class CorrectionTests(unittest.TestCase):
|
||||
def test_reversed_finish_is_not_completion_evidence(self):
|
||||
event = receive('INFO: Finished Backup of VM 100 (00:01:01)\nINFO: Starting Backup of VM 100 (qemu)')
|
||||
self.assertEqual(event.data['backup_outcome'], 'unconfirmed')
|
||||
|
||||
|
||||
def test_interleaved_complete_logs_keep_both_finished_guests(self):
|
||||
message = ('INFO: Starting Backup of VM 100 (qemu)\nINFO: VM Name: web\n'
|
||||
'INFO: Starting Backup of VM 101 (lxc)\nINFO: CT Name: db\n'
|
||||
'INFO: Finished Backup of VM 100 (00:01:01)\nINFO: Finished Backup of VM 101 (00:01:02)')
|
||||
event = receive(message)
|
||||
self.assertEqual(event.data['backup_outcome'], 'confirmed')
|
||||
result, markup = email(event.event_type, event.data, event.severity)
|
||||
self.assertIn('✅ VM web (100)', result['body'])
|
||||
self.assertIn('✅ CT db (101)', result['body'])
|
||||
self.assertNotIn('❔', result['body'])
|
||||
self.assertIn('00:01:01', result['body'])
|
||||
|
||||
|
||||
def test_backup_identity_has_event_scoped_mail_compatible_wrapping(self):
|
||||
result, markup = email('backup_complete', {'hostname': 'n' * 64,
|
||||
'backup_outcome': 'confirmed', 'pve_message': 'INFO: Starting Backup of VM 100 (qemu)\nINFO: VM Name: customerproductionpostgresqlreplicaeuropewestdatacenter01\nINFO: Finished Backup of VM 100 (00:01:01)'})
|
||||
self.assertIn('table-layout:fixed;', markup)
|
||||
title_tag = markup.split('<h2 style="', 1)[1].split('"', 1)[0]
|
||||
self.assertIn('overflow-wrap:break-word;', title_tag)
|
||||
self.assertIn('word-wrap:break-word;', title_tag)
|
||||
_, unrelated = email('node_reconnect', {'hostname': 'node-a'}, 'OK')
|
||||
self.assertNotIn('table-layout:fixed;', unrelated)
|
||||
|
||||
|
||||
def test_catalog_parity_accepts_eventual_valid_slovak_keys(self):
|
||||
from notification_fixture import extract, SCRIPTS
|
||||
import copy
|
||||
import string
|
||||
path = SCRIPTS / 'tests/test_notification_runtime_i18n.py'
|
||||
ns = {'string': string}
|
||||
ns['_placeholders'] = extract(path, '_placeholders', None, ns)
|
||||
parity = extract(path, 'test_runtime_catalog_keys_and_placeholders_match', 'RuntimeCatalogTests', ns)
|
||||
probe = unittest.TestCase()
|
||||
probe.catalogs = {lang: copy.deepcopy(templates._load_runtime_catalog(lang)) for lang in LANGUAGES}
|
||||
parity(probe) # shipped missing keys remain allowed
|
||||
probe.catalogs['sk']['backup'] = copy.deepcopy(probe.catalogs['en']['backup'])
|
||||
probe.catalogs['sk']['channels']['email']['status']['unconfirmed'] = probe.catalogs['en']['channels']['email']['status']['unconfirmed']
|
||||
parity(probe) # generation of exactly the pending keys is legal
|
||||
probe.catalogs['sk']['backup']['confirmedTitle'] = 'Missing hostname token'
|
||||
with self.assertRaises(AssertionError): parity(probe)
|
||||
|
||||
|
||||
def test_actual_neutral_style_is_not_success_green(self):
|
||||
for event, severity, data in (('backup_complete', 'INFO', {'backup_outcome': 'unconfirmed'}),):
|
||||
result, markup = email(event, data, severity)
|
||||
self.assertIn('background:#f9fafb;', markup)
|
||||
self.assertNotIn('background:#f0fdf4;', markup)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_actual_restore_endpoint_warnings_and_counts_reach_email(self):
|
||||
from notification_fixture import extract, SCRIPTS
|
||||
from types import SimpleNamespace
|
||||
for warnings in ('', 'missing module zfs'):
|
||||
events = []
|
||||
ns = {'request': SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
|
||||
'hostname': 'node-a', 'guests': '3', 'stubs': '1', 'stale_nodes': '0',
|
||||
'components': '2', 'duration': '2m', 'warnings': warnings}),
|
||||
'notification_manager': SimpleNamespace(emit_event=lambda **kw: events.append(kw)),
|
||||
'jsonify': lambda value: value}
|
||||
handler = extract(SCRIPTS / 'flask_notification_routes.py', 'internal_restore_event', None, ns)
|
||||
response, status = handler()
|
||||
self.assertEqual(status, 200)
|
||||
event = events[0]
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event['event_type'], event['data'], event['severity'], language)
|
||||
self.assertIn('2m', markup)
|
||||
for line in result['body'].splitlines():
|
||||
if line.strip(): self.assertIn(line.strip(), html.unescape(markup))
|
||||
if warnings: self.assertIn(warnings, markup)
|
||||
|
||||
|
||||
def test_raw_display_hostname_is_substituted_exactly_once(self):
|
||||
for event_type, outcome in (('backup_complete', 'confirmed'), ('backup_complete', 'failed'), ('backup_fail', 'failed')):
|
||||
for hostname in ('Sala {rack} – Zürich', 'node-{vmid}', 'Sala {rack.location}'):
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event_type, {'hostname': hostname,
|
||||
'backup_outcome': outcome, 'pve_message': REPORT}, language=language)
|
||||
self.assertIn(hostname, result['title'])
|
||||
self.assertIn(hostname, html.unescape(markup))
|
||||
|
||||
|
||||
def test_malformed_numeric_report_is_queued_uncertain_and_renderable(self):
|
||||
for size in ('1..5 GiB', '..5 GiB'):
|
||||
message = REPORT.replace('1 GiB ', size.ljust(9)).split('Total size:')[0]
|
||||
event = receive(message)
|
||||
self.assertEqual(event.data['backup_outcome'], 'unconfirmed')
|
||||
result, markup = email(event.event_type, event.data, event.severity)
|
||||
self.assertIn(size, result['body'])
|
||||
|
||||
|
||||
def test_unknown_backup_type_keeps_explicit_err_but_cannot_certify_ok(self):
|
||||
event = receive(REPORT.replace('ok ', 'err '), kind='')
|
||||
self.assertEqual(event.event_type, 'backup_complete')
|
||||
self.assertEqual(event.severity, 'INFO')
|
||||
self.assertEqual(event.data['backup_outcome'], 'failed')
|
||||
result, markup = email(event.event_type, event.data, event.severity)
|
||||
self.assertIn('FAILED', markup)
|
||||
self.assertEqual(receive(REPORT, kind='').data['backup_outcome'], 'unconfirmed')
|
||||
|
||||
|
||||
def test_confirmed_metadata_only_context_is_retained(self):
|
||||
data = {'hostname': 'node-a', 'backup_outcome': 'confirmed',
|
||||
'vmid': '100', 'vmname': 'web {literal}', 'storage': 'PBS', 'size': '1 GiB'}
|
||||
for language in LANGUAGES:
|
||||
result, markup = email('backup_complete', data, language=language)
|
||||
self.assertIn('web {literal} (100)', result['title'])
|
||||
self.assertIn('1 GiB', result['body'])
|
||||
self.assertIn('1 GiB', markup)
|
||||
|
||||
|
||||
def test_explicit_guest_failure_overrides_only_the_linked_ok_row(self):
|
||||
other = '101 db ok 1m 1s 1 GiB ct/101/2026-09-29T17:00:00Z'
|
||||
message = REPORT.replace('\n\nTotal', '\n' + other + '\n\nTotal')
|
||||
diagnostic = '100: 2026-09-29 17:00:00 ERROR: Backup of VM 100 failed - archive write failed'
|
||||
for severity in ('info', 'error'):
|
||||
event = receive(message + '\n' + diagnostic, severity)
|
||||
self.assertEqual(event.data.get('vmid'), '100')
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event.event_type, event.data, event.severity, language)
|
||||
self.assertIn('❌ VM web (100)', result['body'])
|
||||
self.assertNotIn('✅ VM web (100)', markup)
|
||||
self.assertIn('✅ CT db (101)', result['body'])
|
||||
self.assertIn('web (100)', result['title'])
|
||||
|
||||
|
||||
def test_official_week_month_year_durations_stay_confirmed(self):
|
||||
for duration in ('1w', '1w 1m 1s', '1M', '1y'):
|
||||
# Fixed-column widths are unchanged for these bounded values.
|
||||
message = REPORT.replace('1m 1s ', duration.ljust(9))
|
||||
event = receive(message)
|
||||
self.assertEqual(event.data['backup_outcome'], 'confirmed', duration)
|
||||
result, markup = email(event.event_type, event.data, event.severity)
|
||||
self.assertIn(duration, result['body'])
|
||||
|
||||
|
||||
def test_prefixed_error_diagnostics_survive_both_source_severities(self):
|
||||
diagnostic = '100: 2026-09-29 17:00:00 ERROR: archive write failed: permission denied'
|
||||
for severity in ('info', 'error'):
|
||||
event = receive(REPORT + '\n' + diagnostic, severity)
|
||||
self.assertEqual(event.data['backup_outcome'], 'failed')
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event.event_type, event.data, event.severity, language)
|
||||
self.assertIn(diagnostic, result['body'])
|
||||
self.assertIn(diagnostic, html.unescape(markup))
|
||||
# A job-level error must not invent a failed guest.
|
||||
self.assertIn('✅ VM web (100)', result['body'])
|
||||
|
||||
|
||||
def test_official_completed_report_warning_is_distinct_and_retained(self):
|
||||
warning = '100: 2026-09-29 17:00:00 WARN: unable to add notes - permission denied'
|
||||
event = receive(REPORT + '\n' + warning)
|
||||
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event.event_type, event.data, event.severity, language)
|
||||
self.assertIn(warning, result['body'])
|
||||
self.assertIn(warning, html.unescape(markup))
|
||||
|
||||
def test_batch_and_abort_failure_titles_have_no_empty_guest_slot(self):
|
||||
failed = REPORT.replace('ok ', 'err ')
|
||||
failed = failed.replace('\n\nTotal', '\n101 db err 1m 1s 0 B null\n\nTotal')
|
||||
for message in (failed, REPORT.replace('ok ', 'todo '),
|
||||
REPORT.split('100 web')[0] + '\nTotal running time: 0s'):
|
||||
event = receive(message + '\nINFO: vzdump --storage PBS', 'error')
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event.event_type, event.data, event.severity, language)
|
||||
self.assertNotIn('()', result['title'])
|
||||
self.assertIn('PBS', result['title'])
|
||||
self.assertNotIn('web (100)', result['title'])
|
||||
|
||||
def test_subject_only_setup_failure_survives_actual_email(self):
|
||||
reason = 'unable to activate storage PBS'
|
||||
title = 'vzdump backup status (node-a): backup failed: ' + reason
|
||||
message = REPORT.split('100 web')[0] + '\nTotal running time: 0s\nTotal size: 0 B'
|
||||
event = receive(message, 'error', title)
|
||||
for language in LANGUAGES:
|
||||
result, markup = email(event.event_type, event.data, event.severity, language)
|
||||
self.assertIn(reason, result['body'])
|
||||
self.assertIn(reason, html.unescape(markup))
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Final review contracts at actual locale/manual/queued/quiet email seams.
|
||||
All operational dependencies are inert; no manager or route module import.
|
||||
"""
|
||||
import ast
|
||||
import copy
|
||||
import html
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from notification_fixture import templates, SCRIPTS, LANGUAGES, receive
|
||||
from notification_final_fixture import deliver, restore_event
|
||||
|
||||
# Literal native send_notification body, produced by pinned PVE Perl helpers.
|
||||
NATIVE_REPORT = 'Details\n=======\n' + '''VMID Name Status Time Size Filename
|
||||
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
|
||||
|
||||
Total running time: 1m 1s
|
||||
Total size: 1 GiB
|
||||
|
||||
Logs
|
||||
====
|
||||
vzdump --all 1 --storage PBS --mode snapshot
|
||||
|
||||
100: no log available
|
||||
'''
|
||||
|
||||
|
||||
|
||||
class FinalCorrectionsTests(unittest.TestCase):
|
||||
def test_runtime_backup_assertions_accept_generated_and_missing_slovak_title(self):
|
||||
path = SCRIPTS / 'tests/test_notification_runtime_i18n.py'
|
||||
tree = ast.parse(path.read_text())
|
||||
owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'RuntimeCatalogTests')
|
||||
method = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == 'test_special_formatters_digest_and_test_message_are_slovak')
|
||||
start = next(i for i,n in enumerate(method.body) if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='backup' for t in n.targets))
|
||||
stop = next(i for i,n in enumerate(method.body) if isinstance(n,ast.Assign) and any(isinstance(t,ast.Name) and t.id=='manager' for t in n.targets))
|
||||
block = ast.Module(body=method.body[start:stop], type_ignores=[])
|
||||
english = templates._load_runtime_catalog('en')
|
||||
for title in (None, '{hostname}: GENERATED_SK potvrdené'):
|
||||
sk = copy.deepcopy(templates._load_runtime_catalog('sk'))
|
||||
if title is not None: sk.setdefault('backup', {})['confirmedTitle'] = title
|
||||
else: sk.get('backup', {}).pop('confirmedTitle', None)
|
||||
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: english if lang == 'en' else sk):
|
||||
exec(compile(block,str(path),'exec'), {'self':self, 'notification_templates':templates})
|
||||
|
||||
|
||||
def test_native_multiline_failure_diagnostics_survive_receiver_dispatch_once(self):
|
||||
subject = 'vzdump backup status (node-a): backup failed: multiple problems'
|
||||
for diagnostic, report in (
|
||||
('external provider job cleanup failed\njob-abort hook permission denied', NATIVE_REPORT),
|
||||
('job interrupted\njob-abort hook permission denied', NATIVE_REPORT.replace('ok ', 'todo ')),
|
||||
('unable to initialize external provider\njob-abort hook permission denied', NATIVE_REPORT.replace('100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z\n', '')),
|
||||
):
|
||||
event = receive(diagnostic + '\n' + report, 'error', subject)
|
||||
self.assertEqual((event.event_type, event.severity), ('backup_fail', 'CRITICAL'))
|
||||
for language in LANGUAGES:
|
||||
result = deliver(event.event_type, event.data, event.severity, language)
|
||||
for line in diagnostic.splitlines():
|
||||
self.assertEqual(result['body'].count(line), 1)
|
||||
self.assertEqual(result['text'].count(line), 1)
|
||||
if 'ok ' in report:
|
||||
self.assertIn('✅ VM web (100)', result['body'])
|
||||
|
||||
|
||||
def test_restore_endpoint_quiet_release_keeps_warning_counts_and_truthful_footer(self):
|
||||
reason = 'missing module zfs <raw> & {literal}'
|
||||
event = restore_event(reason)
|
||||
self.assertEqual(event['severity'], 'WARNING')
|
||||
for language in LANGUAGES:
|
||||
result = deliver(event['event_type'], event['data'], event['severity'], language, quiet=True)
|
||||
self.assertEqual(result['severity'], 'INFO')
|
||||
buffered_body = result['buffered'][0][3]
|
||||
self.assertEqual(result['text'].count(reason), 1)
|
||||
for line in buffered_body.splitlines():
|
||||
if line.strip(): self.assertIn(line.strip(), result['text'])
|
||||
self.assertIn('2m', result['text'])
|
||||
self.assertNotIn(templates.runtime_message('digest.footer', language), result['body'])
|
||||
self.assertNotIn('script', result['tags'])
|
||||
|
||||
|
||||
|
||||
|
||||
def test_raw_restore_cells_use_event_scoped_mail_wrapping(self):
|
||||
token = 'b' * 64
|
||||
event = restore_event('Boot check: recorded token ' + token + '; verification pending')
|
||||
for language in LANGUAGES:
|
||||
results = [deliver(event['event_type'],event['data'],event['severity'],language,quiet=quiet) for quiet in (False,True)]
|
||||
for result in results:
|
||||
self.assertIn('table-layout:fixed;', result['html'])
|
||||
self.assertIn('word-wrap:break-word;', result['html'])
|
||||
self.assertIn('overflow-wrap:break-word;', result['html'])
|
||||
self.assertEqual(result['text'].count(token), 1)
|
||||
unrelated = deliver('node_reconnect', {'hostname':'node-a'}, 'OK')
|
||||
self.assertNotIn('table-layout:fixed;', unrelated['html'])
|
||||
|
||||
|
||||
def test_manual_failed_and_unconfirmed_backup_keep_short_actionable_reason(self):
|
||||
for outcome, reason in (('failed','PBS permission denied for datastore remote'),
|
||||
('unconfirmed','Task status unavailable: upstream API timed out')):
|
||||
for language in LANGUAGES:
|
||||
result = deliver('backup_complete',{'hostname':'node-a','vmid':'100','vmname':'web',
|
||||
'storage':'PBS','backup_outcome':outcome,'reason':reason},'INFO',language,manual=True)
|
||||
self.assertEqual(result['text'].count(reason), 1)
|
||||
self.assertNotIn('>COMPLETED</span>', result['html'])
|
||||
|
||||
|
||||
def test_backup_reason_threshold_and_raw_body_deduplication(self):
|
||||
for event in ('backup_complete', 'backup_fail'):
|
||||
for length in (79, 80, 81, 120):
|
||||
prefix = '<raw> & {rack.location} '
|
||||
reason = prefix + 'b' * (length - len(prefix))
|
||||
self.assertEqual(len(reason), length)
|
||||
for raw in ('', reason):
|
||||
data = {'hostname':'node-a','backup_outcome':'failed','reason':reason,'pve_message':raw}
|
||||
for manual in (False, True):
|
||||
result = deliver(event,data,'INFO','en',manual=manual)
|
||||
self.assertEqual(result['text'].count(reason), 1, (event,length,raw,manual))
|
||||
self.assertNotIn('raw', result['tags'])
|
||||
|
||||
|
||||
def test_quiet_restore_after_preview_limit_is_not_omitted_or_called_info(self):
|
||||
event = restore_event('missing module zfs')
|
||||
earlier = [dict(event_type='service_fail',severity='WARNING',data={'hostname':'node-a','service_name':f'unit-{i}','reason':'recorded'}) for i in range(8)]
|
||||
result = deliver(event['event_type'],event['data'],event['severity'],quiet=True,quiet_before=earlier)
|
||||
self.assertEqual(result['text'].count('missing module zfs'), 1)
|
||||
self.assertNotIn(templates.runtime_message('digest.lead','en',count=9), result['body'])
|
||||
self.assertEqual(result['data']['_count'], 9)
|
||||
|
||||
|
||||
def test_native_error_block_does_not_deduplicate_a_substring_of_inventory(self):
|
||||
event = receive('web\nexternal provider job cleanup failed\n' + NATIVE_REPORT,
|
||||
'error','vzdump backup status (node-a): backup failed: multiple problems')
|
||||
result = deliver(event.event_type,event.data,event.severity)
|
||||
self.assertEqual(result['body'].splitlines().count('web'), 1)
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Maintainer acceptance at inert actual notification consumers."""
|
||||
import unittest
|
||||
from notification_fixture import templates, receive, LANGUAGES, SCRIPTS, extract
|
||||
from notification_final_fixture import deliver
|
||||
|
||||
NATIVE_REPORT = '''Details
|
||||
=======
|
||||
VMID Name Status Time Size Filename
|
||||
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
|
||||
|
||||
Total running time: 1m 1s
|
||||
Total size: 1 GiB
|
||||
|
||||
Logs
|
||||
====
|
||||
vzdump --all 1 --storage PBS --mode snapshot
|
||||
|
||||
100: 2026-09-29 17:00:00 INFO: Starting Backup of VM 100 (qemu)
|
||||
100: 2026-09-29 17:01:01 INFO: Finished Backup of VM 100 (00:01:01)
|
||||
'''
|
||||
|
||||
class MaintainerFollowupTests(unittest.TestCase):
|
||||
def test_completed_with_warnings_requires_independent_completion(self):
|
||||
warning = '\n100: 2026-09-29 17:00:01 WARN: file changed during backup'
|
||||
event = receive(NATIVE_REPORT + warning)
|
||||
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
|
||||
self.assertEqual((event.event_type,event.severity), ('backup_complete','INFO'))
|
||||
for manual in (False,True):
|
||||
for lang in LANGUAGES:
|
||||
result = deliver(event.event_type,event.data,event.severity,lang,manual=manual)
|
||||
label = templates.runtime_message('backup.warningTitle',lang,hostname='node-a')
|
||||
self.assertIn(label,result['title'])
|
||||
status = templates.runtime_message('channels.email.status.completed_with_warnings',lang)
|
||||
self.assertIn(status,result['text'])
|
||||
self.assertIn('file changed during backup',result['text'])
|
||||
# Manual sends intentionally skip channel emoji enrichment.
|
||||
if not manual: self.assertTrue(result['title'].startswith('💾⚠️'))
|
||||
rich, _ = templates.enrich_with_emojis(event.event_type,result['title'],result['body'],event.data)
|
||||
self.assertTrue(rich.startswith('💾⚠️'))
|
||||
self.assertEqual(receive('WARN: file changed during backup').data['backup_outcome'],'unconfirmed')
|
||||
self.assertEqual(receive(NATIVE_REPORT.split('Total running time:')[0]+warning).data['backup_outcome'],'unconfirmed')
|
||||
self.assertEqual(receive(NATIVE_REPORT+warning+'\nERROR: cleanup failed').data['backup_outcome'],'failed')
|
||||
self.assertEqual(receive(NATIVE_REPORT+warning,'warning').data['backup_outcome'],'completed_with_warnings')
|
||||
self.assertEqual(receive('INFO: Starting Backup of VM 100 (qemu)\nINFO: Finished Backup of VM 100 (00:01:01)'+warning).data['backup_outcome'],'completed_with_warnings')
|
||||
self.assertEqual(receive(NATIVE_REPORT+warning,kind='').data['backup_outcome'],'unconfirmed')
|
||||
|
||||
def test_null_filename_failed_guest_uses_own_start_identity(self):
|
||||
report = NATIVE_REPORT.replace('100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z',
|
||||
'100 web err 1m 1s 0 B null')
|
||||
for kind, prefix in (('qemu','VM'),('lxc','CT')):
|
||||
own = report.replace('VM 100 (qemu)',f'VM 100 ({kind})')
|
||||
# Unrelated guest appears first and must never supply the failed type.
|
||||
message = f'INFO: Starting Backup of VM 999 ({"lxc" if kind == "qemu" else "qemu"})\n' + own
|
||||
event = receive(message,'error','vzdump backup status (raw-node): backup failed')
|
||||
parsed = templates._parse_vzdump_message(message)
|
||||
self.assertEqual(parsed['vms'][0]['type'],kind)
|
||||
result = deliver(event.event_type,event.data,event.severity)
|
||||
self.assertIn(f'{prefix} web (100)',result['title'])
|
||||
self.assertIn(f'❌ {prefix} web (100)',result['body'])
|
||||
for message in (report.replace('Starting Backup of VM 100','Starting Backup of VM 999'),
|
||||
report+'\nINFO: Starting Backup of VM 100 (lxc)'):
|
||||
self.assertEqual(templates._parse_vzdump_message(message)['vms'][0]['type'],'')
|
||||
|
||||
def test_native_subject_keeps_cause_without_host_envelope(self):
|
||||
subject = 'vzdump backup status (raw-host): backup failed: multiple problems'
|
||||
event = receive('ERROR: archive write failed\n'+NATIVE_REPORT,'error',subject)
|
||||
event.data['hostname']='configured-alias'
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event.event_type,event.data,event.severity,lang)
|
||||
self.assertNotIn(subject,result['text'])
|
||||
self.assertEqual(result['text'].count('ERROR: archive write failed'),1)
|
||||
self.assertIn('configured-alias',result['title'])
|
||||
setup=receive('Details\n=======\nVMID Name Status Time Size Filename\n\nTotal running time: 0s\nTotal size: 0 B','error',subject.replace('multiple problems','unable to open storage'))
|
||||
result=deliver(setup.event_type,setup.data,setup.severity)
|
||||
self.assertEqual(result['text'].count('unable to open storage'),1)
|
||||
self.assertNotIn(setup.data['pve_title'],result['text'])
|
||||
unique=receive(NATIVE_REPORT,'error',subject.replace('multiple problems','job-end hook denied'))
|
||||
result=deliver(unique.event_type,unique.data,unique.severity)
|
||||
self.assertIn('job-end hook denied',result['body'])
|
||||
self.assertNotIn('vzdump backup status',result['body'])
|
||||
|
||||
def test_backup_diagnostics_are_bounded_with_principal_cause_and_notice(self):
|
||||
raw = NATIVE_REPORT + '\n' + '\n'.join(f'WARN: repeated warning {i}' for i in range(80)) + '\nERROR: principal archive write failure'
|
||||
event=receive(raw)
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event.event_type,event.data,event.severity,lang)
|
||||
self.assertIn('ERROR: principal archive write failure',result['body'])
|
||||
self.assertLessEqual(len('\n'.join(line for line in result['body'].splitlines() if line.startswith(('WARN:','ERROR:')))),1024)
|
||||
self.assertLessEqual(sum(line.startswith(('WARN:','ERROR:')) for line in result['body'].splitlines()),8)
|
||||
notice=templates.runtime_message('backup.diagnosticsOmitted',lang,count=73)
|
||||
self.assertIn(notice,result['body'])
|
||||
self.assertEqual(event.data['pve_message'],raw)
|
||||
long=receive('ERROR: '+ 'b'*5000,'error','vzdump backup status (node): backup failed')
|
||||
result=deliver(long.event_type,long.data,long.severity)
|
||||
self.assertLess(len(result['body']),1400)
|
||||
self.assertIn('ERROR: '+ 'b'*100,result['body'])
|
||||
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted','en',count=1),result['body'])
|
||||
|
||||
def test_real_quiet_digest_retains_each_backup_outcome_icon(self):
|
||||
samples=[(NATIVE_REPORT,'confirmed','💾✅'),(NATIVE_REPORT+'\nWARN: changed file','completed_with_warnings','💾⚠️'),(NATIVE_REPORT+'\nERROR: write failed','failed','💾❌'),('INFO: Starting Backup of VM 100 (qemu)','unconfirmed','💾❔')]
|
||||
for raw,outcome,icon in samples:
|
||||
event=receive(raw)
|
||||
self.assertEqual(event.data['backup_outcome'],outcome)
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event.event_type,event.data,event.severity,lang,quiet=True)
|
||||
self.assertIn(icon,result['body'])
|
||||
self.assertNotIn('💾❔',result['body']) if outcome!='unconfirmed' else None
|
||||
import types,datetime
|
||||
ns={'datetime':datetime.datetime,'runtime_message':templates.runtime_message,'EVENT_EMOJI':templates.EVENT_EMOJI,'CATEGORY_EMOJI':templates.CATEGORY_EMOJI}
|
||||
compose=extract(SCRIPTS/'notification_manager.py','_compose_digest_body','NotificationManager',ns)
|
||||
target=types.SimpleNamespace(_notification_language=lambda:'en')
|
||||
rows=[(i,'backup_complete','backup',1,icon+' node: Backup','') for i,(_,_,icon) in enumerate(samples)]
|
||||
body=compose(target,rows,use_icons=True)
|
||||
for _,_,icon in samples:self.assertIn(icon,body)
|
||||
plain=compose(target,rows,use_icons=False)
|
||||
for _,_,icon in samples:self.assertNotIn(icon,plain)
|
||||
should=extract(SCRIPTS/'notification_manager.py','_should_buffer_for_digest','NotificationManager',{})
|
||||
self.assertFalse(should(types.SimpleNamespace(_DIGEST_EXEMPT_EVENTS={'backup_complete'},_config={'email.digest_enabled':'true'}),'email','INFO','backup_complete'))
|
||||
|
||||
def test_quiet_restore_details_are_subordinate_in_text_and_email(self):
|
||||
from notification_final_fixture import restore_event
|
||||
event=restore_event('missing module zfs')
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event['event_type'],event['data'],event['severity'],lang,quiet=True)
|
||||
body_lines=result['buffered'][0][3].splitlines()
|
||||
for line in body_lines:
|
||||
if line.strip():
|
||||
self.assertIn(' '+line.strip(),result['body'])
|
||||
self.assertIn(' '+__import__('html').escape(line.strip()),result['html'])
|
||||
self.assertIn('white-space:pre-wrap;',result['html'])
|
||||
self.assertNotIn(templates.runtime_message('digest.footer',lang),result['body'])
|
||||
|
||||
def test_concrete_subject_cause_already_in_error_log_is_not_repeated(self):
|
||||
event=receive(NATIVE_REPORT+'\n100: ERROR: job-end hook denied','error',
|
||||
'vzdump backup status (raw-host): backup failed: job-end hook denied')
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event.event_type,event.data,event.severity,lang)
|
||||
self.assertEqual(result['text'].count('job-end hook denied'),1)
|
||||
self.assertIn('100: ERROR: job-end hook denied',result['body'])
|
||||
|
||||
def test_job_level_subject_cause_survives_warning_cap(self):
|
||||
raw=NATIVE_REPORT+'\n'+'\n'.join('WARN: repeated diagnostic '+str(i) for i in range(80))
|
||||
event=receive(raw,'error','vzdump backup status (raw-host): backup failed: job-end hook denied')
|
||||
for lang in LANGUAGES:
|
||||
result=deliver(event.event_type,event.data,event.severity,lang)
|
||||
self.assertEqual(result['body'].count('job-end hook denied'),1)
|
||||
self.assertNotIn('vzdump backup status',result['body'])
|
||||
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted',lang,count=73),result['body'])
|
||||
|
||||
def test_backup_quiet_email_uses_event_scoped_wrapping(self):
|
||||
event=receive(NATIVE_REPORT+'\nWARN: changed file')
|
||||
result=deliver(event.event_type,event.data,event.severity,'sv',quiet=True)
|
||||
self.assertTrue(result['data'].get('_backup_summary'))
|
||||
self.assertIn('table-layout:fixed;',result['html'])
|
||||
self.assertIn('overflow-wrap:break-word;',result['html'])
|
||||
unrelated=deliver('node_reconnect',{'hostname':'node-a'},'OK')
|
||||
self.assertNotIn('table-layout:fixed;',unrelated['html'])
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Inert producer-to-renderer checks for notification outcome claims."""
|
||||
import ast
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
from notification_fixture import templates as actual_templates
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPTS = ROOT / 'AppImage/scripts'
|
||||
CATALOG = ROOT / 'AppImage/messages/en/common.json'
|
||||
EXPECTED = {'system_restore_completed': {'body': 'Post-restore tasks completed in background.\n\nGuests applied: {guests}\nBind-mount stubs: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}\nThe node is now fully ready to use.'}}
|
||||
|
||||
|
||||
def extract(path, name, owner=None, namespace=None):
|
||||
tree = ast.parse(path.read_text())
|
||||
nodes = tree.body
|
||||
if owner:
|
||||
nodes = next(n.body for n in nodes if isinstance(n, ast.ClassDef) and n.name == owner)
|
||||
node = next(n for n in nodes if isinstance(n, ast.FunctionDef) and n.name == name)
|
||||
node.decorator_list = []
|
||||
ns = namespace if namespace is not None else {}
|
||||
exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), ns)
|
||||
return ns[name]
|
||||
|
||||
|
||||
def renderer(catalog, translated=None):
|
||||
path = SCRIPTS / 'notification_templates.py'
|
||||
tree = ast.parse(path.read_text())
|
||||
templates = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'TEMPLATES' for t in n.targets)))
|
||||
def lookup(obj, key):
|
||||
for part in key.split('.'):
|
||||
obj = obj.get(part) if isinstance(obj, dict) else None
|
||||
return obj
|
||||
def message(key, language='en', **values):
|
||||
source = translated if language == 'it' and translated is not None else catalog
|
||||
namespace = source['runtime']['notifications']
|
||||
value = lookup(namespace, key) or lookup(catalog['runtime']['notifications'], key) or ''
|
||||
return value.format_map(type('Safe', (dict,), {'__missing__': lambda self, k: ''})(values))
|
||||
ns = {'TEMPLATES': templates, 'Dict': dict, 'Any': Any, 'time': time, 're': re,
|
||||
'_get_hostname': lambda: 'node-a',
|
||||
'_load_runtime_catalog': lambda lang: (translated if lang == 'it' and translated is not None
|
||||
else catalog)['runtime']['notifications'],
|
||||
'_catalog_value': lookup, 'runtime_message': message}
|
||||
from typing import Optional
|
||||
ns['Optional'] = Optional
|
||||
extract(path, '_parse_vzdump_table', namespace=ns)
|
||||
extract(path, '_parse_vzdump_message', namespace=ns)
|
||||
extract(path, '_format_vzdump_body', namespace=ns)
|
||||
return templates, extract(path, 'render_template', namespace=ns)
|
||||
|
||||
|
||||
class OutcomeWording(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.catalog = json.loads(CATALOG.read_text())
|
||||
cls.templates, render = renderer(cls.catalog)
|
||||
cls.render = staticmethod(render)
|
||||
|
||||
def test_spanish_restore_uses_maintainer_guests_terminology(self):
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location('isolated_spanish_restore', SCRIPTS / 'notification_templates.py')
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
result = module.render_template('system_restore_completed', {
|
||||
'hostname':'node-a','guests':3,'stubs':0,'stale_nodes':0,
|
||||
'components':1,'duration':'2m','warnings_block':''}, 'es')
|
||||
self.assertIn('Guests aplicados: 3', result['body'])
|
||||
self.assertNotIn('invitados', result['body'].lower())
|
||||
|
||||
def test_settings_labels_stay_at_upstream_values_in_all_locales(self):
|
||||
# Frozen from develop eb7cc548; CI shallow checkouts have no base history.
|
||||
labels = {'en': {'backup_complete': 'Backup complete', 'error_resolved': 'Recovery notification'}, 'de': {'backup_complete': 'Sicherung abgeschlossen', 'error_resolved': 'Wiederherstellungsbenachrichtigung'}, 'es': {'backup_complete': 'Backup completado', 'error_resolved': 'Notificación de recuperación'}, 'fr': {'backup_complete': 'Sauvegarde terminée', 'error_resolved': 'Notification de récupération'}, 'it': {'backup_complete': 'Backup completato', 'error_resolved': 'Notifica di recupero'}, 'pt': {'backup_complete': 'Backup concluído', 'error_resolved': 'Notificação de recuperação'}, 'sk': {'backup_complete': 'Záloha bola dokončená', 'error_resolved': 'Problém bol vyriešený'}, 'sv': {'backup_complete': 'Säkerhetskopieringen är klar', 'error_resolved': 'Återställningsmeddelande'}}
|
||||
for lang in labels:
|
||||
path = f'AppImage/messages/{lang}/common.json'
|
||||
current = json.loads((ROOT / path).read_text())
|
||||
for event in ('backup_complete', 'error_resolved'):
|
||||
with self.subTest(lang=lang, event=event):
|
||||
expected = labels[lang][event]
|
||||
self.assertEqual(current['runtime']['notifications']['templates'][event]['label'], expected)
|
||||
if lang == 'en': self.assertEqual(self.templates[event]['label'], expected)
|
||||
|
||||
def test_exact_four_english_leaves_match_source_and_catalog(self):
|
||||
for event, fields in EXPECTED.items():
|
||||
for field, value in fields.items():
|
||||
with self.subTest(event=event, field=field):
|
||||
self.assertEqual(self.templates[event][field], value)
|
||||
self.assertEqual(self.catalog['runtime']['notifications']['templates'][event][field], value)
|
||||
|
||||
def test_webhook_backup_outcome_is_evidence_based_without_rerouting(self):
|
||||
path = SCRIPTS / 'notification_events.py'
|
||||
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
|
||||
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
|
||||
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
|
||||
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
|
||||
class Event:
|
||||
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
|
||||
class Queue:
|
||||
def __init__(self): self.items = []
|
||||
def put(self, event): self.items.append(event)
|
||||
ns['NotificationEvent'] = Event
|
||||
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
|
||||
class Receiver:
|
||||
_hostname = 'node-a'
|
||||
_classify_pve = classify
|
||||
_map_severity = staticmethod(severity_map)
|
||||
_backup_outcome = staticmethod(backup_outcome)
|
||||
def __init__(self): self._queue = Queue()
|
||||
header = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('VMID','Name','Status','Time','Size','Filename')
|
||||
row_ok = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('104','alpha','OK','00:01:00','1.5 GiB','archive')
|
||||
row_warning = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','WARNINGS','00:01:00','1.5 GiB','archive')
|
||||
row_error = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','ERROR','00:01:00','1.5 GiB','archive')
|
||||
row_err = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('105','beta','err','00:01:00','1.5 GiB','archive')
|
||||
truncated = 'INFO: Log output was too long to be displayed. Please see task log for details.'
|
||||
cases = [
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_err+'\nTotal running time: 00:02:00', 'failed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\n'+header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
|
||||
('vzdump', 'info', header+'\n'+row_err+'\nTotal running time: 00:01:00\n'+truncated, 'failed'),
|
||||
('vzdump', 'warning', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'completed_with_warnings'),
|
||||
('vzdump', 'info', header+'\n'+row_ok, 'unconfirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_ok, 'unconfirmed'),
|
||||
('vzdump', 'warning', header+'\n'+row_err+'\nTotal running time: 00:01:00', 'failed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00\n'+truncated, 'unconfirmed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00\nERROR: archive write failed', 'failed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_error+'\nTotal running time: 00:02:00', 'failed'),
|
||||
('vzdump', 'info', header+'\n'+row_error, 'failed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'confirmed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00', 'unconfirmed'),
|
||||
('vzdump', 'info', header+'\n'+row_ok[:30], 'unconfirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'confirmed'),
|
||||
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'completed_with_warnings'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)', 'unconfirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nWARNING: skipped file', 'completed_with_warnings'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: TASK OK\n104 alpha WARNINGS: 1', 'completed_with_warnings'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Finished Backup of VM 105 (00:01:00)', 'confirmed'),
|
||||
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed for VM 104', 'failed'),
|
||||
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'failed'),
|
||||
('', 'warning', 'Backup scheduled', 'unconfirmed'),
|
||||
('', 'info', 'Backup complete', 'unconfirmed'),
|
||||
]
|
||||
for kind, severity, message, expected in cases:
|
||||
with self.subTest(message=message, severity=severity):
|
||||
receiver = Receiver()
|
||||
receive(receiver, {'fields': {'type': kind}, 'severity': severity,
|
||||
'title': 'Backup', 'message': message})
|
||||
event = receiver._queue.items[0]
|
||||
self.assertEqual(event.event_type, 'backup_complete')
|
||||
self.assertEqual(event.data['backup_outcome'], expected)
|
||||
|
||||
def test_backup_classifier_preserves_confirmed_and_unverified_paths(self):
|
||||
path = SCRIPTS / 'notification_events.py'
|
||||
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
|
||||
classify = extract(path, '_classify_pve', 'ProxmoxHookWatcher', ns)
|
||||
severity_map = extract(path, '_map_severity', 'ProxmoxHookWatcher', ns)
|
||||
backup_outcome = extract(path, '_backup_outcome', 'ProxmoxHookWatcher', ns)
|
||||
class Event:
|
||||
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
|
||||
class Queue:
|
||||
def __init__(self): self.items = []
|
||||
def put(self, event): self.items.append(event)
|
||||
ns['NotificationEvent'] = Event
|
||||
receive = extract(path, 'process_webhook', 'ProxmoxHookWatcher', ns)
|
||||
class Receiver:
|
||||
_hostname = 'node-a'
|
||||
_classify_pve = classify
|
||||
_map_severity = staticmethod(severity_map)
|
||||
_backup_outcome = staticmethod(backup_outcome)
|
||||
def __init__(self): self._queue = Queue()
|
||||
samples = [('vzdump', 'info', 'Backup', 'job finished'),
|
||||
('vzdump', 'warning', 'Backup', 'job incomplete'),
|
||||
('', 'warning', 'backup job', 'Backup scheduled')]
|
||||
for kind, severity, title, message in samples:
|
||||
with self.subTest(kind=kind, message=message):
|
||||
event, entity, _ = classify(None, kind, severity, title, message)
|
||||
self.assertEqual((event, entity), ('backup_complete', 'vm'))
|
||||
receiver = Receiver()
|
||||
reply = receive(receiver, {'fields': {'type': kind}, 'severity': severity,
|
||||
'title': title, 'message': message})
|
||||
self.assertEqual(reply['event_type'], event)
|
||||
self.assertEqual(len(receiver._queue.items), 1)
|
||||
emitted = receiver._queue.items[0]
|
||||
self.assertEqual(emitted.severity, 'WARNING' if severity == 'warning' else 'INFO')
|
||||
output = self.render(emitted.event_type, emitted.data, 'en')
|
||||
self.assertIn('Backup outcome unconfirmed', output['title'])
|
||||
self.assertEqual(output['body'].splitlines()[-1], message) # raw PVE body retained
|
||||
self.assertEqual(classify(None, 'vzdump', 'error', 'Backup', 'failed')[0], 'backup_fail')
|
||||
|
||||
def test_incomplete_vzdump_log_does_not_certify_a_guest(self):
|
||||
from typing import Dict, Optional
|
||||
ns = {'re': re, 'Dict': Dict, 'Optional': Optional, 'Any': Any,
|
||||
'runtime_message': lambda key, lang, **kw: key}
|
||||
extract(SCRIPTS / 'notification_templates.py', '_parse_vzdump_table', namespace=ns)
|
||||
parser = extract(SCRIPTS / 'notification_templates.py', '_parse_vzdump_message', namespace=ns)
|
||||
formatter = extract(SCRIPTS / 'notification_templates.py', '_format_vzdump_body', namespace=ns)
|
||||
incomplete = parser('INFO: Starting Backup of VM 104 (qemu)')
|
||||
table_header = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('VMID','Name','Status','Time','Size','Filename')
|
||||
table_err = '{:<8}{:<22}{:<10}{:<10}{:<14}{}'.format('104','alpha','err','00:01:00','1.5 GiB','archive')
|
||||
failed_table = parser(table_header+'\n'+table_err+'\nTotal running time: 00:01:00')
|
||||
self.assertEqual(failed_table['vms'][0]['status'].lower(), 'error')
|
||||
self.assertIn('❌', formatter(failed_table, False, 'en'))
|
||||
self.assertEqual(incomplete['vms'][0]['status'], 'unknown')
|
||||
self.assertNotIn('✅', formatter(incomplete, False, 'en'))
|
||||
mixed = parser('INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Starting Backup of VM 105 (lxc)')
|
||||
self.assertEqual([vm['status'] for vm in mixed['vms']], ['ok', 'unknown'])
|
||||
formatted = formatter(mixed, False, 'en')
|
||||
self.assertEqual(formatted.count('✅'), 1)
|
||||
self.assertNotIn('❌', formatted)
|
||||
conflicting = parser('INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed\nINFO: Finished Backup of VM 104 (00:01:00)')
|
||||
self.assertEqual(conflicting['vms'][0]['status'], 'error')
|
||||
|
||||
def test_backup_render_preserves_success_and_marks_unconfirmed_and_failure(self):
|
||||
for outcome, message, title_part, body_part in (
|
||||
('confirmed', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'Backup complete', '✅'),
|
||||
('unconfirmed', 'INFO: Starting Backup of VM 104 (qemu)', 'Backup outcome unconfirmed', '❔'),
|
||||
('failed', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed', 'Backup error reported', '❌'),
|
||||
):
|
||||
with self.subTest(outcome=outcome):
|
||||
output = self.render('backup_complete', {'hostname': 'node-a', 'pve_type': 'vzdump',
|
||||
'pve_message': message, 'pve_title': 'Backup complete', 'backup_outcome': outcome}, 'en')
|
||||
self.assertIn(title_part, output['title'])
|
||||
self.assertIn(body_part, output['body'])
|
||||
if outcome != 'confirmed': self.assertNotIn('Backup complete', output['title'])
|
||||
output = self.render('backup_complete', {'hostname': 'node-a', 'vmname': 'vm', 'vmid': '104'}, 'en')
|
||||
self.assertIn('Backup outcome unconfirmed', output['title'])
|
||||
self.assertNotIn('successfully', output['body'])
|
||||
conflict = self.render('backup_complete', {'hostname': 'node-a','backup_outcome':'failed',
|
||||
'pve_message':'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nERROR: archive write failed'}, 'en')
|
||||
self.assertIn('ERROR: archive write failed', conflict['body'])
|
||||
self.assertNotIn('Backup complete', conflict['title'])
|
||||
|
||||
def test_confirmed_title_keeps_single_guest_and_destination_without_misnaming_batches(self):
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location('isolated_backup_title', SCRIPTS / 'notification_templates.py')
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
actual_render = module.render_template
|
||||
log = ('INFO: starting new backup job: vzdump 104 --storage PBS-Cloud --mode snapshot\n'
|
||||
'INFO: Starting Backup of VM 104 (qemu)\nINFO: VM Name: Alpha\n'
|
||||
'INFO: Finished Backup of VM 104 (00:01:00)')
|
||||
single = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
|
||||
'pve_message':log}, 'en')
|
||||
self.assertIn('PBS-Cloud',single['title'])
|
||||
self.assertIn('VM Alpha (104)',single['title'])
|
||||
batch = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
|
||||
'pve_message':log+'\nINFO: Starting Backup of VM 105 (lxc)\n'
|
||||
'INFO: Finished Backup of VM 105 (00:01:00)'}, 'en')
|
||||
self.assertIn('PBS-Cloud',batch['title'])
|
||||
self.assertNotIn('Alpha (104)',batch['title'])
|
||||
no_context = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed'}, 'en')
|
||||
self.assertEqual(no_context['title'], 'node-a: Backup complete')
|
||||
named = actual_render('backup_complete', {'hostname':'node-a','backup_outcome':'confirmed',
|
||||
'pve_message':log.replace('Alpha','Alpha {literal}')}, 'en')
|
||||
self.assertIn('Alpha {literal} (104)', named['title'])
|
||||
|
||||
def test_html_email_badge_and_backup_status_are_context_specific(self):
|
||||
import html
|
||||
path = SCRIPTS / 'notification_channels.py'
|
||||
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
|
||||
with self.subTest(lang=lang):
|
||||
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
|
||||
english = self.catalog['runtime']['notifications']
|
||||
def text(key, data=None, **values):
|
||||
def lookup(source):
|
||||
value = source['channels']
|
||||
for part in key.split('.'):
|
||||
value = value.get(part) if isinstance(value, dict) else None
|
||||
return value
|
||||
return (lookup(catalog) or lookup(english) or '').format(**values)
|
||||
ns = {'Dict': dict, 'Optional': __import__('typing').Optional,
|
||||
'_runtime_text': text, '_runtime_notification_text': lambda key, data=None: ''}
|
||||
build = extract(path, '_build_detail_rows', 'EmailChannel', ns)
|
||||
fmt = extract(path, '_format_html', 'EmailChannel', ns)
|
||||
class Email:
|
||||
from notification_channels import EmailChannel
|
||||
_SEV_STYLE = EmailChannel._SEV_STYLE
|
||||
_SEV_DEFAULT = EmailChannel._SEV_DEFAULT
|
||||
subject_prefix = 'ProxMenux'
|
||||
_build_detail_rows = staticmethod(build)
|
||||
unrelated = fmt(Email(), 'Reconnected', 'Body', 'OK', {'_event_type': 'node_reconnect',
|
||||
'_notification_language': lang, '_group': 'cluster'})
|
||||
self.assertIn('>' + catalog['channels']['email']['severity']['ok'].upper() + '</span>', unrelated)
|
||||
for outcome, status in [('confirmed', 'completed'), ('unconfirmed','unconfirmed'), ('failed','failed')]:
|
||||
email = fmt(Email(), 'Backup', 'Details', 'INFO', {'_event_type': 'backup_complete',
|
||||
'backup_outcome': outcome, '_notification_language': lang, '_group': 'backup'})
|
||||
label = catalog['channels']['email']['status'].get(status) or english['channels']['email']['status'][status]
|
||||
self.assertIn(label, html.unescape(email))
|
||||
badge_label = label.upper()
|
||||
self.assertIn('>' + badge_label + '</span>', html.unescape(email))
|
||||
if outcome == 'failed':
|
||||
self.assertIn('color:#dc2626;font-weight:600;', email)
|
||||
self.assertIn('background:#fef2f2;', email)
|
||||
elif outcome == 'unconfirmed':
|
||||
self.assertIn('background:#f9fafb;', email)
|
||||
else:
|
||||
self.assertIn('background:#f0fdf4;', email)
|
||||
|
||||
def test_actual_all_locale_rendering_and_fallback_for_backup_outcomes(self):
|
||||
import importlib.util
|
||||
module_path = SCRIPTS / 'notification_templates.py'
|
||||
spec = importlib.util.spec_from_file_location('isolated_notification_templates', module_path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module) # templates and catalogs only; no manager or sends
|
||||
samples = {
|
||||
'confirmed': 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)',
|
||||
'unconfirmed': 'INFO: Starting Backup of VM 104 (qemu)',
|
||||
'failed': 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed',
|
||||
}
|
||||
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
|
||||
catalog = json.loads((ROOT / 'AppImage/messages' / lang / 'common.json').read_text())['runtime']['notifications']
|
||||
for state, message in samples.items():
|
||||
with self.subTest(lang=lang, state=state):
|
||||
data = {'hostname':'node-with-a-long-name','backup_outcome':state,'pve_type':'vzdump',
|
||||
'pve_message':message, 'pve_title':'Backup complete', '_notification_language':lang}
|
||||
result = module.render_template('backup_complete', data, lang)
|
||||
if state != 'unconfirmed':
|
||||
key = 'confirmedTitle' if state == 'confirmed' else 'errorTitle'
|
||||
expected_title = (catalog.get('backup', {}).get(key) or
|
||||
self.catalog['runtime']['notifications']['backup'][key]).format(hostname=data['hostname'])
|
||||
self.assertTrue(result['title'].startswith(expected_title), result['title'])
|
||||
else:
|
||||
self.assertEqual(result['title'], module.runtime_message('backup.unconfirmedTitle', lang, hostname=data['hostname']))
|
||||
self.assertNotIn('{hostname}', result['title'])
|
||||
if state == 'unconfirmed':
|
||||
source = (catalog if catalog.get('backup', {}).get('unconfirmedBody')
|
||||
else self.catalog['runtime']['notifications'])
|
||||
self.assertIn(source['backup']['unconfirmedBody'], result['body'])
|
||||
if state == 'failed':
|
||||
self.assertIn(catalog.get('backup', {}).get('errorBody') or
|
||||
self.catalog['runtime']['notifications']['backup']['errorBody'], result['body'])
|
||||
enriched, _ = module.enrich_with_emojis('backup_complete', result['title'], result['body'], data)
|
||||
self.assertTrue(enriched.startswith({'confirmed':'💾✅','unconfirmed':'💾❔','failed':'💾❌'}[state]))
|
||||
restore = module.render_template('system_restore_completed', {'hostname':'node', 'guests':4,
|
||||
'stubs':1,'stale_nodes':2,'components':1,'duration':'2m','warnings_block':'Missing module'},lang)
|
||||
self.assertIn('Missing module',restore['body'])
|
||||
if lang == 'en': self.assertIn('fully ready',restore['body'].lower())
|
||||
|
||||
|
||||
|
||||
def test_warning_and_clean_restore_keep_only_reported_outcome(self):
|
||||
for warnings in ('', '⚠️ Boot sanity: missing modules\n'):
|
||||
with self.subTest(warnings=warnings):
|
||||
events = []
|
||||
ns = {'request': types.SimpleNamespace(remote_addr='127.0.0.1', get_json=lambda **kw: {
|
||||
'hostname': 'node-a', 'guests': '2', 'stubs': '0', 'stale_nodes': '0',
|
||||
'components': 'none', 'duration': '2m',
|
||||
'warnings': 'missing modules' if warnings else ''}),
|
||||
'notification_manager': types.SimpleNamespace(emit_event=lambda **kw: events.append(kw)),
|
||||
'jsonify': lambda obj: obj}
|
||||
handler = extract(SCRIPTS / 'flask_notification_routes.py', 'internal_restore_event', namespace=ns)
|
||||
response, status = handler()
|
||||
self.assertEqual((status, response['event_type']), (200, 'system_restore_completed'))
|
||||
event = events[0]
|
||||
self.assertEqual(event['severity'], 'WARNING' if warnings else 'INFO')
|
||||
result = self.render(event['event_type'], event['data'], 'en')
|
||||
self.assertIn('Post-restore tasks completed', result['body'])
|
||||
self.assertIn('fully ready', result['body'])
|
||||
if warnings: self.assertIn('missing modules', result['body'])
|
||||
|
||||
|
||||
def test_rich_backup_icon_tracks_outcome_and_digest_default_is_neutral(self):
|
||||
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
|
||||
def assign(name):
|
||||
return ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
|
||||
and any(isinstance(t, ast.Name) and t.id == name for t in n.targets)))
|
||||
icons = assign('EVENT_EMOJI')
|
||||
self.assertNotIn('✅', icons['backup_complete'])
|
||||
ns = {'TEMPLATES': assign('TEMPLATES'), 'EVENT_EMOJI': icons,
|
||||
'Dict': dict, 'Any': Any,
|
||||
'CATEGORY_EMOJI': assign('CATEGORY_EMOJI'), 'SEVERITY_ICONS': assign('SEVERITY_ICONS'),
|
||||
'FIELD_EMOJI': assign('FIELD_EMOJI'), '_localized_template_labels': lambda *a: {},
|
||||
'_lxc_update_label_icons': lambda *a: {}}
|
||||
enrich = extract(SCRIPTS / 'notification_templates.py', 'enrich_with_emojis', namespace=ns)
|
||||
for state, icon in [('confirmed', '💾✅'), ('unconfirmed', '💾❔'), ('failed', '💾❌')]:
|
||||
with self.subTest(state=state):
|
||||
title, body = enrich('backup_complete', 'node-a: Backup', 'Backup report',
|
||||
{'backup_outcome': state, '_notification_language': 'en'})
|
||||
self.assertTrue(title.startswith(icon), title)
|
||||
|
||||
def test_uncertain_outcomes_have_no_blanket_success_icon(self):
|
||||
tree = ast.parse((SCRIPTS / 'notification_templates.py').read_text())
|
||||
icon_map = ast.literal_eval(next(n.value for n in tree.body if isinstance(n, ast.Assign)
|
||||
and any(isinstance(t, ast.Name) and t.id == 'EVENT_EMOJI' for t in n.targets)))
|
||||
self.assertNotIn('✅', icon_map['backup_complete']) # buffered digest has no outcome metadata
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Principal diagnostics must not be deduplicated against inventory substrings."""
|
||||
import unittest
|
||||
from notification_fixture import receive, LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
|
||||
|
||||
def noisy_report(name='ordinary-web', storage='PBS', filename='null', cause='denied'):
|
||||
# Frozen native Perl notifier/log-reader shape; fixed-column table is
|
||||
# source-modeled from the official plaintext renderer, not a live PVE run.
|
||||
return ('\nDetails\n=======\nVMID Name Status Time Size Filename\n'
|
||||
f'100 {name} err 1m 1s 0 B {filename}\n\n'
|
||||
'Total running time: 1m 1s\nTotal size: 0 B\n\nLogs\n====\n'
|
||||
f'vzdump --all 1 --storage {storage} --mode snapshot\n\n' +
|
||||
'\n'.join(f'100: 2026-09-29 17:00:00 ERROR: earlier diagnostic {i}' for i in range(12)) +
|
||||
f'\n100: 2026-09-29 17:00:00 ERROR: {cause}\n\n')
|
||||
|
||||
|
||||
class PrincipalInventoryTests(unittest.TestCase):
|
||||
def assert_principal(self, raw, cause):
|
||||
event = receive(raw, 'error', 'vzdump backup status (raw-host): backup failed: ' + cause)
|
||||
for language in LANGUAGES:
|
||||
for manual in (False, True):
|
||||
with self.subTest(language=language, manual=manual):
|
||||
result = deliver(event.event_type, {**event.data, 'hostname':'alias {rack.location}'},
|
||||
event.severity, language, manual=manual)
|
||||
exact = [line for line in result['body'].splitlines()
|
||||
if line.strip() == cause or line.rstrip().endswith('ERROR: ' + cause)]
|
||||
self.assertEqual(len(exact), 1)
|
||||
diagnostics = [line for line in result['body'].splitlines() if 'ERROR:' in line]
|
||||
self.assertLessEqual(len(diagnostics), 8)
|
||||
self.assertLessEqual(len('\n'.join(diagnostics)), 1024)
|
||||
self.assertTrue(all(len(line) <= 512 for line in diagnostics))
|
||||
self.assertEqual(result['data']['pve_message'], raw)
|
||||
self.assertNotIn('raw-host', result['text'])
|
||||
self.assertIn('alias {rack.location}', result['text'])
|
||||
self.assertEqual(result['text'].count('ERROR: ' + cause), 1)
|
||||
|
||||
def test_late_principal_survives_incidental_guest_storage_and_archive(self):
|
||||
for name, storage, filename in (('ordinary-web','PBS','null'), ('denied','PBS','null'),
|
||||
('ordinary-web','denied','null'), ('ordinary-web','PBS','denied.tar')):
|
||||
with self.subTest(name=name, storage=storage, filename=filename):
|
||||
self.assert_principal(noisy_report(name,storage,filename), 'denied')
|
||||
|
||||
def test_raw_principal_braces_and_markup_survive_once(self):
|
||||
cause = 'denied {rack.location} <native>'
|
||||
self.assert_principal(noisy_report('denied', cause=cause), cause)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Frozen PVE 9.2 report through inert actual receiver and renderers.
|
||||
|
||||
No git history, host-management import, notification send or generated fixture.
|
||||
"""
|
||||
import ast
|
||||
import importlib.util
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPTS = ROOT / 'AppImage/scripts'
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import notification_templates as templates
|
||||
|
||||
PVE92 = '''Details
|
||||
=======
|
||||
VMID Name Status Time Size Filename
|
||||
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
|
||||
|
||||
Total running time: 1m 1s
|
||||
Total size: 1 GiB
|
||||
'''
|
||||
|
||||
|
||||
def receiver():
|
||||
tree = ast.parse((SCRIPTS / 'notification_events.py').read_text())
|
||||
owner = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == 'ProxmoxHookWatcher')
|
||||
names = ('_classify_pve', '_map_severity', '_backup_outcome', 'process_webhook')
|
||||
ns = {'re': re, 'capture_journal_context': lambda **kw: ''}
|
||||
class Event:
|
||||
def __init__(self, **kw): self.__dict__.update(kw); self.event_id = 'inert'
|
||||
ns['NotificationEvent'] = Event
|
||||
for name in names:
|
||||
node = next(n for n in owner.body if isinstance(n, ast.FunctionDef) and n.name == name)
|
||||
node.decorator_list = []
|
||||
exec(compile(ast.Module(body=[node], type_ignores=[]), '<inert-webhook>', 'exec'), ns)
|
||||
class Queue:
|
||||
def __init__(self): self.items = []
|
||||
def put(self, event): self.items.append(event)
|
||||
class Receiver:
|
||||
_hostname = 'node-a'
|
||||
_classify_pve = ns['_classify_pve']
|
||||
_map_severity = staticmethod(ns['_map_severity'])
|
||||
_backup_outcome = staticmethod(ns['_backup_outcome'])
|
||||
process_webhook = ns['process_webhook']
|
||||
def __init__(self): self._queue = Queue()
|
||||
return Receiver()
|
||||
|
||||
|
||||
def event_for(message, severity='info'):
|
||||
target = receiver()
|
||||
target.process_webhook({'fields': {'type': 'vzdump'}, 'severity': severity,
|
||||
'title': 'Backup', 'message': message})
|
||||
return target._queue.items[0]
|
||||
|
||||
|
||||
class PVE92Tests(unittest.TestCase):
|
||||
def test_exact_maintainer_report_is_confirmed(self):
|
||||
event = event_for(PVE92)
|
||||
self.assertEqual(event.data['backup_outcome'], 'confirmed')
|
||||
result = templates.render_template(event.event_type, event.data, 'en')
|
||||
self.assertIn('Backup complete', result['title'])
|
||||
self.assertIn('web (100)', result['title'])
|
||||
self.assertIn('✅', result['body'])
|
||||
|
||||
def test_failed_guest_identity_does_not_come_from_timestamp_or_ok_guest(self):
|
||||
failed = '101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z'
|
||||
ok = PVE92.splitlines()[3]
|
||||
for rows in (ok + '\n' + failed, failed + '\n' + ok):
|
||||
for severity in ('info', 'error'):
|
||||
with self.subTest(rows=rows, severity=severity):
|
||||
message = ('INFO: 100 01:01:06 OK\nINFO: Starting Backup of VM 100 (qemu)\n'
|
||||
+ PVE92.replace(ok, rows))
|
||||
event = event_for(message, severity)
|
||||
self.assertEqual(event.data['backup_outcome'], 'failed')
|
||||
self.assertEqual(event.data.get('vmname'), 'db')
|
||||
self.assertEqual(event.data.get('vmid'), '101')
|
||||
result = templates.render_template(event.event_type, event.data, 'en')
|
||||
self.assertIn('db (101)', result['title'])
|
||||
self.assertNotIn('web', result['title'])
|
||||
self.assertNotIn('01:01:06', result['title'])
|
||||
self.assertIn('❌ CT db (101)', result['body'])
|
||||
|
||||
def test_pbs_prefix_is_used_in_parsed_type_and_title(self):
|
||||
for prefix, kind, label in (('vm', 'qemu', 'VM'), ('ct', 'lxc', 'CT'),
|
||||
('other', '', 'VM/CT')):
|
||||
with self.subTest(prefix=prefix):
|
||||
message = PVE92.replace('vm/100/', prefix + '/100/')
|
||||
parsed = templates._parse_vzdump_message(message)
|
||||
self.assertEqual(parsed['vms'][0]['type'], kind)
|
||||
event = event_for(message)
|
||||
result = templates.render_template(event.event_type, event.data, 'en')
|
||||
self.assertIn(label + ' web (100)', result['title'])
|
||||
|
||||
def test_slovak_uses_normal_locale_resolution_without_source_sentence_overrides(self):
|
||||
from unittest.mock import patch
|
||||
data = {'hostname': 'node-a', 'category': 'temperature', 'entity_suffix': '',
|
||||
'reason': 'old', 'duration': '3d', 'original_severity': 'WARNING',
|
||||
'vmname': 'web', 'vmid': '100', 'storage': 'PBS', 'size': '1 GiB',
|
||||
'guests': 3, 'stubs': 0, 'stale_nodes': 0, 'components': 1,
|
||||
'warnings_block': ''}
|
||||
slovak = templates._load_runtime_catalog('sk')
|
||||
english = templates._load_runtime_catalog('en')
|
||||
for event, field in (('system_restore_completed', 'body'),):
|
||||
with self.subTest(event=event, field=field):
|
||||
value = slovak['templates'][event][field]
|
||||
result = templates.render_template(event, data, 'sk')
|
||||
self.assertEqual(result[field], value.format(**data))
|
||||
self.assertNotIn(value, (SCRIPTS / 'notification_templates.py').read_text())
|
||||
# Independently updated and absent leaves use the usual provider.
|
||||
import copy
|
||||
future = copy.deepcopy(slovak)
|
||||
future['templates'][event][field] = 'REVIEWED {hostname}'
|
||||
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: future if lang == 'sk' else english):
|
||||
self.assertEqual(templates.render_template(event, data, 'sk')[field], 'REVIEWED node-a')
|
||||
future['templates'][event].pop(field)
|
||||
with patch.object(templates, '_load_runtime_catalog', side_effect=lambda lang: future if lang == 'sk' else english):
|
||||
self.assertEqual(templates.render_template(event, data, 'sk')[field],
|
||||
templates.render_template(event, data, 'en')[field])
|
||||
|
||||
def test_complete_table_beats_only_truncated_supplemental_log(self):
|
||||
message = ('INFO: Starting Backup of VM 100 (qemu)\n' + PVE92 +
|
||||
'\nLogs\n====\nINFO: Log output was too long to be displayed. Please see task log for details.')
|
||||
self.assertEqual(event_for(message).data['backup_outcome'], 'confirmed')
|
||||
for message, severity, expected in (
|
||||
(PVE92.replace('ok ', 'OK '), 'info', 'confirmed'),
|
||||
(message, 'warning', 'completed_with_warnings'),
|
||||
(message, 'error', 'failed'),
|
||||
(message + '\nERROR: archive write failed', 'info', 'failed'),
|
||||
(message + '\nWARNING: skipped file', 'info', 'completed_with_warnings'),
|
||||
(PVE92.replace('ok ', 'WARNINGS '), 'info', 'unconfirmed'),
|
||||
):
|
||||
with self.subTest(message=message, severity=severity):
|
||||
self.assertEqual(event_for(message, severity).data['backup_outcome'], expected)
|
||||
|
||||
def test_incomplete_and_unrelated_sections_cannot_certify_table(self):
|
||||
for message in (
|
||||
PVE92.split('Total running time:')[0],
|
||||
PVE92.replace('vm/100/2026-09-29T17:00:00Z', ''),
|
||||
PVE92.replace('\n\nTotal', '\n\nLogs\n======\nTotal'),
|
||||
PVE92.replace('\n\nTotal', '\n\nUnrelated section\n100 web ok\nTotal'),
|
||||
PVE92.replace('1m 1s 1 GiB', 'nonsense 1 GiB'),
|
||||
PVE92.replace('1 GiB vm/', 'garbage vm/'),
|
||||
PVE92.replace('1m 1s 1 GiB vm/', '1m 1s'),
|
||||
):
|
||||
with self.subTest(message=message):
|
||||
# Complete guest logs cannot rescue a genuinely incomplete table.
|
||||
message += '\nINFO: Starting Backup of VM 100 (qemu)\nINFO: Finished Backup of VM 100 (00:01:01)'
|
||||
self.assertEqual(event_for(message).data['backup_outcome'], 'unconfirmed')
|
||||
|
||||
def test_blank_line_between_rows_does_not_hide_a_failure(self):
|
||||
message = PVE92.replace('\n\nTotal',
|
||||
'\n\n101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z\n\nTotal')
|
||||
event = event_for(message)
|
||||
self.assertEqual(event.data['backup_outcome'], 'failed')
|
||||
result = templates.render_template(event.event_type, event.data, 'en')
|
||||
self.assertIn('db (101)', result['title'])
|
||||
self.assertIn('❌ CT db (101)', result['body'])
|
||||
|
||||
def test_changed_titles_and_rows_reach_actual_html_email_in_all_locales(self):
|
||||
import html
|
||||
from notification_channels import EmailChannel
|
||||
email = object.__new__(EmailChannel)
|
||||
email.subject_prefix = '[ProxMenux]'
|
||||
failed = PVE92.replace('\n\nTotal',
|
||||
'\n101 db err 1m 1s 0 B ct/101/2026-09-29T17:00:00Z\n\nTotal')
|
||||
for lang in ('en', 'de', 'es', 'fr', 'it', 'pt', 'sk', 'sv'):
|
||||
for message, severity in ((PVE92, 'info'), (failed, 'info'), (failed, 'error')):
|
||||
with self.subTest(lang=lang, severity=severity, message=message):
|
||||
event = event_for(message, severity)
|
||||
result = templates.render_template(event.event_type, event.data, lang)
|
||||
context = {**event.data, '_event_type': event.event_type,
|
||||
'_notification_language': lang, '_group': result['group']}
|
||||
markup = html.unescape(email._format_html(result['title'], result['body'], event.severity, context))
|
||||
self.assertIn(result['title'], markup)
|
||||
if event.data['backup_outcome'] == 'failed':
|
||||
self.assertIn('db (101)', result['title'])
|
||||
self.assertNotIn('web', result['title'])
|
||||
self.assertIn('❌ CT db (101)', markup)
|
||||
status = templates.runtime_message('channels.email.status.failed', lang)
|
||||
else:
|
||||
self.assertIn('VM web (100)', result['title'])
|
||||
self.assertIn('✅ VM web (100)', markup)
|
||||
status = templates.runtime_message('channels.email.status.completed', lang)
|
||||
badge = (templates.runtime_message('channels.email.severity.critical', lang)
|
||||
if event.event_type == 'backup_fail' else status)
|
||||
self.assertIn('>' + badge.upper() + '</span>', markup)
|
||||
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Review regressions at actual consumer seams; no operational imports."""
|
||||
import unittest
|
||||
from notification_fixture import receive, LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
|
||||
class ReviewCorrectionTests(unittest.TestCase):
|
||||
def test_subject_equivalent_late_error_is_reserved_before_cap(self):
|
||||
cause = 'job-end hook denied'
|
||||
# Frozen native Perl notifier/log-reader output; Rust table source-modeled.
|
||||
raw = '\nDetails\n=======\nVMID Name Status Time Size Filename \n100 web err 1m 1s 0 B null \n\nTotal running time: 1m 1s\nTotal size: 0 B\n\nLogs\n====\nvzdump --all 1 --storage PBS --mode snapshot\n\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 0\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 1\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 2\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 3\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 4\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 5\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 6\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 7\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 8\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 9\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 10\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 11\n100: 2026-09-29 17:00:00 ERROR: job-end hook denied\n\n\n'
|
||||
event = receive(raw, 'error', 'vzdump backup status (raw-host): backup failed: ' + cause)
|
||||
for language in LANGUAGES:
|
||||
for manual in (False, True):
|
||||
with self.subTest(language=language, manual=manual):
|
||||
result = deliver(event.event_type, {**event.data, 'hostname':'alias {rack.location}'}, event.severity, language, manual=manual)
|
||||
self.assertEqual(result['text'].count(cause), 1)
|
||||
self.assertNotIn('raw-host', result['text'])
|
||||
diagnostics = [line for line in result['body'].splitlines() if 'ERROR:' in line]
|
||||
self.assertLessEqual(len(diagnostics), 8)
|
||||
self.assertLessEqual(len('\n'.join(diagnostics)), 1024)
|
||||
self.assertTrue(all(len(line) <= 512 for line in diagnostics))
|
||||
self.assertEqual(result['data']['pve_message'], raw)
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Fehlgeschlagen",
|
||||
"completed": "Abgeschlossen",
|
||||
"started": "Gestartet"
|
||||
"started": "Gestartet",
|
||||
"unconfirmed": "Nicht bestätigt",
|
||||
"completed_with_warnings": "Mit Warnungen abgeschlossen"
|
||||
},
|
||||
"report": "{group} Bericht",
|
||||
"details": "Details",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname}: Backup-Ergebnis nicht bestätigt",
|
||||
"confirmedTitle": "{hostname}: Backup abgeschlossen",
|
||||
"confirmedBody": "Backup erfolgreich abgeschlossen.",
|
||||
"errorTitle": "{hostname}: Backup-Fehler gemeldet",
|
||||
"errorBody": "Der Backup-Bericht enthält einen Fehler.",
|
||||
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt.",
|
||||
"warningTitle": "{hostname}: Sicherung mit Warnungen abgeschlossen",
|
||||
"warningBody": "Sicherung mit Warnungen abgeschlossen.",
|
||||
"diagnosticsOmitted": "Weitere Diagnosezeilen oder Text ausgelassen: {count}. Originalbericht bleibt erhalten."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -6556,7 +6556,7 @@
|
||||
"label": "Reinicio del sistema"
|
||||
},
|
||||
"system_restore_completed": {
|
||||
"title": "{hostname}: restauración del host finalizada",
|
||||
"title": "{hostname}: Restauración del host finalizada",
|
||||
"body": "Tareas posteriores a la restauración completadas en segundo plano.\n\nGuests aplicados: {guests}\nStubs de bind mount: {stubs}\nDirectorios de nodos obsoletos eliminados: {stale_nodes}\nComponentes reinstalados: {components}\nDuración: {duration}\n{warnings_block}\nEl nodo está listo para usarse.",
|
||||
"label": "Restauración del host completada"
|
||||
},
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Fallido",
|
||||
"completed": "Completado",
|
||||
"started": "Iniciado"
|
||||
"started": "Iniciado",
|
||||
"unconfirmed": "Sin confirmar",
|
||||
"completed_with_warnings": "Completado con advertencias"
|
||||
},
|
||||
"report": "Informe de {group}",
|
||||
"details": "Detalles",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "Lecturas altas registradas a lo largo de {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname}: Resultado del backup sin confirmar",
|
||||
"confirmedTitle": "{hostname}: Backup completado",
|
||||
"confirmedBody": "Backup completado correctamente.",
|
||||
"errorTitle": "{hostname}: Backup fallido",
|
||||
"errorBody": "El informe del backup contiene un error.",
|
||||
"unconfirmedBody": "El resultado del backup no está confirmado.",
|
||||
"warningTitle": "{hostname}: Backup completado con advertencias",
|
||||
"warningBody": "Backup completado con advertencias.",
|
||||
"diagnosticsOmitted": "Líneas o texto de diagnóstico omitidos: {count}. Se conserva el informe original."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Échec",
|
||||
"completed": "Terminé",
|
||||
"started": "Commencé"
|
||||
"started": "Commencé",
|
||||
"unconfirmed": "Non confirmé",
|
||||
"completed_with_warnings": "Terminée avec avertissements"
|
||||
},
|
||||
"report": "Rapport {group}",
|
||||
"details": "Détails",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "Les relevés élevés s'étendent sur {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname} : résultat de la sauvegarde non confirmé",
|
||||
"confirmedTitle": "{hostname} : sauvegarde terminée",
|
||||
"confirmedBody": "Sauvegarde terminée avec succès.",
|
||||
"errorTitle": "{hostname} : erreur signalée lors de la sauvegarde",
|
||||
"errorBody": "Le rapport de sauvegarde contient une erreur.",
|
||||
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé.",
|
||||
"warningTitle": "{hostname}: Sauvegarde terminée avec avertissements",
|
||||
"warningBody": "Sauvegarde terminée avec avertissements.",
|
||||
"diagnosticsOmitted": "Lignes ou texte de diagnostic omis : {count}. Rapport original conservé."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Fallito",
|
||||
"completed": "Completato",
|
||||
"started": "Iniziato"
|
||||
"started": "Iniziato",
|
||||
"unconfirmed": "Non confermato",
|
||||
"completed_with_warnings": "Completato con avvisi"
|
||||
},
|
||||
"report": "{group} Rapporto",
|
||||
"details": "Dettagli",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "Intervallo dei campioni sopra soglia: {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname}: esito del backup non confermato",
|
||||
"confirmedTitle": "{hostname}: backup completato",
|
||||
"confirmedBody": "Backup completato correttamente.",
|
||||
"errorTitle": "{hostname}: errore segnalato nel backup",
|
||||
"errorBody": "Il rapporto del backup contiene un errore.",
|
||||
"unconfirmedBody": "L’esito del backup non è confermato.",
|
||||
"warningTitle": "{hostname}: Backup completato con avvisi",
|
||||
"warningBody": "Backup completato con avvisi.",
|
||||
"diagnosticsOmitted": "Righe o testo diagnostico omessi: {count}. Il report originale è conservato."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Falhou",
|
||||
"completed": "Concluído",
|
||||
"started": "Iniciado"
|
||||
"started": "Iniciado",
|
||||
"unconfirmed": "Não confirmado",
|
||||
"completed_with_warnings": "Concluído com avisos"
|
||||
},
|
||||
"report": "Relatório {group}",
|
||||
"details": "Detalhes",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "As amostras elevadas abrangem {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname}: resultado do backup não confirmado",
|
||||
"confirmedTitle": "{hostname}: backup concluído",
|
||||
"confirmedBody": "Backup concluído com sucesso.",
|
||||
"errorTitle": "{hostname}: erro comunicado no backup",
|
||||
"errorBody": "O relatório do backup contém um erro.",
|
||||
"unconfirmedBody": "O resultado do backup não está confirmado.",
|
||||
"warningTitle": "{hostname}: Backup concluído com avisos",
|
||||
"warningBody": "Backup concluído com avisos.",
|
||||
"diagnosticsOmitted": "Linhas ou texto de diagnóstico omitidos: {count}. Relatório original preservado."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6978,7 +6978,9 @@
|
||||
"status": {
|
||||
"failed": "Misslyckades",
|
||||
"completed": "Klar",
|
||||
"started": "Startat"
|
||||
"started": "Startat",
|
||||
"unconfirmed": "Obekräftat",
|
||||
"completed_with_warnings": "Klar med varningar"
|
||||
},
|
||||
"report": "{group} Rapportera",
|
||||
"details": "Detaljer",
|
||||
@@ -6988,6 +6990,17 @@
|
||||
},
|
||||
"temperature": {
|
||||
"sampleSpan": "De höga mätvärdena sträcker sig över {duration}."
|
||||
},
|
||||
"backup": {
|
||||
"unconfirmedTitle": "{hostname}: säkerhetskopians resultat obekräftat",
|
||||
"confirmedTitle": "{hostname}: säkerhetskopiering klar",
|
||||
"confirmedBody": "Säkerhetskopieringen slutfördes utan fel.",
|
||||
"errorTitle": "{hostname}: fel rapporterat vid säkerhetskopiering",
|
||||
"errorBody": "Rapporten om säkerhetskopieringen innehåller ett fel.",
|
||||
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat.",
|
||||
"warningTitle": "{hostname}: Säkerhetskopiering klar med varningar",
|
||||
"warningBody": "Säkerhetskopiering klar med varningar.",
|
||||
"diagnosticsOmitted": "Utelämnade diagnosrader eller text: {count}. Originalrapporten bevaras."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1037,12 +1037,33 @@ class EmailChannel(NotificationChannel):
|
||||
|
||||
# Determine group for section header
|
||||
event_type = data.get('_event_type', '')
|
||||
if event_type == 'backup_complete':
|
||||
outcome = data.get('backup_outcome')
|
||||
if outcome == 'confirmed':
|
||||
sev.update(self._SEV_STYLE['OK'])
|
||||
status = 'completed'
|
||||
elif outcome == 'completed_with_warnings':
|
||||
sev.update(self._SEV_STYLE['WARNING'])
|
||||
status = 'completed_with_warnings'
|
||||
elif outcome == 'failed':
|
||||
sev.update(self._SEV_STYLE['CRITICAL'])
|
||||
status = 'failed'
|
||||
else:
|
||||
sev.update(self._SEV_DEFAULT)
|
||||
status = 'unconfirmed'
|
||||
sev['label'] = _runtime_text(f'email.status.{status}', data)
|
||||
group = data.get('_group', 'other')
|
||||
# Keep unbroken recorded text inside the temperature email's table.
|
||||
# Both properties are inline for mail clients; other events retain
|
||||
# their original markup and layout.
|
||||
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if event_type == 'temp_high' else ''
|
||||
temp_table_layout = 'table-layout:fixed;' if event_type == 'temp_high' else ''
|
||||
# Scoped inline mail-compatible wrapping for authoritative raw-context
|
||||
# bodies, including restore bodies released from quiet hours.
|
||||
backup_email = event_type in {'backup_complete', 'backup_fail'}
|
||||
wrap_body = (event_type in {'temp_high', 'system_restore_completed'}
|
||||
or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
|
||||
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
|
||||
temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
|
||||
# Keep wrapping event-scoped; unrelated mail remains byte-identical.
|
||||
context_email = backup_email
|
||||
backup_title_wrap = temp_cell_wrap if context_email else ''
|
||||
backup_metadata_layout = 'table-layout:fixed;' if context_email else ''
|
||||
section_label = _runtime_text(f'email.groups.{group}', data)
|
||||
report_label = _runtime_text('email.report', data, group=section_label)
|
||||
host_label = _runtime_text('email.host', data)
|
||||
@@ -1073,6 +1094,18 @@ class EmailChannel(NotificationChannel):
|
||||
('', html_mod.escape(line.strip()))
|
||||
for line in body.split('\n') if line.strip()
|
||||
)
|
||||
# A metadata-only/manual body may be generic. Keep actionable raw
|
||||
# context once, without restoring duplicated inventory metadata.
|
||||
reason = data.get('reason', '')
|
||||
if backup_email and reason and len(reason) <= 80 and reason not in body:
|
||||
detail_rows.append((html_mod.escape(_runtime_text('email.fields.reason', data)),
|
||||
html_mod.escape(reason)))
|
||||
|
||||
if event_type == 'system_restore_completed' or data.get('_restore_summary'):
|
||||
# The endpoint's warnings_block and task counts live in the
|
||||
# localized body, not the generic services Event row.
|
||||
detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
|
||||
for line in body.split('\n') if line.strip()]
|
||||
|
||||
# ── Fallback: if no structured rows, render body text lines ──
|
||||
if not detail_rows:
|
||||
@@ -1090,6 +1123,7 @@ class EmailChannel(NotificationChannel):
|
||||
|
||||
# ── Render detail rows as HTML table ──
|
||||
rows_html = ''
|
||||
summary_whitespace = 'white-space:pre-wrap;' if data.get('_quiet_hours_summary') and data.get('_restore_summary') else ''
|
||||
for label, value in detail_rows:
|
||||
if label:
|
||||
rows_html += f'''<tr>
|
||||
@@ -1099,13 +1133,14 @@ class EmailChannel(NotificationChannel):
|
||||
else:
|
||||
# Full-width row (no label, just description text)
|
||||
rows_html += f'''<tr>
|
||||
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}">{value}</td>
|
||||
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}{summary_whitespace}">{value}</td>
|
||||
</tr>'''
|
||||
|
||||
# ── Reason / details block (long text, displayed separately) ──
|
||||
reason = data.get('reason', '')
|
||||
reason_html = ''
|
||||
if reason and len(reason) > 80 and not (event_type == 'temp_high' and reason in body):
|
||||
if reason and len(reason) > 80 and not (
|
||||
(event_type == 'temp_high' or backup_email) and reason in body):
|
||||
reason_html = f'''
|
||||
<div style="margin:16px 0 0;padding:12px 16px;border:1px solid #d1d5db;border-radius:6px;">
|
||||
<p style="margin:0 0 4px;font-size:11px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:0.05em;">{_runtime_text('email.details', data)}</p>
|
||||
@@ -1140,15 +1175,15 @@ class EmailChannel(NotificationChannel):
|
||||
|
||||
<!-- Title bar -->
|
||||
<div style="padding:16px 28px;background:{sev['bg']};border-bottom:1px solid {sev['border']};">
|
||||
<h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};">{html_mod.escape(display_title)}</h2>
|
||||
<h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};{backup_title_wrap}">{html_mod.escape(display_title)}</h2>
|
||||
</div>
|
||||
|
||||
<!-- Body -->
|
||||
<div style="padding:24px 28px;">
|
||||
<!-- Metadata -->
|
||||
<table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;">
|
||||
<table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;{backup_metadata_layout}">
|
||||
<tr>
|
||||
<td style="font-size:12px;color:#4b5563;">
|
||||
<td style="font-size:12px;color:#4b5563;{backup_title_wrap}">
|
||||
{html_mod.escape(host_label)}: <strong style="color:#111827;">{html_mod.escape(data.get('hostname', ''))}</strong>
|
||||
</td>
|
||||
<td style="font-size:12px;color:#4b5563;text-align:right;">
|
||||
@@ -1221,7 +1256,9 @@ class EmailChannel(NotificationChannel):
|
||||
v = str(value).strip() if value else ''
|
||||
if not v or v == '0' and original_label not in ('Failures',):
|
||||
return
|
||||
if fmt == 'severity':
|
||||
if fmt == 'backup_error':
|
||||
rows.append((esc(label), f'<span style="color:#dc2626;font-weight:600;">{esc(v)}</span>'))
|
||||
elif fmt == 'severity':
|
||||
sev_colors = {
|
||||
'CRITICAL': '#dc2626', 'WARNING': '#d97706',
|
||||
'INFO': '#2563eb', 'OK': '#16a34a',
|
||||
@@ -1254,9 +1291,14 @@ class EmailChannel(NotificationChannel):
|
||||
# tell which target the backup ran against. Reported gap: emails
|
||||
# showed no way to distinguish which PBS failed with 2+ configured.
|
||||
_add('Storage', data.get('storage') or data.get('storage_name'), 'code')
|
||||
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
|
||||
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
|
||||
status_key = ('failed' if data.get('backup_outcome') == 'failed'
|
||||
else 'completed_with_warnings' if data.get('backup_outcome') == 'completed_with_warnings'
|
||||
else 'unconfirmed')
|
||||
else:
|
||||
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
|
||||
_add('Status', _runtime_text(f'email.status.{status_key}', language_data),
|
||||
'severity' if 'fail' in event_type else '')
|
||||
'backup_error' if status_key == 'failed' else '')
|
||||
_add('Size', data.get('size'))
|
||||
_add('Duration', data.get('duration'))
|
||||
_add('Snapshot', data.get('snapshot_name'), 'code')
|
||||
|
||||
@@ -4289,6 +4289,47 @@ class ProxmoxHookWatcher:
|
||||
def _hostname(self) -> str:
|
||||
return _hostname()
|
||||
|
||||
@staticmethod
|
||||
def _backup_outcome(severity: str, message: str) -> str:
|
||||
"""Distinguish explicit failure, complete guest logs and unknown results."""
|
||||
text = str(message or '')
|
||||
if severity in ('error', 'err', 'critical') or re.search(
|
||||
r'(?im)^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
|
||||
return 'failed'
|
||||
starts = re.findall(r'(?im)\bStarting Backup of VM (\d+)\s*\(', text)
|
||||
finished = re.findall(r'(?im)\bFinished Backup of VM (\d+)\s*\(', text)
|
||||
from notification_templates import _parse_vzdump_table
|
||||
table = _parse_vzdump_table(text)
|
||||
if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
|
||||
return 'failed'
|
||||
warnings = severity in ('warning', 'warn') or bool(re.search(
|
||||
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*[1-9]\d*)', text))
|
||||
if severity not in ('info', 'ok', 'success', 'warning', 'warn'):
|
||||
return 'unconfirmed'
|
||||
completed = 'completed_with_warnings' if warnings else 'confirmed'
|
||||
# A present table is authoritative: do not certify an incomplete table
|
||||
# from a finished guest log, or reject a complete OK table merely
|
||||
# because the extra diagnostic log was truncated before its finishes.
|
||||
if table is not None:
|
||||
return (completed if table['complete'] and
|
||||
all(guest['status'].lower() == 'ok' for guest in table['vms'])
|
||||
else 'unconfirmed')
|
||||
if starts:
|
||||
pending = {}
|
||||
for match in re.finditer(r'(?im)\b(Starting|Finished) Backup of VM (\d+)\s*\(', text):
|
||||
action, vmid = match.groups()
|
||||
if action.lower() == 'starting':
|
||||
pending[vmid] = pending.get(vmid, 0) + 1
|
||||
elif not pending.get(vmid):
|
||||
return 'unconfirmed' # A finish before its start is not evidence.
|
||||
else:
|
||||
pending[vmid] -= 1
|
||||
return completed if not any(pending.values()) else 'unconfirmed'
|
||||
if re.search(
|
||||
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
|
||||
return completed
|
||||
return 'unconfirmed'
|
||||
|
||||
def process_webhook(self, payload: dict) -> dict:
|
||||
"""Process an incoming Proxmox webhook payload.
|
||||
|
||||
@@ -4347,6 +4388,13 @@ class ProxmoxHookWatcher:
|
||||
'title': title or event_type,
|
||||
'job_id': pve_job_id,
|
||||
}
|
||||
if event_type in ('backup_complete', 'backup_fail'):
|
||||
# This is presentation metadata, not a new event/toggle/delivery path.
|
||||
outcome = self._backup_outcome(severity_raw, message)
|
||||
data['backup_outcome'] = (
|
||||
'failed' if event_type == 'backup_fail' or outcome == 'failed' else
|
||||
outcome if pve_type == 'vzdump' else 'unconfirmed'
|
||||
)
|
||||
|
||||
if pve_type == 'replication':
|
||||
replication = self._extract_replication_context(
|
||||
@@ -4440,10 +4488,17 @@ class ProxmoxHookWatcher:
|
||||
if vmids:
|
||||
data['vmid'] = vmids[0]
|
||||
entity_id = vmids[0]
|
||||
# Try to extract VM name from the table line
|
||||
name_m = re.search(r'(\d+)\s+(\S+)\s+(?:OK|ERROR|WARNINGS)', message)
|
||||
if name_m:
|
||||
data['vmname'] = name_m.group(2)
|
||||
from notification_templates import _parse_vzdump_message
|
||||
parsed = _parse_vzdump_message(message) or {}
|
||||
guests = parsed.get('vms', [])
|
||||
if data.get('backup_outcome') == 'failed':
|
||||
guests = [guest for guest in guests if guest.get('status', '').lower() == 'error']
|
||||
if len(guests) == 1:
|
||||
data['vmid'] = guests[0]['vmid']
|
||||
data['vmname'] = guests[0]['name']
|
||||
else:
|
||||
# Do not make one successful guest the subject of a batch failure.
|
||||
data.pop('vmid', None)
|
||||
# Extract size from "Total size: X"
|
||||
size_m = re.search(r'Total size:\s*(.+?)(?:\n|$)', message)
|
||||
if size_m:
|
||||
|
||||
@@ -1364,6 +1364,12 @@ class NotificationManager:
|
||||
|
||||
# Get journal context if available (will be enriched per-channel based on detail_level)
|
||||
raw_journal_context = data.get('_journal_context', '')
|
||||
# Persist a presentation token in the existing title column: buffers
|
||||
# otherwise discard outcome metadata before composition. Old rows
|
||||
# without a token remain neutral; routing and schema are unchanged.
|
||||
buffer_title = title
|
||||
if event_type in ('backup_complete', 'backup_fail'):
|
||||
buffer_title, _ = enrich_with_emojis(event_type, title, '', data)
|
||||
|
||||
for ch_name, channel in channels.items():
|
||||
# ── Per-channel category check ──
|
||||
@@ -1393,7 +1399,7 @@ class NotificationManager:
|
||||
# delivered after Quiet Hours + Daily Digest were merged.
|
||||
if severity != 'CRITICAL' and self._in_quiet_hours(ch_name):
|
||||
self._buffer_quiet_event(ch_name, event_type, event_group,
|
||||
severity, title, body)
|
||||
severity, buffer_title, body)
|
||||
continue
|
||||
|
||||
# ── Per-channel daily digest ──
|
||||
@@ -1406,7 +1412,7 @@ class NotificationManager:
|
||||
# excluded from the digest by `_DIGEST_EXEMPT_EVENTS`.
|
||||
if self._should_buffer_for_digest(ch_name, severity, event_type):
|
||||
self._buffer_digest_event(ch_name, event_type, event_group,
|
||||
severity, title, body)
|
||||
severity, buffer_title, body)
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -1821,7 +1827,8 @@ class NotificationManager:
|
||||
print(f"[NotificationManager] digest cleanup failed for "
|
||||
f"{ch_name}: {e}")
|
||||
|
||||
def _compose_digest_body(self, rows: list, use_icons: bool = False) -> str:
|
||||
def _compose_digest_body(self, rows: list, use_icons: bool = False,
|
||||
quiet_release: bool = False) -> str:
|
||||
"""Render a grouped summary body. rows is a list of
|
||||
(id, event_type, event_group, ts, title, body) tuples ordered
|
||||
by timestamp ASC.
|
||||
@@ -1830,27 +1837,47 @@ class NotificationManager:
|
||||
groups: OrderedDict[str, list] = OrderedDict()
|
||||
for _id, ev_type, group, ts, title, body in rows:
|
||||
label = group or 'other'
|
||||
groups.setdefault(label, []).append((ts, ev_type, title))
|
||||
groups.setdefault(label, []).append((ts, ev_type, title, body))
|
||||
|
||||
language = self._notification_language()
|
||||
lines = [runtime_message('digest.lead', language, count=len(rows))]
|
||||
# The quiet summary title already carries the total; the daily lead
|
||||
# incorrectly calls every buffered WARNING an INFO event.
|
||||
lines = [] if quiet_release else [runtime_message('digest.lead', language, count=len(rows))]
|
||||
for group, items in groups.items():
|
||||
group_label = runtime_message(f'digest.groups.{group}', language) or group.title()
|
||||
group_icon = CATEGORY_EMOJI.get(group, '') if use_icons else ''
|
||||
group_prefix = f'{group_icon} ' if group_icon else ''
|
||||
lines.append(f"{group_prefix}{group_label}: {len(items)}")
|
||||
for ts, ev_type, title in items[:8]:
|
||||
# Quiet hours can buffer restore warnings, unlike the daily INFO
|
||||
# digest. Keep their complete recorded body, even past the usual
|
||||
# title preview limit, without changing either delivery policy.
|
||||
visible_items = [item for index, item in enumerate(items)
|
||||
if index < 8 or (quiet_release and item[1] == 'system_restore_completed')]
|
||||
for ts, ev_type, title, body in visible_items:
|
||||
hhmm = datetime.fromtimestamp(ts).strftime('%H:%M')
|
||||
backup_icon = ''
|
||||
if ev_type in ('backup_complete', 'backup_fail'):
|
||||
for token in ('💾✅', '💾⚠️', '💾❌', '💾❔', '💾'):
|
||||
if title.startswith(token + ' '):
|
||||
backup_icon = token
|
||||
title = title[len(token) + 1:]
|
||||
break
|
||||
backup_icon = backup_icon or ('💾❌' if ev_type == 'backup_fail' else '💾❔')
|
||||
short_title = title.split(': ', 1)[-1] if ': ' in title else title
|
||||
event_icon = (
|
||||
EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
|
||||
backup_icon or EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
|
||||
) if use_icons else ''
|
||||
event_prefix = f'{event_icon} ' if event_icon else ''
|
||||
lines.append(f" • {event_prefix}{hhmm} {short_title}")
|
||||
if len(items) > 8:
|
||||
lines.append(runtime_message('digest.more', language, count=len(items) - 8))
|
||||
if quiet_release and ev_type == 'system_restore_completed' and body:
|
||||
lines.extend(' ' + line.strip() for line in body.splitlines() if line.strip())
|
||||
if len(items) > len(visible_items):
|
||||
lines.append(runtime_message('digest.more', language, count=len(items) - len(visible_items)))
|
||||
lines.append('')
|
||||
lines.append(runtime_message('digest.footer', language))
|
||||
# The daily footer describes live warning delivery, which is not true
|
||||
# for warnings buffered during quiet hours. Do not repeat that claim.
|
||||
if not quiet_release:
|
||||
lines.append(runtime_message('digest.footer', language))
|
||||
return '\n'.join(lines).rstrip() + '\n'
|
||||
|
||||
# ─── Quiet Hours buffer + flush ────────────────────────────
|
||||
@@ -1985,13 +2012,17 @@ class NotificationManager:
|
||||
'digest.quietTitle', language, hostname=host, count=len(rows),
|
||||
)
|
||||
use_icons = self._config.get(f'{ch_name}.rich_format', 'false') == 'true'
|
||||
summary_body = self._compose_digest_body(rows, use_icons=use_icons)
|
||||
quiet_details = any(row[1] in ('backup_complete', 'backup_fail', 'system_restore_completed')
|
||||
for row in rows)
|
||||
summary_body = self._compose_digest_body(rows, use_icons=use_icons, quiet_release=quiet_details)
|
||||
|
||||
result: dict = {'success': False, 'error': ''}
|
||||
try:
|
||||
result = channel.send(
|
||||
summary_title, summary_body, severity='INFO',
|
||||
data={'_quiet_hours_summary': True, '_count': len(rows),
|
||||
'_restore_summary': any(row[1] == 'system_restore_completed' for row in rows),
|
||||
'_backup_summary': any(row[1] in ('backup_complete', 'backup_fail') for row in rows),
|
||||
'_notification_language': language},
|
||||
) or result
|
||||
except Exception as e:
|
||||
@@ -2467,6 +2498,11 @@ class NotificationManager:
|
||||
runtime_data.get('hostname'), self._config,
|
||||
)
|
||||
runtime_data.setdefault('_notification_language', self._notification_language())
|
||||
# Match queued dispatch's presentation context for these outcome
|
||||
# notices; this does not alter event/severity or direct-send policy.
|
||||
if event_type in ('backup_complete', 'backup_fail', 'system_restore_completed'):
|
||||
runtime_data['_event_type'] = event_type
|
||||
runtime_data['_group'] = TEMPLATES[event_type].get('group', 'other')
|
||||
|
||||
# Render template if available
|
||||
if event_type in TEMPLATES and not message:
|
||||
|
||||
@@ -206,6 +206,58 @@ def _format_lxc_update_details(data: Dict[str, Any], language: str) -> str:
|
||||
|
||||
# ─── vzdump message parser ───────────────────────────────────────
|
||||
|
||||
def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
|
||||
"""Read the bounded fixed-column summary for both outcomes and guest details."""
|
||||
lines = message.splitlines()
|
||||
for index, header in enumerate(lines):
|
||||
if not re.match(r'\s*VMID\s+Name\s+Status\b', header, re.IGNORECASE):
|
||||
continue
|
||||
columns = [re.search(r'\b' + name + r'\b', header, re.IGNORECASE)
|
||||
for name in ('VMID', 'Name', 'Status', 'Time', 'Size', 'Filename')]
|
||||
if not all(columns):
|
||||
return {'vms': [], 'complete': False}
|
||||
starts = [column.start() for column in columns if column is not None]
|
||||
if starts != sorted(starts):
|
||||
return {'vms': [], 'complete': False}
|
||||
rows = []
|
||||
valid = True
|
||||
complete = False
|
||||
for line in lines[index + 1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
if re.match(r'\s*Total running time:\s*\S', line, re.IGNORECASE):
|
||||
complete = valid and bool(rows)
|
||||
break
|
||||
# Blanks are allowed, but no unrelated section can extend the table.
|
||||
if not re.match(r'\s*\d+\s+', line):
|
||||
break
|
||||
values = [line[a:b].strip() for a, b in
|
||||
zip(starts, starts[1:] + [len(line)])]
|
||||
vmid, name, status, duration, size, filename = values
|
||||
if not vmid.isdigit():
|
||||
valid = False
|
||||
break
|
||||
valid = bool(valid and all(values)
|
||||
and re.fullmatch(r'(?:\d+:\d{2}:\d{2}|(?:\d+[yMwdhms]\s*)+)', duration)
|
||||
and re.fullmatch(r'\d+(?:\.\d+)?\s*(?:[KMGTPE]i?B?|B)', size, re.IGNORECASE))
|
||||
if status.lower() in ('err', 'error'):
|
||||
status = 'error'
|
||||
kind = ('lxc' if 'lxc' in filename or filename.startswith('ct/') else
|
||||
'qemu' if 'qemu' in filename or filename.startswith('vm/') else '')
|
||||
if filename.lower() == 'null':
|
||||
# Native failed rows lack archives. Match only this row's VMID;
|
||||
# multiple inconsistent starts are not authoritative identity.
|
||||
kinds = set(re.findall(
|
||||
r'(?im)\bStarting Backup of VM ' + re.escape(vmid) + r'\s+\((lxc|qemu)\)',
|
||||
message))
|
||||
kind = kinds.pop() if len(kinds) == 1 else ''
|
||||
rows.append({'vmid': vmid, 'name': name, 'status': status,
|
||||
'time': duration, 'size': size, 'filename': filename,
|
||||
'type': kind})
|
||||
return {'vms': rows, 'complete': bool(complete)}
|
||||
return None
|
||||
|
||||
|
||||
def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
"""Parse a PVE vzdump notification message into structured data.
|
||||
|
||||
@@ -225,53 +277,10 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
|
||||
lines = message.split('\n')
|
||||
|
||||
# ── Strategy 1: classic table (local/NFS/CIFS storage) ──
|
||||
header_idx = -1
|
||||
for i, line in enumerate(lines):
|
||||
if re.match(r'\s*VMID\s+Name\s+Status', line, re.IGNORECASE):
|
||||
header_idx = i
|
||||
break
|
||||
|
||||
if header_idx >= 0:
|
||||
# Use column positions from the header to slice each row.
|
||||
# Header: "VMID Name Status Time Size Filename"
|
||||
header = lines[header_idx]
|
||||
col_starts = []
|
||||
for col_name in ['VMID', 'Name', 'Status', 'Time', 'Size', 'Filename']:
|
||||
idx = header.find(col_name)
|
||||
if idx >= 0:
|
||||
col_starts.append(idx)
|
||||
|
||||
if len(col_starts) == 6:
|
||||
for line in lines[header_idx + 1:]:
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith('Total') or stripped.startswith('Logs') or stripped.startswith('='):
|
||||
break
|
||||
# Pad line to avoid index errors
|
||||
padded = line.ljust(col_starts[-1] + 50)
|
||||
vmid = padded[col_starts[0]:col_starts[1]].strip()
|
||||
name = padded[col_starts[1]:col_starts[2]].strip()
|
||||
status = padded[col_starts[2]:col_starts[3]].strip()
|
||||
time_val = padded[col_starts[3]:col_starts[4]].strip()
|
||||
size = padded[col_starts[4]:col_starts[5]].strip()
|
||||
filename = padded[col_starts[5]:].strip()
|
||||
|
||||
if vmid and vmid.isdigit():
|
||||
# Infer type from filename (vzdump-lxc-NNN or vzdump-qemu-NNN)
|
||||
vm_type = ''
|
||||
if 'lxc' in filename:
|
||||
vm_type = 'lxc'
|
||||
elif 'qemu' in filename:
|
||||
vm_type = 'qemu'
|
||||
vms.append({
|
||||
'vmid': vmid,
|
||||
'name': name,
|
||||
'status': status,
|
||||
'time': time_val,
|
||||
'size': size,
|
||||
'filename': filename,
|
||||
'type': vm_type,
|
||||
})
|
||||
# The same summary rows drive classification, rich bodies and identities.
|
||||
table = _parse_vzdump_table(message)
|
||||
if table is not None:
|
||||
vms = table['vms']
|
||||
|
||||
# ── Strategy 2: log-style (PBS / Proxmox Backup Server) ──
|
||||
# Parse from the full vzdump log lines.
|
||||
@@ -299,7 +308,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
current_vm = {
|
||||
'vmid': m_start.group(1),
|
||||
'name': '',
|
||||
'status': 'ok',
|
||||
'status': 'unknown',
|
||||
'time': '',
|
||||
'size': '',
|
||||
'filename': '',
|
||||
@@ -307,6 +316,18 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
}
|
||||
continue
|
||||
|
||||
# A finish can belong to a guest already stored when another
|
||||
# start arrived. Preserve that guest's actual completion too.
|
||||
prior_finish = re.match(r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
|
||||
if prior_finish:
|
||||
prior = next((vm for vm in reversed(vms)
|
||||
if vm['vmid'] == prior_finish.group(1)), None)
|
||||
if prior is not None:
|
||||
prior['time'] = prior_finish.group(2)
|
||||
if prior['status'] != 'error':
|
||||
prior['status'] = 'ok'
|
||||
continue
|
||||
|
||||
if current_vm:
|
||||
# Guest name
|
||||
m_name = re.match(r'(?:CT|VM) Name:\s*(.+)', clean)
|
||||
@@ -338,15 +359,16 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
# Finished -> duration
|
||||
m_finish = re.match(
|
||||
r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
|
||||
if m_finish:
|
||||
if m_finish and m_finish.group(1) == current_vm['vmid']:
|
||||
current_vm['time'] = m_finish.group(2)
|
||||
current_vm['status'] = 'ok'
|
||||
if current_vm['status'] != 'error':
|
||||
current_vm['status'] = 'ok'
|
||||
vms.append(current_vm)
|
||||
current_vm = None
|
||||
continue
|
||||
|
||||
# Error
|
||||
if clean.startswith('ERROR:') or clean.startswith('TASK ERROR'):
|
||||
if re.match(r'^\s*(?:ERROR:|TASK ERROR)', line, re.IGNORECASE):
|
||||
if current_vm:
|
||||
current_vm['status'] = 'error'
|
||||
|
||||
@@ -354,6 +376,21 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
if current_vm:
|
||||
vms.append(current_vm)
|
||||
|
||||
# Explicit guest-linked failures outrank a contradictory summary OK row.
|
||||
# Job-level/prune errors do not invalidate unrelated successfully saved guests.
|
||||
for line in lines:
|
||||
error = re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:)\s*(.*)',
|
||||
line, re.IGNORECASE)
|
||||
if not error:
|
||||
continue
|
||||
failed_guest = re.search(r'\bBackup of (?:VM|CT) (\d+) failed\b|\bbackup failed for (?:VM|CT) (\d+)\b',
|
||||
error.group(1), re.IGNORECASE)
|
||||
if failed_guest:
|
||||
vmid = failed_guest.group(1) or failed_guest.group(2)
|
||||
for vm in vms:
|
||||
if vm['vmid'] == vmid:
|
||||
vm['status'] = 'error'
|
||||
|
||||
# ── Extract totals ──
|
||||
for line in lines:
|
||||
m_time = re.search(r'Total running time:\s*(.+)', line)
|
||||
@@ -369,7 +406,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
|
||||
sizes_gib = 0.0
|
||||
for vm in vms:
|
||||
s = vm.get('size', '')
|
||||
m = re.match(r'([\d.]+)\s+(.*)', s)
|
||||
m = re.fullmatch(r'(\d+(?:\.\d+)?)\s+([KMGTPE]i?B|B)', s, re.IGNORECASE)
|
||||
if m:
|
||||
val = float(m.group(1))
|
||||
unit = m.group(2).strip().upper()
|
||||
@@ -439,7 +476,7 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
|
||||
|
||||
for vm in parsed.get('vms', []):
|
||||
status = vm.get('status', '').lower()
|
||||
icon = '\u2705' if status == 'ok' else '\u274C'
|
||||
icon = '\u2705' if status == 'ok' else '\u274C' if status == 'error' else '\u2754'
|
||||
|
||||
# Determine VM/CT type prefix
|
||||
vm_type = vm.get('type', '')
|
||||
@@ -501,7 +538,8 @@ def _format_vzdump_body(parsed: Dict[str, Any], is_success: bool,
|
||||
if vm_count > 0 or parsed.get('total_size'):
|
||||
ok_count = sum(1 for v in parsed.get('vms', [])
|
||||
if v.get('status', '').lower() == 'ok')
|
||||
fail_count = vm_count - ok_count
|
||||
fail_count = sum(1 for v in parsed.get('vms', [])
|
||||
if v.get('status', '').lower() == 'error')
|
||||
|
||||
summary_parts = []
|
||||
if vm_count:
|
||||
@@ -1855,6 +1893,50 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
)
|
||||
if localized:
|
||||
template[field] = localized
|
||||
backup_title_target = ''
|
||||
if event_type == 'backup_complete':
|
||||
template['title'] = runtime_message('backup.unconfirmedTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname()) or (
|
||||
str(data.get('hostname') or _get_hostname()) + ': Backup outcome unconfirmed')
|
||||
template['body'] = runtime_message('backup.unconfirmedBody', language) or 'The backup outcome is not confirmed.'
|
||||
outcome = data.get('backup_outcome')
|
||||
if outcome == 'confirmed':
|
||||
template['title'] = runtime_message('backup.confirmedTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
template['body'] = runtime_message('backup.confirmedBody', language)
|
||||
elif outcome == 'completed_with_warnings':
|
||||
template['title'] = runtime_message('backup.warningTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
template['body'] = runtime_message('backup.warningBody', language)
|
||||
elif outcome == 'failed':
|
||||
template['title'] = runtime_message('backup.errorTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
template['body'] = runtime_message('backup.errorBody', language)
|
||||
if event_type == 'backup_fail':
|
||||
template['title'] = runtime_message('backup.errorTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
|
||||
parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
|
||||
storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
|
||||
guests = (parsed_backup or {}).get('vms') or []
|
||||
# Explicit confirmed manual metadata is useful context, not evidence
|
||||
# about an unparsed batch. Only use it when there is no raw report.
|
||||
if not data.get('pve_message') and data.get('backup_outcome') == 'confirmed' and data.get('vmid'):
|
||||
guests = [{'vmid': str(data['vmid']), 'name': str(data.get('vmname') or ''),
|
||||
'type': str(data.get('vm_type') or ''), 'status': 'ok'}]
|
||||
if event_type == 'backup_fail' or data.get('backup_outcome') == 'failed':
|
||||
guests = [guest for guest in guests if guest.get('status', '').lower() == 'error']
|
||||
target = []
|
||||
if storage:
|
||||
target.append(storage)
|
||||
if len(guests) == 1:
|
||||
guest = guests[0]
|
||||
kind = 'VM' if guest.get('type') == 'qemu' else 'CT' if guest.get('type') == 'lxc' else 'VM/CT'
|
||||
name = guest.get('name') or kind
|
||||
target.append(f"{kind} {name} ({guest['vmid']})" if name != kind
|
||||
else f"{kind} {guest['vmid']}")
|
||||
if target:
|
||||
backup_title_target = ' — ' + ' · '.join(target)
|
||||
|
||||
# Ensure hostname is always available
|
||||
variables = {
|
||||
@@ -1884,6 +1966,11 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
'log_file': '',
|
||||
}
|
||||
variables.update(data)
|
||||
if event_type in ('backup_fail', 'backup_complete'):
|
||||
# The provider has already substituted raw Display Names. Insert the
|
||||
# resolved title as a value, never reinterpret its literal braces.
|
||||
variables['_backup_title'] = template['title']
|
||||
template['title'] = '{_backup_title}'
|
||||
|
||||
# Old persisted errors and manual events may lack a complete reading.
|
||||
# Accept plain numeric strings, but never interpret booleans or objects as
|
||||
@@ -1992,12 +2079,37 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
title = template['title'].format_map(safe_vars)
|
||||
except (ValueError, IndexError):
|
||||
title = template['title']
|
||||
title += backup_title_target
|
||||
|
||||
# ── PVE vzdump special formatting ──
|
||||
# When the event came from PVE webhook with a full vzdump message,
|
||||
# parse the table/logs and format a rich body instead of the sparse template.
|
||||
pve_message = data.get('pve_message', '')
|
||||
pve_title = data.get('pve_title', '')
|
||||
backup_diagnostics = []
|
||||
principal_cause = None
|
||||
|
||||
def bounded_backup_diagnostics(lines, principal_cause=None):
|
||||
# 1024 chars matches the repository's small-channel message convention;
|
||||
# 8 lines keeps repeated producer warnings readable. Inventory/title
|
||||
# size is separate: this is not a one-Telegram-message guarantee.
|
||||
unique = list(dict.fromkeys(line for line in lines if line.strip()))
|
||||
principal = principal_cause or next((line for line in unique if re.search(r'\b(?:ERROR:|TASK ERROR:)', line, re.IGNORECASE)), None)
|
||||
if principal:
|
||||
unique.remove(principal)
|
||||
unique.insert(0, principal)
|
||||
shown, budget, omitted = [], 1024, 0
|
||||
for line in unique:
|
||||
if len(shown) >= 8 or budget < 2:
|
||||
omitted += 1
|
||||
continue
|
||||
line_budget = min(budget, 512)
|
||||
rendered = line if len(line) <= line_budget else line[:line_budget - 1] + '…'
|
||||
omitted += int(rendered != line)
|
||||
shown.append(rendered)
|
||||
budget -= len(rendered) + 1
|
||||
if omitted:
|
||||
shown.append(runtime_message('backup.diagnosticsOmitted', language, count=omitted))
|
||||
return '\n'.join(shown)
|
||||
|
||||
# Check for custom formatter function
|
||||
formatter_name = template.get('formatter')
|
||||
@@ -2016,13 +2128,29 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
if parsed:
|
||||
is_success = (event_type == 'backup_complete')
|
||||
body_text = _format_vzdump_body(parsed, is_success, language=language)
|
||||
# Preserve PVE's source title for English, but never leak it into a
|
||||
# deterministic localized notification.
|
||||
if pve_title and requested_language == 'en':
|
||||
title = pve_title
|
||||
diagnostic_lines = [line.strip() for line in pve_message.splitlines()
|
||||
if re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:WARN(?:ING)?:|ERROR:|TASK ERROR)',
|
||||
line, re.IGNORECASE)]
|
||||
if event_type == 'backup_fail' or data.get('backup_outcome') == 'failed':
|
||||
# Native send_notification puts multiline job/setup errors
|
||||
# before Details, while its subject says only "multiple problems".
|
||||
# Keep that raw block when inventory replaces the producer body;
|
||||
# it is job context, not evidence that every guest failed.
|
||||
error_block = re.match(r'\A(.*?)^Details\r?\n=+\s*$',
|
||||
pve_message, re.MULTILINE | re.DOTALL)
|
||||
if error_block:
|
||||
diagnostic_lines = error_block.group(1).rstrip('\r\n').splitlines() + diagnostic_lines
|
||||
backup_diagnostics = [line for line in diagnostic_lines
|
||||
if line.strip() and line not in body_text.splitlines()]
|
||||
else:
|
||||
# Couldn't parse -- use PVE raw message as body
|
||||
body_text = pve_message.strip()
|
||||
# Unparsed diagnostic-only reports remain visible but bounded.
|
||||
body_text = ''
|
||||
backup_diagnostics = pve_message.strip().splitlines()
|
||||
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
|
||||
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
|
||||
else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
|
||||
else 'backup.unconfirmedBody')
|
||||
body_text = (runtime_message(key, language) or template['body']) + '\n' + body_text
|
||||
elif event_type == 'system_mail' and pve_message:
|
||||
# System mail -- use PVE message directly (mail bounce, cron, smartd)
|
||||
body_text = pve_message.strip()[:1000]
|
||||
@@ -2032,6 +2160,51 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
except (ValueError, IndexError):
|
||||
body_text = template['body']
|
||||
|
||||
if event_type == 'backup_complete' and data.get('backup_outcome') == 'confirmed' and not pve_message:
|
||||
context = []
|
||||
if data.get('vmid'):
|
||||
name = str(data.get('vmname') or '')
|
||||
context.append(f"{name} ({data['vmid']})" if name else str(data['vmid']))
|
||||
if data.get('storage'):
|
||||
context.append(str(data['storage']))
|
||||
if data.get('size'):
|
||||
context.append(runtime_message('vzdump.size', language, value=data['size']))
|
||||
if data.get('duration'):
|
||||
context.append(runtime_message('vzdump.duration', language, value=data['duration']))
|
||||
if context:
|
||||
body_text += '\n' + '\n'.join(context)
|
||||
|
||||
# PVE can move a one-line setup/abort reason exclusively into its subject.
|
||||
# Preserve that raw failure context, without using it as a localized title.
|
||||
if event_type in ('backup_complete', 'backup_fail') and (
|
||||
event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
|
||||
source_subject = str(data.get('pve_title') or '').strip()
|
||||
native_failure = re.fullmatch(
|
||||
r'vzdump backup status \([^\r\n]*\): backup failed(?::\s*(.*))?',
|
||||
source_subject, re.IGNORECASE)
|
||||
guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
|
||||
if native_failure:
|
||||
# Before the first guest, too, only the native cause is diagnostic;
|
||||
# the original host/job envelope is not display-name context.
|
||||
source_subject = (native_failure.group(1) or '').strip()
|
||||
elif guest_context:
|
||||
cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
|
||||
source_subject = cause.group(1).strip() if cause else ''
|
||||
if source_subject.lower() == 'multiple problems':
|
||||
source_subject = ''
|
||||
if source_subject and source_subject not in {line.strip() for line in body_text.splitlines()}:
|
||||
# Reserve the subject-equivalent diagnostic BEFORE the cap. Finding
|
||||
# it in uncapped logs is not enough: that late line could be omitted.
|
||||
principal_cause = next((line for line in backup_diagnostics
|
||||
if line.strip() == source_subject or
|
||||
re.split(r'\b(?:TASK ERROR:|ERROR:)\s*', line, maxsplit=1, flags=re.IGNORECASE)[-1].strip() == source_subject), None)
|
||||
if not principal_cause:
|
||||
principal_cause = source_subject
|
||||
backup_diagnostics.insert(0, source_subject)
|
||||
|
||||
if backup_diagnostics:
|
||||
body_text += '\n' + bounded_backup_diagnostics(backup_diagnostics, principal_cause)
|
||||
|
||||
# Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
|
||||
import re as _re
|
||||
body_text = _re.sub(r'\n{3,}', '\n\n', body_text.strip())
|
||||
@@ -2166,7 +2339,7 @@ EVENT_EMOJI = {
|
||||
'host_backup_start': '\U0001F5C4️\U0001F680', # 🗄️🚀 cabinet + rocket
|
||||
'host_backup_complete': '\U0001F5C4️✅', # 🗄️✅ cabinet + check
|
||||
'host_backup_fail': '\U0001F5C4️❌', # 🗄️❌ cabinet + cross
|
||||
'backup_complete': '\U0001F4BE\u2705', # 💾✅ floppy + check
|
||||
'backup_complete': '\U0001F4BE', # 💾 neutral for digests without outcome metadata
|
||||
'backup_warning': '\U0001F4BE\u26A0\uFE0F', # 💾⚠️ floppy + warning
|
||||
'backup_fail': '\U0001F4BE\u274C', # 💾❌ floppy + cross
|
||||
'snapshot_complete': '\U0001F4F8', # camera with flash
|
||||
@@ -2363,6 +2536,10 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
|
||||
severity = data.get('severity', 'INFO')
|
||||
|
||||
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
|
||||
if event_type == 'backup_complete':
|
||||
icon = {
|
||||
'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
|
||||
}.get(str(data.get('backup_outcome') or ''), '💾❔')
|
||||
|
||||
# Build enriched title: replace severity circle with event-specific icon
|
||||
# Current format: "hostname: Something" -> "ICON hostname: Something"
|
||||
|
||||
@@ -68,10 +68,23 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
return result
|
||||
|
||||
en = flatten(self.catalogs["en"])
|
||||
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
|
||||
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
|
||||
"backup.unconfirmedTitle", "channels.email.status.unconfirmed",
|
||||
"backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
|
||||
"channels.email.status.completed_with_warnings"}
|
||||
for language, catalog in self.catalogs.items():
|
||||
translated = flatten(catalog)
|
||||
self.assertEqual(set(translated), set(en), language)
|
||||
for key in en:
|
||||
if language == 'sk':
|
||||
# Missing maintainer-owned leaves may be generated later.
|
||||
# Accept only this bounded gap, and validate every present leaf.
|
||||
self.assertTrue(set(en) - pending_slovak <= set(translated), language)
|
||||
self.assertTrue(set(translated) <= set(en), language)
|
||||
else:
|
||||
self.assertEqual(set(translated), set(en), language)
|
||||
for key in translated:
|
||||
self.assertIsInstance(translated[key], str, f"{language}:{key}")
|
||||
self.assertTrue(translated[key].strip(), f"{language}:{key}")
|
||||
self.assertEqual(_placeholders(translated[key]), _placeholders(en[key]), f"{language}:{key}")
|
||||
|
||||
def test_notification_language_ui_keys_exist_in_both_catalogs(self):
|
||||
@@ -111,6 +124,11 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
with mock.patch.object(notification_templates, "_get_hostname", return_value="HOST-ŽILINA"):
|
||||
for event_type in notification_templates.TEMPLATES:
|
||||
event_values = dict(values)
|
||||
if event_type == "backup_complete":
|
||||
# Legacy catalog fields keep their completion meaning;
|
||||
# the new unknown-outcome keys intentionally omit guest data.
|
||||
# Exercise retained metadata on the explicit confirmed path.
|
||||
event_values["backup_outcome"] = "confirmed"
|
||||
if event_type == "temp_high":
|
||||
# Temperature is a measured numeric contract; arbitrary
|
||||
# DYNAMIC_VALUE is correctly rejected by its fallback.
|
||||
@@ -312,6 +330,7 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
"backup_complete",
|
||||
{
|
||||
"hostname": "pve01", "storage": "pbs-main", "vmname": "alpha", "vmid": "100",
|
||||
"backup_outcome": "confirmed",
|
||||
"pve_title": "Backup job finished",
|
||||
"pve_message": (
|
||||
"INFO: Starting Backup of VM 100 (qemu)\n"
|
||||
@@ -322,7 +341,12 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
},
|
||||
language="sk",
|
||||
)
|
||||
self.assertIn("Záloha dokončená", backup["title"])
|
||||
expected_title = notification_templates.runtime_message(
|
||||
"backup.confirmedTitle", "sk", hostname="pve01",
|
||||
)
|
||||
self.assertTrue(backup["title"].startswith(expected_title + " — "))
|
||||
self.assertIn("pbs-main", backup["title"])
|
||||
self.assertIn("VM alpha (100)", backup["title"])
|
||||
self.assertNotIn("Backup job finished", backup["title"])
|
||||
self.assertIn("Veľkosť: 1.5 GiB", backup["body"])
|
||||
self.assertIn("Trvanie: 00:00:10", backup["body"])
|
||||
@@ -627,6 +651,7 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
"_notification_language": "sk", "_event_type": event_type,
|
||||
"_group": "backup", "hostname": "pve01", "vmid": "100",
|
||||
"vmname": "alpha", "storage": "pbs-main",
|
||||
"backup_outcome": "confirmed" if event_type == "backup_complete" else "unconfirmed",
|
||||
},
|
||||
)
|
||||
self.assertIn(f">{localized_status}<", backup_html)
|
||||
|
||||
@@ -275,8 +275,14 @@ class VzdumpAIIntegrityTests(unittest.TestCase):
|
||||
rendered["body"], "CRITICAL", data,
|
||||
)
|
||||
|
||||
# The failure title now identifies the unique failed guest; inventory
|
||||
# remains exactly once in the detail table, not suppressed from body.
|
||||
inventory = html.split('<!-- Detail table -->', 1)[1].split('</table>', 1)[0]
|
||||
for vmid in range(100, 149):
|
||||
self.assertEqual(html.count(f"guest-{vmid} ({vmid})"), 1, vmid)
|
||||
self.assertEqual(inventory.count(f"guest-{vmid} ({vmid})"), 1, vmid)
|
||||
self.assertIn('guest-148 (148)', rendered['title'])
|
||||
self.assertNotIn('guest-100 (100)', rendered['title'])
|
||||
self.assertEqual(html.count('guest-148 (148)'), 2) # title + inventory
|
||||
self.assertEqual(html.count("49 backups"), 1)
|
||||
self.assertEqual(html.count("1 failed"), 1)
|
||||
self.assertEqual(html.count(">Zlyhalo<"), 1)
|
||||
|
||||
@@ -60,7 +60,7 @@ def _make_long_vzdump_report():
|
||||
|
||||
|
||||
class VzdumpWebhookTruncationTests(unittest.TestCase):
|
||||
def test_truncating_vzdump_report_at_4096_can_create_false_failed_backup(self):
|
||||
def test_truncating_vzdump_report_at_4096_leaves_guest_unconfirmed(self):
|
||||
full_message = _make_long_vzdump_report()
|
||||
truncated_message = full_message[:4096]
|
||||
|
||||
@@ -82,8 +82,8 @@ class VzdumpWebhookTruncationTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(truncated_dockflare["name"], "dockflare")
|
||||
self.assertEqual(truncated_dockflare["status"], "")
|
||||
self.assertIn("❌ dockflare (129)", truncated_body)
|
||||
self.assertIn("❌ 1 failed", truncated_body)
|
||||
self.assertIn("❔ dockflare (129)", truncated_body)
|
||||
self.assertNotIn("❌ 1 failed", truncated_body)
|
||||
|
||||
def test_webhook_handler_does_not_truncate_message_before_parsing(self):
|
||||
source = (SCRIPTS_DIR / "flask_notification_routes.py").read_text()
|
||||
|
||||
Reference in New Issue
Block a user