fix(notifications): preserve evidenced outcomes in backup and health notices

This commit is contained in:
martino
2026-10-01 08:38:32 +02:00
parent 147c401864
commit 08598d2863
20 changed files with 497 additions and 51 deletions
@@ -142,6 +142,10 @@ class CommandDescriptionsTests(unittest.TestCase):
'observation', source['channels']['email']['severity']['observation'])
local['channels']['email']['status'].setdefault(
'unconfirmed', source['channels']['email']['status']['unconfirmed'])
local['channels']['email']['status'].setdefault(
'completed_with_warnings', source['channels']['email']['status']['completed_with_warnings'])
for key, value in source['healthRecovery'].items():
local.setdefault('healthRecovery', {}).setdefault(key, value)
path.write_text(json.dumps(temporary, ensure_ascii=False))
# Model steady state after the bot fills these intentional new
# messages; keep repository locales and all other leaves intact.
@@ -217,10 +217,10 @@ class CorrectionTests(unittest.TestCase):
self.assertIn('✅ VM web (100)', result['body'])
def test_official_prefixed_warning_is_uncertain_and_retained(self):
def test_official_completed_report_warning_is_distinct_and_retained(self):
warning = '100: 2026-09-29 17:00:00 WARN: unable to add notes - permission denied'
event = receive(REPORT + '\n' + warning)
self.assertEqual(event.data['backup_outcome'], 'unconfirmed')
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
for language in LANGUAGES:
result, markup = email(event.event_type, event.data, event.severity, language)
self.assertIn(warning, result['body'])
@@ -0,0 +1,150 @@
"""Maintainer acceptance at inert actual notification consumers."""
import unittest
from notification_fixture import templates, receive, LANGUAGES, SCRIPTS, extract
from notification_final_fixture import deliver
NATIVE_REPORT = '''Details
=======
VMID Name Status Time Size Filename
100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z
Total running time: 1m 1s
Total size: 1 GiB
Logs
====
vzdump --all 1 --storage PBS --mode snapshot
100: 2026-09-29 17:00:00 INFO: Starting Backup of VM 100 (qemu)
100: 2026-09-29 17:01:01 INFO: Finished Backup of VM 100 (00:01:01)
'''
class MaintainerFollowupTests(unittest.TestCase):
def test_completed_with_warnings_requires_independent_completion(self):
warning = '\n100: 2026-09-29 17:00:01 WARN: file changed during backup'
event = receive(NATIVE_REPORT + warning)
self.assertEqual(event.data['backup_outcome'], 'completed_with_warnings')
self.assertEqual((event.event_type,event.severity), ('backup_complete','INFO'))
for manual in (False,True):
for lang in LANGUAGES:
result = deliver(event.event_type,event.data,event.severity,lang,manual=manual)
label = templates.runtime_message('backup.warningTitle',lang,hostname='node-a')
self.assertIn(label,result['title'])
status = templates.runtime_message('channels.email.status.completed_with_warnings',lang)
self.assertIn(status,result['text'])
self.assertIn('file changed during backup',result['text'])
# Manual sends intentionally skip channel emoji enrichment.
if not manual: self.assertTrue(result['title'].startswith('💾⚠️'))
rich, _ = templates.enrich_with_emojis(event.event_type,result['title'],result['body'],event.data)
self.assertTrue(rich.startswith('💾⚠️'))
self.assertEqual(receive('WARN: file changed during backup').data['backup_outcome'],'unconfirmed')
self.assertEqual(receive(NATIVE_REPORT.split('Total running time:')[0]+warning).data['backup_outcome'],'unconfirmed')
self.assertEqual(receive(NATIVE_REPORT+warning+'\nERROR: cleanup failed').data['backup_outcome'],'failed')
self.assertEqual(receive(NATIVE_REPORT+warning,'warning').data['backup_outcome'],'completed_with_warnings')
self.assertEqual(receive('INFO: Starting Backup of VM 100 (qemu)\nINFO: Finished Backup of VM 100 (00:01:01)'+warning).data['backup_outcome'],'completed_with_warnings')
self.assertEqual(receive(NATIVE_REPORT+warning,kind='').data['backup_outcome'],'unconfirmed')
def test_null_filename_failed_guest_uses_own_start_identity(self):
report = NATIVE_REPORT.replace('100 web ok 1m 1s 1 GiB vm/100/2026-09-29T17:00:00Z',
'100 web err 1m 1s 0 B null')
for kind, prefix in (('qemu','VM'),('lxc','CT')):
own = report.replace('VM 100 (qemu)',f'VM 100 ({kind})')
# Unrelated guest appears first and must never supply the failed type.
message = f'INFO: Starting Backup of VM 999 ({"lxc" if kind == "qemu" else "qemu"})\n' + own
event = receive(message,'error','vzdump backup status (raw-node): backup failed')
parsed = templates._parse_vzdump_message(message)
self.assertEqual(parsed['vms'][0]['type'],kind)
result = deliver(event.event_type,event.data,event.severity)
self.assertIn(f'{prefix} web (100)',result['title'])
self.assertIn(f'❌ {prefix} web (100)',result['body'])
for message in (report.replace('Starting Backup of VM 100','Starting Backup of VM 999'),
report+'\nINFO: Starting Backup of VM 100 (lxc)'):
self.assertEqual(templates._parse_vzdump_message(message)['vms'][0]['type'],'')
def test_original_subject_only_retained_when_no_guest_context(self):
subject = 'vzdump backup status (raw-host): backup failed: multiple problems'
event = receive('ERROR: archive write failed\n'+NATIVE_REPORT,'error',subject)
event.data['hostname']='configured-alias'
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertNotIn(subject,result['text'])
self.assertEqual(result['text'].count('ERROR: archive write failed'),1)
self.assertIn('configured-alias',result['title'])
setup=receive('Details\n=======\nVMID Name Status Time Size Filename\n\nTotal running time: 0s\nTotal size: 0 B','error',subject.replace('multiple problems','unable to open storage'))
result=deliver(setup.event_type,setup.data,setup.severity)
self.assertEqual(result['text'].count(setup.data['pve_title']),1)
unique=receive(NATIVE_REPORT,'error',subject.replace('multiple problems','job-end hook denied'))
result=deliver(unique.event_type,unique.data,unique.severity)
self.assertIn('job-end hook denied',result['body'])
self.assertNotIn('vzdump backup status',result['body'])
def test_backup_diagnostics_are_bounded_with_principal_cause_and_notice(self):
raw = NATIVE_REPORT + '\n' + '\n'.join(f'WARN: repeated warning {i}' for i in range(80)) + '\nERROR: principal archive write failure'
event=receive(raw)
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertIn('ERROR: principal archive write failure',result['body'])
self.assertLessEqual(len('\n'.join(line for line in result['body'].splitlines() if line.startswith(('WARN:','ERROR:')))),1024)
self.assertLessEqual(sum(line.startswith(('WARN:','ERROR:')) for line in result['body'].splitlines()),8)
notice=templates.runtime_message('backup.diagnosticsOmitted',lang,count=73)
self.assertIn(notice,result['body'])
self.assertEqual(event.data['pve_message'],raw)
long=receive('ERROR: '+ 'b'*5000,'error','vzdump backup status (node): backup failed')
result=deliver(long.event_type,long.data,long.severity)
self.assertLess(len(result['body']),1400)
self.assertIn('ERROR: '+ 'b'*100,result['body'])
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted','en',count=1),result['body'])
def test_real_quiet_digest_retains_each_backup_outcome_icon(self):
samples=[(NATIVE_REPORT,'confirmed','💾✅'),(NATIVE_REPORT+'\nWARN: changed file','completed_with_warnings','💾⚠️'),(NATIVE_REPORT+'\nERROR: write failed','failed','💾❌'),('INFO: Starting Backup of VM 100 (qemu)','unconfirmed','💾❔')]
for raw,outcome,icon in samples:
event=receive(raw)
self.assertEqual(event.data['backup_outcome'],outcome)
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang,quiet=True)
self.assertIn(icon,result['body'])
self.assertNotIn('💾❔',result['body']) if outcome!='unconfirmed' else None
import types,datetime
ns={'datetime':datetime.datetime,'runtime_message':templates.runtime_message,'EVENT_EMOJI':templates.EVENT_EMOJI,'CATEGORY_EMOJI':templates.CATEGORY_EMOJI}
compose=extract(SCRIPTS/'notification_manager.py','_compose_digest_body','NotificationManager',ns)
target=types.SimpleNamespace(_notification_language=lambda:'en')
rows=[(i,'backup_complete','backup',1,icon+' node: Backup','') for i,(_,_,icon) in enumerate(samples)]
body=compose(target,rows,use_icons=True)
for _,_,icon in samples:self.assertIn(icon,body)
plain=compose(target,rows,use_icons=False)
for _,_,icon in samples:self.assertNotIn(icon,plain)
should=extract(SCRIPTS/'notification_manager.py','_should_buffer_for_digest','NotificationManager',{})
self.assertFalse(should(types.SimpleNamespace(_DIGEST_EXEMPT_EVENTS={'backup_complete'},_config={'email.digest_enabled':'true'}),'email','INFO','backup_complete'))
def test_quiet_restore_details_are_subordinate_in_text_and_email(self):
from notification_final_fixture import restore_event
event=restore_event('missing module zfs')
for lang in LANGUAGES:
result=deliver(event['event_type'],event['data'],event['severity'],lang,quiet=True)
body_lines=result['buffered'][0][3].splitlines()
for line in body_lines:
if line.strip():
self.assertIn(' '+line.strip(),result['body'])
self.assertIn(' '+__import__('html').escape(line.strip()),result['html'])
self.assertIn('white-space:pre-wrap;',result['html'])
self.assertNotIn(templates.runtime_message('digest.footer',lang),result['body'])
def test_job_level_subject_cause_survives_warning_cap(self):
raw=NATIVE_REPORT+'\n'+'\n'.join('WARN: repeated diagnostic '+str(i) for i in range(80))
event=receive(raw,'error','vzdump backup status (raw-host): backup failed: job-end hook denied')
for lang in LANGUAGES:
result=deliver(event.event_type,event.data,event.severity,lang)
self.assertEqual(result['body'].count('job-end hook denied'),1)
self.assertNotIn('vzdump backup status',result['body'])
self.assertIn(templates.runtime_message('backup.diagnosticsOmitted',lang,count=73),result['body'])
def test_backup_quiet_email_uses_event_scoped_wrapping(self):
event=receive(NATIVE_REPORT+'\nWARN: changed file')
result=deliver(event.event_type,event.data,event.severity,'sv',quiet=True)
self.assertTrue(result['data'].get('_backup_summary'))
self.assertIn('table-layout:fixed;',result['html'])
self.assertIn('overflow-wrap:break-word;',result['html'])
unrelated=deliver('node_reconnect',{'hostname':'node-a'},'OK')
self.assertNotIn('table-layout:fixed;',unrelated['html'])
if __name__ == '__main__': unittest.main()
@@ -134,7 +134,7 @@ class OutcomeWording(unittest.TestCase):
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\n'+header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
('vzdump', 'info', header+'\n'+row_ok+'\nTotal running time: 00:01:00\n'+truncated, 'confirmed'),
('vzdump', 'info', header+'\n'+row_err+'\nTotal running time: 00:01:00\n'+truncated, 'failed'),
('vzdump', 'warning', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'unconfirmed'),
('vzdump', 'warning', header+'\n'+row_ok+'\nTotal running time: 00:01:00', 'completed_with_warnings'),
('vzdump', 'info', header+'\n'+row_ok, 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\n'+header+'\n'+row_ok, 'unconfirmed'),
('vzdump', 'warning', header+'\n'+row_err+'\nTotal running time: 00:01:00', 'failed'),
@@ -147,11 +147,11 @@ class OutcomeWording(unittest.TestCase):
('vzdump', 'info', header+'\n'+row_ok+'\n'+row_warning+'\nTotal running time: 00:02:00', 'unconfirmed'),
('vzdump', 'info', header+'\n'+row_ok[:30], 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'confirmed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'unconfirmed'),
('vzdump', 'warning', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nWARNING: skipped file', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: TASK OK\n104 alpha WARNINGS: 1', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nWARNING: skipped file', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: TASK OK\n104 alpha WARNINGS: 1', 'completed_with_warnings'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Finished Backup of VM 105 (00:01:00)', 'unconfirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nINFO: Starting Backup of VM 105 (lxc)\nINFO: Finished Backup of VM 104 (00:01:00)\nINFO: Finished Backup of VM 105 (00:01:00)', 'confirmed'),
('vzdump', 'info', 'INFO: Starting Backup of VM 104 (qemu)\nERROR: backup failed for VM 104', 'failed'),
@@ -128,10 +128,10 @@ class PVE92Tests(unittest.TestCase):
self.assertEqual(event_for(message).data['backup_outcome'], 'confirmed')
for message, severity, expected in (
(PVE92.replace('ok ', 'OK '), 'info', 'confirmed'),
(message, 'warning', 'unconfirmed'),
(message, 'warning', 'completed_with_warnings'),
(message, 'error', 'failed'),
(message + '\nERROR: archive write failed', 'info', 'failed'),
(message + '\nWARNING: skipped file', 'info', 'unconfirmed'),
(message + '\nWARNING: skipped file', 'info', 'completed_with_warnings'),
(PVE92.replace('ok ', 'WARNINGS '), 'info', 'unconfirmed'),
):
with self.subTest(message=message, severity=severity):
@@ -0,0 +1,107 @@
"""Fresh existing-check provenance; extracted consumers, real disposable SQLite."""
import contextlib
import datetime
import json
import os
import sqlite3
import tempfile
import time
import types
import unittest
from unittest.mock import patch
from notification_fixture import extract, SCRIPTS, templates, LANGUAGES
from notification_final_fixture import deliver
class RecoveryEvidenceTests(unittest.TestCase):
def test_cpu_success_provenance_is_persisted_only_after_normal_samples(self):
events=[]
with tempfile.TemporaryDirectory() as scratch:
db=scratch+'/health.sqlite'
conn=sqlite3.connect(db)
conn.execute('CREATE TABLE errors(id INTEGER PRIMARY KEY,error_key TEXT,details TEXT,resolved_at TEXT,resolution_type TEXT,resolution_reason TEXT)')
conn.execute("INSERT INTO errors(error_key,details) VALUES ('cpu_usage','{}')")
conn.commit();conn.close()
@contextlib.contextmanager
def connection():
c=sqlite3.connect(db)
try:yield c
finally:c.close()
ns={'datetime':datetime.datetime,'json':json}
resolve=extract(SCRIPTS/'health_persistence.py','_resolve_error_impl','HealthPersistence',ns)
store=types.SimpleNamespace(_db_connection=connection,_entity_from_details=lambda details:'',_record_event=lambda cursor,kind,key,data:events.append(data))
store.resolve_error=lambda key,reason,**kw:resolve(store,key,reason,**kw)
ns={'Dict':dict,'Any':object,'os':os,'time':time,'health_persistence':store,'psutil':types.SimpleNamespace(cpu_percent=lambda **kw:20,cpu_count=lambda:4)}
check=extract(SCRIPTS/'health_monitor.py','_check_cpu_with_hysteresis','HealthMonitor',ns)
target=types.SimpleNamespace(state_history={'cpu_usage':[{'value':20,'time':time.time()-i*10} for i in range(10)]},CPU_CRITICAL=95,CPU_WARNING=85,CPU_RECOVERY=75,CPU_CRITICAL_DURATION=300,CPU_WARNING_DURATION=300,CPU_RECOVERY_DURATION=120,_check_cpu_temperature=lambda:None)
result=check(target)
self.assertEqual(result['status'],'OK')
self.assertTrue(events[-1].get('check_evidence'),events)
proof=events[-1]['check_evidence']
self.assertEqual(proof['check'],'cpu_usage')
self.assertGreaterEqual(proof['checked_at'],time.time()-5)
# Existing generic resolve callers (cleanup/exclusion) get no proof.
conn=sqlite3.connect(db);conn.execute('UPDATE errors SET resolved_at=NULL');conn.commit();conn.close()
resolve(store,'cpu_usage','No longer present')
self.assertFalse(events[-1].get('check_evidence'))
def test_recovery_query_requires_fresh_same_incident_proof(self):
with tempfile.TemporaryDirectory() as scratch:
db=scratch+'/health.sqlite'
conn=sqlite3.connect(db)
conn.execute('CREATE TABLE errors(id INTEGER PRIMARY KEY,error_key TEXT,first_seen TEXT,last_seen TEXT,resolved_at TEXT,acknowledged INTEGER)')
conn.execute('CREATE TABLE events(id INTEGER PRIMARY KEY,event_type TEXT,error_key TEXT,timestamp TEXT,data TEXT)')
now=datetime.datetime.now(); first=(now-datetime.timedelta(minutes=10)).isoformat(); last=(now-datetime.timedelta(minutes=1)).isoformat(); resolved=now.isoformat()
proof={'check':'cpu_usage','checked_at':now.timestamp()}
conn.execute('INSERT INTO errors VALUES(1,?,?,?,?,0)',('cpu_usage',first,last,resolved))
conn.execute('INSERT INTO events VALUES(1,?,?,?,?)',('resolved','cpu_usage',resolved,json.dumps({'check_evidence':proof})))
conn.commit();conn.close()
@contextlib.contextmanager
def connection(**kwargs):
c=sqlite3.connect(db)
try:yield c
finally:c.close()
ns={'datetime':datetime.datetime,'json':json,'time':time}
tree=(SCRIPTS/'health_persistence.py').read_text()
query=extract(SCRIPTS/'health_persistence.py','get_recovery_evidence','HealthPersistence',ns) if 'def get_recovery_evidence(' in tree else lambda *args:None
store=types.SimpleNamespace(_db_connection=connection)
self.assertEqual(query(store,'cpu_usage',first),proof)
self.assertIsNone(query(store,'cpu_usage','different incident'))
for field,value in [('acknowledged',1),('resolved_at',None),('last_seen',(now+datetime.timedelta(seconds=1)).isoformat())]:
conn=sqlite3.connect(db);conn.execute(f'UPDATE errors SET {field}=?',(value,));conn.commit();conn.close()
self.assertIsNone(query(store,'cpu_usage',first))
conn=sqlite3.connect(db);conn.execute('UPDATE errors SET acknowledged=0,resolved_at=?,last_seen=?',(resolved,last));conn.commit();conn.close()
for bad in (None,{'check':'cpu_usage','checked_at':now.timestamp()-7201},{'check':'storage_removed','checked_at':now.timestamp()},{'check':'cpu_usage','checked_at':float('inf')},{'check':'cpu_usage','checked_at':10**400}):
conn=sqlite3.connect(db);conn.execute('UPDATE events SET data=?',(json.dumps({'check_evidence':bad}),));conn.commit();conn.close()
self.assertIsNone(query(store,'cpu_usage',first))
def test_poller_and_all_consumers_distinguish_proven_recovery_from_disappearance(self):
import sys
ns={'time':time,'json':json,'Dict':dict,'NotificationEvent':lambda *a,**kw:types.SimpleNamespace(event_type=a[0],severity=a[1],data=a[2])}
poll=extract(SCRIPTS/'notification_events.py','_check_persistent_health','PollingCollector',ns)
for proof in (None,{'check':'cpu_usage','checked_at':time.time()}):
events=[]
store=types.SimpleNamespace(get_active_errors=lambda:[],is_error_acknowledged=lambda key:False,get_recovery_evidence=lambda *a:proof)
collector=types.SimpleNamespace(_hostname='node-a',_ENTITY_MAP={'cpu':('node','')},_first_poll_done=True,_known_errors={'cpu_usage':{'category':'cpu','reason':'CPU high','severity':'WARNING','first_seen':'2026-09-30T00:00:00'}},_notified_severity={'cpu_usage':'WARNING'},_last_notified={'cpu_usage':1},_queue=types.SimpleNamespace(put=events.append),_guest_storage_error_is_now_foreign=lambda *a:False,_save_known_errors_meta=lambda:None)
with patch.dict(sys.modules,{'health_persistence':types.SimpleNamespace(health_persistence=store)}):poll(collector)
self.assertEqual(len(events),1)
event=events[0]
self.assertEqual(event.data.get('recovery_outcome'),'resolved' if proof else 'no_longer_reported')
self.assertEqual(event.data['is_recovery'],bool(proof))
for lang in LANGUAGES:
for manual in (False,True):
result=deliver(event.event_type,event.data,event.severity,lang,manual=manual)
if proof:
self.assertIn(templates.runtime_message('healthRecovery.title',lang,hostname='node-a',category='cpu',entity_suffix=''),result['title'])
self.assertIn('background:#f0fdf4;',result['html'])
else:self.assertNotIn('background:#f0fdf4;',result['html'])
self.assertEqual(result['text'].count(event.data['reason']),1)
def test_manual_recovery_flag_alone_is_not_authoritative_evidence(self):
data={'hostname':'node-a','category':'cpu','reason':'Observation disappeared','duration':'1h','original_severity':'WARNING','recovery_outcome':'resolved'}
for lang in LANGUAGES:
for manual in (False,True):
result=deliver('error_resolved',data,'OK',lang,manual=manual)
self.assertNotIn('background:#f0fdf4;',result['html'])
self.assertNotIn(templates.runtime_message('healthRecovery.body',lang,**data),result['body'])
if __name__=='__main__':unittest.main()
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Fehlgeschlagen",
"completed": "Abgeschlossen",
"started": "Gestartet",
"unconfirmed": "Nicht bestätigt"
"unconfirmed": "Nicht bestätigt",
"completed_with_warnings": "Mit Warnungen abgeschlossen"
},
"report": "{group} Bericht",
"details": "Details",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}."
},
"healthRecovery": {"title": "{hostname}: Behoben - {category}{entity_suffix}", "body": "Eine aktuelle Zustandsprüfung hat für {category} wieder einen normalen Zustand festgestellt.\nVorherige Beobachtung: {reason}\nVorheriger Schweregrad: {original_severity}\nZeit seit der ersten Beobachtung: {duration}", "status": "Behoben"},
"backup": {
"confirmedTitle": "{hostname}: Backup abgeschlossen",
"confirmedBody": "Backup erfolgreich abgeschlossen.",
"errorTitle": "{hostname}: Backup-Fehler gemeldet",
"errorBody": "Der Backup-Bericht enthält einen Fehler.",
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt."
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt.",
"warningTitle": "{hostname}: Sicherung mit Warnungen abgeschlossen",
"warningBody": "Sicherung mit Warnungen abgeschlossen.",
"diagnosticsOmitted": "Weitere Diagnosezeilen oder Text ausgelassen: {count}. Originalbericht bleibt erhalten."
}
}
}
File diff suppressed because one or more lines are too long
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Fallido",
"completed": "Completado",
"started": "Iniciado",
"unconfirmed": "Sin confirmar"
"unconfirmed": "Sin confirmar",
"completed_with_warnings": "Completado con advertencias"
},
"report": "Informe de {group}",
"details": "Detalles",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "Lecturas altas registradas a lo largo de {duration}."
},
"healthRecovery": {"title": "{hostname}: Resuelto - {category}{entity_suffix}", "body": "Una comprobación reciente confirma que la condición de {category} volvió a la normalidad.\nObservación anterior: {reason}\nGravedad anterior: {original_severity}\nTiempo desde la primera observación: {duration}", "status": "Resuelto"},
"backup": {
"confirmedTitle": "{hostname}: backup completado",
"confirmedBody": "Backup completado correctamente.",
"errorTitle": "{hostname}: error notificado en el backup",
"errorBody": "El informe del backup contiene un error.",
"unconfirmedBody": "El resultado del backup no está confirmado."
"unconfirmedBody": "El resultado del backup no está confirmado.",
"warningTitle": "{hostname}: Backup completado con advertencias",
"warningBody": "Backup completado con advertencias.",
"diagnosticsOmitted": "Líneas o texto de diagnóstico omitidos: {count}. Se conserva el informe original."
}
}
}
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Échec",
"completed": "Terminé",
"started": "Commencé",
"unconfirmed": "Non confirmé"
"unconfirmed": "Non confirmé",
"completed_with_warnings": "Terminée avec avertissements"
},
"report": "Rapport {group}",
"details": "Détails",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "Les relevés élevés s'étendent sur {duration}."
},
"healthRecovery": {"title": "{hostname} : Résolu - {category}{entity_suffix}", "body": "Un contrôle récent confirme le retour à la normale de la condition {category}.\nObservation précédente : {reason}\nGravité précédente : {original_severity}\nTemps depuis la première observation : {duration}", "status": "Résolu"},
"backup": {
"confirmedTitle": "{hostname} : sauvegarde terminée",
"confirmedBody": "Sauvegarde terminée avec succès.",
"errorTitle": "{hostname} : erreur signalée lors de la sauvegarde",
"errorBody": "Le rapport de sauvegarde contient une erreur.",
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé."
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé.",
"warningTitle": "{hostname}: Sauvegarde terminée avec avertissements",
"warningBody": "Sauvegarde terminée avec avertissements.",
"diagnosticsOmitted": "Lignes ou texte de diagnostic omis : {count}. Rapport original conservé."
}
}
}
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Fallito",
"completed": "Completato",
"started": "Iniziato",
"unconfirmed": "Non confermato"
"unconfirmed": "Non confermato",
"completed_with_warnings": "Completato con avvisi"
},
"report": "{group} Rapporto",
"details": "Dettagli",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "Intervallo dei campioni sopra soglia: {duration}."
},
"healthRecovery": {"title": "{hostname}: Risolto - {category}{entity_suffix}", "body": "Un controllo recente conferma che la condizione {category} è tornata nella norma.\nOsservazione precedente: {reason}\nGravità precedente: {original_severity}\nTempo dalla prima osservazione: {duration}", "status": "Risolto"},
"backup": {
"confirmedTitle": "{hostname}: backup completato",
"confirmedBody": "Backup completato correttamente.",
"errorTitle": "{hostname}: errore segnalato nel backup",
"errorBody": "Il rapporto del backup contiene un errore.",
"unconfirmedBody": "L’esito del backup non è confermato."
"unconfirmedBody": "L’esito del backup non è confermato.",
"warningTitle": "{hostname}: Backup completato con avvisi",
"warningBody": "Backup completato con avvisi.",
"diagnosticsOmitted": "Righe o testo diagnostico omessi: {count}. Il report originale è conservato."
}
}
}
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Falhou",
"completed": "Concluído",
"started": "Iniciado",
"unconfirmed": "Não confirmado"
"unconfirmed": "Não confirmado",
"completed_with_warnings": "Concluído com avisos"
},
"report": "Relatório {group}",
"details": "Detalhes",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "As amostras elevadas abrangem {duration}."
},
"healthRecovery": {"title": "{hostname}: Resolvido - {category}{entity_suffix}", "body": "Uma verificação recente confirma que a condição de {category} voltou ao normal.\nObservação anterior: {reason}\nGravidade anterior: {original_severity}\nTempo desde a primeira observação: {duration}", "status": "Resolvido"},
"backup": {
"confirmedTitle": "{hostname}: backup concluído",
"confirmedBody": "Backup concluído com sucesso.",
"errorTitle": "{hostname}: erro comunicado no backup",
"errorBody": "O relatório do backup contém um erro.",
"unconfirmedBody": "O resultado do backup não está confirmado."
"unconfirmedBody": "O resultado do backup não está confirmado.",
"warningTitle": "{hostname}: Backup concluído com avisos",
"warningBody": "Backup concluído com avisos.",
"diagnosticsOmitted": "Linhas ou texto de diagnóstico omitidos: {count}. Relatório original preservado."
}
}
}
+7 -2
View File
@@ -6980,7 +6980,8 @@
"failed": "Misslyckades",
"completed": "Klar",
"started": "Startat",
"unconfirmed": "Obekräftat"
"unconfirmed": "Obekräftat",
"completed_with_warnings": "Klar med varningar"
},
"report": "{group} Rapportera",
"details": "Detaljer",
@@ -6991,12 +6992,16 @@
"temperature": {
"sampleSpan": "De höga mätvärdena sträcker sig över {duration}."
},
"healthRecovery": {"title": "{hostname}: Åtgärdat - {category}{entity_suffix}", "body": "En aktuell hälsokontroll bekräftar att tillståndet för {category} återgått till det normala.\nTidigare observation: {reason}\nTidigare allvarlighetsgrad: {original_severity}\nTid sedan första observationen: {duration}", "status": "Åtgärdat"},
"backup": {
"confirmedTitle": "{hostname}: säkerhetskopiering klar",
"confirmedBody": "Säkerhetskopieringen slutfördes utan fel.",
"errorTitle": "{hostname}: fel rapporterat vid säkerhetskopiering",
"errorBody": "Rapporten om säkerhetskopieringen innehåller ett fel.",
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat."
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat.",
"warningTitle": "{hostname}: Säkerhetskopiering klar med varningar",
"warningBody": "Säkerhetskopiering klar med varningar.",
"diagnosticsOmitted": "Utelämnade diagnosrader eller text: {count}. Originalrapporten bevaras."
}
}
}
+5 -1
View File
@@ -1452,7 +1452,11 @@ class HealthMonitor:
status = 'OK'
reason = None
# CPU is normal - auto-resolve any existing CPU errors
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal')
evidence = None
if cpu_percent < self.CPU_RECOVERY and len(recovery_samples) >= RECOVERY_MIN_SAMPLES:
evidence = {'check': 'cpu_usage', 'checked_at': current_time}
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal',
check_evidence=evidence)
temp_status = self._check_cpu_temperature()
+50 -3
View File
@@ -744,12 +744,12 @@ class HealthPersistence:
return event_info
def resolve_error(self, error_key: str, reason: str = 'auto-resolved'):
def resolve_error(self, error_key: str, reason: str = 'auto-resolved', *, check_evidence=None):
"""Mark an error as resolved"""
with self._db_lock:
return self._resolve_error_impl(error_key, reason)
return self._resolve_error_impl(error_key, reason, check_evidence=check_evidence)
def _resolve_error_impl(self, error_key, reason):
def _resolve_error_impl(self, error_key, reason, *, check_evidence=None):
with self._db_connection() as conn:
cursor = conn.cursor()
now = datetime.now().isoformat()
@@ -788,12 +788,59 @@ class HealthPersistence:
stored_details = None
self._record_event(cursor, 'resolved', error_key, {
'reason': reason,
# Only explicit current-check callers attach this proof.
# Generic resolve/cleanup remains neutral.
'check_evidence': check_evidence,
'entity': self._entity_from_details(stored_details),
'details': stored_details or {},
})
conn.commit()
def get_recovery_evidence(self, error_key: str, first_seen: str):
"""Return fresh same-incident native check proof, never absence of errors.
Initially only the host CPU check has a stable condition identity. Other
checks, generic clears, excluded/deleted records and legacy events stay
neutral until they have equivalent per-condition provenance.
"""
if error_key != 'cpu_usage' or not first_seen:
return None
try:
with self._db_connection() as conn:
row = conn.execute('''
SELECT first_seen, last_seen, resolved_at, acknowledged
FROM errors WHERE error_key = ? ORDER BY id DESC LIMIT 1
''', (error_key,)).fetchone()
if not row or row[0] != first_seen or not row[2] or row[3]:
return None
event = conn.execute('''
SELECT timestamp, data FROM events
WHERE error_key = ? AND event_type = 'resolved'
ORDER BY id DESC LIMIT 1
''', (error_key,)).fetchone()
if not event:
return None
proof = json.loads(event[1]).get('check_evidence')
if not isinstance(proof, dict) or proof.get('check') != error_key:
return None
checked = proof.get('checked_at')
if not isinstance(checked, (int, float)) or isinstance(checked, bool):
return None
checked = float(checked)
# Reuse the collector's existing two-hour freshness boundary.
now = datetime.now().timestamp()
if not 0 <= now - checked <= 7200:
return None
last_seen = datetime.fromisoformat(row[1]).timestamp()
resolved = datetime.fromisoformat(row[2]).timestamp()
recorded = datetime.fromisoformat(event[0]).timestamp()
if not last_seen <= checked <= resolved <= recorded:
return None
return proof
except (ValueError, TypeError, AttributeError, OverflowError):
return None
def is_error_active(self, error_key: str, category: Optional[str] = None) -> bool:
"""
Check if an error is currently active OR suppressed (dismissed but within suppression period).
+17 -5
View File
@@ -1038,13 +1038,23 @@ class EmailChannel(NotificationChannel):
# Determine group for section header
event_type = data.get('_event_type', '')
if event_type == 'error_resolved':
sev.update(self._SEV_DEFAULT)
sev['label'] = _runtime_text('email.severity.observation', data)
if (data.get('recovery_outcome') == 'resolved'
and data.get('is_recovery') is True
and isinstance(data.get('check_evidence'), dict)
and data['check_evidence'].get('check') == 'cpu_usage'):
sev.update(self._SEV_STYLE['OK'])
sev['label'] = _runtime_notification_text('healthRecovery.status', data)
else:
sev.update(self._SEV_DEFAULT)
sev['label'] = _runtime_text('email.severity.observation', data)
elif event_type == 'backup_complete':
outcome = data.get('backup_outcome')
if outcome == 'confirmed':
sev.update(self._SEV_STYLE['OK'])
status = 'completed'
elif outcome == 'completed_with_warnings':
sev.update(self._SEV_STYLE['WARNING'])
status = 'completed_with_warnings'
elif outcome == 'failed':
sev.update(self._SEV_STYLE['CRITICAL'])
status = 'failed'
@@ -1057,7 +1067,7 @@ class EmailChannel(NotificationChannel):
# bodies, including restore bodies released from quiet hours.
backup_email = event_type in {'backup_complete', 'backup_fail'}
wrap_body = (event_type in {'temp_high', 'system_restore_completed', 'error_resolved'}
or backup_email or data.get('_restore_summary'))
or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
backup_title_wrap = temp_cell_wrap if backup_email else ''
@@ -1103,7 +1113,7 @@ class EmailChannel(NotificationChannel):
# Observation age/disappearance must not become a green OK row.
# The endpoint's warnings_block and task counts live in the
# localized body, not the generic services Event row.
detail_rows = [('', html_mod.escape(line.strip()))
detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
for line in body.split('\n') if line.strip()]
# ── Fallback: if no structured rows, render body text lines ──
@@ -1122,6 +1132,7 @@ class EmailChannel(NotificationChannel):
# ── Render detail rows as HTML table ──
rows_html = ''
summary_whitespace = 'white-space:pre-wrap;' if data.get('_quiet_hours_summary') and data.get('_restore_summary') else ''
for label, value in detail_rows:
if label:
rows_html += f'''<tr>
@@ -1131,7 +1142,7 @@ class EmailChannel(NotificationChannel):
else:
# Full-width row (no label, just description text)
rows_html += f'''<tr>
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}">{value}</td>
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}{summary_whitespace}">{value}</td>
</tr>'''
# ── Reason / details block (long text, displayed separately) ──
@@ -1291,6 +1302,7 @@ class EmailChannel(NotificationChannel):
_add('Storage', data.get('storage') or data.get('storage_name'), 'code')
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
status_key = ('failed' if data.get('backup_outcome') == 'failed'
else 'completed_with_warnings' if data.get('backup_outcome') == 'completed_with_warnings'
else 'unconfirmed')
else:
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
+20 -6
View File
@@ -3223,6 +3223,13 @@ class PollingCollector:
self._last_notified.pop(key, None)
continue
# Disappearance is not recovery. Only same-incident proof written
# by a successful existing native check can certify normality.
try:
recovery_evidence = health_persistence.get_recovery_evidence(key, first_seen)
except Exception:
recovery_evidence = None
# Calculate duration
duration = ''
if first_seen:
@@ -3273,6 +3280,9 @@ class PollingCollector:
else:
clean_reason = 'Condition no longer reported'
if recovery_evidence:
clean_reason = reason_summary
# `original_severity` must match what the user actually saw
# in the most-recent notification for this error, not the
# latest DB severity. See `_notified_severity` docstring at
@@ -3299,7 +3309,9 @@ class PollingCollector:
'original_severity': original_severity,
'first_seen': first_seen,
'duration': duration_label,
'is_recovery': True,
'is_recovery': bool(recovery_evidence),
'recovery_outcome': 'resolved' if recovery_evidence else 'no_longer_reported',
'check_evidence': recovery_evidence,
}
# Spread the original details blob so the resolved notification
# can use the same {storage_name}/{vm_name}/{device} placeholders
@@ -4302,14 +4314,16 @@ class ProxmoxHookWatcher:
table = _parse_vzdump_table(text)
if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
return 'failed'
if severity not in ('info', 'ok', 'success') or re.search(
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*\d+)', text):
warnings = severity in ('warning', 'warn') or bool(re.search(
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*[1-9]\d*)', text))
if severity not in ('info', 'ok', 'success', 'warning', 'warn'):
return 'unconfirmed'
completed = 'completed_with_warnings' if warnings else 'confirmed'
# A present table is authoritative: do not certify an incomplete table
# from a finished guest log, or reject a complete OK table merely
# because the extra diagnostic log was truncated before its finishes.
if table is not None:
return ('confirmed' if table['complete'] and
return (completed if table['complete'] and
all(guest['status'].lower() == 'ok' for guest in table['vms'])
else 'unconfirmed')
if starts:
@@ -4322,10 +4336,10 @@ class ProxmoxHookWatcher:
return 'unconfirmed' # A finish before its start is not evidence.
else:
pending[vmid] -= 1
return 'confirmed' if not any(pending.values()) else 'unconfirmed'
return completed if not any(pending.values()) else 'unconfirmed'
if re.search(
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
return 'confirmed'
return completed
return 'unconfirmed'
def process_webhook(self, payload: dict) -> dict:
+19 -4
View File
@@ -1364,6 +1364,12 @@ class NotificationManager:
# Get journal context if available (will be enriched per-channel based on detail_level)
raw_journal_context = data.get('_journal_context', '')
# Persist a presentation token in the existing title column: buffers
# otherwise discard outcome metadata before composition. Old rows
# without a token remain neutral; routing and schema are unchanged.
buffer_title = title
if event_type in ('backup_complete', 'backup_fail'):
buffer_title, _ = enrich_with_emojis(event_type, title, '', data)
for ch_name, channel in channels.items():
# ── Per-channel category check ──
@@ -1393,7 +1399,7 @@ class NotificationManager:
# delivered after Quiet Hours + Daily Digest were merged.
if severity != 'CRITICAL' and self._in_quiet_hours(ch_name):
self._buffer_quiet_event(ch_name, event_type, event_group,
severity, title, body)
severity, buffer_title, body)
continue
# ── Per-channel daily digest ──
@@ -1406,7 +1412,7 @@ class NotificationManager:
# excluded from the digest by `_DIGEST_EXEMPT_EVENTS`.
if self._should_buffer_for_digest(ch_name, severity, event_type):
self._buffer_digest_event(ch_name, event_type, event_group,
severity, title, body)
severity, buffer_title, body)
continue
try:
@@ -1849,14 +1855,22 @@ class NotificationManager:
if index < 8 or (quiet_release and item[1] == 'system_restore_completed')]
for ts, ev_type, title, body in visible_items:
hhmm = datetime.fromtimestamp(ts).strftime('%H:%M')
backup_icon = ''
if ev_type in ('backup_complete', 'backup_fail'):
for token in ('💾✅', '💾⚠️', '💾❌', '💾❔', '💾'):
if title.startswith(token + ' '):
backup_icon = token
title = title[len(token) + 1:]
break
backup_icon = backup_icon or ('💾❌' if ev_type == 'backup_fail' else '💾❔')
short_title = title.split(': ', 1)[-1] if ': ' in title else title
event_icon = (
EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
backup_icon or EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
) if use_icons else ''
event_prefix = f'{event_icon} ' if event_icon else ''
lines.append(f" • {event_prefix}{hhmm} {short_title}")
if quiet_release and ev_type == 'system_restore_completed' and body:
lines.extend(line for line in body.splitlines() if line.strip())
lines.extend(' ' + line.strip() for line in body.splitlines() if line.strip())
if len(items) > len(visible_items):
lines.append(runtime_message('digest.more', language, count=len(items) - len(visible_items)))
lines.append('')
@@ -2006,6 +2020,7 @@ class NotificationManager:
summary_title, summary_body, severity='INFO',
data={'_quiet_hours_summary': True, '_count': len(rows),
'_restore_summary': any(row[1] == 'system_restore_completed' for row in rows),
'_backup_summary': any(row[1] in ('backup_complete', 'backup_fail') for row in rows),
'_notification_language': language},
) or result
except Exception as e:
+70 -10
View File
@@ -244,6 +244,13 @@ def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
status = 'error'
kind = ('lxc' if 'lxc' in filename or filename.startswith('ct/') else
'qemu' if 'qemu' in filename or filename.startswith('vm/') else '')
if filename.lower() == 'null':
# Native failed rows lack archives. Match only this row's VMID;
# multiple inconsistent starts are not authoritative identity.
kinds = set(re.findall(
r'(?im)\bStarting Backup of VM ' + re.escape(vmid) + r'\s+\((lxc|qemu)\)',
message))
kind = kinds.pop() if len(kinds) == 1 else ''
rows.append({'vmid': vmid, 'name': name, 'status': status,
'time': duration, 'size': size, 'filename': filename,
'type': kind})
@@ -1892,6 +1899,10 @@ def render_template(event_type: str, data: Dict[str, Any],
template['title'] = runtime_message('backup.confirmedTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.confirmedBody', language)
elif outcome == 'completed_with_warnings':
template['title'] = runtime_message('backup.warningTitle', language,
hostname=data.get('hostname') or _get_hostname())
template['body'] = runtime_message('backup.warningBody', language)
elif outcome == 'failed':
template['title'] = runtime_message('backup.errorTitle', language,
hostname=data.get('hostname') or _get_hostname())
@@ -1899,7 +1910,7 @@ def render_template(event_type: str, data: Dict[str, Any],
if event_type == 'backup_fail':
template['title'] = runtime_message('backup.errorTitle', language,
hostname=data.get('hostname') or _get_hostname())
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
guests = (parsed_backup or {}).get('vms') or []
@@ -1950,7 +1961,7 @@ def render_template(event_type: str, data: Dict[str, Any],
'log_file': '',
}
variables.update(data)
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
# The provider has already substituted raw Display Names. Insert the
# resolved title as a value, never reinterpret its literal braces.
variables['_backup_title'] = template['title']
@@ -2059,6 +2070,14 @@ def render_template(event_type: str, data: Dict[str, Any],
return ''
safe_vars = _SafeDict(variables)
if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
and data.get('is_recovery') is True
and isinstance(data.get('check_evidence'), dict)
and data['check_evidence'].get('check') == 'cpu_usage'):
safe_vars['_health_title'] = runtime_message('healthRecovery.title', language, **variables)
safe_vars['_health_body'] = runtime_message('healthRecovery.body', language, **variables)
template['title'] = '{_health_title}'
template['body'] = '{_health_body}'
try:
title = template['title'].format_map(safe_vars)
except (ValueError, IndexError):
@@ -2069,6 +2088,30 @@ def render_template(event_type: str, data: Dict[str, Any],
# When the event came from PVE webhook with a full vzdump message,
# parse the table/logs and format a rich body instead of the sparse template.
pve_message = data.get('pve_message', '')
backup_diagnostics = []
def bounded_backup_diagnostics(lines):
# 1024 chars matches the repository's small-channel message convention;
# 8 lines keeps repeated producer warnings readable. Inventory/title
# size is separate: this is not a one-Telegram-message guarantee.
unique = list(dict.fromkeys(line for line in lines if line.strip()))
principal = next((line for line in unique if re.search(r'\b(?:ERROR:|TASK ERROR:)', line, re.IGNORECASE)), None)
if principal:
unique.remove(principal)
unique.insert(0, principal)
shown, budget, omitted = [], 1024, 0
for line in unique:
if len(shown) >= 8 or budget < 2:
omitted += 1
continue
line_budget = min(budget, 512)
rendered = line if len(line) <= line_budget else line[:line_budget - 1] + '…'
omitted += int(rendered != line)
shown.append(rendered)
budget -= len(rendered) + 1
if omitted:
shown.append(runtime_message('backup.diagnosticsOmitted', language, count=omitted))
return '\n'.join(shown)
# Check for custom formatter function
formatter_name = template.get('formatter')
@@ -2099,15 +2142,15 @@ def render_template(event_type: str, data: Dict[str, Any],
pve_message, re.MULTILINE | re.DOTALL)
if error_block:
diagnostic_lines = error_block.group(1).rstrip('\r\n').splitlines() + diagnostic_lines
if diagnostic_lines:
body_text += '\n' + '\n'.join(
line for line in dict.fromkeys(diagnostic_lines)
if line.strip() and line not in body_text.splitlines())
backup_diagnostics = [line for line in diagnostic_lines
if line.strip() and line not in body_text.splitlines()]
else:
# Couldn't parse -- use PVE raw message as body
body_text = pve_message.strip()
# Unparsed diagnostic-only reports remain visible but bounded.
body_text = ''
backup_diagnostics = pve_message.strip().splitlines()
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
else 'backup.unconfirmedBody')
body_text = runtime_message(key, language) + '\n' + body_text
elif event_type == 'system_mail' and pve_message:
@@ -2138,8 +2181,20 @@ def render_template(event_type: str, data: Dict[str, Any],
if event_type in ('backup_complete', 'backup_fail') and (
event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
source_subject = str(data.get('pve_title') or '').strip()
guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
if guest_context:
# Native single-line job errors live only in the subject. Retain
# that cause, not the redundant job/host envelope or generic count.
cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
source_subject = cause.group(1).strip() if cause else ''
if source_subject.lower() == 'multiple problems':
source_subject = ''
if source_subject and source_subject not in body_text:
body_text += '\n' + source_subject
# A unique job/setup cause must survive a warning-heavy report.
backup_diagnostics.insert(0, source_subject)
if backup_diagnostics:
body_text += '\n' + bounded_backup_diagnostics(backup_diagnostics)
# Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
import re as _re
@@ -2472,9 +2527,14 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
severity = data.get('severity', 'INFO')
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
and data.get('is_recovery') is True
and isinstance(data.get('check_evidence'), dict)
and data['check_evidence'].get('check') == 'cpu_usage'):
icon = '✅'
if event_type == 'backup_complete':
icon = {
'confirmed': '💾✅', 'failed': '💾❌',
'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
}.get(str(data.get('backup_outcome') or ''), '💾❔')
# Build enriched title: replace severity circle with event-specific icon
@@ -74,7 +74,10 @@ class RuntimeCatalogTests(unittest.TestCase):
en = flatten(self.catalogs["en"])
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
"channels.email.severity.observation", "channels.email.status.unconfirmed"}
"channels.email.severity.observation", "channels.email.status.unconfirmed",
"backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
"channels.email.status.completed_with_warnings",
"healthRecovery.title", "healthRecovery.body", "healthRecovery.status"}
for language, catalog in self.catalogs.items():
translated = flatten(catalog)
if language == 'sk':