mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-09 06:56:37 +00:00
fix(notifications): preserve evidenced outcomes in backup and health notices
This commit is contained in:
martino
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Fehlgeschlagen",
|
||||
"completed": "Abgeschlossen",
|
||||
"started": "Gestartet",
|
||||
"unconfirmed": "Nicht bestätigt"
|
||||
"unconfirmed": "Nicht bestätigt",
|
||||
"completed_with_warnings": "Mit Warnungen abgeschlossen"
|
||||
},
|
||||
"report": "{group} Bericht",
|
||||
"details": "Details",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "Die hohen Messwerte erstrecken sich über {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname}: Behoben - {category}{entity_suffix}", "body": "Eine aktuelle Zustandsprüfung hat für {category} wieder einen normalen Zustand festgestellt.\nVorherige Beobachtung: {reason}\nVorheriger Schweregrad: {original_severity}\nZeit seit der ersten Beobachtung: {duration}", "status": "Behoben"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname}: Backup abgeschlossen",
|
||||
"confirmedBody": "Backup erfolgreich abgeschlossen.",
|
||||
"errorTitle": "{hostname}: Backup-Fehler gemeldet",
|
||||
"errorBody": "Der Backup-Bericht enthält einen Fehler.",
|
||||
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt."
|
||||
"unconfirmedBody": "Das Backup-Ergebnis ist nicht bestätigt.",
|
||||
"warningTitle": "{hostname}: Sicherung mit Warnungen abgeschlossen",
|
||||
"warningBody": "Sicherung mit Warnungen abgeschlossen.",
|
||||
"diagnosticsOmitted": "Weitere Diagnosezeilen oder Text ausgelassen: {count}. Originalbericht bleibt erhalten."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Fallido",
|
||||
"completed": "Completado",
|
||||
"started": "Iniciado",
|
||||
"unconfirmed": "Sin confirmar"
|
||||
"unconfirmed": "Sin confirmar",
|
||||
"completed_with_warnings": "Completado con advertencias"
|
||||
},
|
||||
"report": "Informe de {group}",
|
||||
"details": "Detalles",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "Lecturas altas registradas a lo largo de {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname}: Resuelto - {category}{entity_suffix}", "body": "Una comprobación reciente confirma que la condición de {category} volvió a la normalidad.\nObservación anterior: {reason}\nGravedad anterior: {original_severity}\nTiempo desde la primera observación: {duration}", "status": "Resuelto"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname}: backup completado",
|
||||
"confirmedBody": "Backup completado correctamente.",
|
||||
"errorTitle": "{hostname}: error notificado en el backup",
|
||||
"errorBody": "El informe del backup contiene un error.",
|
||||
"unconfirmedBody": "El resultado del backup no está confirmado."
|
||||
"unconfirmedBody": "El resultado del backup no está confirmado.",
|
||||
"warningTitle": "{hostname}: Backup completado con advertencias",
|
||||
"warningBody": "Backup completado con advertencias.",
|
||||
"diagnosticsOmitted": "Líneas o texto de diagnóstico omitidos: {count}. Se conserva el informe original."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Échec",
|
||||
"completed": "Terminé",
|
||||
"started": "Commencé",
|
||||
"unconfirmed": "Non confirmé"
|
||||
"unconfirmed": "Non confirmé",
|
||||
"completed_with_warnings": "Terminée avec avertissements"
|
||||
},
|
||||
"report": "Rapport {group}",
|
||||
"details": "Détails",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "Les relevés élevés s'étendent sur {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname} : Résolu - {category}{entity_suffix}", "body": "Un contrôle récent confirme le retour à la normale de la condition {category}.\nObservation précédente : {reason}\nGravité précédente : {original_severity}\nTemps depuis la première observation : {duration}", "status": "Résolu"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname} : sauvegarde terminée",
|
||||
"confirmedBody": "Sauvegarde terminée avec succès.",
|
||||
"errorTitle": "{hostname} : erreur signalée lors de la sauvegarde",
|
||||
"errorBody": "Le rapport de sauvegarde contient une erreur.",
|
||||
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé."
|
||||
"unconfirmedBody": "Le résultat de la sauvegarde n’est pas confirmé.",
|
||||
"warningTitle": "{hostname}: Sauvegarde terminée avec avertissements",
|
||||
"warningBody": "Sauvegarde terminée avec avertissements.",
|
||||
"diagnosticsOmitted": "Lignes ou texte de diagnostic omis : {count}. Rapport original conservé."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Fallito",
|
||||
"completed": "Completato",
|
||||
"started": "Iniziato",
|
||||
"unconfirmed": "Non confermato"
|
||||
"unconfirmed": "Non confermato",
|
||||
"completed_with_warnings": "Completato con avvisi"
|
||||
},
|
||||
"report": "{group} Rapporto",
|
||||
"details": "Dettagli",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "Intervallo dei campioni sopra soglia: {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname}: Risolto - {category}{entity_suffix}", "body": "Un controllo recente conferma che la condizione {category} è tornata nella norma.\nOsservazione precedente: {reason}\nGravità precedente: {original_severity}\nTempo dalla prima osservazione: {duration}", "status": "Risolto"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname}: backup completato",
|
||||
"confirmedBody": "Backup completato correttamente.",
|
||||
"errorTitle": "{hostname}: errore segnalato nel backup",
|
||||
"errorBody": "Il rapporto del backup contiene un errore.",
|
||||
"unconfirmedBody": "L’esito del backup non è confermato."
|
||||
"unconfirmedBody": "L’esito del backup non è confermato.",
|
||||
"warningTitle": "{hostname}: Backup completato con avvisi",
|
||||
"warningBody": "Backup completato con avvisi.",
|
||||
"diagnosticsOmitted": "Righe o testo diagnostico omessi: {count}. Il report originale è conservato."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Falhou",
|
||||
"completed": "Concluído",
|
||||
"started": "Iniciado",
|
||||
"unconfirmed": "Não confirmado"
|
||||
"unconfirmed": "Não confirmado",
|
||||
"completed_with_warnings": "Concluído com avisos"
|
||||
},
|
||||
"report": "Relatório {group}",
|
||||
"details": "Detalhes",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "As amostras elevadas abrangem {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname}: Resolvido - {category}{entity_suffix}", "body": "Uma verificação recente confirma que a condição de {category} voltou ao normal.\nObservação anterior: {reason}\nGravidade anterior: {original_severity}\nTempo desde a primeira observação: {duration}", "status": "Resolvido"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname}: backup concluído",
|
||||
"confirmedBody": "Backup concluído com sucesso.",
|
||||
"errorTitle": "{hostname}: erro comunicado no backup",
|
||||
"errorBody": "O relatório do backup contém um erro.",
|
||||
"unconfirmedBody": "O resultado do backup não está confirmado."
|
||||
"unconfirmedBody": "O resultado do backup não está confirmado.",
|
||||
"warningTitle": "{hostname}: Backup concluído com avisos",
|
||||
"warningBody": "Backup concluído com avisos.",
|
||||
"diagnosticsOmitted": "Linhas ou texto de diagnóstico omitidos: {count}. Relatório original preservado."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6980,7 +6980,8 @@
|
||||
"failed": "Misslyckades",
|
||||
"completed": "Klar",
|
||||
"started": "Startat",
|
||||
"unconfirmed": "Obekräftat"
|
||||
"unconfirmed": "Obekräftat",
|
||||
"completed_with_warnings": "Klar med varningar"
|
||||
},
|
||||
"report": "{group} Rapportera",
|
||||
"details": "Detaljer",
|
||||
@@ -6991,12 +6992,16 @@
|
||||
"temperature": {
|
||||
"sampleSpan": "De höga mätvärdena sträcker sig över {duration}."
|
||||
},
|
||||
"healthRecovery": {"title": "{hostname}: Åtgärdat - {category}{entity_suffix}", "body": "En aktuell hälsokontroll bekräftar att tillståndet för {category} återgått till det normala.\nTidigare observation: {reason}\nTidigare allvarlighetsgrad: {original_severity}\nTid sedan första observationen: {duration}", "status": "Åtgärdat"},
|
||||
"backup": {
|
||||
"confirmedTitle": "{hostname}: säkerhetskopiering klar",
|
||||
"confirmedBody": "Säkerhetskopieringen slutfördes utan fel.",
|
||||
"errorTitle": "{hostname}: fel rapporterat vid säkerhetskopiering",
|
||||
"errorBody": "Rapporten om säkerhetskopieringen innehåller ett fel.",
|
||||
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat."
|
||||
"unconfirmedBody": "Säkerhetskopieringens resultat är inte bekräftat.",
|
||||
"warningTitle": "{hostname}: Säkerhetskopiering klar med varningar",
|
||||
"warningBody": "Säkerhetskopiering klar med varningar.",
|
||||
"diagnosticsOmitted": "Utelämnade diagnosrader eller text: {count}. Originalrapporten bevaras."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1452,7 +1452,11 @@ class HealthMonitor:
|
||||
status = 'OK'
|
||||
reason = None
|
||||
# CPU is normal - auto-resolve any existing CPU errors
|
||||
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal')
|
||||
evidence = None
|
||||
if cpu_percent < self.CPU_RECOVERY and len(recovery_samples) >= RECOVERY_MIN_SAMPLES:
|
||||
evidence = {'check': 'cpu_usage', 'checked_at': current_time}
|
||||
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal',
|
||||
check_evidence=evidence)
|
||||
|
||||
temp_status = self._check_cpu_temperature()
|
||||
|
||||
|
||||
@@ -744,12 +744,12 @@ class HealthPersistence:
|
||||
|
||||
return event_info
|
||||
|
||||
def resolve_error(self, error_key: str, reason: str = 'auto-resolved'):
|
||||
def resolve_error(self, error_key: str, reason: str = 'auto-resolved', *, check_evidence=None):
|
||||
"""Mark an error as resolved"""
|
||||
with self._db_lock:
|
||||
return self._resolve_error_impl(error_key, reason)
|
||||
return self._resolve_error_impl(error_key, reason, check_evidence=check_evidence)
|
||||
|
||||
def _resolve_error_impl(self, error_key, reason):
|
||||
def _resolve_error_impl(self, error_key, reason, *, check_evidence=None):
|
||||
with self._db_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
now = datetime.now().isoformat()
|
||||
@@ -788,12 +788,59 @@ class HealthPersistence:
|
||||
stored_details = None
|
||||
self._record_event(cursor, 'resolved', error_key, {
|
||||
'reason': reason,
|
||||
# Only explicit current-check callers attach this proof.
|
||||
# Generic resolve/cleanup remains neutral.
|
||||
'check_evidence': check_evidence,
|
||||
'entity': self._entity_from_details(stored_details),
|
||||
'details': stored_details or {},
|
||||
})
|
||||
|
||||
conn.commit()
|
||||
|
||||
def get_recovery_evidence(self, error_key: str, first_seen: str):
|
||||
"""Return fresh same-incident native check proof, never absence of errors.
|
||||
|
||||
Initially only the host CPU check has a stable condition identity. Other
|
||||
checks, generic clears, excluded/deleted records and legacy events stay
|
||||
neutral until they have equivalent per-condition provenance.
|
||||
"""
|
||||
if error_key != 'cpu_usage' or not first_seen:
|
||||
return None
|
||||
try:
|
||||
with self._db_connection() as conn:
|
||||
row = conn.execute('''
|
||||
SELECT first_seen, last_seen, resolved_at, acknowledged
|
||||
FROM errors WHERE error_key = ? ORDER BY id DESC LIMIT 1
|
||||
''', (error_key,)).fetchone()
|
||||
if not row or row[0] != first_seen or not row[2] or row[3]:
|
||||
return None
|
||||
event = conn.execute('''
|
||||
SELECT timestamp, data FROM events
|
||||
WHERE error_key = ? AND event_type = 'resolved'
|
||||
ORDER BY id DESC LIMIT 1
|
||||
''', (error_key,)).fetchone()
|
||||
if not event:
|
||||
return None
|
||||
proof = json.loads(event[1]).get('check_evidence')
|
||||
if not isinstance(proof, dict) or proof.get('check') != error_key:
|
||||
return None
|
||||
checked = proof.get('checked_at')
|
||||
if not isinstance(checked, (int, float)) or isinstance(checked, bool):
|
||||
return None
|
||||
checked = float(checked)
|
||||
# Reuse the collector's existing two-hour freshness boundary.
|
||||
now = datetime.now().timestamp()
|
||||
if not 0 <= now - checked <= 7200:
|
||||
return None
|
||||
last_seen = datetime.fromisoformat(row[1]).timestamp()
|
||||
resolved = datetime.fromisoformat(row[2]).timestamp()
|
||||
recorded = datetime.fromisoformat(event[0]).timestamp()
|
||||
if not last_seen <= checked <= resolved <= recorded:
|
||||
return None
|
||||
return proof
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
return None
|
||||
|
||||
def is_error_active(self, error_key: str, category: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if an error is currently active OR suppressed (dismissed but within suppression period).
|
||||
|
||||
@@ -1038,13 +1038,23 @@ class EmailChannel(NotificationChannel):
|
||||
# Determine group for section header
|
||||
event_type = data.get('_event_type', '')
|
||||
if event_type == 'error_resolved':
|
||||
sev.update(self._SEV_DEFAULT)
|
||||
sev['label'] = _runtime_text('email.severity.observation', data)
|
||||
if (data.get('recovery_outcome') == 'resolved'
|
||||
and data.get('is_recovery') is True
|
||||
and isinstance(data.get('check_evidence'), dict)
|
||||
and data['check_evidence'].get('check') == 'cpu_usage'):
|
||||
sev.update(self._SEV_STYLE['OK'])
|
||||
sev['label'] = _runtime_notification_text('healthRecovery.status', data)
|
||||
else:
|
||||
sev.update(self._SEV_DEFAULT)
|
||||
sev['label'] = _runtime_text('email.severity.observation', data)
|
||||
elif event_type == 'backup_complete':
|
||||
outcome = data.get('backup_outcome')
|
||||
if outcome == 'confirmed':
|
||||
sev.update(self._SEV_STYLE['OK'])
|
||||
status = 'completed'
|
||||
elif outcome == 'completed_with_warnings':
|
||||
sev.update(self._SEV_STYLE['WARNING'])
|
||||
status = 'completed_with_warnings'
|
||||
elif outcome == 'failed':
|
||||
sev.update(self._SEV_STYLE['CRITICAL'])
|
||||
status = 'failed'
|
||||
@@ -1057,7 +1067,7 @@ class EmailChannel(NotificationChannel):
|
||||
# bodies, including restore bodies released from quiet hours.
|
||||
backup_email = event_type in {'backup_complete', 'backup_fail'}
|
||||
wrap_body = (event_type in {'temp_high', 'system_restore_completed', 'error_resolved'}
|
||||
or backup_email or data.get('_restore_summary'))
|
||||
or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
|
||||
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
|
||||
temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
|
||||
backup_title_wrap = temp_cell_wrap if backup_email else ''
|
||||
@@ -1103,7 +1113,7 @@ class EmailChannel(NotificationChannel):
|
||||
# Observation age/disappearance must not become a green OK row.
|
||||
# The endpoint's warnings_block and task counts live in the
|
||||
# localized body, not the generic services Event row.
|
||||
detail_rows = [('', html_mod.escape(line.strip()))
|
||||
detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
|
||||
for line in body.split('\n') if line.strip()]
|
||||
|
||||
# ── Fallback: if no structured rows, render body text lines ──
|
||||
@@ -1122,6 +1132,7 @@ class EmailChannel(NotificationChannel):
|
||||
|
||||
# ── Render detail rows as HTML table ──
|
||||
rows_html = ''
|
||||
summary_whitespace = 'white-space:pre-wrap;' if data.get('_quiet_hours_summary') and data.get('_restore_summary') else ''
|
||||
for label, value in detail_rows:
|
||||
if label:
|
||||
rows_html += f'''<tr>
|
||||
@@ -1131,7 +1142,7 @@ class EmailChannel(NotificationChannel):
|
||||
else:
|
||||
# Full-width row (no label, just description text)
|
||||
rows_html += f'''<tr>
|
||||
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}">{value}</td>
|
||||
<td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}{summary_whitespace}">{value}</td>
|
||||
</tr>'''
|
||||
|
||||
# ── Reason / details block (long text, displayed separately) ──
|
||||
@@ -1291,6 +1302,7 @@ class EmailChannel(NotificationChannel):
|
||||
_add('Storage', data.get('storage') or data.get('storage_name'), 'code')
|
||||
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
|
||||
status_key = ('failed' if data.get('backup_outcome') == 'failed'
|
||||
else 'completed_with_warnings' if data.get('backup_outcome') == 'completed_with_warnings'
|
||||
else 'unconfirmed')
|
||||
else:
|
||||
status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'
|
||||
|
||||
@@ -3223,6 +3223,13 @@ class PollingCollector:
|
||||
self._last_notified.pop(key, None)
|
||||
continue
|
||||
|
||||
# Disappearance is not recovery. Only same-incident proof written
|
||||
# by a successful existing native check can certify normality.
|
||||
try:
|
||||
recovery_evidence = health_persistence.get_recovery_evidence(key, first_seen)
|
||||
except Exception:
|
||||
recovery_evidence = None
|
||||
|
||||
# Calculate duration
|
||||
duration = ''
|
||||
if first_seen:
|
||||
@@ -3273,6 +3280,9 @@ class PollingCollector:
|
||||
else:
|
||||
clean_reason = 'Condition no longer reported'
|
||||
|
||||
if recovery_evidence:
|
||||
clean_reason = reason_summary
|
||||
|
||||
# `original_severity` must match what the user actually saw
|
||||
# in the most-recent notification for this error, not the
|
||||
# latest DB severity. See `_notified_severity` docstring at
|
||||
@@ -3299,7 +3309,9 @@ class PollingCollector:
|
||||
'original_severity': original_severity,
|
||||
'first_seen': first_seen,
|
||||
'duration': duration_label,
|
||||
'is_recovery': True,
|
||||
'is_recovery': bool(recovery_evidence),
|
||||
'recovery_outcome': 'resolved' if recovery_evidence else 'no_longer_reported',
|
||||
'check_evidence': recovery_evidence,
|
||||
}
|
||||
# Spread the original details blob so the resolved notification
|
||||
# can use the same {storage_name}/{vm_name}/{device} placeholders
|
||||
@@ -4302,14 +4314,16 @@ class ProxmoxHookWatcher:
|
||||
table = _parse_vzdump_table(text)
|
||||
if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
|
||||
return 'failed'
|
||||
if severity not in ('info', 'ok', 'success') or re.search(
|
||||
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*\d+)', text):
|
||||
warnings = severity in ('warning', 'warn') or bool(re.search(
|
||||
r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*[1-9]\d*)', text))
|
||||
if severity not in ('info', 'ok', 'success', 'warning', 'warn'):
|
||||
return 'unconfirmed'
|
||||
completed = 'completed_with_warnings' if warnings else 'confirmed'
|
||||
# A present table is authoritative: do not certify an incomplete table
|
||||
# from a finished guest log, or reject a complete OK table merely
|
||||
# because the extra diagnostic log was truncated before its finishes.
|
||||
if table is not None:
|
||||
return ('confirmed' if table['complete'] and
|
||||
return (completed if table['complete'] and
|
||||
all(guest['status'].lower() == 'ok' for guest in table['vms'])
|
||||
else 'unconfirmed')
|
||||
if starts:
|
||||
@@ -4322,10 +4336,10 @@ class ProxmoxHookWatcher:
|
||||
return 'unconfirmed' # A finish before its start is not evidence.
|
||||
else:
|
||||
pending[vmid] -= 1
|
||||
return 'confirmed' if not any(pending.values()) else 'unconfirmed'
|
||||
return completed if not any(pending.values()) else 'unconfirmed'
|
||||
if re.search(
|
||||
r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
|
||||
return 'confirmed'
|
||||
return completed
|
||||
return 'unconfirmed'
|
||||
|
||||
def process_webhook(self, payload: dict) -> dict:
|
||||
|
||||
@@ -1364,6 +1364,12 @@ class NotificationManager:
|
||||
|
||||
# Get journal context if available (will be enriched per-channel based on detail_level)
|
||||
raw_journal_context = data.get('_journal_context', '')
|
||||
# Persist a presentation token in the existing title column: buffers
|
||||
# otherwise discard outcome metadata before composition. Old rows
|
||||
# without a token remain neutral; routing and schema are unchanged.
|
||||
buffer_title = title
|
||||
if event_type in ('backup_complete', 'backup_fail'):
|
||||
buffer_title, _ = enrich_with_emojis(event_type, title, '', data)
|
||||
|
||||
for ch_name, channel in channels.items():
|
||||
# ── Per-channel category check ──
|
||||
@@ -1393,7 +1399,7 @@ class NotificationManager:
|
||||
# delivered after Quiet Hours + Daily Digest were merged.
|
||||
if severity != 'CRITICAL' and self._in_quiet_hours(ch_name):
|
||||
self._buffer_quiet_event(ch_name, event_type, event_group,
|
||||
severity, title, body)
|
||||
severity, buffer_title, body)
|
||||
continue
|
||||
|
||||
# ── Per-channel daily digest ──
|
||||
@@ -1406,7 +1412,7 @@ class NotificationManager:
|
||||
# excluded from the digest by `_DIGEST_EXEMPT_EVENTS`.
|
||||
if self._should_buffer_for_digest(ch_name, severity, event_type):
|
||||
self._buffer_digest_event(ch_name, event_type, event_group,
|
||||
severity, title, body)
|
||||
severity, buffer_title, body)
|
||||
continue
|
||||
|
||||
try:
|
||||
@@ -1849,14 +1855,22 @@ class NotificationManager:
|
||||
if index < 8 or (quiet_release and item[1] == 'system_restore_completed')]
|
||||
for ts, ev_type, title, body in visible_items:
|
||||
hhmm = datetime.fromtimestamp(ts).strftime('%H:%M')
|
||||
backup_icon = ''
|
||||
if ev_type in ('backup_complete', 'backup_fail'):
|
||||
for token in ('💾✅', '💾⚠️', '💾❌', '💾❔', '💾'):
|
||||
if title.startswith(token + ' '):
|
||||
backup_icon = token
|
||||
title = title[len(token) + 1:]
|
||||
break
|
||||
backup_icon = backup_icon or ('💾❌' if ev_type == 'backup_fail' else '💾❔')
|
||||
short_title = title.split(': ', 1)[-1] if ': ' in title else title
|
||||
event_icon = (
|
||||
EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
|
||||
backup_icon or EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
|
||||
) if use_icons else ''
|
||||
event_prefix = f'{event_icon} ' if event_icon else ''
|
||||
lines.append(f" • {event_prefix}{hhmm} {short_title}")
|
||||
if quiet_release and ev_type == 'system_restore_completed' and body:
|
||||
lines.extend(line for line in body.splitlines() if line.strip())
|
||||
lines.extend(' ' + line.strip() for line in body.splitlines() if line.strip())
|
||||
if len(items) > len(visible_items):
|
||||
lines.append(runtime_message('digest.more', language, count=len(items) - len(visible_items)))
|
||||
lines.append('')
|
||||
@@ -2006,6 +2020,7 @@ class NotificationManager:
|
||||
summary_title, summary_body, severity='INFO',
|
||||
data={'_quiet_hours_summary': True, '_count': len(rows),
|
||||
'_restore_summary': any(row[1] == 'system_restore_completed' for row in rows),
|
||||
'_backup_summary': any(row[1] in ('backup_complete', 'backup_fail') for row in rows),
|
||||
'_notification_language': language},
|
||||
) or result
|
||||
except Exception as e:
|
||||
|
||||
@@ -244,6 +244,13 @@ def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
|
||||
status = 'error'
|
||||
kind = ('lxc' if 'lxc' in filename or filename.startswith('ct/') else
|
||||
'qemu' if 'qemu' in filename or filename.startswith('vm/') else '')
|
||||
if filename.lower() == 'null':
|
||||
# Native failed rows lack archives. Match only this row's VMID;
|
||||
# multiple inconsistent starts are not authoritative identity.
|
||||
kinds = set(re.findall(
|
||||
r'(?im)\bStarting Backup of VM ' + re.escape(vmid) + r'\s+\((lxc|qemu)\)',
|
||||
message))
|
||||
kind = kinds.pop() if len(kinds) == 1 else ''
|
||||
rows.append({'vmid': vmid, 'name': name, 'status': status,
|
||||
'time': duration, 'size': size, 'filename': filename,
|
||||
'type': kind})
|
||||
@@ -1892,6 +1899,10 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
template['title'] = runtime_message('backup.confirmedTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
template['body'] = runtime_message('backup.confirmedBody', language)
|
||||
elif outcome == 'completed_with_warnings':
|
||||
template['title'] = runtime_message('backup.warningTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
template['body'] = runtime_message('backup.warningBody', language)
|
||||
elif outcome == 'failed':
|
||||
template['title'] = runtime_message('backup.errorTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
@@ -1899,7 +1910,7 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
if event_type == 'backup_fail':
|
||||
template['title'] = runtime_message('backup.errorTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname())
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
|
||||
parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
|
||||
storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
|
||||
guests = (parsed_backup or {}).get('vms') or []
|
||||
@@ -1950,7 +1961,7 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
'log_file': '',
|
||||
}
|
||||
variables.update(data)
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
|
||||
# The provider has already substituted raw Display Names. Insert the
|
||||
# resolved title as a value, never reinterpret its literal braces.
|
||||
variables['_backup_title'] = template['title']
|
||||
@@ -2059,6 +2070,14 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
return ''
|
||||
|
||||
safe_vars = _SafeDict(variables)
|
||||
if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
|
||||
and data.get('is_recovery') is True
|
||||
and isinstance(data.get('check_evidence'), dict)
|
||||
and data['check_evidence'].get('check') == 'cpu_usage'):
|
||||
safe_vars['_health_title'] = runtime_message('healthRecovery.title', language, **variables)
|
||||
safe_vars['_health_body'] = runtime_message('healthRecovery.body', language, **variables)
|
||||
template['title'] = '{_health_title}'
|
||||
template['body'] = '{_health_body}'
|
||||
try:
|
||||
title = template['title'].format_map(safe_vars)
|
||||
except (ValueError, IndexError):
|
||||
@@ -2069,6 +2088,30 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
# When the event came from PVE webhook with a full vzdump message,
|
||||
# parse the table/logs and format a rich body instead of the sparse template.
|
||||
pve_message = data.get('pve_message', '')
|
||||
backup_diagnostics = []
|
||||
|
||||
def bounded_backup_diagnostics(lines):
|
||||
# 1024 chars matches the repository's small-channel message convention;
|
||||
# 8 lines keeps repeated producer warnings readable. Inventory/title
|
||||
# size is separate: this is not a one-Telegram-message guarantee.
|
||||
unique = list(dict.fromkeys(line for line in lines if line.strip()))
|
||||
principal = next((line for line in unique if re.search(r'\b(?:ERROR:|TASK ERROR:)', line, re.IGNORECASE)), None)
|
||||
if principal:
|
||||
unique.remove(principal)
|
||||
unique.insert(0, principal)
|
||||
shown, budget, omitted = [], 1024, 0
|
||||
for line in unique:
|
||||
if len(shown) >= 8 or budget < 2:
|
||||
omitted += 1
|
||||
continue
|
||||
line_budget = min(budget, 512)
|
||||
rendered = line if len(line) <= line_budget else line[:line_budget - 1] + '…'
|
||||
omitted += int(rendered != line)
|
||||
shown.append(rendered)
|
||||
budget -= len(rendered) + 1
|
||||
if omitted:
|
||||
shown.append(runtime_message('backup.diagnosticsOmitted', language, count=omitted))
|
||||
return '\n'.join(shown)
|
||||
|
||||
# Check for custom formatter function
|
||||
formatter_name = template.get('formatter')
|
||||
@@ -2099,15 +2142,15 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
pve_message, re.MULTILINE | re.DOTALL)
|
||||
if error_block:
|
||||
diagnostic_lines = error_block.group(1).rstrip('\r\n').splitlines() + diagnostic_lines
|
||||
if diagnostic_lines:
|
||||
body_text += '\n' + '\n'.join(
|
||||
line for line in dict.fromkeys(diagnostic_lines)
|
||||
if line.strip() and line not in body_text.splitlines())
|
||||
backup_diagnostics = [line for line in diagnostic_lines
|
||||
if line.strip() and line not in body_text.splitlines()]
|
||||
else:
|
||||
# Couldn't parse -- use PVE raw message as body
|
||||
body_text = pve_message.strip()
|
||||
# Unparsed diagnostic-only reports remain visible but bounded.
|
||||
body_text = ''
|
||||
backup_diagnostics = pve_message.strip().splitlines()
|
||||
if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
|
||||
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
|
||||
else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
|
||||
else 'backup.unconfirmedBody')
|
||||
body_text = runtime_message(key, language) + '\n' + body_text
|
||||
elif event_type == 'system_mail' and pve_message:
|
||||
@@ -2138,8 +2181,20 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
if event_type in ('backup_complete', 'backup_fail') and (
|
||||
event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
|
||||
source_subject = str(data.get('pve_title') or '').strip()
|
||||
guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
|
||||
if guest_context:
|
||||
# Native single-line job errors live only in the subject. Retain
|
||||
# that cause, not the redundant job/host envelope or generic count.
|
||||
cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
|
||||
source_subject = cause.group(1).strip() if cause else ''
|
||||
if source_subject.lower() == 'multiple problems':
|
||||
source_subject = ''
|
||||
if source_subject and source_subject not in body_text:
|
||||
body_text += '\n' + source_subject
|
||||
# A unique job/setup cause must survive a warning-heavy report.
|
||||
backup_diagnostics.insert(0, source_subject)
|
||||
|
||||
if backup_diagnostics:
|
||||
body_text += '\n' + bounded_backup_diagnostics(backup_diagnostics)
|
||||
|
||||
# Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
|
||||
import re as _re
|
||||
@@ -2472,9 +2527,14 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
|
||||
severity = data.get('severity', 'INFO')
|
||||
|
||||
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
|
||||
if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
|
||||
and data.get('is_recovery') is True
|
||||
and isinstance(data.get('check_evidence'), dict)
|
||||
and data['check_evidence'].get('check') == 'cpu_usage'):
|
||||
icon = '✅'
|
||||
if event_type == 'backup_complete':
|
||||
icon = {
|
||||
'confirmed': '💾✅', 'failed': '💾❌',
|
||||
'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
|
||||
}.get(str(data.get('backup_outcome') or ''), '💾❔')
|
||||
|
||||
# Build enriched title: replace severity circle with event-specific icon
|
||||
|
||||
@@ -74,7 +74,10 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
en = flatten(self.catalogs["en"])
|
||||
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
|
||||
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
|
||||
"channels.email.severity.observation", "channels.email.status.unconfirmed"}
|
||||
"channels.email.severity.observation", "channels.email.status.unconfirmed",
|
||||
"backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
|
||||
"channels.email.status.completed_with_warnings",
|
||||
"healthRecovery.title", "healthRecovery.body", "healthRecovery.status"}
|
||||
for language, catalog in self.catalogs.items():
|
||||
translated = flatten(catalog)
|
||||
if language == 'sk':
|
||||
|
||||
Reference in New Issue
Block a user