fix(notifications): preserve evidenced outcomes in backup and health notices

AppImage/scripts/health_monitor.py

@@ -1452,7 +1452,11 @@ class HealthMonitor:
                 status = 'OK'
                 reason = None
                 # CPU is normal - auto-resolve any existing CPU errors
-                health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal')
+                evidence = None
+                if cpu_percent < self.CPU_RECOVERY and len(recovery_samples) >= RECOVERY_MIN_SAMPLES:
+                    evidence = {'check': 'cpu_usage', 'checked_at': current_time}
+                health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal',
+                                                 check_evidence=evidence)
             
             temp_status = self._check_cpu_temperature()
             

AppImage/scripts/health_persistence.py

@@ -744,12 +744,12 @@ class HealthPersistence:
         
         return event_info
     
-    def resolve_error(self, error_key: str, reason: str = 'auto-resolved'):
+    def resolve_error(self, error_key: str, reason: str = 'auto-resolved', *, check_evidence=None):
         """Mark an error as resolved"""
         with self._db_lock:
-            return self._resolve_error_impl(error_key, reason)
+            return self._resolve_error_impl(error_key, reason, check_evidence=check_evidence)
     
-    def _resolve_error_impl(self, error_key, reason):
+    def _resolve_error_impl(self, error_key, reason, *, check_evidence=None):
         with self._db_connection() as conn:
             cursor = conn.cursor()
             now = datetime.now().isoformat()
@@ -788,12 +788,59 @@ class HealthPersistence:
                         stored_details = None
                 self._record_event(cursor, 'resolved', error_key, {
                     'reason': reason,
+                    # Only explicit current-check callers attach this proof.
+                    # Generic resolve/cleanup remains neutral.
+                    'check_evidence': check_evidence,
                     'entity': self._entity_from_details(stored_details),
                     'details': stored_details or {},
                 })
 
             conn.commit()
     
+    def get_recovery_evidence(self, error_key: str, first_seen: str):
+        """Return fresh same-incident native check proof, never absence of errors.
+
+        Initially only the host CPU check has a stable condition identity. Other
+        checks, generic clears, excluded/deleted records and legacy events stay
+        neutral until they have equivalent per-condition provenance.
+        """
+        if error_key != 'cpu_usage' or not first_seen:
+            return None
+        try:
+            with self._db_connection() as conn:
+                row = conn.execute('''
+                    SELECT first_seen, last_seen, resolved_at, acknowledged
+                    FROM errors WHERE error_key = ? ORDER BY id DESC LIMIT 1
+                ''', (error_key,)).fetchone()
+                if not row or row[0] != first_seen or not row[2] or row[3]:
+                    return None
+                event = conn.execute('''
+                    SELECT timestamp, data FROM events
+                    WHERE error_key = ? AND event_type = 'resolved'
+                    ORDER BY id DESC LIMIT 1
+                ''', (error_key,)).fetchone()
+            if not event:
+                return None
+            proof = json.loads(event[1]).get('check_evidence')
+            if not isinstance(proof, dict) or proof.get('check') != error_key:
+                return None
+            checked = proof.get('checked_at')
+            if not isinstance(checked, (int, float)) or isinstance(checked, bool):
+                return None
+            checked = float(checked)
+            # Reuse the collector's existing two-hour freshness boundary.
+            now = datetime.now().timestamp()
+            if not 0 <= now - checked <= 7200:
+                return None
+            last_seen = datetime.fromisoformat(row[1]).timestamp()
+            resolved = datetime.fromisoformat(row[2]).timestamp()
+            recorded = datetime.fromisoformat(event[0]).timestamp()
+            if not last_seen <= checked <= resolved <= recorded:
+                return None
+            return proof
+        except (ValueError, TypeError, AttributeError, OverflowError):
+            return None
+
     def is_error_active(self, error_key: str, category: Optional[str] = None) -> bool:
         """
         Check if an error is currently active OR suppressed (dismissed but within suppression period).

AppImage/scripts/notification_channels.py

@@ -1038,13 +1038,23 @@ class EmailChannel(NotificationChannel):
         # Determine group for section header
         event_type = data.get('_event_type', '')
         if event_type == 'error_resolved':
-            sev.update(self._SEV_DEFAULT)
-            sev['label'] = _runtime_text('email.severity.observation', data)
+            if (data.get('recovery_outcome') == 'resolved'
+                    and data.get('is_recovery') is True
+                    and isinstance(data.get('check_evidence'), dict)
+                    and data['check_evidence'].get('check') == 'cpu_usage'):
+                sev.update(self._SEV_STYLE['OK'])
+                sev['label'] = _runtime_notification_text('healthRecovery.status', data)
+            else:
+                sev.update(self._SEV_DEFAULT)
+                sev['label'] = _runtime_text('email.severity.observation', data)
         elif event_type == 'backup_complete':
             outcome = data.get('backup_outcome')
             if outcome == 'confirmed':
                 sev.update(self._SEV_STYLE['OK'])
                 status = 'completed'
+            elif outcome == 'completed_with_warnings':
+                sev.update(self._SEV_STYLE['WARNING'])
+                status = 'completed_with_warnings'
             elif outcome == 'failed':
                 sev.update(self._SEV_STYLE['CRITICAL'])
                 status = 'failed'
@@ -1057,7 +1067,7 @@ class EmailChannel(NotificationChannel):
         # bodies, including restore bodies released from quiet hours.
         backup_email = event_type in {'backup_complete', 'backup_fail'}
         wrap_body = (event_type in {'temp_high', 'system_restore_completed', 'error_resolved'}
-                     or backup_email or data.get('_restore_summary'))
+                     or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
         temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
         temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
         backup_title_wrap = temp_cell_wrap if backup_email else ''
@@ -1103,7 +1113,7 @@ class EmailChannel(NotificationChannel):
             # Observation age/disappearance must not become a green OK row.
             # The endpoint's warnings_block and task counts live in the
             # localized body, not the generic services Event row.
-            detail_rows = [('', html_mod.escape(line.strip()))
+            detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
                            for line in body.split('\n') if line.strip()]
 
         # ── Fallback: if no structured rows, render body text lines ──
@@ -1122,6 +1132,7 @@ class EmailChannel(NotificationChannel):
 
         # ── Render detail rows as HTML table ──
         rows_html = ''
+        summary_whitespace = 'white-space:pre-wrap;' if data.get('_quiet_hours_summary') and data.get('_restore_summary') else ''
         for label, value in detail_rows:
             if label:
                 rows_html += f'''<tr>
@@ -1131,7 +1142,7 @@ class EmailChannel(NotificationChannel):
             else:
                 # Full-width row (no label, just description text)
                 rows_html += f'''<tr>
-  <td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}">{value}</td>
+  <td colspan="2" style="padding:8px 12px;font-size:13px;color:#1f2937;border-bottom:1px solid #e5e7eb;{temp_cell_wrap}{summary_whitespace}">{value}</td>
 </tr>'''
 
         # ── Reason / details block (long text, displayed separately) ──
@@ -1291,6 +1302,7 @@ class EmailChannel(NotificationChannel):
             _add('Storage', data.get('storage') or data.get('storage_name'), 'code')
             if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
                 status_key = ('failed' if data.get('backup_outcome') == 'failed'
+                              else 'completed_with_warnings' if data.get('backup_outcome') == 'completed_with_warnings'
                               else 'unconfirmed')
             else:
                 status_key = 'failed' if 'fail' in event_type else 'completed' if 'complete' in event_type else 'started'

AppImage/scripts/notification_events.py

@@ -3223,6 +3223,13 @@ class PollingCollector:
                     self._last_notified.pop(key, None)
                     continue
             
+            # Disappearance is not recovery. Only same-incident proof written
+            # by a successful existing native check can certify normality.
+            try:
+                recovery_evidence = health_persistence.get_recovery_evidence(key, first_seen)
+            except Exception:
+                recovery_evidence = None
+
             # Calculate duration
             duration = ''
             if first_seen:
@@ -3273,6 +3280,9 @@ class PollingCollector:
             else:
                 clean_reason = 'Condition no longer reported'
             
+            if recovery_evidence:
+                clean_reason = reason_summary
+
             # `original_severity` must match what the user actually saw
             # in the most-recent notification for this error, not the
             # latest DB severity. See `_notified_severity` docstring at
@@ -3299,7 +3309,9 @@ class PollingCollector:
                 'original_severity': original_severity,
                 'first_seen': first_seen,
                 'duration': duration_label,
-                'is_recovery': True,
+                'is_recovery': bool(recovery_evidence),
+                'recovery_outcome': 'resolved' if recovery_evidence else 'no_longer_reported',
+                'check_evidence': recovery_evidence,
             }
             # Spread the original details blob so the resolved notification
             # can use the same {storage_name}/{vm_name}/{device} placeholders
@@ -4302,14 +4314,16 @@ class ProxmoxHookWatcher:
         table = _parse_vzdump_table(text)
         if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
             return 'failed'
-        if severity not in ('info', 'ok', 'success') or re.search(
-                r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*\d+)', text):
+        warnings = severity in ('warning', 'warn') or bool(re.search(
+                r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*[1-9]\d*)', text))
+        if severity not in ('info', 'ok', 'success', 'warning', 'warn'):
             return 'unconfirmed'
+        completed = 'completed_with_warnings' if warnings else 'confirmed'
         # A present table is authoritative: do not certify an incomplete table
         # from a finished guest log, or reject a complete OK table merely
         # because the extra diagnostic log was truncated before its finishes.
         if table is not None:
-            return ('confirmed' if table['complete'] and
+            return (completed if table['complete'] and
                     all(guest['status'].lower() == 'ok' for guest in table['vms'])
                     else 'unconfirmed')
         if starts:
@@ -4322,10 +4336,10 @@ class ProxmoxHookWatcher:
                     return 'unconfirmed'  # A finish before its start is not evidence.
                 else:
                     pending[vmid] -= 1
-            return 'confirmed' if not any(pending.values()) else 'unconfirmed'
+            return completed if not any(pending.values()) else 'unconfirmed'
         if re.search(
                 r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
-            return 'confirmed'
+            return completed
         return 'unconfirmed'
 
     def process_webhook(self, payload: dict) -> dict:

AppImage/scripts/notification_manager.py

@@ -1364,6 +1364,12 @@ class NotificationManager:
         
         # Get journal context if available (will be enriched per-channel based on detail_level)
         raw_journal_context = data.get('_journal_context', '')
+        # Persist a presentation token in the existing title column: buffers
+        # otherwise discard outcome metadata before composition. Old rows
+        # without a token remain neutral; routing and schema are unchanged.
+        buffer_title = title
+        if event_type in ('backup_complete', 'backup_fail'):
+            buffer_title, _ = enrich_with_emojis(event_type, title, '', data)
         
         for ch_name, channel in channels.items():
             # ── Per-channel category check ──
@@ -1393,7 +1399,7 @@ class NotificationManager:
             # delivered after Quiet Hours + Daily Digest were merged.
             if severity != 'CRITICAL' and self._in_quiet_hours(ch_name):
                 self._buffer_quiet_event(ch_name, event_type, event_group,
-                                          severity, title, body)
+                                          severity, buffer_title, body)
                 continue
 
             # ── Per-channel daily digest ──
@@ -1406,7 +1412,7 @@ class NotificationManager:
             # excluded from the digest by `_DIGEST_EXEMPT_EVENTS`.
             if self._should_buffer_for_digest(ch_name, severity, event_type):
                 self._buffer_digest_event(ch_name, event_type, event_group,
-                                          severity, title, body)
+                                          severity, buffer_title, body)
                 continue
             
             try:
@@ -1849,14 +1855,22 @@ class NotificationManager:
                              if index < 8 or (quiet_release and item[1] == 'system_restore_completed')]
             for ts, ev_type, title, body in visible_items:
                 hhmm = datetime.fromtimestamp(ts).strftime('%H:%M')
+                backup_icon = ''
+                if ev_type in ('backup_complete', 'backup_fail'):
+                    for token in ('💾✅', '💾⚠️', '💾❌', '💾❔', '💾'):
+                        if title.startswith(token + ' '):
+                            backup_icon = token
+                            title = title[len(token) + 1:]
+                            break
+                    backup_icon = backup_icon or ('💾❌' if ev_type == 'backup_fail' else '💾❔')
                 short_title = title.split(': ', 1)[-1] if ': ' in title else title
                 event_icon = (
-                    EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
+                    backup_icon or EVENT_EMOJI.get(ev_type) or CATEGORY_EMOJI.get(group, '')
                 ) if use_icons else ''
                 event_prefix = f'{event_icon} ' if event_icon else ''
                 lines.append(f"  • {event_prefix}{hhmm}  {short_title}")
                 if quiet_release and ev_type == 'system_restore_completed' and body:
-                    lines.extend(line for line in body.splitlines() if line.strip())
+                    lines.extend('    ' + line.strip() for line in body.splitlines() if line.strip())
             if len(items) > len(visible_items):
                 lines.append(runtime_message('digest.more', language, count=len(items) - len(visible_items)))
             lines.append('')
@@ -2006,6 +2020,7 @@ class NotificationManager:
                 summary_title, summary_body, severity='INFO',
                 data={'_quiet_hours_summary': True, '_count': len(rows),
                       '_restore_summary': any(row[1] == 'system_restore_completed' for row in rows),
+                      '_backup_summary': any(row[1] in ('backup_complete', 'backup_fail') for row in rows),
                       '_notification_language': language},
             ) or result
         except Exception as e:

AppImage/scripts/notification_templates.py

@@ -244,6 +244,13 @@ def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
                 status = 'error'
             kind = ('lxc' if 'lxc' in filename or filename.startswith('ct/') else
                     'qemu' if 'qemu' in filename or filename.startswith('vm/') else '')
+            if filename.lower() == 'null':
+                # Native failed rows lack archives. Match only this row's VMID;
+                # multiple inconsistent starts are not authoritative identity.
+                kinds = set(re.findall(
+                    r'(?im)\bStarting Backup of VM ' + re.escape(vmid) + r'\s+\((lxc|qemu)\)',
+                    message))
+                kind = kinds.pop() if len(kinds) == 1 else ''
             rows.append({'vmid': vmid, 'name': name, 'status': status,
                          'time': duration, 'size': size, 'filename': filename,
                          'type': kind})
@@ -1892,6 +1899,10 @@ def render_template(event_type: str, data: Dict[str, Any],
             template['title'] = runtime_message('backup.confirmedTitle', language,
                                                 hostname=data.get('hostname') or _get_hostname())
             template['body'] = runtime_message('backup.confirmedBody', language)
+        elif outcome == 'completed_with_warnings':
+            template['title'] = runtime_message('backup.warningTitle', language,
+                                                hostname=data.get('hostname') or _get_hostname())
+            template['body'] = runtime_message('backup.warningBody', language)
         elif outcome == 'failed':
             template['title'] = runtime_message('backup.errorTitle', language,
                                                 hostname=data.get('hostname') or _get_hostname())
@@ -1899,7 +1910,7 @@ def render_template(event_type: str, data: Dict[str, Any],
     if event_type == 'backup_fail':
         template['title'] = runtime_message('backup.errorTitle', language,
                                             hostname=data.get('hostname') or _get_hostname())
-    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
+    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
         parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
         storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
         guests = (parsed_backup or {}).get('vms') or []
@@ -1950,7 +1961,7 @@ def render_template(event_type: str, data: Dict[str, Any],
         'log_file': '',
     }
     variables.update(data)
-    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
+    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
         # The provider has already substituted raw Display Names. Insert the
         # resolved title as a value, never reinterpret its literal braces.
         variables['_backup_title'] = template['title']
@@ -2059,6 +2070,14 @@ def render_template(event_type: str, data: Dict[str, Any],
             return ''
 
     safe_vars = _SafeDict(variables)
+    if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
+            and data.get('is_recovery') is True
+            and isinstance(data.get('check_evidence'), dict)
+            and data['check_evidence'].get('check') == 'cpu_usage'):
+        safe_vars['_health_title'] = runtime_message('healthRecovery.title', language, **variables)
+        safe_vars['_health_body'] = runtime_message('healthRecovery.body', language, **variables)
+        template['title'] = '{_health_title}'
+        template['body'] = '{_health_body}'
     try:
         title = template['title'].format_map(safe_vars)
     except (ValueError, IndexError):
@@ -2069,6 +2088,30 @@ def render_template(event_type: str, data: Dict[str, Any],
     # When the event came from PVE webhook with a full vzdump message,
     # parse the table/logs and format a rich body instead of the sparse template.
     pve_message = data.get('pve_message', '')
+    backup_diagnostics = []
+
+    def bounded_backup_diagnostics(lines):
+        # 1024 chars matches the repository's small-channel message convention;
+        # 8 lines keeps repeated producer warnings readable. Inventory/title
+        # size is separate: this is not a one-Telegram-message guarantee.
+        unique = list(dict.fromkeys(line for line in lines if line.strip()))
+        principal = next((line for line in unique if re.search(r'\b(?:ERROR:|TASK ERROR:)', line, re.IGNORECASE)), None)
+        if principal:
+            unique.remove(principal)
+            unique.insert(0, principal)
+        shown, budget, omitted = [], 1024, 0
+        for line in unique:
+            if len(shown) >= 8 or budget < 2:
+                omitted += 1
+                continue
+            line_budget = min(budget, 512)
+            rendered = line if len(line) <= line_budget else line[:line_budget - 1] + '…'
+            omitted += int(rendered != line)
+            shown.append(rendered)
+            budget -= len(rendered) + 1
+        if omitted:
+            shown.append(runtime_message('backup.diagnosticsOmitted', language, count=omitted))
+        return '\n'.join(shown)
     
     # Check for custom formatter function
     formatter_name = template.get('formatter')
@@ -2099,15 +2142,15 @@ def render_template(event_type: str, data: Dict[str, Any],
                                        pve_message, re.MULTILINE | re.DOTALL)
                 if error_block:
                     diagnostic_lines = error_block.group(1).rstrip('\r\n').splitlines() + diagnostic_lines
-            if diagnostic_lines:
-                body_text += '\n' + '\n'.join(
-                    line for line in dict.fromkeys(diagnostic_lines)
-                    if line.strip() and line not in body_text.splitlines())
+            backup_diagnostics = [line for line in diagnostic_lines
+                                  if line.strip() and line not in body_text.splitlines()]
         else:
-            # Couldn't parse -- use PVE raw message as body
-            body_text = pve_message.strip()
+            # Unparsed diagnostic-only reports remain visible but bounded.
+            body_text = ''
+            backup_diagnostics = pve_message.strip().splitlines()
         if event_type == 'backup_complete' and data.get('backup_outcome') != 'confirmed':
             key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
+                   else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
                    else 'backup.unconfirmedBody')
             body_text = runtime_message(key, language) + '\n' + body_text
     elif event_type == 'system_mail' and pve_message:
@@ -2138,8 +2181,20 @@ def render_template(event_type: str, data: Dict[str, Any],
     if event_type in ('backup_complete', 'backup_fail') and (
             event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
         source_subject = str(data.get('pve_title') or '').strip()
+        guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
+        if guest_context:
+            # Native single-line job errors live only in the subject. Retain
+            # that cause, not the redundant job/host envelope or generic count.
+            cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
+            source_subject = cause.group(1).strip() if cause else ''
+            if source_subject.lower() == 'multiple problems':
+                source_subject = ''
         if source_subject and source_subject not in body_text:
-            body_text += '\n' + source_subject
+            # A unique job/setup cause must survive a warning-heavy report.
+            backup_diagnostics.insert(0, source_subject)
+
+    if backup_diagnostics:
+        body_text += '\n' + bounded_backup_diagnostics(backup_diagnostics)
 
     # Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
     import re as _re
@@ -2472,9 +2527,14 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
     severity = data.get('severity', 'INFO')
     
     icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
+    if (event_type == 'error_resolved' and data.get('recovery_outcome') == 'resolved'
+            and data.get('is_recovery') is True
+            and isinstance(data.get('check_evidence'), dict)
+            and data['check_evidence'].get('check') == 'cpu_usage'):
+        icon = '✅'
     if event_type == 'backup_complete':
         icon = {
-            'confirmed': '💾✅', 'failed': '💾❌',
+            'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
         }.get(str(data.get('backup_outcome') or ''), '💾❔')
     
     # Build enriched title: replace severity circle with event-specific icon

AppImage/scripts/tests/test_notification_runtime_i18n.py

@@ -74,7 +74,10 @@ class RuntimeCatalogTests(unittest.TestCase):
         en = flatten(self.catalogs["en"])
         pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
                           "backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
-                          "channels.email.severity.observation", "channels.email.status.unconfirmed"}
+                          "channels.email.severity.observation", "channels.email.status.unconfirmed",
+                          "backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
+                          "channels.email.status.completed_with_warnings",
+                          "healthRecovery.title", "healthRecovery.body", "healthRecovery.status"}
         for language, catalog in self.catalogs.items():
             translated = flatten(catalog)
             if language == 'sk':