Correct backup outcome diagnostics and email presentation chains

AppImage/scripts/notification_channels.py

@@ -1053,11 +1053,13 @@ class EmailChannel(NotificationChannel):
                 status = 'unconfirmed'
             sev['label'] = _runtime_text(f'email.status.{status}', data)
         group = data.get('_group', 'other')
-        # Keep unbroken recorded text inside the temperature email's table.
-        # Both properties are inline for mail clients; other events retain
-        # their original markup and layout.
-        temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if event_type == 'temp_high' else ''
-        temp_table_layout = 'table-layout:fixed;' if event_type == 'temp_high' else ''
+        # Scoped inline mail-compatible wrapping: temperature measurements
+        # and backup identities/raw diagnostics. Other events retain layout.
+        backup_email = event_type in {'backup_complete', 'backup_fail'}
+        temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if event_type == 'temp_high' or backup_email else ''
+        temp_table_layout = 'table-layout:fixed;' if event_type == 'temp_high' or backup_email else ''
+        backup_title_wrap = temp_cell_wrap if backup_email else ''
+        backup_metadata_layout = 'table-layout:fixed;' if backup_email else ''
         section_label = _runtime_text(f'email.groups.{group}', data)
         report_label = _runtime_text('email.report', data, group=section_label)
         host_label = _runtime_text('email.host', data)
@@ -1089,6 +1091,13 @@ class EmailChannel(NotificationChannel):
                 for line in body.split('\n') if line.strip()
             )
 
+        if event_type in {'system_restore_completed', 'error_resolved'}:
+            # Observation age/disappearance must not become a green OK row.
+            # The endpoint's warnings_block and task counts live in the
+            # localized body, not the generic services Event row.
+            detail_rows = [('', html_mod.escape(line.strip()))
+                           for line in body.split('\n') if line.strip()]
+
         # ── Fallback: if no structured rows, render body text lines ──
         if not detail_rows:
             for line in body.split('\n'):
@@ -1155,15 +1164,15 @@ class EmailChannel(NotificationChannel):
 
   <!-- Title bar -->
   <div style="padding:16px 28px;background:{sev['bg']};border-bottom:1px solid {sev['border']};">
-    <h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};">{html_mod.escape(display_title)}</h2>
+    <h2 style="margin:0;font-size:15px;font-weight:600;color:{sev['color']};{backup_title_wrap}">{html_mod.escape(display_title)}</h2>
   </div>
 
   <!-- Body -->
   <div style="padding:24px 28px;">
     <!-- Metadata -->
-    <table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;">
+    <table width="100%" cellpadding="0" cellspacing="0" border="0" style="margin-bottom:16px;{backup_metadata_layout}">
       <tr>
-        <td style="font-size:12px;color:#4b5563;">
+        <td style="font-size:12px;color:#4b5563;{backup_title_wrap}">
           {html_mod.escape(host_label)}: <strong style="color:#111827;">{html_mod.escape(data.get('hostname', ''))}</strong>
         </td>
         <td style="font-size:12px;color:#4b5563;text-align:right;">

AppImage/scripts/notification_events.py

@@ -4294,7 +4294,7 @@ class ProxmoxHookWatcher:
         """Distinguish explicit failure, complete guest logs and unknown results."""
         text = str(message or '')
         if severity in ('error', 'err', 'critical') or re.search(
-                r'(?im)^\s*(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
+                r'(?im)^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:|.*\bStatus\s+ERROR\b)', text):
             return 'failed'
         starts = re.findall(r'(?im)\bStarting Backup of VM (\d+)\s*\(', text)
         finished = re.findall(r'(?im)\bFinished Backup of VM (\d+)\s*\(', text)
@@ -4303,7 +4303,7 @@ class ProxmoxHookWatcher:
         if table is not None and any(guest['status'].lower() == 'error' for guest in table['vms']):
             return 'failed'
         if severity not in ('info', 'ok', 'success') or re.search(
-                r'(?im)(?:^\s*WARNING:|\bWARNINGS\s*:\s*\d+)', text):
+                r'(?im)(?:^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?WARN(?:ING)?:|\bWARNINGS\s*:\s*\d+)', text):
             return 'unconfirmed'
         # A present table is authoritative: do not certify an incomplete table
         # from a finished guest log, or reject a complete OK table merely
@@ -4313,7 +4313,16 @@ class ProxmoxHookWatcher:
                     all(guest['status'].lower() == 'ok' for guest in table['vms'])
                     else 'unconfirmed')
         if starts:
-            return 'confirmed' if sorted(starts) == sorted(finished) else 'unconfirmed'
+            pending = {}
+            for match in re.finditer(r'(?im)\b(Starting|Finished) Backup of VM (\d+)\s*\(', text):
+                action, vmid = match.groups()
+                if action.lower() == 'starting':
+                    pending[vmid] = pending.get(vmid, 0) + 1
+                elif not pending.get(vmid):
+                    return 'unconfirmed'  # A finish before its start is not evidence.
+                else:
+                    pending[vmid] -= 1
+            return 'confirmed' if not any(pending.values()) else 'unconfirmed'
         if re.search(
                 r'(?im)^\s*(?:INFO:\s*)?TASK OK\s*$', text):
             return 'confirmed'
@@ -4379,10 +4388,10 @@ class ProxmoxHookWatcher:
         }
         if event_type in ('backup_complete', 'backup_fail'):
             # This is presentation metadata, not a new event/toggle/delivery path.
+            outcome = self._backup_outcome(severity_raw, message)
             data['backup_outcome'] = (
-                'failed' if event_type == 'backup_fail' else
-                self._backup_outcome(severity_raw, message) if pve_type == 'vzdump'
-                else 'unconfirmed'
+                'failed' if event_type == 'backup_fail' or outcome == 'failed' else
+                outcome if pve_type == 'vzdump' else 'unconfirmed'
             )
 
         if pve_type == 'replication':

AppImage/scripts/notification_manager.py

@@ -2467,6 +2467,11 @@ class NotificationManager:
             runtime_data.get('hostname'), self._config,
         )
         runtime_data.setdefault('_notification_language', self._notification_language())
+        # Match queued dispatch's presentation context for these outcome
+        # notices; this does not alter event/severity or direct-send policy.
+        if event_type in ('backup_complete', 'backup_fail', 'error_resolved', 'system_restore_completed'):
+            runtime_data['_event_type'] = event_type
+            runtime_data['_group'] = TEMPLATES[event_type].get('group', 'other')
 
         # Render template if available
         if event_type in TEMPLATES and not message:

AppImage/scripts/notification_templates.py

@@ -238,7 +238,7 @@ def _parse_vzdump_table(message: str) -> Optional[Dict[str, Any]]:
                 valid = False
                 break
             valid = bool(valid and all(values)
-                         and re.fullmatch(r'(?:\d+:\d{2}:\d{2}|(?:\d+[dhms]\s*)+)', duration)
+                         and re.fullmatch(r'(?:\d+:\d{2}:\d{2}|(?:\d+[yMwdhms]\s*)+)', duration)
                          and re.fullmatch(r'\d+(?:\.\d+)?\s*(?:[KMGTPE]i?B?|B)', size, re.IGNORECASE))
             if status.lower() in ('err', 'error'):
                 status = 'error'
@@ -309,6 +309,18 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
                 }
                 continue
             
+            # A finish can belong to a guest already stored when another
+            # start arrived. Preserve that guest's actual completion too.
+            prior_finish = re.match(r'Finished Backup of VM (\d+)\s+\(([^)]+)\)', clean)
+            if prior_finish:
+                prior = next((vm for vm in reversed(vms)
+                              if vm['vmid'] == prior_finish.group(1)), None)
+                if prior is not None:
+                    prior['time'] = prior_finish.group(2)
+                    if prior['status'] != 'error':
+                        prior['status'] = 'ok'
+                    continue
+
             if current_vm:
                 # Guest name
                 m_name = re.match(r'(?:CT|VM) Name:\s*(.+)', clean)
@@ -357,6 +369,21 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
         if current_vm:
             vms.append(current_vm)
     
+    # Explicit guest-linked failures outrank a contradictory summary OK row.
+    # Job-level/prune errors do not invalidate unrelated successfully saved guests.
+    for line in lines:
+        error = re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:ERROR:|TASK ERROR:)\s*(.*)',
+                         line, re.IGNORECASE)
+        if not error:
+            continue
+        failed_guest = re.search(r'\bBackup of (?:VM|CT) (\d+) failed\b|\bbackup failed for (?:VM|CT) (\d+)\b',
+                                 error.group(1), re.IGNORECASE)
+        if failed_guest:
+            vmid = failed_guest.group(1) or failed_guest.group(2)
+            for vm in vms:
+                if vm['vmid'] == vmid:
+                    vm['status'] = 'error'
+
     # ── Extract totals ──
     for line in lines:
         m_time = re.search(r'Total running time:\s*(.+)', line)
@@ -372,7 +399,7 @@ def _parse_vzdump_message(message: str) -> Optional[Dict[str, Any]]:
         sizes_gib = 0.0
         for vm in vms:
             s = vm.get('size', '')
-            m = re.match(r'([\d.]+)\s+(.*)', s)
+            m = re.fullmatch(r'(\d+(?:\.\d+)?)\s+([KMGTPE]i?B|B)', s, re.IGNORECASE)
             if m:
                 val = float(m.group(1))
                 unit = m.group(2).strip().upper()
@@ -1869,11 +1896,19 @@ def render_template(event_type: str, data: Dict[str, Any],
             template['title'] = runtime_message('backup.errorTitle', language,
                                                 hostname=data.get('hostname') or _get_hostname())
             template['body'] = runtime_message('backup.errorBody', language)
-    if event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed'):
+    if event_type == 'backup_fail':
+        template['title'] = runtime_message('backup.errorTitle', language,
+                                            hostname=data.get('hostname') or _get_hostname())
+    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
         parsed_backup = _parse_vzdump_message(str(data.get('pve_message') or ''))
         storage = str((parsed_backup or {}).get('storage_name') or data.get('storage') or '').strip()
         guests = (parsed_backup or {}).get('vms') or []
-        if data.get('backup_outcome') == 'failed':
+        # Explicit confirmed manual metadata is useful context, not evidence
+        # about an unparsed batch. Only use it when there is no raw report.
+        if not data.get('pve_message') and data.get('backup_outcome') == 'confirmed' and data.get('vmid'):
+            guests = [{'vmid': str(data['vmid']), 'name': str(data.get('vmname') or ''),
+                       'type': str(data.get('vm_type') or ''), 'status': 'ok'}]
+        if event_type == 'backup_fail' or data.get('backup_outcome') == 'failed':
             guests = [guest for guest in guests if guest.get('status', '').lower() == 'error']
         target = []
         if storage:
@@ -1915,6 +1950,11 @@ def render_template(event_type: str, data: Dict[str, Any],
         'log_file': '',
     }
     variables.update(data)
+    if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'failed')):
+        # The provider has already substituted raw Display Names. Insert the
+        # resolved title as a value, never reinterpret its literal braces.
+        variables['_backup_title'] = template['title']
+        template['title'] = '{_backup_title}'
 
     # Old persisted errors and manual events may lack a complete reading.
     # Accept plain numeric strings, but never interpret booleans or objects as
@@ -2047,11 +2087,11 @@ def render_template(event_type: str, data: Dict[str, Any],
         if parsed:
             is_success = (event_type == 'backup_complete')
             body_text = _format_vzdump_body(parsed, is_success, language=language)
-            if event_type == 'backup_complete' and data.get('backup_outcome') == 'failed':
-                error_lines = [line.strip() for line in pve_message.splitlines()
-                               if re.match(r'^\s*(?:ERROR:|TASK ERROR)', line, re.IGNORECASE)]
-                if error_lines:
-                    body_text += '\n' + '\n'.join(error_lines)
+            diagnostic_lines = [line.strip() for line in pve_message.splitlines()
+                                if re.match(r'^\s*(?:\d+:\s*)?(?:\d{4}-\d{2}-\d{2}\s+\S+\s+)?(?:WARN(?:ING)?:|ERROR:|TASK ERROR)',
+                                            line, re.IGNORECASE)]
+            if diagnostic_lines:
+                body_text += '\n' + '\n'.join(dict.fromkeys(diagnostic_lines))
         else:
             # Couldn't parse -- use PVE raw message as body
             body_text = pve_message.strip()
@@ -2068,6 +2108,28 @@ def render_template(event_type: str, data: Dict[str, Any],
         except (ValueError, IndexError):
             body_text = template['body']
     
+    if event_type == 'backup_complete' and data.get('backup_outcome') == 'confirmed' and not pve_message:
+        context = []
+        if data.get('vmid'):
+            name = str(data.get('vmname') or '')
+            context.append(f"{name} ({data['vmid']})" if name else str(data['vmid']))
+        if data.get('storage'):
+            context.append(str(data['storage']))
+        if data.get('size'):
+            context.append(runtime_message('vzdump.size', language, value=data['size']))
+        if data.get('duration'):
+            context.append(runtime_message('vzdump.duration', language, value=data['duration']))
+        if context:
+            body_text += '\n' + '\n'.join(context)
+
+    # PVE can move a one-line setup/abort reason exclusively into its subject.
+    # Preserve that raw failure context, without using it as a localized title.
+    if event_type in ('backup_complete', 'backup_fail') and (
+            event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
+        source_subject = str(data.get('pve_title') or '').strip()
+        if source_subject and source_subject not in body_text:
+            body_text += '\n' + source_subject
+
     # Clean up: collapse runs of 3+ blank lines into 1, remove trailing whitespace
     import re as _re
     body_text = _re.sub(r'\n{3,}', '\n\n', body_text.strip())

AppImage/scripts/tests/test_notification_runtime_i18n.py

@@ -77,9 +77,16 @@ class RuntimeCatalogTests(unittest.TestCase):
                           "channels.email.severity.observation", "channels.email.status.unconfirmed"}
         for language, catalog in self.catalogs.items():
             translated = flatten(catalog)
-            expected = set(en) - pending_slovak if language == 'sk' else set(en)
-            self.assertEqual(set(translated), expected, language)
-            for key in expected:
+            if language == 'sk':
+                # Missing maintainer-owned leaves may be generated later.
+                # Accept only this bounded gap, and validate every present leaf.
+                self.assertTrue(set(en) - pending_slovak <= set(translated), language)
+                self.assertTrue(set(translated) <= set(en), language)
+            else:
+                self.assertEqual(set(translated), set(en), language)
+            for key in translated:
+                self.assertIsInstance(translated[key], str, f"{language}:{key}")
+                self.assertTrue(translated[key].strip(), f"{language}:{key}")
                 if language == 'sk' and key in ('templates.backup_complete.title',
                                                 'templates.backup_complete.body'):
                     continue  # exact upstream SK, superseded only at render time

AppImage/scripts/tests/test_vzdump_ai_integrity.py

@@ -275,8 +275,14 @@ class VzdumpAIIntegrityTests(unittest.TestCase):
             rendered["body"], "CRITICAL", data,
         )
 
+        # The failure title now identifies the unique failed guest; inventory
+        # remains exactly once in the detail table, not suppressed from body.
+        inventory = html.split('<!-- Detail table -->', 1)[1].split('</table>', 1)[0]
         for vmid in range(100, 149):
-            self.assertEqual(html.count(f"guest-{vmid} ({vmid})"), 1, vmid)
+            self.assertEqual(inventory.count(f"guest-{vmid} ({vmid})"), 1, vmid)
+        self.assertIn('guest-148 (148)', rendered['title'])
+        self.assertNotIn('guest-100 (100)', rendered['title'])
+        self.assertEqual(html.count('guest-148 (148)'), 2)  # title + inventory
         self.assertEqual(html.count("49 backups"), 1)
         self.assertEqual(html.count("1 failed"), 1)
         self.assertEqual(html.count(">Zlyhalo<"), 1)