set apart host backups of other nodes and of former jobs

- Each host backup carries the node it belongs to, read from its source host or its name (`hostcfg-<node>`, a job of this node or its PBS backup ID); both backup lists also return the node name.
- The backup list offers This node / All nodes when some backup belongs to another node, with This node as the default, a count that follows the choice and a note with the backups not shown.
- Backups whose node cannot be read from their name are listed under All nodes.
- A local backup made on this node by a job that no longer exists is marked as Former job, read from its metadata file; backups without it, manual ones and those of other nodes are not marked.
- New texts in English and Spanish, modelled in the translation steady-state test.
- Tests for the node and the former job read from each backup, and for the cluster node membership notifications.

.github/scripts/tests/test_command_descriptions.py

@@ -99,6 +99,13 @@ class CommandDescriptionsTests(unittest.TestCase):
                     temporary = json.loads(path.read_text())
                     for section, key in (
                         ("archives", "emptyMessage"),
+                        ("archives", "formerJob"),
+                        ("archives", "formerJobTitle"),
+                        ("archives", "hiddenOtherNodes"),
+                        ("archives", "scopeAllNodes"),
+                        ("archives", "scopeAriaLabel"),
+                        ("archives", "scopeShowAll"),
+                        ("archives", "scopeThisNode"),
                         ("destinations", "backupsKeptMessage"),
                         ("destinations", "localAttachHelpMessage"),
                     ):

AppImage/components/host-backup.tsx

@@ -100,11 +100,15 @@ interface RemoteSnapshot {
   encrypted?: boolean
   borg_id?: string             // Borg-only fields when available
   borg_start_iso?: string
+  // The node the backup belongs to, read from its name; null when the name
+  // does not say (another node's backup under a name of its own choosing).
+  node?: string | null
 }
 
 interface RemoteArchivesResp {
   snapshots: RemoteSnapshot[]
   errors: Array<{ backend: string; repo_name: string; error: string }>
+  hostname?: string
 }
 
 interface ExportTask {
@@ -132,6 +136,7 @@ interface UnifiedArchive {
   // One of the two is populated depending on `source`.
   local?: BackupArchive
   remote?: RemoteSnapshot      // for both pbs and borg
+  node: string | null
 }
 
 interface BackupArchive {
@@ -146,6 +151,8 @@ interface BackupArchive {
   kind: "manual" | "scheduled" | "legacy"
   profile: string | null
   source_hostname: string | null
+  node?: string | null           // the node the backup belongs to
+  former_job?: boolean           // made by a job of this node that no longer exists
   // Which detection path identified this archive. Surfaced as a small tooltip
   // hint so the operator knows whether the metadata is authoritative
   // (sidecar) or inferred (filename / tar-peek).
@@ -964,11 +971,14 @@ export function HostBackup() {
     fetcher,
     { refreshInterval: 60000 },
   )
-  const { data: archivesResp, error: archivesErr, mutate: mutateArchives } = useSWR<{ archives: BackupArchive[] }>(
+  const { data: archivesResp, error: archivesErr, mutate: mutateArchives } = useSWR<{ archives: BackupArchive[]; hostname?: string }>(
     "/api/host-backups/archives",
     fetcher,
     { refreshInterval: 30000 },
   )
+  // On a cluster every PBS holds the backups of all nodes: the list shows
+  // this node's by default, and all of them on request.
+  const [archiveScope, setArchiveScope] = useState<"node" | "all">("node")
 
   // Background poll for the pending manual backup, independent of the
   // watch modal being open. Fires mutateArchives/mutateRemoteArchives
@@ -1005,6 +1015,7 @@ export function HostBackup() {
           created_at: a.mtime,
           source_label: a.path.replace(/\/[^/]+$/, "") || "/",
           local: a,
+          node: a.node ?? null,
         })
       }
     }
@@ -1017,12 +1028,21 @@ export function HostBackup() {
           created_at: s.backup_time,
           source_label: `${s.backend.toUpperCase()} ${s.repo_name}`,
           remote: s,
+          node: s.node ?? null,
         })
       }
     }
     out.sort((a, b) => b.created_at - a.created_at)
     return out
   })()
+  const thisNode = archivesResp?.hostname || remoteArchivesResp?.hostname || null
+  // The choice is offered only when some backup belongs to another node or
+  // its node is not known; a single node sees its list as before.
+  const otherNodeArchives = thisNode ? unifiedArchives.filter((u) => u.node !== thisNode).length : 0
+  const visibleArchives = archiveScope === "all" || otherNodeArchives === 0
+    ? unifiedArchives
+    : unifiedArchives.filter((u) => u.node === thisNode)
+  const hiddenArchives = unifiedArchives.length - visibleArchives.length
 
   return (
     <div className="space-y-4 md:space-y-6">
@@ -1278,7 +1298,32 @@ export function HostBackup() {
             <Archive className="h-5 w-5 text-blue-500" />
             <CardTitle className="text-base font-semibold">{t("backup.archives.title")}</CardTitle>
           </div>
-          <Badge variant="outline">{unifiedArchives.length}</Badge>
+          <div className="flex items-center gap-2">
+            {otherNodeArchives > 0 && (
+              <div
+                role="group"
+                aria-label={t("backup.archives.scopeAriaLabel")}
+                className="inline-flex rounded-lg border border-border bg-muted/40 p-1 gap-1"
+              >
+                {(["node", "all"] as const).map((key) => (
+                  <button
+                    key={key}
+                    type="button"
+                    aria-pressed={archiveScope === key}
+                    onClick={() => setArchiveScope(key)}
+                    className={`inline-flex items-center rounded-md px-2.5 py-1 text-xs font-medium transition-colors ${
+                      archiveScope === key
+                        ? "bg-blue-500 text-white shadow-sm"
+                        : "text-muted-foreground hover:text-foreground hover:bg-background/60"
+                    }`}
+                  >
+                    {key === "node" ? t("backup.archives.scopeThisNode") : t("backup.archives.scopeAllNodes")}
+                  </button>
+                ))}
+              </div>
+            )}
+            <Badge variant="outline">{visibleArchives.length}</Badge>
+          </div>
         </CardHeader>
         <CardContent>
           <p className="text-[11px] text-muted-foreground mb-3">
@@ -1301,13 +1346,21 @@ export function HostBackup() {
               <Loader2 className="h-4 w-4 animate-spin" />
               {t("backup.common.loading")}
             </div>
-          ) : unifiedArchives.length === 0 ? (
+          ) : visibleArchives.length === 0 && hiddenArchives === 0 ? (
             <div className="text-sm text-muted-foreground py-4">
               {t("backup.archives.emptyMessage")}
             </div>
           ) : (
             <div className="space-y-2">
-              {unifiedArchives.map((u) => {
+              {hiddenArchives > 0 && (
+                <p className="text-[11px] text-muted-foreground">
+                  {t("backup.archives.hiddenOtherNodes", { count: hiddenArchives })}{" "}
+                  <button type="button" className="text-blue-400 hover:underline" onClick={() => setArchiveScope("all")}>
+                    {t("backup.archives.scopeShowAll")}
+                  </button>
+                </p>
+              )}
+              {visibleArchives.map((u) => {
                 const localKind = u.local?.kind
                 const localJobId = u.local?.job_id
                 const localHost = u.local?.source_hostname
@@ -1366,6 +1419,14 @@ export function HostBackup() {
                             {t("backup.status.manual")}
                           </span>
                         ) : null}
+                        {u.source === "local" && u.local?.former_job && (
+                          <span
+                            className="uppercase tracking-wide text-[10px] px-1.5 py-0.5 rounded bg-amber-500/10 border border-amber-500/40 text-amber-400"
+                            title={t("backup.archives.formerJobTitle")}
+                          >
+                            {t("backup.archives.formerJob")}
+                          </span>
+                        )}
                         {(u.source === "pbs" || u.source === "borg") && u.remote?.backup_id && (
                           <span>{u.source === "pbs" ? t("backup.archives.group") : t("backup.archives.archive")}: <code className="font-mono">{u.remote.backup_id}</code></span>
                         )}
@@ -2035,7 +2096,19 @@ function InspectModal({
                     <div><span className="text-muted-foreground">{t("backup.fields.createdLabel")}</span> {formatMtime(localArc.mtime)}</div>
                     <div><span className="text-muted-foreground">{t("backup.fields.sizeLabel")}</span> {formatBytes(localArc.size_bytes)}</div>
                     <div className="sm:col-span-2"><span className="text-muted-foreground">{t("backup.fields.pathLabel")}</span> <code className="font-mono break-all">{localArc.path}</code></div>
-                    {localArc.job_id && <div><span className="text-muted-foreground">{t("backup.fields.jobIdLabel")}</span> <code className="font-mono">{localArc.job_id}</code></div>}
+                    {localArc.job_id && (
+                      <div>
+                        <span className="text-muted-foreground">{t("backup.fields.jobIdLabel")}</span> <code className="font-mono">{localArc.job_id}</code>
+                        {localArc.former_job && (
+                          <span
+                            className="ml-2 uppercase tracking-wide text-[10px] px-1.5 py-0.5 rounded bg-amber-500/10 border border-amber-500/40 text-amber-400"
+                            title={t("backup.archives.formerJobTitle")}
+                          >
+                            {t("backup.archives.formerJob")}
+                          </span>
+                        )}
+                      </div>
+                    )}
                     {localArc.profile && <div><span className="text-muted-foreground">{t("backup.fields.profileLabel")}</span> <code className="font-mono">{localArc.profile === "default" ? t("backup.profile.default") : localArc.profile === "custom" ? t("backup.profile.custom") : localArc.profile}</code></div>}
                     {localArc.source_hostname && <div><span className="text-muted-foreground">{t("backup.fields.sourceHostLabel")}</span> <code className="font-mono">{localArc.source_hostname}</code></div>}
                     <div><span className="text-muted-foreground">{t("backup.fields.detectedViaLabel")}</span> <code className="font-mono text-[10px]">{localArc.detected_via === "sidecar" ? t("backup.archives.companionFile") : localArc.detected_via}</code></div>

AppImage/messages/en/common.json

@@ -3953,8 +3953,15 @@
       "emptyAfter": "backups yet.",
       "emptyBefore": "No",
       "emptyMessage": "No backup archives found.",
+      "hiddenOtherNodes": "Backups of other nodes not shown: {count}.",
+      "scopeAllNodes": "All nodes",
+      "scopeAriaLabel": "Backups to show",
+      "scopeShowAll": "Show all",
+      "scopeThisNode": "This node",
       "encrypted": "Encrypted",
       "encryptedTitle": "This backup is encrypted",
+      "formerJob": "Former job",
+      "formerJobTitle": "The job that made this backup no longer exists on this node. The backup is kept until it is deleted.",
       "filesInBackup": "Files in backup",
       "group": "Group",
       "host": "Host",

AppImage/messages/es/common.json

@@ -3953,8 +3953,15 @@
       "downloadTitle": "Descargar este backup",
       "emptyAfter": "backups todavía.",
       "emptyBefore": "No",
+      "hiddenOtherNodes": "Backups de otros nodos no mostrados: {count}.",
+      "scopeAllNodes": "Todos los nodos",
+      "scopeAriaLabel": "Backups que se muestran",
+      "scopeShowAll": "Mostrar todos",
+      "scopeThisNode": "Este nodo",
       "encrypted": "cifrado",
       "encryptedTitle": "Este backup está cifrado",
+      "formerJob": "Trabajo anterior",
+      "formerJobTitle": "El trabajo que hizo este backup ya no existe en este nodo. El backup se conserva hasta que se elimine.",
       "filesInBackup": "Archivos del backup",
       "group": "Grupo",
       "host": "Host",

AppImage/scripts/flask_server.py

@@ -16151,6 +16151,56 @@ def _known_job_ids():
     return {os.path.basename(p)[:-len('.env')] for p in env_files}
 
 
+# The date a host backup carries at the end of its name.
+_HOST_BACKUP_STAMP_RE = re.compile(r'-\d{8}_\d{6}$')
+
+
+def _host_backup_own_names(hostname):
+    """Names this node keeps its host backups under: `hostcfg-<node>`, the
+    id of each of its jobs, which names its archives, and the PBS backup ID
+    each job uses."""
+    import glob
+    names = {f'hostcfg-{hostname}'}
+    try:
+        env_files = glob.glob(f'{_BACKUP_JOBS_DIR}/*.env')
+    except OSError:
+        env_files = []
+    for path in env_files:
+        names.add(os.path.basename(path)[:-len('.env')])
+        backup_id = (_parse_job_env(path).get('PBS_BACKUP_ID') or '').strip()
+        if backup_id:
+            names.add(backup_id)
+    return names
+
+
+def _host_backup_from_former_job(info, hostname, job_ids):
+    """True when the job that made this backup no longer exists on this
+    node. Only the sidecar says which job made it, and only a backup made
+    on this node can be measured against this node's jobs; anything else
+    is left unknown."""
+    if info.get('detected_via') != 'sidecar':
+        return False
+    job_id = info.get('job_id')
+    return bool(job_id) and info.get('source_hostname') == hostname and job_id not in job_ids
+
+
+def _host_backup_node(name, hostname, own_names):
+    """The node a host backup belongs to, read from its name, or None when
+    the name does not say: another node's backup stored under a name of its
+    own choosing."""
+    stem = name or ''
+    for suffix in _BACKUP_TAR_SUFFIXES:
+        if stem.endswith(suffix):
+            stem = stem[:-len(suffix)]
+            break
+    stem = _HOST_BACKUP_STAMP_RE.sub('', stem)
+    if stem in own_names:
+        return hostname
+    if stem.startswith('hostcfg-') and len(stem) > len('hostcfg-'):
+        return stem[len('hostcfg-'):]
+    return None
+
+
 # In-process cache for the tar-peek fallback so we don't re-decompress
 # every archive on every Monitor refresh. Keyed by absolute archive
 # path; the cached tuple is (size, mtime, is_proxmenux_backup_bool).
@@ -20398,7 +20448,11 @@ def api_host_backups_remote_archives():
                 errors.append({'backend': 'borg', 'repo_name': target['name'], 'error': err})
             snapshots.extend(items)
 
-    return jsonify({'snapshots': snapshots, 'errors': errors})
+    hostname = socket.gethostname()
+    own_names = _host_backup_own_names(hostname)
+    for snapshot in snapshots:
+        snapshot['node'] = _host_backup_node(snapshot.get('backup_id'), hostname, own_names)
+    return jsonify({'snapshots': snapshots, 'errors': errors, 'hostname': hostname})
 
 
 @app.route('/api/host-backups/remote-archives', methods=['DELETE'])
@@ -20962,6 +21016,7 @@ def api_host_backups_archives():
     seen: set = set()
     hostname = socket.gethostname()
     job_ids = _known_job_ids()
+    own_names = _host_backup_own_names(hostname)
 
     for d in _collect_backup_scan_dirs():
         try:
@@ -20988,10 +21043,15 @@ def api_host_backups_archives():
                 'size_bytes': st.st_size,
                 'mtime': int(st.st_mtime),
                 **info,
+                # An archive on this node's disk whose origin is not known
+                # is taken as this node's.
+                'node': (info.get('source_hostname')
+                         or _host_backup_node(name, hostname, own_names) or hostname),
+                'former_job': _host_backup_from_former_job(info, hostname, job_ids),
             })
 
     archives.sort(key=lambda a: a['mtime'], reverse=True)
-    return jsonify({'archives': archives})
+    return jsonify({'archives': archives, 'hostname': hostname})
 
 
 # ──────────────────────────────────────────────────────────────

AppImage/scripts/tests/test_cluster_node_membership_notifications.py

@@ -0,0 +1,106 @@
+"""Corosync reports a node that leaves or comes back with its number, in a
+membership line that never says "node". The surviving nodes notify it by the
+name the node has in corosync.conf."""
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from queue import Empty, Queue
+from unittest.mock import patch
+
+SCRIPTS = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS))
+import notification_events  # noqa: E402
+
+CONF = """totem {
+  cluster_name: lab
+}
+nodelist {
+  node {
+    name: pve1
+    nodeid: 1
+    ring0_addr: 10.0.0.1
+  }
+  node {
+    name: pve2
+    nodeid: 2
+    ring0_addr: 10.0.0.2
+  }
+  node {
+    name: pve3
+    nodeid: 3
+    ring0_addr: 10.0.0.3
+  }
+}
+"""
+
+
+class ClusterMembershipTests(unittest.TestCase):
+    def setUp(self):
+        folder = tempfile.TemporaryDirectory()
+        self.addCleanup(folder.cleanup)
+        conf = Path(folder.name) / "corosync.conf"
+        conf.write_text(CONF)
+        real_open = open
+
+        def corosync_conf(path, *args, **kwargs):
+            if str(path) in ("/etc/pve/corosync.conf", "/etc/corosync/corosync.conf"):
+                path = conf
+            return real_open(path, *args, **kwargs)
+
+        self.booting = False
+        for item in (patch("builtins.open", corosync_conf),
+                     patch.object(notification_events.socket, "gethostname", return_value="pve1"),
+                     patch.object(notification_events.startup_grace, "is_real_system_boot",
+                                  side_effect=lambda: self.booting),
+                     patch.object(notification_events._shared_state, "is_startup_health_grace",
+                                  side_effect=lambda: self.booting)):
+            item.start()
+            self.addCleanup(item.stop)
+        self.queue = Queue()
+        self.watcher = notification_events.JournalWatcher(self.queue)
+
+    def notified(self, line):
+        self.watcher._check_cluster_events(line, "corosync")
+        events = []
+        while True:
+            try:
+                event = self.queue.get_nowait()
+            except Empty:
+                return events
+            events.append((event.event_type, event.data.get("node_name")))
+
+    def test_a_node_that_leaves_is_named(self):
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.55) was formed. Members left: 2"),
+                         [("node_disconnect", "pve2")])
+
+    def test_a_node_that_comes_back_is_named(self):
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.5a) was formed. Members joined: 2"),
+                         [("node_reconnect", "pve2")])
+
+    def test_every_node_of_the_line_is_notified(self):
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.60) was formed. Members left: 2 3"),
+                         [("node_disconnect", "pve2"), ("node_disconnect", "pve3")])
+
+    def test_this_node_is_not_notified(self):
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.61) was formed. Members joined: 1"), [])
+
+    def test_while_this_host_boots_only_the_joins_are_left_out(self):
+        self.booting = True
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.62) was formed. Members joined: 2"), [])
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.63) was formed. Members left: 3"),
+                         [("node_disconnect", "pve3")])
+
+    def test_a_node_missing_from_the_configuration_keeps_its_number(self):
+        self.assertEqual(self.notified("[TOTEM ] A new membership (1.64) was formed. Members left: 7"),
+                         [("node_disconnect", "7")])
+
+    def test_other_corosync_lines_notify_nothing(self):
+        for line in ("[CFG   ] Node 2 was shut down by sysadmin", "[QUORUM] Members[2]: 1 3",
+                     "[KNET  ] link: host: 2 link: 0 is down"):
+            with self.subTest(line=line):
+                self.assertEqual(self.notified(line), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

AppImage/scripts/tests/test_host_backup_node.py

@@ -0,0 +1,91 @@
+"""Each host backup in the list carries the node it belongs to, read from its
+name, so a cluster node can show its own backups apart from the others, and
+a local backup says when the job that made it no longer exists."""
+import ast
+import os
+from pathlib import Path
+import re
+import shlex
+import tempfile
+import unittest
+
+SERVER = Path(__file__).resolve().parents[1] / 'flask_server.py'
+HELPERS = ('_parse_job_env', '_host_backup_own_names', '_host_backup_node',
+           '_host_backup_from_former_job')
+
+
+class HostBackupNodeTests(unittest.TestCase):
+    def setUp(self):
+        text = SERVER.read_text(encoding='utf-8')
+        tree = ast.parse(text)
+        nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS]
+        self.assertEqual(sorted(n.name for n in nodes), sorted(HELPERS))
+        folder = tempfile.TemporaryDirectory()
+        self.addCleanup(folder.cleanup)
+        self.jobs = folder.name
+        self.scope = {'os': os, 're': re, 'shlex': shlex,
+                      '_BACKUP_JOBS_DIR': self.jobs,
+                      '_BACKUP_TAR_SUFFIXES': ('.tar', '.tar.zst', '.tar.gz'),
+                      '_HOST_BACKUP_STAMP_RE': re.compile(r'-\d{8}_\d{6}$')}
+        exec(compile(ast.Module(body=nodes, type_ignores=[]), str(SERVER), 'exec'), self.scope)
+        self.own_names = self.scope['_host_backup_own_names']
+        self.node = self.scope['_host_backup_node']
+        self.former = self.scope['_host_backup_from_former_job']
+
+    def job(self, job_id, backup_id=None):
+        lines = [f'JOB_ID={job_id}']
+        if backup_id:
+            lines.append(f'PBS_BACKUP_ID={backup_id}')
+        Path(self.jobs, f'{job_id}.env').write_text('\n'.join(lines) + '\n')
+
+    def test_the_default_name_says_the_node(self):
+        own = self.own_names('pve1')
+        self.assertEqual(self.node('hostcfg-pve1', 'pve1', own), 'pve1')
+        self.assertEqual(self.node('hostcfg-pve2', 'pve1', own), 'pve2')
+
+    def test_a_local_archive_name_says_the_node(self):
+        own = self.own_names('pve1')
+        self.assertEqual(self.node('hostcfg-pve2-20261014_031500.tar.zst', 'pve1', own), 'pve2')
+
+    def test_a_job_of_this_node_is_this_node(self):
+        self.job('nightly-pbs', backup_id='my-host-config')
+        own = self.own_names('pve1')
+        self.assertEqual(self.node('nightly-pbs-20261014_031500.tar.zst', 'pve1', own), 'pve1')
+        self.assertEqual(self.node('my-host-config', 'pve1', own), 'pve1')
+
+    def test_a_name_that_does_not_say_is_unknown(self):
+        own = self.own_names('pve1')
+        self.assertIsNone(self.node('custom-backup', 'pve1', own))
+        self.assertIsNone(self.node('hostcfg-', 'pve1', own))
+        self.assertIsNone(self.node(None, 'pve1', own))
+
+    def test_a_missing_jobs_folder_leaves_the_default_name(self):
+        self.scope['_BACKUP_JOBS_DIR'] = os.path.join(self.jobs, 'absent')
+        self.assertEqual(self.own_names('pve1'), {'hostcfg-pve1'})
+
+    def sidecar(self, job_id, hostname='pve1'):
+        return {'kind': 'scheduled', 'job_id': job_id, 'source_hostname': hostname,
+                'detected_via': 'sidecar'}
+
+    def test_a_backup_of_a_job_that_no_longer_exists_is_from_a_former_job(self):
+        self.assertTrue(self.former(self.sidecar('PVE1-local-backuo'), 'pve1', {'PVE1-local-backup'}))
+
+    def test_a_backup_of_an_existing_job_is_not(self):
+        self.assertFalse(self.former(self.sidecar('PVE1-local-backup'), 'pve1', {'PVE1-local-backup'}))
+
+    def test_a_backup_made_on_another_node_is_left_unknown(self):
+        self.assertFalse(self.former(self.sidecar('pve2-nightly', hostname='pve2'), 'pve1', set()))
+
+    def test_a_backup_without_its_sidecar_is_left_unknown(self):
+        for info in ({'kind': 'legacy', 'job_id': None, 'source_hostname': None, 'detected_via': 'tar_peek'},
+                     {'kind': 'manual', 'job_id': None, 'source_hostname': None, 'detected_via': 'hostcfg_prefix'}):
+            self.assertFalse(self.former(info, 'pve1', set()))
+
+    def test_a_manual_backup_has_no_job(self):
+        info = self.sidecar(None)
+        info['kind'] = 'manual'
+        self.assertFalse(self.former(info, 'pve1', set()))
+
+
+if __name__ == '__main__':
+    unittest.main()