diff --git a/.github/scripts/tests/test_fencing_wording.py b/.github/scripts/tests/test_fencing_wording.py index 7e2834ff..ccd66a15 100644 --- a/.github/scripts/tests/test_fencing_wording.py +++ b/.github/scripts/tests/test_fencing_wording.py @@ -82,7 +82,9 @@ class FencingWordingTests(unittest.TestCase): rendered = self.render(event.event_type, event.data, lang) expected = catalog['runtime']['notifications']['templates']['split_brain'] self.assertEqual(rendered['title'], expected['title'].format(hostname='node-a'), lang) - self.assertEqual(rendered['body'], expected['body'], lang) + # The body names the source event, as received, and adds no diagnosis of its own. + self.assertEqual(rendered['body'], expected['body'].replace('{reason}', event.data['reason']), lang) + self.assertIn('Check node state', rendered['body'], lang) self.assertNotIn('split-brain', '\n'.join((rendered['title'], rendered['body'])).lower(), lang) self.assertNotIn('quorum', rendered['body'].lower(), lang) self.assertEqual(webhook(receiver, {'type': 'fencing', 'severity': 'warning', @@ -107,7 +109,8 @@ class FencingWordingTests(unittest.TestCase): self.assertEqual(args[0], 'split_brain') for lang, catalog in self.catalogs.items(): rendered = self.render(args[0], args[2], lang) - self.assertEqual(rendered['body'], catalog['runtime']['notifications']['templates']['split_brain']['body'], lang) + body = catalog['runtime']['notifications']['templates']['split_brain']['body'] + self.assertEqual(rendered['body'], body.replace('{reason}', message), lang) journal = Journal() method(journal, 'quorum lost', 'syslog') self.assertEqual(journal.events[0][0][0], 'node_disconnect') @@ -126,13 +129,13 @@ class FencingWordingTests(unittest.TestCase): def test_every_shipped_locale_renders_cluster_event_without_diagnosis(self): expected = { - 'de': ('{hostname}: Clusterereignis gemeldet', 'Ein Clusterereignis wurde gemeldet. Prüfen Sie den Clusterstatus und das ursprüngliche Ereignis für weitere Informationen.', 'Clusterereignis', 'Clusterereignisse'), - 'es': ('{hostname}: evento del clúster notificado', 'Se ha notificado un evento del clúster. Consulta el estado del clúster y el evento original para obtener más detalles.', 'Evento del clúster', 'Eventos del clúster'), - 'fr': ('{hostname}\u00a0: événement du cluster signalé', 'Un événement du cluster a été signalé. Vérifiez l’état du cluster et l’événement d’origine pour plus de détails.', 'Événement du cluster', 'Événements du cluster'), - 'it': ('{hostname}: evento del cluster segnalato', "È stato segnalato un evento del cluster. Controlla lo stato del cluster e l'evento originale per maggiori dettagli.", 'Evento del cluster', 'Eventi del cluster'), - 'pt': ('{hostname}: evento do cluster comunicado', 'Foi comunicado um evento do cluster. Verifique o estado do cluster e o evento original para obter mais detalhes.', 'Evento do cluster', 'Eventos do cluster'), - 'sk': ('{hostname}: hlásená udalosť klastra', 'Bola hlásená udalosť klastra. Skontrolujte stav klastra a pôvodnú udalosť, kde nájdete ďalšie podrobnosti.', 'Udalosť klastra', 'Udalosti klastra'), - 'sv': ('{hostname}: klusterhändelse rapporterad', 'En klusterhändelse har rapporterats. Kontrollera klusterstatus och den ursprungliga händelsen för mer information.', 'Klusterhändelse', 'Klusterhändelser'), + 'de': ('{hostname}: Clusterereignis gemeldet', 'Ein Clusterereignis wurde gemeldet:\n{reason}', 'Clusterereignis', 'Clusterereignisse'), + 'es': ('{hostname}: evento del clúster notificado', 'Se ha notificado un evento del clúster:\n{reason}', 'Evento del clúster', 'Eventos del clúster'), + 'fr': ('{hostname}\u00a0: événement du cluster signalé', 'Un événement du cluster a été signalé\u00a0:\n{reason}', 'Événement du cluster', 'Événements du cluster'), + 'it': ('{hostname}: evento del cluster segnalato', 'È stato segnalato un evento del cluster:\n{reason}', 'Evento del cluster', 'Eventi del cluster'), + 'pt': ('{hostname}: evento do cluster comunicado', 'Foi comunicado um evento do cluster:\n{reason}', 'Evento do cluster', 'Eventos do cluster'), + 'sk': ('{hostname}: hlásená udalosť klastra', 'Bola hlásená udalosť klastra:\n{reason}', 'Udalosť klastra', 'Udalosti klastra'), + 'sv': ('{hostname}: klusterhändelse rapporterad', 'En klusterhändelse har rapporterats:\n{reason}', 'Klusterhändelse', 'Klusterhändelser'), } self.assertEqual(set(self.catalogs), {'en', *expected}) for lang, (title, body, label, settings) in expected.items(): @@ -141,9 +144,9 @@ class FencingWordingTests(unittest.TestCase): self.assertEqual((leaves['title'], leaves['body'], leaves['label'], self.catalogs[lang]['settings']['notifications']['eventTypes']['split_brain']), (title, body, label, settings)) - result = self.render('split_brain', {}, lang) + result = self.render('split_brain', {'reason': 'fencing node node-b'}, lang) self.assertEqual(result['title'], title.format(hostname='node-a')) - self.assertEqual(result['body'], body) + self.assertEqual(result['body'], body.replace('{reason}', 'fencing node node-b')) self.assertNotIn('quorum', result['body'].lower()) def test_fallback_and_translated_lookup_are_real_for_missing_and_synthetic_values(self): @@ -152,8 +155,8 @@ class FencingWordingTests(unittest.TestCase): for field in FIELDS: catalogs['it']['runtime']['notifications']['templates']['split_brain'].pop(field) _, render = notification_renderer(catalogs) - self.assertEqual(render('split_brain', {}, 'it')['body'], - 'A cluster event was reported. Review cluster status and the source event for details.') + self.assertEqual(render('split_brain', {'reason': 'fencing node node-b'}, 'it')['body'], + 'A cluster event was reported:\nfencing node node-b') catalogs['it']['runtime']['notifications']['templates']['split_brain']['body'] = 'Evento del cluster segnalato.' _, render = notification_renderer(catalogs) self.assertEqual(render('split_brain', {}, 'it')['body'], 'Evento del cluster segnalato.') diff --git a/.github/scripts/tests/test_oci_recovery_inventory_wording.py b/.github/scripts/tests/test_oci_recovery_inventory_wording.py index b4cbce13..efd2721f 100644 --- a/.github/scripts/tests/test_oci_recovery_inventory_wording.py +++ b/.github/scripts/tests/test_oci_recovery_inventory_wording.py @@ -80,6 +80,9 @@ class InventoryMessages(unittest.TestCase): 'Path': Path, 'shutil': SimpleNamespace(rmtree=lambda path: None), 'image_cache': SimpleNamespace(prune=lambda root, lock: []), 'oci_console': SimpleNamespace(remove_log=lambda vmid: events.append(('log', vmid))), + 're': re, 'guest_node': lambda vmid: None, + 'remove_host_state': lambda vmid: events.append(('host', vmid)), + 'release_shared_host_files': lambda hookscripts: None, 'translate': lambda text: text, 'msg_info': lambda text: events.append(('info', text)), 'msg_ok': lambda text: events.append(('ok', text)), @@ -89,6 +92,7 @@ class InventoryMessages(unittest.TestCase): self.assertIn(('info', POST_HOST + ' /bind/saved'), events) self.assertIn(('run', ('pct', 'destroy', '101', '--purge', '1', '--destroy-unreferenced-disks', '1')), events) self.assertIn(('log', 101), events) + self.assertIn(('host', 101), events) scope['translate'] = lambda text: 'Tradotto: ' + text if text == POST_HOST else text events.clear() remove(Path('/inert'), 101) diff --git a/.github/scripts/tests/test_oci_removal_firewall_wording.py b/.github/scripts/tests/test_oci_removal_firewall_wording.py index 2ab3e802..328e760e 100644 --- a/.github/scripts/tests/test_oci_removal_firewall_wording.py +++ b/.github/scripts/tests/test_oci_removal_firewall_wording.py @@ -51,6 +51,7 @@ class RemovalWordings(unittest.TestCase): remover.guest_config = lambda member: None remover.host_directories = lambda root, members: [] remover.private_bridge = lambda primary: bridge + remover.bridge_in_use = lambda bridge, removed: False state = ModuleType('oci_installation_state') state.parse_config = lambda raw: {} scope = {'sys': SimpleNamespace(path=[]), 'source_text': lambda value: value or '', @@ -117,6 +118,7 @@ class RemovalWordings(unittest.TestCase): 'shutil': SimpleNamespace(rmtree=lambda path: None), 'image_cache': SimpleNamespace(prune=lambda root, lock: []), 'oci_console': SimpleNamespace(remove_log=lambda vmid: None), + 'guest_node': lambda vmid: None, 'remove_host_state': lambda vmid: None, 'release_shared_host_files': lambda hookscripts: None, 're': re, 'translate': lambda text: text, 'msg_info': lambda text: events.append(('info', text)), 'msg_ok': lambda text: events.append(('ok', text)), @@ -185,6 +187,7 @@ class RemovalWordings(unittest.TestCase): 'shutil': SimpleNamespace(rmtree=lambda path: None), 'image_cache': SimpleNamespace(prune=lambda root, lock: []), 'oci_console': SimpleNamespace(remove_log=lambda vmid: None), + 'guest_node': lambda vmid: None, 'remove_host_state': lambda vmid: None, 'release_shared_host_files': lambda hookscripts: None, 'translate': lambda text: text, 'msg_info': lambda text: events.append(('info', text)), 'msg_ok': lambda text: events.append(('ok', text)), @@ -262,6 +265,7 @@ class RemovalWordings(unittest.TestCase): 'shutil': SimpleNamespace(rmtree=lambda path: None), 'image_cache': SimpleNamespace(prune=lambda root, lock: []), 'oci_console': SimpleNamespace(remove_log=lambda vmid: None), + 'guest_node': lambda vmid: None, 'remove_host_state': lambda vmid: None, 'release_shared_host_files': lambda hookscripts: None, 're': re, 'translate': lambda text: text, 'msg_info': lambda text: None, 'msg_ok': lambda text: events.append(('ok', text)), 'msg_warn': lambda text: None} extract(REMOVE, 'remove', scope)(Path('/inert'), 101) diff --git a/.github/scripts/tests/test_oci_selection_setup_wording.py b/.github/scripts/tests/test_oci_selection_setup_wording.py index 6e721836..ab63e5d0 100644 --- a/.github/scripts/tests/test_oci_selection_setup_wording.py +++ b/.github/scripts/tests/test_oci_selection_setup_wording.py @@ -40,6 +40,7 @@ class SelectionSetupWording(TestCase): ui = SimpleNamespace(message=Mock(), choose=Mock()) fn = extracted('_interactive_management', os=SimpleNamespace(geteuid=lambda: 0), shutil=SimpleNamespace(which=lambda _: '/fake/pct'), saved_inventory=lambda _: [], + _clean_orphans=lambda project: None, translate=lambda s: s) fn(Path('/fixture'), ui) ui.message.assert_called_once_with(EMPTY, 'OCI management') diff --git a/AppImage/messages/de/common.json b/AppImage/messages/de/common.json index 2bceb45a..4b0a1b40 100644 --- a/AppImage/messages/de/common.json +++ b/AppImage/messages/de/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname}: Clusterereignis gemeldet", - "body": "Ein Clusterereignis wurde gemeldet. Prüfen Sie den Clusterstatus und das ursprüngliche Ereignis für weitere Informationen.", + "body": "Ein Clusterereignis wurde gemeldet:\n{reason}", "label": "Clusterereignis" }, "node_disconnect": { diff --git a/AppImage/messages/en/common.json b/AppImage/messages/en/common.json index ccd1e250..9a8a5de4 100644 --- a/AppImage/messages/en/common.json +++ b/AppImage/messages/en/common.json @@ -6247,5 +6247,5 @@ "cancel": "Cancel" } }, - "runtime": {"notifications":{"templates":{"state_change":{"title":"{hostname}: {category} changed to {current}{entity_suffix}","body":"{category} status changed from {previous} to {current}.\n{reason}","label":"Health state changed"},"new_error":{"title":"{hostname}: New {severity} - {category}{entity_suffix}","body":"{reason}","label":"New health issue"},"error_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"The {category} issue has been resolved.\n{reason}\n🚦 Previous severity: {original_severity}\n⏱️ Duration: {duration}","label":"Recovery notification"},"error_escalated":{"title":"{hostname}: Escalated to {severity} - {category}{entity_suffix}","body":"{reason}","label":"Health issue escalated"},"health_degraded":{"title":"{title_or_default}","body":"{reason}","label":"Health check degraded"},"lxc_updates_available":{"title":"{hostname}: {count} LXC(s) with package updates available","body":"📊 {count} LXC(s) with pending package updates (📦 {total_packages} total, 🔒 {security_count} security):\n\n{ct_list}","label":"LXC updates available (experimental)"},"lxc_update_applied":{"title":"{hostname}: LXC {ct_name} ({vmid}) update {result}","body":"{details}","label":"LXC update applied"},"app_update_available":{"title":"{hostname}: {app_name} update available on CT {vmid}","body":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","label":"App update available"},"docker_stack_update_available":{"title":"{hostname}: Docker updates available on CT {vmid}","body":"Container {ct_name} (CT {vmid}) has {count} Docker update(s):\n{details}","label":"Docker updates available"},"vm_start":{"title":"{hostname}: VM {vmname} ({vmid}) started","body":"Virtual machine {vmname} (ID: {vmid}) is now running.","label":"VM started"},"vm_start_warning":{"title":"{hostname}: VM {vmname} ({vmid}) started with warnings","body":"Virtual machine {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"VM started (warnings)"},"vm_stop":{"title":"{hostname}: VM {vmname} ({vmid}) stopped","body":"Virtual machine {vmname} (ID: {vmid}) has been stopped.","label":"VM stopped"},"vm_shutdown":{"title":"{hostname}: VM {vmname} ({vmid}) shut down","body":"Virtual machine {vmname} (ID: {vmid}) has been cleanly shut down.","label":"VM shutdown"},"vm_fail":{"title":"{hostname}: VM {vmname} ({vmid}) FAILED","body":"Virtual machine {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"VM FAILED"},"vm_restart":{"title":"{hostname}: VM {vmname} ({vmid}) restarted","body":"Virtual machine {vmname} (ID: {vmid}) has been restarted.","label":"VM restarted"},"ct_start":{"title":"{hostname}: CT {vmname} ({vmid}) started","body":"Container {vmname} (ID: {vmid}) is now running.","label":"CT started"},"ct_start_warning":{"title":"{hostname}: CT {vmname} ({vmid}) started with warnings","body":"Container {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"CT started (warnings)"},"ct_stop":{"title":"{hostname}: CT {vmname} ({vmid}) stopped","body":"Container {vmname} (ID: {vmid}) has been stopped.","label":"CT stopped"},"ct_shutdown":{"title":"{hostname}: CT {vmname} ({vmid}) shut down","body":"Container {vmname} (ID: {vmid}) has been cleanly shut down.","label":"CT shutdown"},"ct_restart":{"title":"{hostname}: CT {vmname} ({vmid}) restarted","body":"Container {vmname} (ID: {vmid}) has been restarted.","label":"CT restarted"},"ct_fail":{"title":"{hostname}: CT {vmname} ({vmid}) FAILED","body":"Container {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"CT FAILED"},"migration_start":{"title":"{hostname}: Migration started — {vmname} ({vmid})","body":"Live migration of {vmname} (ID: {vmid}) to node {target_node} has started.","label":"Migration started"},"migration_complete":{"title":"{hostname}: Migration complete — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) successfully migrated to node {target_node}.","label":"Migration complete"},"migration_warning":{"title":"{hostname}: Migration complete with warnings — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) migrated to node {target_node} but encountered warnings.\nWarnings: {reason}","label":"Migration (warnings)"},"migration_fail":{"title":"{hostname}: Migration FAILED — {vmname} ({vmid})","body":"Migration of {vmname} (ID: {vmid}) to node {target_node} failed.\nReason: {reason}","label":"Migration FAILED"},"replication_fail":{"title":"{hostname}: Replication FAILED — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Replication FAILED"},"replication_complete":{"title":"{hostname}: Replication complete — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) completed successfully.","label":"Replication complete"},"backup_start":{"title":"{hostname}: Backup started on {storage}","body":"Backup job started on storage {storage}.\n{reason}","label":"Backup started"},"backup_complete":{"title":"{hostname} → {storage}: Backup complete — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}","label":"Backup complete"},"backup_warning":{"title":"{hostname} → {storage}: Backup complete with warnings — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) on {storage} completed but encountered warnings.\nWarnings: {reason}","label":"Backup (warnings)"},"backup_fail":{"title":"{hostname} → {storage}: Backup FAILED — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) failed on {storage}.\nReason: {reason}","label":"Backup FAILED"},"host_backup_start":{"title":"{hostname}: Host backup started → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nProfile: {profile_mode}","label":"Host backup started"},"host_backup_complete":{"title":"{hostname}: Host backup complete → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nData size: {data_size}\nArchive size: {archive_size}\nDuration: {duration}","label":"Host backup complete"},"host_backup_fail":{"title":"{hostname}: Host backup FAILED → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nDuration before failure: {duration}\nReason: {reason}\nLog: {log_file}","label":"Host backup FAILED"},"snapshot_complete":{"title":"{hostname}: Snapshot created — {vmname} ({vmid})","body":"Snapshot \"{snapshot_name}\" created for {vmname} (ID: {vmid}).","label":"Snapshot created"},"snapshot_fail":{"title":"{hostname}: Snapshot FAILED — {vmname} ({vmid})","body":"Snapshot creation for {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Snapshot FAILED"},"cpu_high":{"title":"{hostname}: High CPU usage — {value}%","body":"CPU usage has reached {value}% on {cores} cores.\n{details}","label":"High CPU usage"},"ram_high":{"title":"{hostname}: High memory usage — {value}%","body":"Memory usage: {used} / {total} ({value}%).\n{details}","label":"High memory usage"},"temp_high":{"title":"{hostname}: High CPU temperature — {value}°C","body":"CPU temperature has reached {value}°C (threshold: {threshold}°C).\n{details}","label":"High temperature"},"disk_space_low":{"title":"{hostname}: Low disk space on {mount}","body":"Filesystem {mount}: {used}% used ({available} available).\nFree up disk space to avoid service disruption.","label":"Low disk space"},"disk_io_error":{"title":"{hostname}: Disk failure detected — {device}","body":"I/O error or disk failure detected on device {device}.\n{reason}","label":"Disk failure / I/O error"},"storage_unavailable":{"title":"{hostname}: Storage unavailable — {storage_name}","body":"PVE storage \"{storage_name}\" (type: {storage_type}) is not accessible.\nReason: {reason}","label":"Storage unavailable"},"smart_test_complete":{"title":"{hostname}: SMART test completed — {device}","body":"SMART {test_type} test on /dev/{device} has completed.\nResult: {result}\nDuration: {duration}","label":"SMART test completed"},"smart_test_failed":{"title":"{hostname}: SMART test FAILED — {device}","body":"SMART {test_type} test on /dev/{device} has failed.\nResult: {result}\nReason: {reason}","label":"SMART test FAILED"},"gpu_mode_switch":{"title":"{hostname}: GPU mode changed to {new_mode}","body":"GPU passthrough mode has been switched.\nGPU: {gpu_name} ({gpu_pci})\nPrevious mode: {old_mode}\nNew mode: {new_mode}\n{details}","label":"GPU mode switched"},"gpu_passthrough_blocked":{"title":"{hostname}: {guest_type} {guest_id} blocked at startup","body":"PCIe passthrough guard prevented {guest_type} {guest_id} ({guest_name}) from starting.\nReason: {reason}\n{details}","label":"GPU passthrough blocked"},"pci_passthrough_conflict":{"title":"{hostname}: PCIe device conflict detected — {device_pci}","body":"A PCIe device is assigned to multiple guests.\nDevice: {device_pci}\nConflicting guests: {guest_list}\nAction required: Stop one of the guests or reassign the device.","label":"PCIe device conflict"},"load_high":{"title":"{hostname}: High system load — {value}","body":"System load average is {value} on {cores} cores.\n{details}","label":"High system load"},"network_down":{"title":"{hostname}: Network connectivity lost{entity_suffix}","body":"The node has lost network connectivity.\nReason: {reason}","label":"Network connectivity lost"},"network_latency":{"title":"{hostname}: High network latency — {value}ms","body":"Latency to gateway: {value}ms (threshold: {threshold}ms).\nThis may indicate network congestion or hardware issues.","label":"High network latency"},"auth_fail":{"title":"{hostname}: Authentication failure","body":"Failed login attempt detected.\nSource IP: {source_ip}\nUser: {username}\nService: {service}","label":"Authentication failure"},"ip_block":{"title":"{hostname}: IP blocked by Fail2Ban","body":"IP address {source_ip} has been banned.\nJail: {jail}\nFailed attempts: {failures}","label":"IP blocked by Fail2Ban"},"firewall_issue":{"title":"{hostname}: Firewall issue detected{entity_suffix}","body":"A firewall configuration issue has been detected.\nReason: {reason}","label":"Firewall issue detected"},"user_permission_change":{"title":"{hostname}: User permission changed","body":"User: {username}\nChange: {change_details}","label":"User permission changed"},"split_brain":{"title":"{hostname}: Cluster event reported","body":"A cluster event was reported. Review cluster status and the source event for details.","label":"Cluster event"},"node_disconnect":{"title":"{hostname}: Node {node_name} disconnected","body":"Node {node_name} has disconnected from the cluster.","label":"Node disconnected"},"node_reconnect":{"title":"{hostname}: Node {node_name} reconnected","body":"Node {node_name} has rejoined the cluster successfully.","label":"Node reconnected"},"system_startup":{"title":"{hostname}: {reason}","body":"{summary}","label":"System startup report"},"system_shutdown":{"title":"{hostname}: System shutting down","body":"The node is shutting down.\n{reason}","label":"System shutting down"},"system_reboot":{"title":"{hostname}: System rebooting","body":"The node is rebooting.\n{reason}","label":"System rebooting"},"system_restore_completed":{"title":"{hostname}: Host restore finished","body":"Post-restore tasks completed in background.\n\nGuests applied: {guests}\nBind-mount stubs: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}\nThe node is now fully ready to use.","label":"Host restore completed"},"system_problem":{"title":"{hostname}: System problem detected{entity_suffix}","body":"A system-level problem has been detected.\nReason: {reason}","label":"System problem detected"},"service_fail":{"title":"{hostname}: Service failed — {service_name}","body":"System service \"{service_name}\" has failed.\nReason: {reason}","label":"Service failed"},"oom_kill":{"title":"{hostname}: OOM Kill — {process}","body":"Process \"{process}\" was killed by the Out-of-Memory manager.\n{reason}","label":"Out of memory kill"},"service_fail_batch":{"title":"{hostname}: {service_count} services failed{entity_suffix}","body":"{reason}","label":"Service fail batch"},"cron_output":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Cron job output (per-cron stdout via mail)"},"system_mail":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Smartd / mail bounces (PVE system mail)"},"apt_listchanges":{"title":"{hostname}: {pve_title}","body":"Upstream package information forwarded by Proxmox VE through apt-listchanges. The following text comes from the package maintainer and is not a ProxMenux recommendation.\n\n{reason}","label":"apt-listchanges package notices"},"webhook_test":{"title":"{hostname}: Webhook test received","body":"PVE webhook connectivity test successful.\n{reason}","label":"Webhook test"},"update_available":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity: {security_count}\nProxmox: {pve_count}\nKernel: {kernel_count}\nImportant packages:\n{important_list}","label":"Updates available (legacy)"},"unknown_persistent":{"title":"{hostname}: Check unavailable - {category}{entity_suffix}","body":"Health check for {category} has been unavailable for 3+ cycles.\n{reason}","label":"Check unavailable"},"health_persistent":{"title":"{hostname}: {count} active health issue(s){entity_suffix}","body":"The following health issues remain unresolved:\n{issue_list}\n\nThis digest is sent once every 24 hours while issues persist.","label":"Active health issues (daily)"},"health_issue_new":{"title":"{hostname}: New health issue — {category}{entity_suffix}","body":"New {severity} issue detected in: {category}\nDetails: {reason}","label":"New health issue"},"health_issue_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"{category} issue has been resolved.\n{reason}\nDuration: {duration}","label":"Health issue resolved"},"update_summary":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity updates: {security_count}\nProxmox-related updates: {pve_count}\nKernel updates: {kernel_count}\nImportant packages:\n{important_list}","label":"Host package updates"},"pve_update":{"title":"{hostname}: Proxmox VE {new_version} available","body":"A new Proxmox VE release is available.\nCurrent: {current_version}\nNew: {new_version}\n{details}","label":"Proxmox VE update available"},"update_complete":{"title":"{hostname}: System update completed","body":"System packages have been successfully updated.\n{details}","label":"Host update completed"},"ai_model_migrated":{"title":"{hostname}: AI model updated","body":"The AI model for notifications has been automatically updated.\nProvider: {provider}\nPrevious model: {old_model}\nNew model: {new_model}\n\n{message}","label":"AI model auto-updated"},"proxmenux_update":{"title":"{hostname}: ProxMenux {new_version} available","body":"A new version of ProxMenux is available.\nCurrent: {current_version}\nNew: {new_version}","label":"ProxMenux update available"},"mount_stale":{"title":"{hostname}: stale remote mount {mount_target}","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is stale{lxc_scope}.\nStat timed out or returned an error: {error}\n\nApps writing to this path will silently land on the underlying filesystem and may fill the disk. Remount or fix connectivity ASAP.","label":"Remote mount stale"},"mount_readonly":{"title":"{hostname}: remote mount {mount_target} is read-only","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is mounted read-only{lxc_scope}. Writes will fail. If this was unintentional, remount with rw.","label":"Remote mount read-only"},"lxc_disk_low":{"title":"{hostname}: CT {vmid} rootfs at {usage_percent}%","body":"CT {vmid} ({name}) rootfs is at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nA full LXC rootfs prevents the container from booting cleanly. Either expand the rootfs (pct resize {vmid} rootfs +1G) or free space inside the container.","label":"LXC rootfs near full"},"vm_disk_low":{"title":"{hostname}: VM {vmid} filesystems at {usage_percent}%","body":"VM {vmid} ({name}) guest filesystems are at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nReported by the QEMU guest agent. Includes every persistent filesystem the guest mounts on a block device — virtual disks and PCI-passthrough drives alike. Free up space inside the guest or expand the affected storage before writes start to fail.","label":"VM filesystems near full"},"lxc_mount_low":{"title":"{hostname}: CT {vmid} mount {mount} at {usage_percent}%","body":"Mount {mount} inside CT {vmid} ({name}) is at {usage_percent}% used.\nFilesystem type: {fstype}\n\nA full mount inside a container often blocks the application silently — writes either fail or, worse, land on the rootfs and trigger the rootfs alert next. Free up space on the mount or expand it.","label":"LXC mount near full"},"pve_storage_full":{"title":"{hostname}: PVE storage {storage_name} at {usage_percent}%","body":"Proxmox storage \"{storage_name}\" (type: {storage_type}) is at {usage_percent}% used.\n\nOnce full, no new VM/CT can be provisioned and existing guests may fail to write. Move/delete unused volumes or expand the underlying pool/LV/RBD image.","label":"PVE storage near full"},"zfs_pool_full":{"title":"{hostname}: ZFS pool {pool_name} at {usage_percent}%","body":"ZFS pool \"{pool_name}\" is at {usage_percent}% capacity.\n\nZFS performance and write reliability degrade sharply above ~80% capacity (CoW needs free space for new blocks). Free up snapshots, prune old datasets, or add more vdevs to the pool.","label":"ZFS pool near full"},"post_install_update":{"title":"{hostname}: {count} ProxMenux optimization update(s) available","body":"{count} ProxMenux optimization update(s) available on this host.\n\n🛠️ Available versions:\n{tool_list}\n\n💡 Apply from:\n • ProxMenux Monitor → Settings → ProxMenux Optimizations\n • Or run the post-install menu (option 2) → \"Apply available updates\"","label":"ProxMenux optimization updates available"},"secure_gateway_update_available":{"title":"{hostname}: {app_name} update available{update_title_suffix}","body":"{app_name} (managed by ProxMenux) has 📦 {package_count} package update(s) pending in its container.\n{version_line}\n\n💡 Open ProxMenux Monitor > Settings > Secure Gateway and click \"Update\" to apply.\n\n🗂️ Packages:\n{package_list}","label":"Secure Gateway update available"},"nvidia_driver_update_available":{"title":"{hostname}: NVIDIA driver update available — v{latest_version}","body":"A newer maintenance release is available for the installed NVIDIA driver branch.\n🔹 Currently installed: v{current_version}\n🟢 Latest available: v{latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\nReinstalling rebuilds the DKMS module against the running kernel and requires a reboot to load the new driver.","label":"NVIDIA driver update available"},"coral_driver_update_available":{"title":"{hostname}: Coral TPU driver update available — {latest_version}","body":"A newer {variant_label} is available.\n🔹 Currently installed: {current_version}\n🟢 Latest available: {latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\n{reboot_note}","label":"Coral TPU driver update available"},"burst_auth_fail":{"title":"{hostname}: +{count} more auth failures in {window}","body":"+{count} additional authentication failures detected in {window} ({total_count} total).\nSources: {entity_list}","label":"Auth failures burst"},"burst_ip_block":{"title":"{hostname}: Fail2Ban banned +{count} more IPs in {window}","body":"+{count} additional IPs banned by Fail2Ban in {window} ({total_count} total).\nIPs: {entity_list}","label":"IP block burst"},"burst_disk_io":{"title":"{hostname}: +{count} more disk I/O errors on {entity_list}","body":"+{count} additional I/O errors detected in {window} ({total_count} total).\nDevices: {entity_list}","label":"Disk I/O burst"},"burst_cluster":{"title":"{hostname}: Cluster flapping detected (+{count} more changes)","body":"Cluster state changed +{count} more times in {window} ({total_count} total).\nNodes: {entity_list}","label":"Cluster flapping burst"},"burst_service_fail":{"title":"{hostname}: +{count} more services failed in {window}","body":"+{count} additional service failures detected in {window} ({total_count} total).\nThis typically indicates a node reboot or PVE service restart.\n\nAdditional failures:\n{details}","label":"Service fail burst"},"burst_system":{"title":"{hostname}: +{count} more system problems in {window}","body":"+{count} additional system problems detected in {window} ({total_count} total).\n\nAdditional issues:\n{details}","label":"System problems burst"},"burst_generic":{"title":"{hostname}: +{count} more {event_type} events in {window}","body":"+{count} additional events of type {event_type} in {window} ({total_count} total).\n\nAdditional events:\n{details}","label":"Generic burst"},"kernel_warning":{"title":"{hostname}: Kernel diagnostic event detected","body":"The kernel recorded a diagnostic event: {kernel_details}","label":"Kernel warnings and diagnostic traces"}},"lxcUpdate":{"status":{"success":"succeeded","failure":"failed","partial":"completed partially","deferred":"deferred","skipped":"skipped"},"source":{"scheduled":"Scheduled","manual":"Manual"},"sourceLabel":"Source","targets":"Targets","osPending":"OS packages pending: {before} → {after}","osUnverified":"OS packages: update command executed; result not verified","applications":"Applications: {items}","applicationsUnverified":"Applications: updater executed; no tracked version change was observed","dockerEngineChange":"Docker Engine: {before} → {after}","dockerEngineVerified":"Docker Engine: verified at {version}","dockerEngineUnverified":"Docker Engine: update command executed; version not verified","dockerImagesPending":"Docker images pending: {before} → {after}","dockerImagesChanged":"Docker images changed: {images}","deferredTargets":"Deferred targets: {targets}","reason":"Reason: {reason}","restartRequired":"Restart required: {value}","yes":"yes","no":"no","verificationPending":"Verification pending until the container is running","verificationWarning":"Verification warning: {error}","duration":"Duration: {duration}"},"fallback":{"unknownTitle":"{hostname}: {event_type}","healthCheckDegraded":"Health check degraded","none":"none"},"fields":{"vmid":"VM/CT","name":"Name","device":"Device","sourceIp":"Source IP","node":"Node","category":"Category","service":"Service","jail":"Jail","user":"User","count":"Count","window":"Window","affected":"Affected"},"vzdump":{"size":"Size: {value}","duration":"Duration: {value}","file":"File","backups":"{count} backups","failed":"{count} failed","total":"Total: {value}","time":"Time: {value}"},"startup":{"issuesTitle":"{hostname}: System startup - {count} issue(s) detected","completeTitle":"{hostname}: System startup completed","operational":"All systems operational.","vmCountOne":"{count} VM","vmCountMany":"{count} VMs","ctCountOne":"{count} CT","ctCountMany":"{count} CTs","started":"✅ {counts} started","vmFailed":"❌ VM failed: {name} - {reason}","ctFailed":"❌ CT failed: {name} - {reason}","unknownError":"unknown error","storageUnavailable":"⚠️ Storage: {count} unavailable ({names})","servicesFailed":"⚠️ Services: {count} failed ({names})","duration":"⏱️ Startup completed in {minutes} min"},"appUpdates":{"singleTitle":"{hostname}: {app_name} update available on CT {vmid}","singleBody":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","emptyTitle":"{hostname}: Application updates available","emptyBody":"Application updates are available.","batchTitle":"{hostname}: {count} application updates available","batchLeadOneContainer":"{count} applications in {container_count} LXC container have a newer version:","batchLeadManyContainers":"{count} applications in {container_count} LXC containers have a newer version:","additional":"… {count} additional application(s)","app":"app","unknown":"unknown"},"healthDegraded":{"categories":{"cpu":"CPU usage and temperature","memory":"Memory and swap","storage":"Storage mounts and space","disks":"Disk I/O and errors","network":"Network interfaces","vms":"VMs and containers","services":"PVE services","logs":"System logs","updates":"System updates","security":"Security"},"severity":{"critical":"Critical","warning":"Warning"},"singleTitle":"{hostname}: {severity} health status – {category}","multipleTitle":"{hostname}: {count} health checks degraded","multipleLine":"• {severity}: {category}\n {reason}","reasons":{"cpuSustained":"CPU is above {threshold}% for {duration}s."}},"digest":{"title":"{hostname}: 24h summary ({timestamp})","readFailed":"digest read failed: {error}","empty":"No INFO events buffered for this digest window.","lead":"{count} INFO events grouped by category:\n","more":" • … and {count} more","footer":"(Critical/Warning events arrived at the time they happened, not in this digest.)","quietTitle":"{hostname}: {count} events buffered during Quiet Hours","groups":{"vm_ct":"VM a CT","backup":"Backup","resources":"Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"Services","health":"Health","updates":"Updates","hardware":"Hardware","system":"System","other":"Other"}},"test":{"title":"ProxMenux Test","welcome":"Welcome to ProxMenux Monitor!","verify":"This is a test message to verify your notification channel is working correctly.","configuration":"Channel configuration:","iconsEnabled":"Icons: enabled","iconsDisabled":"Icons: disabled","aiEnabled":"AI: enabled ({info})","aiDisabled":"AI: disabled","alerts":"You will receive alerts about system events, health status changes, and security incidents.","photoCaption":"You can use this image as the profile photo for your notification bot."},"channels":{"discord":{"category":"Category","host":"Host","severity":"Severity"},"email":{"report":"{group} Report","details":"Details","host":"Host","footer":"ProxMenux Notification Service","severity":{"critical":"Critical","warning":"Warning","info":"Information","ok":"Resolved","default":"Notice"},"groups":{"vm_ct":"Virtual Machine / Container","backup":"Backup & Snapshot","resources":"System Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"System Services","health":"Health Monitor","updates":"System Updates","system":"System","hardware":"Hardware","other":"System Notification"},"fields":{"vmCtId":"VM/CT ID","name":"Name","action":"Action","targetNode":"Target Node","reason":"Reason","storage":"Storage","status":"Status","size":"Size","duration":"Duration","snapshot":"Snapshot","metric":"Metric","currentValue":"Current Value","threshold":"Threshold","cpuCores":"CPU Cores","memory":"Memory","temperature":"Temperature","mountPoint":"Mount Point","usage":"Usage","available":"Available","device":"Device","severity":"Severity","storageName":"Storage Name","type":"Type","interface":"Interface","latency":"Latency","event":"Event","sourceIp":"Source IP","username":"Username","service":"Service","jail":"Jail","failures":"Failures","change":"Change","node":"Node","quorum":"Quorum","nodesAffected":"Nodes Affected","process":"Process","category":"Category","previousSeverity":"Previous Severity","activeIssues":"Active Issues","totalUpdates":"Total Updates","securityUpdates":"Security Updates","proxmoxUpdates":"Proxmox Updates","kernelUpdates":"Kernel Updates","importantPackages":"Important Packages","currentVersion":"Current Version","newVersion":"New Version"},"status":{"failed":"Failed","completed":"Completed","started":"Started"}}}}} + "runtime": {"notifications":{"templates":{"state_change":{"title":"{hostname}: {category} changed to {current}{entity_suffix}","body":"{category} status changed from {previous} to {current}.\n{reason}","label":"Health state changed"},"new_error":{"title":"{hostname}: New {severity} - {category}{entity_suffix}","body":"{reason}","label":"New health issue"},"error_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"The {category} issue has been resolved.\n{reason}\n🚦 Previous severity: {original_severity}\n⏱️ Duration: {duration}","label":"Recovery notification"},"error_escalated":{"title":"{hostname}: Escalated to {severity} - {category}{entity_suffix}","body":"{reason}","label":"Health issue escalated"},"health_degraded":{"title":"{title_or_default}","body":"{reason}","label":"Health check degraded"},"lxc_updates_available":{"title":"{hostname}: {count} LXC(s) with package updates available","body":"📊 {count} LXC(s) with pending package updates (📦 {total_packages} total, 🔒 {security_count} security):\n\n{ct_list}","label":"LXC updates available (experimental)"},"lxc_update_applied":{"title":"{hostname}: LXC {ct_name} ({vmid}) update {result}","body":"{details}","label":"LXC update applied"},"app_update_available":{"title":"{hostname}: {app_name} update available on CT {vmid}","body":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","label":"App update available"},"docker_stack_update_available":{"title":"{hostname}: Docker updates available on CT {vmid}","body":"Container {ct_name} (CT {vmid}) has {count} Docker update(s):\n{details}","label":"Docker updates available"},"vm_start":{"title":"{hostname}: VM {vmname} ({vmid}) started","body":"Virtual machine {vmname} (ID: {vmid}) is now running.","label":"VM started"},"vm_start_warning":{"title":"{hostname}: VM {vmname} ({vmid}) started with warnings","body":"Virtual machine {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"VM started (warnings)"},"vm_stop":{"title":"{hostname}: VM {vmname} ({vmid}) stopped","body":"Virtual machine {vmname} (ID: {vmid}) has been stopped.","label":"VM stopped"},"vm_shutdown":{"title":"{hostname}: VM {vmname} ({vmid}) shut down","body":"Virtual machine {vmname} (ID: {vmid}) has been cleanly shut down.","label":"VM shutdown"},"vm_fail":{"title":"{hostname}: VM {vmname} ({vmid}) FAILED","body":"Virtual machine {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"VM FAILED"},"vm_restart":{"title":"{hostname}: VM {vmname} ({vmid}) restarted","body":"Virtual machine {vmname} (ID: {vmid}) has been restarted.","label":"VM restarted"},"ct_start":{"title":"{hostname}: CT {vmname} ({vmid}) started","body":"Container {vmname} (ID: {vmid}) is now running.","label":"CT started"},"ct_start_warning":{"title":"{hostname}: CT {vmname} ({vmid}) started with warnings","body":"Container {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"CT started (warnings)"},"ct_stop":{"title":"{hostname}: CT {vmname} ({vmid}) stopped","body":"Container {vmname} (ID: {vmid}) has been stopped.","label":"CT stopped"},"ct_shutdown":{"title":"{hostname}: CT {vmname} ({vmid}) shut down","body":"Container {vmname} (ID: {vmid}) has been cleanly shut down.","label":"CT shutdown"},"ct_restart":{"title":"{hostname}: CT {vmname} ({vmid}) restarted","body":"Container {vmname} (ID: {vmid}) has been restarted.","label":"CT restarted"},"ct_fail":{"title":"{hostname}: CT {vmname} ({vmid}) FAILED","body":"Container {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"CT FAILED"},"migration_start":{"title":"{hostname}: Migration started — {vmname} ({vmid})","body":"Live migration of {vmname} (ID: {vmid}) to node {target_node} has started.","label":"Migration started"},"migration_complete":{"title":"{hostname}: Migration complete — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) successfully migrated to node {target_node}.","label":"Migration complete"},"migration_warning":{"title":"{hostname}: Migration complete with warnings — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) migrated to node {target_node} but encountered warnings.\nWarnings: {reason}","label":"Migration (warnings)"},"migration_fail":{"title":"{hostname}: Migration FAILED — {vmname} ({vmid})","body":"Migration of {vmname} (ID: {vmid}) to node {target_node} failed.\nReason: {reason}","label":"Migration FAILED"},"replication_fail":{"title":"{hostname}: Replication FAILED — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Replication FAILED"},"replication_complete":{"title":"{hostname}: Replication complete — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) completed successfully.","label":"Replication complete"},"backup_start":{"title":"{hostname}: Backup started on {storage}","body":"Backup job started on storage {storage}.\n{reason}","label":"Backup started"},"backup_complete":{"title":"{hostname} → {storage}: Backup complete — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}","label":"Backup complete"},"backup_warning":{"title":"{hostname} → {storage}: Backup complete with warnings — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) on {storage} completed but encountered warnings.\nWarnings: {reason}","label":"Backup (warnings)"},"backup_fail":{"title":"{hostname} → {storage}: Backup FAILED — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) failed on {storage}.\nReason: {reason}","label":"Backup FAILED"},"host_backup_start":{"title":"{hostname}: Host backup started → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nProfile: {profile_mode}","label":"Host backup started"},"host_backup_complete":{"title":"{hostname}: Host backup complete → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nData size: {data_size}\nArchive size: {archive_size}\nDuration: {duration}","label":"Host backup complete"},"host_backup_fail":{"title":"{hostname}: Host backup FAILED → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nDuration before failure: {duration}\nReason: {reason}\nLog: {log_file}","label":"Host backup FAILED"},"snapshot_complete":{"title":"{hostname}: Snapshot created — {vmname} ({vmid})","body":"Snapshot \"{snapshot_name}\" created for {vmname} (ID: {vmid}).","label":"Snapshot created"},"snapshot_fail":{"title":"{hostname}: Snapshot FAILED — {vmname} ({vmid})","body":"Snapshot creation for {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Snapshot FAILED"},"cpu_high":{"title":"{hostname}: High CPU usage — {value}%","body":"CPU usage has reached {value}% on {cores} cores.\n{details}","label":"High CPU usage"},"ram_high":{"title":"{hostname}: High memory usage — {value}%","body":"Memory usage: {used} / {total} ({value}%).\n{details}","label":"High memory usage"},"temp_high":{"title":"{hostname}: High CPU temperature — {value}°C","body":"CPU temperature has reached {value}°C (threshold: {threshold}°C).\n{details}","label":"High temperature"},"disk_space_low":{"title":"{hostname}: Low disk space on {mount}","body":"Filesystem {mount}: {used}% used ({available} available).\nFree up disk space to avoid service disruption.","label":"Low disk space"},"disk_io_error":{"title":"{hostname}: Disk failure detected — {device}","body":"I/O error or disk failure detected on device {device}.\n{reason}","label":"Disk failure / I/O error"},"storage_unavailable":{"title":"{hostname}: Storage unavailable — {storage_name}","body":"PVE storage \"{storage_name}\" (type: {storage_type}) is not accessible.\nReason: {reason}","label":"Storage unavailable"},"smart_test_complete":{"title":"{hostname}: SMART test completed — {device}","body":"SMART {test_type} test on /dev/{device} has completed.\nResult: {result}\nDuration: {duration}","label":"SMART test completed"},"smart_test_failed":{"title":"{hostname}: SMART test FAILED — {device}","body":"SMART {test_type} test on /dev/{device} has failed.\nResult: {result}\nReason: {reason}","label":"SMART test FAILED"},"gpu_mode_switch":{"title":"{hostname}: GPU mode changed to {new_mode}","body":"GPU passthrough mode has been switched.\nGPU: {gpu_name} ({gpu_pci})\nPrevious mode: {old_mode}\nNew mode: {new_mode}\n{details}","label":"GPU mode switched"},"gpu_passthrough_blocked":{"title":"{hostname}: {guest_type} {guest_id} blocked at startup","body":"PCIe passthrough guard prevented {guest_type} {guest_id} ({guest_name}) from starting.\nReason: {reason}\n{details}","label":"GPU passthrough blocked"},"pci_passthrough_conflict":{"title":"{hostname}: PCIe device conflict detected — {device_pci}","body":"A PCIe device is assigned to multiple guests.\nDevice: {device_pci}\nConflicting guests: {guest_list}\nAction required: Stop one of the guests or reassign the device.","label":"PCIe device conflict"},"load_high":{"title":"{hostname}: High system load — {value}","body":"System load average is {value} on {cores} cores.\n{details}","label":"High system load"},"network_down":{"title":"{hostname}: Network connectivity lost{entity_suffix}","body":"The node has lost network connectivity.\nReason: {reason}","label":"Network connectivity lost"},"network_latency":{"title":"{hostname}: High network latency — {value}ms","body":"Latency to gateway: {value}ms (threshold: {threshold}ms).\nThis may indicate network congestion or hardware issues.","label":"High network latency"},"auth_fail":{"title":"{hostname}: Authentication failure","body":"Failed login attempt detected.\nSource IP: {source_ip}\nUser: {username}\nService: {service}","label":"Authentication failure"},"ip_block":{"title":"{hostname}: IP blocked by Fail2Ban","body":"IP address {source_ip} has been banned.\nJail: {jail}\nFailed attempts: {failures}","label":"IP blocked by Fail2Ban"},"firewall_issue":{"title":"{hostname}: Firewall issue detected{entity_suffix}","body":"A firewall configuration issue has been detected.\nReason: {reason}","label":"Firewall issue detected"},"user_permission_change":{"title":"{hostname}: User permission changed","body":"User: {username}\nChange: {change_details}","label":"User permission changed"},"split_brain":{"title":"{hostname}: Cluster event reported","body":"A cluster event was reported:\n{reason}","label":"Cluster event"},"node_disconnect":{"title":"{hostname}: Node {node_name} disconnected","body":"Node {node_name} has disconnected from the cluster.","label":"Node disconnected"},"node_reconnect":{"title":"{hostname}: Node {node_name} reconnected","body":"Node {node_name} has rejoined the cluster successfully.","label":"Node reconnected"},"system_startup":{"title":"{hostname}: {reason}","body":"{summary}","label":"System startup report"},"system_shutdown":{"title":"{hostname}: System shutting down","body":"The node is shutting down.\n{reason}","label":"System shutting down"},"system_reboot":{"title":"{hostname}: System rebooting","body":"The node is rebooting.\n{reason}","label":"System rebooting"},"system_restore_completed":{"title":"{hostname}: Host restore finished","body":"Post-restore tasks completed in background.\n\nGuests applied: {guests}\nBind-mount stubs: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}\nThe node is now fully ready to use.","label":"Host restore completed"},"system_problem":{"title":"{hostname}: System problem detected{entity_suffix}","body":"A system-level problem has been detected.\nReason: {reason}","label":"System problem detected"},"service_fail":{"title":"{hostname}: Service failed — {service_name}","body":"System service \"{service_name}\" has failed.\nReason: {reason}","label":"Service failed"},"oom_kill":{"title":"{hostname}: OOM Kill — {process}","body":"Process \"{process}\" was killed by the Out-of-Memory manager.\n{reason}","label":"Out of memory kill"},"service_fail_batch":{"title":"{hostname}: {service_count} services failed{entity_suffix}","body":"{reason}","label":"Service fail batch"},"cron_output":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Cron job output (per-cron stdout via mail)"},"system_mail":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Smartd / mail bounces (PVE system mail)"},"apt_listchanges":{"title":"{hostname}: {pve_title}","body":"Upstream package information forwarded by Proxmox VE through apt-listchanges. The following text comes from the package maintainer and is not a ProxMenux recommendation.\n\n{reason}","label":"apt-listchanges package notices"},"webhook_test":{"title":"{hostname}: Webhook test received","body":"PVE webhook connectivity test successful.\n{reason}","label":"Webhook test"},"update_available":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity: {security_count}\nProxmox: {pve_count}\nKernel: {kernel_count}\nImportant packages:\n{important_list}","label":"Updates available (legacy)"},"unknown_persistent":{"title":"{hostname}: Check unavailable - {category}{entity_suffix}","body":"Health check for {category} has been unavailable for 3+ cycles.\n{reason}","label":"Check unavailable"},"health_persistent":{"title":"{hostname}: {count} active health issue(s){entity_suffix}","body":"The following health issues remain unresolved:\n{issue_list}\n\nThis digest is sent once every 24 hours while issues persist.","label":"Active health issues (daily)"},"health_issue_new":{"title":"{hostname}: New health issue — {category}{entity_suffix}","body":"New {severity} issue detected in: {category}\nDetails: {reason}","label":"New health issue"},"health_issue_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"{category} issue has been resolved.\n{reason}\nDuration: {duration}","label":"Health issue resolved"},"update_summary":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity updates: {security_count}\nProxmox-related updates: {pve_count}\nKernel updates: {kernel_count}\nImportant packages:\n{important_list}","label":"Host package updates"},"pve_update":{"title":"{hostname}: Proxmox VE {new_version} available","body":"A new Proxmox VE release is available.\nCurrent: {current_version}\nNew: {new_version}\n{details}","label":"Proxmox VE update available"},"update_complete":{"title":"{hostname}: System update completed","body":"System packages have been successfully updated.\n{details}","label":"Host update completed"},"ai_model_migrated":{"title":"{hostname}: AI model updated","body":"The AI model for notifications has been automatically updated.\nProvider: {provider}\nPrevious model: {old_model}\nNew model: {new_model}\n\n{message}","label":"AI model auto-updated"},"proxmenux_update":{"title":"{hostname}: ProxMenux {new_version} available","body":"A new version of ProxMenux is available.\nCurrent: {current_version}\nNew: {new_version}","label":"ProxMenux update available"},"mount_stale":{"title":"{hostname}: stale remote mount {mount_target}","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is stale{lxc_scope}.\nStat timed out or returned an error: {error}\n\nApps writing to this path will silently land on the underlying filesystem and may fill the disk. Remount or fix connectivity ASAP.","label":"Remote mount stale"},"mount_readonly":{"title":"{hostname}: remote mount {mount_target} is read-only","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is mounted read-only{lxc_scope}. Writes will fail. If this was unintentional, remount with rw.","label":"Remote mount read-only"},"lxc_disk_low":{"title":"{hostname}: CT {vmid} rootfs at {usage_percent}%","body":"CT {vmid} ({name}) rootfs is at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nA full LXC rootfs prevents the container from booting cleanly. Either expand the rootfs (pct resize {vmid} rootfs +1G) or free space inside the container.","label":"LXC rootfs near full"},"vm_disk_low":{"title":"{hostname}: VM {vmid} filesystems at {usage_percent}%","body":"VM {vmid} ({name}) guest filesystems are at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nReported by the QEMU guest agent. Includes every persistent filesystem the guest mounts on a block device — virtual disks and PCI-passthrough drives alike. Free up space inside the guest or expand the affected storage before writes start to fail.","label":"VM filesystems near full"},"lxc_mount_low":{"title":"{hostname}: CT {vmid} mount {mount} at {usage_percent}%","body":"Mount {mount} inside CT {vmid} ({name}) is at {usage_percent}% used.\nFilesystem type: {fstype}\n\nA full mount inside a container often blocks the application silently — writes either fail or, worse, land on the rootfs and trigger the rootfs alert next. Free up space on the mount or expand it.","label":"LXC mount near full"},"pve_storage_full":{"title":"{hostname}: PVE storage {storage_name} at {usage_percent}%","body":"Proxmox storage \"{storage_name}\" (type: {storage_type}) is at {usage_percent}% used.\n\nOnce full, no new VM/CT can be provisioned and existing guests may fail to write. Move/delete unused volumes or expand the underlying pool/LV/RBD image.","label":"PVE storage near full"},"zfs_pool_full":{"title":"{hostname}: ZFS pool {pool_name} at {usage_percent}%","body":"ZFS pool \"{pool_name}\" is at {usage_percent}% capacity.\n\nZFS performance and write reliability degrade sharply above ~80% capacity (CoW needs free space for new blocks). Free up snapshots, prune old datasets, or add more vdevs to the pool.","label":"ZFS pool near full"},"post_install_update":{"title":"{hostname}: {count} ProxMenux optimization update(s) available","body":"{count} ProxMenux optimization update(s) available on this host.\n\n🛠️ Available versions:\n{tool_list}\n\n💡 Apply from:\n • ProxMenux Monitor → Settings → ProxMenux Optimizations\n • Or run the post-install menu (option 2) → \"Apply available updates\"","label":"ProxMenux optimization updates available"},"secure_gateway_update_available":{"title":"{hostname}: {app_name} update available{update_title_suffix}","body":"{app_name} (managed by ProxMenux) has 📦 {package_count} package update(s) pending in its container.\n{version_line}\n\n💡 Open ProxMenux Monitor > Settings > Secure Gateway and click \"Update\" to apply.\n\n🗂️ Packages:\n{package_list}","label":"Secure Gateway update available"},"nvidia_driver_update_available":{"title":"{hostname}: NVIDIA driver update available — v{latest_version}","body":"A newer maintenance release is available for the installed NVIDIA driver branch.\n🔹 Currently installed: v{current_version}\n🟢 Latest available: v{latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\nReinstalling rebuilds the DKMS module against the running kernel and requires a reboot to load the new driver.","label":"NVIDIA driver update available"},"coral_driver_update_available":{"title":"{hostname}: Coral TPU driver update available — {latest_version}","body":"A newer {variant_label} is available.\n🔹 Currently installed: {current_version}\n🟢 Latest available: {latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\n{reboot_note}","label":"Coral TPU driver update available"},"burst_auth_fail":{"title":"{hostname}: +{count} more auth failures in {window}","body":"+{count} additional authentication failures detected in {window} ({total_count} total).\nSources: {entity_list}","label":"Auth failures burst"},"burst_ip_block":{"title":"{hostname}: Fail2Ban banned +{count} more IPs in {window}","body":"+{count} additional IPs banned by Fail2Ban in {window} ({total_count} total).\nIPs: {entity_list}","label":"IP block burst"},"burst_disk_io":{"title":"{hostname}: +{count} more disk I/O errors on {entity_list}","body":"+{count} additional I/O errors detected in {window} ({total_count} total).\nDevices: {entity_list}","label":"Disk I/O burst"},"burst_cluster":{"title":"{hostname}: Cluster flapping detected (+{count} more changes)","body":"Cluster state changed +{count} more times in {window} ({total_count} total).\nNodes: {entity_list}","label":"Cluster flapping burst"},"burst_service_fail":{"title":"{hostname}: +{count} more services failed in {window}","body":"+{count} additional service failures detected in {window} ({total_count} total).\nThis typically indicates a node reboot or PVE service restart.\n\nAdditional failures:\n{details}","label":"Service fail burst"},"burst_system":{"title":"{hostname}: +{count} more system problems in {window}","body":"+{count} additional system problems detected in {window} ({total_count} total).\n\nAdditional issues:\n{details}","label":"System problems burst"},"burst_generic":{"title":"{hostname}: +{count} more {event_type} events in {window}","body":"+{count} additional events of type {event_type} in {window} ({total_count} total).\n\nAdditional events:\n{details}","label":"Generic burst"},"kernel_warning":{"title":"{hostname}: Kernel diagnostic event detected","body":"The kernel recorded a diagnostic event: {kernel_details}","label":"Kernel warnings and diagnostic traces"}},"lxcUpdate":{"status":{"success":"succeeded","failure":"failed","partial":"completed partially","deferred":"deferred","skipped":"skipped"},"source":{"scheduled":"Scheduled","manual":"Manual"},"sourceLabel":"Source","targets":"Targets","osPending":"OS packages pending: {before} → {after}","osUnverified":"OS packages: update command executed; result not verified","applications":"Applications: {items}","applicationsUnverified":"Applications: updater executed; no tracked version change was observed","dockerEngineChange":"Docker Engine: {before} → {after}","dockerEngineVerified":"Docker Engine: verified at {version}","dockerEngineUnverified":"Docker Engine: update command executed; version not verified","dockerImagesPending":"Docker images pending: {before} → {after}","dockerImagesChanged":"Docker images changed: {images}","deferredTargets":"Deferred targets: {targets}","reason":"Reason: {reason}","restartRequired":"Restart required: {value}","yes":"yes","no":"no","verificationPending":"Verification pending until the container is running","verificationWarning":"Verification warning: {error}","duration":"Duration: {duration}"},"fallback":{"unknownTitle":"{hostname}: {event_type}","healthCheckDegraded":"Health check degraded","none":"none"},"fields":{"vmid":"VM/CT","name":"Name","device":"Device","sourceIp":"Source IP","node":"Node","category":"Category","service":"Service","jail":"Jail","user":"User","count":"Count","window":"Window","affected":"Affected"},"vzdump":{"size":"Size: {value}","duration":"Duration: {value}","file":"File","backups":"{count} backups","failed":"{count} failed","total":"Total: {value}","time":"Time: {value}"},"startup":{"issuesTitle":"{hostname}: System startup - {count} issue(s) detected","completeTitle":"{hostname}: System startup completed","operational":"All systems operational.","vmCountOne":"{count} VM","vmCountMany":"{count} VMs","ctCountOne":"{count} CT","ctCountMany":"{count} CTs","started":"✅ {counts} started","vmFailed":"❌ VM failed: {name} - {reason}","ctFailed":"❌ CT failed: {name} - {reason}","unknownError":"unknown error","storageUnavailable":"⚠️ Storage: {count} unavailable ({names})","servicesFailed":"⚠️ Services: {count} failed ({names})","duration":"⏱️ Startup completed in {minutes} min"},"appUpdates":{"singleTitle":"{hostname}: {app_name} update available on CT {vmid}","singleBody":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","emptyTitle":"{hostname}: Application updates available","emptyBody":"Application updates are available.","batchTitle":"{hostname}: {count} application updates available","batchLeadOneContainer":"{count} applications in {container_count} LXC container have a newer version:","batchLeadManyContainers":"{count} applications in {container_count} LXC containers have a newer version:","additional":"… {count} additional application(s)","app":"app","unknown":"unknown"},"healthDegraded":{"categories":{"cpu":"CPU usage and temperature","memory":"Memory and swap","storage":"Storage mounts and space","disks":"Disk I/O and errors","network":"Network interfaces","vms":"VMs and containers","services":"PVE services","logs":"System logs","updates":"System updates","security":"Security"},"severity":{"critical":"Critical","warning":"Warning"},"singleTitle":"{hostname}: {severity} health status – {category}","multipleTitle":"{hostname}: {count} health checks degraded","multipleLine":"• {severity}: {category}\n {reason}","reasons":{"cpuSustained":"CPU is above {threshold}% for {duration}s."}},"digest":{"title":"{hostname}: 24h summary ({timestamp})","readFailed":"digest read failed: {error}","empty":"No INFO events buffered for this digest window.","lead":"{count} INFO events grouped by category:\n","more":" • … and {count} more","footer":"(Critical/Warning events arrived at the time they happened, not in this digest.)","quietTitle":"{hostname}: {count} events buffered during Quiet Hours","groups":{"vm_ct":"VM a CT","backup":"Backup","resources":"Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"Services","health":"Health","updates":"Updates","hardware":"Hardware","system":"System","other":"Other"}},"test":{"title":"ProxMenux Test","welcome":"Welcome to ProxMenux Monitor!","verify":"This is a test message to verify your notification channel is working correctly.","configuration":"Channel configuration:","iconsEnabled":"Icons: enabled","iconsDisabled":"Icons: disabled","aiEnabled":"AI: enabled ({info})","aiDisabled":"AI: disabled","alerts":"You will receive alerts about system events, health status changes, and security incidents.","photoCaption":"You can use this image as the profile photo for your notification bot."},"channels":{"discord":{"category":"Category","host":"Host","severity":"Severity"},"email":{"report":"{group} Report","details":"Details","host":"Host","footer":"ProxMenux Notification Service","severity":{"critical":"Critical","warning":"Warning","info":"Information","ok":"Resolved","default":"Notice"},"groups":{"vm_ct":"Virtual Machine / Container","backup":"Backup & Snapshot","resources":"System Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"System Services","health":"Health Monitor","updates":"System Updates","system":"System","hardware":"Hardware","other":"System Notification"},"fields":{"vmCtId":"VM/CT ID","name":"Name","action":"Action","targetNode":"Target Node","reason":"Reason","storage":"Storage","status":"Status","size":"Size","duration":"Duration","snapshot":"Snapshot","metric":"Metric","currentValue":"Current Value","threshold":"Threshold","cpuCores":"CPU Cores","memory":"Memory","temperature":"Temperature","mountPoint":"Mount Point","usage":"Usage","available":"Available","device":"Device","severity":"Severity","storageName":"Storage Name","type":"Type","interface":"Interface","latency":"Latency","event":"Event","sourceIp":"Source IP","username":"Username","service":"Service","jail":"Jail","failures":"Failures","change":"Change","node":"Node","quorum":"Quorum","nodesAffected":"Nodes Affected","process":"Process","category":"Category","previousSeverity":"Previous Severity","activeIssues":"Active Issues","totalUpdates":"Total Updates","securityUpdates":"Security Updates","proxmoxUpdates":"Proxmox Updates","kernelUpdates":"Kernel Updates","importantPackages":"Important Packages","currentVersion":"Current Version","newVersion":"New Version"},"status":{"failed":"Failed","completed":"Completed","started":"Started"}}}}} } diff --git a/AppImage/messages/es/common.json b/AppImage/messages/es/common.json index 34a501fb..975968a4 100644 --- a/AppImage/messages/es/common.json +++ b/AppImage/messages/es/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname}: evento del clúster notificado", - "body": "Se ha notificado un evento del clúster. Consulta el estado del clúster y el evento original para obtener más detalles.", + "body": "Se ha notificado un evento del clúster:\n{reason}", "label": "Evento del clúster" }, "node_disconnect": { diff --git a/AppImage/messages/fr/common.json b/AppImage/messages/fr/common.json index 13924861..5724d59e 100644 --- a/AppImage/messages/fr/common.json +++ b/AppImage/messages/fr/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname} : événement du cluster signalé", - "body": "Un événement du cluster a été signalé. Vérifiez l’état du cluster et l’événement d’origine pour plus de détails.", + "body": "Un événement du cluster a été signalé :\n{reason}", "label": "Événement du cluster" }, "node_disconnect": { diff --git a/AppImage/messages/it/common.json b/AppImage/messages/it/common.json index 09d8e600..7dbfa307 100644 --- a/AppImage/messages/it/common.json +++ b/AppImage/messages/it/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname}: evento del cluster segnalato", - "body": "È stato segnalato un evento del cluster. Controlla lo stato del cluster e l'evento originale per maggiori dettagli.", + "body": "È stato segnalato un evento del cluster:\n{reason}", "label": "Evento del cluster" }, "node_disconnect": { diff --git a/AppImage/messages/pt/common.json b/AppImage/messages/pt/common.json index 5d7a7b4f..37859a99 100644 --- a/AppImage/messages/pt/common.json +++ b/AppImage/messages/pt/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname}: evento do cluster comunicado", - "body": "Foi comunicado um evento do cluster. Verifique o estado do cluster e o evento original para obter mais detalhes.", + "body": "Foi comunicado um evento do cluster:\n{reason}", "label": "Evento do cluster" }, "node_disconnect": { diff --git a/AppImage/messages/sk/common.json b/AppImage/messages/sk/common.json index fddccd4e..617e6cef 100644 --- a/AppImage/messages/sk/common.json +++ b/AppImage/messages/sk/common.json @@ -6522,7 +6522,7 @@ }, "split_brain": { "title": "{hostname}: hlásená udalosť klastra", - "body": "Bola hlásená udalosť klastra. Skontrolujte stav klastra a pôvodnú udalosť, kde nájdete ďalšie podrobnosti.", + "body": "Bola hlásená udalosť klastra:\n{reason}", "label": "Udalosť klastra" }, "node_disconnect": { diff --git a/AppImage/messages/sv/common.json b/AppImage/messages/sv/common.json index a42d9e71..6e84f468 100644 --- a/AppImage/messages/sv/common.json +++ b/AppImage/messages/sv/common.json @@ -6523,7 +6523,7 @@ }, "split_brain": { "title": "{hostname}: klusterhändelse rapporterad", - "body": "En klusterhändelse har rapporterats. Kontrollera klusterstatus och den ursprungliga händelsen för mer information.", + "body": "En klusterhändelse har rapporterats:\n{reason}", "label": "Klusterhändelse" }, "node_disconnect": { diff --git a/AppImage/scripts/notification_events.py b/AppImage/scripts/notification_events.py index 2ff24767..6269f6ad 100644 --- a/AppImage/scripts/notification_events.py +++ b/AppImage/scripts/notification_events.py @@ -4367,8 +4367,9 @@ class ProxmoxHookWatcher: # "🔵 constructor: Problema del sistema detectado" / "+1 problema # más del sistema (Problemas adicionales: - problema_del_sistema)" # messages the operator sees on Telegram. Preserve the PVE payload - # as `reason` so both surfaces have concrete text to render. - if event_type == 'system_problem': + # as `reason` so both surfaces have concrete text to render. A cluster + # event (fencing) renders it too: the message is what happened. + if event_type in ('system_problem', 'split_brain'): reason_text = (message or title or '').strip() if reason_text: data['reason'] = reason_text[:500] diff --git a/AppImage/scripts/notification_templates.py b/AppImage/scripts/notification_templates.py index 9d32e2ef..9ce16074 100644 --- a/AppImage/scripts/notification_templates.py +++ b/AppImage/scripts/notification_templates.py @@ -1218,7 +1218,7 @@ TEMPLATES = { # ── Cluster events ── 'split_brain': { 'title': '{hostname}: Cluster event reported', - 'body': 'A cluster event was reported. Review cluster status and the source event for details.', + 'body': 'A cluster event was reported:\n{reason}', 'label': 'Cluster event', 'group': 'cluster', 'default_enabled': True, diff --git a/lang/es.json b/lang/es.json index 60d08564..fcc277f2 100644 --- a/lang/es.json +++ b/lang/es.json @@ -67,7 +67,7 @@ "A legacy gasket-dkms package was found on this host, but no Coral M.2 / PCIe hardware is present.": "Se ha encontrado un paquete heredado gasket-dkms en este host, pero no hay ningún dispositivo Coral M.2/PCIe.", "A managed rootfs and an unprivileged container are required": "Un rootf gestionado y un contenedor no privilegiado son necesarios", "A managed volume with backup enabled is required": "Un volumen gestionado con backup habilitado es necesario", - "A matching managed host firewall rule may also be removed.": "también se puede eliminar una regla de firewall de host administrado coincidente.", + "A matching managed host firewall rule may also be removed.": "También puede eliminarse la regla del cortafuegos del host gestionada por ProxMenux que coincida.", "A member VMID is in use by another guest or is on another node": "Un miembro VMID está en uso por otro invitado o está en otro nodo", "A member configuration changed after the stack was checked": "Una configuración de miembro cambió después de la comprobación de la pila", "A member configuration changed during the preparation": "Una configuración de miembro cambió durante la preparación", @@ -315,7 +315,7 @@ "All images imported and configured successfully": "Todas las imágenes importadas y configuradas correctamente.", "All imports failed": "Todas las importaciones fallaron", "All of them are removed.": "Se eliminan todos.", - "All of them are targeted for removal.": "Todos ellos están destinados a ser eliminados.", + "All of them are targeted for removal.": "Se eliminarán todos.", "All partitions and metadata removed.": "Se eliminaron todas las particiones y metadatos.", "All physical interfaces from backup are present on target": "Todas las interfaces físicas de la copia de seguridad están presentes en el objetivo", "All stack members are updated together. Main CT:": "Todos los miembros de la pila se actualizan juntos. CT principal:", @@ -1076,7 +1076,7 @@ "Container configured (not started):": "Contenedor configurado (no iniciado):", "Container converted to privileged": "Contenedor convertido a privilegiado", "Container created:": "Container created:", - "Container data targeted for deletion:": "Datos del contenedor destinados a ser eliminados:", + "Container data targeted for deletion:": "Datos de los contenedores que se eliminarán:", "Container did not become ready in time. Skipping driver installation.": "El contenedor no estuvo listo a tiempo. Saltarse la instalación del controlador.", "Container did not start in time.": "El contenedor no arrancó a tiempo.", "Container distro": "distribución de contenedores", @@ -1108,7 +1108,7 @@ "Container:": "Contenedor:", "Containers & Docker": "Contenedores y Docker", "Containers returned to their previous state": "Los contenedores regresaron a su estado anterior", - "Containers targeted for removal:": "Contenedores destinados a ser eliminados:", + "Containers targeted for removal:": "Contenedores que se eliminarán:", "Containers that are removed:": "Contenedores que se eliminan:", "Containers that will be created (one LXC per service, on a private network):": "Contenedores que se crearán (un LXC por servicio, en una red privada):", "Containers that will receive this device": "Contenedores que recibirán este dispositivo", @@ -1313,7 +1313,7 @@ "Could not unmount the container filesystem:": "No podía desmontar el sistema de archivos de contenedores:", "Could not unmount — disk may be busy. Removing fstab entry anyway.": "No se pudo desmontar: es posible que el disco esté ocupado. Eliminando la entrada fstab de todos modos.", "Could not update config file.": "No se pudo actualizar el archivo de configuración.", - "Could not verify removal of the managed host firewall rule.": "no se pudo verificar la eliminación de la regla de firewall del host administrado.", + "Could not verify removal of the managed host firewall rule.": "No se pudo verificar la eliminación de la regla del cortafuegos del host gestionada por ProxMenux.", "Could not write to:": "No se pudo escribir a:", "Crafty Controller default login": "Inicio predeterminado de Crafty Controller", "Create Directory": "Crear directorio", @@ -3166,7 +3166,7 @@ "It asks for capabilities or a relaxed confinement profile.": "Pide capacidades o un perfil de confinamiento relajado.", "It asks to see the processes of the host.": "Pide ver los procesos del host.", "It cannot be removed on its own, because the application would stop working: continuing removes the whole application.": "No se puede eliminar por sí solo, porque la aplicación dejaría de funcionar: continuar elimina toda la aplicación.", - "It cannot be removed on its own, because the application would stop working: continuing targets the whole application for removal.": "No se puede eliminar por sí solo, porque la aplicación dejaría de funcionar: continuar apunta a toda la aplicación para su eliminación.", + "It cannot be removed on its own, because the application would stop working: continuing targets the whole application for removal.": "No se puede eliminar por separado, porque la aplicación dejaría de funcionar: si continúas, se eliminará toda la aplicación.", "It is created empty; existing data is not migrated automatically.": "Se crea vacía; los datos existentes no se migran automáticamente.", "It is recommended to create a backup before continuing.": "Se recomienda crear una copia de seguridad antes de continuar.", "It is strongly recommended to create a backup of your container before proceeding with the conversion.": "Se recomienda encarecidamente crear una copia de seguridad de su contenedor antes de continuar con la conversión.", @@ -4519,10 +4519,11 @@ "Privacy-first, self-hosted PDF toolkit": "kit de herramientas PDF autohospedado que prioriza la privacidad", "Private installation record saved": "Registro de instalación privado guardado", "Private network assigned automatically:": "Red privada asignada automáticamente:", + "Private network kept, because other containers still use it:": "Red privada conservada, porque otros contenedores la siguen usando:", "Private network of the application released:": "Red privada de la aplicación liberada:", "Private network of the application that is released:": "Red privada de la aplicación que se libera:", - "Private network release attempted:": "Intento de liberación de red privada:", - "Private network targeted for release if no other guest uses it:": "Red privada destinada a ser liberada si ningún otro invitado la utiliza:", + "Private network release attempted:": "Se ha intentado liberar la red privada:", + "Private network targeted for release if no other guest uses it:": "Red privada que se liberará si ningún otro contenedor o VM la usa:", "Private network:": "Red privada:", "Private personal knowledge management": "Gestión privada de los conocimientos personales", "Privileged": "Privilegiado", @@ -4778,7 +4779,7 @@ "Remote server via SSH (recommended — off-host, dedup across machines)": "servidor remoto a través de SSH (recomendado: fuera del host, desduplicación entre máquinas)", "Remote verified:": "Comprobación remota:", "Remounting CIFS share with open permissions...": "Remontando el recurso compartido CIFS con permisos abiertos...", - "Removal command finished; review any warnings above.": "comando de eliminación finalizado;revise las advertencias anteriores.", + "Removal command finished; review any warnings above.": "Eliminación terminada; revisa los avisos anteriores.", "Remove CIFS Mount": "Quitar montaje CIFS", "Remove CIFS Mount (pvesm or fstab)": "Quitar montaje CIFS (pvesm o fstab)", "Remove CIFS Storage": "Eliminar almacenamiento CIFS", @@ -4824,7 +4825,7 @@ "Remove old repository files:": "Eliminar archivos antiguos del repositorio:", "Remove passthrough device from VM": "Eliminar el dispositivo de paso de la VM", "Remove subscription banner": "Eliminar banner de suscripción", - "Remove the application? Its container disks are deleted, and only a backup can bring them back.": "¿Eliminar la aplicación? Los discos de su contenedor se eliminan y sólo una copia de seguridad puede recuperarlos.", + "Remove the application? Its container disks are deleted, and only a backup can bring them back.": "¿Eliminar la aplicación? Los discos de sus contenedores se eliminan y solo un backup puede recuperarlos.", "Remove the unprivileged flag from configuration:": "Elimine la bandera sin privilegios de la configuración:", "Remove unused packages and their config": "Eliminar paquetes no utilizados y su configuración.", "Remove: delete the application and its containers": "Eliminar: la aplicación y sus contenedores", @@ -6041,7 +6042,7 @@ "The host port is already in use:": "El puerto host ya está en uso:", "The host-monitor firewall bridge does not match the selected bridge": "El bridge del plan de cortafuegos no coincide con el bridge seleccionado", "The host-monitor firewall declaration is invalid": "La declaración de cortafuegos del monitor del host no es válida", - "The host-monitor firewall port does not match exactly one TCP port in the container contract": "El puerto del firewall del monitor de host no coincide exactamente con un puerto TCP en el contrato del contenedor", + "The host-monitor firewall port does not match exactly one TCP port in the container contract": "El puerto del cortafuegos del monitor del host no coincide con exactamente un puerto TCP del contrato del contenedor", "The host-monitor firewall port is invalid": "El puerto del cortafuegos del monitor del host no es válido", "The host-monitor firewall port is not declared as the web port": "El puerto del cortafuegos no está declarado como puerto web", "The host-monitor firewall port is not declared by this profile": "Este perfil no declara el puerto del cortafuegos", @@ -6964,7 +6965,6 @@ "Weixin (WeChat) is an instant messaging, social media, and mobile payment app developed by Tencent.": "Weixin (WeChat) es una aplicación de mensajería instantánea, redes sociales y pago móvil desarrollada por Tencent.", "What cannot be translated:": "Lo que no puede traducirse:", "What do you want to do?": "¿Qué es lo que quieres hacer?", - "What was left of containers that no longer exist has been removed:": "Se ha eliminado lo que quedaba de contenedores que ya no existen:", "What would you like to do?": "¿Qué te gustaría hacer?", "When asked to select a disk, click Load Driver and load the VirtIO drivers.": "Cuando se le solicite seleccionar un disco, haga clic en Cargar controlador y cargue los controladores VirtIO.", "When this GPU is passed through to a VM, the Proxmox host will lose all video output on the physical monitor.": "Cuando esta GPU pasa a una VM, el host Proxmox perderá toda la salida de video en el monitor físico.", diff --git a/oci/remote/oci_remove.py b/oci/remote/oci_remove.py index 653fb597..5dc89729 100644 --- a/oci/remote/oci_remove.py +++ b/oci/remote/oci_remove.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 """Removes an OCI installation: its containers with the volumes they own, the -private network of a multi-container application and its saved record. Host -directories are left exactly as they are.""" +private network of a multi-container application, its saved record and what +it left on the host for those containers. Host directories are left exactly +as they are.""" from __future__ import annotations import argparse @@ -19,10 +20,17 @@ import oci_image_cache as image_cache import oci_instances as instances from oci_installation_state import parse_config import oci_console +import oci_runtime_settings as runtime_settings from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error # The private networks ProxMenux creates for multi-container applications. PRIVATE_STACK_NETWORK = ipaddress.ip_network('10.77.0.0/16') +CLUSTER_NODES = Path('/etc/pve/nodes') +SNIPPETS = Path('/var/lib/vz/snippets') +# The App tab of ProxMenux Monitor keeps one file per VMID. +MONITOR_APPS = Path('/etc/proxmenux/apps') +HOST_MONITOR_INCLUDES = (Path('/etc/pve/proxmenux/host-monitor'), Path('/etc/pve/lxc/proxmenux-host-monitor')) +STACK_HOOK = 'proxmenux-stack-dependencies.sh' def run(*args): @@ -73,6 +81,15 @@ def private_bridge(primary): network = (primary.get('stack') or {}).get('deployment', {}).get('network', {}) bridge = network.get('private_bridge') subnet = network.get('private_subnet') + if not bridge: + # An application of the Arr suite has no stack record: each one is + # independent, and its own leg is on the network the suite shares. + own = primary.get('deployment', {}).get('network', {}) + bridge = own.get('bridge') + try: + subnet = str(ipaddress.ip_interface(own.get('ipv4')).network) + except (TypeError, ValueError): + return None if not bridge or not re.fullmatch(r'vmbr[0-9]+', bridge): return None try: @@ -83,6 +100,126 @@ def private_bridge(primary): return bridge +def guest_node(vmid): + """The cluster node that holds the container's configuration, if any. + `pct config` sees only the local node; a migrated container is elsewhere.""" + for path in CLUSTER_NODES.glob(f'*/lxc/{int(vmid)}.conf'): + return path.parent.parent.name + return None + + +def _unlink(path): + try: + if path.is_file() and not path.is_symlink(): + path.unlink() + except OSError: + pass + + +def remove_host_state(vmid): + """What the installation kept on the host for a container that is gone: + its sysctl include, the Rclone mount hookscript and the views it + published, and its registration in the App tab of ProxMenux Monitor.""" + for include in (runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid)): + _unlink(include) + hook = SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh' + if hook.is_file() and not hook.is_symlink(): + unit = f'proxmenux-rclone-publish-{int(vmid)}.service' + for action in ('stop', 'reset-failed'): + subprocess.run(['systemctl', action, unit], check=False, capture_output=True) + views = re.findall(r'^published(?:_ro)?=(/\S+)$', hook.read_text(errors='ignore'), re.MULTILINE) + for view in views: + # Only an empty directory that is no longer a mount point. + if os.path.isdir(view) and not os.path.ismount(view): + try: + os.rmdir(view) + except OSError: + pass + _unlink(hook) + _unlink(MONITOR_APPS / f'{int(vmid)}.json') + dismissed = MONITOR_APPS / '.oci-dismissed.json' + try: + entries = json.loads(dismissed.read_text()) + except (OSError, ValueError): + return + if isinstance(entries, dict) and entries.pop(str(int(vmid)), None) is not None: + temporary = dismissed.with_name(dismissed.name + '.tmp') + temporary.write_text(json.dumps(entries, indent=2)) + os.replace(temporary, dismissed) + + +def _guest_configs(): + texts = [] + for pattern in ('*/lxc/*.conf', '*/qemu-server/*.conf'): + for path in CLUSTER_NODES.glob(pattern): + texts.append(path.read_text(encoding='utf-8', errors='ignore')) + return '\n'.join(texts) + + +def release_shared_host_files(hookscripts): + """Files several installations share, once no guest of the cluster uses + them: the host-monitor include and the stack dependency hookscript.""" + remaining = _guest_configs() + for include in HOST_MONITOR_INCLUDES: + if include.is_file() and f'lxc.include: {include}' not in remaining: + _unlink(include) + for volume in set(hookscripts): + if STACK_HOOK not in volume or f'hookscript: {volume}' in remaining: + continue + try: + path = Path(instances.command('pvesm', 'path', volume).decode().strip()) + except (subprocess.CalledProcessError, RuntimeError, OSError): + continue + if path.name == STACK_HOOK: + _unlink(path) + + +def _leftovers(vmid): + """Whether anything of the container is still on the host.""" + paths = [runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid), + SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh', MONITOR_APPS / f'{int(vmid)}.json', + *oci_console.LOG_DIR.glob(f'{int(vmid)}.console.log*')] + return any(path.exists() for path in paths) + + +def sweep_orphans(root): + """Leftovers of containers that exist on no node of the cluster: the + record of one deleted from the Proxmox interface, and files an earlier + removal left behind. A record with an operation left halfway is kept, + because its backup may still be needed. Returns the VMIDs cleaned.""" + found = {int(d.name) for d in root.iterdir() if d.name.isdecimal()} if root.is_dir() else set() + for directory, pattern in ((runtime_settings.include_path(0).parent, r'([0-9]+)\.sysctls'), + (runtime_settings.legacy_include_path(0).parent, r'([0-9]+)\.proxmenux-sysctls'), + (oci_console.LOG_DIR, r'([0-9]+)\.console\.log.*'), + (SNIPPETS, r'proxmenux-rclone-([0-9]+)-fuse-hook\.sh')): + if directory.is_dir(): + found.update(int(m.group(1)) for m in (re.fullmatch(pattern, p.name) for p in directory.iterdir()) if m) + # Only the App tab registrations of OCI installs; the other ones belong to + # ordinary containers. + for path in MONITOR_APPS.glob('*.json') if MONITOR_APPS.is_dir() else []: + try: + apps = json.loads(path.read_text()).get('apps') or [] + except (OSError, ValueError, AttributeError): + continue + if path.stem.isdecimal() and any(app.get('installed_via') == 'oci_image' for app in apps): + found.add(int(path.stem)) + cleaned = [] + for vmid in sorted(found): + if instances.guest_exists(vmid): + continue + record = instances.has_contract(root, vmid) + if record and not instances.release_orphan(root, vmid): + continue + if not (record or _leftovers(vmid)): + continue + oci_console.remove_log(vmid) + remove_host_state(vmid) + cleaned.append(vmid) + if cleaned: + release_shared_host_files([]) + return cleaned + + def bridge_in_use(bridge, removed): """Whether a guest that is not being removed still uses the bridge.""" for path in Path('/etc/pve/nodes').glob('*/lxc/*.conf'): @@ -163,10 +300,15 @@ def remove(root, vmid): if record.get('pending_transaction') or record.get('pending_stack_transaction'): raise ValueError(translate('An operation of this installation has not finished; ' 'recover it from the management menu before removing it')) + node = guest_node(member) if guest_config(member) is None else None + if node: + raise ValueError(f"{translate('The container runs on another node of the cluster; migrate it back to this node to remove it:')} " + f"CT {member} ({node})") kept = host_directories(root, members) bridge = private_bridge(primary) incomplete = False msg_info(translate('Removing the containers...')) + hookscripts = [] for member in members: record = instances.read(root, member) config = guest_config(member) @@ -174,13 +316,16 @@ def remove(root, vmid): msg_warn(f"{translate('The container no longer exists:')} CT {member}") incomplete = True oci_console.remove_log(member) + remove_host_state(member) elif instances.identity(config) != record['installation_id']: msg_warn(f"{translate('The VMID belongs to another container now and is not touched:')} CT {member}") incomplete = True else: + hookscripts += re.findall(r'^hookscript: (\S+)$', config.decode(errors='ignore'), re.MULTILINE) subprocess.run(['pct', 'stop', str(member), '--skiplock', '1'], check=False, capture_output=True) run('pct', 'destroy', str(member), '--purge', '1', '--destroy-unreferenced-disks', '1') oci_console.remove_log(member) + remove_host_state(member) msg_ok(f"{translate('Container removed:')} CT {member}") if bridge and not bridge_in_use(bridge, set(members)): released = release_bridge(bridge) @@ -192,6 +337,7 @@ def remove(root, vmid): msg_info(f"{translate('The private network is still used by another container and is kept:')} {bridge}") if not remove_owned_host_firewall(primary): incomplete = True + release_shared_host_files(hookscripts) lifecycle = Path(f'/etc/pve/priv/proxmenux-stack-{primary_id}.json') if lifecycle.exists() and not lifecycle.is_symlink(): lifecycle.unlink() diff --git a/oci/src/proxmenux_oci/management.py b/oci/src/proxmenux_oci/management.py index 879029fa..ed0cb60e 100644 --- a/oci/src/proxmenux_oci/management.py +++ b/oci/src/proxmenux_oci/management.py @@ -140,10 +140,23 @@ def interactive_management(project, ui): ui.message(translate('OCI management could not be completed. Check the backend status; no additional cleanup has been authorized.'), translate('OCI management')) +def _clean_orphans(project): + """Remove, silently, what is left of containers that exist on no node of the cluster.""" + sys.path.insert(0, str(project / 'remote')) + import oci_instances as instances + import oci_remove + try: + with instances.locked(instances.ROOT): + oci_remove.sweep_orphans(instances.ROOT) + except (BlockingIOError, OSError, ValueError): + pass + + def _interactive_management(project, ui): if os.geteuid() != 0 or not shutil.which('pct'): ui.message(translate('This interface runs on the Proxmox node as root. Open OCI manager Apps from the ProxMenux menu on the Proxmox host.'), translate('OCI management')) return + _clean_orphans(project) rows = saved_inventory(project) if not rows: ui.message(translate('No registered OCI containers are available for selection on this host.'), translate('OCI management')) @@ -316,8 +329,10 @@ def _removal_summary(project, vmid): f"{translate('containers of')} {application}. {alone}", ''] text += [translate('Containers targeted for removal:'), *lines, '', translate('Container data targeted for deletion:'), *volumes] - if bridge: + if bridge and not oci_remove.bridge_in_use(bridge, set(members)): text += ['', f"{translate('Private network targeted for release if no other guest uses it:')} {bridge}"] + elif bridge: + text += ['', f"{translate('Private network kept, because other containers still use it:')} {bridge}"] if (primary.get('deployment') or {}).get('host_firewall'): text += ['', translate('A matching managed host firewall rule may also be removed.')] if kept: diff --git a/oci/tests/test_removal_cleanup.py b/oci/tests/test_removal_cleanup.py new file mode 100644 index 00000000..6b5dc7bf --- /dev/null +++ b/oci/tests/test_removal_cleanup.py @@ -0,0 +1,177 @@ +"""Removing an OCI application leaves nothing of it on the host, and nothing another guest still uses is touched.""" + +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "remote")) + +import oci_remove +import oci_runtime_settings as runtime_settings + +HOOK = """#!/usr/bin/env bash +inside=/data/mounts/drive +published={shared}/rw/drive +published_ro={shared}/ro/drive +""" + + +class RemovalCleanupTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.nodes = self.root / "nodes" + (self.nodes / "amd/lxc").mkdir(parents=True) + (self.nodes / "pve2/lxc").mkdir(parents=True) + self.snippets = self.root / "snippets" + self.snippets.mkdir() + self.apps = self.root / "apps" + self.apps.mkdir() + self.cluster = self.root / "proxmenux" + self.cluster.mkdir() + self.legacy = self.root / "legacy" + self.legacy.mkdir() + self.host_monitor = (self.cluster / "host-monitor", self.legacy / "proxmenux-host-monitor") + patches = [ + patch.object(oci_remove, "CLUSTER_NODES", self.nodes), + patch.object(oci_remove, "SNIPPETS", self.snippets), + patch.object(oci_remove, "MONITOR_APPS", self.apps), + patch.object(oci_remove, "HOST_MONITOR_INCLUDES", self.host_monitor), + patch.object(runtime_settings, "include_path", lambda vmid: self.cluster / f"{vmid}.sysctls"), + patch.object(runtime_settings, "legacy_include_path", lambda vmid: self.legacy / f"{vmid}.proxmenux-sysctls"), + patch.object(oci_remove.subprocess, "run"), + ] + for item in patches: + item.start() + self.addCleanup(item.stop) + + def test_every_file_of_the_container_is_removed(self): + (self.cluster / "113.sysctls").write_text("lxc.sysctl.net.ipv4.ip_unprivileged_port_start = 0\n") + (self.legacy / "113.proxmenux-sysctls").write_text("x\n") + shared = self.root / "shared" + for view in ("rw/drive", "ro/drive"): + (shared / view).mkdir(parents=True) + (self.snippets / "proxmenux-rclone-113-fuse-hook.sh").write_text(HOOK.format(shared=shared)) + (self.apps / "113.json").write_text("{}") + (self.apps / ".oci-dismissed.json").write_text(json.dumps({"113": "a", "112": "b"})) + oci_remove.remove_host_state(113) + self.assertFalse((self.cluster / "113.sysctls").exists()) + self.assertFalse((self.legacy / "113.proxmenux-sysctls").exists()) + self.assertFalse((self.snippets / "proxmenux-rclone-113-fuse-hook.sh").exists()) + self.assertFalse((shared / "rw/drive").exists() or (shared / "ro/drive").exists()) + self.assertFalse((self.apps / "113.json").exists()) + self.assertEqual(json.loads((self.apps / ".oci-dismissed.json").read_text()), {"112": "b"}) + + def test_a_published_view_with_content_is_kept(self): + shared = self.root / "shared" + (shared / "rw/drive").mkdir(parents=True) + (shared / "rw/drive/file").write_text("data") + (self.snippets / "proxmenux-rclone-113-fuse-hook.sh").write_text(HOOK.format(shared=shared)) + oci_remove.remove_host_state(113) + self.assertTrue((shared / "rw/drive/file").exists()) + + def test_other_containers_are_not_touched(self): + (self.cluster / "114.sysctls").write_text("x\n") + (self.apps / "114.json").write_text("{}") + oci_remove.remove_host_state(113) + self.assertTrue((self.cluster / "114.sysctls").exists() and (self.apps / "114.json").exists()) + + def test_shared_host_monitor_include_is_kept_while_a_guest_uses_it(self): + self.host_monitor[0].write_text("lxc.namespace.share.net = 1\n") + (self.nodes / "pve2/lxc/120.conf").write_text(f"arch: amd64\nlxc.include: {self.host_monitor[0]}\n") + oci_remove.release_shared_host_files([]) + self.assertTrue(self.host_monitor[0].exists()) + (self.nodes / "pve2/lxc/120.conf").unlink() + oci_remove.release_shared_host_files([]) + self.assertFalse(self.host_monitor[0].exists()) + + def test_stack_hookscript_is_removed_only_when_no_guest_uses_it(self): + hook = self.snippets / "proxmenux-stack-dependencies.sh" + hook.write_text("#!/bin/sh\n") + volume = "local:snippets/proxmenux-stack-dependencies.sh" + with patch.object(oci_remove.instances, "command", return_value=f"{hook}\n".encode()): + (self.nodes / "amd/lxc/130.conf").write_text(f"hookscript: {volume}\n") + oci_remove.release_shared_host_files([volume]) + self.assertTrue(hook.exists()) + (self.nodes / "amd/lxc/130.conf").unlink() + oci_remove.release_shared_host_files([volume, "local:snippets/someone-else.sh"]) + self.assertFalse(hook.exists()) + + def sweep(self, existing=()): + registry = self.root / "instances" + registry.mkdir(exist_ok=True) + logs = self.root / "logs" + logs.mkdir(exist_ok=True) + with patch.object(oci_remove.oci_console, "LOG_DIR", logs), \ + patch.object(oci_remove.instances, "guest_exists", lambda vmid: vmid in existing), \ + patch.object(oci_remove.instances, "has_contract", lambda root, vmid: (root / str(vmid) / "oci-compose.json").exists()), \ + patch.object(oci_remove.instances, "release_orphan", self.retire): + return oci_remove.sweep_orphans(registry), registry, logs + + def retire(self, root, vmid): + record = root / str(vmid) / "oci-compose.json" + if json.loads(record.read_text()).get("pending_transaction"): + return False + record.replace(record.with_name("retired-x.json")) + return True + + def test_sweep_removes_what_is_left_of_containers_that_no_longer_exist(self): + registry = self.root / "instances" + (registry / "151").mkdir(parents=True) + (registry / "151/oci-compose.json").write_text("{}") + (self.legacy / "9901.proxmenux-sysctls").write_text("x\n") + (self.cluster / "100.sysctls").write_text("x\n") + cleaned, registry, logs = self.sweep(existing={100}) + self.assertEqual(cleaned, [151, 9901]) + self.assertTrue((registry / "151/retired-x.json").exists()) + self.assertFalse((self.legacy / "9901.proxmenux-sysctls").exists()) + # A container that exists keeps everything. + self.assertTrue((self.cluster / "100.sysctls").exists()) + # Nothing left: the next visit cleans and reports nothing. + self.assertEqual(self.sweep(existing={100})[0], []) + + def test_sweep_finds_oci_registrations_of_the_app_tab_only(self): + (self.apps / "113.json").write_text(json.dumps({"apps": [{"installed_via": "oci_image"}]})) + (self.apps / "114.json").write_text(json.dumps({"apps": [{"installed_via": "dpkg"}]})) + self.assertEqual(self.sweep()[0], [113]) + self.assertFalse((self.apps / "113.json").exists()) + self.assertTrue((self.apps / "114.json").exists()) + + def test_sweep_keeps_an_operation_left_halfway(self): + registry = self.root / "instances" + (registry / "152").mkdir(parents=True) + (registry / "152/oci-compose.json").write_text(json.dumps({"pending_transaction": "x"})) + (self.cluster / "152.sysctls").write_text("x\n") + self.assertEqual(self.sweep()[0], []) + self.assertTrue((self.cluster / "152.sysctls").exists()) + + def test_private_network_of_a_stack_and_of_an_arr_suite_application(self): + stack = {"stack": {"deployment": {"network": {"private_bridge": "vmbr10", "private_subnet": "10.77.0.0/24"}}}} + suite = {"deployment": {"network": {"bridge": "vmbr11", "ipv4": "10.77.1.31/24"}}} + lan = {"deployment": {"network": {"bridge": "vmbr0", "ipv4": "192.168.0.40/24"}}} + dhcp = {"deployment": {"network": {"bridge": "vmbr0", "ipv4": "dhcp"}}} + self.assertEqual(oci_remove.private_bridge(stack), "vmbr10") + self.assertEqual(oci_remove.private_bridge(suite), "vmbr11") + self.assertIsNone(oci_remove.private_bridge(lan)) + self.assertIsNone(oci_remove.private_bridge(dhcp)) + + def test_the_suite_network_is_kept_while_another_application_uses_it(self): + (self.nodes / "amd/lxc/180.conf").write_text("net0: name=eth0,bridge=vmbr11,ip=10.77.1.30/24\n") + (self.nodes / "amd/lxc/181.conf").write_text("net0: name=eth0,bridge=vmbr11,ip=10.77.1.31/24\n") + with patch.object(oci_remove, "Path", lambda value: self.nodes if value == "/etc/pve/nodes" else Path(value)): + self.assertTrue(oci_remove.bridge_in_use("vmbr11", {181})) + self.assertFalse(oci_remove.bridge_in_use("vmbr11", {180, 181})) + + def test_a_container_on_another_node_is_found(self): + (self.nodes / "pve2/lxc/113.conf").write_text("arch: amd64\n") + self.assertEqual(oci_remove.guest_node(113), "pve2") + self.assertIsNone(oci_remove.guest_node(114)) + + +if __name__ == "__main__": + unittest.main()