diff --git a/.github/scripts/tests/test_fencing_wording.py b/.github/scripts/tests/test_fencing_wording.py index 8bbcce4d..48e580f5 100644 --- a/.github/scripts/tests/test_fencing_wording.py +++ b/.github/scripts/tests/test_fencing_wording.py @@ -111,9 +111,12 @@ class FencingWordingTests(unittest.TestCase): rendered = self.render(args[0], args[2], lang) body = catalog['runtime']['notifications']['templates']['split_brain']['body'] self.assertEqual(rendered['body'], body.replace('{reason}', message), lang) - journal = Journal() - method(journal, 'quorum lost', 'syslog') - self.assertEqual(journal.events[0][0][0], 'node_disconnect') + # Losing quorum is this host's state, not a node that left: the + # membership lines name the nodes that left (issue #605). + for message in ('quorum lost', '[status] notice: node lost quorum'): + journal = Journal() + method(journal, message, 'pmxcfs') + self.assertEqual(journal.events, []) def test_english_family_and_settings_label_align_without_changing_key(self): en = self.catalogs['en'] diff --git a/AppImage/messages/de/common.json b/AppImage/messages/de/common.json index a538e80f..c54776b9 100644 --- a/AppImage/messages/de/common.json +++ b/AppImage/messages/de/common.json @@ -6992,6 +6992,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Der Knoten {node_name} hat seinen Clusterdienst neu gestartet und ist nach {seconds} s wieder beigetreten." + }, "templates": { "state_change": { "title": "{hostname}: {category} geändert zu {current}{entity_suffix}", diff --git a/AppImage/messages/en/common.json b/AppImage/messages/en/common.json index c9e342ab..ede56306 100644 --- a/AppImage/messages/en/common.json +++ b/AppImage/messages/en/common.json @@ -7117,7 +7117,7 @@ "cancel": "Cancel" } }, - "runtime": {"notifications":{"templates":{"state_change":{"title":"{hostname}: {category} changed to {current}{entity_suffix}","body":"{category} status changed from {previous} to {current}.\n{reason}","label":"Health state changed"},"new_error":{"title":"{hostname}: New {severity} - {category}{entity_suffix}","body":"{reason}","label":"New health issue"},"error_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"The {category} issue has been resolved.\n{reason}\n🚦 Previous severity: {original_severity}\n⏱️ Duration: {duration}","label":"Recovery notification"},"error_escalated":{"title":"{hostname}: Escalated to {severity} - {category}{entity_suffix}","body":"{reason}","label":"Health issue escalated"},"health_degraded":{"title":"{title_or_default}","body":"{reason}","label":"Health check degraded"},"lxc_updates_available":{"title":"{hostname}: {count} LXC(s) with package updates available","body":"📊 {count} LXC(s) with pending package updates (📦 {total_packages} total, 🔒 {security_count} security):\n\n{ct_list}","label":"LXC updates available (experimental)"},"lxc_update_applied":{"title":"{hostname}: LXC {ct_name} ({vmid}) update {result}","body":"{details}","label":"LXC update applied"},"oci_update_completed":{"title":"{hostname}: {app_name} updated","body":"{app_name} was updated to its new image and its data was kept.\nContainers: {containers}","label":"OCI application updated"},"oci_update_failed":{"title":"{hostname}: {app_name} update did not complete","body":"The update of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application update failed"},"oci_recreate_completed":{"title":"{hostname}: {app_name} recreated","body":"{app_name} was recreated with its saved configuration and its data was kept.\nContainers: {containers}","label":"OCI application recreated"},"oci_recreate_failed":{"title":"{hostname}: {app_name} recreation did not complete","body":"The recreation of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application recreation failed"},"oci_modify_completed":{"title":"{hostname}: {app_name} modified","body":"{app_name} was modified with its new options and its data was kept.\nContainers: {containers}","label":"OCI application modified"},"oci_modify_failed":{"title":"{hostname}: {app_name} modification did not complete","body":"The modification of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application modification failed"},"oci_watchdog_restarted":{"title":"{hostname}: {app_name} restarted after it stopped","body":"{app_name} stopped on its own and the watchdog started it again.\nContainers: {containers}","label":"OCI application restarted by the watchdog"},"oci_watchdog_failed":{"title":"{hostname}: {app_name} could not be kept running","body":"The watchdog could not keep {app_name} running: it stopped again or did not start after several attempts, and it is left stopped.\nContainers: {containers}\nStart it by hand once the cause is solved; the watchdog then watches it again.","label":"OCI application left stopped by the watchdog"},"app_update_available":{"title":"{hostname}: {app_name} update available on CT {vmid}","body":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","label":"App update available"},"docker_stack_update_available":{"title":"{hostname}: Docker updates available on CT {vmid}","body":"Container {ct_name} (CT {vmid}) has {count} Docker update(s):\n{details}","label":"Docker updates available"},"vm_start":{"title":"{hostname}: VM {vmname} ({vmid}) started","body":"Virtual machine {vmname} (ID: {vmid}) is now running.","label":"VM started"},"vm_start_warning":{"title":"{hostname}: VM {vmname} ({vmid}) started with warnings","body":"Virtual machine {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"VM started (warnings)"},"vm_stop":{"title":"{hostname}: VM {vmname} ({vmid}) stopped","body":"Virtual machine {vmname} (ID: {vmid}) has been stopped.","label":"VM stopped"},"vm_shutdown":{"title":"{hostname}: VM {vmname} ({vmid}) shut down","body":"Virtual machine {vmname} (ID: {vmid}) has been cleanly shut down.","label":"VM shutdown"},"vm_fail":{"title":"{hostname}: VM {vmname} ({vmid}) FAILED","body":"Virtual machine {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"VM FAILED"},"vm_restart":{"title":"{hostname}: VM {vmname} ({vmid}) restarted","body":"Virtual machine {vmname} (ID: {vmid}) has been restarted.","label":"VM restarted"},"ct_start":{"title":"{hostname}: CT {vmname} ({vmid}) started","body":"Container {vmname} (ID: {vmid}) is now running.","label":"CT started"},"ct_start_warning":{"title":"{hostname}: CT {vmname} ({vmid}) started with warnings","body":"Container {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"CT started (warnings)"},"ct_stop":{"title":"{hostname}: CT {vmname} ({vmid}) stopped","body":"Container {vmname} (ID: {vmid}) has been stopped.","label":"CT stopped"},"ct_shutdown":{"title":"{hostname}: CT {vmname} ({vmid}) shut down","body":"Container {vmname} (ID: {vmid}) has been cleanly shut down.","label":"CT shutdown"},"ct_restart":{"title":"{hostname}: CT {vmname} ({vmid}) restarted","body":"Container {vmname} (ID: {vmid}) has been restarted.","label":"CT restarted"},"ct_fail":{"title":"{hostname}: CT {vmname} ({vmid}) FAILED","body":"Container {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"CT FAILED"},"migration_start":{"title":"{hostname}: Migration started — {vmname} ({vmid})","body":"Live migration of {vmname} (ID: {vmid}) to node {target_node} has started.","label":"Migration started"},"migration_complete":{"title":"{hostname}: Migration complete — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) successfully migrated to node {target_node}.","label":"Migration complete"},"migration_warning":{"title":"{hostname}: Migration complete with warnings — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) migrated to node {target_node} but encountered warnings.\nWarnings: {reason}","label":"Migration (warnings)"},"migration_fail":{"title":"{hostname}: Migration FAILED — {vmname} ({vmid})","body":"Migration of {vmname} (ID: {vmid}) to node {target_node} failed.\nReason: {reason}","label":"Migration FAILED"},"replication_fail":{"title":"{hostname}: Replication FAILED — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Replication FAILED"},"replication_complete":{"title":"{hostname}: Replication complete — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) completed successfully.","label":"Replication complete"},"backup_start":{"title":"{hostname}: Backup started on {storage}","body":"Backup job started on storage {storage}.\n{reason}","label":"Backup started"},"backup_complete":{"title":"{hostname} → {storage}: Backup complete — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}","label":"Backup complete"},"backup_warning":{"title":"{hostname} → {storage}: Backup complete with warnings — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) on {storage} completed but encountered warnings.\nWarnings: {reason}","label":"Backup (warnings)"},"backup_fail":{"title":"{hostname} → {storage}: Backup FAILED — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) failed on {storage}.\nReason: {reason}","label":"Backup FAILED"},"host_backup_start":{"title":"{hostname}: Host backup started → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nProfile: {profile_mode}","label":"Host backup started"},"host_backup_complete":{"title":"{hostname}: Host backup complete → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nData size: {data_size}\nArchive size: {archive_size}\nDuration: {duration}","label":"Host backup complete"},"host_backup_fail":{"title":"{hostname}: Host backup FAILED → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nDuration before failure: {duration}\nReason: {reason}\nLog: {log_file}","label":"Host backup FAILED"},"snapshot_complete":{"title":"{hostname}: Snapshot created — {vmname} ({vmid})","body":"Snapshot \"{snapshot_name}\" created for {vmname} (ID: {vmid}).","label":"Snapshot created"},"snapshot_fail":{"title":"{hostname}: Snapshot FAILED — {vmname} ({vmid})","body":"Snapshot creation for {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Snapshot FAILED"},"cpu_high":{"title":"{hostname}: High CPU usage — {value}%","body":"CPU usage has reached {value}% on {cores} cores.\n{details}","label":"High CPU usage"},"ram_high":{"title":"{hostname}: High memory usage — {value}%","body":"Memory usage: {used} / {total} ({value}%).\n{details}","label":"High memory usage"},"temp_high":{"title":"{hostname}: High sensor temperature — {value}°C","body":"Sensor temperature has reached {value}°C (threshold: {threshold}°C).\n{details}","label":"High temperature"},"disk_space_low":{"title":"{hostname}: Low disk space on {mount}","body":"Filesystem {mount}: {used}% used ({available} available).\nFree up disk space to avoid service disruption.","label":"Low disk space"},"disk_io_error":{"title":"{hostname}: Disk failure detected — {device}","body":"I/O error or disk failure detected on device {device}.\n{reason}","label":"Disk failure / I/O error"},"storage_unavailable":{"title":"{hostname}: Storage unavailable — {storage_name}","body":"PVE storage \"{storage_name}\" (type: {storage_type}) is not accessible.\nReason: {reason}","label":"Storage unavailable"},"smart_test_complete":{"title":"{hostname}: SMART test completed — {device}","body":"SMART {test_type} test on /dev/{device} has completed.\nResult: {result}\nDuration: {duration}","label":"SMART test completed"},"smart_test_failed":{"title":"{hostname}: SMART test FAILED — {device}","body":"SMART {test_type} test on /dev/{device} has failed.\nResult: {result}\nReason: {reason}","label":"SMART test FAILED"},"gpu_mode_switch":{"title":"{hostname}: GPU mode changed to {new_mode}","body":"GPU passthrough mode has been switched.\nGPU: {gpu_name} ({gpu_pci})\nPrevious mode: {old_mode}\nNew mode: {new_mode}\n{details}","label":"GPU mode switched"},"gpu_passthrough_blocked":{"title":"{hostname}: {guest_type} {guest_id} blocked at startup","body":"PCIe passthrough guard prevented {guest_type} {guest_id} ({guest_name}) from starting.\nReason: {reason}\n{details}","label":"GPU passthrough blocked"},"pci_passthrough_conflict":{"title":"{hostname}: PCIe device conflict detected — {device_pci}","body":"A PCIe device is assigned to multiple guests.\nDevice: {device_pci}\nConflicting guests: {guest_list}\nAction required: Stop one of the guests or reassign the device.","label":"PCIe device conflict"},"load_high":{"title":"{hostname}: High system load — {value}","body":"System load average is {value} on {cores} cores.\n{details}","label":"High system load"},"network_down":{"title":"{hostname}: Network connectivity lost{entity_suffix}","body":"The node has lost network connectivity.\nReason: {reason}","label":"Network connectivity lost"},"network_latency":{"title":"{hostname}: High network latency — {value}ms","body":"Latency to gateway: {value}ms (threshold: {threshold}ms).\nThis may indicate network congestion or hardware issues.","label":"High network latency"},"auth_fail":{"title":"{hostname}: Authentication failure","body":"Failed login attempt detected.\nSource IP: {source_ip}\nUser: {username}\nService: {service}","label":"Authentication failure"},"ip_block":{"title":"{hostname}: IP blocked by Fail2Ban","body":"IP address {source_ip} has been banned.\nJail: {jail}\nFailed attempts: {failures}","label":"IP blocked by Fail2Ban"},"firewall_issue":{"title":"{hostname}: Firewall issue detected{entity_suffix}","body":"A firewall configuration issue has been detected.\nReason: {reason}","label":"Firewall issue detected"},"user_permission_change":{"title":"{hostname}: User permission changed","body":"User: {username}\nChange: {change_details}","label":"User permission changed"},"split_brain":{"title":"{hostname}: Cluster event reported","body":"A cluster event was reported:\n{reason}","label":"Cluster event"},"node_disconnect":{"title":"{hostname}: Node {node_name} disconnected","body":"Node {node_name} has disconnected from the cluster.","label":"Node disconnected"},"node_reconnect":{"title":"{hostname}: Node {node_name} reconnected","body":"Node {node_name} has rejoined the cluster successfully.","label":"Node reconnected"},"system_startup":{"title":"{hostname}: {reason}","body":"{summary}","label":"System startup report"},"system_shutdown":{"title":"{hostname}: System shutting down","body":"The node is shutting down.\n{reason}","label":"System shutting down"},"system_reboot":{"title":"{hostname}: System rebooting","body":"The node is rebooting.\n{reason}","label":"System rebooting"},"system_restore_completed":{"title":"{hostname}: Host restore finished","body":"Post-restore tasks completed in background.\n\nGuest configurations copied: {guests}\nBind-mount stub directories created: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}","label":"Host restore completed"},"system_problem":{"title":"{hostname}: System problem detected{entity_suffix}","body":"A system-level problem has been detected.\nReason: {reason}","label":"System problem detected"},"service_fail":{"title":"{hostname}: Service failed — {service_name}","body":"System service \"{service_name}\" has failed.\nReason: {reason}","label":"Service failed"},"oom_kill":{"title":"{hostname}: OOM Kill — {process}","body":"Process \"{process}\" was killed by the Out-of-Memory manager.\n{reason}","label":"Out of memory kill"},"service_fail_batch":{"title":"{hostname}: {service_count} services failed{entity_suffix}","body":"{reason}","label":"Service fail batch"},"cron_output":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Cron job output (per-cron stdout via mail)"},"system_mail":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Smartd / mail bounces (PVE system mail)"},"apt_listchanges":{"title":"{hostname}: {pve_title}","body":"Upstream package information forwarded by Proxmox VE through apt-listchanges. The following text comes from the package maintainer and is not a ProxMenux recommendation.\n\n{reason}","label":"apt-listchanges package notices"},"webhook_test":{"title":"{hostname}: Webhook test received","body":"PVE webhook connectivity test successful.\n{reason}","label":"Webhook test"},"update_available":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity: {security_count}\nProxmox: {pve_count}\nKernel: {kernel_count}\nImportant packages:\n{important_list}","label":"Updates available (legacy)"},"unknown_persistent":{"title":"{hostname}: Check unavailable - {category}{entity_suffix}","body":"Health check for {category} has been unavailable for 3+ cycles.\n{reason}","label":"Check unavailable"},"health_persistent":{"title":"{hostname}: {count} active health issue(s){entity_suffix}","body":"The following health issues remain unresolved:\n{issue_list}\n\nThis digest is sent once every 24 hours while issues persist.","label":"Active health issues (daily)"},"health_issue_new":{"title":"{hostname}: New health issue — {category}{entity_suffix}","body":"New {severity} issue detected in: {category}\nDetails: {reason}","label":"New health issue"},"health_issue_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"{category} issue has been resolved.\n{reason}\nDuration: {duration}","label":"Health issue resolved"},"update_summary":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity updates: {security_count}\nProxmox-related updates: {pve_count}\nKernel updates: {kernel_count}\nImportant packages:\n{important_list}","label":"Host package updates"},"pve_update":{"title":"{hostname}: Proxmox VE {new_version} available","body":"A new Proxmox VE release is available.\nCurrent: {current_version}\nNew: {new_version}\n{details}","label":"Proxmox VE update available"},"update_complete":{"title":"{hostname}: System update completed","body":"System packages have been successfully updated.\n{details}","label":"Host update completed"},"ai_model_migrated":{"title":"{hostname}: AI model updated","body":"The AI model for notifications has been automatically updated.\nProvider: {provider}\nPrevious model: {old_model}\nNew model: {new_model}\n\n{message}","label":"AI model auto-updated"},"proxmenux_update":{"title":"{hostname}: ProxMenux {new_version} available","body":"A new version of ProxMenux is available.\nCurrent: {current_version}\nNew: {new_version}","label":"ProxMenux update available"},"mount_stale":{"title":"{hostname}: stale remote mount {mount_target}","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is stale{lxc_scope}.\nStat timed out or returned an error: {error}\n\nApps writing to this path will silently land on the underlying filesystem and may fill the disk. Remount or fix connectivity ASAP.","label":"Remote mount stale"},"mount_readonly":{"title":"{hostname}: remote mount {mount_target} is read-only","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is mounted read-only{lxc_scope}. Writes will fail. If this was unintentional, remount with rw.","label":"Remote mount read-only"},"lxc_disk_low":{"title":"{hostname}: CT {vmid} rootfs at {usage_percent}%","body":"CT {vmid} ({name}) rootfs is at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nA full LXC rootfs prevents the container from booting cleanly. Either expand the rootfs (pct resize {vmid} rootfs +1G) or free space inside the container.","label":"LXC rootfs near full"},"vm_disk_low":{"title":"{hostname}: VM {vmid} filesystems at {usage_percent}%","body":"VM {vmid} ({name}) guest filesystems are at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nReported by the QEMU guest agent. Includes every persistent filesystem the guest mounts on a block device — virtual disks and PCI-passthrough drives alike. Free up space inside the guest or expand the affected storage before writes start to fail.","label":"VM filesystems near full"},"lxc_mount_low":{"title":"{hostname}: CT {vmid} mount {mount} at {usage_percent}%","body":"Mount {mount} inside CT {vmid} ({name}) is at {usage_percent}% used.\nFilesystem type: {fstype}\n\nA full mount inside a container often blocks the application silently — writes either fail or, worse, land on the rootfs and trigger the rootfs alert next. Free up space on the mount or expand it.","label":"LXC mount near full"},"pve_storage_full":{"title":"{hostname}: PVE storage {storage_name} at {usage_percent}%","body":"Proxmox storage \"{storage_name}\" (type: {storage_type}) is at {usage_percent}% used.\n\nOnce full, no new VM/CT can be provisioned and existing guests may fail to write. Move/delete unused volumes or expand the underlying pool/LV/RBD image.","label":"PVE storage near full"},"zfs_pool_full":{"title":"{hostname}: ZFS pool {pool_name} at {usage_percent}%","body":"ZFS pool \"{pool_name}\" is at {usage_percent}% capacity.\n\nZFS performance and write reliability degrade sharply above ~80% capacity (CoW needs free space for new blocks). Free up snapshots, prune old datasets, or add more vdevs to the pool.","label":"ZFS pool near full"},"post_install_update":{"title":"{hostname}: {count} ProxMenux optimization update(s) available","body":"{count} ProxMenux optimization update(s) available on this host.\n\n🛠️ Available versions:\n{tool_list}\n\n💡 Apply from:\n • ProxMenux Monitor → Settings → ProxMenux Optimizations\n • Or open the ProxMenux menu → Settings post-install Proxmox → \"Apply available updates\"","label":"ProxMenux optimization updates available"},"secure_gateway_update_available":{"title":"{hostname}: {app_name} update available{update_title_suffix}","body":"{app_name} (managed by ProxMenux) has 📦 {package_count} package update(s) pending in its container.\n{version_line}\n\n💡 Open ProxMenux Monitor > Security > Secure Gateway and click \"Update\" to apply.\n\n🗂️ Packages:\n{package_list}","label":"Secure Gateway update available"},"nvidia_driver_update_available":{"title":"{hostname}: NVIDIA driver update available — v{latest_version}","body":"A newer maintenance release is available for the installed NVIDIA driver branch.\n🔹 Currently installed: v{current_version}\n🟢 Latest available: v{latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\nReinstalling rebuilds the DKMS module against the running kernel and requires a reboot to load the new driver.","label":"NVIDIA driver update available"},"coral_driver_update_available":{"title":"{hostname}: Coral TPU driver update available — {latest_version}","body":"A newer {variant_label} is available.\n🔹 Currently installed: {current_version}\n🟢 Latest available: {latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\n{reboot_note}","label":"Coral TPU driver update available"},"burst_auth_fail":{"title":"{hostname}: +{count} more auth failures in {window}","body":"+{count} additional authentication failures detected in {window} ({total_count} total).\nSources: {entity_list}","label":"Auth failures burst"},"burst_ip_block":{"title":"{hostname}: Fail2Ban banned +{count} more IPs in {window}","body":"+{count} additional IPs banned by Fail2Ban in {window} ({total_count} total).\nIPs: {entity_list}","label":"IP block burst"},"burst_disk_io":{"title":"{hostname}: +{count} more disk I/O errors on {entity_list}","body":"+{count} additional I/O errors detected in {window} ({total_count} total).\nDevices: {entity_list}","label":"Disk I/O burst"},"burst_cluster":{"title":"{hostname}: Cluster flapping detected (+{count} more changes)","body":"Cluster state changed +{count} more times in {window} ({total_count} total).\nNodes: {entity_list}","label":"Cluster flapping burst"},"burst_service_fail":{"title":"{hostname}: +{count} more services failed in {window}","body":"+{count} additional service failures detected in {window} ({total_count} total).\nThis typically indicates a node reboot or PVE service restart.\n\nAdditional failures:\n{details}","label":"Service fail burst"},"burst_system":{"title":"{hostname}: +{count} more system problems in {window}","body":"+{count} additional system problems detected in {window} ({total_count} total).\n\nAdditional issues:\n{details}","label":"System problems burst"},"burst_generic":{"title":"{hostname}: +{count} more {event_type} events in {window}","body":"+{count} additional events of type {event_type} in {window} ({total_count} total).\n\nAdditional events:\n{details}","label":"Generic burst"},"kernel_warning":{"title":"{hostname}: Kernel diagnostic event detected","body":"The kernel recorded a diagnostic event: {kernel_details}","label":"Kernel warnings and diagnostic traces"}},"lxcUpdate":{"status":{"success":"succeeded","failure":"failed","partial":"completed partially","deferred":"deferred","skipped":"skipped"},"source":{"scheduled":"Scheduled","manual":"Manual"},"sourceLabel":"Source","targets":"Targets","osPending":"OS packages pending: {before} → {after}","osUnverified":"OS packages: update command executed; result not verified","applications":"Applications: {items}","applicationsUnverified":"Applications: updater executed; no tracked version change was observed","dockerEngineChange":"Docker Engine: {before} → {after}","dockerEngineVerified":"Docker Engine: verified at {version}","dockerEngineUnverified":"Docker Engine: update command executed; version not verified","dockerImagesPending":"Docker images pending: {before} → {after}","dockerImagesChanged":"Docker images changed: {images}","deferredTargets":"Deferred targets: {targets}","reason":"Reason: {reason}","restartRequired":"Restart required: {value}","yes":"yes","no":"no","verificationPending":"Verification pending until the container is running","verificationWarning":"Verification warning: {error}","duration":"Duration: {duration}"},"fallback":{"unknownTitle":"{hostname}: {event_type}","healthCheckDegraded":"Health check degraded","none":"none","temperatureAlertTitle":"{hostname}: Sensor temperature alert","temperatureAlertBody":"Temperature alert recorded without a complete measurement payload.","recordedReason":"Recorded reason: {reason}","recordedDetails":"Recorded details: {details}"},"fields":{"vmid":"VM/CT","name":"Name","device":"Device","sourceIp":"Source IP","node":"Node","category":"Category","service":"Service","jail":"Jail","user":"User","count":"Count","window":"Window","affected":"Affected"},"vzdump":{"size":"Size: {value}","duration":"Duration: {value}","file":"File","backups":"{count} backups","failed":"{count} failed","total":"Total: {value}","time":"Time: {value}"},"startup":{"issuesTitle":"{hostname}: System startup - {count} issue(s) detected","completeTitle":"{hostname}: System startup completed","operational":"All systems operational.","vmCountOne":"{count} VM","vmCountMany":"{count} VMs","ctCountOne":"{count} CT","ctCountMany":"{count} CTs","started":"✅ {counts} started","vmFailed":"❌ VM failed: {name} - {reason}","ctFailed":"❌ CT failed: {name} - {reason}","unknownError":"unknown error","storageUnavailable":"⚠️ Storage: {count} unavailable ({names})","servicesFailed":"⚠️ Services: {count} failed ({names})","duration":"⏱️ Startup completed in {minutes} min"},"appUpdates":{"singleTitle":"{hostname}: {app_name} update available on CT {vmid}","singleBody":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","emptyTitle":"{hostname}: Application updates available","emptyBody":"Application updates are available.","batchTitle":"{hostname}: {count} application updates available","batchLeadOneContainer":"{count} applications in {container_count} LXC container have a newer version:","batchLeadManyContainers":"{count} applications in {container_count} LXC containers have a newer version:","additional":"… {count} additional application(s)","app":"app","unknown":"unknown"},"healthDegraded":{"categories":{"cpu":"CPU usage and temperature","memory":"Memory and swap","storage":"Storage mounts and space","disks":"Disk I/O and errors","network":"Network interfaces","vms":"VMs and containers","services":"PVE services","logs":"System logs","updates":"System updates","security":"Security"},"severity":{"critical":"Critical","warning":"Warning"},"singleTitle":"{hostname}: {severity} health status – {category}","multipleTitle":"{hostname}: {count} health checks degraded","multipleLine":"• {severity}: {category}\n {reason}","reasons":{"cpuSustained":"CPU is above {threshold}% for {duration}s."}},"digest":{"title":"{hostname}: 24h summary ({timestamp})","readFailed":"digest read failed: {error}","empty":"No INFO events buffered for this digest window.","lead":"{count} INFO events grouped by category:\n","more":" • … and {count} more","footer":"(Critical/Warning events arrived at the time they happened, not in this digest.)","quietTitle":"{hostname}: {count} events buffered during Quiet Hours","groups":{"vm_ct":"VM a CT","backup":"Backup","resources":"Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"Services","health":"Health","updates":"Updates","hardware":"Hardware","system":"System","other":"Other"}},"test":{"title":"ProxMenux Test","welcome":"Welcome to ProxMenux Monitor!","verify":"This is a test message to verify your notification channel is working correctly.","configuration":"Channel configuration:","iconsEnabled":"Icons: enabled","iconsDisabled":"Icons: disabled","aiEnabled":"AI: enabled ({info})","aiDisabled":"AI: disabled","alerts":"You will receive alerts about system events, health status changes, and security incidents.","photoCaption":"You can use this image as the profile photo for your notification bot."},"channels":{"discord":{"category":"Category","host":"Host","severity":"Severity"},"email":{"report":"{group} Report","details":"Details","host":"Host","footer":"ProxMenux Notification Service","severity":{"critical":"Critical","warning":"Warning","info":"Information","ok":"Resolved","default":"Notice"},"groups":{"vm_ct":"Virtual Machine / Container","backup":"Backup & Snapshot","resources":"System Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"System Services","health":"Health Monitor","updates":"System Updates","system":"System","hardware":"Hardware","other":"System Notification"},"fields":{"vmCtId":"VM/CT ID","name":"Name","action":"Action","targetNode":"Target Node","reason":"Reason","storage":"Storage","status":"Status","size":"Size","duration":"Duration","snapshot":"Snapshot","metric":"Metric","currentValue":"Current Value","threshold":"Threshold","cpuCores":"CPU Cores","memory":"Memory","temperature":"Temperature","mountPoint":"Mount Point","usage":"Usage","available":"Available","device":"Device","severity":"Severity","storageName":"Storage Name","type":"Type","interface":"Interface","latency":"Latency","event":"Event","sourceIp":"Source IP","username":"Username","service":"Service","jail":"Jail","failures":"Failures","change":"Change","node":"Node","quorum":"Quorum","nodesAffected":"Nodes Affected","process":"Process","category":"Category","previousSeverity":"Previous Severity","activeIssues":"Active Issues","totalUpdates":"Total Updates","securityUpdates":"Security Updates","proxmoxUpdates":"Proxmox Updates","kernelUpdates":"Kernel Updates","importantPackages":"Important Packages","currentVersion":"Current Version","newVersion":"New Version"},"status":{"failed":"Failed","completed":"Completed","started":"Started","unconfirmed":"Unconfirmed","completed_with_warnings":"Completed with warnings"}}},"temperature":{"sampleSpan":"High samples span {duration}."},"backup":{"unconfirmedTitle":"{hostname}: Backup outcome unconfirmed","confirmedTitle":"{hostname}: Backup complete","confirmedBody":"Backup completed successfully.","errorTitle":"{hostname}: Backup error reported","errorBody":"The backup report contains an error.","unconfirmedBody":"The backup outcome is not confirmed.","warningTitle":"{hostname}: Backup completed with warnings","warningBody":"Backup completed with warnings.","diagnosticsOmitted":"Additional diagnostic lines or text omitted: {count}. Original report retained.","quotaCause":"A storage quota prevented the backup from completing.","quotaAffected":"Backups {ids}: a storage quota prevented completion.","quotaCheck":"Check the quota of backup storage {storage} before retrying.","causeMissing":"Backups {ids}: the summary does not identify the cause; check the full logs of those tasks."}}}, + "runtime": {"notifications":{"cluster":{"rejoinedBody":"Node {node_name} restarted its cluster service and rejoined after {seconds} s."},"templates":{"state_change":{"title":"{hostname}: {category} changed to {current}{entity_suffix}","body":"{category} status changed from {previous} to {current}.\n{reason}","label":"Health state changed"},"new_error":{"title":"{hostname}: New {severity} - {category}{entity_suffix}","body":"{reason}","label":"New health issue"},"error_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"The {category} issue has been resolved.\n{reason}\n🚦 Previous severity: {original_severity}\n⏱️ Duration: {duration}","label":"Recovery notification"},"error_escalated":{"title":"{hostname}: Escalated to {severity} - {category}{entity_suffix}","body":"{reason}","label":"Health issue escalated"},"health_degraded":{"title":"{title_or_default}","body":"{reason}","label":"Health check degraded"},"lxc_updates_available":{"title":"{hostname}: {count} LXC(s) with package updates available","body":"📊 {count} LXC(s) with pending package updates (📦 {total_packages} total, 🔒 {security_count} security):\n\n{ct_list}","label":"LXC updates available (experimental)"},"lxc_update_applied":{"title":"{hostname}: LXC {ct_name} ({vmid}) update {result}","body":"{details}","label":"LXC update applied"},"oci_update_completed":{"title":"{hostname}: {app_name} updated","body":"{app_name} was updated to its new image and its data was kept.\nContainers: {containers}","label":"OCI application updated"},"oci_update_failed":{"title":"{hostname}: {app_name} update did not complete","body":"The update of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application update failed"},"oci_recreate_completed":{"title":"{hostname}: {app_name} recreated","body":"{app_name} was recreated with its saved configuration and its data was kept.\nContainers: {containers}","label":"OCI application recreated"},"oci_recreate_failed":{"title":"{hostname}: {app_name} recreation did not complete","body":"The recreation of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application recreation failed"},"oci_modify_completed":{"title":"{hostname}: {app_name} modified","body":"{app_name} was modified with its new options and its data was kept.\nContainers: {containers}","label":"OCI application modified"},"oci_modify_failed":{"title":"{hostname}: {app_name} modification did not complete","body":"The modification of {app_name} did not complete.\nReason: {reason}\nContainers: {containers}\nOpen Manage installed OCI applications to check its state.","label":"OCI application modification failed"},"oci_watchdog_restarted":{"title":"{hostname}: {app_name} restarted after it stopped","body":"{app_name} stopped on its own and the watchdog started it again.\nContainers: {containers}","label":"OCI application restarted by the watchdog"},"oci_watchdog_failed":{"title":"{hostname}: {app_name} could not be kept running","body":"The watchdog could not keep {app_name} running: it stopped again or did not start after several attempts, and it is left stopped.\nContainers: {containers}\nStart it by hand once the cause is solved; the watchdog then watches it again.","label":"OCI application left stopped by the watchdog"},"app_update_available":{"title":"{hostname}: {app_name} update available on CT {vmid}","body":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","label":"App update available"},"docker_stack_update_available":{"title":"{hostname}: Docker updates available on CT {vmid}","body":"Container {ct_name} (CT {vmid}) has {count} Docker update(s):\n{details}","label":"Docker updates available"},"vm_start":{"title":"{hostname}: VM {vmname} ({vmid}) started","body":"Virtual machine {vmname} (ID: {vmid}) is now running.","label":"VM started"},"vm_start_warning":{"title":"{hostname}: VM {vmname} ({vmid}) started with warnings","body":"Virtual machine {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"VM started (warnings)"},"vm_stop":{"title":"{hostname}: VM {vmname} ({vmid}) stopped","body":"Virtual machine {vmname} (ID: {vmid}) has been stopped.","label":"VM stopped"},"vm_shutdown":{"title":"{hostname}: VM {vmname} ({vmid}) shut down","body":"Virtual machine {vmname} (ID: {vmid}) has been cleanly shut down.","label":"VM shutdown"},"vm_fail":{"title":"{hostname}: VM {vmname} ({vmid}) FAILED","body":"Virtual machine {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"VM FAILED"},"vm_restart":{"title":"{hostname}: VM {vmname} ({vmid}) restarted","body":"Virtual machine {vmname} (ID: {vmid}) has been restarted.","label":"VM restarted"},"ct_start":{"title":"{hostname}: CT {vmname} ({vmid}) started","body":"Container {vmname} (ID: {vmid}) is now running.","label":"CT started"},"ct_start_warning":{"title":"{hostname}: CT {vmname} ({vmid}) started with warnings","body":"Container {vmname} (ID: {vmid}) started successfully but has warnings.\nWarnings: {reason}","label":"CT started (warnings)"},"ct_stop":{"title":"{hostname}: CT {vmname} ({vmid}) stopped","body":"Container {vmname} (ID: {vmid}) has been stopped.","label":"CT stopped"},"ct_shutdown":{"title":"{hostname}: CT {vmname} ({vmid}) shut down","body":"Container {vmname} (ID: {vmid}) has been cleanly shut down.","label":"CT shutdown"},"ct_restart":{"title":"{hostname}: CT {vmname} ({vmid}) restarted","body":"Container {vmname} (ID: {vmid}) has been restarted.","label":"CT restarted"},"ct_fail":{"title":"{hostname}: CT {vmname} ({vmid}) FAILED","body":"Container {vmname} (ID: {vmid}) has crashed or failed to start.\nReason: {reason}","label":"CT FAILED"},"migration_start":{"title":"{hostname}: Migration started — {vmname} ({vmid})","body":"Live migration of {vmname} (ID: {vmid}) to node {target_node} has started.","label":"Migration started"},"migration_complete":{"title":"{hostname}: Migration complete — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) successfully migrated to node {target_node}.","label":"Migration complete"},"migration_warning":{"title":"{hostname}: Migration complete with warnings — {vmname} ({vmid})","body":"{vmname} (ID: {vmid}) migrated to node {target_node} but encountered warnings.\nWarnings: {reason}","label":"Migration (warnings)"},"migration_fail":{"title":"{hostname}: Migration FAILED — {vmname} ({vmid})","body":"Migration of {vmname} (ID: {vmid}) to node {target_node} failed.\nReason: {reason}","label":"Migration FAILED"},"replication_fail":{"title":"{hostname}: Replication FAILED — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Replication FAILED"},"replication_complete":{"title":"{hostname}: Replication complete — {vmname} ({vmid})","body":"Replication of {vmname} (ID: {vmid}) completed successfully.","label":"Replication complete"},"backup_start":{"title":"{hostname}: Backup started on {storage}","body":"Backup job started on storage {storage}.\n{reason}","label":"Backup started"},"backup_complete":{"title":"{hostname} → {storage}: Backup complete — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}","label":"Backup complete"},"backup_warning":{"title":"{hostname} → {storage}: Backup complete with warnings — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) on {storage} completed but encountered warnings.\nWarnings: {reason}","label":"Backup (warnings)"},"backup_fail":{"title":"{hostname} → {storage}: Backup FAILED — {vmname} ({vmid})","body":"Backup of {vmname} (ID: {vmid}) failed on {storage}.\nReason: {reason}","label":"Backup FAILED"},"host_backup_start":{"title":"{hostname}: Host backup started → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nProfile: {profile_mode}","label":"Host backup started"},"host_backup_complete":{"title":"{hostname}: Host backup complete → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nData size: {data_size}\nArchive size: {archive_size}\nDuration: {duration}","label":"Host backup complete"},"host_backup_fail":{"title":"{hostname}: Host backup FAILED → {backend_label}","body":"Job: {job_id}\nBackend: {backend_label}\nDestination: {destination}\nDuration before failure: {duration}\nReason: {reason}\nLog: {log_file}","label":"Host backup FAILED"},"snapshot_complete":{"title":"{hostname}: Snapshot created — {vmname} ({vmid})","body":"Snapshot \"{snapshot_name}\" created for {vmname} (ID: {vmid}).","label":"Snapshot created"},"snapshot_fail":{"title":"{hostname}: Snapshot FAILED — {vmname} ({vmid})","body":"Snapshot creation for {vmname} (ID: {vmid}) failed.\nReason: {reason}","label":"Snapshot FAILED"},"cpu_high":{"title":"{hostname}: High CPU usage — {value}%","body":"CPU usage has reached {value}% on {cores} cores.\n{details}","label":"High CPU usage"},"ram_high":{"title":"{hostname}: High memory usage — {value}%","body":"Memory usage: {used} / {total} ({value}%).\n{details}","label":"High memory usage"},"temp_high":{"title":"{hostname}: High sensor temperature — {value}°C","body":"Sensor temperature has reached {value}°C (threshold: {threshold}°C).\n{details}","label":"High temperature"},"disk_space_low":{"title":"{hostname}: Low disk space on {mount}","body":"Filesystem {mount}: {used}% used ({available} available).\nFree up disk space to avoid service disruption.","label":"Low disk space"},"disk_io_error":{"title":"{hostname}: Disk failure detected — {device}","body":"I/O error or disk failure detected on device {device}.\n{reason}","label":"Disk failure / I/O error"},"storage_unavailable":{"title":"{hostname}: Storage unavailable — {storage_name}","body":"PVE storage \"{storage_name}\" (type: {storage_type}) is not accessible.\nReason: {reason}","label":"Storage unavailable"},"smart_test_complete":{"title":"{hostname}: SMART test completed — {device}","body":"SMART {test_type} test on /dev/{device} has completed.\nResult: {result}\nDuration: {duration}","label":"SMART test completed"},"smart_test_failed":{"title":"{hostname}: SMART test FAILED — {device}","body":"SMART {test_type} test on /dev/{device} has failed.\nResult: {result}\nReason: {reason}","label":"SMART test FAILED"},"gpu_mode_switch":{"title":"{hostname}: GPU mode changed to {new_mode}","body":"GPU passthrough mode has been switched.\nGPU: {gpu_name} ({gpu_pci})\nPrevious mode: {old_mode}\nNew mode: {new_mode}\n{details}","label":"GPU mode switched"},"gpu_passthrough_blocked":{"title":"{hostname}: {guest_type} {guest_id} blocked at startup","body":"PCIe passthrough guard prevented {guest_type} {guest_id} ({guest_name}) from starting.\nReason: {reason}\n{details}","label":"GPU passthrough blocked"},"pci_passthrough_conflict":{"title":"{hostname}: PCIe device conflict detected — {device_pci}","body":"A PCIe device is assigned to multiple guests.\nDevice: {device_pci}\nConflicting guests: {guest_list}\nAction required: Stop one of the guests or reassign the device.","label":"PCIe device conflict"},"load_high":{"title":"{hostname}: High system load — {value}","body":"System load average is {value} on {cores} cores.\n{details}","label":"High system load"},"network_down":{"title":"{hostname}: Network connectivity lost{entity_suffix}","body":"The node has lost network connectivity.\nReason: {reason}","label":"Network connectivity lost"},"network_latency":{"title":"{hostname}: High network latency — {value}ms","body":"Latency to gateway: {value}ms (threshold: {threshold}ms).\nThis may indicate network congestion or hardware issues.","label":"High network latency"},"auth_fail":{"title":"{hostname}: Authentication failure","body":"Failed login attempt detected.\nSource IP: {source_ip}\nUser: {username}\nService: {service}","label":"Authentication failure"},"ip_block":{"title":"{hostname}: IP blocked by Fail2Ban","body":"IP address {source_ip} has been banned.\nJail: {jail}\nFailed attempts: {failures}","label":"IP blocked by Fail2Ban"},"firewall_issue":{"title":"{hostname}: Firewall issue detected{entity_suffix}","body":"A firewall configuration issue has been detected.\nReason: {reason}","label":"Firewall issue detected"},"user_permission_change":{"title":"{hostname}: User permission changed","body":"User: {username}\nChange: {change_details}","label":"User permission changed"},"split_brain":{"title":"{hostname}: Cluster event reported","body":"A cluster event was reported:\n{reason}","label":"Cluster event"},"node_disconnect":{"title":"{hostname}: Node {node_name} disconnected","body":"Node {node_name} has disconnected from the cluster.","label":"Node disconnected"},"node_reconnect":{"title":"{hostname}: Node {node_name} reconnected","body":"Node {node_name} has rejoined the cluster successfully.","label":"Node reconnected"},"system_startup":{"title":"{hostname}: {reason}","body":"{summary}","label":"System startup report"},"system_shutdown":{"title":"{hostname}: System shutting down","body":"The node is shutting down.\n{reason}","label":"System shutting down"},"system_reboot":{"title":"{hostname}: System rebooting","body":"The node is rebooting.\n{reason}","label":"System rebooting"},"system_restore_completed":{"title":"{hostname}: Host restore finished","body":"Post-restore tasks completed in background.\n\nGuest configurations copied: {guests}\nBind-mount stub directories created: {stubs}\nStale node dirs removed: {stale_nodes}\nComponents reinstalled: {components}\nDuration: {duration}\n{warnings_block}","label":"Host restore completed"},"system_problem":{"title":"{hostname}: System problem detected{entity_suffix}","body":"A system-level problem has been detected.\nReason: {reason}","label":"System problem detected"},"service_fail":{"title":"{hostname}: Service failed — {service_name}","body":"System service \"{service_name}\" has failed.\nReason: {reason}","label":"Service failed"},"oom_kill":{"title":"{hostname}: OOM Kill — {process}","body":"Process \"{process}\" was killed by the Out-of-Memory manager.\n{reason}","label":"Out of memory kill"},"service_fail_batch":{"title":"{hostname}: {service_count} services failed{entity_suffix}","body":"{reason}","label":"Service fail batch"},"cron_output":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Cron job output (per-cron stdout via mail)"},"system_mail":{"title":"{hostname}: {pve_title}","body":"{reason}","label":"Smartd / mail bounces (PVE system mail)"},"apt_listchanges":{"title":"{hostname}: {pve_title}","body":"Upstream package information forwarded by Proxmox VE through apt-listchanges. The following text comes from the package maintainer and is not a ProxMenux recommendation.\n\n{reason}","label":"apt-listchanges package notices"},"webhook_test":{"title":"{hostname}: Webhook test received","body":"PVE webhook connectivity test successful.\n{reason}","label":"Webhook test"},"update_available":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity: {security_count}\nProxmox: {pve_count}\nKernel: {kernel_count}\nImportant packages:\n{important_list}","label":"Updates available (legacy)"},"unknown_persistent":{"title":"{hostname}: Check unavailable - {category}{entity_suffix}","body":"Health check for {category} has been unavailable for 3+ cycles.\n{reason}","label":"Check unavailable"},"health_persistent":{"title":"{hostname}: {count} active health issue(s){entity_suffix}","body":"The following health issues remain unresolved:\n{issue_list}\n\nThis digest is sent once every 24 hours while issues persist.","label":"Active health issues (daily)"},"health_issue_new":{"title":"{hostname}: New health issue — {category}{entity_suffix}","body":"New {severity} issue detected in: {category}\nDetails: {reason}","label":"New health issue"},"health_issue_resolved":{"title":"{hostname}: Resolved - {category}{entity_suffix}","body":"{category} issue has been resolved.\n{reason}\nDuration: {duration}","label":"Health issue resolved"},"update_summary":{"title":"{hostname}: Updates available","body":"Total updates: {total_count}\nSecurity updates: {security_count}\nProxmox-related updates: {pve_count}\nKernel updates: {kernel_count}\nImportant packages:\n{important_list}","label":"Host package updates"},"pve_update":{"title":"{hostname}: Proxmox VE {new_version} available","body":"A new Proxmox VE release is available.\nCurrent: {current_version}\nNew: {new_version}\n{details}","label":"Proxmox VE update available"},"update_complete":{"title":"{hostname}: System update completed","body":"System packages have been successfully updated.\n{details}","label":"Host update completed"},"ai_model_migrated":{"title":"{hostname}: AI model updated","body":"The AI model for notifications has been automatically updated.\nProvider: {provider}\nPrevious model: {old_model}\nNew model: {new_model}\n\n{message}","label":"AI model auto-updated"},"proxmenux_update":{"title":"{hostname}: ProxMenux {new_version} available","body":"A new version of ProxMenux is available.\nCurrent: {current_version}\nNew: {new_version}","label":"ProxMenux update available"},"mount_stale":{"title":"{hostname}: stale remote mount {mount_target}","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is stale{lxc_scope}.\nStat timed out or returned an error: {error}\n\nApps writing to this path will silently land on the underlying filesystem and may fill the disk. Remount or fix connectivity ASAP.","label":"Remote mount stale"},"mount_readonly":{"title":"{hostname}: remote mount {mount_target} is read-only","body":"Remote mount {mount_target} ({fstype}) from {mount_source} is mounted read-only{lxc_scope}. Writes will fail. If this was unintentional, remount with rw.","label":"Remote mount read-only"},"lxc_disk_low":{"title":"{hostname}: CT {vmid} rootfs at {usage_percent}%","body":"CT {vmid} ({name}) rootfs is at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nA full LXC rootfs prevents the container from booting cleanly. Either expand the rootfs (pct resize {vmid} rootfs +1G) or free space inside the container.","label":"LXC rootfs near full"},"vm_disk_low":{"title":"{hostname}: VM {vmid} filesystems at {usage_percent}%","body":"VM {vmid} ({name}) guest filesystems are at {usage_percent}% ({disk_bytes_human} / {maxdisk_bytes_human}).\n\nReported by the QEMU guest agent. Includes every persistent filesystem the guest mounts on a block device — virtual disks and PCI-passthrough drives alike. Free up space inside the guest or expand the affected storage before writes start to fail.","label":"VM filesystems near full"},"lxc_mount_low":{"title":"{hostname}: CT {vmid} mount {mount} at {usage_percent}%","body":"Mount {mount} inside CT {vmid} ({name}) is at {usage_percent}% used.\nFilesystem type: {fstype}\n\nA full mount inside a container often blocks the application silently — writes either fail or, worse, land on the rootfs and trigger the rootfs alert next. Free up space on the mount or expand it.","label":"LXC mount near full"},"pve_storage_full":{"title":"{hostname}: PVE storage {storage_name} at {usage_percent}%","body":"Proxmox storage \"{storage_name}\" (type: {storage_type}) is at {usage_percent}% used.\n\nOnce full, no new VM/CT can be provisioned and existing guests may fail to write. Move/delete unused volumes or expand the underlying pool/LV/RBD image.","label":"PVE storage near full"},"zfs_pool_full":{"title":"{hostname}: ZFS pool {pool_name} at {usage_percent}%","body":"ZFS pool \"{pool_name}\" is at {usage_percent}% capacity.\n\nZFS performance and write reliability degrade sharply above ~80% capacity (CoW needs free space for new blocks). Free up snapshots, prune old datasets, or add more vdevs to the pool.","label":"ZFS pool near full"},"post_install_update":{"title":"{hostname}: {count} ProxMenux optimization update(s) available","body":"{count} ProxMenux optimization update(s) available on this host.\n\n🛠️ Available versions:\n{tool_list}\n\n💡 Apply from:\n • ProxMenux Monitor → Settings → ProxMenux Optimizations\n • Or open the ProxMenux menu → Settings post-install Proxmox → \"Apply available updates\"","label":"ProxMenux optimization updates available"},"secure_gateway_update_available":{"title":"{hostname}: {app_name} update available{update_title_suffix}","body":"{app_name} (managed by ProxMenux) has 📦 {package_count} package update(s) pending in its container.\n{version_line}\n\n💡 Open ProxMenux Monitor > Security > Secure Gateway and click \"Update\" to apply.\n\n🗂️ Packages:\n{package_list}","label":"Secure Gateway update available"},"nvidia_driver_update_available":{"title":"{hostname}: NVIDIA driver update available — v{latest_version}","body":"A newer maintenance release is available for the installed NVIDIA driver branch.\n🔹 Currently installed: v{current_version}\n🟢 Latest available: v{latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\nReinstalling rebuilds the DKMS module against the running kernel and requires a reboot to load the new driver.","label":"NVIDIA driver update available"},"coral_driver_update_available":{"title":"{hostname}: Coral TPU driver update available — {latest_version}","body":"A newer {variant_label} is available.\n🔹 Currently installed: {current_version}\n🟢 Latest available: {latest_version}\n\n{upgrade_reason}\n\n💡 To reinstall:\n • From the ProxMenux post-install menu: {menu_label}\n\n{reboot_note}","label":"Coral TPU driver update available"},"burst_auth_fail":{"title":"{hostname}: +{count} more auth failures in {window}","body":"+{count} additional authentication failures detected in {window} ({total_count} total).\nSources: {entity_list}","label":"Auth failures burst"},"burst_ip_block":{"title":"{hostname}: Fail2Ban banned +{count} more IPs in {window}","body":"+{count} additional IPs banned by Fail2Ban in {window} ({total_count} total).\nIPs: {entity_list}","label":"IP block burst"},"burst_disk_io":{"title":"{hostname}: +{count} more disk I/O errors on {entity_list}","body":"+{count} additional I/O errors detected in {window} ({total_count} total).\nDevices: {entity_list}","label":"Disk I/O burst"},"burst_cluster":{"title":"{hostname}: Cluster flapping detected (+{count} more changes)","body":"Cluster state changed +{count} more times in {window} ({total_count} total).\nNodes: {entity_list}","label":"Cluster flapping burst"},"burst_service_fail":{"title":"{hostname}: +{count} more services failed in {window}","body":"+{count} additional service failures detected in {window} ({total_count} total).\nThis typically indicates a node reboot or PVE service restart.\n\nAdditional failures:\n{details}","label":"Service fail burst"},"burst_system":{"title":"{hostname}: +{count} more system problems in {window}","body":"+{count} additional system problems detected in {window} ({total_count} total).\n\nAdditional issues:\n{details}","label":"System problems burst"},"burst_generic":{"title":"{hostname}: +{count} more {event_type} events in {window}","body":"+{count} additional events of type {event_type} in {window} ({total_count} total).\n\nAdditional events:\n{details}","label":"Generic burst"},"kernel_warning":{"title":"{hostname}: Kernel diagnostic event detected","body":"The kernel recorded a diagnostic event: {kernel_details}","label":"Kernel warnings and diagnostic traces"}},"lxcUpdate":{"status":{"success":"succeeded","failure":"failed","partial":"completed partially","deferred":"deferred","skipped":"skipped"},"source":{"scheduled":"Scheduled","manual":"Manual"},"sourceLabel":"Source","targets":"Targets","osPending":"OS packages pending: {before} → {after}","osUnverified":"OS packages: update command executed; result not verified","applications":"Applications: {items}","applicationsUnverified":"Applications: updater executed; no tracked version change was observed","dockerEngineChange":"Docker Engine: {before} → {after}","dockerEngineVerified":"Docker Engine: verified at {version}","dockerEngineUnverified":"Docker Engine: update command executed; version not verified","dockerImagesPending":"Docker images pending: {before} → {after}","dockerImagesChanged":"Docker images changed: {images}","deferredTargets":"Deferred targets: {targets}","reason":"Reason: {reason}","restartRequired":"Restart required: {value}","yes":"yes","no":"no","verificationPending":"Verification pending until the container is running","verificationWarning":"Verification warning: {error}","duration":"Duration: {duration}"},"fallback":{"unknownTitle":"{hostname}: {event_type}","healthCheckDegraded":"Health check degraded","none":"none","temperatureAlertTitle":"{hostname}: Sensor temperature alert","temperatureAlertBody":"Temperature alert recorded without a complete measurement payload.","recordedReason":"Recorded reason: {reason}","recordedDetails":"Recorded details: {details}"},"fields":{"vmid":"VM/CT","name":"Name","device":"Device","sourceIp":"Source IP","node":"Node","category":"Category","service":"Service","jail":"Jail","user":"User","count":"Count","window":"Window","affected":"Affected"},"vzdump":{"size":"Size: {value}","duration":"Duration: {value}","file":"File","backups":"{count} backups","failed":"{count} failed","total":"Total: {value}","time":"Time: {value}"},"startup":{"issuesTitle":"{hostname}: System startup - {count} issue(s) detected","completeTitle":"{hostname}: System startup completed","operational":"All systems operational.","vmCountOne":"{count} VM","vmCountMany":"{count} VMs","ctCountOne":"{count} CT","ctCountMany":"{count} CTs","started":"✅ {counts} started","vmFailed":"❌ VM failed: {name} - {reason}","ctFailed":"❌ CT failed: {name} - {reason}","unknownError":"unknown error","storageUnavailable":"⚠️ Storage: {count} unavailable ({names})","servicesFailed":"⚠️ Services: {count} failed ({names})","duration":"⏱️ Startup completed in {minutes} min"},"appUpdates":{"singleTitle":"{hostname}: {app_name} update available on CT {vmid}","singleBody":"{app_name} on CT {vmid} ({ct_name}) has a new version:\n {installed} → {latest}","emptyTitle":"{hostname}: Application updates available","emptyBody":"Application updates are available.","batchTitle":"{hostname}: {count} application updates available","batchLeadOneContainer":"{count} applications in {container_count} LXC container have a newer version:","batchLeadManyContainers":"{count} applications in {container_count} LXC containers have a newer version:","additional":"… {count} additional application(s)","app":"app","unknown":"unknown"},"healthDegraded":{"categories":{"cpu":"CPU usage and temperature","memory":"Memory and swap","storage":"Storage mounts and space","disks":"Disk I/O and errors","network":"Network interfaces","vms":"VMs and containers","services":"PVE services","logs":"System logs","updates":"System updates","security":"Security"},"severity":{"critical":"Critical","warning":"Warning"},"singleTitle":"{hostname}: {severity} health status – {category}","multipleTitle":"{hostname}: {count} health checks degraded","multipleLine":"• {severity}: {category}\n {reason}","reasons":{"cpuSustained":"CPU is above {threshold}% for {duration}s."}},"digest":{"title":"{hostname}: 24h summary ({timestamp})","readFailed":"digest read failed: {error}","empty":"No INFO events buffered for this digest window.","lead":"{count} INFO events grouped by category:\n","more":" • … and {count} more","footer":"(Critical/Warning events arrived at the time they happened, not in this digest.)","quietTitle":"{hostname}: {count} events buffered during Quiet Hours","groups":{"vm_ct":"VM a CT","backup":"Backup","resources":"Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"Services","health":"Health","updates":"Updates","hardware":"Hardware","system":"System","other":"Other"}},"test":{"title":"ProxMenux Test","welcome":"Welcome to ProxMenux Monitor!","verify":"This is a test message to verify your notification channel is working correctly.","configuration":"Channel configuration:","iconsEnabled":"Icons: enabled","iconsDisabled":"Icons: disabled","aiEnabled":"AI: enabled ({info})","aiDisabled":"AI: disabled","alerts":"You will receive alerts about system events, health status changes, and security incidents.","photoCaption":"You can use this image as the profile photo for your notification bot."},"channels":{"discord":{"category":"Category","host":"Host","severity":"Severity"},"email":{"report":"{group} Report","details":"Details","host":"Host","footer":"ProxMenux Notification Service","severity":{"critical":"Critical","warning":"Warning","info":"Information","ok":"Resolved","default":"Notice"},"groups":{"vm_ct":"Virtual Machine / Container","backup":"Backup & Snapshot","resources":"System Resources","storage":"Storage","network":"Network","security":"Security","cluster":"Cluster","services":"System Services","health":"Health Monitor","updates":"System Updates","system":"System","hardware":"Hardware","other":"System Notification"},"fields":{"vmCtId":"VM/CT ID","name":"Name","action":"Action","targetNode":"Target Node","reason":"Reason","storage":"Storage","status":"Status","size":"Size","duration":"Duration","snapshot":"Snapshot","metric":"Metric","currentValue":"Current Value","threshold":"Threshold","cpuCores":"CPU Cores","memory":"Memory","temperature":"Temperature","mountPoint":"Mount Point","usage":"Usage","available":"Available","device":"Device","severity":"Severity","storageName":"Storage Name","type":"Type","interface":"Interface","latency":"Latency","event":"Event","sourceIp":"Source IP","username":"Username","service":"Service","jail":"Jail","failures":"Failures","change":"Change","node":"Node","quorum":"Quorum","nodesAffected":"Nodes Affected","process":"Process","category":"Category","previousSeverity":"Previous Severity","activeIssues":"Active Issues","totalUpdates":"Total Updates","securityUpdates":"Security Updates","proxmoxUpdates":"Proxmox Updates","kernelUpdates":"Kernel Updates","importantPackages":"Important Packages","currentVersion":"Current Version","newVersion":"New Version"},"status":{"failed":"Failed","completed":"Completed","started":"Started","unconfirmed":"Unconfirmed","completed_with_warnings":"Completed with warnings"}}},"temperature":{"sampleSpan":"High samples span {duration}."},"backup":{"unconfirmedTitle":"{hostname}: Backup outcome unconfirmed","confirmedTitle":"{hostname}: Backup complete","confirmedBody":"Backup completed successfully.","errorTitle":"{hostname}: Backup error reported","errorBody":"The backup report contains an error.","unconfirmedBody":"The backup outcome is not confirmed.","warningTitle":"{hostname}: Backup completed with warnings","warningBody":"Backup completed with warnings.","diagnosticsOmitted":"Additional diagnostic lines or text omitted: {count}. Original report retained.","quotaCause":"A storage quota prevented the backup from completing.","quotaAffected":"Backups {ids}: a storage quota prevented completion.","quotaCheck":"Check the quota of backup storage {storage} before retrying.","causeMissing":"Backups {ids}: the summary does not identify the cause; check the full logs of those tasks."}}}, "monitorLinks": { "thisNode": "This node", "checking": "Checking nodes…", diff --git a/AppImage/messages/es/common.json b/AppImage/messages/es/common.json index cf1140a0..b25b8d03 100644 --- a/AppImage/messages/es/common.json +++ b/AppImage/messages/es/common.json @@ -7145,6 +7145,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "El nodo {node_name} reinició su servicio de clúster y volvió a unirse tras {seconds} s." + }, "templates": { "state_change": { "title": "{hostname}: {category} cambió a {current}{entity_suffix}", diff --git a/AppImage/messages/fr/common.json b/AppImage/messages/fr/common.json index 24e6186f..eda4df07 100644 --- a/AppImage/messages/fr/common.json +++ b/AppImage/messages/fr/common.json @@ -6992,6 +6992,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Le nœud {node_name} a redémarré son service de cluster et l'a rejoint après {seconds} s." + }, "templates": { "state_change": { "title": "{hostname} : {category} remplacé par {current}{entity_suffix}", diff --git a/AppImage/messages/it/common.json b/AppImage/messages/it/common.json index 0d08798a..57a774db 100644 --- a/AppImage/messages/it/common.json +++ b/AppImage/messages/it/common.json @@ -6992,6 +6992,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Il nodo {node_name} ha riavviato il servizio del cluster ed è rientrato dopo {seconds} s." + }, "templates": { "state_change": { "title": "{hostname}: {category} cambiato in {current}{entity_suffix}", diff --git a/AppImage/messages/pt/common.json b/AppImage/messages/pt/common.json index 15003692..4e055791 100644 --- a/AppImage/messages/pt/common.json +++ b/AppImage/messages/pt/common.json @@ -6992,6 +6992,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "O nó {node_name} reiniciou o serviço do cluster e voltou a entrar após {seconds} s." + }, "templates": { "state_change": { "title": "{hostname}: {category} alterado para {current}{entity_suffix}", diff --git a/AppImage/messages/ru/common.json b/AppImage/messages/ru/common.json index 2ff56a98..b2c9c84e 100644 --- a/AppImage/messages/ru/common.json +++ b/AppImage/messages/ru/common.json @@ -7144,6 +7144,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Узел {node_name} перезапустил службу кластера и снова присоединился через {seconds} с." + }, "templates": { "state_change": { "title": "{hostname}: {category} — новое состояние: {current}{entity_suffix}", diff --git a/AppImage/messages/sk/common.json b/AppImage/messages/sk/common.json index 3522d998..3f1b2b10 100644 --- a/AppImage/messages/sk/common.json +++ b/AppImage/messages/sk/common.json @@ -6991,6 +6991,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Uzol {node_name} reštartoval službu klastra a po {seconds} s sa znova pripojil." + }, "templates": { "state_change": { "title": "{hostname}: {category} sa zmenilo na {current}{entity_suffix}", diff --git a/AppImage/messages/sv/common.json b/AppImage/messages/sv/common.json index e8e74e1b..af9d3c6a 100644 --- a/AppImage/messages/sv/common.json +++ b/AppImage/messages/sv/common.json @@ -6992,6 +6992,9 @@ }, "runtime": { "notifications": { + "cluster": { + "rejoinedBody": "Noden {node_name} startade om sin klustertjänst och anslöt igen efter {seconds} s." + }, "templates": { "state_change": { "title": "{hostname}: {category} har ändrats till {current}{entity_suffix}", diff --git a/AppImage/scripts/notification_events.py b/AppImage/scripts/notification_events.py index 9a6ff5fb..a76067ad 100644 --- a/AppImage/scripts/notification_events.py +++ b/AppImage/scripts/notification_events.py @@ -35,6 +35,10 @@ from proxmox_known_errors import analyze_oom_event, format_oom_diagnosis # This provides a single source of truth for all grace period logic import startup_grace +# Serializes JournalWatcher emits: held cluster events fire from a timer. +_JOURNAL_EMIT_LOCK = threading.RLock() + + class _SharedState: """Wrapper around centralized startup_grace module for backwards compatibility. @@ -1872,14 +1876,18 @@ class JournalWatcher: # the surviving nodes when a peer leaves or comes back: # [TOTEM ] A new membership (1.55) was formed. Members left: 2 # [TOTEM ] A new membership (1.5a) was formed. Members joined: 2 - # The line carries node IDs and no "node" keyword, so the generic - # match below never fired for a plain node loss in a quorate cluster. + # Only these lines identify a node. pmxcfs' "node lost quorum" is this + # host losing quorum while a peer is away, never a node of that name. if 'a new membership' in msg_lower and 'was formed' in msg_lower: names = self._corosync_node_names() local_name = socket.gethostname().split('.')[0] - for keyword, event_type, severity in ( - ('left', 'node_disconnect', 'CRITICAL'), - ('joined', 'node_reconnect', 'INFO')): + local_ids = {node_id for node_id, name in names.items() if name == local_name} + joined = re.search(r'\bjoined:((?:\s+\d+)+)', msg) + if joined and local_ids & set(joined.group(1).split()): + # Our own corosync (re)started: the peers it now sees never + # left the cluster, so they are not reconnections. + self._cluster_own_join_at = time.time() + for keyword, event_type in (('left', 'node_disconnect'), ('joined', 'node_reconnect')): id_match = re.search(r'\b%s:((?:\s+\d+)+)' % keyword, msg) if not id_match: continue @@ -1887,28 +1895,64 @@ class JournalWatcher: node_name = names.get(node_id, node_id) if node_name == local_name: continue # our own corosync (re)starting - if (event_type == 'node_reconnect' - and startup_grace.is_real_system_boot() - and _shared_state.is_startup_health_grace()): - continue # this host just booted: it is us joining them - self._emit(event_type, severity, { - 'node_name': node_name, - 'hostname': self._hostname, - }, entity='cluster', entity_id=node_name) + if event_type == 'node_disconnect': + self._hold_node_disconnect(node_name) + else: + self._node_rejoined(node_name) return - # Node disconnect - if (('quorum' in msg_lower and 'lost' in msg_lower) or - ('node' in msg_lower and any(w in msg_lower for w in ['left', 'offline', 'lost']))): - - node_match = re.search(r'[Nn]ode\s+(\S+)', msg) - node_name = node_match.group(1) if node_match else 'unknown' - - self._emit('node_disconnect', 'CRITICAL', { + # A corosync restart (package update, maintenance) takes a node out of the + # membership for a few seconds. Report a departure only when the node is + # still away after this grace; a quick return is reported as one rejoin. + CLUSTER_LEAVE_GRACE_SECONDS = 60 + CLUSTER_OWN_JOIN_WINDOW_SECONDS = 30 + + def _cluster_pending(self) -> Dict[str, Any]: + if not hasattr(self, '_cluster_pending_nodes'): + self._cluster_pending_nodes = {} + return self._cluster_pending_nodes + + def _hold_node_disconnect(self, node_name: str): + pending = self._cluster_pending() + with _JOURNAL_EMIT_LOCK: + if node_name in pending: + return + timer = threading.Timer(self.CLUSTER_LEAVE_GRACE_SECONDS, + self._report_node_disconnect, args=(node_name,)) + timer.daemon = True + pending[node_name] = (time.time(), timer) + timer.start() + + def _report_node_disconnect(self, node_name: str): + with _JOURNAL_EMIT_LOCK: + if self._cluster_pending().pop(node_name, None) is None: + return # it came back in time + self._emit_unlocked('node_disconnect', 'CRITICAL', { 'node_name': node_name, 'hostname': self._hostname, }, entity='cluster', entity_id=node_name) - + + def _node_rejoined(self, node_name: str): + with _JOURNAL_EMIT_LOCK: + held = self._cluster_pending().pop(node_name, None) + if held is not None: + left_at, timer = held + timer.cancel() + self._emit('node_reconnect', 'INFO', { + 'node_name': node_name, + 'downtime_seconds': max(1, int(round(time.time() - left_at))), + 'hostname': self._hostname, + }, entity='cluster', entity_id=node_name) + return + if time.time() - getattr(self, '_cluster_own_join_at', 0) < self.CLUSTER_OWN_JOIN_WINDOW_SECONDS: + return # we joined them, they did not come back + if startup_grace.is_real_system_boot() and _shared_state.is_startup_health_grace(): + return # this host just booted: it is us joining them + self._emit('node_reconnect', 'INFO', { + 'node_name': node_name, + 'hostname': self._hostname, + }, entity='cluster', entity_id=node_name) + def _corosync_node_names(self) -> Dict[str, str]: """Map corosync node IDs to node names from corosync.conf.""" for path in ('/etc/pve/corosync.conf', '/etc/corosync/corosync.conf'): @@ -2022,6 +2066,12 @@ class JournalWatcher: def _emit(self, event_type: str, severity: str, data: Dict, entity: str = 'node', entity_id: str = ''): """Emit event to queue with short-term deduplication (30s window).""" + # A held cluster event is emitted from its timer thread. + with _JOURNAL_EMIT_LOCK: + self._emit_unlocked(event_type, severity, data, entity, entity_id) + + def _emit_unlocked(self, event_type: str, severity: str, data: Dict, + entity: str = 'node', entity_id: str = ''): event = NotificationEvent( event_type, severity, data, source='journal', entity=entity, entity_id=entity_id, diff --git a/AppImage/scripts/notification_templates.py b/AppImage/scripts/notification_templates.py index f7529500..8598cd18 100644 --- a/AppImage/scripts/notification_templates.py +++ b/AppImage/scripts/notification_templates.py @@ -1967,6 +1967,13 @@ def render_template(event_type: str, data: Dict[str, Any], ) if localized: template[field] = localized + if event_type == 'node_reconnect' and data.get('downtime_seconds'): + # A node that left and came back within the grace: one notice. + rejoined = runtime_message('cluster.rejoinedBody', language, + node_name=data.get('node_name', ''), + seconds=data['downtime_seconds']) + if rejoined: + template['body'] = rejoined backup_title_target = '' if event_type == 'backup_complete': template['title'] = runtime_message('backup.unconfirmedTitle', language, diff --git a/AppImage/scripts/tests/test_cluster_membership_grace.py b/AppImage/scripts/tests/test_cluster_membership_grace.py new file mode 100644 index 00000000..349865dd --- /dev/null +++ b/AppImage/scripts/tests/test_cluster_membership_grace.py @@ -0,0 +1,146 @@ +"""A corosync restart is not a node failure: departures are held for a grace, +a quick return is one rejoin notice, and only membership lines name a node. +The journal lines below were captured from a lab cluster during +`systemctl restart corosync` on one of its nodes.""" +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import notification_events as ne # noqa: E402 +import notification_templates as nt # noqa: E402 + +NAMES = {'1': 'pmxlab1', '2': 'pmxlab2', '3': 'pmxlab3'} + + +class Clock: + def __init__(self): + self.now = 1_000_000.0 + + def __call__(self): + return self.now + + +class FakeTimer: + started = [] + + def __init__(self, interval, function, args=()): + self.interval, self.function, self.args = interval, function, args + self.cancelled = False + self.daemon = False + + def start(self): + FakeTimer.started.append(self) + + def cancel(self): + self.cancelled = True + + +class MembershipGraceTests(unittest.TestCase): + def watcher(self, hostname): + sub = type('Watcher', (ne.JournalWatcher,), {'_hostname': hostname}) + watcher = object.__new__(sub) + watcher._recent_events = {} + watcher._dedup_window = 30 + watcher.events = [] + + class Queue: + def put(self, event): + watcher.events.append((event.event_type, event.severity, dict(event.data))) + watcher._queue = Queue() + watcher._corosync_node_names = lambda: NAMES + return watcher + + def setUp(self): + FakeTimer.started = [] + self.clock = Clock() + for target, value in ((ne.threading, 'Timer'),): + patcher = patch.object(target, value, FakeTimer) + patcher.start() + self.addCleanup(patcher.stop) + for patcher in (patch.object(ne.time, 'time', self.clock), + patch.object(ne.startup_grace, 'is_real_system_boot', return_value=False)): + patcher.start() + self.addCleanup(patcher.stop) + + def feed(self, watcher, host, lines): + with patch.object(ne.socket, 'gethostname', return_value=host): + for delay, ident, msg in lines: + self.clock.now += delay + watcher._check_cluster_events(msg, ident) + + def expire_timers(self): + self.clock.now += ne.JournalWatcher.CLUSTER_LEAVE_GRACE_SECONDS + for timer in list(FakeTimer.started): + if not timer.cancelled: + timer.function(*timer.args) + + def test_a_peer_corosync_restart_is_one_rejoin_notice(self): + watcher = self.watcher('pmxlab1') + self.feed(watcher, 'pmxlab1', [ + (0, 'corosync', ' [CFG ] Node 2 was shut down by sysadmin'), + (0, 'corosync', ' [QUORUM] Sync left[1]: 2'), + (0, 'corosync', ' [TOTEM ] A new membership (1.59) was formed. Members left: 2'), + (0, 'corosync', ' [QUORUM] This node is within the non-primary component and will NOT provide any services.'), + (0, 'pmxcfs', '[status] notice: node lost quorum'), + (3, 'corosync', ' [TOTEM ] A new membership (1.5e) was formed. Members joined: 2'), + (0, 'pmxcfs', '[status] notice: node has quorum'), + ]) + self.expire_timers() + self.assertEqual(watcher.events, [('node_reconnect', 'INFO', { + 'node_name': 'pmxlab2', 'downtime_seconds': 3, 'hostname': 'pmxlab1'})]) + + def test_a_node_that_stays_away_is_reported_after_the_grace(self): + watcher = self.watcher('pmxlab1') + self.feed(watcher, 'pmxlab1', [ + (0, 'corosync', ' [TOTEM ] A new membership (1.59) was formed. Members left: 2'), + ]) + self.assertEqual(watcher.events, []) + self.expire_timers() + self.assertEqual(watcher.events, [('node_disconnect', 'CRITICAL', + {'node_name': 'pmxlab2', 'hostname': 'pmxlab1'})]) + self.feed(watcher, 'pmxlab1', [ + (600, 'corosync', ' [TOTEM ] A new membership (1.5e) was formed. Members joined: 2'), + ]) + self.assertEqual(watcher.events[-1], ('node_reconnect', 'INFO', + {'node_name': 'pmxlab2', 'hostname': 'pmxlab1'})) + + def test_our_own_restart_does_not_report_the_peers_as_reconnected(self): + watcher = self.watcher('pmxlab2') + self.feed(watcher, 'pmxlab2', [ + (0, 'corosync', ' [CFG ] Node 2 was shut down by sysadmin'), + (0, 'corosync', ' [MAIN ] Node was shut down by a signal'), + (0, 'pmxcfs', '[status] notice: node lost quorum'), + (1, 'corosync', ' [TOTEM ] A new membership (2.5a) was formed. Members joined: 2'), + (2, 'corosync', ' [TOTEM ] A new membership (1.5e) was formed. Members joined: 1'), + (4, 'pmxcfs', '[status] notice: node has quorum'), + ]) + self.expire_timers() + self.assertEqual(watcher.events, []) + + def test_lines_that_only_mention_a_node_name_nothing(self): + watcher = self.watcher('pmxlab1') + self.feed(watcher, 'pmxlab1', [ + (0, 'pmxcfs', '[status] notice: node lost quorum'), + (0, 'node_exporter', 'level=warn msg="node offline, lost connection"'), + (0, 'app', 'worker node left the pool'), + (0, '', 'quorum lost by votequorum'), + (0, 'corosync', ' [QUORUM] Sync left[1]: 2'), + ]) + self.expire_timers() + self.assertEqual(watcher.events, []) + + def test_the_rejoin_notice_says_how_long_the_node_was_away(self): + rendered = nt.render_template('node_reconnect', { + 'node_name': 'pve', 'downtime_seconds': 3, 'hostname': 'pve1'}, language='es') + self.assertIn('pve', rendered['body']) + self.assertIn('3 s', rendered['body']) + self.assertNotIn('{', rendered['body']) + plain = nt.render_template('node_reconnect', {'node_name': 'pve', 'hostname': 'pve1'}, language='en') + self.assertIn('rejoined the cluster successfully', plain['body']) + + +if __name__ == '__main__': + unittest.main() diff --git a/AppImage/scripts/tests/test_cluster_node_membership_notifications.py b/AppImage/scripts/tests/test_cluster_node_membership_notifications.py index a831cfa3..006f2431 100644 --- a/AppImage/scripts/tests/test_cluster_node_membership_notifications.py +++ b/AppImage/scripts/tests/test_cluster_node_membership_notifications.py @@ -35,8 +35,23 @@ nodelist { """ +class GraceTimer: + """Departures wait for a grace; these tests let it run out at once.""" + pending = [] + + def __init__(self, interval, function, args=()): + self.function, self.args, self.cancelled, self.daemon = function, args, False, False + + def start(self): + GraceTimer.pending.append(self) + + def cancel(self): + self.cancelled = True + + class ClusterMembershipTests(unittest.TestCase): def setUp(self): + GraceTimer.pending = [] folder = tempfile.TemporaryDirectory() self.addCleanup(folder.cleanup) conf = Path(folder.name) / "corosync.conf" @@ -54,7 +69,8 @@ class ClusterMembershipTests(unittest.TestCase): patch.object(notification_events.startup_grace, "is_real_system_boot", side_effect=lambda: self.booting), patch.object(notification_events._shared_state, "is_startup_health_grace", - side_effect=lambda: self.booting)): + side_effect=lambda: self.booting), + patch.object(notification_events.threading, "Timer", GraceTimer)): item.start() self.addCleanup(item.stop) self.queue = Queue() @@ -62,6 +78,10 @@ class ClusterMembershipTests(unittest.TestCase): def notified(self, line): self.watcher._check_cluster_events(line, "corosync") + while GraceTimer.pending: + timer = GraceTimer.pending.pop(0) + if not timer.cancelled: + timer.function(*timer.args) events = [] while True: try: