Add audit and reports page, and a change journal

ProxMenux modifies the host: it rewrites configuration files, installs packages, enables services. Until now nobody could say afterwards what had changed, and showing the script does not answer that question — a four-hundred-line function may alter two values, and the reader has no way to know which two. This adds the two halves of an answer.

The change journal records what ProxMenux does as it does it. Eleven bash primitives capture the previous state, apply the change and record it in the same step, writing to a spool that the Monitor reads back. One hundred and thirteen functions across twenty-five scripts are instrumented, covering post-install, shared storage, security tooling, container conversions, disk operations and the PVE 8 to 9 upgrade path. The page shows the difference — rotate 7 becoming rotate 14 — and never the script. Restore and backup scripts are deliberately left out: a restore puts the host back to a state some other script already recorded.

The Audit and reports page answers the other half: what state is this host in, regardless of who put it there. Forty-three checks across seven areas read the host and classify each result as critical, warning, observation, conformant, unverified or not applicable, with the evidence they read attached to each one. A declared policy lets the reader say what this particular host is expected to do — which guests must have a backup, which storages are essential — so the report judges the host against its own intent rather than a generic template. An inventory records the hardware, network and guest topology behind those readings, a comparison shows what moved between two runs, and six report profiles produce a printable document scoped to what the reader needs. Everything is available in the eight supported languages.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

AppImage/scripts/tests/test_auth_manager_setup.py

@@ -109,5 +109,105 @@ class SetupAuthTests(unittest.TestCase):
             self.assertEqual(config[key], value)
 
 
+class ChangePasswordTests(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp_dir.cleanup)
+        config_dir = Path(self.temp_dir.name)
+        config_patch = mock.patch.multiple(
+            auth_manager,
+            CONFIG_DIR=config_dir,
+            AUTH_CONFIG_FILE=config_dir / "auth.json",
+        )
+        config_patch.start()
+        self.addCleanup(config_patch.stop)
+
+        self.current_password = "CurrentPass1!"
+        self.new_password = "Replacement2!"
+        auth_manager.AUTH_CONFIG_FILE.write_text(json.dumps({
+            "enabled": True,
+            "configured": True,
+            "declined": False,
+            "username": "admin",
+            "password_hash": auth_manager.hash_password(self.current_password),
+            "totp_enabled": False,
+            "totp_secret": None,
+            "backup_codes": [],
+        }))
+
+    def read_config(self):
+        return json.loads(auth_manager.AUTH_CONFIG_FILE.read_text())
+
+    def test_missing_current_password_is_rejected_without_exception(self):
+        self.assertFalse(auth_manager.verify_password(None, self.read_config()["password_hash"]))
+
+        success, message = auth_manager.change_password(None, self.new_password)
+
+        self.assertFalse(success)
+        self.assertEqual(message, "Current password is incorrect")
+
+    def test_password_change_without_2fa(self):
+        success, message = auth_manager.change_password(
+            self.current_password, self.new_password
+        )
+
+        self.assertTrue(success, message)
+        self.assertTrue(auth_manager.verify_password(
+            self.new_password, self.read_config()["password_hash"]
+        ))
+
+    def test_password_change_requires_2fa_when_enabled(self):
+        config = self.read_config()
+        config["totp_enabled"] = True
+        auth_manager.AUTH_CONFIG_FILE.write_text(json.dumps(config))
+
+        success, message = auth_manager.change_password(
+            self.current_password, self.new_password
+        )
+
+        self.assertFalse(success)
+        self.assertEqual(message, "2FA code required to change password")
+        self.assertTrue(auth_manager.verify_password(
+            self.current_password, self.read_config()["password_hash"]
+        ))
+
+    def test_password_change_accepts_valid_2fa_code(self):
+        config = self.read_config()
+        config["totp_enabled"] = True
+        auth_manager.AUTH_CONFIG_FILE.write_text(json.dumps(config))
+
+        with mock.patch.object(
+            auth_manager, "verify_totp", return_value=(True, "accepted")
+        ) as verify_totp:
+            success, message = auth_manager.change_password(
+                self.current_password, self.new_password, "123456"
+            )
+
+        self.assertTrue(success, message)
+        verify_totp.assert_called_once_with("admin", "123456", use_backup=False)
+        self.assertTrue(auth_manager.verify_password(
+            self.new_password, self.read_config()["password_hash"]
+        ))
+
+    def test_password_change_rejects_invalid_2fa_and_preserves_password(self):
+        config = self.read_config()
+        config["totp_enabled"] = True
+        auth_manager.AUTH_CONFIG_FILE.write_text(json.dumps(config))
+
+        with mock.patch.object(
+            auth_manager, "verify_totp", return_value=(False, "rejected")
+        ) as verify_totp:
+            success, message = auth_manager.change_password(
+                self.current_password, self.new_password, "000000"
+            )
+
+        self.assertFalse(success)
+        self.assertEqual(message, "Invalid 2FA code")
+        self.assertEqual(verify_totp.call_count, 2)
+        self.assertTrue(auth_manager.verify_password(
+            self.current_password, self.read_config()["password_hash"]
+        ))
+
+
 if __name__ == "__main__":
     unittest.main()

AppImage/scripts/tests/test_cluster_guest_storage_ownership.py

@@ -0,0 +1,129 @@
+import sys
+import unittest
+from pathlib import Path
+from queue import Queue
+from types import ModuleType, SimpleNamespace
+from unittest.mock import patch
+
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+# These modules normally bind the live `/usr/local/share/proxmenux` database
+# while importing. Ownership tests need no host state, so provide the same
+# narrow dependency boundary used by the production functions below.
+_health_persistence_module = ModuleType("health_persistence")
+_health_persistence_module.health_persistence = SimpleNamespace(
+    cleanup_old_errors=lambda: None,
+)
+_health_persistence_module.disk_base_name = lambda name: str(name).replace("/dev/", "")
+sys.modules.setdefault("health_persistence", _health_persistence_module)
+
+sys.modules.setdefault("psutil", ModuleType("psutil"))
+
+flask_server = SimpleNamespace(
+    get_proxmox_node_name=lambda: "fixture",
+    get_cached_pvesh_cluster_resources_vm=lambda: [],
+    get_cached_vm_disk=lambda _vmid: None,
+)
+sys.modules.setdefault("flask_server", flask_server)
+
+import health_monitor  # noqa: E402
+import notification_events  # noqa: E402
+
+
+class _Persistence:
+    def __init__(self):
+        self.recorded = []
+        self.cleared = []
+
+    def record_error(self, **kwargs):
+        self.recorded.append(kwargs)
+
+    def get_active_errors(self, *args, **kwargs):
+        return []
+
+    def clear_error(self, key):
+        self.cleared.append(key)
+
+
+class ClusterGuestStorageOwnershipTests(unittest.TestCase):
+    def setUp(self):
+        self.monitor = health_monitor.HealthMonitor.__new__(health_monitor.HealthMonitor)
+        self.persistence = _Persistence()
+        self.resources = [
+            {
+                "type": "lxc", "node": "hades", "status": "running",
+                "vmid": 128, "name": "plex", "disk": 94, "maxdisk": 100,
+            },
+            {
+                "type": "lxc", "node": "poseidon", "status": "running",
+                "vmid": 129, "name": "remote", "disk": 99, "maxdisk": 100,
+            },
+        ]
+
+    def test_lxc_capacity_records_only_guests_owned_by_local_node(self):
+        with (
+            patch.object(health_monitor, "MOUNT_MONITOR_AVAILABLE", False),
+            patch.object(health_monitor, "health_persistence", self.persistence),
+            patch.object(flask_server, "get_proxmox_node_name", return_value="hades"),
+            patch.object(flask_server, "get_cached_pvesh_cluster_resources_vm", return_value=self.resources),
+        ):
+            result = self.monitor._check_lxc_disk_usage()
+
+        self.assertEqual(result["status"], "WARNING")
+        self.assertEqual([row["error_key"] for row in self.persistence.recorded], ["lxc_disk_128"])
+        self.assertEqual(self.persistence.recorded[0]["details"]["node"], "hades")
+        self.assertNotIn("CT 129", result["checks"])
+
+    def test_vm_capacity_does_not_probe_remote_guest_agent(self):
+        resources = [
+            {"type": "qemu", "node": "hades", "status": "running", "vmid": 201, "name": "local"},
+            {"type": "qemu", "node": "poseidon", "status": "running", "vmid": 202, "name": "remote"},
+        ]
+
+        def disk_for(vmid):
+            if vmid == 201:
+                return (94, 100)
+            raise AssertionError("remote VM was probed")
+
+        with (
+            patch.object(health_monitor, "health_persistence", self.persistence),
+            patch.object(flask_server, "get_proxmox_node_name", return_value="hades"),
+            patch.object(flask_server, "get_cached_pvesh_cluster_resources_vm", return_value=resources),
+            patch.object(flask_server, "get_cached_vm_disk", side_effect=disk_for),
+        ):
+            result = self.monitor._check_vm_disk_usage()
+
+        self.assertEqual(result["status"], "WARNING")
+        self.assertEqual([row["error_key"] for row in self.persistence.recorded], ["vm_disk_201"])
+        self.assertEqual(self.persistence.recorded[0]["details"]["node"], "hades")
+
+    def test_foreign_legacy_record_is_not_a_recovery(self):
+        collector = notification_events.PollingCollector(Queue())
+        resources = [{"type": "lxc", "node": "poseidon", "vmid": 128}]
+        with (
+            patch.object(flask_server, "get_proxmox_node_name", return_value="hades"),
+            patch.object(flask_server, "get_cached_pvesh_cluster_resources_vm", return_value=resources),
+        ):
+            foreign = collector._guest_storage_error_is_now_foreign(
+                "lxc_disk_128", {"details": {"vmid": "128"}}
+            )
+        self.assertTrue(foreign)
+
+    def test_local_recovery_remains_a_recovery(self):
+        collector = notification_events.PollingCollector(Queue())
+        resources = [{"type": "lxc", "node": "hades", "vmid": 128}]
+        with (
+            patch.object(flask_server, "get_proxmox_node_name", return_value="hades"),
+            patch.object(flask_server, "get_cached_pvesh_cluster_resources_vm", return_value=resources),
+        ):
+            foreign = collector._guest_storage_error_is_now_foreign(
+                "lxc_disk_128", {"details": {"vmid": "128", "node": "hades"}}
+            )
+        self.assertFalse(foreign)
+
+
+if __name__ == "__main__":
+    unittest.main()

AppImage/scripts/tests/test_kernel_trace_notifications.py

@@ -0,0 +1,86 @@
+import json
+import sys
+import unittest
+from pathlib import Path
+from queue import Empty, Queue
+
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+APPIMAGE_DIR = SCRIPTS_DIR.parent
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import notification_events  # noqa: E402
+import notification_templates  # noqa: E402
+
+
+class KernelTraceNotificationTests(unittest.TestCase):
+    def setUp(self):
+        self.queue = Queue()
+        self.watcher = notification_events.JournalWatcher(self.queue)
+
+    def _check(self, message, *, syslog_id="kernel", transport="kernel"):
+        self.watcher._check_kernel_critical(
+            message,
+            syslog_id,
+            4,
+            {
+                "_TRANSPORT": transport,
+                "__REALTIME_TIMESTAMP": "1788883200000000",
+            },
+        )
+
+    def test_bare_call_trace_is_not_an_event(self):
+        self._check("Call Trace:")
+        with self.assertRaises(Empty):
+            self.queue.get_nowait()
+
+    def test_kernel_warning_carries_attributable_fields(self):
+        self._check(
+            "WARNING: CPU: 2 PID: 418 Comm: z_wr_iss at arc_evict_state+0x12/0x80"
+        )
+        event = self.queue.get_nowait()
+        self.assertEqual(event.event_type, "kernel_warning")
+        self.assertEqual(event.severity, "WARNING")
+        self.assertIn("Type: Kernel warning", event.data["kernel_details"])
+        self.assertIn("Process: z_wr_iss (PID 418)", event.data["kernel_details"])
+        self.assertIn("Component: arc_evict_state", event.data["kernel_details"])
+        self.assertIn("Recorded: 2026-", event.data["kernel_details"])
+        self.assertIn("WARNING: CPU", event.data["_journal_context"])
+
+        self._check("Call Trace:")
+        with self.assertRaises(Empty):
+            self.queue.get_nowait()
+
+    def test_application_text_cannot_impersonate_kernel_warning(self):
+        self._check(
+            "WARNING: CPU: 0 PID: 99 Comm: example at fake_function+0x1/0x2",
+            syslog_id="systemd",
+            transport="stdout",
+        )
+        with self.assertRaises(Empty):
+            self.queue.get_nowait()
+
+    def test_blocked_task_is_identified(self):
+        self._check("INFO: task txg_sync:812 blocked for more than 120 seconds.")
+        event = self.queue.get_nowait()
+        self.assertEqual(event.event_type, "kernel_warning")
+        self.assertIn("Type: Blocked kernel task", event.data["kernel_details"])
+        self.assertIn("Process: txg_sync", event.data["kernel_details"])
+
+    def test_event_is_visible_and_translated_in_every_monitor_locale(self):
+        services = notification_templates.get_event_types_by_group()["services"]
+        self.assertIn("kernel_warning", {item["type"] for item in services})
+        for locale in ("en", "es", "de", "fr", "it", "pt", "sk", "sv"):
+            messages = json.loads(
+                (APPIMAGE_DIR / "messages" / locale / "common.json").read_text(
+                    encoding="utf-8"
+                )
+            )
+            self.assertTrue(
+                messages["settings"]["notifications"]["eventTypes"]["kernel_warning"]
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

AppImage/scripts/tests/test_lxc_app_notification_batch.py

@@ -0,0 +1,140 @@
+import json
+import sys
+import tempfile
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import lxc_apps
+import notification_templates
+
+
+def _app(app_id, name, installed, latest, **extra):
+    return {
+        "id": app_id,
+        "name": name,
+        "state": {
+            "installed_version": installed,
+            "latest_version": latest,
+            "update_available": True,
+        },
+        **extra,
+    }
+
+
+class _FakeNotificationManager:
+    def __init__(self):
+        self.calls = []
+
+    def emit_event(self, **kwargs):
+        self.calls.append(kwargs)
+        return {"success": True}
+
+
+class AppUpdateNotificationBatchTests(unittest.TestCase):
+    def _write_sidecar(self, directory, vmid, apps):
+        Path(directory, f"{vmid}.json").write_text(
+            json.dumps({"vmid": vmid, "apps": apps}),
+            encoding="utf-8",
+        )
+
+    def _emit(self, sidecars):
+        fake = _FakeNotificationManager()
+        module = types.SimpleNamespace(notification_manager=fake)
+        with tempfile.TemporaryDirectory() as directory:
+            for vmid, apps in sidecars.items():
+                self._write_sidecar(directory, vmid, apps)
+            with (
+                mock.patch.object(lxc_apps, "_APPS_DIR", directory),
+                mock.patch.dict(sys.modules, {"notification_manager": module}),
+            ):
+                count = lxc_apps.emit_all_pending_updates()
+        return count, fake.calls
+
+    def test_multiple_updates_are_sent_as_one_sorted_batch(self):
+        count, calls = self._emit({
+            115: [
+                _app("redis", "Redis", "7.0.15-1", "8.10.1"),
+                _app("docmost", "Docmost", "0.23.2", "0.95.0"),
+            ],
+            100: [_app("adguard", "AdGuard Home", "0.107.78", "0.107.79")],
+        })
+
+        self.assertEqual(count, 3)
+        self.assertEqual(len(calls), 1)
+        event = calls[0]
+        self.assertEqual(event["event_type"], "app_update_available")
+        self.assertEqual(event["entity"], "node")
+        self.assertTrue(event["entity_id"].startswith("batch:"))
+        self.assertEqual(event["data"]["count"], 3)
+        self.assertEqual(event["data"]["container_count"], 2)
+        self.assertEqual(
+            [(item["vmid"], item["app_name"]) for item in event["data"]["updates"]],
+            [(100, "AdGuard Home"), (115, "Docmost"), (115, "Redis")],
+        )
+
+    def test_single_update_keeps_the_individual_event_shape(self):
+        count, calls = self._emit({
+            101: [_app("npm", "Nginx Proxy Manager", "2.9.19", "2.15.1")],
+        })
+
+        self.assertEqual(count, 1)
+        self.assertEqual(len(calls), 1)
+        event = calls[0]
+        self.assertEqual(event["entity"], "ct")
+        self.assertNotIn("updates", event["data"])
+        self.assertEqual(event["data"]["vmid"], 101)
+        self.assertEqual(event["data"]["latest"], "2.15.1")
+
+    def test_batch_respects_opt_outs_and_docker_delegation(self):
+        count, calls = self._emit({
+            110: [
+                _app("silent", "Silent", "1.0", "2.0", notifications_enabled=False),
+                _app("docker", "Docker", "1.0", "2.0", helper_slug="docker"),
+                _app("portainer", "Portainer", "2.0", "2.1", update_via="docker"),
+            ],
+        })
+
+        self.assertEqual(count, 0)
+        self.assertEqual(calls, [])
+
+    def test_check_all_can_refresh_without_emitting_individual_events(self):
+        sidecar = {"vmid": 120, "apps": [{"id": "one"}, {"id": "two"}]}
+        with (
+            mock.patch.object(lxc_apps, "_read_sidecar", return_value=sidecar),
+            mock.patch.object(lxc_apps, "check_app") as check,
+        ):
+            lxc_apps.check_all(120, force=False, notify=False)
+
+        self.assertEqual(check.call_count, 2)
+        check.assert_any_call(120, "one", force=False, notify=False)
+        check.assert_any_call(120, "two", force=False, notify=False)
+
+    def test_batch_formatter_groups_versions_by_container(self):
+        rendered = notification_templates.render_template(
+            "app_update_available",
+            {
+                "hostname": "pve01",
+                "updates": [
+                    {"vmid": 115, "app_name": "Redis", "installed": "7.0", "latest": "8.1"},
+                    {"vmid": 100, "app_name": "AdGuard Home", "installed": "1.0", "latest": "1.1"},
+                    {"vmid": 115, "app_name": "Docmost", "installed": "0.2", "latest": "0.9"},
+                ],
+            },
+        )
+
+        self.assertEqual(rendered["title"], "pve01: 3 application updates available")
+        self.assertIn("3 applications in 2 LXC containers", rendered["body"])
+        self.assertLess(rendered["body"].index("CT 100"), rendered["body"].index("CT 115"))
+        self.assertIn("• Docmost: 0.2 → 0.9", rendered["body"])
+        self.assertIn("• Redis: 7.0 → 8.1", rendered["body"])
+
+
+if __name__ == "__main__":
+    unittest.main()

AppImage/scripts/tests/test_notification_burst_toggle_inheritance.py

@@ -0,0 +1,66 @@
+import sys
+import unittest
+from pathlib import Path
+
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import notification_manager  # noqa: E402
+
+
+class RecordingChannel:
+    def __init__(self):
+        self.calls = 0
+
+    def send(self, title, body, severity, data):
+        self.calls += 1
+        return {"success": True, "error": ""}
+
+
+class NotificationBurstToggleInheritanceTests(unittest.TestCase):
+    def setUp(self):
+        self.channel = RecordingChannel()
+        self.manager = notification_manager.NotificationManager()
+        self.manager._channels = {"email": self.channel}
+        self.manager._config = {
+            "email.enabled": "true",
+            "email.events.services": "true",
+            "email.rich_format": "false",
+            "email.event.kernel_warning": "false",
+            "ai_enabled": "false",
+        }
+
+    def test_hidden_summary_inherits_source_event_toggle(self):
+        delivered = self.manager._dispatch_to_channels(
+            "host: +1 more system problem",
+            "One additional issue",
+            "WARNING",
+            "burst_system",
+            {"event_type": "kernel_warning", "hostname": "host"},
+            "aggregator",
+        )
+        self.assertFalse(delivered)
+        self.assertEqual(self.channel.calls, 0)
+
+    def test_generic_summary_inherits_source_event_category(self):
+        self.manager._config.update({
+            "email.event.oom_kill": "true",
+            "email.events.services": "false",
+            "email.events.other": "true",
+        })
+        delivered = self.manager._dispatch_to_channels(
+            "host: related events",
+            "One additional issue",
+            "WARNING",
+            "burst_generic",
+            {"event_type": "oom_kill", "hostname": "host"},
+            "aggregator",
+        )
+        self.assertFalse(delivered)
+        self.assertEqual(self.channel.calls, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()