fix(oci): remove everything an OCI installation leaves, and name cluster events

oci/remote/oci_remove.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python3
 """Removes an OCI installation: its containers with the volumes they own, the
-private network of a multi-container application and its saved record. Host
-directories are left exactly as they are."""
+private network of a multi-container application, its saved record and what
+it left on the host for those containers. Host directories are left exactly
+as they are."""
 from __future__ import annotations
 
 import argparse
@@ -19,10 +20,17 @@ import oci_image_cache as image_cache
 import oci_instances as instances
 from oci_installation_state import parse_config
 import oci_console
+import oci_runtime_settings as runtime_settings
 from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error
 
 # The private networks ProxMenux creates for multi-container applications.
 PRIVATE_STACK_NETWORK = ipaddress.ip_network('10.77.0.0/16')
+CLUSTER_NODES = Path('/etc/pve/nodes')
+SNIPPETS = Path('/var/lib/vz/snippets')
+# The App tab of ProxMenux Monitor keeps one file per VMID.
+MONITOR_APPS = Path('/etc/proxmenux/apps')
+HOST_MONITOR_INCLUDES = (Path('/etc/pve/proxmenux/host-monitor'), Path('/etc/pve/lxc/proxmenux-host-monitor'))
+STACK_HOOK = 'proxmenux-stack-dependencies.sh'
 
 
 def run(*args):
@@ -73,6 +81,15 @@ def private_bridge(primary):
     network = (primary.get('stack') or {}).get('deployment', {}).get('network', {})
     bridge = network.get('private_bridge')
     subnet = network.get('private_subnet')
+    if not bridge:
+        # An application of the Arr suite has no stack record: each one is
+        # independent, and its own leg is on the network the suite shares.
+        own = primary.get('deployment', {}).get('network', {})
+        bridge = own.get('bridge')
+        try:
+            subnet = str(ipaddress.ip_interface(own.get('ipv4')).network)
+        except (TypeError, ValueError):
+            return None
     if not bridge or not re.fullmatch(r'vmbr[0-9]+', bridge):
         return None
     try:
@@ -83,6 +100,126 @@ def private_bridge(primary):
     return bridge
 
 
+def guest_node(vmid):
+    """The cluster node that holds the container's configuration, if any.
+    `pct config` sees only the local node; a migrated container is elsewhere."""
+    for path in CLUSTER_NODES.glob(f'*/lxc/{int(vmid)}.conf'):
+        return path.parent.parent.name
+    return None
+
+
+def _unlink(path):
+    try:
+        if path.is_file() and not path.is_symlink():
+            path.unlink()
+    except OSError:
+        pass
+
+
+def remove_host_state(vmid):
+    """What the installation kept on the host for a container that is gone:
+    its sysctl include, the Rclone mount hookscript and the views it
+    published, and its registration in the App tab of ProxMenux Monitor."""
+    for include in (runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid)):
+        _unlink(include)
+    hook = SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh'
+    if hook.is_file() and not hook.is_symlink():
+        unit = f'proxmenux-rclone-publish-{int(vmid)}.service'
+        for action in ('stop', 'reset-failed'):
+            subprocess.run(['systemctl', action, unit], check=False, capture_output=True)
+        views = re.findall(r'^published(?:_ro)?=(/\S+)$', hook.read_text(errors='ignore'), re.MULTILINE)
+        for view in views:
+            # Only an empty directory that is no longer a mount point.
+            if os.path.isdir(view) and not os.path.ismount(view):
+                try:
+                    os.rmdir(view)
+                except OSError:
+                    pass
+        _unlink(hook)
+    _unlink(MONITOR_APPS / f'{int(vmid)}.json')
+    dismissed = MONITOR_APPS / '.oci-dismissed.json'
+    try:
+        entries = json.loads(dismissed.read_text())
+    except (OSError, ValueError):
+        return
+    if isinstance(entries, dict) and entries.pop(str(int(vmid)), None) is not None:
+        temporary = dismissed.with_name(dismissed.name + '.tmp')
+        temporary.write_text(json.dumps(entries, indent=2))
+        os.replace(temporary, dismissed)
+
+
+def _guest_configs():
+    texts = []
+    for pattern in ('*/lxc/*.conf', '*/qemu-server/*.conf'):
+        for path in CLUSTER_NODES.glob(pattern):
+            texts.append(path.read_text(encoding='utf-8', errors='ignore'))
+    return '\n'.join(texts)
+
+
+def release_shared_host_files(hookscripts):
+    """Files several installations share, once no guest of the cluster uses
+    them: the host-monitor include and the stack dependency hookscript."""
+    remaining = _guest_configs()
+    for include in HOST_MONITOR_INCLUDES:
+        if include.is_file() and f'lxc.include: {include}' not in remaining:
+            _unlink(include)
+    for volume in set(hookscripts):
+        if STACK_HOOK not in volume or f'hookscript: {volume}' in remaining:
+            continue
+        try:
+            path = Path(instances.command('pvesm', 'path', volume).decode().strip())
+        except (subprocess.CalledProcessError, RuntimeError, OSError):
+            continue
+        if path.name == STACK_HOOK:
+            _unlink(path)
+
+
+def _leftovers(vmid):
+    """Whether anything of the container is still on the host."""
+    paths = [runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid),
+             SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh', MONITOR_APPS / f'{int(vmid)}.json',
+             *oci_console.LOG_DIR.glob(f'{int(vmid)}.console.log*')]
+    return any(path.exists() for path in paths)
+
+
+def sweep_orphans(root):
+    """Leftovers of containers that exist on no node of the cluster: the
+    record of one deleted from the Proxmox interface, and files an earlier
+    removal left behind. A record with an operation left halfway is kept,
+    because its backup may still be needed. Returns the VMIDs cleaned."""
+    found = {int(d.name) for d in root.iterdir() if d.name.isdecimal()} if root.is_dir() else set()
+    for directory, pattern in ((runtime_settings.include_path(0).parent, r'([0-9]+)\.sysctls'),
+                               (runtime_settings.legacy_include_path(0).parent, r'([0-9]+)\.proxmenux-sysctls'),
+                               (oci_console.LOG_DIR, r'([0-9]+)\.console\.log.*'),
+                               (SNIPPETS, r'proxmenux-rclone-([0-9]+)-fuse-hook\.sh')):
+        if directory.is_dir():
+            found.update(int(m.group(1)) for m in (re.fullmatch(pattern, p.name) for p in directory.iterdir()) if m)
+    # Only the App tab registrations of OCI installs; the other ones belong to
+    # ordinary containers.
+    for path in MONITOR_APPS.glob('*.json') if MONITOR_APPS.is_dir() else []:
+        try:
+            apps = json.loads(path.read_text()).get('apps') or []
+        except (OSError, ValueError, AttributeError):
+            continue
+        if path.stem.isdecimal() and any(app.get('installed_via') == 'oci_image' for app in apps):
+            found.add(int(path.stem))
+    cleaned = []
+    for vmid in sorted(found):
+        if instances.guest_exists(vmid):
+            continue
+        record = instances.has_contract(root, vmid)
+        if record and not instances.release_orphan(root, vmid):
+            continue
+        if not (record or _leftovers(vmid)):
+            continue
+        oci_console.remove_log(vmid)
+        remove_host_state(vmid)
+        cleaned.append(vmid)
+    if cleaned:
+        release_shared_host_files([])
+    return cleaned
+
+
 def bridge_in_use(bridge, removed):
     """Whether a guest that is not being removed still uses the bridge."""
     for path in Path('/etc/pve/nodes').glob('*/lxc/*.conf'):
@@ -163,10 +300,15 @@ def remove(root, vmid):
         if record.get('pending_transaction') or record.get('pending_stack_transaction'):
             raise ValueError(translate('An operation of this installation has not finished; '
                                        'recover it from the management menu before removing it'))
+        node = guest_node(member) if guest_config(member) is None else None
+        if node:
+            raise ValueError(f"{translate('The container runs on another node of the cluster; migrate it back to this node to remove it:')} "
+                             f"CT {member} ({node})")
     kept = host_directories(root, members)
     bridge = private_bridge(primary)
     incomplete = False
     msg_info(translate('Removing the containers...'))
+    hookscripts = []
     for member in members:
         record = instances.read(root, member)
         config = guest_config(member)
@@ -174,13 +316,16 @@ def remove(root, vmid):
             msg_warn(f"{translate('The container no longer exists:')} CT {member}")
             incomplete = True
             oci_console.remove_log(member)
+            remove_host_state(member)
         elif instances.identity(config) != record['installation_id']:
             msg_warn(f"{translate('The VMID belongs to another container now and is not touched:')} CT {member}")
             incomplete = True
         else:
+            hookscripts += re.findall(r'^hookscript: (\S+)$', config.decode(errors='ignore'), re.MULTILINE)
             subprocess.run(['pct', 'stop', str(member), '--skiplock', '1'], check=False, capture_output=True)
             run('pct', 'destroy', str(member), '--purge', '1', '--destroy-unreferenced-disks', '1')
             oci_console.remove_log(member)
+            remove_host_state(member)
             msg_ok(f"{translate('Container removed:')} CT {member}")
     if bridge and not bridge_in_use(bridge, set(members)):
         released = release_bridge(bridge)
@@ -192,6 +337,7 @@ def remove(root, vmid):
         msg_info(f"{translate('The private network is still used by another container and is kept:')} {bridge}")
     if not remove_owned_host_firewall(primary):
         incomplete = True
+    release_shared_host_files(hookscripts)
     lifecycle = Path(f'/etc/pve/priv/proxmenux-stack-{primary_id}.json')
     if lifecycle.exists() and not lifecycle.is_symlink():
         lifecycle.unlink()

oci/src/proxmenux_oci/management.py

@@ -140,10 +140,23 @@ def interactive_management(project, ui):
         ui.message(translate('OCI management could not be completed. Check the backend status; no additional cleanup has been authorized.'), translate('OCI management'))
 
 
+def _clean_orphans(project):
+    """Remove, silently, what is left of containers that exist on no node of the cluster."""
+    sys.path.insert(0, str(project / 'remote'))
+    import oci_instances as instances
+    import oci_remove
+    try:
+        with instances.locked(instances.ROOT):
+            oci_remove.sweep_orphans(instances.ROOT)
+    except (BlockingIOError, OSError, ValueError):
+        pass
+
+
 def _interactive_management(project, ui):
     if os.geteuid() != 0 or not shutil.which('pct'):
         ui.message(translate('This interface runs on the Proxmox node as root. Open OCI manager Apps from the ProxMenux menu on the Proxmox host.'), translate('OCI management'))
         return
+    _clean_orphans(project)
     rows = saved_inventory(project)
     if not rows:
         ui.message(translate('No registered OCI containers are available for selection on this host.'), translate('OCI management'))
@@ -316,8 +329,10 @@ def _removal_summary(project, vmid):
                  f"{translate('containers of')} {application}. {alone}", '']
     text += [translate('Containers targeted for removal:'), *lines, '',
              translate('Container data targeted for deletion:'), *volumes]
-    if bridge:
+    if bridge and not oci_remove.bridge_in_use(bridge, set(members)):
         text += ['', f"{translate('Private network targeted for release if no other guest uses it:')} {bridge}"]
+    elif bridge:
+        text += ['', f"{translate('Private network kept, because other containers still use it:')} {bridge}"]
     if (primary.get('deployment') or {}).get('host_firewall'):
         text += ['', translate('A matching managed host firewall rule may also be removed.')]
     if kept:

oci/tests/test_removal_cleanup.py

@@ -0,0 +1,177 @@
+"""Removing an OCI application leaves nothing of it on the host, and nothing another guest still uses is touched."""
+
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "remote"))
+
+import oci_remove
+import oci_runtime_settings as runtime_settings
+
+HOOK = """#!/usr/bin/env bash
+inside=/data/mounts/drive
+published={shared}/rw/drive
+published_ro={shared}/ro/drive
+"""
+
+
+class RemovalCleanupTests(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.root = Path(tmp.name)
+        self.nodes = self.root / "nodes"
+        (self.nodes / "amd/lxc").mkdir(parents=True)
+        (self.nodes / "pve2/lxc").mkdir(parents=True)
+        self.snippets = self.root / "snippets"
+        self.snippets.mkdir()
+        self.apps = self.root / "apps"
+        self.apps.mkdir()
+        self.cluster = self.root / "proxmenux"
+        self.cluster.mkdir()
+        self.legacy = self.root / "legacy"
+        self.legacy.mkdir()
+        self.host_monitor = (self.cluster / "host-monitor", self.legacy / "proxmenux-host-monitor")
+        patches = [
+            patch.object(oci_remove, "CLUSTER_NODES", self.nodes),
+            patch.object(oci_remove, "SNIPPETS", self.snippets),
+            patch.object(oci_remove, "MONITOR_APPS", self.apps),
+            patch.object(oci_remove, "HOST_MONITOR_INCLUDES", self.host_monitor),
+            patch.object(runtime_settings, "include_path", lambda vmid: self.cluster / f"{vmid}.sysctls"),
+            patch.object(runtime_settings, "legacy_include_path", lambda vmid: self.legacy / f"{vmid}.proxmenux-sysctls"),
+            patch.object(oci_remove.subprocess, "run"),
+        ]
+        for item in patches:
+            item.start()
+            self.addCleanup(item.stop)
+
+    def test_every_file_of_the_container_is_removed(self):
+        (self.cluster / "113.sysctls").write_text("lxc.sysctl.net.ipv4.ip_unprivileged_port_start = 0\n")
+        (self.legacy / "113.proxmenux-sysctls").write_text("x\n")
+        shared = self.root / "shared"
+        for view in ("rw/drive", "ro/drive"):
+            (shared / view).mkdir(parents=True)
+        (self.snippets / "proxmenux-rclone-113-fuse-hook.sh").write_text(HOOK.format(shared=shared))
+        (self.apps / "113.json").write_text("{}")
+        (self.apps / ".oci-dismissed.json").write_text(json.dumps({"113": "a", "112": "b"}))
+        oci_remove.remove_host_state(113)
+        self.assertFalse((self.cluster / "113.sysctls").exists())
+        self.assertFalse((self.legacy / "113.proxmenux-sysctls").exists())
+        self.assertFalse((self.snippets / "proxmenux-rclone-113-fuse-hook.sh").exists())
+        self.assertFalse((shared / "rw/drive").exists() or (shared / "ro/drive").exists())
+        self.assertFalse((self.apps / "113.json").exists())
+        self.assertEqual(json.loads((self.apps / ".oci-dismissed.json").read_text()), {"112": "b"})
+
+    def test_a_published_view_with_content_is_kept(self):
+        shared = self.root / "shared"
+        (shared / "rw/drive").mkdir(parents=True)
+        (shared / "rw/drive/file").write_text("data")
+        (self.snippets / "proxmenux-rclone-113-fuse-hook.sh").write_text(HOOK.format(shared=shared))
+        oci_remove.remove_host_state(113)
+        self.assertTrue((shared / "rw/drive/file").exists())
+
+    def test_other_containers_are_not_touched(self):
+        (self.cluster / "114.sysctls").write_text("x\n")
+        (self.apps / "114.json").write_text("{}")
+        oci_remove.remove_host_state(113)
+        self.assertTrue((self.cluster / "114.sysctls").exists() and (self.apps / "114.json").exists())
+
+    def test_shared_host_monitor_include_is_kept_while_a_guest_uses_it(self):
+        self.host_monitor[0].write_text("lxc.namespace.share.net = 1\n")
+        (self.nodes / "pve2/lxc/120.conf").write_text(f"arch: amd64\nlxc.include: {self.host_monitor[0]}\n")
+        oci_remove.release_shared_host_files([])
+        self.assertTrue(self.host_monitor[0].exists())
+        (self.nodes / "pve2/lxc/120.conf").unlink()
+        oci_remove.release_shared_host_files([])
+        self.assertFalse(self.host_monitor[0].exists())
+
+    def test_stack_hookscript_is_removed_only_when_no_guest_uses_it(self):
+        hook = self.snippets / "proxmenux-stack-dependencies.sh"
+        hook.write_text("#!/bin/sh\n")
+        volume = "local:snippets/proxmenux-stack-dependencies.sh"
+        with patch.object(oci_remove.instances, "command", return_value=f"{hook}\n".encode()):
+            (self.nodes / "amd/lxc/130.conf").write_text(f"hookscript: {volume}\n")
+            oci_remove.release_shared_host_files([volume])
+            self.assertTrue(hook.exists())
+            (self.nodes / "amd/lxc/130.conf").unlink()
+            oci_remove.release_shared_host_files([volume, "local:snippets/someone-else.sh"])
+        self.assertFalse(hook.exists())
+
+    def sweep(self, existing=()):
+        registry = self.root / "instances"
+        registry.mkdir(exist_ok=True)
+        logs = self.root / "logs"
+        logs.mkdir(exist_ok=True)
+        with patch.object(oci_remove.oci_console, "LOG_DIR", logs), \
+             patch.object(oci_remove.instances, "guest_exists", lambda vmid: vmid in existing), \
+             patch.object(oci_remove.instances, "has_contract", lambda root, vmid: (root / str(vmid) / "oci-compose.json").exists()), \
+             patch.object(oci_remove.instances, "release_orphan", self.retire):
+            return oci_remove.sweep_orphans(registry), registry, logs
+
+    def retire(self, root, vmid):
+        record = root / str(vmid) / "oci-compose.json"
+        if json.loads(record.read_text()).get("pending_transaction"):
+            return False
+        record.replace(record.with_name("retired-x.json"))
+        return True
+
+    def test_sweep_removes_what_is_left_of_containers_that_no_longer_exist(self):
+        registry = self.root / "instances"
+        (registry / "151").mkdir(parents=True)
+        (registry / "151/oci-compose.json").write_text("{}")
+        (self.legacy / "9901.proxmenux-sysctls").write_text("x\n")
+        (self.cluster / "100.sysctls").write_text("x\n")
+        cleaned, registry, logs = self.sweep(existing={100})
+        self.assertEqual(cleaned, [151, 9901])
+        self.assertTrue((registry / "151/retired-x.json").exists())
+        self.assertFalse((self.legacy / "9901.proxmenux-sysctls").exists())
+        # A container that exists keeps everything.
+        self.assertTrue((self.cluster / "100.sysctls").exists())
+        # Nothing left: the next visit cleans and reports nothing.
+        self.assertEqual(self.sweep(existing={100})[0], [])
+
+    def test_sweep_finds_oci_registrations_of_the_app_tab_only(self):
+        (self.apps / "113.json").write_text(json.dumps({"apps": [{"installed_via": "oci_image"}]}))
+        (self.apps / "114.json").write_text(json.dumps({"apps": [{"installed_via": "dpkg"}]}))
+        self.assertEqual(self.sweep()[0], [113])
+        self.assertFalse((self.apps / "113.json").exists())
+        self.assertTrue((self.apps / "114.json").exists())
+
+    def test_sweep_keeps_an_operation_left_halfway(self):
+        registry = self.root / "instances"
+        (registry / "152").mkdir(parents=True)
+        (registry / "152/oci-compose.json").write_text(json.dumps({"pending_transaction": "x"}))
+        (self.cluster / "152.sysctls").write_text("x\n")
+        self.assertEqual(self.sweep()[0], [])
+        self.assertTrue((self.cluster / "152.sysctls").exists())
+
+    def test_private_network_of_a_stack_and_of_an_arr_suite_application(self):
+        stack = {"stack": {"deployment": {"network": {"private_bridge": "vmbr10", "private_subnet": "10.77.0.0/24"}}}}
+        suite = {"deployment": {"network": {"bridge": "vmbr11", "ipv4": "10.77.1.31/24"}}}
+        lan = {"deployment": {"network": {"bridge": "vmbr0", "ipv4": "192.168.0.40/24"}}}
+        dhcp = {"deployment": {"network": {"bridge": "vmbr0", "ipv4": "dhcp"}}}
+        self.assertEqual(oci_remove.private_bridge(stack), "vmbr10")
+        self.assertEqual(oci_remove.private_bridge(suite), "vmbr11")
+        self.assertIsNone(oci_remove.private_bridge(lan))
+        self.assertIsNone(oci_remove.private_bridge(dhcp))
+
+    def test_the_suite_network_is_kept_while_another_application_uses_it(self):
+        (self.nodes / "amd/lxc/180.conf").write_text("net0: name=eth0,bridge=vmbr11,ip=10.77.1.30/24\n")
+        (self.nodes / "amd/lxc/181.conf").write_text("net0: name=eth0,bridge=vmbr11,ip=10.77.1.31/24\n")
+        with patch.object(oci_remove, "Path", lambda value: self.nodes if value == "/etc/pve/nodes" else Path(value)):
+            self.assertTrue(oci_remove.bridge_in_use("vmbr11", {181}))
+            self.assertFalse(oci_remove.bridge_in_use("vmbr11", {180, 181}))
+
+    def test_a_container_on_another_node_is_found(self):
+        (self.nodes / "pve2/lxc/113.conf").write_text("arch: amd64\n")
+        self.assertEqual(oci_remove.guest_node(113), "pve2")
+        self.assertIsNone(oci_remove.guest_node(114))
+
+
+if __name__ == "__main__":
+    unittest.main()