fix(oci): start OCI containers on any node of a cluster

oci/README.md

@@ -485,9 +485,11 @@ so the monitor sees the host's real processor count.
 
 Proxmox does not accept `lxc.namespace.share.*` directly in CT configuration.
 The installer uses supported `lxc.include` referencing the static companion
-`/etc/pve/lxc/proxmenux-host-monitor`. This file persists across host reboots but
-is NOT included in a CT vzdump. Preserve/recreate it when restoring on another
-host, in addition to restoring managed volumes. Full restore/image replacement
+`/etc/pve/proxmenux/host-monitor`. `/etc/pve/proxmenux` is the same on every node
+of a cluster, unlike `/etc/pve/lxc`, which is the folder of the local node, so a
+migrated container finds it. It is NOT included in a CT vzdump: copy it when
+restoring outside the cluster, in addition to restoring managed volumes.
+Installations made before this keep `/etc/pve/lxc/proxmenux-host-monitor`. Full restore/image replacement
 have not been tested. No custom supervisor or image entrypoint is introduced.
 
 DeepSeek OCR remains deferred at the user's request: registry inspection on

oci/catalog/apps/glances.json

@@ -37,7 +37,7 @@
       "Privileged access to the host PID/network and unconfined AppArmor: requires confirmation. Trusted networks only; the host firewall is the one that applies.",
       "Neither docker.sock nor the host root directory is mounted. Docker inventory, SMART and capacity of all filesystems are not guaranteed.",
       "LXCFS is skipped only in the monitor CT so that the limited container RAM/CPU is not shown.",
-      "Uses lxc.include with /etc/pve/lxc/proxmenux-host-monitor; this file is not included in vzdump and must be kept when restoring on another host.",
+      "Uses lxc.include with /etc/pve/proxmenux/host-monitor, which every node of the cluster shares; vzdump does not include it, so it must be copied when restoring outside the cluster.",
       "The CPU limit is applied as cpulimit, without hiding processors through affinity."
     ],
     "mini_changelog": [],
@@ -489,7 +489,7 @@
       "host_network_namespace": true,
       "host_memory_bytes": 16110522368,
       "shutdown_start_passed": true,
-      "companion_include": "/etc/pve/lxc/proxmenux-host-monitor",
+      "companion_include": "/etc/pve/proxmenux/host-monitor",
       "companion_included_in_vzdump": false,
       "rolling_tag_validation": "only-observed-digest",
       "host_cpu_count": 16,

oci/catalog/apps/netdata.json

@@ -37,7 +37,7 @@
       "Privileged access to the host PID/network and unconfined AppArmor: requires confirmation. Trusted networks only; the host firewall is the one that applies.",
       "Neither docker.sock nor the host root directory is mounted. Docker inventory, SMART and capacity of all filesystems are not guaranteed.",
       "LXCFS is skipped only in the monitor CT so that the limited container RAM/CPU is not shown.",
-      "Uses lxc.include with /etc/pve/lxc/proxmenux-host-monitor; this file is not included in vzdump and must be kept when restoring on another host.",
+      "Uses lxc.include with /etc/pve/proxmenux/host-monitor, which every node of the cluster shares; vzdump does not include it, so it must be copied when restoring outside the cluster.",
       "The CPU limit is applied as cpulimit, without hiding processors through affinity."
     ],
     "mini_changelog": [],
@@ -657,7 +657,7 @@
       "host_network_namespace": true,
       "host_memory_bytes": 16110522368,
       "shutdown_start_passed": true,
-      "companion_include": "/etc/pve/lxc/proxmenux-host-monitor",
+      "companion_include": "/etc/pve/proxmenux/host-monitor",
       "companion_included_in_vzdump": false,
       "rolling_tag_validation": "only-observed-digest",
       "cgroup_charts_observed": 144,

oci/catalog/overlays/glances.json

@@ -6,7 +6,7 @@
       "Privileged access to the host PID/network and unconfined AppArmor: requires confirmation. Trusted networks only; the host firewall is the one that applies.",
       "Neither docker.sock nor the host root directory is mounted. Docker inventory, SMART and capacity of all filesystems are not guaranteed.",
       "LXCFS is skipped only in the monitor CT so that the limited container RAM/CPU is not shown.",
-      "Uses lxc.include with /etc/pve/lxc/proxmenux-host-monitor; this file is not included in vzdump and must be kept when restoring on another host.",
+      "Uses lxc.include with /etc/pve/proxmenux/host-monitor, which every node of the cluster shares; vzdump does not include it, so it must be copied when restoring outside the cluster.",
       "The CPU limit is applied as cpulimit, without hiding processors through affinity."
     ]
   },
@@ -102,7 +102,7 @@
       "host_network_namespace": true,
       "host_memory_bytes": 16110522368,
       "shutdown_start_passed": true,
-      "companion_include": "/etc/pve/lxc/proxmenux-host-monitor",
+      "companion_include": "/etc/pve/proxmenux/host-monitor",
       "companion_included_in_vzdump": false,
       "rolling_tag_validation": "only-observed-digest",
       "host_cpu_count": 16,

oci/catalog/overlays/netdata.json

@@ -6,7 +6,7 @@
       "Privileged access to the host PID/network and unconfined AppArmor: requires confirmation. Trusted networks only; the host firewall is the one that applies.",
       "Neither docker.sock nor the host root directory is mounted. Docker inventory, SMART and capacity of all filesystems are not guaranteed.",
       "LXCFS is skipped only in the monitor CT so that the limited container RAM/CPU is not shown.",
-      "Uses lxc.include with /etc/pve/lxc/proxmenux-host-monitor; this file is not included in vzdump and must be kept when restoring on another host.",
+      "Uses lxc.include with /etc/pve/proxmenux/host-monitor, which every node of the cluster shares; vzdump does not include it, so it must be copied when restoring outside the cluster.",
       "The CPU limit is applied as cpulimit, without hiding processors through affinity."
     ]
   },
@@ -152,7 +152,7 @@
       "host_network_namespace": true,
       "host_memory_bytes": 16110522368,
       "shutdown_start_passed": true,
-      "companion_include": "/etc/pve/lxc/proxmenux-host-monitor",
+      "companion_include": "/etc/pve/proxmenux/host-monitor",
       "companion_included_in_vzdump": false,
       "rolling_tag_validation": "only-observed-digest",
       "cgroup_charts_observed": 144,

oci/remote/install_oci.sh

@@ -286,8 +286,11 @@ apply_host_monitor() {
   [[ -n ${HOST_MONITOR:-} ]] || return 0
   # PVE permits lxc.include but not namespace keys directly in the CT config.
   # This static, cluster-persistent companion must accompany cross-host restores.
-  local include=/etc/pve/lxc/proxmenux-host-monitor native
+  # /etc/pve/proxmenux is the same on every node of a cluster; /etc/pve/lxc
+  # is the folder of this node only.
+  local include=/etc/pve/proxmenux/host-monitor native
   native=$'lxc.namespace.share.pid = 1\nlxc.namespace.share.net = 1'
+  mkdir -p /etc/pve/proxmenux || die "$(translate "Could not create the ProxMenux folder in /etc/pve")"
   if [[ -e $include ]]; then
     [[ $(cat "$include") == "$native" ]] || die "$(translate "A different host monitor include already exists; it is not overwritten:") $include"
   else
@@ -300,7 +303,7 @@ apply_host_monitor() {
   # Do not remove the Proxmox pre-start, autodev or post-stop hooks.
   set_lxc_directive lxc.hook.mount ""
   msg_ok "$(translate "Host monitor configured: shared PID and network namespaces, LXCFS disabled in this container")"
-  msg_info2 "$(translate "If you restore this container on another Proxmox host, copy this file to the same path first, because the container backup does not include it:") $include"
+  msg_info2 "$(translate "Every node of this cluster already has this file. If you restore this container on any other Proxmox host, copy it to the same path first, because the container backup does not include it:") $include"
 }
 
 verify_host_monitor() {
@@ -1661,7 +1664,8 @@ fi
 
 SYSCTL_COUNT=$(jq '.security.sysctls? // [] | length' "$DEPLOYMENT_FILE")
 if (( SYSCTL_COUNT > 0 )); then
-  SYSCTL_INCLUDE="/etc/pve/lxc/${VMID}.proxmenux-sysctls"
+  SYSCTL_INCLUDE="/etc/pve/proxmenux/${VMID}.sysctls"
+  mkdir -p /etc/pve/proxmenux || die "$(translate "Could not create the ProxMenux folder in /etc/pve")"
   SYSCTL_TEMP=$(mktemp)
   while IFS=$'\t' read -r SYSCTL_NAME SYSCTL_VALUE; do
     [[ -n $SYSCTL_NAME ]] || continue

oci/remote/oci_console.py

@@ -28,14 +28,20 @@ LOG_DIR = Path('/var/log/proxmenux/oci')
 # Each start is marked in the console log by a pre-start hook. The hook runs
 # the script only when it exists and always succeeds: a hook that fails would
 # stop the container from starting.
-START_MARK_SCRIPT = Path(__file__).resolve().with_name('oci_console_mark.sh')
+# The installed engine, not this file: the installer runs a copy of the
+# engine from a temporary directory that is gone once it finishes.
+START_MARK_SCRIPT = Path('/usr/local/share/proxmenux/oci/engine/remote/oci_console_mark.sh')
 START_MARK = '=== ProxMenux: container started '
 
 
 def start_mark_hook(vmid: int) -> str:
     script = START_MARK_SCRIPT
+    # The hook runs before liblxc opens the console log, and liblxc refuses to
+    # start a container whose log directory is missing: on another cluster
+    # node, after a migration, it may never have been created.
     # `test`, not `[`: a bracket in the configuration reads as a snapshot section.
-    return f"lxc.hook.pre-start: /bin/sh -c 'test -x {script} && {script} {int(vmid)}; exit 0'"
+    return (f"lxc.hook.pre-start: /bin/sh -c 'mkdir -p {LOG_DIR}; "
+            f"test -x {script} && {script} {int(vmid)}; exit 0'")
 LOGROTATE = Path('/etc/logrotate.d/proxmenux-oci')
 # copytruncate, because liblxc keeps the file open for as long as the
 # container runs; moving it away would leave the application writing into the

oci/remote/oci_runtime_settings.py

@@ -44,7 +44,17 @@ def tmpfs_lines(deployment):
     return result
 
 
+# /etc/pve/lxc is the folder of the local node only; this one is the same on
+# every node of a cluster, so a migrated container still finds its include.
+CLUSTER_DIR = Path('/etc/pve/proxmenux')
+
+
 def include_path(vmid):
+    return CLUSTER_DIR / f'{int(vmid)}.sysctls'
+
+
+def legacy_include_path(vmid):
+    """Where installations made before the cluster folder keep the include."""
     return Path(f'/etc/pve/lxc/{int(vmid)}.proxmenux-sysctls')
 
 
@@ -56,10 +66,11 @@ def check(config, deployment, vmid):
         raise ValueError(translate('The tmpfs mounts of the container differ from the saved record'))
     includes = [line.split(': ',1)[1] for line in lines if line.startswith('lxc.include: ')]
     content = sysctl_content(deployment)
-    if includes != ([str(include_path(vmid))] if content else []):
+    allowed = [[str(include_path(vmid))], [str(legacy_include_path(vmid))]] if content else [[]]
+    if includes not in allowed:
         raise ValueError(translate('The sysctl include is unknown or differs from the saved record'))
     if content:
-        path = include_path(vmid)
+        path = Path(includes[0])
         info = path.lstat()
         if not stat.S_ISREG(info.st_mode) or info.st_uid != 0 or info.st_mode & 0o022:
             raise ValueError(translate('The sysctl include is not a safe host file'))
@@ -82,8 +93,8 @@ def check_recovery(config, state):
         if line.startswith('lxc.mount.entry: tmpfs ') and line.split(': ', 1)[1] not in permitted:
             raise ValueError(translate('A tmpfs mount is not part of the journal; recovery blocked'))
         if line.startswith('lxc.include: '):
-            path = include_path(state['vmid'])
-            if line.split(': ', 1)[1] != str(path):
+            path = Path(line.split(': ', 1)[1])
+            if path not in (include_path(state['vmid']), legacy_include_path(state['vmid'])):
                 raise ValueError(translate('An include is not part of the journal; recovery blocked'))
             contents = {sysctl_content(plan) for plan in plans} - {''}
             info = path.lstat()
@@ -99,6 +110,7 @@ def restore(deployment, vmid):
     path = include_path(vmid)
     if path.is_symlink():
         raise ValueError(translate('The sysctl include is not restored over a symbolic link'))
+    path.parent.mkdir(parents=True, exist_ok=True)
     fd, temporary = tempfile.mkstemp(dir=path.parent, prefix='.oci-sysctl-')
     try:
         with os.fdopen(fd, 'w') as output:
@@ -107,7 +119,7 @@ def restore(deployment, vmid):
             os.fsync(output.fileno())
         # pmxcfs uses fixed permissions; ordinary filesystem fixtures still
         # receive an explicit restrictive mode.
-        if path.parent != Path('/etc/pve/lxc'):
+        if Path('/etc/pve') not in path.parents:
             os.chmod(temporary, 0o640)
         os.replace(temporary, path)
     finally:

oci/tests/test_cluster_paths.py

@@ -0,0 +1,80 @@
+"""A migrated container finds what it needs on any node of a cluster."""
+
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "remote"))
+
+import oci_console
+import oci_runtime_settings as runtime_settings
+
+DEPLOYMENT = {"security": {"sysctls": [{"name": "net.ipv4.ip_unprivileged_port_start", "value": "0"}]}}
+
+
+class StartHookTests(unittest.TestCase):
+    def test_hook_creates_the_log_directory_before_liblxc_opens_it(self):
+        hook = oci_console.start_mark_hook(101)
+        self.assertTrue(hook.startswith("lxc.hook.pre-start: /bin/sh -c 'mkdir -p /var/log/proxmenux/oci; "))
+        self.assertIn("test -x", hook)
+        self.assertTrue(hook.endswith("101; exit 0'"))
+        self.assertNotIn("[", hook)
+
+    def test_hook_runs_the_installed_engine_not_the_installer_copy(self):
+        # The installer runs from a temporary copy that is removed afterwards.
+        self.assertIn("test -x /usr/local/share/proxmenux/oci/engine/remote/oci_console_mark.sh && ",
+                      oci_console.start_mark_hook(101))
+        self.assertTrue((ROOT / "remote/oci_console_mark.sh").is_file())
+
+
+class SysctlIncludeTests(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.root = Path(tmp.name)
+        content = runtime_settings.sysctl_content(DEPLOYMENT)
+        self.new = self.root / "proxmenux/101.sysctls"
+        self.old = self.root / "lxc/101.proxmenux-sysctls"
+        for path in (self.new, self.old):
+            path.parent.mkdir(parents=True)
+            path.write_text(content)
+            path.chmod(0o640)
+        for name, value in (("include_path", lambda vmid: self.new), ("legacy_include_path", lambda vmid: self.old)):
+            patcher = patch.object(runtime_settings, name, value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+
+    def config(self, include):
+        return f"arch: amd64\nlxc.include: {include}\n".encode()
+
+    def test_cluster_path_and_the_path_of_earlier_installs_are_accepted(self):
+        runtime_settings.check(self.config(self.new), DEPLOYMENT, 101)
+        runtime_settings.check(self.config(self.old), DEPLOYMENT, 101)
+
+    def test_any_other_include_is_refused(self):
+        with self.assertRaises(ValueError):
+            runtime_settings.check(self.config(self.root / "elsewhere"), DEPLOYMENT, 101)
+
+    def test_recovery_accepts_both_paths_only(self):
+        state = {"vmid": 101, "record": {"deployment": DEPLOYMENT}}
+        runtime_settings.check_recovery(self.config(self.new), state)
+        runtime_settings.check_recovery(self.config(self.old), state)
+        with self.assertRaises(ValueError):
+            runtime_settings.check_recovery(self.config(self.root / "elsewhere"), state)
+
+    def test_restore_writes_the_cluster_path(self):
+        self.new.unlink()
+        self.new.parent.rmdir()
+        runtime_settings.restore(DEPLOYMENT, 101)
+        self.assertEqual(self.new.read_text(), runtime_settings.sysctl_content(DEPLOYMENT))
+
+    def test_the_cluster_folder_is_shared_by_every_node(self):
+        self.assertEqual(runtime_settings.CLUSTER_DIR, Path("/etc/pve/proxmenux"))
+        self.assertNotIn(Path("/etc/pve/lxc"), runtime_settings.CLUSTER_DIR.parents)
+
+
+if __name__ == "__main__":
+    unittest.main()