Pre-release 1.2.7: OCI fixes, 12 new catalog applications and docs updates

Last changes before version 1.2.7.

OCI manager Apps
- Multi-container applications start while one of their containers is
  being backed up: the start hook waits until vzdump releases it and does
  not start it twice. The DHCP client of a dependency no longer keeps the
  lock of the hook, so later starts do not hang, and the watchdog waits
  while a dependency is locked. Installing or updating ProxMenux replaces
  the published copies of the hook already in place.
- Images built from scratch with a numeric user, such as ddns-updater,
  install: the container is created from a copy of the image that adds
  the /etc/passwd and /etc/group Proxmox VE needs, and the application
  runs with the user the image declares. The downloaded image stays
  unchanged. An image with a named user and no user database stops
  before anything is created, with a message that says why.

Catalog
- Twelve new applications: MDBListarr, CrossWatch, Free Games Claimer
  Remaster, Cloudreve, Poznote and AceStream HTTP Proxy, suggested in
  discussion #440, and the Dockur images of Chrony, Dnsmasq, Tor,
  Stunnel, strfry and Statping. The catalog lists 347 applications.
- Their prompts, labels and notes are translated into the seven
  languages of the menus.

Documentation
- OCI manager: architecture, lifecycle and stacks pages revised, with
  larger diagrams and new screenshots.
- Contributors: the page now thanks those developing, reviewing,
  translating and testing ProxMenux today and those who helped in its
  first steps, with new avatars.

Tests for the stack start hook, the watchdog, the user database check
of OCI images and the Tor installation summary.

oci/tests/test_stack_dependency_hook.py

@@ -0,0 +1,144 @@
+"""The start hook of a multi-container application waits for a dependency
+that is being backed up instead of failing the start."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+HOOK = ROOT / "remote" / "stack_dependency_hook.sh"
+
+PCT = r"""#!/bin/bash
+state=$STUB_STATE
+id=${2:-}
+case $1 in
+  status) echo "status: $(cat "$state/$id.status")" ;;
+  config)
+    [[ -f $state/$id.status ]] || exit 2
+    echo "arch: amd64"
+    if [[ -f $state/$id.lock ]]; then
+      left=$(cat "$state/$id.left")
+      if (( left <= 0 )); then
+        rm -f "$state/$id.lock"
+        [[ -f $state/$id.restarted-by-vzdump ]] && echo running >"$state/$id.status"
+      else
+        echo $((left - 1)) >"$state/$id.left"
+        echo "lock: $(cat "$state/$id.lock")"
+      fi
+    fi
+    ;;
+  start)
+    echo "start $id" >>"$state/calls"
+    if [[ -f $state/$id.lock ]]; then echo "CT is locked ($(cat "$state/$id.lock"))" >&2; exit 1; fi
+    echo running >"$state/$id.status"
+    # Like the DHCP client Proxmox leaves running for the container.
+    /bin/sleep 30 >/dev/null 2>&1 &
+    echo $! >>"$state/left-running"
+    ;;
+  *) exit 0 ;;
+esac
+"""
+
+
+@unittest.skipUnless(shutil.which("flock") and shutil.which("jq"), "needs flock and jq")
+class StackDependencyHook(unittest.TestCase):
+    def setUp(self):
+        folder = tempfile.TemporaryDirectory()
+        self.addCleanup(folder.cleanup)
+        self.root = Path(folder.name)
+        self.state = self.root / "state"
+        bin_dir = self.root / "bin"
+        for directory in (self.state, bin_dir, self.root / "priv", self.root / "lock"):
+            directory.mkdir()
+        (bin_dir / "pct").write_text(PCT)
+        (bin_dir / "sleep").write_text("#!/bin/bash\nexit 0\n")
+        for name in ("pct", "sleep"):
+            (bin_dir / name).chmod(0o755)
+        script = HOOK.read_text()
+        for old, new in (("PATH=/usr/sbin:/usr/bin:/sbin:/bin", f"PATH={bin_dir}:{os.environ['PATH']}"),
+                         ("/etc/pve/priv/", f"{self.root}/priv/"), ("/run/lock/", f"{self.root}/lock/")):
+            self.assertIn(old, script)
+            script = script.replace(old, new)
+        self.hook = self.root / "hook.sh"
+        self.hook.write_text(script)
+        (self.root / "priv" / "proxmenux-stack-110.json").write_text(json.dumps({"schema": 1, "dependencies": [
+            {"vmid": 112, "label": "PostgreSQL", "healthcheck": {"type": "running", "timeout_seconds": 10}},
+            {"vmid": 111, "label": "Immich Machine Learning",
+             "healthcheck": {"type": "running", "timeout_seconds": 10}}]}))
+        for vmid in (110, 111, 112):
+            self.container(vmid, "running")
+        self.addCleanup(self.stop_left_running)
+
+    def stop_left_running(self):
+        path = self.state / "left-running"
+        for pid in path.read_text().split() if path.exists() else []:
+            subprocess.run(["kill", pid], capture_output=True)
+
+    def container(self, vmid, status, lock=None, polls=0, restarted_by_vzdump=False):
+        (self.state / f"{vmid}.status").write_text(status + "\n")
+        if lock:
+            (self.state / f"{vmid}.lock").write_text(lock + "\n")
+            (self.state / f"{vmid}.left").write_text(f"{polls}\n")
+        if restarted_by_vzdump:
+            (self.state / f"{vmid}.restarted-by-vzdump").touch()
+
+    def start(self, **environment):
+        env = dict(os.environ, STUB_STATE=str(self.state), **environment)
+        return subprocess.run(["bash", str(self.hook), "110", "pre-start"], capture_output=True, text=True, env=env)
+
+    def calls(self):
+        path = self.state / "calls"
+        return path.read_text().split("\n")[:-1] if path.exists() else []
+
+    def test_a_dependency_stopped_for_its_backup_is_waited_for_and_then_started(self):
+        self.container(111, "stopped", lock="backup", polls=3)
+        result = self.start()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("is locked (backup); waiting until it is released", result.stdout)
+        self.assertIn("Immich Machine Learning (CT 111): ready.", result.stdout)
+        self.assertEqual(self.calls(), ["start 111"])
+
+    def test_what_a_started_dependency_leaves_running_does_not_keep_the_lock(self):
+        self.container(111, "stopped")
+        result = self.start()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.calls(), ["start 111"])
+        free = subprocess.run(["flock", "-n", str(self.root / "lock" / "proxmenux-stack-110.lock"), "true"])
+        self.assertEqual(free.returncode, 0)
+
+    def test_a_dependency_that_vzdump_starts_again_is_not_started_twice(self):
+        self.container(111, "stopped", lock="backup", polls=2, restarted_by_vzdump=True)
+        result = self.start()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.calls(), [])
+
+    def test_a_lock_that_is_never_released_ends_the_wait(self):
+        self.container(111, "stopped", lock="mounted", polls=1000)
+        result = self.start(PROXMENUX_STACK_OTHER_LOCK_WAIT="20")
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("is still locked (mounted) after 20s", result.stderr)
+        self.assertEqual(self.calls(), [])
+
+    def test_running_dependencies_are_not_waited_for(self):
+        self.container(112, "running", lock="backup", polls=1000)
+        result = self.start()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertNotIn("waiting", result.stdout)
+
+    def test_updates_replace_only_published_copies_of_the_hook(self):
+        script = HOOK.read_text()
+        self.assertIn("--refresh", script)
+        for digest in ("c28b69acd7cc8e66d3aec3e8b2ec86bda0e4f91193ec78549f1965f2c859c0d1",
+                       "02c50ef17f2411ef22f4d382c51b34a7a0974a446d4a155ecb63fb1e805a7c59"):
+            self.assertIn(digest, script)
+        for installer in ("install_proxmenux.sh", "install_proxmenux_beta.sh"):
+            self.assertIn("stack_dependency_hook.sh\" --refresh", (ROOT.parent / installer).read_text())
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_tor_install_summary.py

@@ -0,0 +1,39 @@
+"""Protocol-only services explain their addresses without inventing a web UI."""
+import json
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT/'src'))
+from proxmenux_oci import i18n
+from proxmenux_oci.cli import _print_installation_summary
+
+
+class TorSummaryTests(unittest.TestCase):
+    def test_final_summary_explains_proxy_settings_in_spanish(self):
+        template=json.loads((ROOT/'catalog/curated/tor.json').read_text())
+        self.assertFalse(template['first_run']['endpoints'])
+        self.assertIsNone(template['catalog_ui']['launch']['port'])
+        notes=template['proxmox']['installer_profile']['completion_notes']
+        language_file=next(path for path in (ROOT.parent/'lang/es.json',ROOT.parent.parent/'lang/es.json')
+                           if path.is_file())
+        translations=json.loads(language_file.read_text())
+        for note in notes:
+            self.assertIn(note,translations)
+        with patch.object(i18n,'language',return_value='es'), patch.object(i18n,'_cache',translations), \
+                patch('proxmenux_oci.cli.console') as console:
+            _print_installation_summary({
+                'vmid':129,'ip':'192.0.2.24','urls':[],
+                'service_addresses':['192.0.2.24:9050','192.0.2.24:8118'],
+                'completion_notes':notes})
+        shown=[call.args[0] for call in console.msg_note.call_args_list]
+        self.assertIn('Tor es un proxy, no una página web. No tiene interfaz web.',shown)
+        self.assertTrue(any('SOCKS5' in line and '9050' in line for line in shown))
+        self.assertTrue(any('HTTP CONNECT' in line and '8118' in line for line in shown))
+        self.assertTrue(any('Solo las aplicaciones configuradas' in line for line in shown))
+
+
+if __name__=='__main__':
+    unittest.main()

oci/tests/test_user_databases.py

@@ -0,0 +1,193 @@
+"""Images whose User Proxmox cannot resolve are stopped before pct create."""
+from __future__ import annotations
+
+import gzip
+import hashlib
+import io
+import json
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "remote"))
+import oci_runtime as runtime
+
+
+def _tar(entries):
+    data = io.BytesIO()
+    with tarfile.open(fileobj=data, mode="w") as archive:
+        for name, kind in entries:
+            info = tarfile.TarInfo(name)
+            if kind == "dir":
+                info.type = tarfile.DIRTYPE
+                archive.addfile(info)
+            else:
+                body = b"root:x:0:0::/:/bin/sh\n"
+                info.size = len(body)
+                archive.addfile(info, io.BytesIO(body))
+    return data.getvalue()
+
+
+def _blob(archive, data):
+    digest = hashlib.sha256(data).hexdigest()
+    info = tarfile.TarInfo(f"blobs/sha256/{digest}")
+    info.size = len(data)
+    archive.addfile(info, io.BytesIO(data))
+    return f"sha256:{digest}", len(data)
+
+
+def build(path, layers, user="1000:1000", compression="gzip"):
+    """layers: list of entry lists, bottom first."""
+    with tarfile.open(path, "w") as archive:
+        descriptors, diff_ids = [], []
+        for entries in layers:
+            raw = _tar(entries)
+            diff_ids.append("sha256:" + hashlib.sha256(raw).hexdigest())
+            if compression == "gzip":
+                data, media = gzip.compress(raw), "application/vnd.oci.image.layer.v1.tar+gzip"
+            elif compression == "zstd":
+                data = subprocess.run(["zstd", "-q", "-c"], input=raw, capture_output=True, check=True).stdout
+                media = "application/vnd.oci.image.layer.v1.tar+zstd"
+            else:
+                data, media = raw, "application/vnd.oci.image.layer.v1.tar"
+            digest, size = _blob(archive, data)
+            descriptors.append({"mediaType": media, "digest": digest, "size": size})
+        config = {"architecture": "amd64", "os": "linux", "config": {"User": user} if user else {},
+                  "rootfs": {"type": "layers", "diff_ids": diff_ids}}
+        config_digest, config_size = _blob(archive, json.dumps(config).encode())
+        manifest = {"schemaVersion": 2, "config": {"digest": config_digest, "size": config_size},
+                    "layers": descriptors}
+        manifest_digest, manifest_size = _blob(archive, json.dumps(manifest).encode())
+        index = json.dumps({"schemaVersion": 2, "manifests": [
+            {"digest": manifest_digest, "size": manifest_size}]}).encode()
+        info = tarfile.TarInfo("index.json")
+        info.size = len(index)
+        archive.addfile(info, io.BytesIO(index))
+
+
+class UserDatabaseTests(unittest.TestCase):
+    def setUp(self):
+        self.folder = Path(tempfile.mkdtemp())
+        self.addCleanup(shutil.rmtree, self.folder)
+        self.path = self.folder / "image.tar"
+
+    def test_scratch_image_with_a_numeric_user_lacks_both_files(self):
+        build(self.path, [[("updater", "dir"), ("updater/app", "file")]])
+        self.assertEqual(runtime.missing_user_databases(self.path), ["/etc/passwd", "/etc/group"])
+
+    def test_image_with_its_user_databases_is_accepted(self):
+        build(self.path, [[("etc", "dir"), ("etc/passwd", "file"), ("etc/group", "file")], [("app", "file")]])
+        self.assertEqual(runtime.missing_user_databases(self.path), [])
+
+    def test_image_without_user_reads_no_layer(self):
+        build(self.path, [[("app", "file")]], user="")
+        with patch.object(runtime, "_layer_entries", side_effect=AssertionError("layers read")):
+            self.assertEqual(runtime.missing_user_databases(self.path), [])
+
+    def test_a_later_layer_that_removes_passwd_is_respected(self):
+        build(self.path, [[("etc", "dir"), ("etc/passwd", "file"), ("etc/group", "file")],
+                          [("etc/.wh.passwd", "file")]])
+        self.assertEqual(runtime.missing_user_databases(self.path), ["/etc/passwd"])
+
+    def test_a_later_layer_that_removes_etc_is_respected(self):
+        build(self.path, [[("etc", "dir"), ("etc/passwd", "file"), ("etc/group", "file")],
+                          [(".wh.etc", "file")]])
+        self.assertEqual(runtime.missing_user_databases(self.path), ["/etc/passwd", "/etc/group"])
+
+    def test_paths_with_a_leading_dot_slash_are_recognised(self):
+        build(self.path, [[("./etc/", "dir"), ("./etc/passwd", "file"), ("./etc/group", "file")]])
+        self.assertEqual(runtime.missing_user_databases(self.path), [])
+
+    def test_uncompressed_layers_are_read(self):
+        build(self.path, [[("etc/passwd", "file"), ("etc/group", "file")]], compression="none")
+        self.assertEqual(runtime.missing_user_databases(self.path), [])
+
+    @unittest.skipUnless(shutil.which("zstd"), "zstd is not installed")
+    def test_zstd_layers_are_read(self):
+        build(self.path, [[("app", "file")]], compression="zstd")
+        self.assertEqual(runtime.missing_user_databases(self.path), ["/etc/passwd", "/etc/group"])
+
+    def test_command_line_reports_missing_files_with_exit_code_3(self):
+        build(self.path, [[("app", "file")]])
+        result = subprocess.run([sys.executable, str(ROOT / "remote/oci_runtime.py"), "--user-databases",
+                                 str(self.path)], capture_output=True, text=True)
+        self.assertEqual(result.returncode, 3)
+        self.assertEqual(result.stdout.split(), ["/etc/passwd", "/etc/group"])
+
+    def test_installer_creates_from_a_copy_with_the_missing_files(self):
+        script = (ROOT / "remote/install_oci.sh").read_text()
+        check = script.index('--user-databases "$ARCHIVE_PATH"')
+        create = script.index('oci_create_container "${CREATE_ARGS[@]}"')
+        self.assertLess(check, create)
+        block = script[check:create]
+        self.assertIn("user_databases_rc == 3", block)
+        self.assertIn('--add-user-databases "$ARCHIVE_PATH" "$USER_DATABASES_ARCHIVE"', block)
+        self.assertIn("CREATE_ARGS[1]=$USER_DATABASES_ARCHIVE", block)
+        self.assertIn('rm -f "$USER_DATABASES_ARCHIVE"', script[create:create + 300])
+
+
+class AddUserDatabaseTests(unittest.TestCase):
+    def setUp(self):
+        self.folder = Path(tempfile.mkdtemp())
+        self.addCleanup(shutil.rmtree, self.folder)
+        self.source = self.folder / "image.tar"
+        self.copy = self.folder / "copy.tar"
+
+    def read(self, path):
+        with tarfile.open(path) as archive:
+            def blob(digest):
+                return archive.extractfile(f"blobs/sha256/{digest.split(':')[1]}").read()
+            index = json.load(archive.extractfile("index.json"))
+            descriptor = index["manifests"][0]
+            manifest_data = blob(descriptor["digest"])
+            manifest = json.loads(manifest_data)
+            config_data = blob(manifest["config"]["digest"])
+            top = blob(manifest["layers"][-1]["digest"])
+        return index, descriptor, manifest_data, manifest, config_data, top
+
+    def test_a_scratch_image_gains_both_files_and_the_original_is_kept(self):
+        build(self.source, [[("updater", "dir"), ("updater/app", "file")]])
+        original = self.source.read_bytes()
+        self.assertEqual(runtime.add_user_databases(self.source, self.copy), ["/etc/passwd", "/etc/group"])
+        self.assertEqual(self.source.read_bytes(), original)
+        self.assertEqual(runtime.missing_user_databases(self.copy), [])
+        index, descriptor, manifest_data, manifest, config_data, top = self.read(self.copy)
+        self.assertEqual(len(index["manifests"]), 1)
+        self.assertEqual(descriptor["digest"], "sha256:" + hashlib.sha256(manifest_data).hexdigest())
+        self.assertEqual(manifest["config"]["digest"], "sha256:" + hashlib.sha256(config_data).hexdigest())
+        config = json.loads(config_data)
+        self.assertEqual(config["config"]["User"], "1000:1000")
+        self.assertEqual(len(config["rootfs"]["diff_ids"]), len(manifest["layers"]))
+        self.assertEqual(config["rootfs"]["diff_ids"][-1], "sha256:" + hashlib.sha256(top).hexdigest())
+        with tarfile.open(fileobj=io.BytesIO(top)) as layer:
+            self.assertEqual(layer.extractfile("etc/passwd").read(), b"root:x:0:0:root:/root:/sbin/nologin\n")
+            self.assertEqual({item.uid for item in layer.getmembers()}, {0})
+
+    def test_only_the_missing_file_is_added(self):
+        build(self.source, [[("etc", "dir"), ("etc/passwd", "file")]])
+        self.assertEqual(runtime.add_user_databases(self.source, self.copy), ["/etc/group"])
+        with tarfile.open(fileobj=io.BytesIO(self.read(self.copy)[5])) as layer:
+            self.assertEqual(sorted(layer.getnames()), ["etc", "etc/group"])
+
+    def test_an_image_that_needs_nothing_is_not_copied(self):
+        build(self.source, [[("etc/passwd", "file"), ("etc/group", "file")]])
+        self.assertEqual(runtime.add_user_databases(self.source, self.copy), [])
+        self.assertFalse(self.copy.exists())
+
+    def test_a_named_user_is_not_invented(self):
+        for user in ("app", "1000:app"):
+            build(self.source, [[("app", "file")]], user=user)
+            with self.assertRaises(runtime.RuntimeResolutionError):
+                runtime.add_user_databases(self.source, self.copy)
+            self.assertFalse(self.copy.exists())
+            self.assertEqual(list(self.folder.glob("*.partial")), [])
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_watchdog.py

@@ -205,5 +205,54 @@ class Switch(unittest.TestCase):
             self.assertEqual(record["stack"]["members"], [{"vmid": 101}, {"vmid": 102}])
 
 
+class DependencyLocks(unittest.TestCase):
+    """A multi-container application waits while a container it depends on is busy."""
+    def setUp(self):
+        from unittest.mock import patch
+        folder = tempfile.TemporaryDirectory()
+        self.addCleanup(folder.cleanup)
+        self.root = Path(folder.name)
+        (self.root / "lxc").mkdir()
+        (self.root / "priv").mkdir()
+        for target, value in (("CONFIGS", self.root / "lxc"), ("STACK_CONTRACTS", self.root / "priv"),
+                              ("HA_RESOURCES", self.root / "resources.cfg")):
+            patcher = patch.object(watchdog, target, value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+        patcher = patch.object(watchdog.oci_operation_notice, "MARKERS", self.root / "markers")
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        patcher = patch.object(watchdog.subprocess, "run")
+        run = patcher.start()
+        run.return_value.stdout = "running\n"
+        self.addCleanup(patcher.stop)
+        for vmid in (110, 111, 112):
+            self.config(vmid, "arch: amd64\n")
+        (self.root / "priv" / "proxmenux-stack-110.json").write_text(json.dumps(
+            {"schema": 1, "dependencies": [{"vmid": 112, "label": "PostgreSQL"}, {"vmid": 111, "label": "ML"}]}))
+
+    def config(self, vmid, text):
+        (self.root / "lxc" / f"{vmid}.conf").write_text(text)
+
+    def test_a_dependency_being_backed_up_keeps_the_application_waiting(self):
+        self.assertFalse(watchdog.is_busy(110, 1000))
+        self.config(111, "arch: amd64\nlock: backup\n")
+        self.assertTrue(watchdog.is_busy(110, 1000))
+
+    def test_a_lock_in_a_snapshot_of_a_dependency_does_not_count(self):
+        self.config(111, "arch: amd64\n\n[before]\nlock: backup\n")
+        self.assertFalse(watchdog.is_busy(110, 1000))
+
+    def test_a_removed_dependency_does_not_keep_the_application_waiting(self):
+        (self.root / "lxc" / "112.conf").unlink()
+        self.assertFalse(watchdog.is_busy(110, 1000))
+
+    def test_an_application_without_dependencies_only_looks_at_itself(self):
+        self.config(111, "arch: amd64\nlock: backup\n")
+        self.assertFalse(watchdog.is_busy(112, 1000))
+        (self.root / "lxc" / "112.conf").unlink()
+        self.assertTrue(watchdog.is_busy(112, 1000))
+
+
 if __name__ == "__main__":
     unittest.main()