feat(oci): coherent advanced installer flow, container creation progress and USB device list

2026-10-08 22:46:41 +00:00 · 2026-09-29 20:20:36 +02:00
parent db4ef3f29b
commit 135df4b2c5
20 changed files with 752 additions and 180 deletions
@@ -0,0 +1,159 @@
+"""The advanced installer asks for each data path's storage next to its size."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+from proxmenux_oci.catalog import Catalog
+from proxmenux_oci.installer import ADVANCED_MODE, DEFAULT_MODE, build_deployment
+
+STORAGES = [{"storage": "Public", "type": "nfs", "avail": 9 * 2**40},
+            {"storage": "local-lvm", "type": "lvmthin", "avail": 700 * 2**30}]
+
+
+class RecordingUI:
+    """Answers every question with its default, or with a scripted answer."""
+
+    def __init__(self, answers=None):
+        self.answers = answers or {}
+        self.asked = []
+
+    def _answer(self, text, default):
+        self.asked.append(text)
+        return self.answers.get(text, default)
+
+    def ask(self, text, default=None, required=True):
+        return self._answer(text, default if default is not None else "")
+
+    def password(self, text, required=True):
+        return self._answer(text, "secret")
+
+    def choose(self, text, options, default=None):
+        return self._answer(text, default)
+
+    def confirm(self, text, default=False):
+        return self._answer(text, default)
+
+    def checklist(self, text, options, default=None):
+        return self._answer(text, default)
+
+    def info(self, text):
+        pass
+
+    def message(self, text):
+        pass
+
+
+def storages(content):
+    return STORAGES if content == "rootdir" else [{"storage": "local", "type": "dir", "avail": 2**35}]
+
+
+def addresses(ui, bridge, names, *args):
+    return {name: "dhcp" for name in names}, None
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch("proxmenux_oci.installer.host.storages", side_effect=storages)
+@patch("proxmenux_oci.installer.host.bridges", return_value=[{"iface": "vmbr0", "cidr": "192.0.2.10/24"}])
+@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
+@patch("proxmenux_oci.installer.access.ask_ipv4", return_value=("dhcp", None))
+@patch("proxmenux_oci.installer.access.ask_addresses", side_effect=addresses)
+class AdvancedFlowOrderTests(unittest.TestCase):
+    catalog = Catalog(ROOT)
+
+    def build(self, app, ui, mode=ADVANCED_MODE):
+        return build_deployment(self.catalog.compose(app), ui, mode)
+
+    def test_each_volume_asks_its_storage_and_then_its_size(self, *_):
+        ui = RecordingUI({"Storage for /data/movies": "Public"})
+        plan = self.build("jellyfin", ui)
+        volumes = {m["container_path"]: m["source"] for m in plan["mounts"]}
+        self.assertEqual(volumes, {"/config": "local-lvm", "/data/tvshows": "local-lvm",
+                                   "/data/movies": "Public"})
+        asked = ui.asked
+        self.assertEqual(asked.index("Size in GB of /config"), asked.index("Storage for /config") + 1)
+        self.assertNotIn("Storage for persistent data", asked)
+        self.assertNotIn("Where to store /config", asked)
+
+    def test_volumes_are_proposed_on_the_rootfs_storage(self, *_):
+        ui = RecordingUI({"Storage for rootfs": "Public"})
+        plan = self.build("jellyfin", ui)
+        self.assertEqual({m["source"] for m in plan["mounts"]}, {"Public"})
+
+    def test_system_resources_then_data_then_network_then_start(self, *_):
+        ui = RecordingUI()
+        self.build("jellyfin", ui)
+        asked = ui.asked
+        order = ["Memory in MB", "Storage for rootfs", "Rootfs size in GB", "Storage for /config",
+                 "Where to store /data/movies", "Network bridge", "Value for TZ",
+                 "Start with Proxmox", "Start when finished"]
+        self.assertEqual([asked.index(text) for text in order], sorted(asked.index(text) for text in order))
+        self.assertEqual(asked[-2:], ["Start with Proxmox", "Start when finished"])
+
+    def test_single_option_and_system_paths_are_not_asked(self, *_):
+        ui = RecordingUI()
+        plan = self.build("frigate", ui)
+        self.assertNotIn("Where to store /etc/localtime", ui.asked)
+        self.assertNotIn("Host path for /etc/localtime", ui.asked)
+        self.assertIn("Where to store /media/frigate", ui.asked)
+        localtime = next(m for m in plan["mounts"] if m["container_path"] == "/etc/localtime")
+        self.assertEqual((localtime["type"], localtime["source"]), ("host-bind", "/etc/localtime"))
+
+    def test_default_mode_asks_no_storage(self, *_):
+        ui = RecordingUI()
+        plan = self.build("jellyfin", ui, DEFAULT_MODE)
+        self.assertFalse([text for text in ui.asked if text.startswith("Storage for")])
+        self.assertEqual({m["source"] for m in plan["mounts"]}, {"local-lvm"})
+
+    def assert_follows(self, asked, *texts):
+        positions = [asked.index(text) for text in texts]
+        self.assertEqual(positions, list(range(positions[0], positions[0] + len(texts))), texts)
+
+    def test_stack_asks_each_member_path_with_its_storage_and_size(self, *_):
+        ui = RecordingUI({"Storage for linkwarden-postgres: /var/lib/postgresql": "Public"})
+        plan = self.build("linkwarden", ui)
+        asked = ui.asked
+        self.assertNotIn("Storage for persistent data", asked)
+        self.assert_follows(asked, "Where to store linkwarden-postgres: /var/lib/postgresql",
+                            "Storage for linkwarden-postgres: /var/lib/postgresql",
+                            "Size in GB of linkwarden-postgres: /var/lib/postgresql")
+        self.assertLess(asked.index("Add extra paths to this stack"), asked.index("Access bridge"))
+        self.assertLess(asked.index("Access bridge"), asked.index("Timezone"))
+        self.assertEqual(asked[-1], "Start the stack with Proxmox")
+        sources = {(s["name"], m["container_path"]): m["source"]
+                   for s in plan["services"] for m in s["deployment"]["mounts"]}
+        self.assertEqual(sources[("linkwarden-postgres", "/var/lib/postgresql")], "Public")
+        self.assertTrue(all(s["deployment"]["onboot"] is False for s in plan["services"]))
+
+    def test_arr_suite_asks_data_first_and_start_last(self, *_):
+        ui = RecordingUI()
+        plan = self.build("suite-arr", ui)
+        asked = ui.asked
+        self.assertLess(asked.index("Shared host media directory"), asked.index("Access bridge"))
+        self.assertLess(asked.index("Add extra paths to this stack"), asked.index("Access bridge"))
+        self.assertEqual(asked[-1], "Start each LXC with Proxmox (no coordinated startup)")
+        self.assertTrue(all("onboot" in s["deployment"] for s in plan["services"]))
+
+    def test_special_stacks_ask_each_storage_next_to_its_size(self, *_):
+        ui = RecordingUI()
+        self.build("immich", ui)
+        self.assert_follows(ui.asked, "Where to store the Immich library", "Storage for the Immich library",
+                            "Library size in GB", "Local storage for PostgreSQL", "PostgreSQL volume size in GB")
+        ui = RecordingUI()
+        self.build("paperless-ngx", ui)
+        self.assert_follows(ui.asked, "Storage for Paperless data and documents", "Data volume size in GB",
+                            "Documents volume size in GB", "Local storage for PostgreSQL",
+                            "PostgreSQL volume size in GB")
+        ui = RecordingUI()
+        self.build("tandoor", ui)
+        self.assert_follows(ui.asked, "Storage for staticfiles", "staticfiles volume size in GB",
+                            "Local storage for PostgreSQL", "PostgreSQL volume size in GB")
+        self.assertEqual(ui.asked[-2:], ["Start the stack with Proxmox", "Start when finished"])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,56 @@
+"""The container creation progress knows how big the image is once unpacked."""
+
+import gzip
+import hashlib
+import io
+import json
+from pathlib import Path
+import sys
+import tarfile
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "remote"))
+
+from verify_oci_archive import extracted_size
+
+
+def zstd_frame(content_size: int) -> bytes:
+    # Magic, a descriptor with an 8-byte content size and single segment, the size.
+    return b"\x28\xb5\x2f\xfd" + bytes([0b11100000]) + content_size.to_bytes(8, "little") + b"\x00" * 16
+
+
+class ExtractedSizeTests(unittest.TestCase):
+    def archive(self, layers):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        path = Path(tmp.name) / "image.tar"
+        blobs = {}
+
+        def add(data: bytes) -> str:
+            digest = "sha256:" + hashlib.sha256(data).hexdigest()
+            blobs[digest] = data
+            return digest
+
+        manifest = {"layers": [{"digest": add(data), "mediaType": "x"} for data in layers]}
+        index = {"manifests": [{"digest": add(json.dumps(manifest).encode())}]}
+        with tarfile.open(path, "w") as tar:
+            for name, data in [("oci-layout", b"{}"), ("index.json", json.dumps(index).encode()),
+                               *((f"blobs/{d.replace(':', '/')}", b) for d, b in blobs.items())]:
+                info = tarfile.TarInfo(name)
+                info.size = len(data)
+                tar.addfile(info, io.BytesIO(data))
+        return path
+
+    def test_gzip_zstd_and_plain_layers_add_up(self):
+        path = self.archive([gzip.compress(b"a" * 300_000), zstd_frame(5 << 20), b"p" * 1000])
+        self.assertEqual(extracted_size(path), 300_000 + (5 << 20) + 1000)
+
+    def test_zstd_layer_without_content_size_is_unknown(self):
+        frame = b"\x28\xb5\x2f\xfd" + bytes([0b00000000, 0x50]) + b"\x00" * 16
+        self.assertIsNone(extracted_size(self.archive([frame])))
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,94 @@
+"""USB devices are offered by name, as the Monitor shows them, with the node an LXC receives."""
+
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import Mock, patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+from proxmenux_oci import host
+from proxmenux_oci.extra_devices import ask_extra_devices, choose_usb_device
+
+LSUSB = """Bus 003 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub
+Bus 003 Device 002: ID 0463:ffff MGE UPS Systems UPS
+Bus 003 Device 003: ID 0658:0200 Sigma Designs, Inc. Aeotec Z-Stick Gen5 (ZW090) - UZB
+Bus 003 Device 004: ID 1cf1:0030 Dresden Elektronik ZigBee gateway [ConBee II]
+Bus 003 Device 005: ID 0781:5581 SanDisk Corp. Ultra
+"""
+
+
+class UsbDevicesTests(unittest.TestCase):
+    def setUp(self):
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        self.root = Path(tmp.name)
+        pci = self.root / "sys/devices/pci0000:00/usb3"
+        self.device(pci, "usb3", "1d6b", "09", 1)
+        self.device(pci, "3-3", "0463", "00", 2, product="Ellipse ECO", interface="03")
+        self.device(pci, "3-8", "0658", "02", 3, tty="ttyACM1")
+        self.device(pci, "3-9", "1cf1", "02", 4, product="ConBee II", tty="ttyACM0")
+        self.device(pci, "3-10", "0781", "00", 5, product="Ultra", interface="08")
+
+    def device(self, parent, name, vendor, device_class, number, product="", interface="", tty=""):
+        path = parent / name
+        path.mkdir(parents=True)
+        for key, value in {"idVendor": vendor, "idProduct": "0001", "bDeviceClass": device_class,
+                           "busnum": "3", "devnum": str(number)}.items():
+            (path / key).write_text(value + "\n")
+        if product:
+            (path / "product").write_text(product + "\n")
+        if interface:
+            (path / f"{name}:1.0").mkdir()
+            (path / f"{name}:1.0" / "bInterfaceClass").write_text(interface + "\n")
+        link = self.root / "sys/bus/usb/devices" / name
+        link.parent.mkdir(parents=True, exist_ok=True)
+        os.symlink(path, link)
+        if tty:
+            port = path / f"{name}:1.0" / "tty" / tty
+            port.mkdir(parents=True)
+            (self.root / "sys/class/tty").mkdir(parents=True, exist_ok=True)
+            (self.root / "sys/class/tty" / tty).mkdir()
+            os.symlink(path / f"{name}:1.0", self.root / "sys/class/tty" / tty / "device")
+
+    def test_devices_are_named_as_the_monitor_names_them(self):
+        rows = host.usb_devices(self.root, LSUSB)
+        self.assertEqual(rows, [
+            {"path": "/dev/bus/usb/003/002", "name": "Ellipse ECO", "kind": "UPS"},
+            {"path": "/dev/ttyACM1", "name": "Sigma Designs, Inc. Aeotec Z-Stick Gen5 (ZW090) - UZB",
+             "kind": "Communications"},
+            {"path": "/dev/ttyACM0", "name": "ConBee II", "kind": "Communications"},
+        ])
+
+    def test_the_menu_lists_the_devices_and_leaves_attached_ones_out(self):
+        ui = Mock()
+        ui.choose.return_value = "/dev/ttyACM1"
+        with patch.object(host, "usb_devices", return_value=host.usb_devices(self.root, LSUSB)):
+            self.assertEqual(choose_usb_device(ui, "USB", attached={"/dev/ttyACM0"}), "/dev/ttyACM1")
+        tags = [tag for tag, _label in ui.choose.call_args.args[1]]
+        self.assertEqual(tags, ["/dev/bus/usb/003/002", "/dev/ttyACM1", "manual"])
+        ui.ask.assert_not_called()
+
+    def test_without_devices_the_node_is_typed(self):
+        ui = Mock()
+        ui.ask.return_value = "/dev/ttyUSB0"
+        with patch.object(host, "usb_devices", return_value=[]):
+            self.assertEqual(choose_usb_device(ui, "USB"), "/dev/ttyUSB0")
+        ui.message.assert_called_once()
+        ui.choose.assert_not_called()
+
+    def test_extra_usb_device_comes_from_the_list(self):
+        ui = Mock()
+        ui.confirm.side_effect = [True, False]
+        ui.choose.side_effect = ["usb", "/dev/ttyACM0"]
+        with patch.object(host, "usb_devices", return_value=host.usb_devices(self.root, LSUSB)):
+            devices = ask_extra_devices(ui, [], True)
+        self.assertEqual([(d["host_path"], d["container_path"]) for d in devices],
+                         [("/dev/ttyACM0", "/dev/ttyACM0")])
+
+
+if __name__ == "__main__":
+    unittest.main()