feat(oci): VLAN tag and updates that keep Proxmox changes

oci/tests/test_external_tags.py

@@ -67,13 +67,11 @@ class ExternalTagsTests(unittest.TestCase):
         self.assertEqual(transaction.external_changes(record(), current), {'cores': 4, 'memory': 2048})
 
     def test_tag_changes_do_not_bypass_identity_or_network_checks(self):
-        for current in (config('homelab').replace(MARKER.encode(), b'22222222-2222-4222-8222-222222222222'),
-                        config('homelab').replace(b'bridge=vmbr0', b'bridge=vmbr2')):
-            with self.subTest(config=current):
-                with self.assertRaises(ValueError):
-                    reconcile.propose(record(), current)
-                with self.assertRaises(ValueError):
-                    transaction.external_changes(record(), current)
+        current = config('homelab').replace(MARKER.encode(), b'22222222-2222-4222-8222-222222222222')
+        with self.assertRaises(ValueError):
+            reconcile.propose(record(), current)
+        with self.assertRaises(ValueError):
+            transaction.external_changes(record(), current + b'arch: arm64\n')
 
     def test_new_volume_still_requires_adoption_when_tags_also_change(self):
         saved = record()
@@ -90,6 +88,51 @@ class ExternalTagsTests(unittest.TestCase):
             self.assertEqual(json.loads(path.read_text()), saved)
 
 
+    def test_a_vlan_tag_set_in_proxmox_is_taken_into_the_record(self):
+        for saved_net, current_net, expected in ((b'ip=dhcp\nnet1', b'ip=dhcp,tag=4\nnet1', 4),
+                                                 (b'ip=dhcp,tag=4\nnet1', b'ip=dhcp\nnet1', None)):
+            with self.subTest(expected=expected), tempfile.TemporaryDirectory() as folder:
+                saved = record()
+                before = config('oci;proxmenux').replace(b'ip=dhcp\nnet1', saved_net)
+                saved['observed'].update(config=before.decode(), config_sha256=sha(before))
+                root = Path(folder)
+                path = root / '200/oci-compose.json'
+                path.parent.mkdir()
+                path.write_text(json.dumps(saved))
+                current = before.replace(saved_net, current_net)
+                with patch.object(reconcile.instances, 'ROOT', root):
+                    proposal = reconcile.propose(saved, current)
+                self.assertEqual(proposal['candidate']['deployment']['network']['vlan_tag'], expected)
+                self.assertEqual(proposal['details'], [])
+
+    def test_a_host_directory_taken_away_in_proxmox_leaves_the_record(self):
+        saved = record()
+        before = config('oci;proxmenux') + b'mp0: /srv/share,mp=/share,backup=0\n'
+        saved['observed'].update(config=before.decode(), config_sha256=sha(before))
+        saved['deployment']['mounts'] = [{'type': 'host-bind', 'container_path': '/share', 'source': '/srv/share'}]
+        with tempfile.TemporaryDirectory() as folder:
+            root = Path(folder)
+            path = root / '200/oci-compose.json'
+            path.parent.mkdir()
+            path.write_text(json.dumps(saved))
+            with patch.object(reconcile.instances, 'ROOT', root):
+                proposal = reconcile.propose(saved, config('oci;proxmenux'))
+        self.assertEqual(proposal['candidate']['deployment']['mounts'], [])
+        self.assertFalse(proposal['review'])
+
+    def test_a_data_volume_taken_away_in_proxmox_still_stops_the_update(self):
+        saved = record()
+        before = config('oci;proxmenux') + b'mp0: local-lvm:vm-200-disk-1,mp=/data,size=8G,backup=1\n'
+        saved['observed'].update(config=before.decode(), config_sha256=sha(before))
+        with self.assertRaisesRegex(ValueError, 'mp0'):
+            reconcile.propose(saved, config('oci;proxmenux'))
+
+    def test_a_network_changed_in_proxmox_is_taken_into_the_record(self):
+        current = config('oci;proxmenux').replace(b'bridge=vmbr0', b'bridge=vmbr9')
+        self.assertEqual(transaction.external_changes(record(), current), {'bridge': 'vmbr9'})
+        self.assertEqual(transaction.external_changes(record(), current, adopt=False), {})
+
+
 class Prepared(Exception):
     """Stop a mocked transaction before it can stop or modify any container."""
 

oci/tests/test_guessed_endpoint_healthcheck.py

@@ -38,7 +38,7 @@ class GuessedEndpointCheckTests(unittest.TestCase):
             fake = Path(directory) / 'curl'
             fake.write_text(f'#!/bin/sh\nprintf %s {code}\nexit {exit_code}\n')
             fake.chmod(0o755)
-            script = (probe_function() + '\nCURL_ARGS=()\nHC_URL=http://192.0.2.1:3306/\n'
+            script = (probe_function() + '\noci_curl() { shift; curl "$@"; }\nCURL_ARGS=()\nHC_URL=http://192.0.2.1:3306/\n'
                       f'HC_ANY_STATUS={any_status}\nHC_ACCEPT_NON_HTTP={accept_non_http}\nhealthcheck_probe\n')
             env = {**os.environ, 'PATH': f'{directory}:{os.environ["PATH"]}'}
             return subprocess.run(['bash', '-c', script], env=env, capture_output=True).returncode

oci/tests/test_instance_reconcile.py

@@ -78,7 +78,7 @@ class ProposalTests(unittest.TestCase):
         self.assertIsNone(reconcile.propose(record, updated))
         self.assertEqual(reconcile.transaction.external_changes(record, updated), {})
 
-    def test_notes_cannot_hide_identity_or_network_changes(self):
+    def test_notes_cannot_hide_identity_or_platform_changes(self):
         marker = '11111111-1111-1111-1111-111111111111'
         config = f'description: Old notes proxmenux-instance={marker}\n'.encode()
         record = {
@@ -88,7 +88,7 @@ class ProposalTests(unittest.TestCase):
         }
         with self.assertRaises(ValueError):
             reconcile.propose(record, b'description: Different instance\n')
-        changed = f'description: New notes proxmenux-instance={marker}\nnet0: name=eth0,bridge=vmbr1\n'.encode()
+        changed = f'description: New notes proxmenux-instance={marker}\narch: arm64\n'.encode()
         with self.assertRaises(ValueError):
             reconcile.propose(record, changed)
         with self.assertRaises(ValueError):
@@ -197,10 +197,11 @@ class EnvironmentAdoptionTests(unittest.TestCase):
         proposal = self.propose(record, 'APP_PASSWORD=first-secret\0PATH=/bin')
         self.assertEqual(proposal['details'], [])
 
-    def test_removing_an_image_default_is_refused(self):
+    def test_an_image_default_removed_in_proxmox_comes_back_with_its_value(self):
         record = self.record('PATH=/bin\0APP_PASSWORD=first-secret')
-        with self.assertRaisesRegex(ValueError, 'env'):
-            self.propose(record, 'APP_PASSWORD=first-secret')
+        proposal = self.propose(record, 'APP_PASSWORD=first-secret')
+        self.assertFalse(proposal['review'])
+        self.assertNotIn('PATH', [item['name'] for item in proposal['candidate']['deployment']['environment']])
 
     def test_removing_a_saved_variable_removes_it_from_the_contract(self):
         record = self.record('PATH=/bin\0APP_PASSWORD=first-secret')
@@ -211,9 +212,9 @@ class EnvironmentAdoptionTests(unittest.TestCase):
     def test_other_changes_next_to_the_variables_are_still_refused(self):
         record = self.record('PATH=/bin')
         config = (f'description: proxmenux-instance={self.marker}\nenv: PATH=/bin\0A=1\n'
-                  'net0: name=eth0,bridge=vmbr1\n').encode()
+                  'arch: arm64\n').encode()
         with (patch.object(reconcile.instances, 'identity', return_value=self.marker),
-              self.assertRaisesRegex(ValueError, 'net0')):
+              self.assertRaisesRegex(ValueError, 'arch')):
             reconcile.propose(record, config)
 
 

oci/tests/test_native_exec_web_readiness.py

@@ -55,7 +55,7 @@ class NativeExecWebReadinessTests(unittest.TestCase):
                         adapter.healthcheck(4800)
                 else:
                     adapter.healthcheck(4800)
-            self.assertEqual(probes, ['pct', 'curl'])
+            self.assertEqual(probes, ['pct', 'pct', 'curl'])
         finally:
             server.shutdown(); server.server_close(); thread.join(timeout=2)
 

oci/tests/test_native_modify.py

@@ -61,6 +61,24 @@ class NativePlanTests(unittest.TestCase):
         self.assertEqual(fields['tag'], '20'); self.assertEqual(fields['ip6'], 'auto')
         self.assertEqual(fields['ip'], '192.0.2.2/24'); self.assertEqual(fields['gw'], '192.0.2.1')
 
+    def test_the_vlan_tag_is_set_changed_and_removed(self):
+        for saved, wanted, expected in ((None, 30, '30'), (20, 30, '30'), (20, None, None)):
+            with self.subTest(saved=saved, wanted=wanted):
+                self.record['deployment']['network'].pop('vlan_tag', None)
+                self.candidate['deployment']['network'].pop('vlan_tag', None)
+                if saved: self.record['deployment']['network']['vlan_tag'] = saved
+                if wanted: self.candidate['deployment']['network']['vlan_tag'] = wanted
+                self.candidate['deployment']['network']['ipv4'] = '192.0.2.2/24'
+                with patch.object(Path, 'is_dir', return_value=True):
+                    result = native.plan(self.record, self.candidate, self.config)
+                fields = dict(x.split('=',1) for x in result['values']['net0'].split(','))
+                self.assertEqual(fields.get('tag'), expected)
+
+    def test_an_invalid_vlan_tag_is_rejected_before_stopping(self):
+        self.candidate['deployment']['network']['vlan_tag'] = 5000
+        with patch.object(Path, 'is_dir', return_value=True), self.assertRaises(ValueError):
+            native.plan(self.record, self.candidate, self.config)
+
     def test_invalid_network_is_rejected_before_stopping(self):
         for value in ('nonsense', '2001:db8::1/64'):
             self.candidate['deployment']['network']['ipv4'] = value

oci/tests/test_recreation_i18n.py

@@ -131,7 +131,7 @@ class SlovakEditorTests(unittest.TestCase):
         self.assertIn('Vlastná cesta nemôže skryť systémové adresáre', self.text(ui))
 
     def test_network_gateway_explains_outbound_route(self):
-        ui = EditorUI(['static'], ['vmbr0', '192.0.2.42/24', ''], confirms=[])
+        ui = EditorUI(['static'], ['vmbr0', '192.0.2.42/24', '', ''], confirms=[])
         with patch.object(installer.host, 'bridges', return_value=[]), \
                 patch.object(installer.host, 'address_answers', return_value=False):
             recreation.edit_setting(copy.deepcopy(self.original), ui, 'network')

oci/tests/test_vlan_tag.py

@@ -0,0 +1,175 @@
+"""The VLAN tag of the access interface is a setting of the installation."""
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(ROOT / "remote"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import oci_installation_state as state
+import oci_instance_transaction as transaction
+from proxmenux_oci import network
+
+
+class Answer:
+    def __init__(self, value):
+        self.value = value
+
+    def ask(self, text, default=None, required=True):
+        return self.value if self.value is not None else default
+
+
+def record(net0):
+    config = f"arch: amd64\nmemory: 1024\nnet0: {net0}\n"
+    return {"observed": {"config": config, "config_sha256": transaction.sha(config.encode())},
+            "installation_id": "x"}, config.encode()
+
+
+class Question(unittest.TestCase):
+    def test_empty_means_no_tag_and_the_current_one_is_offered(self):
+        self.assertIsNone(network.ask_vlan_tag(Answer("")))
+        self.assertEqual(network.ask_vlan_tag(Answer(None), 40), 40)
+        self.assertEqual(network.ask_vlan_tag(Answer(" 4 ")), 4)
+
+    def test_only_a_valid_vlan_is_accepted(self):
+        for value in ("0", "4095", "abc", "1,2", "-1"):
+            with self.subTest(value=value), self.assertRaises(ValueError):
+                network.ask_vlan_tag(Answer(value))
+
+
+class TagSetInProxmox(unittest.TestCase):
+    NET = "name=eth0,bridge=vmbr0,firewall=1,hwaddr=BC:24:11:98:91:D2,ip=dhcp,type=veth"
+
+    def changes(self, before, after, adopt=True):
+        saved, _ = record(before)
+        return transaction.external_changes(saved, record(after)[1], adopt=adopt)
+
+    def test_an_application_takes_the_tag_into_its_record(self):
+        self.assertEqual(self.changes(self.NET, self.NET + ",tag=4"), {"vlan_tag": 4})
+
+    def test_a_tag_removed_in_proxmox_is_taken_as_well(self):
+        self.assertEqual(self.changes(self.NET + ",tag=4", self.NET), {"vlan_tag": None})
+
+    def test_a_stack_member_keeps_the_line_as_it_is(self):
+        self.assertEqual(self.changes(self.NET, self.NET + ",tag=4", adopt=False), {})
+
+    def test_the_rest_of_the_network_line_is_taken_as_well(self):
+        self.assertEqual(self.changes(self.NET, self.NET.replace("vmbr0", "vmbr1") + ",tag=4"),
+                         {"bridge": "vmbr1", "vlan_tag": 4})
+
+
+class Kept(unittest.TestCase):
+    def test_the_update_names_the_tag_it_keeps(self):
+        import oci_update_current as update
+        with patch.object(update, "translate", side_effect=str):
+            self.assertEqual(update.kept_settings({"vlan_tag": 4}, {"network": {"vlan_tag": 4}}), ["VLAN tag"])
+
+
+class Carried(unittest.TestCase):
+    CONFIG = b"memory: 512\nnameserver: 9.9.9.9\nnet0: name=eth0,mtu=1400,bridge=vmbr0\nprotection: 1\nfeatures: keyctl=1\n"
+
+    def kept(self, candidate_network=None):
+        saved = {"deployment": {"network": {"bridge": "vmbr0"}, "security": {}}}
+        new = {"deployment": {"network": candidate_network or {"bridge": "vmbr0"}, "security": {}}}
+        return transaction.kept_settings(saved, new, state.parse_config(self.CONFIG))
+
+    def test_what_the_user_set_goes_back_to_the_rebuilt_container(self):
+        kept = self.kept()
+        self.assertEqual(kept["nameserver"], "9.9.9.9")
+        self.assertEqual(kept["features"], "keyctl=1")
+        self.assertEqual(kept["net0"], "name=eth0,mtu=1400,bridge=vmbr0")
+
+    def test_protection_comes_back_only_at_the_end(self):
+        self.assertNotIn("protection", self.kept())
+
+    def test_only_a_command_of_the_user_goes_back(self):
+        config = state.parse_config(b"entrypoint: /usr/bin/app --flag\n")
+        same = {"deployment": {}}
+        self.assertNotIn("entrypoint", transaction.kept_settings(same, same, config))
+        custom = {"deployment": {"custom_entrypoint": "/usr/bin/app --flag"}}
+        self.assertEqual(transaction.kept_settings(custom, custom, config)["entrypoint"], "/usr/bin/app --flag")
+
+    def test_a_network_edited_in_modify_is_not_overwritten(self):
+        self.assertNotIn("net0", self.kept({"bridge": "vmbr1"}))
+
+    def test_a_start_command_set_in_proxmox_is_taken(self):
+        saved, config = record("name=eth0,bridge=vmbr0")
+        self.assertEqual(transaction.external_changes(saved, config + b"entrypoint: /bin/app\n"),
+                         {"entrypoint": "/bin/app"})
+
+    def test_the_snapshot_a_backup_leaves_is_not_a_change(self):
+        saved, config = record("name=eth0,bridge=vmbr0")
+        self.assertEqual(transaction.external_changes(saved, config + b"parent: vzdump\n"), {})
+
+    def test_a_snapshot_of_the_user_is_not_discarded(self):
+        saved, config = record("name=eth0,bridge=vmbr0")
+        with self.assertRaises(ValueError):
+            transaction.external_changes(saved, config + b"parent: before-upgrade\n")
+
+    def test_raw_lines_of_the_user_are_told_from_those_of_the_engine(self):
+        before = "lxc.init.cwd: /\nlxc.cgroup2.devices.allow: c 1:1 rwm\n"
+        saved = {"observed": {"config": before}, "deployment": {"custom_lxc": ["lxc.cgroup2.devices.allow: c 1:1 rwm"]}}
+        config = (before + "lxc.mount.entry: /a b none bind 0 0\nlxc.environment.runtime: A=1\n").encode()
+        self.assertEqual(transaction.custom_lxc(saved, config),
+                         ["lxc.cgroup2.devices.allow: c 1:1 rwm", "lxc.mount.entry: /a b none bind 0 0"])
+
+
+class Probe(unittest.TestCase):
+    def curl(self, net):
+        def run(*args):
+            return b"1234\n" if args[0] == "lxc-info" else f"memory: 512\n{net}\n".encode()
+        return state.container_curl(108, run)
+
+    def test_the_host_looks_at_an_untagged_container(self):
+        self.assertEqual(self.curl("net0: name=eth0,bridge=vmbr0,ip=dhcp"), ["curl"])
+
+    def test_a_tagged_interface_is_looked_at_from_inside(self):
+        expected = ["nsenter", "--net=/proc/1234/ns/net", "curl"]
+        self.assertEqual(self.curl("net0: name=eth0,bridge=vmbr0,tag=4,ip=dhcp"), expected)
+        self.assertEqual(self.curl("net1: name=lan,bridge=vmbr0,tag=4"), expected)
+
+    def test_every_web_check_goes_through_it(self):
+        for name in ("install_oci.sh", "install_paperless_stack.sh", "install_tandoor_stack.sh",
+                     "install_nextcloud_stack.sh"):
+            self.assertIn("oci_curl", (ROOT / "remote" / name).read_text(), name)
+        for name in ("oci_instance_transaction.py", "oci_stack_native.py", "install_generic_stack.py",
+                     "haos_healthcheck.py"):
+            self.assertIn("container_curl", (ROOT / "remote" / name).read_text(), name)
+
+
+class Creation(unittest.TestCase):
+    def option(self, network_values):
+        with tempfile.TemporaryDirectory() as directory:
+            deployment = Path(directory) / "deployment.json"
+            deployment.write_text(json.dumps({"network": network_values}))
+            text = (ROOT / "remote/oci_ui.sh").read_text()
+            start = text.index("oci_vlan_option() {")
+            function = text[start:text.index("\n}", start) + 2]
+            return subprocess.run(
+                ["/bin/bash", "-c", f'DEPLOYMENT_FILE={deployment}\n{function}\n'
+                 'oci_vlan_option vlan_tag; code=$?; echo "$code:$OCI_VLAN_OPTION"'],
+                capture_output=True, text=True, check=True).stdout.strip()
+
+    def test_the_option_of_the_interface(self):
+        self.assertEqual(self.option({"vlan_tag": 4}), "0:,tag=4")
+        self.assertEqual(self.option({}), "0:")
+
+    def test_an_invalid_tag_stops_the_installation(self):
+        self.assertEqual(self.option({"vlan_tag": 5000}), "1:")
+        self.assertEqual(self.option({"vlan_tag": "4,trunks=1"}), "1:")
+
+    def test_every_installer_applies_it(self):
+        for name in ("install_oci.sh", "install_immich_stack.sh", "install_paperless_stack.sh",
+                     "install_nextcloud_stack.sh", "install_tandoor_stack.sh"):
+            self.assertIn("oci_vlan_option", (ROOT / "remote" / name).read_text(), name)
+        self.assertIn("frontend_vlan_tag", (ROOT / "remote/install_generic_stack.py").read_text())
+
+
+if __name__ == "__main__":
+    unittest.main()