fix(oci): carry verified stack metadata into native backups

oci/README.md

@@ -256,6 +256,11 @@ The native exit-status monitor currently accepts LXC 7.0.0, the verified protoco
 version. Other versions refuse one-shot execution rather than treating a stopped
 container as proof of success.
 
+Before each native stack backup, the engine writes the validated original
+installation record into every stopped member and reads it back. A missing or
+unverifiable copy aborts the backup, so database and worker archives do not
+silently lose the metadata needed to register the restored stack.
+
 The bridges are local to the installation node; they are not a cluster SDN.
 Unsupported network and runtime semantics are reported before installation.
 Support for several services does not imply complete Docker Compose compatibility.

oci/remote/oci_carried_record.py

@@ -31,7 +31,7 @@ import sys
 import uuid
 
 import oci_instances as instances
-from oci_installation_state import sha
+from oci_installation_state import sha, parse_config
 from oci_ui import translate
 
 DIRECTORY = '.proxmenux'
@@ -40,6 +40,7 @@ KIND = 'proxmenux.oci-carried-record'
 STAMP = 'carried.sha256'
 CONFIG_CACHE = 'carried-config.json'
 LOCAL_CONFIGS = Path('/etc/pve/lxc')
+ROOTFS = Path('/var/lib/lxc')
 LIMIT = 16 * 1024 * 1024
 STACK_CONTRACT = '/etc/pve/priv/proxmenux-stack-{}.json'
 NVIDIA_HOOK = re.compile(r'/usr/local/lib/proxmenux/oci/nvidia-mount-([a-f0-9]{64})\.sh')
@@ -400,6 +401,75 @@ def carry(root, vmid, mount_stopped=True, verify=False, adopted=False):
     return outcome
 
 
+def carry_backup(root, vmid, original_record, validated_config_text):
+    """Require a verified original record inside a stopped CT before vzdump.
+
+    A coordinated transaction has already marked the host registry pending.
+    The explicit original is the validated plan record, never that pending
+    record or an untrusted copy from inside the guest. Unlike best-effort
+    carry(), any missing copy, configuration drift or I/O failure aborts backup.
+    """
+    failure = translate('The carried record could not be verified before the backup')
+    if isinstance(validated_config_text, bytes):
+        validated_config_text = validated_config_text.decode('utf-8')
+    if (not isinstance(original_record, dict) or original_record.get('vmid') != vmid
+            or original_record.get('status') != 'installed'
+            or original_record.get('pending_transaction') or original_record.get('pending_stack_transaction')
+            or not isinstance(validated_config_text, str)):
+        raise ValueError(failure)
+    identity = original_record.get('installation_id')
+    if not identity or instances.identity(validated_config_text.encode()) != identity:
+        raise ValueError(failure)
+
+    def checked_stopped():
+        status = _run('pct', 'status', str(int(vmid)))
+        if status.returncode or status.stdout.strip() != 'status: stopped':
+            raise ValueError(failure)
+
+    def checked_configuration():
+        result = _run('pct', 'config', str(int(vmid)))
+        if result.returncode or instances.identity(result.stdout.encode()) != identity:
+            raise ValueError(failure)
+        def projection(text):
+            if re.search(r'^\[[^]\r\n]+\]\s*$', text, re.MULTILINE):
+                raise ValueError(failure)
+            values = parse_config(text.encode())
+            # The identity is checked independently; notes and maintenance
+            # locks do not change the image, mounts, network, user or devices.
+            values.pop('description', None)
+            lock = values.pop('lock', None)
+            if lock not in (None, 'backup', 'mounted'):
+                raise ValueError(failure)
+            return values
+        if projection(result.stdout) != projection(validated_config_text):
+            raise ValueError(failure)
+        return result.stdout
+
+    checked_stopped()
+    config = checked_configuration()
+    if _host_directory_missing(vmid) or _run('pct', 'mount', str(int(vmid))).returncode:
+        raise ValueError(failure)
+    try:
+        checked_stopped()
+        checked_configuration()
+        rootfs = ROOTFS / str(int(vmid)) / 'rootfs'
+        value = bundle(original_record, config)
+        value['saved_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
+        value['generation'] = uuid.uuid4().hex
+        write_copy(rootfs, value, mapped_root(config))
+        copied = read_copy(rootfs)
+        if copied != value or digest(copied) != digest(value):
+            raise ValueError(failure)
+        checked_stopped()
+        checked_configuration()
+    finally:
+        if _run('pct', 'unmount', str(int(vmid))).returncode:
+            raise ValueError(failure)
+    instances.write(instances.location(root, vmid).parent / STAMP,
+                    {'digest': digest(value), 'generation': value['generation']})
+    return {'status': 'carried', 'generation': value['generation'], 'digest': digest(value)}
+
+
 def registered(root):
     if not root.is_dir():
         return []

oci/remote/oci_stack_native.py

@@ -14,6 +14,7 @@ import tempfile
 import uuid
 
 import oci_image_cache as image_cache
+import oci_carried_record as carried
 import oci_compose_jobs as compose_jobs
 import oci_instances as instances
 import oci_instance_transaction as member_tx
@@ -417,6 +418,8 @@ class NativeAdapter:
                      f"{self.records[self.plan['primary_vmid']]['deployment'].get(work_backup.KEY)}")
         directory = base / ('backup-%s' % vmid)
         private_directory(directory)
+        config = member_tx.owned(vmid, identity).decode()
+        carried.carry_backup(self.root, vmid, self.records[vmid], config)
         archive = member_tx.verified_backup(vmid, directory, 'zstd',
                                             translate('The backup of a member could not be identified'))
         return {'archive': str(archive), 'sha256': member_tx.filehash(archive)}

oci/src/proxmenux_oci/installer.py

@@ -1598,7 +1598,7 @@ def _run_remote_install(
             shutil.copy2(project_root / 'remote' / 'oci_installation_state.py', temporary / 'oci_installation_state.py')
             shutil.copy2(project_root / 'remote' / 'oci_instances.py', temporary / 'oci_instances.py')
             shutil.copy2(project_root / 'remote' / 'oci_image_mountpoints.py', temporary / 'oci_image_mountpoints.py')
-            for helper in ('oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'):
+            for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'):
                 shutil.copy2(project_root / 'remote' / helper, temporary / helper)
             if deployment_kind == 'generic-multi-lxc-stack':
                 shutil.copy2(project_root / 'remote' / 'install_generic_stack.py', temporary / 'install_generic_stack.py')
@@ -1635,7 +1635,7 @@ def _run_remote_install(
             archive.add(project_root / 'remote' / 'haos_healthcheck.py', arcname='haos_healthcheck.py')
             archive.add(project_root / 'remote' / 'oci_installation_state.py', arcname='oci_installation_state.py')
             archive.add(project_root / 'remote' / 'oci_instances.py', arcname='oci_instances.py')
-            for helper in ('oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_image_mountpoints.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'):
+            for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_image_mountpoints.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'):
                 archive.add(project_root / 'remote' / helper, arcname=helper)
             if deployment_kind == 'generic-multi-lxc-stack':
                 archive.add(project_root / 'remote' / 'install_generic_stack.py', arcname='install_generic_stack.py')

oci/tests/test_backup_carried_record.py

@@ -0,0 +1,86 @@
+"""A backup must contain its validated original metadata, despite a pending registry."""
+import copy
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'remote'))
+import oci_carried_record as carried
+
+VMID = 4801
+IDENTITY = '488ed3cb-1145-477a-b90f-c46777d69fb2'
+CONFIG = ('description: <!-- proxmenux-instance='+IDENTITY+' -->\n'
+          'rootfs: local-zfs:subvol-4801-disk-0,size=8G\n'
+          'unprivileged: 1\nlxc.init.uid: 999\n'
+          'net0: name=eth0,bridge=vmbr10,ip=10.77.0.31/24\n')
+ORIGINAL = {'vmid': VMID, 'installation_id': IDENTITY, 'status': 'installed',
+            'deployment': {}, 'observed': {'config': CONFIG}}
+
+class BackupCarriedRecordTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory(); self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name)/'registry'; self.rootfs_base = Path(temporary.name)/'lxc'
+        self.rootfs = self.rootfs_base/str(VMID)/'rootfs'; self.rootfs.mkdir(parents=True)
+        self.config = CONFIG; self.status = 'stopped'; self.unmount_code = 0; self.commands = []
+        def run(*args):
+            self.commands.append(args)
+            value = self.config if args[1]=='config' else 'status: '+self.status if args[1]=='status' else ''
+            code = self.unmount_code if args[1]=='unmount' else 0
+            return SimpleNamespace(returncode=code, stdout=value)
+        for name, value in [('_run',run),('_host_directory_missing',lambda _:False),
+                            ('ROOTFS',self.rootfs_base),('mapped_root',lambda _: (os.getuid(),os.getgid()))]:
+            context=patch.object(carried,name,value);context.start();self.addCleanup(context.stop)
+
+    def carry(self, original=None):
+        return carried.carry_backup(self.root,VMID,original or ORIGINAL,CONFIG)
+
+    def test_pending_host_registry_does_not_replace_the_validated_original(self):
+        pending=copy.deepcopy(ORIGINAL);pending.update(status='updating',pending_stack_transaction='/transaction')
+        carried.instances.write(carried.instances.location(self.root,VMID),pending)
+        result=self.carry()
+        copy_in_backup=carried.read_copy(self.rootfs)
+        self.assertEqual(copy_in_backup['record'],ORIGINAL)
+        self.assertNotIn('pending_stack_transaction',copy_in_backup['record'])
+        self.assertEqual(result['digest'],carried.digest(copy_in_backup))
+        self.assertEqual(self.commands.count(('pct','mount',str(VMID))),1)
+        self.assertEqual(self.commands.count(('pct','unmount',str(VMID))),1)
+        self.assertEqual(json.loads(carried.instances.location(self.root,VMID).read_text()),pending)
+
+    def test_notes_and_backup_lock_changes_do_not_hide_identity(self):
+        self.config=CONFIG.replace('description: ','description: user notes ')+'lock: backup\n'
+        self.carry()
+        self.assertEqual(carried.read_copy(self.rootfs)['record'],ORIGINAL)
+
+    def test_runtime_network_mount_user_and_identity_drift_are_refused(self):
+        for before,after in [('10.77.0.31','10.77.0.99'),('disk-0','disk-1'),('uid: 999','uid: 1000'),(IDENTITY,'11111111-1111-4111-8111-111111111111')]:
+            with self.subTest(after=after):
+                self.config=CONFIG.replace(before,after);self.commands=[]
+                with self.assertRaises(ValueError):self.carry()
+                self.assertNotIn(('pct','mount',str(VMID)),self.commands)
+
+    def test_running_guest_and_pending_original_are_refused(self):
+        self.status='running'
+        with self.assertRaises(ValueError):self.carry()
+        self.status='stopped'
+        for field,value in [('status','updating'),('pending_transaction','/transaction'),('pending_stack_transaction','/stack')]:
+            record=dict(ORIGINAL,**{field:value})
+            with self.subTest(field=field),self.assertRaises(ValueError):self.carry(record)
+
+    def test_write_failure_is_not_silenced_and_always_unmounts(self):
+        with patch.object(carried,'write_copy',side_effect=OSError('disk full')):
+            with self.assertRaisesRegex(OSError,'disk full'):self.carry()
+        self.assertIn(('pct','unmount',str(VMID)),self.commands)
+
+    def test_roundtrip_mismatch_and_unmount_failure_abort_before_success_stamp(self):
+        with patch.object(carried,'read_copy',return_value=None):
+            with self.assertRaises(ValueError):self.carry()
+        self.assertFalse((carried.instances.location(self.root,VMID).parent/carried.STAMP).exists())
+        self.unmount_code=1
+        with self.assertRaises(ValueError):self.carry()
+
+if __name__=='__main__':unittest.main()

oci/tests/test_compose_job_integration.py

@@ -41,6 +41,7 @@ class JobInstallIntegrationTests(unittest.TestCase):
             def inspect(command,*args):
                 staged=Path(command[1]).parent/'oci_compose_jobs.py'
                 self.assertEqual(staged.read_bytes(),(ROOT/'remote/oci_compose_jobs.py').read_bytes())
+                self.assertEqual((staged.parent/'oci_carried_record.py').read_bytes(),(ROOT/'remote/oci_carried_record.py').read_bytes())
                 return None
             with patch.object(installer,'_stream_process',side_effect=inspect):
                 installer._run_remote_install(ROOT,template,{'deployment_kind':'generic-multi-lxc-stack'},'local',True)
@@ -52,6 +53,7 @@ class JobInstallIntegrationTests(unittest.TestCase):
                 self.assertEqual(command[0],'ssh')
                 with tarfile.open(fileobj=io.BytesIO(data),mode='r:gz') as archive:
                     self.assertEqual(archive.extractfile('oci_compose_jobs.py').read(),(ROOT/'remote/oci_compose_jobs.py').read_bytes())
+                    self.assertEqual(archive.extractfile('oci_carried_record.py').read(),(ROOT/'remote/oci_carried_record.py').read_bytes())
                 return None
             with patch.object(installer,'_stream_process',side_effect=inspect):
                 installer._run_remote_install(ROOT,template,{'deployment_kind':'generic-multi-lxc-stack'},'test-node',True)

oci/tests/test_stack_backup_carried_metadata.py

@@ -0,0 +1,44 @@
+"""A coordinated native backup must contain its verified, original restore recipe."""
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+ROOT=Path(__file__).resolve().parents[1]
+sys.path.insert(0,str(ROOT/'remote'))
+import oci_stack_native as native
+
+class StackBackupCarriedMetadataTests(unittest.TestCase):
+    def adapter(self, folder):
+        adapter=object.__new__(native.NativeAdapter)
+        adapter.root=Path('/unused/registry')
+        adapter.journal=Path(folder)/'transaction.json'
+        adapter.plan={'primary_vmid':4901}
+        adapter.records={4901:{'vmid':4901,'installation_id':'original','status':'installed'}}
+        return adapter
+
+    def test_restore_metadata_is_verified_before_archive_creation(self):
+        with tempfile.TemporaryDirectory() as folder:
+            adapter=self.adapter(folder);events=[]
+            archive=Path(folder)/'archive.tar.zst'
+            with patch.object(adapter,'validate'),patch.object(adapter,'work_base',return_value=Path(folder)), \
+                 patch.object(native.member_tx,'owned',return_value=b'validated original config') as owned, \
+                 patch.object(native.carried,'carry_backup',side_effect=lambda *args:events.append(('carry',args)),create=True), \
+                 patch.object(native.member_tx,'verified_backup',side_effect=lambda *args:events.append(('backup',args)) or archive), \
+                 patch.object(native.member_tx,'filehash',return_value='verified-sha'):
+                result=adapter.backup(4901,'original')
+            owned.assert_called_once_with(4901,'original')
+            self.assertEqual([event[0] for event in events],['carry','backup'])
+            self.assertEqual(events[0][1],(adapter.root,4901,adapter.records[4901],'validated original config'))
+            self.assertEqual(result,{'archive':str(archive),'sha256':'verified-sha'})
+
+    def test_missing_restore_metadata_prevents_archive_creation(self):
+        with tempfile.TemporaryDirectory() as folder:
+            adapter=self.adapter(folder)
+            with patch.object(adapter,'validate'),patch.object(adapter,'work_base',return_value=Path(folder)), \
+                 patch.object(native.member_tx,'owned',return_value=b'validated original config'), \
+                 patch.object(native.carried,'carry_backup',side_effect=ValueError('metadata not verified'),create=True), \
+                 patch.object(native.member_tx,'verified_backup') as backup:
+                with self.assertRaisesRegex(ValueError,'metadata not verified'):
+                    adapter.backup(4901,'original')
+            backup.assert_not_called()