#!/usr/bin/env python3 """Private OCI instance contracts. Does not recreate or update containers.""" from __future__ import annotations import argparse import contextlib import copy from contextlib import contextmanager import datetime import fcntl import json import os from pathlib import Path import re import subprocess import sys import tempfile import uuid from oci_installation_state import command, image_from_archive, private_directory, sha from oci_host_mounts import capture_sources from oci_accelerators import capture as capture_gpu_devices from oci_ui import translate, msg_error, msg_ok ROOT = Path('/usr/local/share/proxmenux/oci/apps') MARKER = 'proxmenux-instance=' ACTIVE = {'installing', 'assembling', 'updating', 'recovering'} def now(): return datetime.datetime.now(datetime.timezone.utc).isoformat() def location(root, vmid): if not isinstance(vmid, int) or vmid < 100: raise ValueError(translate('Invalid VMID')) path = root / str(vmid) if path.is_symlink(): raise ValueError(translate('Unsafe instance directory')) return path / 'oci-compose.json' @contextmanager def locked(root): private_directory(root) path = root / '.lock' if path.is_symlink(): raise ValueError(translate('Unsafe registry lock')) inherited = os.environ.get('PROXMENUX_INSTANCE_LOCK_FD') if inherited: fd = int(inherited) if os.fstat(fd).st_ino != path.stat().st_ino or os.fstat(fd).st_dev != path.stat().st_dev: raise ValueError(translate('Wrong inherited registry lock')) fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) yield else: with path.open('a') as lock: os.chmod(path, 0o600) fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) yield def write(path, value): private_directory(path.parent) if path.is_symlink(): raise ValueError(translate('Unsafe instance record')) fd, tmp = tempfile.mkstemp(dir=path.parent, prefix='.compose-') try: with os.fdopen(fd, 'w') as out: json.dump(value, out, indent=2) out.write('\n') out.flush() os.fsync(out.fileno()) os.replace(tmp, path) fd = os.open(path.parent, os.O_RDONLY) try: os.fsync(fd) finally: os.close(fd) finally: if os.path.exists(tmp): os.unlink(tmp) def read(root, vmid): path = location(root, vmid) if path.is_symlink(): raise ValueError(translate('Unsafe instance record')) value = json.loads(path.read_text()) if value.get('schema_version') != 1 or value.get('vmid') != vmid: raise ValueError(translate('Incompatible instance record')) uuid.UUID(value['installation_id']) return value def has_contract(root, vmid): path = location(root, vmid) if path.is_symlink(): raise ValueError(translate('Unsafe instance record')) if path.parent.exists() and not path.parent.is_dir(): raise ValueError(translate('Incompatible instance directory')) return path.exists() def guest_exists(vmid): nodes = Path('/etc/pve/nodes') return any(nodes.glob(f'*/lxc/{vmid}.conf')) or any(nodes.glob(f'*/qemu-server/{vmid}.conf')) def release_orphan(root, vmid): """A record whose guest no longer exists on any node does not own the VMID: a failed install, or a container deleted from the Proxmox UI. An update or recovery left halfway keeps it, because its backup may still be needed. The record stays in the same directory as history.""" path = location(root, vmid) if not path.exists(): return True previous = read(root, vmid) if (previous.get('status') in ('updating', 'recovering') or previous.get('pending_transaction') or previous.get('pending_stack_transaction') or guest_exists(vmid)): return False path.replace(path.with_name(f"retired-{previous['installation_id']}.json")) return True def prepare(root, vmid, template, deployment): path = location(root, vmid) if not release_orphan(root, vmid): raise ValueError(f"VMID {vmid} {translate('belongs to another OCI installation')}") deployment = dict(deployment, vmid=vmid) value = {'schema_version': 1, 'vmid': vmid, 'installation_id': str(uuid.uuid4()), 'created_at': now(), 'status': 'installing', 'template': template, 'deployment': deployment, 'observed': None, 'automatic_update_enabled': False} write(path, value) return value def observe(vmid, installation_id, archive, digest, image=None): """Collect evidence before changing the current desired-state contract. An installation observed again passes its saved image metadata, since the downloaded archive may have been removed.""" config = command('pct', 'config', str(vmid)) if identity(config) != installation_id: raise ValueError(translate('The container identity does not match')) resources = json.loads(command('pvesh', 'get', '/cluster/resources', '--type', 'vm', '--output-format', 'json')) node = next(r['node'] for r in resources if r.get('type') == 'lxc' and int(r['vmid']) == vmid) api_config = command('pvesh', 'get', f'/nodes/{node}/lxc/{vmid}/config', '--output-format', 'json') observed = { 'config': config.decode(), 'config_sha256': sha(config), 'api_config_sha256': api_hash(api_config), 'image': image if image is not None else image_from_archive(archive), 'archive_path': archive, 'resolved_registry_digest': digest} sources = capture_sources(config) if sources: observed['host_bind_sources'] = sources gpu = capture_gpu_devices(config) if gpu: observed['gpu_devices'] = gpu return observed def finish(root, vmid, archive, digest): value = read(root, vmid) observed = observe(vmid, value['installation_id'], archive, digest) value.update(status='assembling' if value['deployment'].get('stack_managed') else 'installed', completed_at=now(), observed=observed) write(location(root, vmid), value) def publish_stack(root, primary_id, template, deployment, members): """Preserve each recipe AND a complete, nonrecursive copy in the principal.""" records = {} for member in members: vmid = int(member['vmid']) record = read(root, vmid) if identity(command('pct', 'config', str(vmid))) != record['installation_id']: raise ValueError(translate('A stack member has a different identity')) records[vmid] = record if primary_id is not None and primary_id not in records: raise ValueError(translate('The main member of the stack is missing')) stack_id = records[primary_id]['installation_id'] if primary_id is not None else None for member in members: vmid = int(member['vmid']) record = records[vmid] if 'deployment' in member: record['deployment'] = copy.deepcopy(member['deployment']) record['deployment']['vmid'] = vmid record['deployment']['stack_managed'] = primary_id is not None write(location(root, vmid), record) finish(root, vmid, record['observed']['archive_path'], record['observed']['resolved_registry_digest']) record = read(root, vmid) record['status'] = 'installed' if stack_id: record['stack_member'] = {'stack_id': stack_id, 'primary_vmid': primary_id, 'name': member['name']} records[vmid] = record if primary_id is not None: recipes = [copy.deepcopy(records[int(m['vmid'])]) for m in members] for recipe in recipes: recipe.pop('stack', None) records[primary_id]['stack'] = { 'id': stack_id, 'template': copy.deepcopy(template), 'deployment': copy.deepcopy(deployment), 'members': recipes, 'reconstruction_requires_volume_verification': True, } records[primary_id]['stack']['deployment']['services'] = copy.deepcopy(members) # Persist recovery recipes first, before publishing member completion. write(location(root, primary_id), records[primary_id]) for vmid, record in records.items(): if vmid != primary_id: write(location(root, vmid), record) def identity(config): # Description is URL-escaped by pct; UUID characters remain unchanged. text = config.decode() try: description = json.loads(text).get('description', '') except ValueError: description = next((line[len('description: '):] for line in text.splitlines() if line.startswith('description: ')), '') match = re.search(r'proxmenux-instance=([0-9a-f-]{36})(?![0-9a-f-])', description) return match.group(1) if match else None def save_assembly(root, primary_id, template, deployment, members): path = location(root, primary_id).parent / 'stack-assembly.json' if path.exists() or path.is_symlink(): raise ValueError(translate('A pending stack assembly already exists; it is not overwritten')) ids = [int(m['vmid']) for m in members] if primary_id not in ids or len(set(ids)) != len(ids): raise ValueError(translate('Invalid stack members')) expected = {str(vmid): read(root, vmid)['installation_id'] for vmid in ids} write(path, {'schema_version': 1, 'primary_vmid': primary_id, 'created_at': now(), 'expected_installation_ids': expected, 'template': template, 'deployment': deployment, 'services': members}) def resume_assembly(root, primary_id): path = location(root, primary_id).parent / 'stack-assembly.json' if path.is_symlink(): raise ValueError(translate('Unsafe stack assembly')) saved = json.loads(path.read_text()) ids = [int(m['vmid']) for m in saved['services']] if (saved.get('schema_version') != 1 or saved['primary_vmid'] != primary_id or primary_id not in ids or len(set(ids)) != len(ids) or set(saved['expected_installation_ids']) != {str(vmid) for vmid in ids}): raise ValueError(translate('Incompatible stack assembly')) for vmid in ids: expected = saved['expected_installation_ids'][str(vmid)] if read(root, vmid)['installation_id'] != expected: raise ValueError(translate('The record of a member was replaced; the assembly is not resumed')) if identity(command('pct', 'config', str(vmid))) != expected: raise ValueError(translate('The container of a member was replaced; the assembly is not resumed')) publish_stack(root, primary_id, saved['template'], saved['deployment'], saved['services']) path.unlink() def api_hash(config): return sha(json.dumps(json.loads(config), sort_keys=True, separators=(',', ':')).encode()) def reconcile(root, resources, configs): """Caller holds lock and fetched a complete cluster inventory and configs.""" if not isinstance(resources, list): raise ValueError(translate('Invalid Proxmox inventory')) for resource in resources: if (not isinstance(resource, dict) or resource.get('type') not in ('lxc', 'qemu') or not isinstance(resource.get('vmid'), int)): raise ValueError(translate('Incomplete or incompatible Proxmox inventory')) present = {int(r['vmid']): r for r in resources if r.get('type') in ('lxc', 'qemu')} decisions = [] pending = set() for directory in root.iterdir(): if directory.name.isdecimal(): journal = location(root, int(directory.name)).parent / 'stack-assembly.json' if journal.is_symlink(): raise ValueError(translate('Unsafe stack assembly')) if journal.exists(): saved = json.loads(journal.read_text()) if saved.get('schema_version') != 1: raise ValueError(translate('Incompatible stack assembly')) pending.update(int(vmid) for vmid in saved['expected_installation_ids']) # Plan everything before removing anything: malformed records fail closed. for directory in sorted(root.iterdir()): if not directory.name.isdecimal(): continue vmid = int(directory.name) if not has_contract(root, vmid): # Orphan cleanup intentionally retains transaction history/backups. continue value = read(root, vmid) row = {'vmid': vmid, 'status': value['status'], 'action': 'keep'} if value['status'] in ACTIVE or vmid in pending or value.get('pending_stack_transaction'): row['reason'] = 'operation-in-progress' elif vmid not in present or present[vmid]['type'] != 'lxc': row.update(action='delete', reason='container-absent-or-replaced') else: config = configs[vmid] marker = identity(config) if marker and marker != value['installation_id']: row.update(action='delete', reason='different-installation') elif not marker: row['reason'] = 'identity-unconfirmed' else: row['reason'] = 'matched' baseline = (value.get('observed') or {}).get('api_config_sha256') row['configuration_changed'] = api_hash(config) != baseline if baseline else None decisions.append(row) if value.get('stack'): row['missing_members'] = [m['vmid'] for m in value['stack']['members'] if m['vmid'] not in present] row['replaced_members'] = [m['vmid'] for m in value['stack']['members'] if m['vmid'] in present and (present[m['vmid']]['type'] != 'lxc' or (m['vmid'] in configs and identity(configs[m['vmid']]) not in (None, m['installation_id'])))] for row in decisions: if row['action'] == 'delete': path = location(root, row['vmid']) path.unlink() # Never recursively delete history, backups, or unexpected files. try: path.parent.rmdir() except OSError: pass return decisions def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--root', type=Path, default=ROOT) sub = parser.add_subparsers(dest='action', required=True) for action in ('prepare', 'complete', 'failed'): p = sub.add_parser(action) p.add_argument('vmid', type=int) if action == 'prepare': p.add_argument('--template', required=True) p.add_argument('--deployment', required=True) if action == 'complete': p.add_argument('--archive', required=True) p.add_argument('--digest', required=True) sub.add_parser('reconcile') resume = sub.add_parser('resume-stack-recording') resume.add_argument('vmid', type=int) args = parser.parse_args() if os.geteuid() != 0: parser.error(translate('Root privileges are required')) try: with locked(args.root): if args.action == 'resume-stack-recording': resume_assembly(args.root, args.vmid) msg_ok(translate('Stack records saved; no container was reinstalled.')) elif args.action == 'prepare': value = prepare(args.root, args.vmid, json.loads(Path(args.template).read_text()), json.loads(Path(args.deployment).read_text())) print(value['installation_id']) elif args.action == 'complete': finish(args.root, args.vmid, args.archive, args.digest) elif args.action == 'failed': value = read(args.root, args.vmid) value.update(status='failed', failed_at=now()) write(location(args.root, args.vmid), value) else: resources = json.loads(command('pvesh', 'get', '/cluster/resources', '--type', 'vm', '--output-format', 'json')) configs = {} wanted = set() for directory in args.root.iterdir(): if directory.name.isdecimal(): if not has_contract(args.root, int(directory.name)): continue record = read(args.root, int(directory.name)) wanted.add(record['vmid']) wanted.update(m['vmid'] for m in record.get('stack', {}).get('members', [])) for r in resources: if r.get('type') == 'lxc' and int(r['vmid']) in wanted: configs[int(r['vmid'])] = command('pvesh', 'get', f"/nodes/{r['node']}/lxc/{r['vmid']}/config", '--output-format', 'json') # pvesh JSON contains the description and all repeated LXC entries. print(json.dumps(reconcile(args.root, resources, configs), indent=2)) return 0 except (OSError, ValueError, KeyError, StopIteration, RuntimeError, subprocess.TimeoutExpired) as exc: # stdout carries data read by the installers; the error goes to stderr. with contextlib.redirect_stdout(sys.stderr): msg_error(f"{translate('The instance record operation did not complete; no container was modified.')} ({exc})") return 1 if __name__ == '__main__': raise SystemExit(main())