diff --git a/.github/scripts/tests/test_oci_recovery_inventory_wording.py b/.github/scripts/tests/test_oci_recovery_inventory_wording.py index 21e48f6a..91a82435 100644 --- a/.github/scripts/tests/test_oci_recovery_inventory_wording.py +++ b/.github/scripts/tests/test_oci_recovery_inventory_wording.py @@ -32,6 +32,7 @@ def extracted(path, name, scope): if name == 'remove': # Operational cache cleanup is exercised by the engine tests; keep wording tests inert. scope.setdefault('compose_jobs', SimpleNamespace(forget_completed=lambda *args: None)) + scope.setdefault('public_routes', SimpleNamespace(remove=lambda *args: None)) node = next(n for n in ast.parse(path.read_text()).body if isinstance(n, ast.FunctionDef) and n.name == name) exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), scope) diff --git a/.github/scripts/tests/test_oci_removal_firewall_wording.py b/.github/scripts/tests/test_oci_removal_firewall_wording.py index 779eeae3..b18861e2 100644 --- a/.github/scripts/tests/test_oci_removal_firewall_wording.py +++ b/.github/scripts/tests/test_oci_removal_firewall_wording.py @@ -38,6 +38,7 @@ def extract(path, name, scope): if name == 'remove': # Operational cache cleanup is exercised by the engine tests; keep wording tests inert. scope.setdefault('compose_jobs', SimpleNamespace(forget_completed=lambda *args: None)) + scope.setdefault('public_routes', SimpleNamespace(remove=lambda *args: None)) node = next(n for n in ast.parse(path.read_text()).body if isinstance(n, ast.FunctionDef) and n.name == name) exec(compile(ast.Module(body=[node], type_ignores=[]), str(path), 'exec'), scope) return scope[name] diff --git a/oci/README.md b/oci/README.md index cb49bb14..031576bc 100644 --- a/oci/README.md +++ b/oci/README.md @@ -242,6 +242,12 @@ variables, such as `NEXTAUTH_URL` or `ROOT_URL`, must contain the intended URL i the supplied Compose or environment file. Changing a frontend IP later also requires updating its application URLs. +Private members receive specific routes to their own stack's static frontend +addresses before the application starts. These routes stay inside the container's +network namespace and use the frontend's private address as the next hop. Native +update and recovery preserve their ownership records and rebuild the required +host configuration, including when a restore changes container IDs. + Repeated identical volume sources share one host directory, selected once; separate configuration volumes stay independent. Host directories are not part of a normal Proxmox container backup. Newly created private data volumes use the diff --git a/oci/remote/install_generic_stack.py b/oci/remote/install_generic_stack.py index 17538a10..c6696442 100644 --- a/oci/remote/install_generic_stack.py +++ b/oci/remote/install_generic_stack.py @@ -26,6 +26,7 @@ import xml.etree.ElementTree as ET from allocate_private_network import allocate_network, existing_bridges, existing_networks import oci_instances import oci_nested_mounts +import oci_stack_public_routes from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error, msg_info2, stop_spinner, log HERE = Path(__file__).resolve().parent @@ -306,6 +307,45 @@ def prepare_public_environment(services): _set_public_value(service, key, binding['value'].replace('@STACK_LAN_IP@', str(address))) +def install_public_routes(services, primary_id, bridge): + """Private members can reach only this stack's own static frontend IPs.""" + frontends = [service for service in services if service.get('main') or service.get('frontend')] + targets = {} + for service in frontends: + address = service.get('frontend_ipv4') + if not address or address == 'dhcp': + continue + destination = str(ipaddress.IPv4Interface(address).ip) + if destination in targets: + raise RuntimeError('Duplicate stack frontend address') + targets[destination] = service + if not targets: + return + records = {service['vmid']: oci_instances.read(oci_instances.ROOT, service['vmid']) for service in services} + stack_id = records[primary_id]['installation_id'] + for service in services: + if service.get('main') or service.get('frontend'): + continue + vmid = service['vmid'] + identity = records[vmid]['installation_id'] + spec = {'schema': 1, 'stack_id': stack_id, 'primary_vmid': primary_id, + 'vmid': vmid, 'installation_id': identity, 'private_bridge': bridge, + 'private_address': service['ip'] + '/24', 'routes': [ + {'destination': destination, 'via': target['ip'], 'target_vmid': target['vmid'], + 'target_installation_id': records[target['vmid']]['installation_id']} + for destination, target in sorted(targets.items())]} + plan = service['deployment'] + plan['stack_public_routes'] = spec + oci_stack_public_routes.validate(plan, vmid, identity) + config = Path(f'/etc/pve/lxc/{vmid}.conf') + text = config.read_text() + # Never replace another include or accept a stale route contract. + oci_stack_public_routes.check(text, {}, vmid, require_host_assets=False) + line = oci_stack_public_routes.install(plan, vmid, identity) + config.write_text(text.rstrip('\n') + '\n' + line + '\n') + oci_stack_public_routes.check(config.read_text(), plan, vmid, identity) + + def bind_completed_jobs(services): """All members and private aliases exist before a job can be authorized.""" from oci_compose_jobs import job_spec_from_record @@ -1022,6 +1062,7 @@ def main(): result.update(completion_notes=list(deployment.get('completion_notes',[])), log=LOG) print('PROXMENUX_RESULT='+base64.b64encode(json.dumps(result).encode()).decode(),flush=True) return + install_public_routes(services, primary_id, bridge) bind_completed_jobs(services) hook_spec = Path(tmp)/'lifecycle.json' write_json(hook_spec,{'schema':1,'stack':deployment['stack_name'],'dependencies':[ diff --git a/oci/remote/install_oci.sh b/oci/remote/install_oci.sh index f6bc3460..916415a9 100755 --- a/oci/remote/install_oci.sh +++ b/oci/remote/install_oci.sh @@ -2107,6 +2107,16 @@ apply_rlimits apply_host_monitor apply_kept_settings +# Recreated stack members retain their owned pre-init routes. Append rather +# than replacing lxc.include: other managed runtime includes must survive. +if jq -e '.stack_public_routes != null' "$DEPLOYMENT_FILE" >/dev/null; then + ROUTE_INCLUDE=$(python3 "${SCRIPT_DIR}/oci_stack_public_routes.py" install "$VMID" \ + --deployment "$DEPLOYMENT_FILE" --installation-id "$INSTANCE_ID") \ + || die "Managed public route installation failed" + [[ -n "$ROUTE_INCLUDE" ]] || die "Managed public route include is missing" + grep -Fxq "$ROUTE_INCLUDE" "$CONF" || printf '%s\n' "$ROUTE_INCLUDE" >>"$CONF" +fi + while IFS= read -r REPAIR_ENCODED; do [[ -n $REPAIR_ENCODED ]] || continue run_pre_start_repair "$(printf '%s' "$REPAIR_ENCODED" | base64 -d)" diff --git a/oci/remote/oci_instance_transaction.py b/oci/remote/oci_instance_transaction.py index f1f6a096..35b88352 100644 --- a/oci/remote/oci_instance_transaction.py +++ b/oci/remote/oci_instance_transaction.py @@ -33,6 +33,7 @@ import oci_host_mounts as host_mounts import oci_nested_mounts as nested_mounts import oci_accelerators as gpu_devices import oci_runtime_settings as runtime_settings +import oci_stack_public_routes as public_routes import oci_image_cache as image_cache import oci_work_backup as work_backup import oci_ui @@ -52,7 +53,7 @@ BASIC = {'arch', 'cmode', 'console', 'tty', 'cores', 'cpulimit', 'cpuunits', 'de # Their output is data (and may hold saved secrets); it is never logged. DATA_COMMANDS = {('pct', 'config'), ('pvesh', 'get'), ('pct', 'df')} # Kept across a modification: the watchdog and where the backups are made. -HOST_PREFERENCES = ('watchdog', 'work_storage') +HOST_PREFERENCES = ('watchdog', 'work_storage', 'stack_public_routes') LOG_DIR = Path(os.environ.get('OCI_LOG_DIR', '/var/log/proxmenux/oci')) ANSI = re.compile(r'\x1b\[[0-9;?]*[A-Za-z]') SPINNER_FRAME = re.compile('^ ?[' + ''.join(oci_ui.FRAMES) + ']') @@ -476,7 +477,7 @@ def candidate_contract(record, operation, proposal=None): # of what is being edited: it stays as the record has it now. for key in HOST_PREFERENCES: if key in record['deployment']: - result['deployment'][key] = record['deployment'][key] + result['deployment'][key] = copy.deepcopy(record['deployment'][key]) else: result['deployment'].pop(key, None) return with_default_healthcheck(result) @@ -519,6 +520,11 @@ def external_changes(record, config, adopt=True): return values +def without_public_routes(config, vmid): + """The managed include is validated separately from sysctl includes.""" + return b'\n'.join(line for line in config.splitlines() if not public_routes.own(line, vmid)) + b'\n' + + def preflight(record, candidate, config, coordinated=None, in_place=False, baseline=True): deployment = record['deployment'] desired = candidate['deployment'] @@ -555,7 +561,10 @@ def preflight(record, candidate, config, coordinated=None, in_place=False, basel files = runtime_settings.file_binds(config) if files: runtime_keys.add('lxc.mount.entry') - runtime_settings.check(config, deployment, record['vmid']) + public_routes.check(config, deployment, record['vmid'], record['installation_id']) + if deployment.get('stack_public_routes'): + runtime_keys.add('lxc.include') + runtime_settings.check(without_public_routes(config, record['vmid']), deployment, record['vmid']) coordinated_keys = {'hookscript', *KEPT_AS_IS} if coordinated else set(KEPT_AS_IS) preserved_keys = {'nameserver', 'searchdomain', 'timezone', 'protection'} if in_place else set() detached = unused_volumes(config) @@ -680,7 +689,8 @@ def freeze_host_sources(record, candidate, acknowledge_external_data, files=()): def check_runtime_mounts(config, deployment): - runtime_settings.check(config, deployment, deployment['vmid']) + public_routes.check(config, deployment, deployment['vmid']) + runtime_settings.check(without_public_routes(config, deployment['vmid']), deployment, deployment['vmid']) nested_mounts.check(config) actual = mounts(config) files = runtime_settings.file_binds(config) @@ -1303,6 +1313,7 @@ def complete_recovery(root, journal, state): raise ValueError(translate('The configuration changed after the backup was restored; the recovery is not confirmed')) cleanup_restored_format_dirs(vmid, state['record']['deployment'], state['record']['installation_id']) runtime_settings.restore(state['record']['deployment'], vmid) + public_routes.restore(state['record']['deployment'], vmid, state['record']['installation_id']) check_runtime_mounts(config, state['record']['deployment']) gpu_devices.check(config, state['record']['deployment']) if show: diff --git a/oci/remote/oci_remove.py b/oci/remote/oci_remove.py index 59645be3..57420777 100644 --- a/oci/remote/oci_remove.py +++ b/oci/remote/oci_remove.py @@ -20,6 +20,7 @@ import oci_image_cache as image_cache import oci_work_backup as work_backup import oci_instances as instances import oci_compose_jobs as compose_jobs +import oci_stack_public_routes as public_routes from oci_installation_state import parse_config import oci_console import oci_runtime_settings as runtime_settings @@ -198,7 +199,7 @@ def _leftovers(vmid): """Whether anything of the container is still on the host.""" paths = [runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid), runtime_settings.seccomp_path(vmid), SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh', MONITOR_APPS / f'{int(vmid)}.json', - CLUSTER_RECORDS / f'{int(vmid)}.json', + CLUSTER_RECORDS / f'{int(vmid)}.json', public_routes.include_path(vmid), public_routes.contract_path(vmid), *oci_console.LOG_DIR.glob(f'{int(vmid)}.console.log*')] return any(path.exists() for path in paths) @@ -214,7 +215,8 @@ def sweep_orphans(root): (runtime_settings.seccomp_path(0).parent, r'([0-9]+)\.proxmenux-seccomp'), (oci_console.LOG_DIR, r'([0-9]+)\.console\.log.*'), (SNIPPETS, r'proxmenux-rclone-([0-9]+)-fuse-hook\.sh'), - (CLUSTER_RECORDS, r'([0-9]+)\.json')): + (CLUSTER_RECORDS, r'([0-9]+)\.json'), + (public_routes.DIRECTORY, r'([0-9]+)\.(?:conf|json)')): if directory.is_dir(): found.update(int(m.group(1)) for m in (re.fullmatch(pattern, p.name) for p in directory.iterdir()) if m) # Only the App tab registrations of OCI installs; the other ones belong to @@ -231,10 +233,17 @@ def sweep_orphans(root): if instances.guest_exists(vmid): continue record = instances.has_contract(root, vmid) + route_assets = public_routes.contract_path(vmid).exists() or public_routes.include_path(vmid).exists() + route_identity = instances.read(root, vmid)['installation_id'] if record and route_assets else None if record and not instances.release_orphan(root, vmid): continue if not (record or _leftovers(vmid)): continue + try: + public_routes.remove(vmid, route_identity) + except (ValueError, OSError): + # Changed or unowned assets remain for inspection, never guessed. + continue oci_console.remove_log(vmid) remove_host_state(vmid) cleaned.append(vmid) @@ -340,6 +349,7 @@ def remove(root, vmid): msg_warn(f"{translate('The container no longer exists:')} CT {member}") incomplete = True oci_console.remove_log(member) + public_routes.remove(member, record['installation_id']) remove_host_state(member) elif instances.identity(config) != record['installation_id']: msg_warn(f"{translate('The VMID belongs to another container now and is not touched:')} CT {member}") @@ -349,6 +359,7 @@ def remove(root, vmid): subprocess.run(['pct', 'stop', str(member), '--skiplock', '1'], check=False, capture_output=True) run('pct', 'destroy', str(member), '--purge', '1', '--destroy-unreferenced-disks', '1') oci_console.remove_log(member) + public_routes.remove(member, record['installation_id']) remove_host_state(member) msg_ok(f"{translate('Container removed:')} CT {member}") if bridge and not bridge_in_use(bridge, set(members)): diff --git a/oci/remote/oci_restore_recovery.py b/oci/remote/oci_restore_recovery.py index 32fc35a1..7ce305a7 100644 --- a/oci/remote/oci_restore_recovery.py +++ b/oci/remote/oci_restore_recovery.py @@ -31,6 +31,7 @@ import tempfile import oci_carried_record as carried import oci_console +import oci_stack_public_routes as public_routes import oci_instances as instances import oci_compose_jobs as compose_jobs import oci_stack_plan @@ -59,7 +60,7 @@ HOST_MONITOR_CONTENT = 'lxc.namespace.share.pid = 1\nlxc.namespace.share.net = 1 # them from, so they are not offered again. UNRECOVERABLE = '.unrecoverable.json' # The fields of a record that name a container of the application. -VMID_KEYS = ('vmid', 'primary_vmid', 'base_vmid') +VMID_KEYS = ('vmid', 'primary_vmid', 'base_vmid', 'target_vmid') def run(*args): @@ -287,7 +288,7 @@ def renumbered_lines(text, old, new): f'lxc.include: {runtime_settings.legacy_include_path(old)}': f'lxc.include: {runtime_settings.include_path(new)}'} hook = re.compile(rf'(hookscript: \S+:snippets/proxmenux-rclone-){int(old)}(-fuse-hook\.sh)') lines = [hook.sub(rf'\g<1>{int(new)}\g<2>', swaps.get(line, line)) if line.startswith('hookscript: ') - else swaps.get(line, line) for line in text.split('\n')] + else swaps.get(line, public_routes.renumber_line(line, old, new)) for line in text.split('\n')] return '\n'.join(lines) @@ -441,6 +442,11 @@ def check_host_resources(plan): vmid, label = entry['vmid'], f"CT {entry['vmid']}" values = parse_config(entry['config'].encode()) deployment = entry['copy']['record']['deployment'] + projected = renumbered_lines(entry['config'], entry.get('recorded', vmid), vmid) + try: + public_routes.check(projected, deployment, vmid, entry.get('installation_id'), require_host_assets=False) + except ValueError as error: + plan['blockers'].append(f"{label}: {error}") directories = [value.split(',', 1)[0] for key, value in values.items() if re.fullmatch(r'mp[0-9]+', key) and value.startswith('/')] devices = [oci_stack_modify.device_path(value) for key, value in values.items() @@ -482,6 +488,9 @@ def check_host_resources(plan): if key != 'lxc.include': continue path = Path(value) + if any(public_routes.own(f'lxc.include: {path}', number) for number in {vmid, entry.get('recorded', vmid)}): + # Canonical assets are emitted from the validated deployment before start. + continue if path in HOST_MONITOR: if path.exists() and path.read_text() != HOST_MONITOR_CONTENT: plan['blockers'].append(f"{label}: {translate('a different host monitor include already exists:')} {path}") @@ -708,12 +717,27 @@ def check_records(root, plan): plan['blockers'].append(f"CT {entry['vmid']}: {translate('this host keeps a record of another installation with an operation pending')}") +def check_public_route_members(root, plan): + """Guest-carried routes can target only members validated in this recovery.""" + restored = {entry['vmid']: entry['copy']['record'] for entry in plan['restored'] if entry.get('copy')} + try: + records = {member['vmid']: restored.get(member['vmid']) or instances.read(root, member['vmid']) + for member in plan['members']} + if not any(record.get('deployment', {}).get('stack_public_routes') for record in records.values()): + return + oci_stack_plan.validate_public_route_members(plan['primary'], records) + except (ValueError, KeyError, OSError) as error: + plan['blockers'].append(translate('Invalid stack contract') + ': ' + str(error)) + + def prepare(root): """Every restored application of this node, with what prevents its recovery and what the user has to know.""" plans = applications(root, [examine(entry) for entry in pending(root)]) for plan in plans: check_records(root, plan) + if not plan['blockers']: + check_public_route_members(root, plan) if not plan['blockers']: check_networks(plan) check_host_resources(plan) @@ -740,6 +764,7 @@ def restore_host_files(entry): vmid = entry['vmid'] deployment = entry['copy']['record']['deployment'] renumber_config(entry) + public_routes.restore(deployment, vmid, entry.get('installation_id')) for line in entry['config'].splitlines(): key, _, value = line.partition(': ') if key == 'lxc.include': diff --git a/oci/remote/oci_stack_native.py b/oci/remote/oci_stack_native.py index c8ef8f72..adbf6dda 100644 --- a/oci/remote/oci_stack_native.py +++ b/oci/remote/oci_stack_native.py @@ -15,6 +15,7 @@ import uuid import oci_image_cache as image_cache import oci_carried_record as carried +import oci_stack_public_routes as public_routes import oci_compose_jobs as compose_jobs import oci_instances as instances import oci_instance_transaction as member_tx @@ -227,6 +228,7 @@ class NativeAdapter: if row['type'] != 'lxc' or row.get('node') != socket.gethostname().split('.')[0]: raise ValueError(translate('A member VMID is in use by another guest or is on another node')) config = instances.command('pct', 'config', str(vmid)) + public_routes.check(config, record['deployment'], vmid, record['installation_id']) if instances.identity(config) != record['installation_id']: raise ValueError(translate('The identity of a member was replaced')) if (not self.journal.exists() or not self.state().get('replacement_intent')) and not instances.same_config_except_notes(record, config, ignore_tags=True): diff --git a/oci/remote/oci_stack_plan.py b/oci/remote/oci_stack_plan.py index 80f7eb67..c71099a8 100644 --- a/oci/remote/oci_stack_plan.py +++ b/oci/remote/oci_stack_plan.py @@ -1,5 +1,6 @@ """Read-only validation of a coordinated stack operation, before host changes.""" import copy +import oci_stack_public_routes as public_routes import re import uuid @@ -32,6 +33,27 @@ def validate_job_check(check, record=None): return job +def validate_public_route_members(primary_id, records): + """A route may name only the frontends of this exact recorded stack.""" + for vmid, record in records.items(): + spec = public_routes.validate(record.get('deployment', {}), vmid, record['installation_id']) + if spec is None: + continue + primary = records.get(primary_id) + if (primary is None or spec['primary_vmid'] != primary_id + or spec['stack_id'] != primary['installation_id']): + raise ValueError(translate('Invalid stack contract')) + members = {member['vmid']: member['installation_id'] + for member in primary.get('stack', {}).get('members', [])} + if members.get(vmid) != record['installation_id']: + raise ValueError(translate('Invalid stack contract')) + for route in spec['routes']: + target = records.get(route['target_vmid']) + if (target is None or target['installation_id'] != route['target_installation_id'] + or members.get(route['target_vmid']) != route['target_installation_id']): + raise ValueError(translate('Invalid stack contract')) + + def build(primary, records, inventory, operation): """Inventory maps VMIDs to installation UUIDs, including unrelated guests. @@ -43,6 +65,7 @@ def build(primary, records, inventory, operation): stack = primary.get('stack') if not stack or not stack.get('members'): raise ValueError(translate('This is not a coordinated stack')) + validate_public_route_members(primary['vmid'], records) snapshots = stack['members'] ids = [member['vmid'] for member in snapshots] if any(type(vmid) is not int or vmid < 100 for vmid in ids) or len(set(ids)) != len(ids): diff --git a/oci/remote/oci_stack_public_routes.py b/oci/remote/oci_stack_public_routes.py new file mode 100644 index 00000000..0c0f52e8 --- /dev/null +++ b/oci/remote/oci_stack_public_routes.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Owned /32 routes to a stack's own frontends, installed before OCI init.""" +import argparse +import copy +import fcntl +import ipaddress +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import tempfile +import uuid + +DIRECTORY = Path('/etc/pve/priv/proxmenux-stack-public-routes') +HELPER = Path('/usr/local/share/proxmenux/oci/engine/remote/oci_stack_public_routes.py') +CONFIGS = Path('/etc/pve/lxc') +ROOT_UID = 0 +PRIVATE = ipaddress.ip_network('10.77.0.0/16') +IP = '/usr/sbin/ip' +NSENTER = '/usr/bin/nsenter' +NS_GET_NSTYPE = 0xb703 +CLONE_NEWNET = 0x40000000 + + +class RouteError(ValueError): + pass + + +def _vmid(value): + if type(value) is not int or not 100 <= value < 1000000000: + raise RouteError('Invalid public route VMID') + return value + + +def _identity(value): + try: + result = str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError) as error: + raise RouteError('Invalid public route identity') from error + if result != value: + raise RouteError('Invalid public route identity') + return result + + +def include_path(vmid): + return DIRECTORY / f'{_vmid(vmid)}.conf' + + +def contract_path(vmid): + return DIRECTORY / f'{_vmid(vmid)}.json' + + +def _address(value): + if not isinstance(value, str): + raise RouteError('Invalid public route address') + try: + address = ipaddress.IPv4Address(value) + except (ValueError, TypeError) as error: + raise RouteError('Invalid public route address') from error + if address.is_loopback or address.is_multicast or address.is_unspecified or address.is_link_local: + raise RouteError('Invalid public route address') + return address + + +def validate(deployment, vmid, identity=None): + spec = deployment.get('stack_public_routes') + if spec is None: + return None + required = {'schema', 'stack_id', 'primary_vmid', 'vmid', 'installation_id', + 'private_bridge', 'private_address', 'routes'} + if not isinstance(spec, dict) or set(spec) != required or type(spec['schema']) is not int or spec['schema'] != 1: + raise RouteError('Invalid public route contract') + if _vmid(spec['vmid']) != _vmid(vmid): + raise RouteError('Public route owner differs') + _vmid(spec['primary_vmid']) + _identity(spec['stack_id']) + if _identity(spec['installation_id']) != (identity or spec['installation_id']): + raise RouteError('Public route owner identity differs') + if not isinstance(spec['private_bridge'], str) or not re.fullmatch(r'vmbr[0-9]+', spec['private_bridge']): + raise RouteError('Invalid public route bridge') + if not isinstance(spec['private_address'], str): + raise RouteError('Invalid public route private address') + try: + source = ipaddress.IPv4Interface(spec['private_address']) + except (ValueError, TypeError) as error: + raise RouteError('Invalid public route private address') from error + if source.network.prefixlen != 24 or source.ip not in PRIVATE or source.ip in (source.network.network_address, source.network.broadcast_address): + raise RouteError('Invalid public route private address') + routes = spec['routes'] + if not isinstance(routes, list) or not routes or len(routes) > 64: + raise RouteError('Invalid public route list') + destinations = set() + for route in routes: + if not isinstance(route, dict) or set(route) != {'destination', 'via', 'target_vmid', 'target_installation_id'}: + raise RouteError('Invalid public route entry') + destination, via = _address(route['destination']), _address(route['via']) + if (destination in source.network or destination in destinations or via not in source.network + or via in (source.ip, source.network.network_address, source.network.broadcast_address) + or _vmid(route['target_vmid']) == vmid): + raise RouteError('Invalid public route destination or next hop') + _identity(route['target_installation_id']) + destinations.add(destination) + return copy.deepcopy(spec) + + +def include_content(): + return f'lxc.hook.start-host = /usr/bin/python3 -I {HELPER} start\n' + + +def own(line, vmid): + if isinstance(line, bytes): + line = line.decode() + return line.rstrip('\n') == f'lxc.include: {include_path(vmid)}' + + +def renumber_line(line, old, new): + binary = isinstance(line, bytes) + text = line.decode() if binary else line + if own(text, old): + text = f'lxc.include: {include_path(new)}' + ('\n' if text.endswith('\n') else '') + return text.encode() if binary else text + + +def _read(path): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != ROOT_UID or info.st_mode & 0o022: + raise RouteError('Unsafe managed public route file') + if info.st_size > 262144: + raise RouteError('Managed public route file is too large') + return path.read_bytes() + + +def _managed_directory(path, create=False): + if create: + path.mkdir(parents=True, exist_ok=True) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != ROOT_UID or info.st_mode & 0o022: + raise RouteError('Unsafe managed public route directory') + + +def _write(path, data, mode=0o600): + _managed_directory(path.parent, create=True) + if path.is_symlink(): + raise RouteError('Managed public route path is a symbolic link') + fd, temporary = tempfile.mkstemp(prefix='.public-routes-', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as output: + output.write(data) + output.flush() + os.fsync(output.fileno()) + # pmxcfs supplies the permissions of files below its private directory. + if Path('/etc/pve') not in path.parents: + os.chmod(temporary, mode) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def install(deployment, vmid, identity=None): + spec = validate(deployment, vmid, identity) + if spec is None: + return + # The installed engine is the authority; upgrades replace it normally. + _managed_directory(HELPER.parent) + _read(HELPER) + _write(contract_path(vmid), (json.dumps(spec, sort_keys=True) + '\n').encode()) + _write(include_path(vmid), include_content().encode()) + return f'lxc.include: {include_path(vmid)}' + + +restore = install +publish = install + + +def check(config, deployment, vmid, identity=None, require_host_assets=True): + spec = validate(deployment, vmid, identity) + text = config.decode() if isinstance(config, bytes) else config + includes = [line for line in text.splitlines() if line.startswith('lxc.include: ') + and str(DIRECTORY) + '/' in line] + expected = [f'lxc.include: {include_path(vmid)}'] if spec else [] + if includes != expected: + raise RouteError('Managed public route include differs from the contract') + if spec and require_host_assets: + _managed_directory(DIRECTORY) + _managed_directory(HELPER.parent) + if _read(include_path(vmid)) != include_content().encode(): + raise RouteError('Managed public route include was changed') + if json.loads(_read(contract_path(vmid))) != spec: + raise RouteError('Managed public route contract was changed') + _read(HELPER) + return spec + + +def remove(vmid, identity=None): + path = contract_path(vmid) + if not path.exists() and not path.is_symlink(): + if include_path(vmid).exists() or include_path(vmid).is_symlink(): + raise RouteError('Public route include has no ownership contract') + return False + _managed_directory(DIRECTORY) + spec = json.loads(_read(path)) + validate({'stack_public_routes': spec}, vmid, identity) + include = include_path(vmid) + if include.exists() or include.is_symlink(): + if _read(include) != include_content().encode(): + raise RouteError('Managed public route include was changed') + include.unlink() + path.unlink() + return True + + +def _config(vmid): + text = _read(CONFIGS / f'{_vmid(vmid)}.conf').decode() + matches = re.findall(r'proxmenux-instance=([0-9a-f-]{36})(?![0-9a-f-])', text) + if len(set(matches)) != 1: + raise RouteError('Public route container identity is unknown') + values = {} + for line in text.splitlines(): + if line.startswith('#') or ': ' not in line: + continue + key, value = line.split(': ', 1) + if key in ('net0', 'net1'): + if key in values: + raise RouteError('Ambiguous public route network') + parts = [part.split('=', 1) for part in value.split(',') if '=' in part] + if len({key for key, _ in parts}) != len(parts): + raise RouteError('Ambiguous public route network') + values[key] = dict(parts) + return matches[0], values, text + + +def live_contract(spec): + """Validate actual PVE ownership and both ends of every generated route.""" + vmid = spec['vmid'] + identity, networks, text = _config(vmid) + check(text, {'stack_public_routes': spec}, vmid, identity) + source = networks.get('net0', {}) + if source.get('bridge') != spec['private_bridge'] or source.get('ip') != spec['private_address'] or source.get('name') != 'eth0': + raise RouteError('Public route private interface differs') + primary_identity, primary_networks, _ = _config(spec['primary_vmid']) + if primary_identity != spec['stack_id'] or primary_networks.get('net0', {}).get('bridge') != spec['private_bridge']: + raise RouteError('Public route stack ownership differs') + for route in spec['routes']: + target_identity, target_networks, _ = _config(route['target_vmid']) + private, frontend = target_networks.get('net0', {}), target_networks.get('net1', {}) + try: + private_ip = ipaddress.IPv4Interface(private.get('ip')) + public_ip = ipaddress.IPv4Interface(frontend.get('ip')) + except (ValueError, TypeError) as error: + raise RouteError('Public route target is not a static frontend') from error + if (target_identity != route['target_installation_id'] or private.get('bridge') != spec['private_bridge'] + or private_ip.network != ipaddress.IPv4Interface(spec['private_address']).network + or str(private_ip.ip) != route['via'] or str(public_ip.ip) != route['destination']): + raise RouteError('Public route target ownership or addresses differ') + mac = source.get('hwaddr', '').lower() + if not re.fullmatch(r'(?:[0-9a-f]{2}:){5}[0-9a-f]{2}', mac): + raise RouteError('Public route private hardware address is unknown') + return mac + + +def namespace_fd(path): + if not isinstance(path, str) or not re.fullmatch(r'/proc/[1-9][0-9]*/(?:fd/[0-9]+|ns/net)', path): + raise RouteError('LXC did not provide a network namespace') + fd = os.open(path, os.O_RDONLY) + try: + if fcntl.ioctl(fd, NS_GET_NSTYPE) != CLONE_NEWNET: + raise RouteError('LXC namespace is not a network namespace') + info, host = os.fstat(fd), os.stat('/proc/self/ns/net') + if (info.st_dev, info.st_ino) == (host.st_dev, host.st_ino): + raise RouteError('Refusing to configure the host network namespace') + return fd + except BaseException: + os.close(fd) + raise + + +def _namespace_command(fd, *args): + process = subprocess.run([NSENTER, f'--net=/proc/self/fd/{fd}', '--', IP, *args], + pass_fds=(fd,), capture_output=True, check=True, timeout=15) + return process.stdout + + +def apply(spec, fd, mac): + source = ipaddress.IPv4Interface(spec['private_address']) + interfaces = json.loads(_namespace_command(fd, '-j', 'address', 'show', 'dev', 'eth0')) + if (len(interfaces) != 1 or interfaces[0].get('address', '').lower() != mac + or not any(item.get('family') == 'inet' and item.get('local') == str(source.ip) + and item.get('prefixlen') == source.network.prefixlen + for item in interfaces[0].get('addr_info', []))): + raise RouteError('Network namespace does not belong to the public route owner') + for route in spec['routes']: + _namespace_command(fd, '-4', 'route', 'replace', route['destination'] + '/32', + 'via', route['via'], 'dev', 'eth0', 'src', str(source.ip)) + + +def start(): + if os.geteuid() != 0: + raise RouteError('Public routes require root') + name = os.environ.get('LXC_NAME', '') + if not re.fullmatch(r'[1-9][0-9]*', name): + raise RouteError('LXC did not provide its container identity') + vmid = _vmid(int(name)) + spec = json.loads(_read(contract_path(vmid))) + validate({'stack_public_routes': spec}, vmid) + mac = live_contract(spec) + pid = os.environ.get('LXC_PID', '') + if not re.fullmatch(r'[1-9][0-9]*', pid): + raise RouteError('LXC did not provide its init process identity') + fd = namespace_fd(f'/proc/{pid}/ns/net') + try: + apply(spec, fd, mac) + finally: + os.close(fd) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='operation', required=True) + start_parser = commands.add_parser('start') + start_parser.add_argument('hook_arguments', nargs='*') + install_parser = commands.add_parser('install') + install_parser.add_argument('vmid', type=int) + install_parser.add_argument('--deployment', required=True) + install_parser.add_argument('--installation-id', required=True) + args = parser.parse_args() + if os.geteuid() != 0: + raise RouteError('Public routes require root') + if args.operation == 'start': + # Proxmox's network scripts use the default LXC version-0 hook ABI. + expected = [os.environ.get('LXC_NAME', ''), 'lxc', 'start-host'] + if args.hook_arguments != expected: + raise RouteError('Unexpected public route hook invocation') + start() + else: + line = install(json.loads(Path(args.deployment).read_text()), args.vmid, args.installation_id) + if line: + print(line) + + +if __name__ == '__main__': + try: + main() + except (ValueError, OSError, subprocess.SubprocessError) as error: + detail = str(error) + if isinstance(error, subprocess.CalledProcessError) and error.stderr: + output = error.stderr.decode(errors='replace') if isinstance(error.stderr, bytes) else error.stderr + detail += ': ' + output.strip()[:1000] + raise SystemExit('Managed public routes failed: ' + detail) diff --git a/oci/src/proxmenux_oci/installer.py b/oci/src/proxmenux_oci/installer.py index 95bfecbb..e6b930a0 100644 --- a/oci/src/proxmenux_oci/installer.py +++ b/oci/src/proxmenux_oci/installer.py @@ -1598,7 +1598,7 @@ def _run_remote_install( shutil.copy2(project_root / 'remote' / 'oci_installation_state.py', temporary / 'oci_installation_state.py') shutil.copy2(project_root / 'remote' / 'oci_instances.py', temporary / 'oci_instances.py') shutil.copy2(project_root / 'remote' / 'oci_image_mountpoints.py', temporary / 'oci_image_mountpoints.py') - for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'): + for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_stack_public_routes.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'): shutil.copy2(project_root / 'remote' / helper, temporary / helper) if deployment_kind == 'generic-multi-lxc-stack': shutil.copy2(project_root / 'remote' / 'install_generic_stack.py', temporary / 'install_generic_stack.py') @@ -1635,7 +1635,7 @@ def _run_remote_install( archive.add(project_root / 'remote' / 'haos_healthcheck.py', arcname='haos_healthcheck.py') archive.add(project_root / 'remote' / 'oci_installation_state.py', arcname='oci_installation_state.py') archive.add(project_root / 'remote' / 'oci_instances.py', arcname='oci_instances.py') - for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_image_mountpoints.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'): + for helper in ('oci_carried_record.py', 'oci_compose_jobs.py', 'oci_stack_public_routes.py', 'oci_ui.sh', 'oci_ui.py', 'oci_description.py', 'oci_console.py', 'oci_native_stack.py', 'oci_native_stack.sh', 'oci_instance_transaction.py', 'oci_managed_volumes.py', 'oci_image_mountpoints.py', 'oci_host_mounts.py', 'oci_nested_mounts.py', 'oci_runtime_settings.py', 'oci_gpu_devices.py', 'oci_accelerators.py', 'oci_nvidia_runtime.py', 'oci_nvidia_refresh.py', 'oci_nvidia_dynamic.py', 'oci_update_current.py', 'oci_resource_modify.py', 'oci_native_modify.py', 'oci_stack_replay.py', 'oci_stack_plan.py', 'oci_stack_transaction.py', 'oci_stack_native.py', 'oci_image_cache.py', 'nvidia_lxc_mount_lab.sh', 'oci_nvidia_setup.sh', 'oci_immich_ml.sh', 'oci_rocm_check.py'): archive.add(project_root / 'remote' / helper, arcname=helper) if deployment_kind == 'generic-multi-lxc-stack': archive.add(project_root / 'remote' / 'install_generic_stack.py', arcname='install_generic_stack.py') diff --git a/oci/tests/test_compose_job_integration.py b/oci/tests/test_compose_job_integration.py index 866888cb..1dea12fe 100644 --- a/oci/tests/test_compose_job_integration.py +++ b/oci/tests/test_compose_job_integration.py @@ -42,6 +42,7 @@ class JobInstallIntegrationTests(unittest.TestCase): staged=Path(command[1]).parent/'oci_compose_jobs.py' self.assertEqual(staged.read_bytes(),(ROOT/'remote/oci_compose_jobs.py').read_bytes()) self.assertEqual((staged.parent/'oci_carried_record.py').read_bytes(),(ROOT/'remote/oci_carried_record.py').read_bytes()) + self.assertEqual((staged.parent/'oci_stack_public_routes.py').read_bytes(),(ROOT/'remote/oci_stack_public_routes.py').read_bytes()) return None with patch.object(installer,'_stream_process',side_effect=inspect): installer._run_remote_install(ROOT,template,{'deployment_kind':'generic-multi-lxc-stack'},'local',True) @@ -54,6 +55,7 @@ class JobInstallIntegrationTests(unittest.TestCase): with tarfile.open(fileobj=io.BytesIO(data),mode='r:gz') as archive: self.assertEqual(archive.extractfile('oci_compose_jobs.py').read(),(ROOT/'remote/oci_compose_jobs.py').read_bytes()) self.assertEqual(archive.extractfile('oci_carried_record.py').read(),(ROOT/'remote/oci_carried_record.py').read_bytes()) + self.assertEqual(archive.extractfile('oci_stack_public_routes.py').read(),(ROOT/'remote/oci_stack_public_routes.py').read_bytes()) return None with patch.object(installer,'_stream_process',side_effect=inspect): installer._run_remote_install(ROOT,template,{'deployment_kind':'generic-multi-lxc-stack'},'test-node',True) diff --git a/oci/tests/test_stack_public_route_install.py b/oci/tests/test_stack_public_route_install.py new file mode 100644 index 00000000..d789b807 --- /dev/null +++ b/oci/tests/test_stack_public_route_install.py @@ -0,0 +1,68 @@ +"""Persist route contracts before startup without losing other LXC includes.""" +import copy +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'remote')) +import install_generic_stack as runtime +import oci_stack_public_routes as routes + +class PublicRouteInstallTests(unittest.TestCase): + def test_private_members_route_to_all_static_frontends_with_real_identities(self): + services = [ + {'name':'web','vmid':4810,'main':True,'ip':'10.77.0.30','frontend_ipv4':'10.0.1.246/24','deployment':{}}, + {'name':'mail','vmid':4811,'main':False,'frontend':True,'ip':'10.77.0.31','frontend_ipv4':'10.0.1.245/24','deployment':{}}, + {'name':'worker','vmid':4812,'main':False,'ip':'10.77.0.32','deployment':{}}, + ] + identities = {service['vmid']: {'installation_id':str(uuid.uuid4())} for service in services} + with tempfile.TemporaryDirectory() as directory: + config = Path(directory) / '4812.conf' + config.write_text('hostname: worker\nlxc.include: /existing/sysctl.conf\n') + def path(value): + return config if str(value) == '/etc/pve/lxc/4812.conf' else Path(value) + def install(deployment, vmid, identity): + routes.validate(deployment,vmid,identity) + return f'lxc.include: {routes.include_path(vmid)}' + with patch.object(runtime,'Path',side_effect=path), patch.object(runtime.oci_instances,'read',side_effect=lambda root,vmid:identities[vmid]), patch.object(routes,'install',side_effect=install), patch.object(routes,'check',wraps=lambda text,plan,vmid,identity=None,**kw:routes.validate(plan,vmid,identity)): + runtime.install_public_routes(services,4810,'vmbr10') + self.assertIn('lxc.include: /existing/sysctl.conf\n',config.read_text()) + spec=services[2]['deployment']['stack_public_routes'] + self.assertEqual(spec['installation_id'],identities[4812]['installation_id']) + self.assertEqual(spec['stack_id'],identities[4810]['installation_id']) + self.assertEqual(spec['primary_vmid'],4810) + self.assertEqual(spec['routes'],[ + {'destination':'10.0.1.245','via':'10.77.0.31','target_vmid':4811,'target_installation_id':identities[4811]['installation_id']}, + {'destination':'10.0.1.246','via':'10.77.0.30','target_vmid':4810,'target_installation_id':identities[4810]['installation_id']}]) + self.assertNotIn('stack_public_routes',services[0]['deployment']) + self.assertNotIn('stack_public_routes',services[1]['deployment']) + + def test_dhcp_without_static_targets_does_not_invent_routes(self): + services=[{'main':True,'frontend_ipv4':'dhcp','deployment':{}}] + with patch.object(runtime.oci_instances,'read') as read: + runtime.install_public_routes(services,4810,'vmbr10') + read.assert_not_called() + + def test_duplicate_static_ip_rejected_before_read_or_write(self): + services=[{'main':True,'frontend_ipv4':'10.0.1.246/24'}, {'frontend':True,'frontend_ipv4':'10.0.1.246/24'}] + with patch.object(runtime.oci_instances,'read') as read: + with self.assertRaisesRegex(RuntimeError,'Duplicate'):runtime.install_public_routes(services,4810,'vmbr10') + read.assert_not_called() + + def test_recreate_route_include_is_last_after_other_managed_includes(self): + source=(ROOT/'remote/install_oci.sh').read_text() + flow=source[source.index('prepare_managed_format_directories "/var/lib/lxc/'):] + self.assertLess(flow.index('apply_host_monitor\n'),flow.index('ROUTE_INCLUDE=$(python3')) + self.assertLess(flow.index('apply_kept_settings\n'),flow.index('ROUTE_INCLUDE=$(python3')) + + def test_routes_installed_before_jobs_and_main_start(self): + source=(ROOT/'remote/install_generic_stack.py').read_text() + main=source[source.index('def main('):] + self.assertLess(main.index('install_public_routes(services, primary_id, bridge)'),main.index('bind_completed_jobs(services)')) + self.assertLess(main.index('install_public_routes(services, primary_id, bridge)'),main.index("run('pct','start',primary_id")) + +if __name__ == '__main__':unittest.main() diff --git a/oci/tests/test_stack_public_routes.py b/oci/tests/test_stack_public_routes.py new file mode 100644 index 00000000..0c53977e --- /dev/null +++ b/oci/tests/test_stack_public_routes.py @@ -0,0 +1,199 @@ +"""Managed routes cannot configure an unowned guest or the host namespace.""" +import copy +from contextlib import ExitStack +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'remote')) +import oci_stack_public_routes as routes + +OWNER = '11111111-1111-4111-8111-111111111111' +PRIMARY = '22222222-2222-4222-8222-222222222222' +MAC = '02:11:22:33:44:55' + + +def fixture(): + return {'stack_public_routes': {'schema': 1, 'stack_id': PRIMARY, 'primary_vmid': 4900, + 'vmid': 4901, 'installation_id': OWNER, 'private_bridge': 'vmbr15', + 'private_address': '10.77.5.31/24', 'routes': [ + {'destination': '172.30.249.90', 'via': '10.77.5.30', + 'target_vmid': 4900, 'target_installation_id': PRIMARY}]}} + + +class PublicRouteTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + self.stack = ExitStack() + self.addCleanup(self.stack.close) + for key, value in {'DIRECTORY': root/'owned', 'HELPER': root/'helper.py', + 'CONFIGS': root/'configs', 'ROOT_UID': os.getuid()}.items(): + self.stack.enter_context(patch.object(routes, key, value)) + routes.CONFIGS.mkdir() + routes.HELPER.write_text(Path(routes.__file__).read_text()) + routes.HELPER.chmod(0o644) + self.deployment = fixture() + self.spec = self.deployment['stack_public_routes'] + + def config(self, vmid, identity, private, public=None, mac=MAC): + text = f'# proxmenux-instance={identity}\nnet0: name=eth0,bridge=vmbr15,ip={private},hwaddr={mac}\n' + if public: + text += f'net1: name=eth1,bridge=vmbr909,ip={public}\n' + if vmid == 4901: + text += f'lxc.include: {routes.include_path(vmid)}\n' + path = routes.CONFIGS/f'{vmid}.conf' + path.write_text(text) + path.chmod(0o600) + return text + + def install(self): + return routes.install(self.deployment, 4901, OWNER) + + def test_install_check_canonical_include_and_root_owned_contract(self): + line = self.install() + self.assertEqual(line, f'lxc.include: {routes.include_path(4901)}') + self.assertNotIn('lxc.hook.version', routes.include_path(4901).read_text()) + self.assertIn(' -I ', routes.include_content()) + self.assertEqual(routes.check(line, self.deployment, 4901, OWNER), self.spec) + self.assertEqual(routes.contract_path(4901).stat().st_mode & 0o777, 0o600) + self.assertEqual(routes.HELPER.stat().st_mode & 0o777, 0o644) + + def test_restore_can_validate_without_assets_but_not_a_foreign_include(self): + line = f'lxc.include: {routes.include_path(4901)}\n' + self.assertEqual(routes.check(line, self.deployment, 4901, OWNER, require_host_assets=False), self.spec) + with self.assertRaises(routes.RouteError): + routes.check(line.replace('4901.conf','4902.conf'), self.deployment, 4901, OWNER, require_host_assets=False) + self.assertEqual(routes.renumber_line(line.encode(),4901,5901), f'lxc.include: {routes.include_path(5901)}\n'.encode()) + + def test_reused_vmid_cannot_remove_another_installation_contract(self): + self.install() + with self.assertRaises(routes.RouteError):routes.remove(4901, PRIMARY) + self.assertTrue(routes.contract_path(4901).exists()) + self.assertTrue(routes.remove(4901, OWNER)) + self.assertFalse(routes.include_path(4901).exists()) + + def test_bad_identity_and_unsafe_routes_fail_before_commands(self): + changes = [lambda s:s.update(vmid=4902), lambda s:s.update(installation_id=PRIMARY), + lambda s:s['routes'][0].update(destination='0.0.0.0/0'), + lambda s:s['routes'][0].update(destination='127.0.0.1'), + lambda s:s['routes'][0].update(via='172.30.249.1'), + lambda s:s['routes'][0].update(destination=True), + lambda s:s['routes'].append(copy.deepcopy(s['routes'][0]))] + with patch.object(routes.subprocess,'run') as run: + for change in changes: + value=copy.deepcopy(self.deployment);change(value['stack_public_routes']) + with self.subTest(value=value),self.assertRaises(routes.RouteError):routes.validate(value,4901,OWNER) + run.assert_not_called() + + def test_symlink_and_unsafe_helper_fail_closed(self): + routes.DIRECTORY.mkdir() + target=Path(self.temp.name)/'other';target.write_text('do not touch') + routes.include_path(4901).symlink_to(target) + with self.assertRaises(routes.RouteError):self.install() + self.assertEqual(target.read_text(),'do not touch') + routes.include_path(4901).unlink() + routes.HELPER.chmod(0o666) + with self.assertRaises(routes.RouteError):self.install() + + def test_symlink_directory_and_foreign_file_owner_are_rejected(self): + outside=Path(self.temp.name)/'outside';outside.mkdir() + routes.DIRECTORY.symlink_to(outside, target_is_directory=True) + with self.assertRaises(routes.RouteError):self.install() + unsafe=SimpleNamespace(lstat=lambda:SimpleNamespace(st_mode=stat.S_IFREG|0o600,st_uid=routes.ROOT_UID+1,st_size=10)) + with self.assertRaises(routes.RouteError):routes._read(unsafe) + + def test_live_identity_and_frontend_addresses_are_verified(self): + self.install() + self.config(4901,OWNER,'10.77.5.31/24') + self.config(4900,PRIMARY,'10.77.5.30/24','172.30.249.90/24') + self.assertEqual(routes.live_contract(self.spec),MAC) + self.config(4900,OWNER,'10.77.5.30/24','172.30.249.90/24') + with self.assertRaises(routes.RouteError):routes.live_contract(self.spec) + self.config(4900,PRIMARY,'10.77.5.30/24','172.30.249.91/24') + with self.assertRaises(routes.RouteError):routes.live_contract(self.spec) + + def test_tampered_or_world_writable_contract_is_rejected(self): + self.install();routes.contract_path(4901).chmod(0o666) + with self.assertRaises(routes.RouteError):routes.check(f'lxc.include: {routes.include_path(4901)}',self.deployment,4901,OWNER) + + def test_namespace_is_a_net_namespace_and_never_the_host(self): + with patch.object(routes.os,'open',return_value=17),patch.object(routes.os,'close') as close, \ + patch.object(routes.fcntl,'ioctl',return_value=routes.CLONE_NEWNET), \ + patch.object(routes.os,'fstat',return_value=SimpleNamespace(st_dev=1,st_ino=9)), \ + patch.object(routes.os,'stat',return_value=SimpleNamespace(st_dev=1,st_ino=9)): + with self.assertRaisesRegex(routes.RouteError,'host network'):routes.namespace_fd('/proc/123/fd/4') + close.assert_called_once_with(17) + with patch.object(routes.os,'open',return_value=17),patch.object(routes.os,'close'), \ + patch.object(routes.fcntl,'ioctl',return_value=0): + with self.assertRaisesRegex(routes.RouteError,'not a network'):routes.namespace_fd('/proc/123/fd/4') + with patch.object(routes.os,'open') as opened: + for value in (None,'/proc/self/ns/net','/etc/passwd','/proc/123/ns/mnt'): + with self.assertRaises(routes.RouteError):routes.namespace_fd(value) + opened.assert_not_called() + + def test_routes_are_only_32_in_verified_namespace_with_no_shell(self): + interface=[{'address':MAC,'addr_info':[{'family':'inet','local':'10.77.5.31','prefixlen':24}]}] + with patch.object(routes,'_namespace_command',side_effect=[json.dumps(interface).encode(),b'']) as command: + routes.apply(self.spec,17,MAC) + self.assertEqual(command.call_args_list[0].args,(17,'-j','address','show','dev','eth0')) + self.assertEqual(command.call_args.args,(17,'-4','route','replace','172.30.249.90/32','via','10.77.5.30','dev','eth0','src','10.77.5.31')) + with patch.object(routes.subprocess,'run',return_value=SimpleNamespace(stdout=b'ok')) as run: + routes._namespace_command(17,'-4','route','show') + self.assertEqual(run.call_args.args[0][:3],[routes.NSENTER,'--net=/proc/self/fd/17','--']) + self.assertEqual(run.call_args.kwargs['pass_fds'],(17,));self.assertNotIn('shell',run.call_args.kwargs) + + def test_engine_helper_can_change_normally_without_stale_snapshot_adoption(self): + routes.HELPER.write_text('# trusted engine update, same stable path\n') + self.install() + self.assertEqual(routes.HELPER.read_text(),'# trusted engine update, same stable path\n') + + def test_namespace_with_wrong_owner_address_cannot_receive_routes(self): + for value in ([], [{'address':'02:aa:bb:cc:dd:ee','addr_info':[{'family':'inet','local':'10.77.5.31','prefixlen':24}]}]): + with patch.object(routes,'_namespace_command',return_value=json.dumps(value).encode()) as command: + with self.assertRaises(routes.RouteError):routes.apply(self.spec,17,MAC) + self.assertEqual(command.call_count,1) + + def test_ip_failure_propagates_so_startup_cannot_report_success(self): + interface=[{'address':MAC,'addr_info':[{'family':'inet','local':'10.77.5.31','prefixlen':24}]}] + with patch.object(routes,'_namespace_command',side_effect=[json.dumps(interface).encode(),subprocess.CalledProcessError(2,['ip'])]): + with self.assertRaises(subprocess.CalledProcessError):routes.apply(self.spec,17,MAC) + + def test_start_uses_only_the_runtime_pid_namespace(self): + self.install() + with patch.dict(os.environ,{'LXC_NAME':'4901','LXC_PID':'12345'},clear=True), \ + patch.object(routes.os,'geteuid',return_value=0), \ + patch.object(routes,'live_contract',return_value=MAC), \ + patch.object(routes,'namespace_fd',return_value=17) as namespace, \ + patch.object(routes,'apply') as apply, patch.object(routes.os,'close') as close: + routes.start() + namespace.assert_called_once_with('/proc/12345/ns/net') + apply.assert_called_once_with(self.spec,17,MAC) + close.assert_called_once_with(17) + + def test_default_hook_arguments_must_match_owner_and_phase(self): + with patch.dict(os.environ,{'LXC_NAME':'4901'},clear=True), \ + patch.object(routes.os,'geteuid',return_value=0),patch.object(routes,'start') as start: + with patch.object(sys,'argv',['helper','start','4901','lxc','start-host']):routes.main() + start.assert_called_once() + start.reset_mock() + with patch.object(sys,'argv',['helper','start','4902','lxc','start-host']), self.assertRaises(routes.RouteError):routes.main() + with patch.object(sys,'argv',['helper','start','4901','lxc','post-stop']), self.assertRaises(routes.RouteError):routes.main() + start.assert_not_called() + + def test_missing_lxc_pid_is_not_replaced_by_a_host_fallback(self): + self.install() + with patch.dict(os.environ,{'LXC_NAME':'4901'},clear=True),patch.object(routes.os,'geteuid',return_value=0),patch.object(routes,'live_contract',return_value=MAC),patch.object(routes,'apply') as apply: + with self.assertRaises(routes.RouteError):routes.start() + apply.assert_not_called() + + +if __name__=='__main__':unittest.main() diff --git a/oci/tests/test_stack_public_routes_lifecycle.py b/oci/tests/test_stack_public_routes_lifecycle.py new file mode 100644 index 00000000..89710780 --- /dev/null +++ b/oci/tests/test_stack_public_routes_lifecycle.py @@ -0,0 +1,131 @@ +"""Managed frontend /32 routes remain bounded and portable across lifecycle APIs.""" +import copy +import contextlib +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'remote')) +import oci_instance_transaction as tx +import oci_restore_recovery as recovery +import oci_stack_plan as plans +import oci_stack_public_routes as routes +import oci_remove as removal + +MAIN='11111111-1111-4111-8111-111111111111' +WORKER='22222222-2222-4222-8222-222222222222' +FOREIGN='33333333-3333-4333-8333-333333333333' + +def deployment(): + return {'vmid':4901,'mounts':[],'stack_public_routes':{'schema':1,'stack_id':MAIN, + 'primary_vmid':4900,'vmid':4901,'installation_id':WORKER,'private_bridge':'vmbr15', + 'private_address':'10.77.5.31/24','routes':[{'destination':'192.0.2.10','via':'10.77.5.30', + 'target_vmid':4900,'target_installation_id':MAIN}]}} + +def records(): + return {4900:{'vmid':4900,'installation_id':MAIN,'deployment':{},'stack':{'members':[ + {'vmid':4900,'installation_id':MAIN},{'vmid':4901,'installation_id':WORKER}]}}, + 4901:{'vmid':4901,'installation_id':WORKER,'deployment':deployment()}} + +class ManagedRouteLifecycleTests(unittest.TestCase): + @contextlib.contextmanager + def assets(self,folder): + with patch.multiple(routes,DIRECTORY=Path(folder)/'private',HELPER=Path(folder)/'canonical.py',ROOT_UID=os.geteuid()): + # The installed engine is trusted; guest-carried code is never used. + routes.HELPER.write_bytes(Path(routes.__file__).read_bytes()) + routes.HELPER.chmod(0o755) + yield + + def test_managed_include_coexists_with_sysctl_validation_and_rejects_unknown_include(self): + with tempfile.TemporaryDirectory() as folder,self.assets(folder): + plan=deployment();routes.install(plan,4901,WORKER) + config=(f'description: proxmenux-instance={WORKER}\nrootfs: local-zfs:subvol-4901-disk-0,size=8G\n' + f'lxc.include: {routes.include_path(4901)}\n').encode() + tx.check_runtime_mounts(config,plan) + with self.assertRaisesRegex(ValueError,'sysctl include'): + tx.check_runtime_mounts(config+b'lxc.include: /unknown/profile.conf\n',plan) + with self.assertRaisesRegex(ValueError,'differs from the contract'): + tx.check_runtime_mounts(config,{'vmid':4901,'mounts':[]}) + + def test_plan_rejects_foreign_target_even_if_supplied_in_record_inventory(self): + good=records();plans.validate_public_route_members(4900,good) + bad=copy.deepcopy(good);bad[4999]={'vmid':4999,'installation_id':FOREIGN,'deployment':{}} + route=bad[4901]['deployment']['stack_public_routes']['routes'][0] + route.update(target_vmid=4999,target_installation_id=FOREIGN) + with self.assertRaisesRegex(ValueError,'Invalid stack contract'): + plans.validate_public_route_members(4900,bad) + + def test_plan_rejects_changed_target_and_stack_identities(self): + for field in ('target_installation_id','stack_id'): + bad=records();spec=bad[4901]['deployment']['stack_public_routes'] + if field=='stack_id':spec[field]=FOREIGN + else:spec['routes'][0][field]=FOREIGN + with self.subTest(field=field),self.assertRaisesRegex(ValueError,'Invalid stack contract'): + plans.validate_public_route_members(4900,bad) + + def test_restore_renumbers_only_managed_paths_and_all_route_member_ids(self): + with tempfile.TemporaryDirectory() as folder,self.assets(folder): + mapped=recovery.renumber(records(),{4900:4800,4901:4801}) + # Mapping dictionary keys are not VMID-valued fields: select records by their new vmid. + mapped={value['vmid']:value for value in mapped.values()} + plans.validate_public_route_members(4800,mapped) + spec=mapped[4801]['deployment']['stack_public_routes'] + self.assertEqual((spec['vmid'],spec['primary_vmid'],spec['routes'][0]['target_vmid']),(4801,4800,4800)) + self.assertEqual(spec['installation_id'],WORKER) + original=f'lxc.include: {routes.include_path(4901)}\nlxc.include: /unrelated/4901.conf\n' + changed=recovery.renumbered_lines(original,4901,4801) + self.assertIn(f'lxc.include: {routes.include_path(4801)}',changed) + self.assertIn('lxc.include: /unrelated/4901.conf',changed) + routes.check(changed.splitlines()[0]+'\n',mapped[4801]['deployment'],4801,WORKER,require_host_assets=False) + + def test_restore_republishes_only_canonical_host_code_before_start(self): + with tempfile.TemporaryDirectory() as folder,self.assets(folder): + plan=recovery.renumber(deployment(),{4900:4800,4901:4801}) + entry={'vmid':4801,'installation_id':WORKER,'config':f'lxc.include: {routes.include_path(4801)}\n', + 'copy':{'record':{'deployment':plan},'untrusted_script':'do not execute'}} + with patch.object(recovery,'renumber_config'): + recovery.restore_host_files(entry) + self.assertEqual(routes.HELPER.read_bytes(),Path(routes.__file__).read_bytes()) + self.assertEqual(json.loads(routes.contract_path(4801).read_text())['routes'][0]['target_vmid'],4800) + routes.check(entry['config'],plan,4801,WORKER) + + def test_restore_plan_refuses_foreign_route_before_host_assets_are_created(self): + bad=records();bad[4901]['deployment']['stack_public_routes']['routes'][0]['target_vmid']=4999 + plan={'primary':4900,'members':[{'vmid':v} for v in bad],'restored':[ + {'vmid':v,'copy':{'record':record}} for v,record in bad.items()],'blockers':[]} + recovery.check_public_route_members(Path('/unused'),plan) + self.assertTrue(any('Invalid stack contract' in value for value in plan['blockers'])) + + def test_recreate_keeps_managed_routes_outside_editable_desired_state(self): + old={'vmid':4901,'installation_id':WORKER,'status':'installed', + 'template':{'id':'fixture','proxmox':{'installer_profile':{}}, + 'first_run':{'endpoints':[{'scheme':'http','port':80,'path':'/'}]}}, + 'deployment':deployment()} + candidate=copy.deepcopy(old);candidate['deployment'].pop('stack_public_routes') + candidate['deployment']['resources']={'cores':2,'memory_mb':1024} + result=tx.candidate_contract(old,'recreate',{'operation':'recreate','candidate':candidate}) + self.assertEqual(result['deployment']['stack_public_routes'],old['deployment']['stack_public_routes']) + result['deployment']['stack_public_routes']['routes'][0]['destination']='192.0.2.11' + self.assertEqual(old['deployment']['stack_public_routes']['routes'][0]['destination'],'192.0.2.10') + + def test_orphan_sweep_cleans_only_absent_owned_routes_and_keeps_pending_or_reused_ids(self): + for case in ('absent','pending','reused','changed'): + with self.subTest(case=case),tempfile.TemporaryDirectory() as folder,self.assets(folder): + root=Path(folder)/'records';root.mkdir() + routes.install(deployment(),4901,WORKER) + if case=='changed':routes.include_path(4901).write_text('foreign hook') + with patch.object(removal.instances,'guest_exists',return_value=case=='reused'), \ + patch.object(removal.instances,'has_contract',return_value=case=='pending'), \ + patch.object(removal.instances,'read',return_value={'installation_id':WORKER}), \ + patch.object(removal.instances,'release_orphan',return_value=False), \ + patch.object(removal,'remove_host_state'), \ + patch.object(removal.oci_console,'remove_log'), \ + patch.object(removal,'release_shared_host_files'): + cleaned=removal.sweep_orphans(root) + self.assertEqual(cleaned,[4901] if case=='absent' else []) + self.assertEqual(routes.contract_path(4901).exists(),case!='absent') + self.assertEqual(routes.include_path(4901).exists(),case!='absent')