diff --git a/.github/scripts/tests/test_oci_wording.py b/.github/scripts/tests/test_oci_wording.py index 76499b5c..9de358d1 100644 --- a/.github/scripts/tests/test_oci_wording.py +++ b/.github/scripts/tests/test_oci_wording.py @@ -20,8 +20,7 @@ AFTER = 'The recovery did not complete. Review the log and choose "{choice}" aga REGISTRY_ADVICE = ('Check the image reference and registry access. If the registry is unavailable, ' 'try again later. For a private image, sign in from Registry accounts in the OCI menu.') NEW_KEYS = ( - 'The Compose file describes several services:', - 'Only one service at a time can be installed this way.', + 'Containers of the application:', 'No docker run command was given', 'Image selection was cancelled', 'Could not inspect the image in its registry:', @@ -77,15 +76,16 @@ class OciWordingTests(TestCase): setattr(module, '_cache', {NEW_KEYS[0]: 'Several services (translated):'}) self.assertEqual(module.translate(NEW_KEYS[0]), 'Several services (translated):') - def test_multiple_services_named_not_images(self): + def test_multiple_services_keep_their_names_and_enter_stack_conversion(self): + # Reusing one image twice still creates two distinct Compose services. + document = {'services': {'alpha': {'image': 'same:latest'}, + 'beta': {'image': 'same:latest'}}} + converted = object() + stack = Mock(return_value=converted) template = function(CUSTOM, 'template_from_compose', - _services=lambda doc: doc['services'], _check_mounts=lambda _: None) - with self.assertRaises(ConversionError) as caught: - template(json.dumps({'services': {'alpha': {'image': 'same:latest'}, - 'beta': {'image': 'same:latest'}}})) - self.assertEqual(str(caught.exception), - 'The Compose file describes several services: alpha, beta. ' - 'Only one service at a time can be installed this way.') + _services=lambda doc: doc['services'], _template_from_stack=stack) + self.assertIs(template(json.dumps(document)), converted) + stack.assert_called_once_with(document, None) def test_empty_paste_is_specific_and_nonempty_unchanged(self): for title, expected in ((None, 'No Compose file was given'), diff --git a/oci/README.md b/oci/README.md index e5c404b1..a8ff200d 100644 --- a/oci/README.md +++ b/oci/README.md @@ -209,7 +209,7 @@ The native path checks the saved configuration and installation identity, keeps ## From a Compose file or a docker run command to an oci-compose -The converter reads a Compose file with one service. A `docker run` command and an image reference alone are first written as a Compose file and follow the same path, so the three produce the same kind of oci-compose. +The converter reads Compose files with one or several services. Multi-service files use the same native stack compiler as catalog applications: each service runs in its own LXC, with dependency ordering and private networking. The first service that publishes a port is selected as the main service, unless `x-casaos.main` specifies another one. Every image is checked for registry access before installation, and written tags and digests are preserved. Each web application uses its own published container port for its health check and receives its own LAN address. Repeated identical volume sources share one host directory, whose location is selected once; separate configuration volumes remain independent. Unsupported stack features (including external networks and application services without a supported health check) are reported before installation. A `docker run` command and an image reference alone are first written as a Compose file and follow the same path, so the three produce the same kind of oci-compose. Starting from this definition: @@ -251,7 +251,7 @@ each key becomes a field of the oci-compose: | `stop_grace_period` | `proxmox.defaults.shutdown_timeout_seconds` | The time a shutdown waits | | `labels`, `logging` | Kept in `container_contract.original_compose` only | Not applied: they are read by Docker tools | -What has no translation stops the installation and is named: `pid: host`, a Compose file with several services given to this option, `configs`, external secrets, a `build` section without a published image, and shell substitutions such as `$(command)`. +What has no translation stops the installation and is named: `pid: host`, `configs`, external secrets, a `build` section without a published image, and shell substitutions such as `$(command)`. Besides those fields, the converter writes the blocks that describe the translation itself: where the definition came from, the keys it understood, the adaptations it applied and the state of each validation. They are listed in [Blocks ProxMenux writes](#blocks-proxmenux-writes). diff --git a/oci/remote/install_generic_stack.py b/oci/remote/install_generic_stack.py index 6d7794be..a1ab4a68 100644 --- a/oci/remote/install_generic_stack.py +++ b/oci/remote/install_generic_stack.py @@ -421,6 +421,12 @@ def attach_mounts(service, temporary): if not source.is_absolute() or ',' in str(source) or '\n' in str(source): raise RuntimeError(translate('Invalid shared path')) if not source.exists(): + shared_owner = mount.get('host_owner') + if shared_owner is not None: + if (not isinstance(shared_owner, list) or len(shared_owner) != 2 + or any(type(value) is not int or not 0 <= value <= 2**32-2 for value in shared_owner)): + raise RuntimeError(translate('Invalid shared path')) + owner = (*shared_owner, owner[2]) source.mkdir(parents=True) os.chown(source,*owner[:2]) source.chmod(owner[2]) @@ -883,7 +889,8 @@ def main(): refresh_notes(services) # The credentials of the main service come from its own installation. result.update(vmid=primary_id,ip=lan,stack_vmids={s['name']:s['vmid'] for s in services}, - urls=[{'label':'Web UI','url':public_url}], + urls=(lan_access_urls(services, subnet) if sum(bool(s['main'] or s.get('frontend')) for s in services) > 1 + else [{'label':'Web UI','url':public_url}]), credentials=result.get('credentials') or []) result.update(completion_notes=[note.replace('{main_vmid}',str(primary_id)) for note in deployment.get('completion_notes', [])], log=LOG) diff --git a/oci/src/proxmenux_oci/custom.py b/oci/src/proxmenux_oci/custom.py index acc14355..1e42c719 100644 --- a/oci/src/proxmenux_oci/custom.py +++ b/oci/src/proxmenux_oci/custom.py @@ -2,6 +2,7 @@ is translated into the same template the catalog applications use.""" from __future__ import annotations +import copy import json import os import re @@ -20,7 +21,7 @@ from .catalog import TIME_SETTINGS from .cli import install_template from .i18n import source_text from .installer import ADVANCED_MODE, DEFAULT_MODE -from .converter import ConversionError +from .converter import ConversionError, _image_contract from .i18n import N_, translate from .registry_auth import RegistryError, refuses_access, validate_image from .ui import UserCancelled @@ -107,22 +108,14 @@ def _check_mounts(service: dict[str, Any]) -> None: def _keep_reference(template: dict[str, Any], reference: str) -> None: """The catalog normalizes every image to its latest tag; an image given by hand keeps the tag or the digest that was written.""" - repository, _, digest = reference.partition('@') - tag = '' - if ':' in repository.rsplit('/', 1)[-1]: - repository, _, tag = repository.rpartition(':') - image = template['container_contract']['image'] - image['reference'] = reference if (tag or digest) else f'{repository}:latest' - image['repository'] = repository - image['tag'] = tag or ('' if digest else 'latest') - image['digest'] = f'@{digest}' if digest else None - if tag or digest: + image = _image_contract(reference) + if image.get('digest') or ':' in reference.split('@', 1)[0].rsplit('/', 1)[-1]: image['pull_policy'] = 'resolve-written-reference-to-architecture-digest-at-install' + template['container_contract']['image'] = image def template_from_compose(text: str, title: str | None = None) -> dict[str, Any]: - """The template of a Compose file with a single service. The metadata the - importer expects is taken from the service itself.""" + """Translate a Compose application through the native single or stack installer.""" try: compose = yaml.safe_load(text) except yaml.YAMLError as error: @@ -131,9 +124,7 @@ def template_from_compose(text: str, title: str | None = None) -> dict[str, Any] raise ConversionError(translate('The Compose file does not contain a Compose document')) services = _services(compose) if len(services) > 1: - raise ConversionError( - f"{translate('The Compose file describes several services:')} {', '.join(services)}. " - f"{translate('Only one service at a time can be installed this way.')}") + return _template_from_stack(compose, title) name = next(iter(services)) service = services[name] or {} if not service.get('image'): @@ -181,6 +172,63 @@ def template_from_compose(text: str, title: str | None = None) -> dict[str, Any] return template +def _template_from_stack(compose: dict[str, Any], title: str | None) -> dict[str, Any]: + """Reuse the catalog stack compiler, keeping explicitly selected image versions.""" + from .stack import apply_stack_support + + services = _services(compose) + singles = {} + for name, service in services.items(): + if not isinstance(service, dict): + raise ConversionError(f"{translate('The service declares no image:')} {name}") + singles[name] = template_from_compose(_dump({'services': {name: service}}), title) + metadata = compose.get('x-casaos', {}) + if metadata is None: + metadata = {} + if not isinstance(metadata, dict): + raise ConversionError('x-casaos.main no identifica un servicio valido') + main = metadata.get('main') or next( + (name for name, service in services.items() if _published_port(service)), next(iter(services))) + if not isinstance(main, str) or main not in services: + raise ConversionError('x-casaos.main no identifica un servicio valido') + document = copy.deepcopy(compose) + document['x-casaos'] = {**metadata, 'main': main, + 'title': {'en_US': title or main}} + port = _published_port(services[main]) + if port: + document['x-casaos'].update(port_map=port, scheme='http', index='/') + identifier = normalize_app_id(str(compose.get('name') or title or main)) + template = convert_casaos_compose(_dump(document), 'local', '', + f'local/{identifier}/docker-compose.yml', '') + template['proxmox']['custom_compose_stack'] = True + _keep_reference(template, str(services[main]['image'])) + # Catalog imports use latest. A pasted Compose must keep every written tag/digest. + for service in template['compose_stack']['services']: + reference = singles[service['name']]['container_contract']['image']['reference'] + service['image'] = reference + service['compose']['image'] = reference + if 'environment' in services[service['name']]: + service['compose']['environment'] = copy.deepcopy(services[service['name']]['environment']) + service['compose']['volumes'] = [ + mount for mount in service['compose'].get('volumes', []) + if (mount.get('target') if isinstance(mount, dict) + else str(mount).split(':')[-1]) not in TIME_SETTINGS] + template['container_contract']['original_compose'] = _dump({ + **template['compose_stack']['top_level'], + 'services': {service['name']: service['compose'] for service in template['compose_stack']['services']}}) + for service in template['container_contract']['related_services']: + service['image'] = singles[service['name']]['container_contract']['image']['reference'] + preparations = list({entry['container_path']: entry for single in singles.values() + for entry in single['proxmox'].get('installer_profile', {}).get('volume_preparations', [])}.values()) + template['proxmox'].setdefault('installer_profile', {})['volume_preparations'] = preparations + apply_stack_support(template) + # Keep compiler diagnostics visible at the custom-image boundary. + if template['proxmox'].get('generic_stack_review'): + template['compatibility']['automatic_install_candidate'] = False + template['compatibility']['untranslated_blockers'] = template['proxmox']['generic_stack_review'] + return template + + def compose_from_image(reference: str, title: str | None = None) -> str: """A Compose file written from what the image declares: its ports, its persistent paths and its variables.""" @@ -589,7 +637,7 @@ def ignored_settings(text: str) -> list[str]: notes.append(translate('The container takes its time zone from Proxmox, so the time files ' 'of the host are not attached to it.')) break - if compose.get('networks') or compose.get('volumes'): + if len(compose.get('services') or {}) == 1 and (compose.get('networks') or compose.get('volumes')): notes.append(translate('The container gets its own address and its own volumes, so the networks ' 'and volumes declared in the file are not used.')) return notes @@ -642,6 +690,10 @@ def describe(template: dict[str, Any]) -> str: profile = proxmox.get('installer_profile', {}) or {} security = proxmox.get('security_profile', {}) or {} lines = [f"{translate('Image') + ':':<14} {contract['image']['reference']}"] + services = (template.get('compose_stack') or {}).get('services') or [] + if len(services) > 1: + lines += ['', translate('Containers of the application:')] + lines += [f" {service['name']}: {service['image']}" for service in services] endpoints = template.get('first_run', {}).get('endpoints') or [] if endpoints: # The address is read from the published port; the image does not say @@ -679,7 +731,7 @@ def describe(template: dict[str, Any]) -> str: if environment: lines += ['', translate('Variables the installation asks for:')] lines += [f" {item['name']}" - + (f" = {item['example']}" if item.get('example') and not item['sensitive'] else '') + + (f" = {item['example']}" if item.get('example') and not item['sensitive'] and len(services) <= 1 else '') + ('' if item.get('required', True) else f" ({translate('optional')})") for item in environment] devices = profile.get('device_requests') or [] @@ -744,13 +796,17 @@ def _explore(ui) -> None: template = template_from_compose(text) title = translate('Image that is not in the catalog') summary = describe(template) + (describe_oci_compose(template) if written else '') - reference = template['container_contract']['image']['reference'] - available, report = registry_report(reference) - if not available and _registry_refused(reference): - from .registry_accounts import offer_sign_in - if offer_sign_in(ui, reference, title): - available, report = registry_report(reference) - summary += '\n\n' + report + stack_services = (template.get('compose_stack') or {}).get('services') or [] + references = list(dict.fromkeys(service['image'] for service in stack_services)) if len(stack_services) > 1 else [template['container_contract']['image']['reference']] + available = True + for reference in references: + accessible, report = registry_report(reference) + if not accessible and _registry_refused(reference): + from .registry_accounts import offer_sign_in + if offer_sign_in(ui, reference, title): + accessible, report = registry_report(reference) + available = available and accessible + summary += '\n\n' + report if notes: summary += '\n\n' + '\n'.join(notes) if not available: @@ -762,7 +818,7 @@ def _explore(ui) -> None: ui.message(f"{translate('This image cannot be installed as it is described:')}\n\n{summary}", title) return # An oci-compose already says which of its values are secret and which it generates. - secrets = [] if written else _ask_secrets(template) + secrets = [] if written or len(stack_services) > 1 else _ask_secrets(template) if secrets: summary += ('\n\n' + translate('These values are asked during the installation:') + ' ' + ', '.join(secrets)) diff --git a/oci/src/proxmenux_oci/stack.py b/oci/src/proxmenux_oci/stack.py index 3b878d3f..63c5cb8b 100644 --- a/oci/src/proxmenux_oci/stack.py +++ b/oci/src/proxmenux_oci/stack.py @@ -5,13 +5,14 @@ import copy import base64 import re import secrets +import hashlib from urllib.parse import urlsplit, urlunsplit, quote import yaml from .casaos import convert_casaos_compose, canonical_image_repository from .converter import ConversionError -from .converter import _split_short_mount +from .converter import _image_contract, _split_short_mount from .i18n import translate @@ -143,7 +144,10 @@ def service_template(parent, service, main=False): parent['catalog_ui'].get('category_label'), service['name'], ) # Preserve selected image and resolved stack values; the importer normalizes secrets. - single['container_contract']['image']['reference'] = service['image'] + image = _image_contract(service['image']) + if image.get('digest') or ':' in service['image'].split('@', 1)[0].rsplit('/', 1)[-1]: + image['pull_policy'] = 'resolve-written-reference-to-architecture-digest-at-install' + single['container_contract']['image'] = image single['container_contract']['environment'] = [ {'name': k, 'example': v, 'required': True, 'sensitive': True, 'prompt_user': False} for k, v in c['environment'].items() @@ -166,13 +170,41 @@ def service_template(parent, service, main=False): return single +def service_endpoint(service): + """A custom web service is checked on its own container port, not a host mapping.""" + for port in service['compose'].get('ports') or []: + if isinstance(port, dict): + protocol = port.get('protocol', 'tcp') + target = port.get('target') + else: + raw, _, protocol = str(port).partition('/') + protocol = protocol or 'tcp' + target = raw.rsplit(':', 1)[-1] + if protocol == 'tcp' and str(target).isdigit() and 0 < int(target) < 65536: + return {'scheme': 'http', 'port': int(target), 'path': '/'} + return None + + +def shared_sources(services): + """Source identities mounted more than once, before storage is chosen.""" + users = {} + for service in services: + for mount in normalized_mounts(service['compose'].get('volumes', [])): + source = mount.get('source') + if source: + users.setdefault(source, []).append(service['name']) + return {source for source, names in users.items() if len(names) > 1} + + def assess(template): """Reject untranslated semantics before any host resources are allocated.""" errors = [] + custom = template.get('proxmox', {}).get('custom_compose_stack', False) try: services = ordered_services(template) top = template['compose_stack'].get('top_level', {}) - if any(k not in {'name', 'networks', 'volumes', 'version'} for k in top): + if any(k not in {'name', 'networks', 'volumes', 'version'} + and not (custom and str(k).startswith('x-')) for k in top): raise StackError(translate('Top-level configs, secrets and other global options are not yet supported')) networks = top.get('networks', {}) if len(networks) > 1 or any(v and any(not ((k == 'driver' and x == 'bridge') or (k == 'name' and isinstance(x,str))) for k,x in v.items()) for v in networks.values()): @@ -185,17 +217,18 @@ def assess(template): for key in ('profiles','build','configs','secrets','env_file','pid','privileged','devices','runtime','cap_add','security_opt','network_mode','extra_hosts'): if c.get(key): errors.append(f"{s['name']}: {key} {translate('needs stack review')}") - if s['name'] != template['compose_stack']['main_service'] and kind(s['image']) == 'application': + if (s['name'] != template['compose_stack']['main_service'] and kind(s['image']) == 'application' + and not (custom and service_endpoint(s))): errors.append(f"{s['name']}: {translate('health and persistence profile not yet defined')}") if kind(s['image']) != 'application' and (c.get('command') or c.get('entrypoint')) and not authenticated_redis(s): errors.append(f"{s['name']}: {translate('custom dependency commands are not yet supported')}") for m in normalized_mounts(c.get('volumes', [])): source = m.get('source') - if source is not None and source in seen_sources: + if source is not None and source in seen_sources and not custom: errors.append(translate('Volumes shared between services are not yet supported')) if source is not None: seen_sources.add(source) - if set(m) - {'type','source','target','read_only'} or m.get('read_only'): + if set(m) - {'type','source','target','read_only'} or (m.get('read_only') and not custom): errors.append(translate('Volume options are not yet supported')) if m['target'].startswith(('/etc/', '/var/run/', '/run/', '/dev/', '/proc/', '/sys/')): errors.append(translate('System path mounts are not yet supported')) @@ -209,6 +242,15 @@ def assess(template): errors.append(f'{s["name"]}: {translate("unsupported external credential or boolean")} {variable}') single = service_template(template,s) errors.extend(f'{s["name"]}: {b}' for b in single['compatibility']['untranslated_blockers']) + if custom: + for source in shared_sources(services): + owners = set() + for service in services: + if any(m.get('source') == source for m in normalized_mounts(service['compose'].get('volumes', []))): + env = environment(service['compose'].get('environment')) + owners.add((env.get('PUID', '1000'), env.get('PGID', '1000'))) + if len(owners) > 1: + errors.append(f"{source}: {translate('Volumes shared between services are not yet supported')}") if not template['first_run'].get('endpoints'): errors.append(translate('Main endpoint not yet defined')) resolve_environments(services, 'UTC', template.get('proxmox', {}).get('stack_generators', {})) @@ -300,6 +342,9 @@ def build_stack(template, ui, mode='advanced'): raise StackError('; '.join(problems)) services = copy.deepcopy(ordered_services(template)) defaults = template['proxmox']['defaults'] + custom = template.get('proxmox', {}).get('custom_compose_stack', False) + shared = shared_sources(services) if custom else set() + shared_paths = {} # A default installation takes the name, the VMID and the settings from the # recipe; it still asks the storage, the address and how the stack starts. quiet = DefaultsUI() if mode == DEFAULT_MODE else ui @@ -332,8 +377,31 @@ def build_stack(template, ui, mode='advanced'): # they go is asked in a default installation too; the databases and # caches beside it keep the values of the recipe. asked = mode != DEFAULT_MODE or s['name'] == template['compose_stack']['main_service'] + source_mounts = {m['target']: m for m in normalized_mounts(s['compose'].get('volumes', []))} for m in plan['mounts']: label = f"{s['name']}: {m['container_path']}" + source_mount = source_mounts.get(m['container_path'], {}) + source = source_mount.get('source') + if source in shared: + if source not in shared_paths: + slug = re.sub(r'[^A-Za-z0-9_.-]+', '-', source).strip('.-') or 'data' + identity = hashlib.sha256(source.encode()).hexdigest()[:8] + default_path = source if source.startswith('/') else '/mnt/oci-shared/'+name+'/shared/'+slug+'-'+identity + shared_paths[source] = ui.ask(f"{translate('Host path for')} {source}", default_path) + chosen = shared_paths[source] + if not chosen.startswith('/') or any(c in chosen for c in ',\n\r') or '..' in chosen.split('/'): + raise StackError(translate('Invalid volume target')) + if chosen.rstrip('/') in {'', '/etc', '/var', '/usr', '/root', '/dev', '/proc', '/sys', '/run', '/boot', '/bin', '/sbin', '/lib', '/lib64'}: + raise StackError(translate('Invalid volume target')) + env = draft_envs[s['name']] + uid, gid = env.get('PUID', '1000'), env.get('PGID', '1000') + if not str(uid).isdigit() or not str(gid).isdigit() or int(uid) > 65535 or int(gid) > 65535: + raise StackError(translate('Invalid resources')) + offset = 100000 if plan.get('security', {}).get('unprivileged', True) else 0 + m.update(type='host-bind', source=shared_paths[source], backup=False, size_gb=None, + read_only=bool(source_mount.get('read_only')), + host_owner=[int(uid)+offset, int(gid)+offset]) + continue mount_mode = (ui.choose(f"{translate('Where to store')} {label}", [('managed-volume', translate('Container volume (included in backups)')), ('host-bind', translate('Host directory (not included in Proxmox backups)'))], @@ -358,7 +426,9 @@ def build_stack(template, ui, mode='advanced'): from .custom_mounts import ask_stack_custom_mounts ask_stack_custom_mounts(ui, drafts, volumes) bridge = ask_bridge(ui, translate('Access bridge'), defaults['bridge'], mode) - addresses, gateway = access.ask_addresses(ui, bridge, ['']) + frontend_names = [s['name'] for s in services if s['name'] == template['compose_stack']['main_service'] or (custom and service_endpoint(s))] + address_labels = frontend_names if custom else [''] + addresses, gateway = access.ask_addresses(ui, bridge, address_labels) timezone = quiet.ask(translate('Timezone'), host.timezone()) envs = resolve_environments(services, timezone, generators) for group in template.get('proxmox', {}).get('stack_optional_environment', []): @@ -418,13 +488,16 @@ def build_stack(template, ui, mode='advanced'): check = "REDISCLI_AUTH=${REDISCLI_AUTH:-$(tr '\\000' '\\n' < /proc/1/environ | sed -n 's/^REDISCLI_AUTH=//p')}; export REDISCLI_AUTH; " + check health = {'type':'exec','timeout_seconds':90,'argv':['sh','-c',check]} else: - endpoint = template['first_run']['endpoints'][0] + endpoint = service_endpoint(s) if custom else template['first_run']['endpoints'][0] + if endpoint is None: + raise StackError(f"{s['name']}: {translate('Main endpoint not yet defined')}") health = {'type':'http','timeout_seconds':360,'endpoint':endpoint} plans.append({'name':s['name'],'main':main,'kind':k,'offset':0 if main else len(plans)+1, 'aliases':list(dict.fromkeys([s['name'],s['compose'].get('container_name',s['name'])])), 'template':single,'deployment':plan,'healthcheck':health}) - if main: - plans[-1]['frontend_ipv4'] = addresses[''] + if s['name'] in frontend_names: + plans[-1]['frontend'] = True + plans[-1]['frontend_ipv4'] = addresses[s['name']] if custom else addresses[''] if mode != DEFAULT_MODE: from .extra_devices import ask_stack_extra_devices ask_stack_extra_devices(ui, plans) diff --git a/oci/tests/test_custom_compose_stacks.py b/oci/tests/test_custom_compose_stacks.py new file mode 100644 index 00000000..18ca3888 --- /dev/null +++ b/oci/tests/test_custom_compose_stacks.py @@ -0,0 +1,111 @@ +"""Pasted Compose files enter the same checked stack flow as catalog recipes.""" +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from proxmenux_oci import custom +from proxmenux_oci.converter import ConversionError +from proxmenux_oci.stack import ordered_services, resolve_environments, service_template + +COMPOSE = '''services: + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: supplied-password + volumes: [dbdata:/var/lib/postgresql/data] + app: + image: adminer:4 + ports: ['8080:8080'] + depends_on: [db] +volumes: + dbdata: +''' + + +class CustomComposeStacks(unittest.TestCase): + def test_stack_uses_existing_installer_and_preserves_versions_and_environment(self): + template = custom.template_from_compose(COMPOSE) + self.assertTrue(template['compatibility']['automatic_install_candidate']) + self.assertEqual(template['proxmox']['installer_profile']['stack_driver'], 'generic-multi-lxc-stack') + self.assertEqual(template['compose_stack']['main_service'], 'app') + services = ordered_services(template) + self.assertEqual([s['name'] for s in services], ['db', 'app']) + self.assertEqual([s['image'] for s in services], ['postgres:16', 'adminer:4']) + self.assertEqual(resolve_environments(services, 'UTC')['db']['POSTGRES_PASSWORD'], 'supplied-password') + self.assertEqual(service_template(template, services[0])['container_contract']['image']['reference'], 'postgres:16') + self.assertNotIn('supplied-password', custom.describe(template)) + + def test_stack_summary_hides_credentials_embedded_in_a_url(self): + document = COMPOSE.replace( + " ports:", + " environment: {DATABASE_URL: 'postgres://user:embedded-secret@db/app'}\n ports:") + template = custom.template_from_compose(document) + summary = custom.describe(template) + self.assertIn('DATABASE_URL', summary) + self.assertNotIn('embedded-secret', summary) + self.assertNotIn('postgres://user:', summary) + app = next(service for service in ordered_services(template) if service['name'] == 'app') + self.assertIn('embedded-secret', app['compose']['environment']['DATABASE_URL']) + + def test_invalid_stack_metadata_has_a_conversion_error(self): + for metadata in ('x-casaos: not-a-mapping', 'x-casaos: [app]', + 'x-casaos: {main: missing-service}'): + with self.subTest(metadata=metadata), self.assertRaises(ConversionError): + custom.template_from_compose(COMPOSE + metadata + '\n') + + def test_written_digest_and_tag_survive_in_full_image_contracts(self): + digest = 'sha256:' + 'a' * 64 + reference = 'ghcr.io/example/adminer:4@' + digest + template = custom.template_from_compose(COMPOSE.replace('adminer:4', reference)) + main_image = template['container_contract']['image'] + for image in (main_image, service_template(template, next( + service for service in ordered_services(template) if service['name'] == 'app'), + main=True)['container_contract']['image']): + self.assertEqual(image['reference'], reference) + self.assertEqual(image['registry'], 'ghcr.io') + self.assertEqual(image['repository'], 'ghcr.io/example/adminer') + self.assertEqual(image['tag'], '4') + self.assertEqual(image['digest'], digest) + self.assertEqual(image['pull_policy'], + 'resolve-written-reference-to-architecture-digest-at-install') + dependency = service_template(template, ordered_services(template)[0])['container_contract']['image'] + self.assertEqual(dependency['reference'], 'postgres:16') + self.assertEqual(dependency['repository'], 'postgres') + self.assertEqual(dependency['tag'], '16') + self.assertIsNone(dependency['digest']) + + def test_reject_unsafe_dependency_mount_or_build(self): + for value in (' volumes: [/var/run/docker.sock:/var/run/docker.sock]', + ' build: .'): + with self.subTest(value=value), self.assertRaises(ConversionError): + custom.template_from_compose(COMPOSE.replace(' volumes: [dbdata:/var/lib/postgresql/data]', value)) + + def test_shared_storage_is_supported_by_custom_stack(self): + template = custom.template_from_compose(COMPOSE.replace(" ports:", " volumes: [dbdata:/shared]\n ports:")) + self.assertTrue(template['compatibility']['automatic_install_candidate']) + self.assertEqual(template['compatibility']['untranslated_blockers'], []) + + def test_every_image_checked_before_install(self): + class UI: + def message(self, text, title): + self.text = text + ui = UI() + with patch.object(custom, 'read_definition', return_value=(COMPOSE, 'Compose')), \ + patch.object(custom, 'registry_report', side_effect=[(True, 'db ok'), (False, 'app missing')]) as report, \ + patch.object(custom, '_registry_refused', return_value=False), \ + patch.object(custom, 'install_template') as install: + custom.explore(ui) + self.assertEqual([c.args[0] for c in report.call_args_list], ['postgres:16', 'adminer:4']) + install.assert_not_called() + self.assertIn('app missing', ui.text) + + def test_single_image_still_uses_single_container_flow(self): + template = custom.template_from_compose('services: {app: {image: nginx:1.27}}') + self.assertNotEqual(template['proxmox'].get('installer_profile', {}).get('stack_driver'), 'generic-multi-lxc-stack') + self.assertEqual(template['container_contract']['image']['reference'], 'nginx:1.27') + + +if __name__ == '__main__': + unittest.main() diff --git a/oci/tests/test_custom_multiapp_stack.py b/oci/tests/test_custom_multiapp_stack.py new file mode 100644 index 00000000..1258201b --- /dev/null +++ b/oci/tests/test_custom_multiapp_stack.py @@ -0,0 +1,121 @@ +"""Custom stacks keep each app's endpoint and actual shared storage identity.""" +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from proxmenux_oci.casaos import convert_casaos_compose +from proxmenux_oci.stack import apply_stack_support, assess, build_stack, DefaultsUI, service_template +from proxmenux_oci.installer import DEFAULT_MODE, ADVANCED_MODE + +COMPOSE = '''name: arr-test +services: + sonarr: + image: lscr.io/linuxserver/sonarr:latest + ports: ['8989:8989'] + environment: {PUID: '1000', PGID: '1000'} + volumes: [sonarr-config:/config, shared-data:/data] + radarr: + image: lscr.io/linuxserver/radarr:latest + ports: ['7878:7878'] + environment: {PUID: '1000', PGID: '1000'} + volumes: [radarr-config:/config, shared-data:/data] +volumes: + sonarr-config: + radarr-config: + shared-data: +x-casaos: + main: sonarr + port_map: '8989' + scheme: http + index: / + title: {en_US: Arr Test} +''' + + +def template(custom=True, text=COMPOSE): + result = convert_casaos_compose(text, 'local', '', 'local/arr-test/docker-compose.yml', '') + result['proxmox']['custom_compose_stack'] = custom + apply_stack_support(result) + return result + + +class UI(DefaultsUI): + def __init__(self): + self.asks = [] + def ask(self, text, default=None, **kwargs): + self.asks.append(text) + return default or '' + + +class CustomMultiAppStackTests(unittest.TestCase): + def test_custom_opt_in_supports_two_web_apps_but_catalog_remains_guarded(self): + self.assertEqual(assess(template()), []) + self.assertTrue(assess(template(custom=False))) + + def test_app_without_own_port_is_rejected(self): + result = template(text=COMPOSE.replace(" ports: ['7878:7878']\n", '')) + self.assertTrue(any('health and persistence' in e for e in assess(result))) + + def test_service_image_contract_preserves_tag_and_digest(self): + parent = template() + service = parent['compose_stack']['services'][0] + for reference, repository, tag, digest in ( + ('ghcr.io/example/app:2.4', 'ghcr.io/example/app', '2.4', None), + ('ghcr.io/example/app:2.4@sha256:'+'a'*64, 'ghcr.io/example/app', '2.4', 'sha256:'+'a'*64), + ): + with self.subTest(reference=reference): + service['image'] = reference + service['compose']['image'] = reference + image = service_template(parent, service)['container_contract']['image'] + self.assertEqual((image['reference'], image['repository'], image['tag'], image['digest']), + (reference, repository, tag, digest)) + self.assertEqual(image['registry'], 'ghcr.io') + + def test_compose_extension_metadata_is_accepted_only_for_custom(self): + text = 'x-common-environment: &common {PUID: \'1000\'}\n' + COMPOSE + self.assertEqual(assess(template(text=text)), []) + self.assertTrue(assess(template(custom=False, text=text))) + + def test_shared_owners_must_match(self): + result = template(text=COMPOSE.replace("environment: {PUID: '1000', PGID: '1000'}", "environment: {PUID: '1001', PGID: '1000'}", 1)) + self.assertTrue(any('shared' in e for e in assess(result))) + + def test_advanced_addresses_are_kept_per_service(self): + ui = UI() + supplied = {'sonarr': '192.0.2.21/24', 'radarr': '192.0.2.22/24'} + with patch('proxmenux_oci.installer.ask_storage', return_value='local-zfs'), \ + patch('proxmenux_oci.installer.ask_bridge', return_value='vmbr0'), \ + patch('proxmenux_oci.host.timezone', return_value='UTC'), \ + patch('proxmenux_oci.custom_mounts.ask_stack_custom_mounts'), \ + patch('proxmenux_oci.extra_devices.ask_stack_extra_devices'), \ + patch('proxmenux_oci.network.ask_addresses', return_value=(supplied, '192.0.2.1')): + plan = build_stack(template(), ui, ADVANCED_MODE) + self.assertEqual({s['name']:s['frontend_ipv4'] for s in plan['services']}, supplied) + self.assertEqual(plan['network']['frontend_gateway'], '192.0.2.1') + + def test_build_assigns_distinct_endpoints_and_shared_bind_source(self): + ui = UI() + with patch('proxmenux_oci.installer.ask_storage', return_value='local-zfs'), \ + patch('proxmenux_oci.installer.ask_bridge', return_value='vmbr0'), \ + patch('proxmenux_oci.host.timezone', return_value='UTC'), \ + patch('proxmenux_oci.network.ask_addresses', side_effect=lambda ui, bridge, labels: ({name:'dhcp' for name in labels}, None)): + plan = build_stack(template(), ui, DEFAULT_MODE) + members = {s['name']:s for s in plan['services']} + self.assertEqual(members['sonarr']['healthcheck']['endpoint']['port'], 8989) + self.assertEqual(members['radarr']['healthcheck']['endpoint']['port'], 7878) + for member in members.values(): + self.assertTrue(member['frontend']) + self.assertEqual(member['frontend_ipv4'], 'dhcp') + binds = [next(m for m in s['deployment']['mounts'] if m['container_path']=='/data') for s in members.values()] + self.assertEqual(binds[0]['source'], binds[1]['source']) + self.assertEqual(binds[0]['host_owner'], [101000, 101000]) + self.assertTrue(all(m['type']=='host-bind' and m['backup'] is False for m in binds)) + self.assertEqual(len([text for text in ui.asks if text.endswith('shared-data')]), 1) + configs = [next(m for m in s['deployment']['mounts'] if m['container_path']=='/config') for s in members.values()] + self.assertTrue(all(m['type']=='managed-volume' for m in configs)) + + +if __name__ == '__main__': + unittest.main() diff --git a/web/messages/de/docs/oci-manager/custom-image.json b/web/messages/de/docs/oci-manager/custom-image.json index 225e88dd..86136676 100644 --- a/web/messages/de/docs/oci-manager/custom-image.json +++ b/web/messages/de/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Ein Dockerfile ohne veröffentlichtes Image: Das Image muss zuerst gebaut und in einer OCI-Registry veröffentlicht werden.", - "Eine Compose-Datei mit mehreren Diensten: Diese Option installiert jeweils einen Dienst in einem Container, auch wenn mehrere Dienste dasselbe Image verwenden.", + "Compose-Dateien mit mehreren Diensten verwenden einen nativen LXC pro Dienst. Jede Webanwendung benötigt einen eigenen veröffentlichten Port; Volumes mit derselben Quelle können ein Hostverzeichnis gemeinsam nutzen. Nicht unterstützte Optionen, externe Netzwerke und Dienste ohne unterstützte Zustandsprüfung verhindern weiterhin die Installation.", "configs, externe Geheimnisse oder Geräteformate, die sich nicht eindeutig umsetzen lassen.", "Shell-Ersetzungen wie $(command): Stattdessen muss der resultierende Wert angegeben werden.", "Nicht erkannte docker run-Optionen und --env-file: Verwenden Sie stattdessen eine Compose-Datei mit expliziten Werten." diff --git a/web/messages/en/docs/oci-manager/custom-image.json b/web/messages/en/docs/oci-manager/custom-image.json index 5ae7cb95..9c0b60b2 100644 --- a/web/messages/en/docs/oci-manager/custom-image.json +++ b/web/messages/en/docs/oci-manager/custom-image.json @@ -192,7 +192,7 @@ "list": { "items": [ "A Dockerfile without a published image: the image has to be built and published in an OCI registry first.", - "A Compose file with several services is blocked: this option installs one service in one container at a time, even when several services use the same image.", + "Multi-service Compose files use one native LXC per service. Each web application needs its own published endpoint; identical volume sources can share one host directory. Unsupported options, external networks and services without a supported health check still block installation.", "configs, external secrets or device formats that cannot be translated unambiguously.", "Shell substitutions such as $(command): the resulting value has to be written instead.", "docker run options that are not recognised, and --env-file: an explicit Compose file is read instead." diff --git a/web/messages/es/docs/oci-manager/custom-image.json b/web/messages/es/docs/oci-manager/custom-image.json index ed75b621..448a0b8e 100644 --- a/web/messages/es/docs/oci-manager/custom-image.json +++ b/web/messages/es/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Un Dockerfile sin imagen publicada: primero hay que construir la imagen y publicarla en un registro OCI.", - "Un archivo Compose con varios servicios: se bloquea porque esta opción instala un único servicio por contenedor, aunque varios servicios utilicen la misma imagen.", + "Los archivos Compose con varios servicios usan un LXC nativo por servicio. Cada aplicación web necesita su propio puerto publicado; los volúmenes con el mismo origen pueden compartir un directorio del host. Las opciones no compatibles, las redes externas y los servicios sin una comprobación de salud compatible siguen bloqueando la instalación.", "configs, secretos externos o formatos de dispositivo que no puedan adaptarse de forma inequívoca.", "Sustituciones de shell como $(command): hay que escribir directamente el valor resultante.", "Opciones de docker run no reconocidas y --env-file: utiliza un archivo Compose con los valores explícitos." diff --git a/web/messages/fr/docs/oci-manager/custom-image.json b/web/messages/fr/docs/oci-manager/custom-image.json index 6817b296..c78622d5 100644 --- a/web/messages/fr/docs/oci-manager/custom-image.json +++ b/web/messages/fr/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Un Dockerfile sans image publiée : l’image doit d’abord être construite et publiée dans un registre OCI.", - "Un fichier Compose à plusieurs services : cette option installe un seul service par conteneur, même si plusieurs services utilisent la même image.", + "Les fichiers Compose à plusieurs services utilisent un LXC natif par service. Chaque application web doit publier son propre port ; les volumes de même source peuvent partager un répertoire de l’hôte. Les options non prises en charge, les réseaux externes et les services sans contrôle de santé compatible bloquent toujours l’installation.", "configs, secrets externes ou formats de périphériques impossibles à adapter sans ambiguïté.", "Substitutions shell comme $(command) : il faut saisir directement la valeur résultante.", "Options docker run non reconnues et --env-file : utilisez plutôt un fichier Compose aux valeurs explicites." diff --git a/web/messages/it/docs/oci-manager/custom-image.json b/web/messages/it/docs/oci-manager/custom-image.json index 7d48a5b7..03777ba6 100644 --- a/web/messages/it/docs/oci-manager/custom-image.json +++ b/web/messages/it/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Un Dockerfile senza immagine pubblicata: prima occorre creare l’immagine e pubblicarla in un registro OCI.", - "Un file Compose con più servizi: viene bloccato perché questa opzione installa un solo servizio per container, anche se più servizi usano la stessa immagine.", + "I file Compose con più servizi usano un LXC nativo per servizio. Ogni applicazione web deve pubblicare la propria porta; i volumi con la stessa origine possono condividere una directory dell’host. Le opzioni non supportate, le reti esterne e i servizi senza un controllo di integrità supportato continuano a bloccare l’installazione.", "configs, segreti esterni o formati di dispositivi che non possono essere adattati senza ambiguità.", "Sostituzioni della shell come $(command): occorre inserire direttamente il valore risultante.", "Opzioni docker run non riconosciute e --env-file: usa invece un file Compose con valori espliciti." diff --git a/web/messages/pt/docs/oci-manager/custom-image.json b/web/messages/pt/docs/oci-manager/custom-image.json index d8fc1551..93df9a37 100644 --- a/web/messages/pt/docs/oci-manager/custom-image.json +++ b/web/messages/pt/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Um Dockerfile sem imagem publicada: primeiro é necessário construir a imagem e publicá-la num registo OCI.", - "Um ficheiro Compose com vários serviços: é bloqueado porque esta opção instala um único serviço por contentor, mesmo que vários serviços utilizem a mesma imagem.", + "Os ficheiros Compose com vários serviços usam um LXC nativo por serviço. Cada aplicação web precisa da sua própria porta publicada; os volumes com a mesma origem podem partilhar um diretório do anfitrião. As opções não suportadas, as redes externas e os serviços sem uma verificação de saúde compatível continuam a bloquear a instalação.", "configs, segredos externos ou formatos de dispositivo que não possam ser adaptados sem ambiguidade.", "Substituições de shell como $(command): é necessário escrever diretamente o valor resultante.", "Opções docker run não reconhecidas e --env-file: utilize antes um ficheiro Compose com valores explícitos." diff --git a/web/messages/sk/docs/oci-manager/custom-image.json b/web/messages/sk/docs/oci-manager/custom-image.json index 7122dedf..b90a1530 100644 --- a/web/messages/sk/docs/oci-manager/custom-image.json +++ b/web/messages/sk/docs/oci-manager/custom-image.json @@ -265,7 +265,7 @@ "list": { "items": [ "Dockerfile bez publikovaného obrazu: obraz sa musí najprv zostaviť a zverejniť v registri OCI.", - "Súbor Compose s viacerými službami sa zablokuje, pretože táto voľba inštaluje iba jednu službu do jedného kontajnera, aj keď viaceré služby používajú rovnaký obraz.", + "Súbory Compose s viacerými službami používajú samostatný natívny LXC pre každú službu. Každá webová aplikácia potrebuje vlastný publikovaný port; zväzky s rovnakým zdrojom môžu zdieľať adresár hostiteľa. Nepodporované voľby, externé siete a služby bez podporovanej kontroly stavu naďalej blokujú inštaláciu.", "configs, externé tajné údaje alebo formáty zariadení, ktoré nemožno jednoznačne prispôsobiť.", "Nahradenia shellu, napríklad $(command): namiesto nich treba zapísať výslednú hodnotu.", "Nerozpoznané voľby docker run a --env-file: namiesto nich použite súbor Compose s výslovne uvedenými hodnotami."