feat(oci): VLAN tag and updates that keep Proxmox changes

oci/remote/haos_healthcheck.py

@@ -11,6 +11,7 @@ import traceback
 import urllib.error
 import urllib.request
 
+from oci_installation_state import container_curl
 from oci_ui import log, msg_progress, translate
 
 
@@ -79,7 +80,14 @@ def containers_ready(items):
             and bool(by_name['homeassistant'].get('Config', {}).get('Image')))
 
 
-def http_ready(url, timeout=5):
+def http_ready(vmid, url, timeout=5):
+    curl = container_curl(vmid, lambda *args: capture(args).encode())
+    if curl != ['curl']:
+        try:
+            return capture([*curl, '-sS', '--noproxy', '*', '-o', '/dev/null', '-w', '%{http_code}',
+                            '--max-time', str(timeout), url], timeout=timeout + 1).strip() == '200'
+        except subprocess.SubprocessError:
+            return False
     # Do not route private healthchecks through an environment HTTP proxy.
     opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
     try:
@@ -111,10 +119,10 @@ def probe(vmid, ip, remaining):
     supervisor = json.loads(run(docker + ['exec', 'hassio_cli', 'ha', '--raw-json', 'supervisor', 'info']))
     if not supervisor_ready(supervisor):
         raise Pending(translate('Supervisor does not confirm healthy and supported yet'))
-    if not http_ready(f'http://{ip}:4357/', max(0.1, min(5, remaining()))):
+    if not http_ready(vmid, f'http://{ip}:4357/', max(0.1, min(5, remaining()))):
         raise Pending(translate('Observer is not responding on port 4357'))
     for port in (8123, 80):
-        if http_ready(f'http://{ip}:{port}/', max(0.1, min(5, remaining()))):
+        if http_ready(vmid, f'http://{ip}:{port}/', max(0.1, min(5, remaining()))):
             return [{'label': 'Home Assistant', 'url': f'http://{ip}:{port}/'},
                     {'label': 'Home Assistant Observer', 'url': f'http://{ip}:4357/'}]
     raise Pending(translate('Core is running, but not responding over HTTP on 80/8123'))

oci/remote/install_generic_stack.py

@@ -662,6 +662,16 @@ def attach_mounts(service, temporary):
     oci_nested_mounts.follow(vmid)
 
 
+def web_answers(vmid, url, opener):
+    from oci_installation_state import container_curl
+    curl = container_curl(vmid, lambda *args: run(*args).encode())
+    if curl != ['curl']:
+        run(*curl, '-fsS', '--noproxy', '*', '-o', '/dev/null', '--max-time', '4', url)
+        return True
+    with opener.open(url,timeout=4) as response:
+        return response.status<400
+
+
 def wait_web(primary, url):
     deadline = time.monotonic()+primary['healthcheck']['timeout_seconds']
     opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
@@ -669,16 +679,15 @@ def wait_web(primary, url):
         if run('pct','status',primary['vmid']).strip()!='status: running':
             raise RuntimeError(f"{translate('The container stopped:')} {primary['name']} (CT {primary['vmid']})")
         try:
-            with opener.open(url,timeout=4) as response:
-                if response.status<400:
-                    check = primary['healthcheck']
-                    if check.get('type') == 'exec':
-                        remaining = max(1, deadline-time.monotonic())
-                        process = subprocess.run(['pct', 'exec', str(primary['vmid']), '--', *check['argv']],
-                                                 capture_output=True, timeout=min(30, remaining), check=False)
-                        if process.returncode != 0:
-                            raise RuntimeError('Health check has not succeeded yet')
-                    return
+            if web_answers(primary['vmid'], url, opener):
+                check = primary['healthcheck']
+                if check.get('type') == 'exec':
+                    remaining = max(1, deadline-time.monotonic())
+                    process = subprocess.run(['pct', 'exec', str(primary['vmid']), '--', *check['argv']],
+                                             capture_output=True, timeout=min(30, remaining), check=False)
+                    if process.returncode != 0:
+                        raise RuntimeError('Health check has not succeeded yet')
+                return
         except Exception:
             pass
         if time.monotonic()>=deadline:
@@ -1050,7 +1059,9 @@ def main():
                     run('pct','set',s['vmid'],'--startup',f'order={order*10},up=5,down=30')
                 if s['main'] or s.get('frontend'):
                     address = access_address(s.get('frontend_ipv4') or 'dhcp', deployment['network'].get('frontend_gateway'))
-                    run('pct','set',s['vmid'],'--net1',f"name=eth1,bridge={deployment['network']['frontend_bridge']},{address},host-managed=1,firewall=1,type=veth")
+                    vlan_tag = deployment['network'].get('frontend_vlan_tag')
+                    tag = f",tag={int(vlan_tag)}" if vlan_tag else ''
+                    run('pct','set',s['vmid'],'--net1',f"name=eth1,bridge={deployment['network']['frontend_bridge']},{address}{tag},host-managed=1,firewall=1,type=veth")
                 assign_health_address(s)
             # Existing CT/network configurations now reserve every allocated ID and subnet.
             # Never retain the allocation lock while startup/hooks wait for another registry operation.

oci/remote/install_immich_stack.sh

@@ -140,10 +140,11 @@ ML_FRONTEND_IPV4=$(jq -r '.network.machine_learning_frontend_ipv4 // "dhcp"' "$D
 FRONTEND_GATEWAY=$(jq -r '.network.frontend_gateway // empty' "$DEPLOYMENT_FILE")
 oci_access_net "$FRONTEND_IPV4" "$FRONTEND_GATEWAY" \
   || die "$(translate "Invalid access address:") $FRONTEND_IPV4 $FRONTEND_GATEWAY"
-FRONTEND_NET=$OCI_ACCESS_NET
+oci_vlan_option frontend_vlan_tag || die "$(translate "The VLAN tag must be a number from 1 to 4094")"
+FRONTEND_NET=$OCI_ACCESS_NET$OCI_VLAN_OPTION
 oci_access_net "$ML_FRONTEND_IPV4" "$FRONTEND_GATEWAY" \
   || die "$(translate "Invalid access address:") $ML_FRONTEND_IPV4 $FRONTEND_GATEWAY"
-ML_FRONTEND_NET=$OCI_ACCESS_NET
+ML_FRONTEND_NET=$OCI_ACCESS_NET$OCI_VLAN_OPTION
 PRIVATE_BRIDGE=$(jqr '.network.private_bridge')
 PRIVATE_SUBNET=$(jqr '.network.private_subnet')
 PRIVATE_HOST_ADDRESS=$(jqr '.network.private_host_address')

oci/remote/install_nextcloud_stack.sh

@@ -132,7 +132,8 @@ FRONTEND_IPV4=$(jqr '.network.frontend_ipv4')
 FRONTEND_GATEWAY=$(jq -r '.network.frontend_gateway // empty' "$DEPLOYMENT_FILE")
 oci_access_net "$FRONTEND_IPV4" "$FRONTEND_GATEWAY" \
   || die "$(translate "Invalid access address:") $FRONTEND_IPV4 $FRONTEND_GATEWAY"
-FRONTEND_NET=$OCI_ACCESS_NET
+oci_vlan_option frontend_vlan_tag || die "$(translate "The VLAN tag must be a number from 1 to 4094")"
+FRONTEND_NET=$OCI_ACCESS_NET$OCI_VLAN_OPTION
 PRIVATE_BRIDGE=$(jqr '.network.private_bridge')
 PRIVATE_SUBNET=$(jqr '.network.private_subnet')
 PRIVATE_HOST_ADDRESS=$(jqr '.network.private_host_address')
@@ -511,7 +512,7 @@ if (( START_AFTER == 1 )); then
       | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \
       | grep -vFx "$APPLICATION_IP" | head -n 1 || true)
     if [[ -n $APPLICATION_LAN_IP ]] \
-      && curl -fsS -H 'Host: localhost' \
+      && oci_curl "$APPLICATION_ID" -fsS -H 'Host: localhost' \
         "http://${APPLICATION_LAN_IP}/status.php" >/dev/null 2>&1; then
       break
     fi
@@ -521,7 +522,7 @@ if (( START_AFTER == 1 )); then
     || die "$(translate "The application did not get an address on the access network:") Nextcloud"
   # Nextcloud 34 validates status.php against trusted_domains before the LAN IP
   # can be registered. localhost is the official image's initial trusted host.
-  STATUS_JSON=$(curl -fsS -H 'Host: localhost' \
+  STATUS_JSON=$(oci_curl "$APPLICATION_ID" -fsS -H 'Host: localhost' \
     "http://${APPLICATION_LAN_IP}/status.php")
   jq -e '.installed == true and .maintenance == false and .needsDbUpgrade == false' \
     <<<"$STATUS_JSON" >/dev/null \

oci/remote/install_oci.sh

@@ -361,10 +361,10 @@ cleanup_canonical_inputs() {
 healthcheck_probe() {
   local code status=0
   if [[ ${HC_ANY_STATUS:-false} == true ]]; then
-    code=$(curl "${CURL_ARGS[@]}" "$HC_URL" 2>/dev/null) || status=$?
+    code=$(oci_curl "$VMID" "${CURL_ARGS[@]}" "$HC_URL" 2>/dev/null) || status=$?
     (( status == 0 )) && [[ $code =~ ^[1-4][0-9][0-9]$ ]] && return 0
   else
-    curl "${CURL_ARGS[@]}" "$HC_URL" >/dev/null 2>&1 || status=$?
+    oci_curl "$VMID" "${CURL_ARGS[@]}" "$HC_URL" >/dev/null 2>&1 || status=$?
     (( status == 0 )) && return 0
   fi
   # A port taken from the compose file may serve a database or SSH: an answer
@@ -1044,12 +1044,13 @@ apply_extra_hosts() {
 }
 
 # Settings an update gives back to the rebuilt container before it starts:
-# the LAN leg of a suite or stack member, its start order and current tags.
+# the LAN leg of a suite or stack member, its start order and current tags,
+# and what the user set in Proxmox.
 apply_kept_settings() {
   local key value
   while IFS=$'\t' read -r key value; do
     [[ -n $key ]] || continue
-    [[ $key == net1 || $key == startup || $key == tags ]] \
+    [[ $key =~ ^(net[01]|startup|tags|features|nameserver|searchdomain|timezone|cpulimit|cmode|console|tty|hookscript|entrypoint)$ ]] \
       || die "$(translate "Unsupported kept setting:") $key"
     if [[ $key == tags && -z $value ]]; then
       oci_quiet pct set "$VMID" --delete tags
@@ -1057,6 +1058,13 @@ apply_kept_settings() {
       oci_quiet pct set "$VMID" "--$key" "$value"
     fi || die "$(translate "Could not restore the container setting:") $key"
   done < <(jq -r '.kept_proxmox_settings // {} | to_entries[] | [.key, .value] | @tsv' "$DEPLOYMENT_FILE")
+  while IFS= read -r value; do
+    [[ -n $value ]] || continue
+    [[ $value =~ ^lxc\.[A-Za-z0-9._-]+:\  ]] \
+      || die "$(translate "Unsupported kept setting:") $value"
+    printf '%s\n' "$value" >>"$CONF" \
+      || die "$(translate "Could not restore the container setting:") $value"
+  done < <(jq -r '.custom_lxc // [] | .[]' "$DEPLOYMENT_FILE")
 }
 
 apply_runtime_user() {
@@ -1714,6 +1722,8 @@ NET="name=eth0,bridge=${BRIDGE},firewall=${FIREWALL},host-managed=1,ip=${IPV4},t
 if [[ -n $GATEWAY ]]; then
   NET="${NET},gw=${GATEWAY}"
 fi
+oci_vlan_option vlan_tag || die "$(translate "The VLAN tag must be a number from 1 to 4094")"
+NET="${NET}${OCI_VLAN_OPTION}"
 if [[ -n $MAC_ADDRESS ]]; then
   [[ $MAC_ADDRESS =~ ^[0-9A-Fa-f]{2}(:[0-9A-Fa-f]{2}){5}$ ]] \
     || die "$(translate "Invalid MAC address:") $MAC_ADDRESS"

oci/remote/install_paperless_stack.sh

@@ -132,7 +132,8 @@ FRONTEND_IPV4=$(jqr '.network.frontend_ipv4')
 FRONTEND_GATEWAY=$(jq -r '.network.frontend_gateway // empty' "$DEPLOYMENT_FILE")
 oci_access_net "$FRONTEND_IPV4" "$FRONTEND_GATEWAY" \
   || die "$(translate "Invalid access address:") $FRONTEND_IPV4 $FRONTEND_GATEWAY"
-FRONTEND_NET=$OCI_ACCESS_NET
+oci_vlan_option frontend_vlan_tag || die "$(translate "The VLAN tag must be a number from 1 to 4094")"
+FRONTEND_NET=$OCI_ACCESS_NET$OCI_VLAN_OPTION
 PRIVATE_BRIDGE=$(jqr '.network.private_bridge')
 PRIVATE_SUBNET=$(jqr '.network.private_subnet')
 PRIVATE_HOST_ADDRESS=$(jqr '.network.private_host_address')
@@ -513,7 +514,7 @@ if (( START_AFTER == 1 )); then
       | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \
       | grep -vFx "$APPLICATION_IP" | head -n 1 || true)
     if [[ -n $APPLICATION_LAN_IP ]] \
-      && curl -fsS "http://${APPLICATION_LAN_IP}:8000/" >/dev/null 2>&1; then
+      && oci_curl "$APPLICATION_ID" -fsS "http://${APPLICATION_LAN_IP}:8000/" >/dev/null 2>&1; then
       break
     fi
     if [[ $(pct status "$APPLICATION_ID" 2>/dev/null) != *running* ]]; then
@@ -523,7 +524,7 @@ if (( START_AFTER == 1 )); then
   done
   [[ -n $APPLICATION_LAN_IP ]] \
     || die "$(translate "The application did not get an address on the access network:") Paperless-ngx"
-  curl -fsS "http://${APPLICATION_LAN_IP}:8000/" >/dev/null 2>>"$OCI_LOG" \
+  oci_curl "$APPLICATION_ID" -fsS "http://${APPLICATION_LAN_IP}:8000/" >/dev/null 2>>"$OCI_LOG" \
     || die "$(translate "The application did not pass its HTTP check:") Paperless-ngx"
   msg_ok "$(translate "Application responding:") http://${APPLICATION_LAN_IP}:8000/"
 fi

oci/remote/install_tandoor_stack.sh

@@ -145,7 +145,8 @@ FRONTEND_IPV4=$(jqr '.network.frontend_ipv4')
 FRONTEND_GATEWAY=$(jq -r '.network.frontend_gateway // empty' "$DEPLOYMENT_FILE")
 oci_access_net "$FRONTEND_IPV4" "$FRONTEND_GATEWAY" \
   || die "$(translate "Invalid access address:") $FRONTEND_IPV4 $FRONTEND_GATEWAY"
-FRONTEND_NET=$OCI_ACCESS_NET
+oci_vlan_option frontend_vlan_tag || die "$(translate "The VLAN tag must be a number from 1 to 4094")"
+FRONTEND_NET=$OCI_ACCESS_NET$OCI_VLAN_OPTION
 PRIVATE_BRIDGE=$(jqr '.network.private_bridge')
 PRIVATE_SUBNET=$(jqr '.network.private_subnet')
 PRIVATE_HOST_ADDRESS=$(jqr '.network.private_host_address')
@@ -475,7 +476,7 @@ if (( START_AFTER == 1 )); then
       | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' \
       | grep -vFx "$APPLICATION_IP" | head -n 1 || true)
     if [[ -n $APPLICATION_LAN_IP ]] \
-      && curl -fsS "http://${APPLICATION_LAN_IP}/" >/dev/null 2>&1; then
+      && oci_curl "$APPLICATION_ID" -fsS "http://${APPLICATION_LAN_IP}/" >/dev/null 2>&1; then
       break
     fi
     if [[ $(pct status "$APPLICATION_ID" 2>/dev/null) != *running* ]]; then
@@ -488,7 +489,7 @@ if (( START_AFTER == 1 )); then
     print_first_boot_diagnostics "$APPLICATION_ID" tandoor
     die "$(translate "The application did not get an address on the access network:") Tandoor"
   fi
-  curl -fsS "http://${APPLICATION_LAN_IP}/" >/dev/null 2>>"$OCI_LOG" \
+  oci_curl "$APPLICATION_ID" -fsS "http://${APPLICATION_LAN_IP}/" >/dev/null 2>>"$OCI_LOG" \
     || { print_first_boot_diagnostics "$APPLICATION_ID" tandoor; \
       die "$(translate "The application did not pass its HTTP check:") Tandoor"; }
   msg_ok "$(translate "Application responding:") http://${APPLICATION_LAN_IP}/"

oci/remote/oci_installation_state.py

@@ -136,6 +136,16 @@ def parse_config(data):
     return values
 
 
+def container_curl(vmid, run=command):
+    """curl for an address of this container. The host may have no route to the
+    VLAN of an interface: there it runs inside the network of the container."""
+    config = parse_config(run('pct', 'config', str(vmid)))
+    if not any(re.fullmatch(r'net[0-9]+', key) and re.search(r'(^|,)tag=', value) for key, value in config.items()):
+        return ['curl']
+    pid = run('lxc-info', '-n', str(vmid), '-pH').decode().strip()
+    return ['nsenter', f'--net=/proc/{pid}/ns/net', 'curl']
+
+
 def private_directory(path):
     path.mkdir(parents=True, exist_ok=True, mode=0o700)
     if path.is_symlink() or path.stat().st_uid != os.geteuid():

oci/remote/oci_instance_reconcile.py

@@ -94,15 +94,11 @@ ENVIRONMENT_KEYS = ('env', 'lxc.environment.runtime')
 
 def _environment(record, config):
     """Variables changed in Proxmox since the last operation. Values are
-    applied but never listed: they may be passwords."""
+    applied but never listed: they may be passwords. A variable the image
+    defines and the user removed comes back with its value, as in Docker."""
     before = native_modify.environment(record['observed']['config'].encode())
     now = native_modify.environment(config)
-    defaults = {item.split('=', 1)[0] for item in record['observed']['image'].get('defaults', {}).get('Env', [])
-                if '=' in item}
     removed = sorted(before.keys() - now.keys())
-    # An image default comes back with a new container; its absence cannot be kept.
-    if any(name in defaults for name in removed):
-        raise ValueError(f"{translate('These manual changes cannot be adopted safely:')} env")
     changed = sorted(name for name in now if before.get(name) != now[name])
     return {name: now[name] for name in changed}, removed
 
@@ -126,14 +122,30 @@ def _adopt_environment(deployment, values, removed):
     return details
 
 
-def propose(record, config):
-    if (record.get('status') != 'installed' or record.get('stack') or record.get('stack_member')
-            or record.get('native_stack_intent') or record.get('deployment', {}).get('stack_managed')):
+def _storage_change(key, value, deployment, vmid):
+    """A disk resized or moved in Proxmox: the record follows it."""
+    if key == 'rootfs':
+        storage = value.split(':', 1)[0]
+        size = dict(part.split('=', 1) for part in value.split(',')[1:] if '=' in part).get('size')
+        deployment['rootfs'].update(storage=storage, size_gb=_managed_size(size))
+        return
+    current = _mount(key, value, vmid)
+    for mount in deployment.get('mounts', []):
+        if mount.get('container_path') == current['container_path']:
+            mount.update({field: current[field] for field in ('source', 'size_gb', 'read_only')})
+
+
+def propose(record, config, member=False):
+    if (record.get('status') != 'installed' or (not member and (
+            record.get('stack') or record.get('stack_member') or record.get('native_stack_intent')
+            or record.get('deployment', {}).get('stack_managed')))):
         raise ValueError(translate('Only an installed standalone OCI instance can adopt external changes'))
     if instances.identity(config) != record['installation_id']:
         raise ValueError(translate('The container identity changed; nothing was adopted'))
-    before = parse_config(record['observed']['config'].encode())
-    current = parse_config(config)
+    if transaction.nested_mounts.stale(config):
+        transaction.nested_mounts.follow(record['vmid'])
+        config = instances.command('pct', 'config', str(record['vmid']))
+    before, current = transaction.settings(record['observed']['config'].encode()), transaction.settings(config)
     changed = sorted(key for key in before.keys() | current.keys() if before.get(key) != current.get(key))
     # Notes do not describe a mount, device or runtime setting. The OCI marker
     # was checked above, so a presentation-only change needs no adoption.
@@ -143,16 +155,41 @@ def propose(record, config):
     if any(key in ENVIRONMENT_KEYS for key in changed):
         changed = [key for key in changed if key not in ENVIRONMENT_KEYS]
         environment = _environment(record, config)
+    custom = transaction.custom_lxc(record, config)
+    custom_keys = {line.split(': ', 1)[0] for line in (*custom, *record['deployment'].get('custom_lxc', []))}
+    custom_changed = sorted(custom) != sorted(record['deployment'].get('custom_lxc', []))
+    changed = [key for key in changed if key not in custom_keys]
     new_keys = [key for key in changed if key not in before and re.fullmatch(r'(mp|dev)[0-9]+', key)]
-    unsupported = [key for key in changed if key not in new_keys and key not in transaction.ADOPTABLE
-                   and key not in transaction.KEPT_AS_IS]
+    storage = [key for key in changed if key in current and key in before and re.fullmatch(r'rootfs|mp[0-9]+', key)]
+    detached = [key for key in changed if key not in current and (
+        re.fullmatch(r'dev[0-9]+', key) or (re.fullmatch(r'mp[0-9]+', key) and before[key].startswith('/')))]
+    unsupported = [key for key in changed if key not in new_keys and key not in storage and key not in detached
+                   and key != 'net0'
+                   and key not in transaction.ADOPTABLE and key not in transaction.KEPT_AS_IS]
     if unsupported:
         raise ValueError(f"{translate('These manual changes cannot be adopted safely:')} {', '.join(unsupported)}")
-    if not new_keys and environment is None:
+    if not member and not (new_keys or storage or detached or custom_changed or environment is not None
+                           or {'net0', 'hostname', 'entrypoint'} & set(changed)):
         return None
     candidate = copy.deepcopy(record)
     deployment = candidate['deployment']
+    if custom:
+        deployment['custom_lxc'] = custom
+    else:
+        deployment.pop('custom_lxc', None)
     details = _adopt_environment(deployment, *environment) if environment else []
+    for key in storage:
+        _storage_change(key, current[key], deployment, record['vmid'])
+    # A host directory or a device taken away in Proxmox holds no data of the container.
+    for key in detached:
+        fields = dict(part.split('=', 1) for part in before[key].split(',') if '=' in part)
+        if key.startswith('mp'):
+            deployment['mounts'] = [mount for mount in deployment.get('mounts', [])
+                                    if mount.get('container_path') != fields.get('mp')]
+        else:
+            path = fields.get('path') or before[key].split(',', 1)[0]
+            deployment['devices'] = [device for device in deployment.get('devices', [])
+                                     if device.get('host_path') != path]
     for key in new_keys:
         if key.startswith('mp'):
             mount = _mount(key, current[key], record['vmid'])
@@ -164,12 +201,14 @@ def propose(record, config):
             deployment.setdefault('devices', []).append(device)
             kind = 'GPU' if gpu_devices.gpu_path(device['host_path']) else 'USB/Coral/NPU'
             details.append(f"{key}: {device['host_path']} ({kind})")
-    skipped = [*new_keys, *(ENVIRONMENT_KEYS if environment else ())]
+    skipped = [*new_keys, *storage, *detached, *custom_keys, *(ENVIRONMENT_KEYS if environment else ())]
     filtered = b'\n'.join(line for line in config.splitlines()
                           if not any(line.startswith(key.encode() + b': ') for key in skipped))
-    if environment:
+    if environment or storage or detached or custom_keys:
         filtered += b''.join(b'\n' + line for line in record['observed']['config'].encode().splitlines()
-                             if any(line.startswith(key.encode() + b': ') for key in ENVIRONMENT_KEYS))
+                             if any(line.startswith(key.encode() + b': ')
+                                    for key in (*storage, *detached, *custom_keys,
+                                                *(ENVIRONMENT_KEYS if environment else ()))))
     filtered += b'\n'
     for key, value in transaction.external_changes(record, filtered).items():
         section, name = transaction.ADOPTABLE[key]
@@ -179,31 +218,36 @@ def propose(record, config):
     candidate['observed']['config_sha256'] = sha(config)
     candidate['observed']['gpu_devices'] = gpu_devices.capture(config)
     candidate['observed']['host_bind_sources'] = host_mounts.capture_sources(config)
-    transaction.preflight(candidate, copy.deepcopy(candidate), config)
-    return {'candidate': candidate, 'details': details,
+    transaction.preflight(candidate, copy.deepcopy(candidate), config, coordinated=member)
+    # A disk or a device is reviewed; the rest is what the user already decided.
+    return {'candidate': candidate, 'details': details, 'review': bool(new_keys),
             'previous_record_hash': sha(instances.location(instances.ROOT, record['vmid']).read_bytes()),
             'config_sha256': sha(config)}
 
 
 def commit(root, vmid, proposal):
     with instances.locked(root):
-        record = instances.read(root, vmid)
-        path = instances.location(root, vmid)
-        if sha(path.read_bytes()) != proposal['previous_record_hash']:
-            raise ValueError(translate('The OCI contract changed while reviewing; nothing was adopted'))
-        config = instances.command('pct', 'config', str(vmid))
-        if sha(config) != proposal['config_sha256']:
-            raise ValueError(translate('The LXC changed while reviewing; nothing was adopted'))
-        checked = propose(record, config)
-        if checked is None or checked['details'] != proposal['details']:
-            raise ValueError(translate('The proposed changes no longer match; nothing was adopted'))
-        candidate = checked['candidate']
-        observed = instances.observe(vmid, record['installation_id'], record['observed']['archive_path'],
-                                     record['observed']['resolved_registry_digest'], record['observed']['image'])
-        if observed['config_sha256'] != proposal['config_sha256']:
-            raise ValueError(translate('The LXC changed during verification; nothing was adopted'))
-        candidate['observed'] = observed
-        history = path.parent / 'history'
-        instances.write(history / f'before-reconciliation-{uuid.uuid4().hex}.json', record)
-        instances.write(path, candidate)
-        return candidate
+        return commit_locked(root, vmid, proposal)
+
+
+def commit_locked(root, vmid, proposal, member=False):
+    record = instances.read(root, vmid)
+    path = instances.location(root, vmid)
+    if sha(path.read_bytes()) != proposal['previous_record_hash']:
+        raise ValueError(translate('The OCI contract changed while reviewing; nothing was adopted'))
+    config = instances.command('pct', 'config', str(vmid))
+    if sha(config) != proposal['config_sha256']:
+        raise ValueError(translate('The LXC changed while reviewing; nothing was adopted'))
+    checked = propose(record, config, member)
+    if checked is None or checked['details'] != proposal['details']:
+        raise ValueError(translate('The proposed changes no longer match; nothing was adopted'))
+    candidate = checked['candidate']
+    observed = instances.observe(vmid, record['installation_id'], record['observed']['archive_path'],
+                                 record['observed']['resolved_registry_digest'], record['observed']['image'])
+    if observed['config_sha256'] != proposal['config_sha256']:
+        raise ValueError(translate('The LXC changed during verification; nothing was adopted'))
+    candidate['observed'] = observed
+    history = path.parent / 'history'
+    instances.write(history / f'before-reconciliation-{uuid.uuid4().hex}.json', record)
+    instances.write(path, candidate)
+    return candidate

oci/remote/oci_instance_transaction.py

@@ -27,7 +27,7 @@ import uuid
 from urllib.parse import unquote
 
 import oci_instances as instances
-from oci_installation_state import image_from_archive, parse_config, private_directory, sha
+from oci_installation_state import container_curl, image_from_archive, parse_config, private_directory, sha
 from verify_oci_archive import verify_archive
 import oci_host_mounts as host_mounts
 import oci_nested_mounts as nested_mounts
@@ -484,11 +484,78 @@ def candidate_contract(record, operation, proposal=None):
 
 
 # Proxmox settings the rebuilt container gets back exactly as they are: the
-# LAN leg ProxMenux adds to a suite or stack member, the start order and tags.
-KEPT_AS_IS = ('net1', 'startup', 'tags')
+# LAN leg ProxMenux adds to a suite or stack member, the start order, the tags
+# and what the user set there. ProxMenux does not decide any of them.
+KEPT_AS_IS = ('net1', 'startup', 'tags', 'features', 'nameserver', 'searchdomain', 'timezone',
+              'cpulimit', 'cmode', 'console', 'tty', 'hookscript', 'protection')
 ADOPTABLE = {'memory': ('resources', 'memory_mb'), 'swap': ('resources', 'swap_mb'),
              'cores': ('resources', 'cores'), 'cpulimit': ('resources', 'cores'),
-             'cpuunits': ('resources', 'cpu_units'), 'onboot': (None, 'onboot')}
+             'cpuunits': ('resources', 'cpu_units'), 'onboot': (None, 'onboot'),
+             'hostname': (None, 'hostname'), 'entrypoint': (None, 'custom_entrypoint'),
+             'bridge': ('network', 'bridge'), 'ipv4': ('network', 'ipv4'),
+             'gateway': ('network', 'gateway'), 'firewall': ('network', 'firewall'),
+             'vlan_tag': ('network', 'vlan_tag')}
+
+
+def network_fields(net):
+    """What a network line says that the record of an application keeps."""
+    fields = dict(item.split('=', 1) for item in (net or '').split(',') if '=' in item)
+    tag = fields.get('tag')
+    return {'bridge': fields.get('bridge'), 'ipv4': fields.get('ip'), 'gateway': fields.get('gw'),
+            'firewall': fields.get('firewall', '0') == '1', 'vlan_tag': int(tag) if tag else None}
+
+
+# What the engine writes itself among the lxc.* lines of a container.
+ENGINE_LXC = ('lxc.environment', 'lxc.init.', 'lxc.signal.', 'lxc.console.', 'lxc.hook.', 'lxc.include',
+              'lxc.idmap', 'lxc.mount.auto')
+
+
+def without_backup_snapshot(config):
+    """The configuration without the mark of a snapshot a backup left behind:
+    it is not a setting, and a snapshot of the user is not this one."""
+    return b''.join(line for line in config.splitlines(keepends=True) if line.strip() != b'parent: vzdump')
+
+
+def drop_backup_snapshot(vmid):
+    """A disk with a snapshot cannot be moved. The one an interrupted backup
+    left goes the way Proxmox removes it before its next backup."""
+    config = run('pct', 'config', str(vmid))
+    if without_backup_snapshot(config) != config and 'lock' not in parse_config(config):
+        run('pct', 'delsnapshot', str(vmid), 'vzdump', '--force', '1')
+
+
+def settings(config):
+    """The settings of a configuration to compare with those saved: without
+    that mark, and without the lines that follow the mount points."""
+    return parse_config(nested_mounts.without(without_backup_snapshot(config)))
+
+
+def custom_lxc(record, config):
+    """The raw lxc.* lines the user added in Proxmox: the ones saved before that
+    are still there, and the new ones. A rebuilt container gets them back."""
+    before = record['observed']['config'].splitlines()
+    saved = record['deployment'].get('custom_lxc', [])
+    lines = config.decode().splitlines()
+    return [line for line in lines if line.startswith('lxc.') and ': ' in line
+            and not line.startswith(ENGINE_LXC) and not nested_mounts.own(line)
+            and (line in saved or line not in before)]
+
+
+def kept_settings(record, candidate, cfg):
+    """What a rebuilt container takes from the one it replaces. The network and
+    the features go back as they are unless Modify has just changed them.
+    Protection comes back at the end: a protected container cannot be replaced."""
+    kept = {key: cfg[key] for key in KEPT_AS_IS if key in cfg and key != 'protection'}
+    if 'cores' not in cfg:
+        kept.pop('cpulimit', None)
+    if candidate['deployment'].get('security') != record['deployment'].get('security'):
+        kept.pop('features', None)
+    if 'net0' in cfg and candidate['deployment'].get('network') == record['deployment'].get('network'):
+        kept['net0'] = cfg['net0']
+    # Only a command the user set: the one of the image changes with the image.
+    if candidate['deployment'].get('custom_entrypoint') and 'entrypoint' in cfg:
+        kept['entrypoint'] = cfg['entrypoint']
+    return kept
 
 
 def external_changes(record, config, adopt=True):
@@ -496,7 +563,7 @@ def external_changes(record, config, adopt=True):
     values. Any change the new container cannot keep is refused."""
     if sha(config) == record['observed']['config_sha256']:
         return {}
-    before, now = parse_config(record['observed']['config'].encode()), parse_config(config)
+    before, now = settings(record['observed']['config'].encode()), settings(config)
     changed = sorted(key for key in before.keys() | now.keys() if before.get(key) != now.get(key))
     if ('description' in changed
             and instances.identity(record['observed']['config'].encode()) == record['installation_id']
@@ -504,10 +571,19 @@ def external_changes(record, config, adopt=True):
         changed.remove('description')
     cores_key = 'cpulimit' if 'cpulimit' in before and 'cores' not in before else 'cores'
     values = {}
+    # The network line stays as the user left it: a stack carries it over as
+    # it is, and an application also takes it into its record.
+    if 'net0' in changed:
+        changed.remove('net0')
+        if adopt:
+            old_net, new_net = network_fields(before.get('net0')), network_fields(now.get('net0'))
+            values.update({key: value for key, value in new_net.items() if old_net[key] != value})
     for key in changed if adopt else ():
         value = now.get(key)
         if key == 'onboot':
             values[key] = value == '1'
+        elif key in ('hostname', 'entrypoint') and value:
+            values[key] = value
         elif key == 'cpuunits' and value is None:
             values[key] = None
         elif key in ('memory', 'swap', 'cpuunits', cores_key) and value and re.fullmatch(r'[0-9]+', value):
@@ -540,6 +616,10 @@ def preflight(record, candidate, config, coordinated=None, in_place=False, basel
         target = desired.setdefault(section, {}) if section else desired
         if target.get(name) == recorded.get(name):
             target[name] = value
+    # Lines the user added are carried over and are not part of what is checked.
+    custom = deployment.get('custom_lxc', [])
+    config = b''.join(line for line in without_backup_snapshot(config).splitlines(keepends=True)
+                      if line.decode().rstrip('\n') not in custom)
     cfg = parse_config(config)
     for key in ('lxc.init.uid', 'lxc.init.gid'):
         if key in cfg and not re.fullmatch(r'[0-9]+', cfg[key]):
@@ -818,7 +898,7 @@ def healthcheck(vmid, template):
         ok = False
         if ip:
             any_status = check.get('accept_any_status', False)
-            args = ['curl', '-sS' if any_status else '-fsS', '--noproxy', '*', '-o', '/dev/null',
+            args = [*container_curl(vmid, run), '-sS' if any_status else '-fsS', '--noproxy', '*', '-o', '/dev/null',
                     '--max-time', str(check.get('request_timeout_seconds', 5))]
             if any_status:
                 args += ['-w', '%{http_code}']
@@ -933,6 +1013,8 @@ def commit(root, journal, state):
     gpu_devices.verify(state.get('desired_gpu_devices', {}))
     gpu_devices.check(config, state['candidate_contract']['deployment'])
     checkpoint(journal, state, 'committing')
+    if state.get('protected'):
+        run('pct', 'set', str(state['vmid']), '--protection', '1')
     result = copy.deepcopy(state['candidate_contract'])
     result.update(status='installed', completed_at=instances.now(),
                   last_transaction=state['id'],
@@ -1087,6 +1169,7 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
         if 'stack' in expected:
             record['stack'] = copy.deepcopy(expected['stack'])
     candidate = candidate_contract(record, operation, proposal)
+    drop_backup_snapshot(vmid)
     before = run('pct', 'config', str(vmid))
     cfg, actual, mac = preflight(record, candidate, before, coordinated)
     description = original_description({'record': record, 'before_config': before.decode()})
@@ -1140,7 +1223,8 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
              'external_data_acknowledged': acknowledge_external_data,
              'original_gpu_devices': original_gpu, 'desired_gpu_devices': desired_gpu,
              'was_running': run('pct', 'status', str(vmid)).strip() == b'status: running'}
-    state['kept_config'] = {key: cfg[key] for key in KEPT_AS_IS if key in cfg}
+    state['kept_config'] = kept_settings(record, candidate, cfg)
+    state['protected'] = cfg.get('protection') == '1'
     # The installer adds default tags. Preserve an explicit absence too, so
     # deleting all tags in Proxmox does not bring them back on the next update.
     state['kept_config']['tags'] = cfg.get('tags', '')
@@ -1150,11 +1234,9 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
         state['runtime_deployment']['kept_proxmox_settings'] = state['kept_config']
     if coordinated:
         state['coordinated'] = coordinated
-        state['preserved_stack_config'] = {key: cfg[key] for key in ('net1', 'startup', 'hookscript') if key in cfg}
-        state['preserved_stack_config']['tags'] = cfg.get('tags', '')
-        if operation == 'update':
-            state['preserved_stack_config'].update({key: cfg[key] for key in
-                ('net0', 'features', 'cmode', 'console', 'tty', 'cpuunits') if key in cfg})
+        state['preserved_stack_config'] = dict(state['kept_config'])
+        if operation == 'update' and 'cpuunits' in cfg:
+            state['preserved_stack_config']['cpuunits'] = cfg['cpuunits']
     checkpoint(journal, state, 'prepared')
     pending = copy.deepcopy(record)
     pending.update(status='updating', pending_transaction=str(journal), transaction_id=state['id'])
@@ -1227,6 +1309,8 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
     firewall = Path(f'/etc/pve/firewall/{vmid}.fw')
     state['firewall_config'] = firewall.read_text() if firewall.exists() else None
     checkpoint(journal, state, 'replacing-root')
+    if state.get('protected'):
+        run('pct', 'set', str(vmid), '--delete', 'protection')
     run('pct', 'destroy', str(vmid))
     checkpoint(journal, state, 'installing-candidate')
     if show:

oci/remote/oci_native_modify.py

@@ -42,8 +42,9 @@ def environment(config):
             entries.extend(line.split(': ', 1)[1].split('\0'))
     for entry in entries:
         name, sep, value = entry.partition('=')
-        if not sep or name in result:
+        if not sep:
             raise ValueError(translate('Duplicate variable in the contract; review it before editing'))
+        # A variable set twice takes its last value, as it does in LXC.
         result[name] = value
     return result
 
@@ -129,7 +130,7 @@ def plan(record, candidate, config):
                 values[native] = str(value)
     if 'network' in changed:
         network = new['network']
-        if set(network) - {'bridge', 'ipv4', 'gateway', 'firewall', 'host_managed', 'mac_address'}:
+        if set(network) - {'bridge', 'ipv4', 'gateway', 'firewall', 'host_managed', 'mac_address', 'vlan_tag'}:
             raise ValueError(translate('Unsupported network settings'))
         if not re.fullmatch(r'[A-Za-z0-9_.:-]+', network['bridge']):
             raise ValueError(translate('Invalid bridge'))
@@ -146,6 +147,11 @@ def plan(record, candidate, config):
         fields.update(bridge=network['bridge'], ip=address)
         if gateway: fields['gw'] = gateway
         else: fields.pop('gw', None)
+        vlan_tag = network.get('vlan_tag')
+        if vlan_tag is not None and (isinstance(vlan_tag, bool) or not isinstance(vlan_tag, int) or not 1 <= vlan_tag <= 4094):
+            raise ValueError(translate('The VLAN tag must be a number from 1 to 4094'))
+        if vlan_tag: fields['tag'] = str(vlan_tag)
+        elif 'vlan_tag' in old['network']: fields.pop('tag', None)
         for key, native in (('firewall', 'firewall'), ('host_managed', 'host-managed')):
             if old['network'].get(key) != network.get(key):
                 fields[native] = '1' if network.get(key) else '0'

oci/remote/oci_nested_mounts.py

@@ -65,6 +65,12 @@ def check(config):
         raise ValueError(translate('A container mount has a source, backup or permission different from the saved record'))
 
 
+def stale(config):
+    """Whether a mount point taken away left its lines behind."""
+    wanted = {line for group in groups(config) for line in group}
+    return any(own(line) and line not in wanted for line in config.decode().splitlines())
+
+
 def follow(vmid):
     """Write the lines of the mount points the container has now, and no
     others. Any snapshot section is left as it is."""

oci/remote/oci_runtime_settings.py

@@ -204,12 +204,14 @@ def filter_config(config, deployment, runtime_config=None):
     declared = set(tmpfs_lines(deployment, runtime_config if runtime_config is not None else config))
     files = {m['source'] for m in deployment.get('mounts', []) if m.get('type') == 'host-bind'}
 
+    custom = {line.encode() for line in deployment.get('custom_lxc', [])}
+
     def own(line):
         entry = line.decode().strip().split(': ', 1)[1]
         match = FILE_BIND.fullmatch(entry)
         return entry in declared or bool(match and match.group(1) in files)
     return b''.join(line for line in config.splitlines(keepends=True)
-                    if not line.startswith(b'lxc.include: ') and not (
+                    if not line.startswith(b'lxc.include: ') and line.rstrip(b'\n') not in custom and not (
                         line.startswith(b'lxc.mount.entry: ') and own(line)))
 
 

oci/remote/oci_stack_modify.py

@@ -183,14 +183,36 @@ def register(root, vmid):
                                   if path not in listed and (gpu_devices.gpu_path(path)
                                                              or gpu_devices.peripheral_path(path))]
     instances.write(instances.location(root, vmid), record)
+    snapshot(root, vmid)
+
+
+def adopt(root, vmid):
+    """Take into the record what was changed in Proxmox on a member, so an
+    operation on the stack keeps it. The caller holds the registry lock."""
+    record = instances.read(root, vmid)
+    config = instances.command('pct', 'config', str(vmid))
+    if instances.same_config_except_notes(record, config, ignore_tags=True):
+        return
+    if (record['deployment'].get('replay_profile') or {}).get('adapter') in CONVERTERS:
+        register(root, vmid)
+        return
+    proposal = reconcile.propose(record, config, member=True)
+    if proposal:
+        reconcile.commit_locked(root, vmid, proposal, member=True)
+        snapshot(root, vmid)
+
+
+def snapshot(root, vmid):
+    """Refresh the copy of a member its main container keeps."""
+    record = instances.read(root, vmid)
     primary_id = (record.get('stack_member') or {}).get('primary_vmid', vmid)
     primary = instances.read(root, primary_id)
-    snapshot = instances.read(root, vmid)
-    snapshot.pop('stack', None)
+    copied = instances.read(root, vmid)
+    copied.pop('stack', None)
     members = primary.get('stack', {}).get('members', [])
     for index, member in enumerate(members):
         if member.get('vmid') == vmid:
-            members[index] = snapshot
+            members[index] = copied
     instances.write(instances.location(root, primary_id), primary)
 
 

oci/remote/oci_stack_native.py

@@ -23,7 +23,7 @@ import oci_stack_plan
 import oci_stack_transaction as stack_tx
 import oci_stack_replay as replay
 import oci_work_backup as work_backup
-from oci_installation_state import image_from_archive, parse_config, private_directory, sha
+from oci_installation_state import container_curl, image_from_archive, parse_config, private_directory, sha
 from oci_update_current import resolve_archive
 from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error
 
@@ -545,9 +545,9 @@ class NativeAdapter:
                     # Main/custom web services retain both contracts: an exec
                     # check never replaces the advertised HTTP readiness probe.
                     if check.get('url') and (self.services[vmid].get('frontend') or check.get('endpoint')):
-                        member_tx.run('curl', '-fsS', '--noproxy', '*', '--max-time', '5', check['url'])
+                        member_tx.run(*container_curl(vmid, member_tx.run), '-fsS', '--noproxy', '*', '--max-time', '5', check['url'])
                 elif check['type'] == 'http':
-                    member_tx.run('curl', '-fsS', '--noproxy', '*', '--max-time', '5', check['url'])
+                    member_tx.run(*container_curl(vmid, member_tx.run), '-fsS', '--noproxy', '*', '--max-time', '5', check['url'])
                 elif check['type'] == 'tcp':
                     with socket.create_connection(tcp_target(check), timeout=5):
                         pass
@@ -848,6 +848,10 @@ def run(vmid, recover=False, acknowledge_external_data=False, keep_backup=None,
             result = stack_tx.execute(journal, adapter)
             msg_ok(translate('Stack recovery completed; every member is back to its previous installation.'))
             return result
+        import oci_stack_modify
+        for member in primary['stack']['members']:
+            oci_stack_modify.adopt(root, member['vmid'])
+        primary = instances.read(root, primary_id)
         records, inventory = {}, {}
         for snapshot in primary['stack']['members']:
             member_id = snapshot['vmid']

oci/remote/oci_stack_replay.py

@@ -507,8 +507,11 @@ def normalize(record):
     names = set()
     def add_environment(value):
         name, equals, content = value.partition('=')
-        if not equals or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name) or name in names:
+        if not equals or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', name):
             raise ValueError(translate('Ambiguous or invalid environment variable'))
+        # A variable set twice takes its last value, as it does in LXC.
+        if name in names:
+            environment[:] = [item for item in environment if item['name'] != name]
         names.add(name)
         environment.append({'name': name, 'value': content})
     for line in config.splitlines():

oci/remote/oci_ui.sh

@@ -228,6 +228,29 @@ oci_log_tail() {
   tail -n "${1:-15}" "$OCI_LOG" | sed "s/^/${_OCI_TAB}  /"
 }
 
+# The host may have no route to the VLAN of an interface: there curl runs
+# inside the network of the container.
+oci_curl() {
+  local vmid=$1
+  shift
+  if grep -Eq '^net[0-9]+:.*,tag=' "/etc/pve/lxc/${vmid}.conf" 2>/dev/null; then
+    nsenter --net="/proc/$(lxc-info -n "$vmid" -pH)/ns/net" curl "$@"
+  else
+    curl "$@"
+  fi
+}
+
+# tag= option of an access interface from the given network key of the
+# deployment, in OCI_VLAN_OPTION. Returns 1 when the tag is not valid.
+oci_vlan_option() {
+  local tag
+  OCI_VLAN_OPTION=""
+  tag=$(jq -r --arg key "$1" '.network[$key] // empty' "$DEPLOYMENT_FILE")
+  [[ -z $tag ]] && return 0
+  [[ $tag =~ ^[0-9]+$ ]] && (( tag >= 1 && tag <= 4094 )) || return 1
+  OCI_VLAN_OPTION=",tag=${tag}"
+}
+
 # ip= and gw= options of an access interface, DHCP or a static IPv4 with an
 # optional gateway, in OCI_ACCESS_NET. Returns 1 when a value is not valid.
 oci_access_net() {

oci/remote/oci_update_current.py

@@ -162,13 +162,16 @@ def kept_settings(changes, deployment):
     """Names of the settings changed in Proxmox that the new container keeps."""
     labels = {'memory': translate('Memory'), 'swap': translate('Swap'), 'cores': translate('CPU cores'),
               'cpulimit': translate('CPU cores'), 'cpuunits': translate('CPU priority'),
-              'onboot': translate('Start with Proxmox')}
+              'onboot': translate('Start with Proxmox'), 'hostname': translate('Hostname'),
+              'entrypoint': translate('Start command'),
+              'bridge': translate('Network'), 'ipv4': translate('Network'), 'gateway': translate('Network'),
+              'firewall': translate('Network'), 'vlan_tag': translate('VLAN tag')}
     kept = []
     for key, value in changes.items():
         section, name = transaction.ADOPTABLE[key]
         if (deployment.get(section, {}) if section else deployment).get(name) == value:
             kept.append(labels[key])
-    return kept
+    return list(dict.fromkeys(kept))
 
 
 def application_name(record, vmid):