Merge remote-tracking branch 'origin/develop' into HEAD

# Conflicts:
#	.github/workflows/test-oci-registries.yml

oci/remote/install_oci.sh

@@ -359,13 +359,18 @@ cleanup_canonical_inputs() {
 # A derived check accepts any HTTP answer below 500: the application is up,
 # even when its first page is a redirect or asks for a login.
 healthcheck_probe() {
+  local code status=0
   if [[ ${HC_ANY_STATUS:-false} == true ]]; then
-    local code
-    code=$(curl "${CURL_ARGS[@]}" "$HC_URL" 2>/dev/null) || return 1
-    [[ $code =~ ^[1-4][0-9][0-9]$ ]]
+    code=$(curl "${CURL_ARGS[@]}" "$HC_URL" 2>/dev/null) || status=$?
+    (( status == 0 )) && [[ $code =~ ^[1-4][0-9][0-9]$ ]] && return 0
   else
-    curl "${CURL_ARGS[@]}" "$HC_URL" 2>/dev/null
+    curl "${CURL_ARGS[@]}" "$HC_URL" >/dev/null 2>&1 || status=$?
+    (( status == 0 )) && return 0
   fi
+  # A port taken from the compose file may serve a database or SSH: an answer
+  # in another protocol (HTTP/0.9, weird or empty reply, reset) means it is up.
+  [[ ${HC_ACCEPT_NON_HTTP:-false} == true ]] \
+    && (( status == 1 || status == 8 || status == 52 || status == 56 ))
 }
 
 cleanup_failed_install() {
@@ -1025,15 +1030,18 @@ apply_extra_hosts() {
 }
 
 # Settings an update gives back to the rebuilt container before it starts:
-# the LAN leg of a suite or stack member and its start order.
+# the LAN leg of a suite or stack member, its start order and current tags.
 apply_kept_settings() {
   local key value
   while IFS=$'\t' read -r key value; do
     [[ -n $key ]] || continue
-    [[ $key == net1 || $key == startup ]] \
+    [[ $key == net1 || $key == startup || $key == tags ]] \
       || die "$(translate "Unsupported kept setting:") $key"
-    oci_quiet pct set "$VMID" "--$key" "$value" \
-      || die "$(translate "Could not restore the container setting:") $key"
+    if [[ $key == tags && -z $value ]]; then
+      oci_quiet pct set "$VMID" --delete tags
+    else
+      oci_quiet pct set "$VMID" "--$key" "$value"
+    fi || die "$(translate "Could not restore the container setting:") $key"
   done < <(jq -r '.kept_proxmox_settings // {} | to_entries[] | [.key, .value] | @tsv' "$DEPLOYMENT_FILE")
 }
 
@@ -1091,7 +1099,7 @@ ensure_runtime_home() {
   passwd_file="${rootfs}/etc/passwd"
   mount_ct_rootfs
   if ! home=$(python3 "$OCI_RUNTIME_RESOLVER" --home "$passwd_file" "$uid" 2>>"${OCI_LOG:-/dev/stderr}"); then
-    oci_quiet pct unmount "$VMID" || true
+    oci_quiet pct unmount "$VMID" || die "$(translate "Could not unmount the container filesystem")"
     return 0
   fi
   oci_quiet pct unmount "$VMID" || die "$(translate "Could not unmount the container filesystem")"
@@ -1230,6 +1238,17 @@ apply_installer_profile() {
       fi
       [[ -d ${rootfs}${target} ]] \
         || { oci_log "The volume to prepare does not exist: $target"; failed=1; break; }
+      # A reused volume may intentionally be mounted read-only (for example,
+      # /run/secrets). Do not remove files or change ownership/mode on it.
+      # Use the most specific containing mount: an RW child can override RO.
+      if [[ $(jq -r --arg target "$target" '
+        [.mounts[]? | select(.container_path as $path |
+          $target == $path or ($target | startswith($path + "/")))]
+        | sort_by(.container_path | length) | last | .read_only // false' \
+        "$DEPLOYMENT_FILE") == true ]]; then
+        oci_log "Skipping preparation of read-only volume: $target"
+        continue
+      fi
       if [[ $remove_lost_found == true ]]; then
         rm -rf "${rootfs}${target}/lost+found" || { failed=1; break; }
       fi
@@ -2253,6 +2272,7 @@ if [[ $START_AFTER == 1 && $HAS_STARTUP_HEALTHCHECK == 1 ]]; then
   [[ -n $IP ]] || die "$(translate "The healthcheck cannot run without an IP address")"
   HC_URL="${HC_SCHEME}://${IP}:${HC_PORT}${HC_PATH}"
   HC_ANY_STATUS=$(jq -r '.accept_any_status // false' <<<"$STARTUP_HEALTHCHECK")
+  HC_ACCEPT_NON_HTTP=$(jq -r '.accept_non_http // false' <<<"$STARTUP_HEALTHCHECK")
   if [[ $HC_ANY_STATUS == true ]]; then
     CURL_ARGS=(-sS --noproxy '*' -o /dev/null -w '%{http_code}' --max-time "$HC_REQUEST_TIMEOUT")
   else

oci/remote/oci_carried_record.py

@@ -38,6 +38,8 @@ DIRECTORY = '.proxmenux'
 NAME = 'oci-record.json'
 KIND = 'proxmenux.oci-carried-record'
 STAMP = 'carried.sha256'
+CONFIG_CACHE = 'carried-config.json'
+LOCAL_CONFIGS = Path('/etc/pve/lxc')
 LIMIT = 16 * 1024 * 1024
 STACK_CONTRACT = '/etc/pve/priv/proxmenux-stack-{}.json'
 NVIDIA_HOOK = re.compile(r'/usr/local/lib/proxmenux/oci/nvidia-mount-([a-f0-9]{64})\.sh')
@@ -279,6 +281,47 @@ def read_stamp(path):
     return value if isinstance(value, dict) else {}
 
 
+def configuration(root, vmid, verify=False):
+    """Reuse pct's formatted output only while its local source is unchanged.
+
+    Opening Manage checks every carried copy. Starting a fresh pct process
+    for each unchanged guest dominates that check. The private host cache is
+    keyed by the complete native configuration, including notes, pending
+    changes and snapshot sections. Lifecycle verification always calls pct.
+    An unavailable source, invalid cache or concurrent edit falls back to pct.
+    """
+    source = LOCAL_CONFIGS / f'{int(vmid)}.conf'
+    cache = instances.location(root, vmid).parent / CONFIG_CACHE
+
+    def fingerprint():
+        try:
+            return sha(source.read_bytes())
+        except OSError:
+            return None
+
+    before = fingerprint()
+    if before is not None and not verify:
+        try:
+            if not cache.is_symlink() and cache.stat().st_size <= LIMIT:
+                value = json.loads(cache.read_text())
+                if (isinstance(value, dict) and value.get('schema_version') == 1
+                        and value.get('source_sha256') == before
+                        and isinstance(value.get('config'), str) and value['config']
+                        and fingerprint() == before):
+                    return value['config']
+        except (OSError, ValueError):
+            pass
+    result = _run('pct', 'config', str(vmid))
+    if result.returncode != 0:
+        return None
+    # Do not associate output from one revision with another revision's hash.
+    if before is not None and fingerprint() == before:
+        with contextlib.suppress(OSError, ValueError):
+            instances.write(cache, {'schema_version': 1, 'source_sha256': before,
+                                    'config': result.stdout})
+    return result.stdout
+
+
 def superseded(record, existing, stamp):
     """Whether the container carries a copy this host did not write, with a
     record that differs from the one of this host. That is a container that
@@ -308,10 +351,10 @@ def carry(root, vmid, mount_stopped=True, verify=False, adopted=False):
     if record.get('status') != 'installed' or record.get('pending_transaction') \
             or record.get('pending_stack_transaction'):
         return 'skipped'
-    result = _run('pct', 'config', str(vmid))
-    if result.returncode != 0 or instances.identity(result.stdout.encode()) != record['installation_id']:
+    config = configuration(root, vmid, verify)
+    if config is None or instances.identity(config.encode()) != record['installation_id']:
         return 'skipped'
-    value = bundle(record, result.stdout)
+    value = bundle(record, config)
     expected = digest(value)
     record_path = instances.location(root, vmid)
     path = record_path.parent / STAMP
@@ -340,7 +383,7 @@ def carry(root, vmid, mount_stopped=True, verify=False, adopted=False):
             else:
                 value['saved_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
                 value['generation'] = uuid.uuid4().hex
-                write_copy(rootfs, value, mapped_root(result.stdout))
+                write_copy(rootfs, value, mapped_root(config))
                 stamp = {'generation': value['generation']}
                 outcome = 'carried'
         if outcome == 'stale':

oci/remote/oci_instance_transaction.py

@@ -352,6 +352,22 @@ def profile_keys(deployment):
     return keys
 
 
+GUESSED_ENDPOINT = 'compose-first-tcp-port-fallback'
+# curl exit codes for a port that accepted the connection and answered in a
+# protocol other than HTTP: HTTP/0.9 refused, weird reply, empty reply, reset.
+NON_HTTP_ANSWER = (1, 8, 52, 56)
+
+
+def probe_http(args):
+    """Run one curl probe and return its output and exit code."""
+    log('$ ' + shlex.join(str(arg) for arg in args))
+    try:
+        result = subprocess.run(args, capture_output=True, timeout=60, close_fds=True)
+    except subprocess.TimeoutExpired:
+        return b'', -1
+    return result.stdout, result.returncode
+
+
 def effective_healthcheck(template):
     """The template's own health check or, when it has none, one derived from
     its web address: any HTTP answer below 500 means the application is up."""
@@ -367,9 +383,14 @@ def effective_healthcheck(template):
         path = str(endpoint.get('path') or '/')
         if (endpoint.get('scheme') in ('http', 'https') and str(endpoint.get('port', '')).isdigit()
                 and path.startswith('/') and not any(c.isspace() for c in path)):
-            return {'scheme': endpoint['scheme'], 'port': int(endpoint['port']), 'path': path,
-                    'timeout_seconds': 300, 'request_timeout_seconds': 10, 'stability_seconds': 4,
-                    'verify_tls': False, 'accept_any_status': True}
+            check = {'scheme': endpoint['scheme'], 'port': int(endpoint['port']), 'path': path,
+                     'timeout_seconds': 300, 'request_timeout_seconds': 10, 'stability_seconds': 4,
+                     'verify_tls': False, 'accept_any_status': True}
+            # A port taken from the compose file because the recipe names no
+            # web address may serve a database or SSH: any answer counts.
+            if endpoint.get('source') == GUESSED_ENDPOINT:
+                check['accept_non_http'] = True
+            return check
     # Without a web address, the new image must at least keep the container running.
     return {'type': 'running', 'timeout_seconds': 90, 'stability_seconds': 20}
 
@@ -415,10 +436,9 @@ def candidate_contract(record, operation, proposal=None):
     return with_default_healthcheck(result)
 
 
-# Resource settings edited in Proxmox that the new container keeps.
 # Proxmox settings the rebuilt container gets back exactly as they are: the
-# LAN leg ProxMenux adds to a suite or stack member, and the start order.
-KEPT_AS_IS = ('net1', 'startup')
+# LAN leg ProxMenux adds to a suite or stack member, the start order and tags.
+KEPT_AS_IS = ('net1', 'startup', 'tags')
 ADOPTABLE = {'memory': ('resources', 'memory_mb'), 'swap': ('resources', 'swap_mb'),
              'cores': ('resources', 'cores'), 'cpulimit': ('resources', 'cores'),
              'cpuunits': ('resources', 'cpu_units'), 'onboot': (None, 'onboot')}
@@ -734,11 +754,10 @@ def healthcheck(vmid, template):
                 args += ['-w', '%{http_code}']
             if not check.get('verify_tls', False):
                 args.append('-k')
-            try:
-                answer = run(*args, f'{scheme}://{ip}:{port}{path}')
-                ok = not any_status or re.fullmatch(rb'[1-4][0-9][0-9]', answer.strip()) is not None
-            except RuntimeError:
-                pass
+            answer, status = probe_http([*args, f'{scheme}://{ip}:{port}{path}'])
+            ok = status == 0 and (not any_status or re.fullmatch(rb'[1-4][0-9][0-9]', answer.strip()) is not None)
+            if not ok and check.get('accept_non_http') and status in NON_HTTP_ANSWER:
+                ok = True
         if ok:
             stable_since = stable_since or time.monotonic()
             if time.monotonic() - stable_since >= stability:
@@ -1051,6 +1070,9 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
              'original_gpu_devices': original_gpu, 'desired_gpu_devices': desired_gpu,
              'was_running': run('pct', 'status', str(vmid)).strip() == b'status: running'}
     state['kept_config'] = {key: cfg[key] for key in KEPT_AS_IS if key in cfg}
+    # The installer adds default tags. Preserve an explicit absence too, so
+    # deleting all tags in Proxmox does not bring them back on the next update.
+    state['kept_config']['tags'] = cfg.get('tags', '')
     if not coordinated and state['kept_config']:
         # Applied by the installer before the new container starts, so the
         # application has its LAN leg from the first second.
@@ -1058,9 +1080,10 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
     if coordinated:
         state['coordinated'] = coordinated
         state['preserved_stack_config'] = {key: cfg[key] for key in ('net1', 'startup', 'hookscript') if key in cfg}
+        state['preserved_stack_config']['tags'] = cfg.get('tags', '')
         if operation == 'update':
             state['preserved_stack_config'].update({key: cfg[key] for key in
-                ('net0', 'features', 'cmode', 'console', 'tty', 'cpuunits', 'tags') if key in cfg})
+                ('net0', 'features', 'cmode', 'console', 'tty', 'cpuunits') if key in cfg})
     checkpoint(journal, state, 'prepared')
     pending = copy.deepcopy(record)
     pending.update(status='updating', pending_transaction=str(journal), transaction_id=state['id'])
@@ -1141,7 +1164,10 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
     restore_firewall(vmid, state)
     if coordinated:
         for key, value in state['preserved_stack_config'].items():
-            run('pct', 'set', str(vmid), '--' + key, value)
+            if key == 'tags' and not value:
+                run('pct', 'set', str(vmid), '--delete', 'tags')
+            else:
+                run('pct', 'set', str(vmid), '--' + key, value)
         check_runtime_mounts(run('pct', 'config', str(vmid)), candidate['deployment'])
         gpu_devices.check(run('pct', 'config', str(vmid)), candidate['deployment'])
         state['staged_config_sha256'] = sha(owned(vmid, record['installation_id']))

oci/remote/oci_instances.py

@@ -230,8 +230,9 @@ def identity(config):
     return match.group(1) if match else None
 
 
-def same_config_except_notes(record, config):
-    """Accept a presentation-only note edit, never a changed OCI identity or LXC setting."""
+def same_config_except_notes(record, config, ignore_tags=False):
+    """Accept a presentation-only note edit, never a changed OCI identity or LXC setting.
+    With `ignore_tags`, Proxmox tags may differ too: the operation keeps them as they are."""
     previous = record['observed']['config'].encode()
     expected = record['installation_id']
     if identity(previous) != expected or identity(config) != expected:
@@ -242,7 +243,8 @@ def same_config_except_notes(record, config):
         notes = [line for line in lines if line.startswith(b'description: ')]
         if len(notes) != 1:
             return None
-        return b''.join(line for line in lines if not line.startswith(b'description: '))
+        dropped = (b'description: ', b'tags: ') if ignore_tags else (b'description: ',)
+        return b''.join(line for line in lines if not line.startswith(dropped))
 
     original = without_notes(previous)
     return original is not None and original == without_notes(config)

oci/remote/oci_runtime.py

@@ -4,7 +4,9 @@
 from __future__ import annotations
 
 import json
+import os
 import shlex
+import stat
 import sys
 import tarfile
 from pathlib import Path
@@ -90,14 +92,39 @@ def effective_entrypoint(
 
 
 def home_directory(passwd_path: Path, uid: int) -> str | None:
-    """Return the safe home directory recorded for a container UID."""
-    for line in passwd_path.read_text(encoding="utf-8").splitlines():
+    """Read a regular image passwd file; only an absent UID defaults to /."""
+    if uid < 0:
+        return None
+    # The caller supplies <mounted rootfs>/etc/passwd. Pin etc so an image
+    # symlink or a rename during lookup cannot redirect the host-side read.
+    directory = os.open(passwd_path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+    try:
+        if not stat.S_ISREG(os.stat(passwd_path.name, dir_fd=directory, follow_symlinks=False).st_mode):
+            return None
+        descriptor = os.open(
+            passwd_path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory,
+        )
+        try:
+            # Recheck the opened object, not its name; a FIFO replacement
+            # must neither block open() nor reach a read operation.
+            if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+                return None
+            with os.fdopen(descriptor, encoding="utf-8", closefd=False) as stream:
+                content = stream.read()
+        finally:
+            os.close(descriptor)
+    finally:
+        os.close(directory)
+    for line in content.splitlines():
         fields = line.split(":")
-        if len(fields) < 7 or fields[2] != str(uid):
+        if len(fields) < 3 or fields[2] != str(uid):
             continue
+        if len(fields) != 7:
+            return None
         home = fields[5]
-        return home if home.startswith("/") else None
-    return None
+        safe = home.startswith("/") and not any(ord(char) < 32 or ord(char) == 127 for char in home)
+        return home if safe else None
+    return "/"
 
 
 def main() -> int:

oci/remote/oci_stack_native.py

@@ -192,7 +192,7 @@ class NativeAdapter:
                 config = instances.command('pct', 'config', str(vmid))
                 if instances.identity(config) != record['installation_id']:
                     raise ValueError(translate('The identity of a member was replaced'))
-                if (not self.journal.exists() or not self.state().get('replacement_intent')) and not instances.same_config_except_notes(record, config):
+                if (not self.journal.exists() or not self.state().get('replacement_intent')) and not instances.same_config_except_notes(record, config, ignore_tags=True):
                     raise ValueError(translate('A member configuration changed during the preparation'))
             else:
                 if Path('/etc/pve/lxc/%s.conf' % vmid).exists():