feat(oci): coherent advanced installer flow, container creation progress and USB device list

oci/remote/install_immich_stack.sh

@@ -361,7 +361,7 @@ fi
 TAGS="media;oci;proxmenux"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$DB_ID" "$DB_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$DB_ID" "$DB_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${DATABASE_STORAGE}:${DATABASE_SIZE},mp=/var/lib/postgresql/data,backup=1" \
   --hostname "${STACK_NAME}-db" --cores 2 --memory 2048 --swap 512 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${DB_ADDRESS},type=veth" \
@@ -390,7 +390,7 @@ oci_quiet pct unmount "$DB_ID"
 msg_ok "$(translate "Container created:") CT $DB_ID (PostgreSQL)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$VALKEY_ID" "$VALKEY_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
+oci_create_container "$VALKEY_ID" "$VALKEY_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
   --mp0 "${ROOTFS_STORAGE}:4,mp=/data,backup=1" \
   --hostname "${STACK_NAME}-valkey" --cores 1 --memory 512 --swap 256 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${VALKEY_ADDRESS},type=veth" \
@@ -412,7 +412,7 @@ set_lxc_directive "$VALKEY_ID" lxc.signal.halt SIGTERM
 msg_ok "$(translate "Container created:") CT $VALKEY_ID (Valkey)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$ML_ID" "$ML_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:12" \
+oci_create_container "$ML_ID" "$ML_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:12" \
   --mp0 "${ROOTFS_STORAGE}:${MODEL_CACHE_SIZE},mp=/cache,backup=1" \
   --hostname "${STACK_NAME}-ml" "${ML_CPU_ARGS[@]}" --memory "$ML_MEMORY" --swap "$ML_SWAP" \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${ML_FRONTEND_NET},type=veth" \
@@ -447,7 +447,7 @@ if [[ $VIDEO_ACCELERATION == vaapi ]]; then
 fi
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$SERVER_ID" "$SERVER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:16" \
+oci_create_container "$SERVER_ID" "$SERVER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:16" \
   --mp0 "$SERVER_MEDIA_MOUNT" --hostname "${STACK_NAME}-server" \
   --cores 4 --memory 3072 --swap 1024 \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \

oci/remote/install_nextcloud_stack.sh

@@ -343,7 +343,7 @@ fi
 TAGS="productivity;oci;proxmenux"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${DATABASE_STORAGE}:${DATABASE_SIZE},mp=/var/lib/postgresql,backup=1" \
   --hostname "${STACK_NAME}-db" --cores 2 --memory 1024 --swap 512 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${DATABASE_ADDRESS},type=veth" \
@@ -380,7 +380,7 @@ set_runtime_env "$DATABASE_ID" TZ "$TIMEZONE"
 msg_ok "$(translate "Container created:") CT $DATABASE_ID (PostgreSQL)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$CACHE_ID" "$CACHE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
+oci_create_container "$CACHE_ID" "$CACHE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
   --hostname "${STACK_NAME}-redis" --cores 1 --memory 512 --swap 256 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${CACHE_ADDRESS},type=veth" \
   --unprivileged 1 --features nesting=1 --cmode console --onboot "$ONBOOT" \
@@ -393,7 +393,7 @@ set_lxc_directive "$CACHE_ID" lxc.signal.halt SIGTERM
 msg_ok "$(translate "Container created:") CT $CACHE_ID (Redis)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "$APPLICATION_MOUNT" --hostname "$STACK_NAME" \
   --cores 2 --memory 2048 --swap 1024 \
   --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${FRONTEND_NET},type=veth" \

oci/remote/install_oci.sh

@@ -1630,7 +1630,7 @@ fi
 
 # pct keeps the OCI Entrypoint/Cmd/Env/User/WorkingDir/StopSignal metadata.
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "${CREATE_ARGS[@]}" \
+oci_create_container "${CREATE_ARGS[@]}" \
   || die "$(translate "Could not create the container:") CT $VMID"
 CT_CREATED=1
 msg_ok "$(translate "Container created:") CT $VMID ($HOSTNAME)"

oci/remote/install_paperless_stack.sh

@@ -359,7 +359,7 @@ fi
 TAGS="productivity;oci;proxmenux"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${DATABASE_STORAGE}:${DATABASE_SIZE},mp=/var/lib/postgresql,backup=1" \
   --hostname "${STACK_NAME}-db" --cores 2 --memory 1024 --swap 512 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${DATABASE_ADDRESS},type=veth" \
@@ -393,7 +393,7 @@ set_runtime_env "$DATABASE_ID" TZ "$TIMEZONE"
 msg_ok "$(translate "Container created:") CT $DATABASE_ID (PostgreSQL)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$BROKER_ID" "$BROKER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
+oci_create_container "$BROKER_ID" "$BROKER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
   --mp0 "${APPLICATION_STORAGE}:${BROKER_SIZE},mp=/data,backup=1" \
   --hostname "${STACK_NAME}-valkey" --cores 1 --memory 512 --swap 256 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${BROKER_ADDRESS},type=veth" \
@@ -410,7 +410,7 @@ set_lxc_directive "$BROKER_ID" lxc.signal.halt SIGTERM
 msg_ok "$(translate "Container created:") CT $BROKER_ID (Valkey)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${APPLICATION_STORAGE}:${DATA_SIZE},mp=/usr/src/paperless/data,backup=1" \
   --mp1 "${APPLICATION_STORAGE}:${MEDIA_SIZE},mp=/usr/src/paperless/media,backup=1" \
   --mp2 "$EXPORT_MOUNT" --mp3 "$CONSUME_MOUNT" --hostname "$STACK_NAME" \

oci/remote/install_tandoor_stack.sh

@@ -352,7 +352,7 @@ fi
 TAGS="productivity;oci;proxmenux"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
+oci_create_container "$DATABASE_ID" "$DATABASE_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:4" \
   --mp0 "${DATABASE_STORAGE}:${DATABASE_SIZE},mp=/var/lib/postgresql/data,backup=1" \
   --hostname "${STACK_NAME}-db" --cores 2 --memory 1024 --swap 512 \
   --net0 "name=eth0,bridge=${PRIVATE_BRIDGE},firewall=1,host-managed=1,ip=${DATABASE_ADDRESS},type=veth" \
@@ -389,7 +389,7 @@ set_runtime_env "$DATABASE_ID" TZ "$TIMEZONE"
 msg_ok "$(translate "Container created:") CT $DATABASE_ID (PostgreSQL)"
 
 msg_info "$(translate "Creating the container...")"
-oci_quiet pct create "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
+oci_create_container "$APPLICATION_ID" "$APPLICATION_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:8" \
   --mp0 "${APPLICATION_STORAGE}:${STATIC_SIZE},mp=/opt/recipes/staticfiles,backup=1" \
   --mp1 "$MEDIA_MOUNT" --hostname "$STACK_NAME" \
   --cores 2 --memory 2048 --swap 512 \

oci/remote/oci_ui.sh

@@ -115,6 +115,49 @@ oci_quiet() {
   fi
 }
 
+# `pct create` of an OCI archive, showing how much of the image is already
+# unpacked into the rootfs. Arguments: VMID ARCHIVE [pct create options...]
+oci_create_container() {
+  local vmid=$1 archive=$2
+  shift 2
+  local path=$archive rootfs="/var/lib/lxc/${vmid}/rootfs" log=${OCI_LOG:-/dev/null}
+  local total_mib=0 base_mib="" used_mib extracted_mib percentage pid status=0 elapsed=0 mounted=0 extracting=0
+  [[ $path == /* ]] || path=$(pvesm path "$archive" 2>/dev/null || true)
+  if [[ -n ${VERIFY_OCI_ARCHIVE:-} && -f $path ]]; then
+    total_mib=$(python3 "$VERIFY_OCI_ARCHIVE" --extracted-mib "$path" 2>/dev/null || printf '0')
+    [[ $total_mib =~ ^[0-9]+$ ]] || total_mib=0
+  fi
+  pct create "$vmid" "$archive" "$@" >>"$log" 2>&1 &
+  pid=$!
+  while kill -0 "$pid" 2>/dev/null; do
+    if mountpoint -q "$rootfs" 2>/dev/null; then
+      mounted=1
+      used_mib=$(df -BM --output=used "$rootfs" 2>/dev/null | tail -n 1 | tr -dc '0-9' || true)
+      used_mib=${used_mib:-0}
+      base_mib=${base_mib:-$used_mib}
+      extracted_mib=$(( used_mib > base_mib ? used_mib - base_mib : 0 ))
+      (( extracted_mib > 0 )) && extracting=1
+      if (( ! extracting )); then
+        msg_progress "$(translate "Creating the container: reading the image...") ${elapsed}s"
+      elif (( total_mib > 0 )); then
+        (( extracted_mib < total_mib )) || extracted_mib=$(( total_mib * 99 / 100 ))
+        percentage=$(( extracted_mib * 100 / total_mib ))
+        msg_progress "$(translate "Creating the container: extracting the image") ${extracted_mib} / ${total_mib} MiB (${percentage}%), ${elapsed}s"
+      else
+        msg_progress "$(translate "Creating the container: extracting the image") ${extracted_mib} MiB, ${elapsed}s"
+      fi
+    elif (( mounted )); then
+      msg_progress "$(translate "Creating the container: applying the image configuration...") ${elapsed}s"
+    else
+      msg_progress "$(translate "Creating the container: preparing its disk...") ${elapsed}s"
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+  done
+  wait "$pid" || status=$?
+  return "$status"
+}
+
 # Last lines of the log, for the error report.
 oci_log_tail() {
   [[ -n $OCI_LOG && -s $OCI_LOG ]] || return 0

oci/remote/verify_oci_archive.py

@@ -5,7 +5,9 @@ from __future__ import annotations
 
 import argparse
 import hashlib
+import json
 import os
+import struct
 import sys
 import tarfile
 import zlib
@@ -114,10 +116,63 @@ def verify_archive(path: Path) -> None:
     progress(f"OCI integrity verified: {path}")
 
 
+def _layer_extracted_size(raw, member: tarfile.TarInfo) -> int | None:
+    """What a layer takes once unpacked: the gzip trailer, the frame header of
+    zstd or the tar itself."""
+    raw.seek(member.offset_data)
+    head = raw.read(18)
+    if head.startswith(b"\x1f\x8b"):
+        raw.seek(member.offset_data + member.size - 4)
+        size = struct.unpack("<I", raw.read(4))[0]
+        # The trailer keeps the size modulo 4 GiB.
+        while size < member.size:
+            size += 1 << 32
+        return size
+    if head.startswith(b"\x28\xb5\x2f\xfd") and len(head) >= 6:
+        descriptor = head[4]
+        single_segment = descriptor >> 5 & 1
+        field = (1 if single_segment else 0, 2, 4, 8)[descriptor >> 6]
+        if not field:
+            return None
+        start = 5 + (0 if single_segment else 1) + (0, 1, 2, 4)[descriptor & 3]
+        value = int.from_bytes(head[start:start + field], "little")
+        return value + 256 if field == 2 else value
+    return member.size
+
+
+def extracted_size(path: Path) -> int | None:
+    """Estimated size of the image once its layers are unpacked, or None when
+    a layer does not record it."""
+    with tarfile.open(path, mode="r:") as archive, path.open("rb") as raw:
+        members = {member.name.lstrip("./"): member for member in archive.getmembers()}
+
+        def blob(digest: str) -> tarfile.TarInfo:
+            return members["blobs/" + digest.replace(":", "/")]
+
+        index = json.load(archive.extractfile(members["index.json"]))
+        manifest = json.load(archive.extractfile(blob(index["manifests"][0]["digest"])))
+        total = 0
+        for layer in manifest.get("layers", []):
+            size = _layer_extracted_size(raw, blob(layer["digest"]))
+            if size is None:
+                return None
+            total += size
+        return total
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("archive", type=Path)
+    parser.add_argument("--extracted-mib", action="store_true",
+                        help="print the estimated unpacked size in MiB (0 when unknown)")
     args = parser.parse_args()
+    if args.extracted_mib:
+        try:
+            size = extracted_size(args.archive)
+        except (tarfile.TarError, OSError, KeyError, ValueError, IndexError):
+            size = None
+        print((size or 0) >> 20)
+        return 0
     try:
         verify_archive(args.archive)
     except VerificationError as exc: