feat(oci): GPU selection per host and per image, one notification per update, and App tab for stack containers

- Immich asks what runs its video and its recognition in one menu, in both modes, and gives the GPU to the server and to Machine learning; AMD uses ROCm - Frigate, Ollama, llama.cpp, Faster Whisper and Piper take the image built for the chosen GPU - The acceleration menu offers only what the host can run - An update or a recreation sends one notification with its result instead of the stop, backup and start of each container - A private bridge with nothing connected is not reported as down - Secondary containers of a stack appear in the App tab with their version and logo; Secure Gateway shows the same update state in both views - A mistyped value in the wizard asks the same question again
2026-10-18 11:32:04 +00:00 · 2026-10-02 21:45:38 +02:00
parent 20ee21c08f
commit 9b5cefb81a
55 changed files with 2294 additions and 113 deletions
@@ -121,7 +121,7 @@ MEDIA_SIZE=$(jq -r '.media.size_gb // empty' "$DEPLOYMENT_FILE")
 MEDIA_ROOT=$(jq -r '.media.host_path // empty' "$DEPLOYMENT_FILE")
 TIMEZONE=$(jqr '.timezone')
 APPLICATION_CORES=$(jqr '.resources.cores // 4')
-APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 3072')
+APPLICATION_MEMORY=$(jqr '.resources.memory_mb // 4096')
 APPLICATION_SWAP=$(jqr '.resources.swap_mb // 1024')
 [[ $APPLICATION_CORES =~ ^[1-9][0-9]*$ && $APPLICATION_MEMORY =~ ^[1-9][0-9]*$ && $APPLICATION_SWAP =~ ^[0-9]+$ ]] \
  || die "$(translate "Invalid resources")"
@@ -149,6 +149,8 @@ ML_IP=${ML_ADDRESS%/*}
 DB_IP=${DB_ADDRESS%/*}
 VALKEY_IP=${VALKEY_ADDRESS%/*}
 VIDEO_ACCELERATION=$(jqr '.video_transcoding.acceleration')
+[[ $VIDEO_ACCELERATION == cpu || $VIDEO_ACCELERATION == vaapi || $VIDEO_ACCELERATION == nvenc ]] \
+  || die "$(translate "Video transcoding profile not implemented:") $VIDEO_ACCELERATION"
 RENDER_DEVICE=$(jq -r '.video_transcoding.render_device // empty' "$DEPLOYMENT_FILE")
 VAAPI_DRIVER=$(jqr '.video_transcoding.driver')
 MODEL_CACHE_SIZE=$(jqr '.machine_learning.model_cache_size_gb')
@@ -417,7 +419,7 @@ set_lxc_directive "$VALKEY_ID" lxc.signal.halt SIGTERM
 msg_ok "$(translate "Container created:") CT $VALKEY_ID (Valkey)"

 msg_info "$(translate "Creating the container...")"
-oci_create_container "$ML_ID" "$ML_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:12" \
+oci_create_container "$ML_ID" "$ML_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:${ML_ROOTFS_SIZE}" \
  --mp0 "${ROOTFS_STORAGE}:${MODEL_CACHE_SIZE},mp=/cache,backup=1" \
  --hostname "${STACK_NAME}-ml" "${ML_CPU_ARGS[@]}" --memory "$ML_MEMORY" --swap "$ML_SWAP" \
  --net0 "name=eth0,bridge=${FRONTEND_BRIDGE},firewall=1,host-managed=1,${ML_FRONTEND_NET},type=veth" \
@@ -442,7 +444,7 @@ rm -rf "$ML_ROOT/cache/lost+found"
 chown 100000:100000 "$ML_ROOT/cache"
 chmod 0755 "$ML_ROOT/cache"
 oci_quiet pct unmount "$ML_ID"
-msg_ok "$(translate "Container created:") CT $ML_ID ($(translate "Machine learning"))"
+msg_ok "$(translate "Container created:") CT $ML_ID (Machine learning)"

 SERVER_DEVICE_ARGS=()
 if [[ $VIDEO_ACCELERATION == vaapi ]]; then
@@ -463,6 +465,8 @@ oci_create_container "$SERVER_ID" "$SERVER_ARCHIVE" --rootfs "${ROOTFS_STORAGE}:
 created_ids+=("$SERVER_ID")
 oci_apply_extra_mounts "$SERVER_ID"
 oci_apply_extra_devices "$SERVER_ID"
+# NVENC needs the video capability on top of what recognition uses.
+[[ $VIDEO_ACCELERATION != nvenc ]] || configure_immich_nvidia "$SERVER_ID" "compute,video,utility"

 oci_quiet pct mount "$SERVER_ID"
 SERVER_ROOT="/var/lib/lxc/${SERVER_ID}/rootfs"
@@ -555,7 +559,7 @@ if (( START_AFTER == 1 )); then
  oci_quiet pct start "$VALKEY_ID"
  wait_command Valkey 30 pct exec "$VALKEY_ID" -- valkey-cli -h "$VALKEY_IP" ping
  msg_ok "$(translate "Service ready:") Valkey"
-  ML_LABEL=$(translate "Machine learning")
+  ML_LABEL="Machine learning"
  msg_info "$(translate "Starting the service:") $ML_LABEL"
  oci_quiet pct start "$ML_ID"
  wait_command "$ML_LABEL" 60 curl -fsS "http://${ML_IP}:3003/ping"
@@ -1,9 +1,22 @@
 # Immich ML prerequisites and native GPU setup; no host driver installation.
 validate_immich_ml_profile() {
-  ML_CPU_ARGS=(--cores 2)
-  ML_MEMORY=2048
+  ML_CPU_ARGS=(--cores 4)
+  ML_MEMORY=4096
+  ML_ROOTFS_SIZE=12
  case "$ML_ACCELERATION" in
    cpu) ;;
+    rocm)
+      ML_RENDER_DEVICE=$(jq -er '.machine_learning.render_device' "$DEPLOYMENT_FILE")
+      [[ $ML_RENDER_DEVICE =~ ^/dev/dri/renderD[0-9]+$ && -c $ML_RENDER_DEVICE ]] \
+        || die "$(translate "The selected AMD render device does not exist:") $ML_RENDER_DEVICE"
+      [[ $(cat "/sys/class/drm/${ML_RENDER_DEVICE##*/}/device/vendor") == 0x1002 ]] \
+        || die "$(translate "ROCm requires the render device of an AMD GPU")"
+      [[ -c /dev/kfd ]] || die "$(translate "ROCm requires /dev/kfd on the host")"
+      ML_MEMORY=8192
+      # The ROCm image carries the whole AMD runtime and is several times
+      # larger than the others.
+      ML_ROOTFS_SIZE=40
+      ;;
    openvino)
      ML_RENDER_DEVICE=$(jq -er '.machine_learning.render_device' "$DEPLOYMENT_FILE")
      [[ $ML_RENDER_DEVICE =~ ^/dev/dri/renderD[0-9]+$ && -c $ML_RENDER_DEVICE ]] \
@@ -57,34 +70,46 @@ validate_immich_ml_profile() {
  esac
 }

+# Gives one container of the stack the NVIDIA GPU through the dynamic hook.
+# Arguments: VMID CAPABILITIES
+configure_immich_nvidia() {
+  # Isolate the shared standalone installer's runtime context from the stack.
+  (
+    VMID=$1
+    CONF="/etc/pve/lxc/${VMID}.conf"
+    UNPRIVILEGED_FLAG=1
+    DEVICE='{"kind":"nvidia-runtime","runtime_mode":"dynamic"}'
+    NVIDIA_GID_ENV=""
+    DEVICE_INDEX=0
+    while grep -q "^dev${DEVICE_INDEX}:" "$CONF"; do DEVICE_INDEX=$((DEVICE_INDEX + 1)); done
+    fragment=$(mktemp)
+    trap 'rm -f "$fragment"' EXIT
+    jq -nc --arg capabilities "$2" \
+      '{environment:[{name:"NVIDIA_DRIVER_CAPABILITIES",value:$capabilities}]}' >"$fragment"
+    DEPLOYMENT_FILE=$fragment
+    add_character_device() {
+      local path=$1 mode gid
+      [[ -c $path && $path == /dev/nvidia* ]] || die "$(translate "Invalid NVIDIA device:") $path"
+      mode="0$(stat -c %a "$path")"
+      gid=$(stat -c %g "$path")
+      oci_quiet pct set "$VMID" "--dev${DEVICE_INDEX}" "path=${path},mode=${mode},gid=${gid},deny-write=0"
+      DEVICE_INDEX=$((DEVICE_INDEX + 1))
+    }
+    configure_nvidia_runtime
+  )
+}
+
 configure_immich_ml_gpu() {
  case "$ML_ACCELERATION" in
    openvino)
      oci_quiet pct set "$ML_ID" --dev0 "path=${ML_RENDER_DEVICE},gid=$(stat -c %g "$ML_RENDER_DEVICE"),mode=0660"
      ;;
+    rocm)
+      oci_quiet pct set "$ML_ID" --dev0 "path=${ML_RENDER_DEVICE},gid=$(stat -c %g "$ML_RENDER_DEVICE"),mode=0660"
+      oci_quiet pct set "$ML_ID" --dev1 "path=/dev/kfd,gid=$(stat -c %g /dev/kfd),mode=0660"
+      ;;
    cuda)
-      # Isolate the shared standalone installer's runtime context from the stack.
-      (
-        VMID=$ML_ID
-        CONF="/etc/pve/lxc/${ML_ID}.conf"
-        UNPRIVILEGED_FLAG=1
-        DEVICE='{"kind":"nvidia-runtime","runtime_mode":"dynamic"}'
-        NVIDIA_GID_ENV=""
-        DEVICE_INDEX=0
-        fragment=$(mktemp)
-        trap 'rm -f "$fragment"' EXIT
-        printf '%s\n' '{"environment":[{"name":"NVIDIA_DRIVER_CAPABILITIES","value":"compute,utility"}]}' >"$fragment"
-        DEPLOYMENT_FILE=$fragment
-        add_character_device() {
-          local path=$1 mode gid
-          [[ -c $path && $path == /dev/nvidia* ]] || die "$(translate "Invalid NVIDIA device:") $path"
-          mode="0$(stat -c %a "$path")"
-          gid=$(stat -c %g "$path")
-          oci_quiet pct set "$VMID" "--dev${DEVICE_INDEX}" "path=${path},mode=${mode},gid=${gid},deny-write=0"
-          DEVICE_INDEX=$((DEVICE_INDEX + 1))
-        }
-        configure_nvidia_runtime
-      )
+      configure_immich_nvidia "$ML_ID" "compute,utility"
      ;;
  esac
 }
@@ -101,6 +126,8 @@ if profile == "openvino":
    assert "OpenVINOExecutionProvider" in ort.get_available_providers()
    devices = ort.capi._pybind_state.get_available_openvino_device_ids()
    assert any(device.startswith("GPU") for device in devices), devices
+elif profile == "rocm":
+    assert "MIGraphXExecutionProvider" in ort.get_available_providers(), ort.get_available_providers()
 else:
    assert profile == "cuda"
    assert "CUDAExecutionProvider" in ort.get_available_providers()
@@ -40,6 +40,9 @@ def begin(root, primary, template, deployment, members, adapter):
                'mounts': []}
        if Path(adapter).name == 'install_immich_stack.sh' and name == 'machine-learning':
            plan['machine_learning'] = copy.deepcopy(deployment.get('machine_learning', {'acceleration': 'cpu'}))
+        if Path(adapter).name == 'install_immich_stack.sh' and name == 'server':
+            # An update must know that the server transcodes with NVIDIA.
+            plan['video_transcoding'] = copy.deepcopy(deployment.get('video_transcoding', {'acceleration': 'cpu'}))
        if Path(adapter).name in oci_stack_replay.FILES:
            plan['replay_profile'] = {'adapter': Path(adapter).name, 'role': name}
            if not oci_stack_replay.FILES[Path(adapter).name][name]:
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""What the Monitor is told about an update or a recreation.
+
+While the operation runs, every container it touches is stopped, backed up
+and started again. Those steps belong to the operation, so the containers are
+marked for the Monitor to keep their stop, start and backup notices to itself,
+and one notification with the result is sent when it ends.
+"""
+from __future__ import annotations
+
+import contextlib
+import json
+from pathlib import Path
+import socket
+import ssl
+import time
+import urllib.request
+
+MARKERS = Path('/run/proxmenux/oci-operations')
+ENDPOINTS = ('http://127.0.0.1:8008/api/internal/oci-event', 'https://127.0.0.1:8008/api/internal/oci-event')
+
+
+def _write(vmid, data):
+    try:
+        MARKERS.mkdir(parents=True, exist_ok=True)
+        path = MARKERS / str(int(vmid))
+        temporary = path.with_suffix('.tmp')
+        temporary.write_text(json.dumps(data))
+        temporary.replace(path)
+    except OSError:
+        pass
+
+
+def begin(vmids):
+    started = time.time()
+    for vmid in vmids:
+        _write(vmid, {'started': started, 'ended': None})
+
+
+def end(vmids):
+    # The notices of the last start can arrive after the operation returned;
+    # the Monitor keeps the mark for a short while after `ended`.
+    ended = time.time()
+    for vmid in vmids:
+        _write(vmid, {'started': ended, 'ended': ended})
+
+
+def notify(event, data):
+    """Best effort: the operation never depends on the Monitor answering."""
+    payload = json.dumps({'event': event, 'hostname': socket.gethostname(), **data}).encode()
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+    for url in ENDPOINTS:
+        request = urllib.request.Request(url, data=payload, headers={'Content-Type': 'application/json'})
+        try:
+            with urllib.request.urlopen(request, timeout=5, context=context if url.startswith('https') else None):
+                return True
+        except (OSError, ValueError):
+            continue
+    return False
+
+
+@contextlib.contextmanager
+def operation(vmids, kind, application, primary=None):
+    """Mark the containers for the length of an update or a recreation and
+    report how it ended. `kind` is 'update' or 'recreate'."""
+    vmids = [int(vmid) for vmid in vmids]
+    data = {'app_name': str(application), 'vmid': int(primary if primary is not None else vmids[0]),
+            'containers': ', '.join(f'CT {vmid}' for vmid in vmids)}
+    begin(vmids)
+    try:
+        yield
+    except BaseException as error:
+        end(vmids)
+        notify(f'oci_{kind}_failed', {**data, 'reason': str(error) or type(error).__name__})
+        raise
+    end(vmids)
+    notify(f'oci_{kind}_completed', data)
@@ -208,6 +208,15 @@ def modify(root, vmid, changes):
    backup = instances.location(root, vmid).parent / f"config-before-recreate-{time.strftime('%Y%m%d-%H%M%S')}.conf"
    backup.write_text(run('pct', 'config', str(vmid)))
    backup.chmod(0o600)
+    import oci_operation_notice
+    import oci_update_current
+    primary_id = (record.get('stack_member') or {}).get('primary_vmid', vmid)
+    name = oci_update_current.application_name(instances.read(root, primary_id), primary_id)
+    with oci_operation_notice.operation([vmid], 'recreate', name, primary_id):
+        _modify(root, vmid, changes)
+
+
+def _modify(root, vmid, changes):
    running = is_running(vmid)
    if running:
        msg_info(translate('Stopping the container...'))
@@ -373,6 +373,10 @@ class NativeAdapter:
                deployment = self.records[vmid]['deployment']
                if deployment.get('replay_profile') == {'adapter': 'install_immich_stack.sh', 'role': 'machine-learning'}:
                    acceleration = deployment.get('machine_learning', {}).get('acceleration', 'cpu')
+                    if acceleration == 'rocm':
+                        member_tx.run('pct', 'exec', str(vmid), '--', 'python', '-c',
+                            'import onnxruntime as ort; '
+                            'assert "MIGraphXExecutionProvider" in ort.get_available_providers()')
                    if acceleration in ('openvino', 'cuda'):
                        member_tx.run('pct', 'exec', str(vmid), '--', 'python', '-c',
                            'import sys,ctypes,onnxruntime as ort; p=sys.argv[1]; '
@@ -674,7 +678,11 @@ def run(vmid, recover=False, acknowledge_external_data=False, keep_backup=None):
                msg_ok(f"{translate('Backup created in')} {keep_backup}")
            else:
                adapter.keep_backup = keep_backup
-        result = stack_tx.execute(journal, adapter, plan)
+        import oci_operation_notice
+        import oci_update_current
+        with oci_operation_notice.operation([member['vmid'] for member in plan['members']], 'update',
+                                            oci_update_current.application_name(primary, primary_id), primary_id):
+            result = stack_tx.execute(journal, adapter, plan)
        msg_ok(translate('Stack update completed. Data kept.'))
        return result

@@ -68,9 +68,13 @@ def immich_record(record):
    if shlex.split(runtime.get('entrypoint', '')) != expected[role]:
        raise ValueError(translate('The Immich startup was modified or cannot be reproduced'))
    acceleration = record['deployment'].get('machine_learning', {}).get('acceleration', 'cpu')
-    if role == 'machine-learning' and acceleration not in ('cpu', 'openvino', 'cuda'):
+    if role == 'machine-learning' and acceleration not in ('cpu', 'openvino', 'cuda', 'rocm'):
        raise ValueError(translate('Immich GPU profile not validated'))
-    cuda = role == 'machine-learning' and acceleration == 'cuda'
+    video = record['deployment'].get('video_transcoding', {}).get('acceleration', 'cpu')
+    # NVIDIA reaches Machine learning for recognition and the server for NVENC.
+    capabilities = ('compute,utility' if role == 'machine-learning' and acceleration == 'cuda'
+                    else 'compute,video,utility' if role == 'server' and video == 'nvenc' else None)
+    cuda = capabilities is not None
    devices = [{'kind': 'nvidia-runtime', 'runtime_mode': 'dynamic'}] if cuda else []
    for item in projection['native_devices']:
        fields = dict(p.split('=', 1) for p in item['value'].split(',') if '=' in p)
@@ -80,17 +84,24 @@ def immich_record(record):
        if oci_gpu_devices.peripheral_path(path):
            devices.append(peripheral_device(fields))
            continue
+        rocm = role == 'machine-learning' and acceleration == 'rocm'
+        if rocm and path == '/dev/kfd':
+            # The compute interface ROCm needs beside the render node.
+            devices.append({'kind': 'character-device', 'host_path': path, 'container_path': path,
+                            'gid_strategy': 'host-device-gid', 'mode': fields.get('mode', '0660')})
+            continue
        if not path or not re.fullmatch(r'/dev/dri/renderD[0-9]+', path):
            raise ValueError(translate('Immich device without a validated translation'))
+        vendors = (['0x1002'] if rocm else ['0x8086'] if role == 'machine-learning' else ['0x8086', '0x1002'])
        devices.append({'kind': 'character-device', 'host_path': path, 'container_path': path,
                        'gid_strategy': 'host-device-gid', 'mode': fields.get('mode', '0660'),
-                        'drm_vendor_ids': ['0x8086'] if role == 'machine-learning' else ['0x8086', '0x1002']})
+                        'drm_vendor_ids': vendors})
    if projection['preserved_raw_runtime'] and not cuda:
        raise ValueError(translate('Immich runtime without a validated translation'))
    if cuda:
        import oci_accelerators
        candidate = {'devices': devices, 'environment': [
-            {'name': 'NVIDIA_DRIVER_CAPABILITIES', 'value': 'compute,utility'}]}
+            {'name': 'NVIDIA_DRIVER_CAPABILITIES', 'value': capabilities}]}
        oci_accelerators.check(record['observed']['config'].encode(), candidate)
    translated = {'compose_entrypoint': expected[role], 'command': []}
    for native, target in (('lxc.init.cwd', 'working_directory'), ('lxc.signal.halt', 'halt_signal')):
@@ -101,8 +112,10 @@ def immich_record(record):
    if cuda:
        result['deployment']['environment'] = [e for e in result['deployment']['environment']
                                             if e['name'] != 'NVIDIA_DRIVER_CAPABILITIES']
-        result['deployment']['environment'].append({'name': 'NVIDIA_DRIVER_CAPABILITIES', 'value': 'compute,utility'})
+        result['deployment']['environment'].append({'name': 'NVIDIA_DRIVER_CAPABILITIES', 'value': capabilities})
    result['deployment']['machine_learning'] = copy.deepcopy(record['deployment'].get('machine_learning', {}))
+    if 'video_transcoding' in record['deployment']:
+        result['deployment']['video_transcoding'] = copy.deepcopy(record['deployment']['video_transcoding'])
    return result


@@ -137,6 +137,17 @@ def kept_settings(changes, deployment):
    return kept


+def application_name(record, vmid):
+    """The name the user knows the application by, for its notifications."""
+    for template in ((record.get('stack') or {}).get('template') or {}, record.get('template') or {}):
+        title = (template.get('catalog_ui') or {}).get('title')
+        if isinstance(title, dict):
+            title = title.get('en_US') or next(iter(title.values()), '')
+        if isinstance(title, str) and title.strip():
+            return title.strip()
+    return f'CT {vmid}'
+
+
 def update(vmid, acknowledge_external_data=False, proposal=None, keep_backup=None):
    operation = 'recreate' if proposal is not None else 'update'
    msg_info(translate('Checking the container before the update...') if operation == 'update'
@@ -169,9 +180,11 @@ def update(vmid, acknowledge_external_data=False, proposal=None, keep_backup=Non
        kept = kept_settings(changes, desired['deployment'])
        if kept:
            msg_info2(f"{translate('Keeping the settings changed in Proxmox:')} {', '.join(kept)}")
-        transaction.apply(instances.ROOT, vmid, archive, operation, proposal=proposal,
-                          registry_digest=digest, acknowledge_external_data=acknowledge_external_data,
-                          keep_backup=file_storage)
+        import oci_operation_notice
+        with oci_operation_notice.operation([vmid], operation, application_name(record, vmid)):
+            transaction.apply(instances.ROOT, vmid, archive, operation, proposal=proposal,
+                              registry_digest=digest, acknowledge_external_data=acknowledge_external_data,
+                              keep_backup=file_storage)


 def main():