overhaul app tracking and update orchestration

- Generate and ship a verified 389-app tracking catalog with 23 runtime overrides, fallback detectors, ports, logos, and Docker Hub tag previews.
- Support modern Proxmox VE Helper-Scripts markers, historical installations, and official or manual app deployments.
- Rework the LXC App and Updates tabs with cached suggestions, explicit discovery, version tracking, web links, custom updaters, and complete i18n.
- Add independent OS, app, Docker Engine, Docker image, bulk, and scheduled update targets.
- Add digest-based Docker inventory, Compose dependency grouping, safe standalone-container recreation with rollback, and package-scoped Docker Engine updates.
- Refresh per-LXC caches after lifecycle and update tasks, then emit idempotent notifications based on the verified final state.
- Harden Coral USB recovery by removing orphaned gasket DKMS registrations and validating that dpkg is healthy before reporting success.
This commit is contained in:
MacRimi
2026-08-23 12:43:03 +02:00
parent 7244201810
commit 0251f77331
27 changed files with 11631 additions and 893 deletions
+229
View File
@@ -190,6 +190,215 @@ cleanup_broken_gasket_dkms() {
esac
}
# ============================================================
# Orphan gasket-dkms detection and assisted cleanup
# ============================================================
# The legacy Coral installer (`scripts/install_coral_pve.sh`, retired
# in April 2026) unconditionally installed the gasket-dkms .deb even
# on USB-only hosts. On modern kernels (6.12+) the upstream `gasket
# 1.0` source no longer compiles, so DKMS autoinstall fails, dpkg
# leaves the package half-configured, and every subsequent apt-get
# call errors out.
#
# On hosts without Coral PCIe/M.2 hardware, this package is pure
# residue with no functional purpose — the Coral USB path uses
# libedgetpu1 in userspace and does not need the kernel driver.
# We detect that combination (gasket-dkms present + no PCIe device
# on the bus) and offer explicit, opt-in cleanup.
#
# Design guardrails:
# * Only offered when CORAL_PCIE_COUNT == 0. Never runs on hosts
# with a Coral PCIe/M.2 device present, even if the package is
# broken — those users need the package, and the fix is a
# rebuild (via `install_gasket_apex_dkms`), not a purge.
# * User confirmation always required — nothing removes silently.
# * The Coral USB path (libedgetpu1-std / -max) is never touched.
# Set by `detect_orphan_gasket_dkms`. Empty when no orphan state
# is present; otherwise one of "healthy_orphan" (package installed
# cleanly but hardware absent) or "broken_orphan" (package in a
# half-configured / half-installed / unpacked state and hardware
# absent — this is DavidOliMar's case and blocks apt).
GASKET_ORPHAN_STATE=""
detect_orphan_gasket_dkms() {
GASKET_ORPHAN_STATE=""
# Hardware present -> not orphan, never touched by this flow.
[[ "$CORAL_PCIE_COUNT" -gt 0 ]] && return 0
local pkg_status
pkg_status=$(dpkg-query -W -f='${Status}' gasket-dkms 2>/dev/null || echo "")
[[ -z "$pkg_status" ]] && return 0 # package not installed at all
if [[ "$pkg_status" == *"ok installed"* ]]; then
GASKET_ORPHAN_STATE="healthy_orphan"
elif [[ "$pkg_status" == *"half-configured"* \
|| "$pkg_status" == *"half-installed"* \
|| "$pkg_status" == *"unpacked"* \
|| "$pkg_status" == *"failed-config"* \
|| "$pkg_status" == *"reinst-required"* ]]; then
GASKET_ORPHAN_STATE="broken_orphan"
fi
}
cleanup_orphan_gasket_dkms() {
# Return codes:
# 0 cleanup completed and every final verification passed
# 1 operator cancelled before any change was made
# 2 cleanup ran, but dpkg/DKMS could not be verified as healthy
local msg=""
msg+="\n$(translate 'A legacy gasket-dkms package was found on this host, but no Coral M.2 / PCIe hardware is present.')\n\n"
msg+="$(translate 'This package was installed by older versions of the ProxMenux Coral installer that placed the M.2 kernel driver on every system, including USB-only setups. It is not needed for Coral USB devices, which use libedgetpu1-std / libedgetpu1-max only.')\n\n"
if [[ "$GASKET_ORPHAN_STATE" == "broken_orphan" ]]; then
msg+="\Z1\Zb$(translate 'The package is currently in a broken state and is blocking apt updates on this system.')\Zn\n\n"
fi
msg+="\Zb$(translate 'This cleanup will:')\Zn\n"
msg+="$(translate 'Purge the gasket-dkms package')\n"
msg+="$(translate 'Remove every registered gasket DKMS version')\n"
msg+="$(translate 'Run apt-get install -f to complete any pending package configurations')\n\n"
if [[ "$CORAL_USB_COUNT" -gt 0 || "$CORAL_USB_INSTALLED" == "true" ]]; then
msg+="\Z2$(translate 'Your Coral USB device and its runtime (libedgetpu1) will NOT be affected.')\Zn\n\n"
fi
msg+="$(translate 'If you have a Coral M.2 / PCIe device that is physically installed but not detected by lspci, cancel here and check your hardware first before proceeding.')\n\n"
msg+="\Zb$(translate 'Do you want to proceed with the cleanup?')\Zn"
if ! dialog --backtitle "ProxMenux" --colors \
--title "$(translate 'Legacy gasket-dkms detected')" \
--defaultno --yesno "$msg" 24 84; then
return 1
fi
show_proxmenux_logo
msg_title "$(translate 'Cleanup legacy gasket-dkms')"
export DEBIAN_FRONTEND=noninteractive
msg_info "$(translate 'Purging gasket-dkms package...')"
# Try the clean apt path first; fall back to dpkg force flags if the
# package state prevents apt from resolving the removal itself.
if ! apt-get remove --purge -y gasket-dkms >>"$LOG_FILE" 2>&1; then
dpkg --remove --force-remove-reinstreq gasket-dkms >>"$LOG_FILE" 2>&1 || true
dpkg --purge --force-all gasket-dkms >>"$LOG_FILE" 2>&1 || true
fi
# A host can retain more than the historical gasket/1.0 entry. Read
# every version known by DKMS and also include stale version trees
# that a broken package configuration may have left behind.
local versions=""
local version=""
local dkms_remove_failed=0
if command -v dkms >/dev/null 2>&1; then
versions=$({
dkms status 2>/dev/null \
| awk -F'[,/ ]+' '/^gasket/ {print $2}'
if [[ -d /var/lib/dkms/gasket ]]; then
find /var/lib/dkms/gasket -mindepth 1 -maxdepth 1 -type d \
-exec basename {} \; 2>/dev/null
fi
} | sed '/^$/d' | sort -u)
if [[ -n "$versions" ]]; then
msg_info "$(translate 'Removing every registered gasket DKMS version...')"
while IFS= read -r version; do
[[ -z "$version" ]] && continue
if ! dkms remove -m gasket -v "$version" --all >>"$LOG_FILE" 2>&1; then
dkms_remove_failed=1
fi
done <<<"$versions"
if [[ "$dkms_remove_failed" -eq 0 ]]; then
msg_ok "$(translate 'DKMS registrations removed.')"
else
msg_warn "$(translate 'Some DKMS removals reported errors; final verification will determine the result.')"
fi
fi
fi
local repair_failed=0
msg_info "$(translate 'Completing pending package configurations...')"
if apt-get install -f -y >>"$LOG_FILE" 2>&1; then
msg_ok "$(translate 'Package configurations completed.')"
else
repair_failed=1
msg_warn "$(translate 'Some packages still need attention; review') ${LOG_FILE}"
fi
# Final verification is authoritative. Any dpkg state whose second
# character is not "n" (not installed) or "c" (only config files)
# still represents package payload or unfinished package work.
local package_state=""
local package_remnant=""
local dkms_remnant=""
local audit_output=""
package_state=$(dpkg -l gasket-dkms 2>/dev/null \
| awk '$2 == "gasket-dkms" {print $1; exit}')
if [[ -n "$package_state" && ! "$package_state" =~ ^.[nc] ]]; then
package_remnant="$package_state"
fi
if command -v dkms >/dev/null 2>&1; then
dkms_remnant=$(dkms status 2>/dev/null \
| grep -E '^gasket([,/ ]|$)' \
| head -n1)
fi
audit_output=$(dpkg --audit 2>&1 || true)
if [[ -n "$audit_output" ]]; then
{
echo "---- dpkg --audit after legacy gasket-dkms cleanup ----"
printf '%s\n' "$audit_output"
} >>"$LOG_FILE"
fi
if [[ -n "$package_remnant" ]]; then
repair_failed=1
msg_warn "$(translate 'gasket-dkms is still reported by dpkg in state:') ${package_remnant}. $(translate 'Manual review is required.')"
else
msg_ok "$(translate 'gasket-dkms has been fully removed from this system.')"
fi
if [[ -n "$dkms_remnant" ]]; then
repair_failed=1
msg_warn "$(translate 'A gasket DKMS registration is still present:') ${dkms_remnant}"
else
msg_ok "$(translate 'No gasket DKMS registrations remain.')"
fi
if [[ -n "$audit_output" ]]; then
repair_failed=1
msg_warn "$(translate 'dpkg still reports unfinished package work; review') ${LOG_FILE}"
else
msg_ok "$(translate 'The dpkg package database is clean.')"
fi
# Clear the component marker only after gasket itself is confirmed
# absent. A separate dpkg audit problem must still make the overall
# operation fail, but should not leave a false Coral PCIe component.
if [[ -z "$package_remnant" && -z "$dkms_remnant" ]]; then
if declare -f update_component_status >/dev/null 2>&1; then
update_component_status "coral_driver" "removed" "" "gpu" '{}' >/dev/null 2>&1 || true
fi
rm -f /var/lib/proxmenux/coral_gasket_version 2>/dev/null || true
fi
if [[ "$repair_failed" -ne 0 ]]; then
echo
msg_error "$(translate 'Legacy gasket-dkms cleanup could not be verified as complete.')"
msg_warn "$(translate 'No reboot was started. Review the log before retrying:') ${LOG_FILE}"
return 2
fi
echo
msg_success "$(translate 'Cleanup completed. A reboot is recommended to fully apply pending kernel package configurations.')"
restart_prompt
return 0
}
clone_gasket_sources() {
# Primary: feranick/gasket-driver — community fork, actively maintained,
# carries patches for kernel 6.10/6.12/6.13.
@@ -655,10 +864,12 @@ restart_prompt() {
# Main orchestrator
# ============================================================
main() {
local cleanup_rc=0
: >"$LOG_FILE"
detect_coral_hardware
detect_coral_install_state
detect_orphan_gasket_dkms
# No hardware AND no leftover install → nothing to do.
if [[ "$CORAL_PCIE_COUNT" -eq 0 && "$CORAL_USB_COUNT" -eq 0 ]] \
@@ -667,6 +878,24 @@ main() {
exit 0
fi
# Legacy gasket-dkms package left behind by the retired installer
# (see detect_orphan_gasket_dkms header). Offer explicit cleanup
# before the normal action menu so the user sees a curated fix
# instead of a broken install/remove flow. If the operator cancels,
# we still fall through to the standard menu (they may want to act
# on the USB runtime independently).
if [[ -n "$GASKET_ORPHAN_STATE" ]]; then
if cleanup_orphan_gasket_dkms; then
exit 0
else
cleanup_rc=$?
# Return 1 means the operator cancelled and may still use the
# standard menu. A failed repair must stop here instead of
# continuing as though the package manager were healthy.
[[ "$cleanup_rc" -eq 1 ]] || exit "$cleanup_rc"
fi
fi
# If something is already installed, offer reinstall/uninstall choice.
# Same UX as nvidia_installer.sh. When nothing is installed yet,
# ACTION="install" automatically.
+140 -35
View File
@@ -10,14 +10,28 @@
# BACKUP — "1" to snapshot with vzdump first, "0" to skip
# BACKUP_STORAGE — PVE storage name for vzdump (required when BACKUP=1)
# RESTART — "1" to `pct reboot` after update, "0" to skip
# UPDATE_COMMAND — optional; user-defined bash string. When set
# RUN_HELPER — "1" to run the verified community-scripts
# updater referenced by /usr/bin/update, "0"
# to leave it alone. Never inferred from names.
# UPDATE_COMMAND — optional user-defined bash string. When set
# and TARGET is "app" or "both", the script
# runs this VIA sh -c inside the CT instead of
# /usr/bin/update. This IS the one place we
# runs it VIA sh -c inside the CT. A custom
# command always replaces RUN_HELPER for safety.
# This IS the one place we
# intentionally use sh -c with a variable
# payload — the threat model matches "user
# typed it via pct exec themselves"; ProxMenux
# does not compose or interpret the command.
# ALLOW_HELPER_WITH_CUSTOM — "1" only for an explicit multi-app plan
# where RUN_HELPER belongs to one registered app and
# UPDATE_COMMAND contains other registered apps. The
# default "0" preserves the custom-replaces-helper rule
# for single-app and legacy callers.
# DOCKER_STANDALONE_TARGETS — optional comma-separated Docker container
# names. Each is recreated transactionally by the
# protected host-side Docker recreation helper.
# UPDATE_DOCKER_ENGINE — "1" to update only the installed Docker Engine
# package stack, without upgrading unrelated OS packages.
#
# Exit codes:
# 0 everything requested completed OK
@@ -26,8 +40,9 @@
# 3 pre-update backup failed (abort so the user still has a rollback)
# 4 OS update failed OR OS family not supported for automated updates
# 5 TARGET=app requested but no update method (neither UPDATE_COMMAND
# nor /usr/bin/update) available in the CT
# nor explicitly-enabled verified helper) available in the CT
# 6 post-update restart failed
# 7 another ProxMenux update is already running for this CT
#
# The frontend surfaces exit code + duration in a follow-up POST to
# /api/lxc-updates/<vmid>/applied so the notification event fires with
@@ -40,6 +55,41 @@ set -o pipefail
: "${TARGET:?TARGET is required}"
BACKUP="${BACKUP:-0}"
RESTART="${RESTART:-0}"
RUN_HELPER="${RUN_HELPER:-0}"
UPDATE_COMMAND="${UPDATE_COMMAND:-}"
ALLOW_HELPER_WITH_CUSTOM="${ALLOW_HELPER_WITH_CUSTOM:-0}"
DOCKER_STANDALONE_TARGETS="${DOCKER_STANDALONE_TARGETS:-}"
UPDATE_DOCKER_ENGINE="${UPDATE_DOCKER_ENGINE:-0}"
if [[ ! "$VMID" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: VMID must be a positive integer." >&2
exit 1
fi
if [[ "$TARGET" != "os" && "$TARGET" != "app" && "$TARGET" != "both" ]]; then
echo "ERROR: TARGET must be os, app, or both." >&2
exit 4
fi
if [[ "$RUN_HELPER" != "0" && "$RUN_HELPER" != "1" ]]; then
echo "ERROR: RUN_HELPER must be 0 or 1." >&2
exit 5
fi
if [[ "$ALLOW_HELPER_WITH_CUSTOM" != "0" && "$ALLOW_HELPER_WITH_CUSTOM" != "1" ]]; then
echo "ERROR: ALLOW_HELPER_WITH_CUSTOM must be 0 or 1." >&2
exit 5
fi
if [[ "$UPDATE_DOCKER_ENGINE" != "0" && "$UPDATE_DOCKER_ENGINE" != "1" ]]; then
echo "ERROR: UPDATE_DOCKER_ENGINE must be 0 or 1." >&2
exit 5
fi
# One update per CT at a time, regardless of whether it came from the
# UI or the scheduler. The descriptor remains open for this process.
LOCK_DIR="${PROXMENUX_LOCK_DIR:-/run/lock}"
exec 9>"${LOCK_DIR}/proxmenux-lxc-update-${VMID}.lock"
if ! flock -n 9; then
echo "ERROR: another ProxMenux update is already running for CT $VMID." >&2
exit 7
fi
STARTED_AT=$(date -Iseconds)
NODE=$(hostname)
@@ -48,6 +98,7 @@ echo "Started: $STARTED_AT"
echo "Target: $TARGET"
echo "Backup: $BACKUP${BACKUP_STORAGE:+ (storage: $BACKUP_STORAGE)}"
echo "Restart: $RESTART"
echo "Helper: $RUN_HELPER"
echo
# 1) CT must exist on this node.
@@ -56,14 +107,35 @@ if ! pct list | awk 'NR>1 {print $1}' | grep -qE "^${VMID}$"; then
exit 1
fi
# 2) CT must be running for pct exec. Auto-start stopped CTs.
# 2) CT must be running for pct exec. Auto-start stopped CTs, then
# restore their original stopped state on every exit path.
STATE=$(pct status "$VMID" | awk '{print $2}')
STARTED_BY_PROXMENUX=0
restore_original_state() {
local rc=$?
trap - EXIT INT TERM
if [[ "$STARTED_BY_PROXMENUX" == "1" ]]; then
echo
echo "Restoring original state: stopping CT $VMID"
if ! pct shutdown "$VMID" --timeout 60; then
echo "ERROR: update finished but CT $VMID could not be returned to its original stopped state." >&2
if [[ "$rc" -eq 0 ]]; then
rc=6
fi
fi
fi
exit "$rc"
}
trap restore_original_state EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
if [[ "$STATE" != "running" ]]; then
echo "CT is $STATE. Starting it before applying updates…"
if ! pct start "$VMID"; then
echo "ERROR: failed to start CT $VMID." >&2
exit 2
fi
STARTED_BY_PROXMENUX=1
# give the CT a moment for services to come up
sleep 3
fi
@@ -123,41 +195,55 @@ if [[ "$TARGET" == "os" || "$TARGET" == "both" ]]; then
echo
fi
# 6) Application update. Precedence:
# a) /usr/bin/update present (community-scripts convention)
# runs the community-scripts helper FROM THE HOST with CTID
# env var. Their build.func framework requires CTID + host-only
# `pveversion`, so `pct exec ... /usr/bin/update` inside the CT
# always fails ("You need to set 'CTID' variable"). We parse
# the ct/<slug>.sh URL from /usr/bin/update and re-fetch it
# here with CTID set. PHS_SILENT=1 keeps it non-interactive.
# b) UPDATE_COMMAND env var set → run it verbatim via `sh -c`
# 6) Application update. Explicit methods only:
# a) RUN_HELPER=1 + a valid /usr/bin/update wrapper
# parses the ct/<slug>.sh URL from the wrapper, canonicalises it
# to the official repository, then runs the current helper
# inside the CT with PHS_SILENT=1.
# b) UPDATE_COMMAND set → run it verbatim via `sh -c`
# inside the CT. The one intentional shell-exec-with-variable
# in ProxMenux — see header comment for threat-model rationale.
# Both can run in the same invocation: the helper first (if
# present), then the per-app custom commands.
# UPDATE_COMMAND always wins if a legacy caller also sets RUN_HELPER.
# A hostname/tag/cache guess is never executable evidence.
if [[ "$TARGET" == "app" || "$TARGET" == "both" ]]; then
APP_METHOD_RAN=0
UPDATE_URL=""
RESOLVED_SLUG=""
if pct exec "$VMID" -- test -f /usr/bin/update 2>/dev/null; then
UPDATE_URL=$(pct exec "$VMID" -- cat /usr/bin/update 2>/dev/null | grep -oE 'https?://[^"'"'"' ]+ct/[a-zA-Z0-9._-]+\.sh' | head -1)
RESOLVED_SLUG=$(echo "$UPDATE_URL" | sed -nE 's|.*/ct/([a-zA-Z0-9._-]+)\.sh$|\1|p')
if [[ -n "$UPDATE_COMMAND" && "$RUN_HELPER" == "1" && "$ALLOW_HELPER_WITH_CUSTOM" != "1" ]]; then
echo "Custom update command configured; skipping Proxmox VE Helper-Scripts updater."
RUN_HELPER=0
fi
# HELPER_SLUG env is a passthrough from the backend when the CT no
# longer carries /usr/bin/update (older installs where the file was
# removed) but the community-scripts slug is known via hostname
# match against the helpers_cache. Lets us run the same host-side
# updater without requiring the on-CT marker file.
if [[ -z "$RESOLVED_SLUG" && -n "$HELPER_SLUG" ]]; then
if [[ "$HELPER_SLUG" =~ ^[a-zA-Z0-9._-]+$ ]]; then
RESOLVED_SLUG="$HELPER_SLUG"
UPDATE_URL="https://raw.githubusercontent.com/community-scripts/ProxmoxVE/main/ct/${RESOLVED_SLUG}.sh"
else
echo "WARN: HELPER_SLUG contains invalid characters — ignored." >&2
if [[ "$UPDATE_DOCKER_ENGINE" == "1" ]]; then
echo "--- Updating Docker Engine only ---"
if ! python3 /usr/local/share/proxmenux/monitor-app/usr/bin/update_docker_engine.py \
--vmid "$VMID"; then
echo "ERROR: Docker Engine update failed." >&2
APP_FAILED=1
fi
APP_METHOD_RAN=1
echo
fi
if [[ -n "$UPDATE_URL" && -n "$RESOLVED_SLUG" ]]; then
if [[ "$RUN_HELPER" == "1" ]]; then
UPDATE_URL=""
RESOLVED_SLUG=""
if pct exec "$VMID" -- test -f /usr/bin/update 2>/dev/null; then
UPDATE_URL=$(pct exec "$VMID" -- cat /usr/bin/update 2>/dev/null | grep -oE 'https?://[^"'"'"' ]+ct/[a-zA-Z0-9._-]+\.sh' | head -1)
RESOLVED_SLUG=$(echo "$UPDATE_URL" | sed -nE 's|.*/ct/([a-zA-Z0-9._-]+)\.sh$|\1|p')
fi
case "$RESOLVED_SLUG" in
alpine|archlinux|archlinux-vm|debian|fedora|gentoo|opensuse|ubuntu)
echo "ERROR: /usr/bin/update references the base-OS helper '$RESOLVED_SLUG', not an application updater." >&2
APP_FAILED=1
RESOLVED_SLUG=""
;;
esac
if [[ -z "$RESOLVED_SLUG" ]]; then
if [[ "$APP_FAILED" -eq 0 ]]; then
echo "ERROR: RUN_HELPER=1 but /usr/bin/update contains no valid community-scripts app reference." >&2
APP_FAILED=1
fi
else
# Never execute the arbitrary URL embedded in the CT. The slug is
# constrained by the parser; fetch the canonical upstream path.
UPDATE_URL="https://raw.githubusercontent.com/community-scripts/ProxmoxVE/main/ct/${RESOLVED_SLUG}.sh"
echo "--- Running community-scripts helper (slug: $RESOLVED_SLUG) ---"
# Community-scripts' build.func in start() dispatches on
# `command -v pveversion`: present → install_script (whiptail
@@ -184,6 +270,7 @@ if [[ "$TARGET" == "app" || "$TARGET" == "both" ]]; then
fi
APP_METHOD_RAN=1
echo
fi
fi
if [[ -n "$UPDATE_COMMAND" ]]; then
echo "--- Running user-defined update command ---"
@@ -195,9 +282,27 @@ if [[ "$TARGET" == "app" || "$TARGET" == "both" ]]; then
APP_METHOD_RAN=1
echo
fi
if [[ -n "$DOCKER_STANDALONE_TARGETS" ]]; then
IFS=',' read -r -a DOCKER_TARGETS <<< "$DOCKER_STANDALONE_TARGETS"
for DOCKER_CONTAINER in "${DOCKER_TARGETS[@]}"; do
if [[ ! "$DOCKER_CONTAINER" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$ ]]; then
echo "ERROR: invalid Docker container target '$DOCKER_CONTAINER'." >&2
APP_FAILED=1
continue
fi
echo "--- Recreating standalone Docker container: $DOCKER_CONTAINER ---"
if ! python3 /usr/local/share/proxmenux/monitor-app/usr/bin/recreate_docker_container.py \
--vmid "$VMID" --container "$DOCKER_CONTAINER"; then
echo "ERROR: protected Docker recreation failed for '$DOCKER_CONTAINER'." >&2
APP_FAILED=1
fi
APP_METHOD_RAN=1
echo
done
fi
if [[ "$APP_METHOD_RAN" -eq 0 ]]; then
if [[ "$TARGET" == "app" ]]; then
echo "ERROR: TARGET=app but no update method (UPDATE_COMMAND unset AND /usr/bin/update missing) in CT $VMID." >&2
echo "ERROR: TARGET=app but no update method was explicitly selected for CT $VMID." >&2
exit 5
else
echo "No app update method available in this CT — skipping app update step."
@@ -209,7 +314,7 @@ fi
# 7) If either branch failed, abort here BEFORE the optional reboot so
# the CT stays in the pre-update state and the user can inspect it.
if (( OS_FAILED || APP_FAILED )); then
echo "=== Update FAILED — CT left running for inspection. ==="
echo "=== Update FAILED. ==="
exit 4
fi