mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-01 18:32:13 +00:00
272 lines
10 KiB
Bash
Executable File
272 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
|
|
HOOK_NAME=proxmenux-stack-dependencies.sh
|
|
COMPOSE_JOB_HELPER=/var/lib/vz/snippets/proxmenux-compose-jobs.py
|
|
# Longest wait for a dependency that is being backed up, and for any other lock.
|
|
BACKUP_LOCK_WAIT=${PROXMENUX_STACK_BACKUP_LOCK_WAIT:-14400}
|
|
OTHER_LOCK_WAIT=${PROXMENUX_STACK_OTHER_LOCK_WAIT:-300}
|
|
# vzdump starts again a guest that was running before its backup right after releasing it.
|
|
RESTART_GRACE=${PROXMENUX_STACK_RESTART_GRACE:-20}
|
|
POLL=${PROXMENUX_STACK_POLL:-5}
|
|
# Earlier published versions of this script, replaced on update.
|
|
PREVIOUS_RELEASES=(
|
|
4bc477bcc35e229d1c964c9372838f0973cbe6f1e3e41f0fb486e9b724247310
|
|
243b0dca81b1b155ca169a4d0e11cdb8ca55974c65ede6a4a9be8f4d536a141e
|
|
c28b69acd7cc8e66d3aec3e8b2ec86bda0e4f91193ec78549f1965f2c859c0d1
|
|
02c50ef17f2411ef22f4d382c51b34a7a0974a446d4a155ecb63fb1e805a7c59
|
|
)
|
|
|
|
die() {
|
|
printf 'ERROR: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
find_snippet_storage() {
|
|
local storage
|
|
if pvesm status --content snippets 2>/dev/null \
|
|
| awk 'NR > 1 && $1 == "local" && $3 == "active" {found=1} END {exit !found}'; then
|
|
printf 'local'
|
|
return
|
|
fi
|
|
storage=$(pvesm status --content snippets 2>/dev/null \
|
|
| awk 'NR > 1 && $3 == "active" {print $1; exit}')
|
|
if [[ -z $storage ]]; then
|
|
# A new Proxmox installation accepts no snippets on any storage.
|
|
enable_local_snippets || die "No active storage supports snippets"
|
|
storage=local
|
|
fi
|
|
printf '%s' "$storage"
|
|
}
|
|
|
|
enable_local_snippets() {
|
|
local content
|
|
content=$(pvesh get /storage/local --output-format json 2>/dev/null | jq -r '.content // empty')
|
|
[[ -n $content ]] || return 1
|
|
pvesm set local --content "${content},snippets" >/dev/null 2>&1 || return 1
|
|
pvesm status --content snippets 2>/dev/null \
|
|
| awk 'NR > 1 && $1 == "local" && $3 == "active" {found=1} END {exit !found}'
|
|
}
|
|
|
|
install_hook() {
|
|
local main_id=${1:?missing main VMID} source_config=${2:?missing lifecycle JSON}
|
|
local storage hook_volume hook_path target_config
|
|
[[ $main_id =~ ^[0-9]+$ ]] || die "Invalid main VMID"
|
|
jq -e '
|
|
.schema == 1 and
|
|
(.dependencies | type == "array") and
|
|
(.dependencies | length > 0) and
|
|
(all(.dependencies[];
|
|
(.vmid | type == "number") and
|
|
(.label | type == "string") and
|
|
(.healthcheck.type | IN("exec", "http", "running", "tcp", "completed")) and
|
|
(.healthcheck.timeout_seconds | type == "number") and
|
|
(if .healthcheck.type == "tcp" then
|
|
(.healthcheck.host | type == "string") and
|
|
(.healthcheck.host | test("^([0-9]{1,3}\\.){3}[0-9]{1,3}$")) and
|
|
(.healthcheck.host | split(".") | all(.[]; tonumber < 256)) and
|
|
(.healthcheck.port | type == "number") and
|
|
(.healthcheck.port > 0 and .healthcheck.port < 65536 and .healthcheck.port == (.healthcheck.port | floor))
|
|
elif .healthcheck.type == "completed" then
|
|
(.healthcheck.job | type == "object") and
|
|
(.healthcheck.job.installation_id | type == "string") and
|
|
(.healthcheck.job.installation_id | test("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")) and
|
|
(.healthcheck.job.image_digest | type == "string") and
|
|
(.healthcheck.job.image_digest | test("^sha256:[0-9a-f]{64}$")) and
|
|
(.healthcheck.job.expected_config_fingerprint | type == "string") and
|
|
(.healthcheck.job.expected_config_fingerprint | test("^[0-9a-f]{64}$")) and
|
|
(.healthcheck.job.timeout_seconds == .healthcheck.timeout_seconds) and
|
|
(.healthcheck.timeout_seconds >= 30 and .healthcheck.timeout_seconds <= 3600 and .healthcheck.timeout_seconds == (.healthcheck.timeout_seconds | floor))
|
|
else true end)
|
|
))
|
|
' "$source_config" >/dev/null || die "Invalid dependency contract"
|
|
|
|
if jq -e 'any(.dependencies[]; .healthcheck.type == "completed")' "$source_config" >/dev/null; then
|
|
local helper_source helper_temporary
|
|
helper_source="$(dirname -- "$0")/oci_compose_jobs.py"
|
|
[[ -f $helper_source && ! -L $helper_source ]] || die "Native OCI job helper is missing"
|
|
install -d -m 0755 "$(dirname -- "$COMPOSE_JOB_HELPER")"
|
|
helper_temporary=$(mktemp "${COMPOSE_JOB_HELPER}.XXXXXX")
|
|
if ! install -m 0755 "$helper_source" "$helper_temporary" || ! mv -f -- "$helper_temporary" "$COMPOSE_JOB_HELPER"; then
|
|
rm -f -- "$helper_temporary"
|
|
die "Could not install the persistent native OCI job helper"
|
|
fi
|
|
fi
|
|
|
|
storage=$(find_snippet_storage)
|
|
hook_volume="${storage}:snippets/${HOOK_NAME}"
|
|
hook_path=$(pvesm path "$hook_volume")
|
|
install -D -m 0755 "$0" "$hook_path"
|
|
|
|
target_config="/etc/pve/priv/proxmenux-stack-${main_id}.json"
|
|
umask 077
|
|
cat "$source_config" >"$target_config"
|
|
pct set "$main_id" --hookscript "$hook_volume" >/dev/null
|
|
printf 'Proxmox hookscript installed: CT %s starts its dependencies through %s\n' \
|
|
"$main_id" "$hook_volume"
|
|
}
|
|
|
|
# The installed copies of an earlier release take this version; a copy
|
|
# changed by hand is left as it is.
|
|
refresh_hooks() {
|
|
local volume path current known
|
|
while IFS= read -r volume; do
|
|
path=$(pvesm path "$volume" 2>/dev/null) || continue
|
|
[[ -f $path && ! -L $path ]] || continue
|
|
current=$(sha256sum "$path" | cut -d' ' -f1)
|
|
for known in "${PREVIOUS_RELEASES[@]}"; do
|
|
if [[ $current == "$known" ]]; then
|
|
install -m 0755 "$0" "$path.new" && mv -f "$path.new" "$path"
|
|
printf 'Updated %s\n' "$path"
|
|
break
|
|
fi
|
|
done
|
|
done < <(sed -n "s/^hookscript:[[:space:]]*\([^[:space:]]*:snippets\/${HOOK_NAME//./\\.}\)[[:space:]]*$/\1/p" \
|
|
/etc/pve/lxc/*.conf 2>/dev/null | sort -u)
|
|
}
|
|
|
|
is_running() {
|
|
[[ $(pct status "$1" 2>/dev/null || true) == "status: running" ]]
|
|
}
|
|
|
|
container_lock() {
|
|
pct config "$1" 2>/dev/null | sed -n '/^lock:/{s/^lock:[[:space:]]*//p;q;}' || true
|
|
}
|
|
|
|
# A dependency stopped for its own backup stays locked until vzdump has
|
|
# saved it, and vzdump starts it again when it was running before.
|
|
wait_for_release() {
|
|
local id=$1 label=$2 lock limit waited=0
|
|
lock=$(container_lock "$id")
|
|
[[ -n $lock ]] || return 0
|
|
limit=$OTHER_LOCK_WAIT
|
|
if [[ $lock == backup ]]; then
|
|
limit=$BACKUP_LOCK_WAIT
|
|
fi
|
|
printf 'Dependency %s (CT %s) is locked (%s); waiting until it is released...\n' "$label" "$id" "$lock"
|
|
while lock=$(container_lock "$id"); [[ -n $lock ]]; do
|
|
(( waited < limit )) || die "$label (CT $id) is still locked ($lock) after ${waited}s"
|
|
sleep "$POLL"
|
|
waited=$((waited + POLL))
|
|
done
|
|
printf 'Dependency %s (CT %s) was released.\n' "$label" "$id"
|
|
waited=0
|
|
while (( waited < RESTART_GRACE )) && ! is_running "$id"; do
|
|
sleep 2
|
|
waited=$((waited + 2))
|
|
done
|
|
}
|
|
|
|
dependency_is_healthy() {
|
|
local id=$1 healthcheck=$2 type url
|
|
type=$(jq -r '.type' <<<"$healthcheck")
|
|
case "$type" in
|
|
running)
|
|
[[ $(pct status "$id" 2>/dev/null || true) == "status: running" ]]
|
|
;;
|
|
exec)
|
|
local -a command=()
|
|
mapfile -t command < <(jq -r '.argv[]' <<<"$healthcheck")
|
|
((${#command[@]} > 0)) || return 1
|
|
pct exec "$id" -- "${command[@]}" >/dev/null 2>&1 9>&-
|
|
;;
|
|
http)
|
|
url=$(jq -r '.url' <<<"$healthcheck")
|
|
curl -fsS --max-time 3 "$url" >/dev/null 2>&1
|
|
;;
|
|
tcp)
|
|
local host port
|
|
host=$(jq -r '.host' <<<"$healthcheck")
|
|
port=$(jq -r '.port' <<<"$healthcheck")
|
|
python3 - "$host" "$port" <<'PYTCP' >/dev/null 2>&1
|
|
import ipaddress, socket, sys
|
|
host = str(ipaddress.IPv4Address(sys.argv[1]))
|
|
port = int(sys.argv[2])
|
|
if not 0 < port < 65536:
|
|
raise ValueError('Invalid TCP port')
|
|
with socket.create_connection((host, port), timeout=3):
|
|
pass
|
|
PYTCP
|
|
;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
start_dependencies() {
|
|
local main_id=$1 config="/etc/pve/priv/proxmenux-stack-${1}.json"
|
|
local encoded dependency id label healthcheck timeout elapsed
|
|
[[ -r $config ]] || die "Missing dependency contract for main CT $main_id"
|
|
|
|
exec 9>"/run/lock/proxmenux-stack-${main_id}.lock"
|
|
flock 9
|
|
while IFS= read -r encoded; do
|
|
[[ -n $encoded ]] || continue
|
|
dependency=$(base64 -d <<<"$encoded")
|
|
id=$(jq -r '.vmid' <<<"$dependency")
|
|
label=$(jq -r '.label' <<<"$dependency")
|
|
healthcheck=$(jq -c '.healthcheck' <<<"$dependency")
|
|
timeout=$(jq -r '.healthcheck.timeout_seconds' <<<"$dependency")
|
|
[[ $id =~ ^[0-9]+$ && $timeout =~ ^[0-9]+$ && $timeout -gt 0 ]] \
|
|
|| die "Invalid dependency in $config"
|
|
pct config "$id" >/dev/null 2>&1 \
|
|
|| die "Dependency $label (CT $id) does not exist"
|
|
|
|
if [[ $(jq -r '.type' <<<"$healthcheck") == completed ]]; then
|
|
[[ -f $COMPOSE_JOB_HELPER && ! -L $COMPOSE_JOB_HELPER ]] || die "Native OCI job helper is missing"
|
|
local helper_owner helper_mode
|
|
helper_owner=$(stat -c '%u' "$COMPOSE_JOB_HELPER")
|
|
helper_mode=$(stat -c '%a' "$COMPOSE_JOB_HELPER")
|
|
[[ $helper_owner == 0 && $helper_mode =~ ^[0-7]{3,4}$ ]] && (( (8#$helper_mode & 8#022) == 0 )) \
|
|
|| die "Native OCI job helper is not owned safely by root"
|
|
printf 'Waiting for completed job %s (CT %s)...\n' "$label" "$id"
|
|
python3 -I "$COMPOSE_JOB_HELPER" ensure "$id" --spec-json "$(jq -c '.job' <<<"$healthcheck")" >/dev/null 9>&- \
|
|
|| die "Job $label (CT $id) did not complete successfully"
|
|
printf 'Dependency %s (CT %s): ready.\n' "$label" "$id"
|
|
continue
|
|
fi
|
|
|
|
is_running "$id" || wait_for_release "$id" "$label"
|
|
if ! is_running "$id"; then
|
|
printf 'Starting dependency %s (CT %s)...\n' "$label" "$id"
|
|
# What pct start leaves running, such as the DHCP client, must not keep the lock.
|
|
pct start "$id" 9>&- || is_running "$id" || die "Could not start $label (CT $id)"
|
|
else
|
|
printf 'Dependency %s (CT %s) was already running.\n' "$label" "$id"
|
|
fi
|
|
|
|
elapsed=0
|
|
while (( elapsed < timeout )); do
|
|
if dependency_is_healthy "$id" "$healthcheck"; then
|
|
printf 'Dependency %s (CT %s): ready.\n' "$label" "$id"
|
|
break
|
|
fi
|
|
[[ $(pct status "$id" 2>/dev/null || true) == "status: running" ]] \
|
|
|| die "$label (CT $id) stopped while starting"
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
(( elapsed < timeout )) \
|
|
|| die "$label (CT $id) did not pass its health check within ${timeout}s"
|
|
done < <(jq -r '.dependencies[] | @base64' "$config")
|
|
}
|
|
|
|
if [[ ${1:-} == "--install" ]]; then
|
|
shift
|
|
install_hook "$@"
|
|
exit 0
|
|
fi
|
|
if [[ ${1:-} == "--refresh" ]]; then
|
|
refresh_hooks
|
|
exit 0
|
|
fi
|
|
|
|
vmid=${1:?missing VMID}
|
|
phase=${2:?missing lifecycle phase}
|
|
case "$phase" in
|
|
pre-start) start_dependencies "$vmid" ;;
|
|
post-start|pre-stop|post-stop) ;;
|
|
*) die "Unknown lifecycle phase: $phase" ;;
|
|
esac
|