From 14583ad9f69ff09918eb5a287e040ebd4221e69e Mon Sep 17 00:00:00 2001 From: MacRimi Date: Fri, 25 Sep 2026 22:53:35 +0200 Subject: [PATCH] Back up /var/lib/proxmenux whole in the default host profile --- .../backup_restore/apply_pending_restore.sh | 30 ++++++- scripts/backup_restore/backup_host.sh | 13 ++- .../backup_restore/lib_host_backup_common.sh | 24 +++++- scripts/backup_restore/test_backup_restore.sh | 80 ++++++++++++++++++- .../en/docs/backup-restore/how-it-works.json | 4 +- .../es/docs/backup-restore/how-it-works.json | 4 +- 6 files changed, 145 insertions(+), 10 deletions(-) diff --git a/scripts/backup_restore/apply_pending_restore.sh b/scripts/backup_restore/apply_pending_restore.sh index e97e1118..2dbc0e88 100755 --- a/scripts/backup_restore/apply_pending_restore.sh +++ b/scripts/backup_restore/apply_pending_restore.sh @@ -10,6 +10,18 @@ LOG_DIR="${PMX_RESTORE_LOG_DIR:-/var/log/proxmenux}" DEST_PREFIX="${PMX_RESTORE_DEST_PREFIX:-/}" PRE_BACKUP_BASE="${PMX_RESTORE_PRE_BACKUP_BASE:-/var/lib/proxmenux/pre-restore}" RECOVERY_BASE="${PMX_RESTORE_RECOVERY_BASE:-/var/lib/proxmenux/recovery}" +# Same list as hb_state_dir_excludes: restore machinery stays with this host. +STATE_DIR_EXCLUDES=( + --exclude=/restore-pending/ + --exclude=/pre-restore/ + --exclude=/recovery/ + --exclude=/restore-history/ + --exclude=/restore-state.json + --exclude=/cluster-apply-pending + --exclude=/post-restore-maintenance-pending + --exclude=/exports/ + --exclude=/helpers_cache.json +) mkdir -p "$LOG_DIR" "$PENDING_BASE/completed" >/dev/null 2>&1 || true LOG_FILE="${LOG_DIR}/proxmenux-restore-onboot-$(date +%Y%m%d_%H%M%S).log" @@ -157,27 +169,39 @@ while IFS= read -r rel; do fi fi + state_excludes=() + [[ "$rel" == "var/lib/proxmenux" ]] && state_excludes=("${STATE_DIR_EXCLUDES[@]}") + if [[ -e "$dst" ]]; then mkdir -p "$backup_root/$(dirname "$rel")" >/dev/null 2>&1 || true - cp -a "$dst" "$backup_root/$rel" >/dev/null 2>&1 || true + if (( ${#state_excludes[@]} )); then + # backup_root lives inside /var/lib/proxmenux/pre-restore. + mkdir -p "$backup_root/$rel" >/dev/null 2>&1 || true + rsync -aAXH "${state_excludes[@]}" "$dst/" "$backup_root/$rel/" >/dev/null 2>&1 || true + else + cp -a "$dst" "$backup_root/$rel" >/dev/null 2>&1 || true + fi fi if [[ -d "$src" ]]; then mkdir -p "$dst" >/dev/null 2>&1 || true + [[ "$rel" == "var/lib/proxmenux" && -d "$src/backup-jobs" ]] && state_jobs=1 || state_jobs=0 # When an exclude list is present, drop --delete so the host's # copy of the excluded file isn't removed by rsync after being # skipped from the source side. if [[ ${#RSYNC_EXCLUDES[@]} -gt 0 ]]; then - if rsync -aAXH "${RSYNC_EXCLUDES[@]}" "$src/" "$dst/" >/dev/null 2>&1; then + if rsync -aAXH "${RSYNC_EXCLUDES[@]}" "${state_excludes[@]}" "$src/" "$dst/" >/dev/null 2>&1; then ((applied++)) [[ "$rel" == "var/lib/proxmenux/backup-jobs" || "$rel" == "var/lib/proxmenux/backup-jobs/"* ]] && jobs_restored=1 + (( state_jobs )) && jobs_restored=1 else ((failed++)) fi else - if rsync -aAXH --delete "$src/" "$dst/" >/dev/null 2>&1; then + if rsync -aAXH --delete "${state_excludes[@]}" "$src/" "$dst/" >/dev/null 2>&1; then ((applied++)) [[ "$rel" == "var/lib/proxmenux/backup-jobs" || "$rel" == "var/lib/proxmenux/backup-jobs/"* ]] && jobs_restored=1 + (( state_jobs )) && jobs_restored=1 else ((failed++)) fi diff --git a/scripts/backup_restore/backup_host.sh b/scripts/backup_restore/backup_host.sh index 35b82d60..6794dc98 100755 --- a/scripts/backup_restore/backup_host.sh +++ b/scripts/backup_restore/backup_host.sh @@ -1754,10 +1754,19 @@ _rs_apply() { [[ "${HB_RESTORE_INCLUDE_ZFS:-0}" != "1" ]] && { ((skipped++)); continue; } fi + local -a state_excludes=() + [[ "$rel" == "var/lib/proxmenux" ]] && mapfile -t state_excludes < <(hb_state_dir_excludes) + # Save current before overwriting if [[ -e "$dst" ]]; then mkdir -p "$backup_root/$(dirname "$rel")" - cp -a "$dst" "$backup_root/$rel" 2>/dev/null || true + if (( ${#state_excludes[@]} )); then + # backup_root lives inside /var/lib/proxmenux/pre-restore. + mkdir -p "$backup_root/$rel" + rsync -aAXH "${state_excludes[@]}" "$dst/" "$backup_root/$rel/" 2>/dev/null || true + else + cp -a "$dst" "$backup_root/$rel" 2>/dev/null || true + fi fi # Apply @@ -1787,9 +1796,11 @@ _rs_apply() { --exclude "restore-pending/" ) fi + rsync_extra+=("${state_excludes[@]}") if rsync -aAXH --delete "${rsync_extra[@]}" "$src/" "$dst/" 2>/dev/null; then ((applied++)) [[ "$rel" == "var/lib/proxmenux/backup-jobs" || "$rel" == "var/lib/proxmenux/backup-jobs/"* ]] && jobs_restored=1 + [[ "$rel" == "var/lib/proxmenux" && -d "$src/backup-jobs" ]] && jobs_restored=1 else ((skipped++)) fi diff --git a/scripts/backup_restore/lib_host_backup_common.sh b/scripts/backup_restore/lib_host_backup_common.sh index 74f494b2..a1573e33 100755 --- a/scripts/backup_restore/lib_host_backup_common.sh +++ b/scripts/backup_restore/lib_host_backup_common.sh @@ -131,7 +131,7 @@ hb_default_profile_paths() { "/usr/local/bin" "/usr/local/sbin" "/usr/local/share/proxmenux" - "$HB_BACKUP_JOBS_DIR" + "/var/lib/proxmenux" # backup jobs and host state; restore machinery excluded # ── Root home (rsync excludes volatile dirs) ───────── "/root" @@ -144,6 +144,22 @@ hb_default_profile_paths() { printf '%s\n' "${paths[@]}" } +# rsync excludes for /var/lib/proxmenux, relative to that directory. +# Restore staging, rollback copies and post-restore markers belong to the +# host that runs the restore: they are neither backed up nor overwritten. +hb_state_dir_excludes() { + printf '%s\n' \ + --exclude=/restore-pending/ \ + --exclude=/pre-restore/ \ + --exclude=/recovery/ \ + --exclude=/restore-history/ \ + --exclude=/restore-state.json \ + --exclude=/cluster-apply-pending \ + --exclude=/post-restore-maintenance-pending \ + --exclude=/exports/ \ + --exclude=/helpers_cache.json +} + # ========================================================== # PATH CLASSIFICATION (restore safety) # Returns: dangerous | reboot | hot @@ -715,6 +731,12 @@ hb_prepare_staging() { ) fi + if (( ! operator_added )) && [[ "$rel" == "var/lib/proxmenux" ]]; then + local -a state_excludes=() + mapfile -t state_excludes < <(hb_state_dir_excludes) + rsync_opts+=("${state_excludes[@]}") + fi + if rsync "${rsync_opts[@]}" "$p/" "$target/"; then echo "$rel" >> "$selected_file" else diff --git a/scripts/backup_restore/test_backup_restore.sh b/scripts/backup_restore/test_backup_restore.sh index fac633b8..0a58d19c 100755 --- a/scripts/backup_restore/test_backup_restore.sh +++ b/scripts/backup_restore/test_backup_restore.sh @@ -230,7 +230,7 @@ staging_state_tests() { # shellcheck source=/dev/null source "$LIB_SCRIPT" - if hb_default_profile_paths | grep -Fxq '/var/lib/proxmenux/backup-jobs'; then + if hb_default_profile_paths | grep -Fxq '/var/lib/proxmenux'; then pass "Default profile includes scheduled backup job definitions" else fail "Default profile does not include scheduled backup job definitions" @@ -579,6 +579,83 @@ EOJ fi } +pending_state_dir_restore_tests() { + log "\n=== Pending restore of the ProxMenux state directory (sandbox) ===" + if ! help mapfile >/dev/null 2>&1; then + skip "Pending state-directory restore test requires the Linux runtime." + return + fi + local pending_base="$TMP_ROOT/state-restore-pending" + local logs_dir="$TMP_ROOT/state-restore-logs" + local target_root="$TMP_ROOT/state-restore-target" + local target_state="$target_root/var/lib/proxmenux" + local pre_backup_base="$target_state/pre-restore" + local recovery_base="$target_state/recovery" + local source_state="$pending_base/r3/rootfs/var/lib/proxmenux" + + mkdir -p "$source_state/backup-jobs" "$source_state/oci-installations" \ + "$source_state/restore-pending/old" "$target_state/restore-history" \ + "$target_state/exports" "$pre_backup_base/earlier" + cat > "$source_state/backup-jobs/stateful.env" <<'EOJ' +JOB_ID=stateful +BACKEND=local +ON_CALENDAR=daily +PROFILE_MODE=custom +ENABLED=1 +LOCAL_DEST_DIR=/var/lib/vz/dump +LOCAL_ARCHIVE_EXT=tar.gz +EOJ + echo "/etc/hosts" > "$source_state/backup-jobs/stateful.paths" + echo '{}' > "$source_state/oci-installations/app.json" + echo "source" > "$source_state/restore-pending/old/marker" + echo "source" > "$source_state/cluster-apply-pending" + echo "target" > "$target_state/restore-history/entry.json" + echo "target" > "$target_state/exports/report.pdf" + echo "target" > "$target_state/stale.json" + echo "target" > "$pre_backup_base/earlier/keep" + echo "var/lib/proxmenux" > "$pending_base/r3/apply-on-boot.list" + echo "HB_RESTORE_INCLUDE_ZFS=0" > "$pending_base/r3/plan.env" + ln -sfn "$pending_base/r3" "$pending_base/current" + + if PMX_RESTORE_PENDING_BASE="$pending_base" PMX_RESTORE_LOG_DIR="$logs_dir" \ + PMX_RESTORE_DEST_PREFIX="$target_root" PMX_RESTORE_PRE_BACKUP_BASE="$pre_backup_base" \ + PMX_RESTORE_RECOVERY_BASE="$recovery_base" \ + bash "$APPLY_ONBOOT" >>"$REPORT_FILE" 2>&1; then + pass "Pending restore applies the state directory" + else + fail "Pending restore failed while applying the state directory" + return + fi + + if [[ -f "$target_state/backup-jobs/stateful.env" && -f "$target_state/oci-installations/app.json" && \ + -f "$target_root/etc/systemd/system/proxmenux-backup-stateful.timer" ]]; then + pass "State directory restore brings jobs and state, and rebuilds the timer" + else + fail "State directory restore did not bring jobs, state or the timer" + fi + + if [[ ! -e "$target_state/restore-pending/old" && ! -e "$target_state/cluster-apply-pending" ]]; then + pass "Restore staging and post-boot markers from the backup are not applied" + else + fail "Restore staging or post-boot markers from the backup were applied" + fi + + if [[ -f "$target_state/restore-history/entry.json" && -f "$target_state/exports/report.pdf" && \ + -f "$pre_backup_base/earlier/keep" && ! -e "$target_state/stale.json" ]]; then + pass "Excluded host entries survive the restore; other entries follow the backup" + else + fail "The restore removed excluded host entries or kept entries absent from the backup" + fi + + local saved + saved=$(find "$pre_backup_base" -mindepth 1 -maxdepth 1 -name '*-onboot' | head -1) + if [[ -n "$saved" && -f "$saved/var/lib/proxmenux/stale.json" && ! -e "$saved/var/lib/proxmenux/pre-restore" ]]; then + pass "Previous state is saved without copying the rollback directory into itself" + else + fail "Previous state was not saved, or the rollback directory was copied into itself" + fi +} + main() { log "ProxMenux backup/restore test matrix" log "Report: $REPORT_FILE" @@ -591,6 +668,7 @@ main() { scheduler_e2e_tests pending_restore_tests pending_jobs_restore_tests + pending_state_dir_restore_tests log "\n=== Summary ===" log "PASS=$PASS" diff --git a/web/messages/en/docs/backup-restore/how-it-works.json b/web/messages/en/docs/backup-restore/how-it-works.json index 917e6b18..dc68cee6 100644 --- a/web/messages/en/docs/backup-restore/how-it-works.json +++ b/web/messages/en/docs/backup-restore/how-it-works.json @@ -66,8 +66,8 @@ }, { "category": "ProxMenux binaries & root", - "paths": "/usr/local/bin, /usr/local/sbin, /usr/local/share/proxmenux, /root (volatile subdirs excluded)", - "why": "ProxMenux-installed binaries and per-user configuration under /root. Volatile paths (.bash_history, .cache/, tmp/, .local/share/Trash/) are excluded from the copy." + "paths": "/usr/local/bin, /usr/local/sbin, /usr/local/share/proxmenux, /var/lib/proxmenux, /root (volatile subdirs excluded)", + "why": "ProxMenux-installed binaries and per-user configuration under /root. Volatile paths (.bash_history, .cache/, tmp/, .local/share/Trash/) are excluded from the copy. /var/lib/proxmenux holds the scheduled backup jobs and the host state kept by ProxMenux; the restore staging, the rollback copies and the post-restore markers are excluded, because they belong to the host that runs a restore." }, { "category": "ZFS state (conditional)", diff --git a/web/messages/es/docs/backup-restore/how-it-works.json b/web/messages/es/docs/backup-restore/how-it-works.json index f5c19b6c..0c5ec8c7 100644 --- a/web/messages/es/docs/backup-restore/how-it-works.json +++ b/web/messages/es/docs/backup-restore/how-it-works.json @@ -66,8 +66,8 @@ }, { "category": "Binarios ProxMenux y root", - "paths": "/usr/local/bin, /usr/local/sbin, /usr/local/share/proxmenux, /root (subdirs volátiles excluidos)", - "why": "Binarios instalados por ProxMenux y configuración por usuario bajo /root. Las rutas volátiles (.bash_history, .cache/, tmp/, .local/share/Trash/) quedan fuera de la copia." + "paths": "/usr/local/bin, /usr/local/sbin, /usr/local/share/proxmenux, /var/lib/proxmenux, /root (subdirs volátiles excluidos)", + "why": "Binarios instalados por ProxMenux y configuración por usuario bajo /root. Las rutas volátiles (.bash_history, .cache/, tmp/, .local/share/Trash/) quedan fuera de la copia. /var/lib/proxmenux guarda los trabajos de backup programados y el estado del host que mantiene ProxMenux; quedan fuera la preparación de una restauración, las copias de rollback y las marcas posteriores a la restauración, porque pertenecen al host que la ejecuta." }, { "category": "Estado ZFS (condicional)",