Merge pull request #341 from f3rs3n/fix/validate-pending-restore

fix(backup): reject incomplete pending restore staging
This commit is contained in:
MacRimi
2026-09-08 20:38:54 +02:00
committed by GitHub
2 changed files with 331 additions and 63 deletions
+94 -63
View File
@@ -2300,59 +2300,71 @@ _rs_prepare_pending_restore() {
local pending_base="/var/lib/proxmenux/restore-pending"
local restore_id pending_dir created_at
restore_id="$(date +%Y%m%d_%H%M%S)"
pending_dir="${pending_base}/${restore_id}"
mkdir -p "$pending_base/completed" || return 1
# Never reuse a published tree, even for two restores in the same second.
pending_dir=$(mktemp -d "${pending_base}/${restore_id}.XXXXXX") || return 1
restore_id="${pending_dir##*/}"
created_at="$(date -Iseconds)"
mkdir -p "$pending_dir/rootfs" "$pending_dir/metadata" "$pending_base/completed" || return 1
# Keep all writes private; callers use if, so do not rely on errexit.
if ! (
mkdir -p "$pending_dir/rootfs" "$pending_dir/metadata" || exit 1
local rel src dst
: > "$pending_dir/apply-on-boot.list"
for rel in "${pending_paths[@]}"; do
src="$staging_root/rootfs/$rel"
[[ -e "$src" ]] || continue
dst="$pending_dir/rootfs/$rel"
mkdir -p "$(dirname "$dst")"
if [[ -d "$src" ]]; then
mkdir -p "$dst"
rsync -aAXH --delete "$src/" "$dst/" 2>/dev/null || true
else
cp -a "$src" "$dst" 2>/dev/null || true
local rel src dst
: > "$pending_dir/apply-on-boot.list" || exit 1
for rel in "${pending_paths[@]}"; do
src="$staging_root/rootfs/$rel"
[[ -e "$src" ]] || {
msg_error "$(translate "Pending restore source is missing:") $rel"
exit 1
}
dst="$pending_dir/rootfs/$rel"
mkdir -p "$(dirname "$dst")" || exit 1
if [[ -d "$src" ]]; then
mkdir -p "$dst" || exit 1
if ! rsync -aAXH --delete "$src/" "$dst/" 2>/dev/null; then
msg_error "$(translate "Could not stage pending restore path:") $rel"
exit 1
fi
else
if ! cp -a "$src" "$dst" 2>/dev/null; then
msg_error "$(translate "Could not stage pending restore path:") $rel"
exit 1
fi
fi
echo "$rel" >> "$pending_dir/apply-on-boot.list" || exit 1
done
if [[ ! -s "$pending_dir/apply-on-boot.list" ]]; then
msg_warn "$(translate "Nothing to schedule for reboot from selected paths.")"
exit 1
fi
echo "$rel" >> "$pending_dir/apply-on-boot.list"
done
if [[ ! -s "$pending_dir/apply-on-boot.list" ]]; then
rm -rf "$pending_dir"
msg_warn "$(translate "Nothing to schedule for reboot from selected paths.")"
return 1
fi
if [[ -d "$staging_root/metadata" ]]; then
cp -a "$staging_root/metadata/." "$pending_dir/metadata/" 2>/dev/null || exit 1
fi
[[ -d "$staging_root/metadata" ]] && cp -a "$staging_root/metadata/." "$pending_dir/metadata/" 2>/dev/null || true
# Persist the rollback plan: VMs/LXCs/components that exist
# on the host but not in the backup. apply_cluster_postboot.sh
# surfaces a read-only "what differs" report from this file
# after the boot. When the operator opted into destructive
# rollback (HB_ROLLBACK_EXECUTE=1) we also destroy those
# guests RIGHT NOW — before the reboot — so the operator
# sees `qm destroy` live in the Monitor terminal AND the
# next boot comes up without those guests reserving hardware
# (critical for GPU passthrough: a stale VM with hostpci
# entries makes Proxmox auto-bind the GPU to vfio-pci before
# the nvidia driver can claim it).
local _rb_script="${SCRIPT_DIR}/restore/compute_rollback_plan.sh"
[[ ! -x "$_rb_script" ]] && _rb_script="${LOCAL_SCRIPTS:-/usr/local/share/proxmenux/scripts}/backup_restore/restore/compute_rollback_plan.sh"
if [[ -x "$_rb_script" ]]; then
bash "$_rb_script" "$staging_root" > "$pending_dir/rollback.json" 2>/dev/null || exit 1
if [[ ! -s "$pending_dir/rollback.json" ]]; then
rm -f "$pending_dir/rollback.json" || exit 1
fi
fi
# Persist the rollback plan: VMs/LXCs/components that exist
# on the host but not in the backup. apply_cluster_postboot.sh
# surfaces a read-only "what differs" report from this file
# after the boot. When the operator opted into destructive
# rollback (HB_ROLLBACK_EXECUTE=1) we also destroy those
# guests RIGHT NOW — before the reboot — so the operator
# sees `qm destroy` live in the Monitor terminal AND the
# next boot comes up without those guests reserving hardware
# (critical for GPU passthrough: a stale VM with hostpci
# entries makes Proxmox auto-bind the GPU to vfio-pci before
# the nvidia driver can claim it).
local _rb_script="${SCRIPT_DIR}/restore/compute_rollback_plan.sh"
[[ ! -x "$_rb_script" ]] && _rb_script="${LOCAL_SCRIPTS:-/usr/local/share/proxmenux/scripts}/backup_restore/restore/compute_rollback_plan.sh"
if [[ -x "$_rb_script" ]]; then
bash "$_rb_script" "$staging_root" > "$pending_dir/rollback.json" 2>/dev/null || true
[[ ! -s "$pending_dir/rollback.json" ]] && rm -f "$pending_dir/rollback.json"
fi
if [[ "${HB_ROLLBACK_EXECUTE:-0}" == "1" && -s "$pending_dir/rollback.json" ]] \
&& command -v jq >/dev/null 2>&1; then
_rs_execute_rollback "$pending_dir/rollback.json"
fi
cat > "$pending_dir/plan.env" <<EOF
cat > "$pending_dir/plan.env" <<EOF || exit 1
RESTORE_ID=${restore_id}
CREATED_AT=${created_at}
HB_RESTORE_INCLUDE_ZFS=${HB_RESTORE_INCLUDE_ZFS:-0}
@@ -2361,26 +2373,45 @@ HB_COMPAT_CROSS_VERSION=${HB_COMPAT_CROSS_VERSION:-0}
HB_COMPAT_KERNEL_DIRECTION=${HB_COMPAT_KERNEL_DIRECTION:-same}
HB_HYDRATION_APPLIED=${HB_HYDRATION_APPLIED:-0}
EOF
# Persist hardware-drift skips so apply_pending_restore.sh can filter
# them at boot. The RS_SKIP_PATHS env var only lives in the restore
# menu session; without writing it to disk, paths that would break
# the boot (stale EFI UUIDs, foreign zpool.cache, ...) leaked through
# to the post-boot apply and ended up corrupting the bootloader.
if [[ -n "${RS_SKIP_PATHS:-}" ]]; then
printf '%s\n' "$RS_SKIP_PATHS" > "$pending_dir/rs-skip-paths.txt"
chmod 600 "$pending_dir/rs-skip-paths.txt"
fi
echo "pending" > "$pending_dir/state"
ln -sfn "$pending_dir" "$pending_base/current"
_rs_install_pending_service_unit "$onboot_script"
systemctl daemon-reload >/dev/null 2>&1 || true
if ! systemctl enable proxmenux-restore-onboot.service >/dev/null 2>&1; then
msg_error "$(translate "Could not enable on-boot restore service.")"
# Persist hardware-drift skips so apply_pending_restore.sh can filter
# them at boot. The RS_SKIP_PATHS env var only lives in the restore
# menu session; without writing it to disk, paths that would break
# the boot (stale EFI UUIDs, foreign zpool.cache, ...) leaked through
# to the post-boot apply and ended up corrupting the bootloader.
if [[ -n "${RS_SKIP_PATHS:-}" ]]; then
printf '%s\n' "$RS_SKIP_PATHS" > "$pending_dir/rs-skip-paths.txt" || exit 1
chmod 600 "$pending_dir/rs-skip-paths.txt" || exit 1
fi
echo "pending" > "$pending_dir/state" || exit 1
); then
rm -rf "$pending_dir"
msg_error "$(translate "Could not stage pending restore. Nothing new was scheduled.")"
return 1
fi
# current is the boot consumer's readiness boundary (state alone is not).
# Prepare a sibling-filesystem link privately, then rename over current;
# never unlink the old job first or let mv follow a directory symlink.
if ! ln -s "$pending_dir" "$pending_dir/.current" \
|| ! _rs_install_pending_service_unit "$onboot_script" \
|| ! systemctl daemon-reload >/dev/null 2>&1 \
|| ! systemctl enable proxmenux-restore-onboot.service >/dev/null 2>&1; then
rm -rf "$pending_dir"
msg_error "$(translate "Could not prepare on-boot restore service. Nothing new was scheduled.")"
return 1
fi
if ! mv -Tf "$pending_dir/.current" "$pending_base/current"; then
rm -rf "$pending_dir"
msg_error "$(translate "Could not publish pending restore. Previous pending restore was kept.")"
return 1
fi
# Destructive rollback is allowed only after staging and publication succeed.
if [[ "${HB_ROLLBACK_EXECUTE:-0}" == "1" && -s "$pending_dir/rollback.json" ]] \
&& command -v jq >/dev/null 2>&1; then
_rs_execute_rollback "$pending_dir/rollback.json"
fi
echo -e ""
echo -e "${TAB}${BGN}$(translate "Pending restore ID:")${CL} ${BL}${restore_id}${CL}"
echo -e "${TAB}${BGN}$(translate "Pending restore dir:")${CL} ${BL}${pending_dir}${CL}"
+237
View File
@@ -0,0 +1,237 @@
"""Fixture-only pending staging tests; never source/run complete host scripts.
Only extracted preparation runs. Commands are allowlisted, filesystem arguments
must stay below TemporaryDirectory, and service/UI/rollback calls are mocked.
ln/readlink model pointers with regular files: no symlinks or devices are created.
"""
import os
from pathlib import Path
import re
import subprocess
import tempfile
import unittest
REPO = Path(__file__).resolve().parents[1]
SOURCE = REPO / 'scripts/backup_restore/backup_host.sh'
DISPATCH = r'''
import os, pathlib, shutil, subprocess, sys, tempfile
root = pathlib.Path(os.environ['FIXTURE'])
cmd, *args = sys.argv[1:]
mode = os.environ.get('FAIL', '')
def checked(s):
p = pathlib.Path(s)
if not p.is_absolute() or not p.resolve().is_relative_to(root):
raise RuntimeError('out-of-fixture path: ' + s)
return p
paths = [checked(a) for a in args if a.startswith('/')]
with (root / 'commands').open('a') as f:
f.write(cmd + ' ' + ' '.join(args) + '\n')
if cmd == 'date':
print('20260907_120000' if args[0].startswith('+') else '2026-09-07T12:00:00+02:00')
elif cmd in ('dirname', 'basename'):
print(str(pathlib.Path(args[0]).parent) if cmd == 'dirname' else pathlib.Path(args[0]).name)
elif cmd == 'mktemp':
template = paths[-1]
print(tempfile.mkdtemp(prefix=template.name.replace('XXXXXX', ''), dir=template.parent))
elif cmd == 'ln':
if mode == 'link': sys.exit(1)
paths[-1].write_text(str(paths[-2]))
elif cmd == 'readlink':
print(paths[-1].read_text())
elif cmd == 'rsync':
if mode == 'directory':
(paths[-1] / 'partial').write_text('incomplete')
sys.exit(11)
sys.exit(subprocess.call(['/usr/bin/rsync', *args]))
elif cmd == 'cp':
if mode == 'file' and '/rootfs/' in str(paths[0]):
paths[-1].write_text('incomplete')
sys.exit(1)
if mode == 'metadata' and '/metadata' in str(paths[0]): sys.exit(1)
sys.exit(subprocess.call(['/usr/bin/cp', *args]))
elif cmd == 'mv':
if mode == 'publish': sys.exit(1)
sys.exit(subprocess.call(['/usr/bin/mv', *args]))
elif cmd == 'cat':
if mode == 'plan': sys.exit(1)
sys.stdout.write(sys.stdin.read() if not paths else paths[0].read_text())
elif cmd in ('mkdir', 'rm', 'chmod'):
if cmd == mode and (cmd == 'chmod' or any('/rootfs' in str(p) for p in paths)): sys.exit(1)
sys.exit(subprocess.call(['/usr/bin/' + cmd, *args]))
else:
raise RuntimeError('unexpected command: ' + cmd)
'''
def extract(name):
match = re.search(r'^' + re.escape(name) + r'\(\) \{\n.*?^\}', SOURCE.read_text(), re.M | re.S)
if not match:
raise AssertionError('missing function ' + name)
return match.group()
class PendingRestoreTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='pmx-pending-test-')
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.put('stage/rootfs/etc/network/interfaces', 'backup network\n')
self.put('stage/rootfs/etc/hostname', 'backup host\n')
self.put('stage/metadata/selected_paths.txt', 'etc/network\netc/hostname\n')
self.put('scripts/backup_restore/apply_pending_restore.sh', '# inert\n').chmod(0o700)
self.put('scripts/backup_restore/restore/compute_rollback_plan.sh', '# inert\n').chmod(0o700)
self.put('dispatch.py', DISPATCH)
(self.root / 'empty-path').mkdir()
def put(self, path, content):
p = self.root / path
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(content)
return p
def run_prepare(self, fail='', paths=('etc/network', 'etc/hostname')):
code = extract('_rs_prepare_pending_restore')
literal = 'local pending_base="/var/lib/proxmenux/restore-pending"'
self.assertEqual(code.count(literal), 1)
code = code.replace(literal, 'local pending_base="$FIXTURE/pending"')
wrappers = '\n'.join(f'{c}() {{ /usr/bin/python3 "$FIXTURE/dispatch.py" {c} "$@"; }}'
for c in ('mkdir', 'rm', 'cp', 'rsync', 'ln', 'mv', 'mktemp', 'cat', 'chmod', 'date', 'dirname', 'basename', 'readlink'))
code += '\n' + wrappers + r'''
LOCAL_SCRIPTS="$FIXTURE/scripts"
SCRIPT_DIR="$LOCAL_SCRIPTS/backup_restore"
translate() { printf '%s' "$1"; }
msg_ok() { printf 'OK:%s\n' "$*"; }
msg_warn() { printf 'WARN:%s\n' "$*"; }
msg_error() { printf 'ERROR:%s\n' "$*"; }
HB_ROLLBACK_EXECUTE=1
bash() { [[ "$FAIL" != rollback_plan ]] || return 1; builtin printf '{}\n'; }
jq() { :; }
echo() {
[[ "$FAIL:$1" != list:etc/network && "$FAIL:$1" != state:pending ]] || return 1
builtin echo "$@"
}
printf() {
[[ "$FAIL:${2:-}" != skips:/etc/kernel/foreign ]] || return 1
builtin printf "$@"
}
_rs_execute_rollback() { builtin printf 'rollback\n' >> "$FIXTURE/rollback"; }
_rs_install_pending_service_unit() {
printf 'install\n' >> "$FIXTURE/services"
[[ "$FAIL" != install ]]
}
systemctl() {
printf '%s\n' "$*" >> "$FIXTURE/services"
[[ "$FAIL" != "$1" ]]
}
# Conditional invocation deliberately disables inherited errexit: production
# callers use this form, so every I/O failure needs an explicit check.
if _rs_prepare_pending_restore "$FIXTURE/stage" "$@"; then exit 0; else exit 1; fi
'''
env = {'PATH': str(self.root / 'empty-path'), 'HOME': str(self.root),
'LC_ALL': 'C', 'FIXTURE': str(self.root), 'FAIL': fail,
'RS_SKIP_PATHS': '/etc/kernel/foreign'}
result = subprocess.run(['/bin/bash', '--noprofile', '--norc', '-c', code, 'fixture', *paths],
cwd=self.root, env=env, capture_output=True, text=True, timeout=30)
self.assertNotIn('out-of-fixture', result.stderr)
self.assertNotIn('command not found', result.stderr)
return result
def boot_eligible(self):
# Execute only the real consumer's entry checks, never its apply loop,
# service installation, host writes, logging setup, or reboot helpers.
boot = (REPO / 'scripts/backup_restore/apply_pending_restore.sh').read_text()
gate = boot[boot.index('if [[ ! -e "$CURRENT_LINK" ]]; then'):
boot.index('echo "Pending dir:')]
code = r'''
CURRENT_LINK="$FIXTURE/pending/current"
readlink() { /usr/bin/python3 "$FIXTURE/dispatch.py" readlink "$@"; }
rm() { /usr/bin/python3 "$FIXTURE/dispatch.py" rm "$@"; }
''' + gate + '\nprintf "BOOT_ELIGIBLE\\n"\n'
result = subprocess.run(['/bin/bash', '--noprofile', '--norc', '-c', code],
cwd=self.root, env={'PATH': str(self.root / 'empty-path'),
'FIXTURE': str(self.root)},
capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
return 'BOOT_ELIGIBLE' in result.stdout
def assert_failed_unpublished(self, result):
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertNotIn('OK:', result.stdout)
self.assertFalse((self.root / 'pending/current').exists(), 'partial tree is boot-eligible')
self.assertFalse((self.root / 'services').exists(), 'service touched after staging failure')
self.assertFalse(self.boot_eligible(), 'consumer accepted failed staging')
def test_failed_directory_copy_is_not_scheduled(self):
self.assert_failed_unpublished(self.run_prepare('directory'))
def test_failed_replacement_preserves_old_pending(self):
for mode in ('directory', 'file', 'metadata', 'plan'):
with self.subTest(mode=mode):
self.setUp()
first = self.run_prepare()
self.assertEqual(first.returncode, 0, first.stderr)
current = self.root / 'pending/current'
old = Path(current.read_text())
before = {str(p.relative_to(old)): p.read_bytes() for p in old.rglob('*') if p.is_file()}
(self.root / 'services').unlink()
(self.root / 'rollback').unlink()
result = self.run_prepare(mode)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(current.read_text(), str(old))
after = {str(p.relative_to(old)): p.read_bytes() for p in old.rglob('*') if p.is_file()}
self.assertEqual(after, before, 'same-second replacement damaged old pending tree')
self.assertFalse((self.root / 'services').exists())
self.assertFalse((self.root / 'rollback').exists())
self.assertTrue(self.boot_eligible())
def test_staging_io_failure_is_not_published(self):
for mode in ('metadata', 'plan', 'mkdir', 'list', 'state', 'skips', 'chmod', 'rollback_plan'):
with self.subTest(mode=mode):
self.setUp()
self.assert_failed_unpublished(self.run_prepare(mode))
self.assertFalse((self.root / 'rollback').exists())
def test_publication_failure_preserves_current(self):
for mode in ('install', 'daemon-reload', 'enable', 'link', 'publish'):
for existing in (False, True):
with self.subTest(mode=mode, existing=existing):
self.setUp()
current = self.root / 'pending/current'
old = None
if existing:
self.assertEqual(self.run_prepare().returncode, 0)
old = current.read_text()
(self.root / 'rollback').unlink()
result = self.run_prepare(mode)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertNotIn('OK:', result.stdout)
self.assertEqual(current.read_text() if current.exists() else None, old)
self.assertFalse((self.root / 'rollback').exists())
def test_missing_selected_source_aborts_whole_stage(self):
self.assert_failed_unpublished(self.run_prepare(paths=('etc/network', 'etc/missing')))
def test_success_publishes_complete_restore(self):
result = self.run_prepare()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('OK:', result.stdout)
pending = Path((self.root / 'pending/current').read_text())
self.assertEqual((pending / 'rootfs/etc/network/interfaces').read_text(), 'backup network\n')
self.assertEqual((pending / 'rootfs/etc/hostname').read_text(), 'backup host\n')
self.assertEqual((pending / 'apply-on-boot.list').read_text(), 'etc/network\netc/hostname\n')
self.assertEqual((pending / 'state').read_text(), 'pending\n')
self.assertEqual((pending / 'rs-skip-paths.txt').read_text(), '/etc/kernel/foreign\n')
self.assertIn(f'RESTORE_ID={pending.name}\n', (pending / 'plan.env').read_text())
self.assertEqual((pending / 'metadata/selected_paths.txt').read_bytes(),
(self.root / 'stage/metadata/selected_paths.txt').read_bytes())
self.assertEqual((self.root / 'services').read_text(), 'install\ndaemon-reload\nenable proxmenux-restore-onboot.service\n')
self.assertTrue((self.root / 'rollback').exists())
self.assertTrue(self.boot_eligible())
def test_failed_file_copy_is_not_scheduled(self):
self.assert_failed_unpublished(self.run_prepare('file'))
if __name__ == '__main__':
unittest.main(verbosity=2)