Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-11-01 10:22:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: skip scheduled updates for stopped LXC

Browse Source
This commit is contained in:
VAIO73
2026-10-16 17:21:34 +00:00
parent 3000c1cd66
commit c2ad35ca73
6 changed files with 163 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AppImage/scripts/flask_server.py
+29 -1
View File
@@ -22245,6 +22245,7 @@ def api_proxmenux_self_update_status():
# so scheduled and manual applies cannot overlap.
_APPLY_UPDATES_SCRIPT = _LXC_APPLY_UPDATES_SCRIPT
_SCHEDULED_STOPPED_EXIT = 8
_DOCKER_ENGINE_INTEGRATED_COMMAND = (
'python3 /usr/local/share/proxmenux/monitor-app/usr/bin/'
'update_docker_engine.py --vmid "$VMID"'
@@ -22286,6 +22287,23 @@ def _append_lxc_update_log(path: str | None, text: str) -> None:
print(f'[ProxMenux] scheduler: could not append update log: {exc}', flush=True)
def _scheduled_runner_reported_stopped(result, log_path: str | None, marker: str) -> bool:
"""Return true only for the scheduler's own pre-start stopped marker."""
if result.returncode != _SCHEDULED_STOPPED_EXIT or not marker:
return False
if log_path:
try:
with open(log_path, 'rb') as stream:
stream.seek(0, os.SEEK_END)
stream.seek(max(0, stream.tell() - 4096))
output = stream.read().decode('utf-8', errors='replace')
except OSError:
return False
else:
output = (result.stdout or '') + (result.stderr or '')
return marker in output
def _prune_lxc_update_logs(vmid: int) -> None:
try:
candidates = sorted(
@@ -22679,6 +22697,7 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
requested_targets = _normalise_schedule_targets(sched)
targets = list(requested_targets)
before = _lxc_update_snapshot(vmid)
requested_target = sched.get("target") or "both"
deferred_targets: list[str] = []
reasons: list[str] = []
reboot_required: bool | None = None
@@ -22743,7 +22762,10 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
'finalization': finalization,
}
requested_target = sched.get("target") or "both"
if not originally_running:
reasons.append('container is stopped; scheduled update was skipped')
return finish('skipped', requested_target, [])
if requested_targets == ['oci_image']:
# A container installed by OCI manager Apps is updated by replacing
# its image, through the same flow as the OCI menu, unattended.
@@ -22887,6 +22909,9 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
target = "both" if has_os_target and has_app_targets else ("os" if has_os_target else "app")
env = dict(os.environ)
env["VMID"] = str(vmid)
env["EXECUTION_MODE"] = "scheduled"
scheduled_stopped_marker = f"PROXMENUX_SCHEDULED_STOPPED:{run_id}"
env["SCHEDULED_STOPPED_MARKER"] = scheduled_stopped_marker
env["TARGET"] = target
env["BACKUP"] = "1" if sched.get("backup") else "0"
env["BACKUP_STORAGE"] = sched.get("backup_storage") or ""
@@ -22952,6 +22977,9 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
text=True,
timeout=60 * 60,
)
if _scheduled_runner_reported_stopped(r, log_path, scheduled_stopped_marker):
reasons.append('container was stopped before scheduled update could begin')
return finish('skipped', target, [])
reboot_required, reboot_packages, reboot_check_error = (
_inspect_lxc_reboot_requirement(
vmid,
scripts/lxc/apply_updates.sh
+5
View File
@@ -163,6 +163,11 @@ fi
# 2) CT must be running for pct exec. Auto-start stopped CTs, then
# restore their original stopped state on every exit path.
STATE=$(pct status "$VMID" | awk '{print $2}')
if [[ "$STATE" != "running" && "${EXECUTION_MODE:-}" == "scheduled" ]]; then
[[ -n "${SCHEDULED_STOPPED_MARKER:-}" ]] && printf '%s\n' "$SCHEDULED_STOPPED_MARKER"
echo "SKIPPED: CT $VMID is stopped; a scheduled update will not start it."
exit 8
fi
STARTED_BY_PROXMENUX=0
OBSERVATION_ATTEMPTED=0
capture_before_stop() {
tests/lxc_updates/test_apply_updates.sh
+9
View File
@@ -162,6 +162,15 @@ run_case stopped_restore RUN_HELPER=0 UPDATE_COMMAND='update-custom' MOCK_INITIA
[[ $(count 'pct start 101') -eq 1 ]] || fail "stopped CT was not started exactly once"
[[ $(count 'pct shutdown 101 --timeout 60') -eq 1 ]] || fail "stopped CT state was not restored"
run_case scheduled_stopped RUN_HELPER=0 UPDATE_COMMAND='update-custom' \
EXECUTION_MODE=scheduled SCHEDULED_STOPPED_MARKER='fixture-stopped-marker' MOCK_INITIAL_STATE=stopped
[[ $CASE_RC -eq 8 ]] || fail "scheduled_stopped returned $CASE_RC"
[[ $(count 'pct start 101') -eq 0 ]] || fail "scheduled update started a stopped CT"
[[ $(count 'pct exec') -eq 0 ]] || fail "scheduled update executed inside a stopped CT"
[[ $(count CUSTOM_EXEC) -eq 0 ]] || fail "scheduled update ran a custom updater"
grep -qF 'fixture-stopped-marker' "$TEST_ROOT/scheduled_stopped.out" \
|| fail "scheduled stopped result did not include its marker"
run_case locked RUN_HELPER=0 UPDATE_COMMAND='update-custom' MOCK_FLOCK_FAIL=1
[[ $CASE_RC -eq 7 ]] || fail "locked expected 7, got $CASE_RC"
[[ $(count CUSTOM_EXEC) -eq 0 ]] || fail "locked run executed an updater"
tests/lxc_updates/test_stopped_update_observation.py
+5 -2
View File
@@ -119,7 +119,8 @@ raise SystemExit(h.main())
self.write("sleep", "#!/bin/sh\nexit 0\n")
self.write("flock", "#!/bin/sh\nexit 0\n")
names = {"_finalize_lxc_update", "_terminal_lxc_update_completed",
"api_lxc_updates_applied", "_run_scheduled_update"}
"api_lxc_updates_applied", "_run_scheduled_update",
"_scheduled_runner_reported_stopped"}
nodes = [n for n in ast.parse(observed.SERVER.read_text()).body
if isinstance(n, ast.FunctionDef) and n.name in names]
self.scope = self.fixture.scope
@@ -134,6 +135,7 @@ raise SystemExit(h.main())
"_json_list": lambda value: json.loads(value or '[]'),
"_lxc_update_finalizations": {}, "_lxc_update_finalization_lock": threading.RLock(),
"_LXC_UPDATE_FINALIZATION_TTL": 3600,
"_SCHEDULED_STOPPED_EXIT": 8,
"_fast_guest_status": lambda *_: self.state.read_text(),
"_lxc_update_snapshot": lambda *_: {"ct_name": "fixture"},
"_lxc_update_target_labels": lambda *_: ["OS"],
@@ -315,6 +317,7 @@ raise SystemExit(h.main())
self.scope['notification_manager'].emit_event.assert_called_once()
def test_scheduled_path_executes_real_runner_and_real_finalization(self):
self.state.write_text("running")
def run(command, **kwargs):
kwargs['env'].update({key: value for key, value in self.env.items()
if key != 'RUN_ID'})
@@ -322,7 +325,7 @@ raise SystemExit(h.main())
self.scope['subprocess'] = SimpleNamespace(run=run, TimeoutExpired=subprocess.TimeoutExpired)
result = self.scope['_run_scheduled_update'](100, {'targets': ['os'], 'target': 'os'})
self.assertEqual(result['status'], 'success', result)
self.assertEqual(self.state.read_text(), 'stopped')
self.assertEqual(self.state.read_text(), 'running')
self.assertEqual(self.fixture.row()['update_check']['count'], 0)
def test_legacy_cli_without_run_id_preserves_update_and_original_state(self):
tests/lxc_updates/test_update_contract.py
+114
View File
@@ -1,4 +1,6 @@
import ast
import json
import os
import sys
import subprocess
import tempfile
@@ -312,6 +314,118 @@ class AppCacheWriteThroughContractTests(unittest.TestCase):
self.assertIn("before_snapshot=before", scheduler_block)
class ScheduledStoppedContainerTests(unittest.TestCase):
@staticmethod
def _worker(scope):
source = ast.parse((REPO_ROOT / "AppImage" / "scripts" / "flask_server.py").read_text())
node = next(item for item in source.body
if isinstance(item, ast.FunctionDef) and item.name == "_run_scheduled_update")
exec(compile(ast.Module(body=[node], type_ignores=[]), "scheduled-update", "exec"), scope)
return scope["_run_scheduled_update"]
@staticmethod
def _stopped_marker_reader(scope):
source = ast.parse((REPO_ROOT / "AppImage" / "scripts" / "flask_server.py").read_text())
node = next(item for item in source.body
if isinstance(item, ast.FunctionDef) and item.name == "_scheduled_runner_reported_stopped")
exec(compile(ast.Module(body=[node], type_ignores=[]), "scheduled-marker", "exec"), scope)
return scope["_scheduled_runner_reported_stopped"]
def test_stopped_container_skips_before_runner_or_any_guest_effect(self):
runner = MagicMock()
finalizer = MagicMock(return_value={"recorded": True})
append_log = MagicMock()
scope = {
"time": SimpleNamespace(monotonic=MagicMock(return_value=10.0)),
"uuid": SimpleNamespace(uuid4=lambda: SimpleNamespace(hex="a" * 32)),
"datetime": __import__("datetime").datetime,
"subprocess": SimpleNamespace(run=runner, STDOUT=object(), TimeoutExpired=subprocess.TimeoutExpired),
"_create_lxc_update_log": lambda _vmid, _run_id: ("fixture.log", "/tmp/fixture.log"),
"_fast_guest_status": MagicMock(return_value="stopped"),
"_normalise_schedule_targets": lambda _sched: ["os"],
"_lxc_update_snapshot": lambda _vmid: {"ct_name": "fixture"},
"_lxc_update_target_labels": lambda *_: ["OS"],
"_append_lxc_update_log": append_log,
"_prune_lxc_update_logs": MagicMock(),
"_finalize_lxc_update": finalizer,
}
outcome = self._worker(scope)(101, {"target": "os"})
self.assertEqual(outcome["status"], "skipped")
self.assertEqual(outcome["executed_targets"], [])
self.assertIn("container is stopped", outcome["reason"])
runner.assert_not_called()
self.assertEqual(finalizer.call_args.kwargs["status"], "skipped")
self.assertEqual(finalizer.call_args.kwargs["source"], "scheduled")
def test_stopped_marker_requires_the_scheduler_token_not_only_exit_code(self):
marker = "PROXMENUX_SCHEDULED_STOPPED:fixture"
reader = self._stopped_marker_reader({"_SCHEDULED_STOPPED_EXIT": 8, "os": os})
self.assertTrue(reader(SimpleNamespace(returncode=8, stdout=marker, stderr=""), None, marker))
self.assertFalse(reader(SimpleNamespace(returncode=8, stdout="", stderr=""), None, marker))
self.assertFalse(reader(SimpleNamespace(returncode=4, stdout=marker, stderr=""), None, marker))
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log:
log.write("prefix\\n" + marker + "\\n")
log.flush()
self.assertTrue(reader(SimpleNamespace(returncode=8, stdout="", stderr=""), log.name, marker))
def test_runner_race_to_stopped_is_reported_as_skipped_without_reboot_probe(self):
runner = MagicMock(return_value=SimpleNamespace(returncode=8, stdout="", stderr=""))
finalizer = MagicMock(return_value={"recorded": True})
reboot_probe = MagicMock()
stopped_marker = MagicMock(return_value=True)
lxc = SimpleNamespace(
_read_sidecar=lambda _vmid: {"apps": []},
partition_scheduled_release_targets=lambda targets, _apps: (set(), targets),
)
scope = {
"time": SimpleNamespace(monotonic=MagicMock(return_value=10.0)),
"uuid": SimpleNamespace(uuid4=lambda: SimpleNamespace(hex="b" * 32)),
"datetime": __import__("datetime").datetime,
"subprocess": SimpleNamespace(run=runner, STDOUT=object(), TimeoutExpired=subprocess.TimeoutExpired),
"os": os,
"json": json,
"_APPLY_UPDATES_SCRIPT": "/bin/true",
"_SCHEDULED_STOPPED_EXIT": 8,
"_DOCKER_ENGINE_INTEGRATED_COMMAND": "integrated-docker",
"_create_lxc_update_log": lambda _vmid, _run_id: (None, None),
"_fast_guest_status": MagicMock(return_value="running"),
"_normalise_schedule_targets": lambda _sched: ["os"],
"_lxc_update_snapshot": lambda _vmid: {"ct_name": "fixture"},
"_lxc_update_target_labels": lambda *_: ["OS"],
"_append_lxc_update_log": MagicMock(),
"_prune_lxc_update_logs": MagicMock(),
"_finalize_lxc_update": finalizer,
"_scheduled_helper_enabled": lambda *_: False,
"_compose_scheduled_update_command": lambda *_: "",
"_inspect_lxc_reboot_requirement": reboot_probe,
"_scheduled_runner_reported_stopped": stopped_marker,
}
with patch.dict(sys.modules, {"lxc_apps": lxc}):
outcome = self._worker(scope)(101, {"target": "os"})
self.assertEqual(outcome["status"], "skipped")
self.assertEqual(outcome["executed_targets"], [])
self.assertIn("before scheduled update could begin", outcome["reason"])
self.assertEqual(runner.call_args.args[0], ["bash", "/bin/true"])
self.assertEqual(runner.call_args.kwargs["env"]["EXECUTION_MODE"], "scheduled")
self.assertTrue(runner.call_args.kwargs["env"]["SCHEDULED_STOPPED_MARKER"].startswith(
"PROXMENUX_SCHEDULED_STOPPED:scheduled-"))
stopped_marker.assert_called_once()
reboot_probe.assert_not_called()
stopped_marker.reset_mock()
stopped_marker.return_value = False
reboot_probe.return_value = (None, [], None)
outcome = self._worker(scope)(101, {"target": "os"})
self.assertEqual(outcome["status"], "failure")
self.assertIn("exited with code 8", outcome["reason"])
stopped_marker.assert_called_once()
reboot_probe.assert_called_once()
class DockerStackNotificationTests(unittest.TestCase):
def _docker_app(self):
return {
tests/lxc_updates/test_update_method_choice.py
+1
View File
@@ -37,6 +37,7 @@ def routes():
_lxc_update_target_labels=lambda *_: [],
_append_lxc_update_log=lambda *_: None,
_prune_lxc_update_logs=lambda *_: None,
_scheduled_runner_reported_stopped=lambda *_: False,
_finalize_lxc_update=Mock(return_value={}),
_inspect_lxc_reboot_requirement=lambda *_, **__: (False, [], None))
exec(compile(ast.Module(body=nodes, type_ignores=[]), 'update-routes', 'exec'), ns)