diff --git a/AppImage/scripts/flask_server.py b/AppImage/scripts/flask_server.py index 13da0971..965f548a 100644 --- a/AppImage/scripts/flask_server.py +++ b/AppImage/scripts/flask_server.py @@ -22245,6 +22245,7 @@ def api_proxmenux_self_update_status(): # so scheduled and manual applies cannot overlap. _APPLY_UPDATES_SCRIPT = _LXC_APPLY_UPDATES_SCRIPT +_SCHEDULED_STOPPED_EXIT = 8 _DOCKER_ENGINE_INTEGRATED_COMMAND = ( 'python3 /usr/local/share/proxmenux/monitor-app/usr/bin/' 'update_docker_engine.py --vmid "$VMID"' @@ -22286,6 +22287,23 @@ def _append_lxc_update_log(path: str | None, text: str) -> None: print(f'[ProxMenux] scheduler: could not append update log: {exc}', flush=True) +def _scheduled_runner_reported_stopped(result, log_path: str | None, marker: str) -> bool: + """Return true only for the scheduler's own pre-start stopped marker.""" + if result.returncode != _SCHEDULED_STOPPED_EXIT or not marker: + return False + if log_path: + try: + with open(log_path, 'rb') as stream: + stream.seek(0, os.SEEK_END) + stream.seek(max(0, stream.tell() - 4096)) + output = stream.read().decode('utf-8', errors='replace') + except OSError: + return False + else: + output = (result.stdout or '') + (result.stderr or '') + return marker in output + + def _prune_lxc_update_logs(vmid: int) -> None: try: candidates = sorted( @@ -22679,6 +22697,7 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict: requested_targets = _normalise_schedule_targets(sched) targets = list(requested_targets) before = _lxc_update_snapshot(vmid) + requested_target = sched.get("target") or "both" deferred_targets: list[str] = [] reasons: list[str] = [] reboot_required: bool | None = None @@ -22743,7 +22762,10 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict: 'finalization': finalization, } - requested_target = sched.get("target") or "both" + if not originally_running: + reasons.append('container is stopped; scheduled update was skipped') + return finish('skipped', requested_target, []) + if requested_targets == ['oci_image']: # A container installed by OCI manager Apps is updated by replacing # its image, through the same flow as the OCI menu, unattended. @@ -22887,6 +22909,9 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict: target = "both" if has_os_target and has_app_targets else ("os" if has_os_target else "app") env = dict(os.environ) env["VMID"] = str(vmid) + env["EXECUTION_MODE"] = "scheduled" + scheduled_stopped_marker = f"PROXMENUX_SCHEDULED_STOPPED:{run_id}" + env["SCHEDULED_STOPPED_MARKER"] = scheduled_stopped_marker env["TARGET"] = target env["BACKUP"] = "1" if sched.get("backup") else "0" env["BACKUP_STORAGE"] = sched.get("backup_storage") or "" @@ -22952,6 +22977,9 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict: text=True, timeout=60 * 60, ) + if _scheduled_runner_reported_stopped(r, log_path, scheduled_stopped_marker): + reasons.append('container was stopped before scheduled update could begin') + return finish('skipped', target, []) reboot_required, reboot_packages, reboot_check_error = ( _inspect_lxc_reboot_requirement( vmid, diff --git a/scripts/lxc/apply_updates.sh b/scripts/lxc/apply_updates.sh index 5737e66b..9e3e0665 100755 --- a/scripts/lxc/apply_updates.sh +++ b/scripts/lxc/apply_updates.sh @@ -163,6 +163,11 @@ fi # 2) CT must be running for pct exec. Auto-start stopped CTs, then # restore their original stopped state on every exit path. STATE=$(pct status "$VMID" | awk '{print $2}') +if [[ "$STATE" != "running" && "${EXECUTION_MODE:-}" == "scheduled" ]]; then + [[ -n "${SCHEDULED_STOPPED_MARKER:-}" ]] && printf '%s\n' "$SCHEDULED_STOPPED_MARKER" + echo "SKIPPED: CT $VMID is stopped; a scheduled update will not start it." + exit 8 +fi STARTED_BY_PROXMENUX=0 OBSERVATION_ATTEMPTED=0 capture_before_stop() { diff --git a/tests/lxc_updates/test_apply_updates.sh b/tests/lxc_updates/test_apply_updates.sh index 8b2ed500..1e42e2a0 100644 --- a/tests/lxc_updates/test_apply_updates.sh +++ b/tests/lxc_updates/test_apply_updates.sh @@ -162,6 +162,15 @@ run_case stopped_restore RUN_HELPER=0 UPDATE_COMMAND='update-custom' MOCK_INITIA [[ $(count 'pct start 101') -eq 1 ]] || fail "stopped CT was not started exactly once" [[ $(count 'pct shutdown 101 --timeout 60') -eq 1 ]] || fail "stopped CT state was not restored" +run_case scheduled_stopped RUN_HELPER=0 UPDATE_COMMAND='update-custom' \ + EXECUTION_MODE=scheduled SCHEDULED_STOPPED_MARKER='fixture-stopped-marker' MOCK_INITIAL_STATE=stopped +[[ $CASE_RC -eq 8 ]] || fail "scheduled_stopped returned $CASE_RC" +[[ $(count 'pct start 101') -eq 0 ]] || fail "scheduled update started a stopped CT" +[[ $(count 'pct exec') -eq 0 ]] || fail "scheduled update executed inside a stopped CT" +[[ $(count CUSTOM_EXEC) -eq 0 ]] || fail "scheduled update ran a custom updater" +grep -qF 'fixture-stopped-marker' "$TEST_ROOT/scheduled_stopped.out" \ + || fail "scheduled stopped result did not include its marker" + run_case locked RUN_HELPER=0 UPDATE_COMMAND='update-custom' MOCK_FLOCK_FAIL=1 [[ $CASE_RC -eq 7 ]] || fail "locked expected 7, got $CASE_RC" [[ $(count CUSTOM_EXEC) -eq 0 ]] || fail "locked run executed an updater" diff --git a/tests/lxc_updates/test_stopped_update_observation.py b/tests/lxc_updates/test_stopped_update_observation.py index 2f348324..50d518b2 100644 --- a/tests/lxc_updates/test_stopped_update_observation.py +++ b/tests/lxc_updates/test_stopped_update_observation.py @@ -119,7 +119,8 @@ raise SystemExit(h.main()) self.write("sleep", "#!/bin/sh\nexit 0\n") self.write("flock", "#!/bin/sh\nexit 0\n") names = {"_finalize_lxc_update", "_terminal_lxc_update_completed", - "api_lxc_updates_applied", "_run_scheduled_update"} + "api_lxc_updates_applied", "_run_scheduled_update", + "_scheduled_runner_reported_stopped"} nodes = [n for n in ast.parse(observed.SERVER.read_text()).body if isinstance(n, ast.FunctionDef) and n.name in names] self.scope = self.fixture.scope @@ -134,6 +135,7 @@ raise SystemExit(h.main()) "_json_list": lambda value: json.loads(value or '[]'), "_lxc_update_finalizations": {}, "_lxc_update_finalization_lock": threading.RLock(), "_LXC_UPDATE_FINALIZATION_TTL": 3600, + "_SCHEDULED_STOPPED_EXIT": 8, "_fast_guest_status": lambda *_: self.state.read_text(), "_lxc_update_snapshot": lambda *_: {"ct_name": "fixture"}, "_lxc_update_target_labels": lambda *_: ["OS"], @@ -315,6 +317,7 @@ raise SystemExit(h.main()) self.scope['notification_manager'].emit_event.assert_called_once() def test_scheduled_path_executes_real_runner_and_real_finalization(self): + self.state.write_text("running") def run(command, **kwargs): kwargs['env'].update({key: value for key, value in self.env.items() if key != 'RUN_ID'}) @@ -322,7 +325,7 @@ raise SystemExit(h.main()) self.scope['subprocess'] = SimpleNamespace(run=run, TimeoutExpired=subprocess.TimeoutExpired) result = self.scope['_run_scheduled_update'](100, {'targets': ['os'], 'target': 'os'}) self.assertEqual(result['status'], 'success', result) - self.assertEqual(self.state.read_text(), 'stopped') + self.assertEqual(self.state.read_text(), 'running') self.assertEqual(self.fixture.row()['update_check']['count'], 0) def test_legacy_cli_without_run_id_preserves_update_and_original_state(self): diff --git a/tests/lxc_updates/test_update_contract.py b/tests/lxc_updates/test_update_contract.py index 5466a7e2..029cd7a5 100644 --- a/tests/lxc_updates/test_update_contract.py +++ b/tests/lxc_updates/test_update_contract.py @@ -1,4 +1,6 @@ import ast +import json +import os import sys import subprocess import tempfile @@ -312,6 +314,118 @@ class AppCacheWriteThroughContractTests(unittest.TestCase): self.assertIn("before_snapshot=before", scheduler_block) +class ScheduledStoppedContainerTests(unittest.TestCase): + @staticmethod + def _worker(scope): + source = ast.parse((REPO_ROOT / "AppImage" / "scripts" / "flask_server.py").read_text()) + node = next(item for item in source.body + if isinstance(item, ast.FunctionDef) and item.name == "_run_scheduled_update") + exec(compile(ast.Module(body=[node], type_ignores=[]), "scheduled-update", "exec"), scope) + return scope["_run_scheduled_update"] + + @staticmethod + def _stopped_marker_reader(scope): + source = ast.parse((REPO_ROOT / "AppImage" / "scripts" / "flask_server.py").read_text()) + node = next(item for item in source.body + if isinstance(item, ast.FunctionDef) and item.name == "_scheduled_runner_reported_stopped") + exec(compile(ast.Module(body=[node], type_ignores=[]), "scheduled-marker", "exec"), scope) + return scope["_scheduled_runner_reported_stopped"] + + def test_stopped_container_skips_before_runner_or_any_guest_effect(self): + runner = MagicMock() + finalizer = MagicMock(return_value={"recorded": True}) + append_log = MagicMock() + scope = { + "time": SimpleNamespace(monotonic=MagicMock(return_value=10.0)), + "uuid": SimpleNamespace(uuid4=lambda: SimpleNamespace(hex="a" * 32)), + "datetime": __import__("datetime").datetime, + "subprocess": SimpleNamespace(run=runner, STDOUT=object(), TimeoutExpired=subprocess.TimeoutExpired), + "_create_lxc_update_log": lambda _vmid, _run_id: ("fixture.log", "/tmp/fixture.log"), + "_fast_guest_status": MagicMock(return_value="stopped"), + "_normalise_schedule_targets": lambda _sched: ["os"], + "_lxc_update_snapshot": lambda _vmid: {"ct_name": "fixture"}, + "_lxc_update_target_labels": lambda *_: ["OS"], + "_append_lxc_update_log": append_log, + "_prune_lxc_update_logs": MagicMock(), + "_finalize_lxc_update": finalizer, + } + + outcome = self._worker(scope)(101, {"target": "os"}) + + self.assertEqual(outcome["status"], "skipped") + self.assertEqual(outcome["executed_targets"], []) + self.assertIn("container is stopped", outcome["reason"]) + runner.assert_not_called() + self.assertEqual(finalizer.call_args.kwargs["status"], "skipped") + self.assertEqual(finalizer.call_args.kwargs["source"], "scheduled") + + def test_stopped_marker_requires_the_scheduler_token_not_only_exit_code(self): + marker = "PROXMENUX_SCHEDULED_STOPPED:fixture" + reader = self._stopped_marker_reader({"_SCHEDULED_STOPPED_EXIT": 8, "os": os}) + self.assertTrue(reader(SimpleNamespace(returncode=8, stdout=marker, stderr=""), None, marker)) + self.assertFalse(reader(SimpleNamespace(returncode=8, stdout="", stderr=""), None, marker)) + self.assertFalse(reader(SimpleNamespace(returncode=4, stdout=marker, stderr=""), None, marker)) + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log: + log.write("prefix\\n" + marker + "\\n") + log.flush() + self.assertTrue(reader(SimpleNamespace(returncode=8, stdout="", stderr=""), log.name, marker)) + + def test_runner_race_to_stopped_is_reported_as_skipped_without_reboot_probe(self): + runner = MagicMock(return_value=SimpleNamespace(returncode=8, stdout="", stderr="")) + finalizer = MagicMock(return_value={"recorded": True}) + reboot_probe = MagicMock() + stopped_marker = MagicMock(return_value=True) + lxc = SimpleNamespace( + _read_sidecar=lambda _vmid: {"apps": []}, + partition_scheduled_release_targets=lambda targets, _apps: (set(), targets), + ) + scope = { + "time": SimpleNamespace(monotonic=MagicMock(return_value=10.0)), + "uuid": SimpleNamespace(uuid4=lambda: SimpleNamespace(hex="b" * 32)), + "datetime": __import__("datetime").datetime, + "subprocess": SimpleNamespace(run=runner, STDOUT=object(), TimeoutExpired=subprocess.TimeoutExpired), + "os": os, + "json": json, + "_APPLY_UPDATES_SCRIPT": "/bin/true", + "_SCHEDULED_STOPPED_EXIT": 8, + "_DOCKER_ENGINE_INTEGRATED_COMMAND": "integrated-docker", + "_create_lxc_update_log": lambda _vmid, _run_id: (None, None), + "_fast_guest_status": MagicMock(return_value="running"), + "_normalise_schedule_targets": lambda _sched: ["os"], + "_lxc_update_snapshot": lambda _vmid: {"ct_name": "fixture"}, + "_lxc_update_target_labels": lambda *_: ["OS"], + "_append_lxc_update_log": MagicMock(), + "_prune_lxc_update_logs": MagicMock(), + "_finalize_lxc_update": finalizer, + "_scheduled_helper_enabled": lambda *_: False, + "_compose_scheduled_update_command": lambda *_: "", + "_inspect_lxc_reboot_requirement": reboot_probe, + "_scheduled_runner_reported_stopped": stopped_marker, + } + + with patch.dict(sys.modules, {"lxc_apps": lxc}): + outcome = self._worker(scope)(101, {"target": "os"}) + + self.assertEqual(outcome["status"], "skipped") + self.assertEqual(outcome["executed_targets"], []) + self.assertIn("before scheduled update could begin", outcome["reason"]) + self.assertEqual(runner.call_args.args[0], ["bash", "/bin/true"]) + self.assertEqual(runner.call_args.kwargs["env"]["EXECUTION_MODE"], "scheduled") + self.assertTrue(runner.call_args.kwargs["env"]["SCHEDULED_STOPPED_MARKER"].startswith( + "PROXMENUX_SCHEDULED_STOPPED:scheduled-")) + stopped_marker.assert_called_once() + reboot_probe.assert_not_called() + + stopped_marker.reset_mock() + stopped_marker.return_value = False + reboot_probe.return_value = (None, [], None) + outcome = self._worker(scope)(101, {"target": "os"}) + self.assertEqual(outcome["status"], "failure") + self.assertIn("exited with code 8", outcome["reason"]) + stopped_marker.assert_called_once() + reboot_probe.assert_called_once() + + class DockerStackNotificationTests(unittest.TestCase): def _docker_app(self): return { diff --git a/tests/lxc_updates/test_update_method_choice.py b/tests/lxc_updates/test_update_method_choice.py index affacbeb..84124962 100644 --- a/tests/lxc_updates/test_update_method_choice.py +++ b/tests/lxc_updates/test_update_method_choice.py @@ -37,6 +37,7 @@ def routes(): _lxc_update_target_labels=lambda *_: [], _append_lxc_update_log=lambda *_: None, _prune_lxc_update_logs=lambda *_: None, + _scheduled_runner_reported_stopped=lambda *_: False, _finalize_lxc_update=Mock(return_value={}), _inspect_lxc_reboot_requirement=lambda *_, **__: (False, [], None)) exec(compile(ast.Module(body=nodes, type_ignores=[]), 'update-routes', 'exec'), ns)