"""Real inert processes and actual Flask/ticket/socket handler boundaries.""" import ast import base64 import io import importlib.util import json import os import pty import re import select import fcntl import struct import termios from pathlib import Path import shlex import subprocess import sys import tempfile import threading import time from types import ModuleType, SimpleNamespace import unittest from unittest.mock import Mock, patch from flask import Blueprint, Flask, jsonify, request ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "AppImage/scripts")) import lxc_update_jobs as jobs def wait_for(predicate, timeout=6): deadline = time.monotonic() + timeout while not predicate(): if time.monotonic() >= deadline: raise AssertionError("Timed out waiting for fixture") time.sleep(0.01) class JobTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.folder = Path(self.temp.name) self.script = self.folder / "apply.sh" self.gate = self.folder / "backup-done" self.events = self.folder / "events" self.script.write_text( f"echo backup-start; echo backup >> {shlex.quote(str(self.events))}\n" f"while [ ! -e {shlex.quote(str(self.gate))} ]; do sleep .02; done\n" f"echo update-finished; echo update >> {shlex.quote(str(self.events))}\n" ) self.allowed = patch.object(jobs, "SCRIPTS", {str(self.script)}) self.allowed.start() self.addCleanup(self.allowed.stop) jobs._jobs.clear() self.callback = Mock() self.addCleanup(self.stop_jobs) def stop_jobs(self): for job in list(jobs._jobs.values()): if job.state != "complete": job.cancel() wait_for(lambda: job.state == "complete") jobs._jobs.clear() def params(self, run="run-1", vmid="100"): return dict(VMID=vmid, RUN_ID=run, TARGET="app", CT_NAME="fixture", BACKUP="1") def start(self, run="run-1", vmid="100"): return jobs.start(run, str(self.script), self.params(run, vmid), self.callback) def finish(self, job): self.gate.touch() wait_for(lambda: job.state == "complete") def test_real_backup_and_update_continue_without_any_viewer(self): job = self.start() wait_for(lambda: self.events.exists()) self.assertEqual(self.events.read_text().splitlines(), ["backup"]) self.assertEqual(job.state, "running") self.finish(job) self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) self.assertEqual(job.exit_code, 0) self.callback.assert_called_once() self.assertEqual(self.callback.call_args.args[:3], (str(self.script), self.params(), 0)) self.assertIn("update-finished", job.read(0)["text"]) def test_same_run_reconnect_is_idempotent_and_other_ct_cannot_attach(self): job = self.start() self.assertIs(self.start(), job) with self.assertRaises(KeyError): jobs.get(job.id, 200) with self.assertRaises(KeyError): jobs.get("lost-run", 100) self.finish(job) self.assertIs(jobs.get(job.id, 100), job) self.assertIs(self.start(), job) self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) self.callback.assert_called_once() def test_different_run_for_same_ct_rejected_before_second_spawn(self): job = self.start() with patch.object(jobs.subprocess, "Popen") as spawn: with self.assertRaises(jobs.JobConflict): self.start("run-2") spawn.assert_not_called() self.finish(job) other = self.start("run-2") wait_for(lambda: other.state == "complete") self.assertEqual(self.events.read_text().splitlines().count("update"), 2) def test_concurrent_same_id_starts_once(self): calls, errors = [], [] barrier = threading.Barrier(6) def attempt(): barrier.wait() try: calls.append(self.start()) except Exception as exc: errors.append(exc) threads = [threading.Thread(target=attempt) for _ in range(6)] for thread in threads: thread.start() for thread in threads: thread.join() self.assertFalse(errors) self.assertEqual(len({id(job) for job in calls}), 1) self.finish(calls[0]) self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) def test_invalid_request_has_zero_process_and_zero_tempfile_effects(self): bad = [ ("../bad", str(self.script), self.params()), ("run-1", "/bin/bash", self.params()), ("run-1", str(self.script), dict(self.params(), LD_PRELOAD="/bad")), ("run-1", str(self.script), dict(self.params(), VMID="0")), ("run-1", str(self.script), dict(self.params(), RUN_ID="other")), ("run-1", str(self.script), dict(self.params(), UPDATE_COMMAND=123)), ("run-1", str(self.script), dict(self.params(), TARGET="unexpected")), ] with patch.object(jobs.subprocess, "Popen") as spawn, \ patch.object(jobs.tempfile, "mkstemp") as files: for identity, script, params in bad: with self.subTest(params=params), self.assertRaises(ValueError): jobs.start(identity, script, params, self.callback) spawn.assert_not_called() files.assert_not_called() self.assertFalse(jobs.list_jobs()) def test_request_id_cannot_be_rebound_to_another_command(self): job = self.start() with patch.object(jobs.subprocess, "Popen") as spawn: with self.assertRaises(jobs.JobConflict): jobs.start("run-1", str(self.script), dict(self.params(), UPDATE_COMMAND="different"), self.callback) spawn.assert_not_called() self.finish(job) def test_failure_and_launch_error_are_not_reported_as_success(self): self.script.write_text("echo failed-backup; exit 3\n") job = self.start() wait_for(lambda: job.state == "complete") self.assertEqual(job.exit_code, 3) self.assertFalse(self.events.exists()) self.callback.assert_called_once() with patch.object(jobs.subprocess, "Popen", side_effect=OSError("cannot execute")): with self.assertRaises(OSError): self.start("run-2") self.assertNotIn("run-2", jobs._jobs) def test_cancel_is_explicit_and_kills_the_inert_process_group(self): job = self.start() wait_for(lambda: self.events.exists()) job.input({"type": "cancel"}) wait_for(lambda: job.state == "complete") self.assertEqual(job.exit_code, 130) self.assertEqual(self.events.read_text().splitlines(), ["backup"]) self.callback.assert_called_once() def test_timeout_is_bounded_even_if_the_child_closes_its_terminal(self): self.script.write_text("exec /dev/null 2>&1\nwhile :; do sleep .02; done\n") with patch.object(jobs, "MAX_RUNTIME", .15): job = self.start() wait_for(lambda: job.state == "complete") self.assertEqual(job.exit_code, 124) self.callback.assert_called_once() def test_cancel_waits_for_the_runners_original_state_cleanup(self): # apply_updates.sh's EXIT trap may need a 60s package probe followed # by `pct shutdown --timeout 60`. A two-second kill truncates that trap. self.script.write_text( f"trap 'sleep 2.3; echo restored >> {shlex.quote(str(self.events))}; exit 0' TERM\n" + self.script.read_text()) job = self.start() wait_for(lambda: self.events.exists()) job.cancel() wait_for(lambda: job.state == 'complete') self.assertEqual(job.exit_code, 130) self.assertEqual(self.events.read_text().splitlines(), ['backup', 'restored']) self.assertNotIn('update', self.events.read_text()) self.callback.assert_called_once() self.assertGreater(jobs.STOP_GRACE, 120) def test_timeout_still_reaps_a_process_that_ignores_termination(self): self.script.write_text("trap '' TERM\necho ready\nwhile :; do sleep .02; done\n") with patch.object(jobs, 'MAX_RUNTIME', .15), patch.object(jobs, 'STOP_GRACE', .1): job = self.start() wait_for(lambda: job.state == 'complete') self.assertEqual(job.exit_code, 124) self.assertIsNotNone(job.process.poll()) self.callback.assert_called_once() def test_output_is_bounded_and_cursor_replay_is_read_only(self): job = self.start() wait_for(lambda: self.events.exists()) with patch.object(jobs, "MAX_OUTPUT", 32): job._append("x" * 100) snap = job.read(0) self.assertEqual(len(snap["text"]), 32) self.assertTrue(snap["truncated"]) self.assertEqual(job.read(snap["offset"])["text"], "") self.finish(job) def test_unknown_interaction_response_never_writes_a_file(self): job = self.start() with patch.object(jobs.os, "open") as files: with self.assertRaises(ValueError): job.input({"type": "interaction_response", "id": "not-pending", "value": "yes"}) files.assert_not_called() self.finish(job) def test_large_paste_cannot_block_output_cancel_or_timeout(self): self.script.write_text("stty raw -echo\necho ready\nwhile :; do echo progress; sleep .02; done\n") for run, cancel in [("cancel-paste", True), ("timeout-paste", False)]: with self.subTest(run=run), patch.object(jobs, "MAX_RUNTIME", .5), \ patch.object(jobs, "STOP_GRACE", .1): job = self.start(run) wait_for(lambda: "ready" in job.read(0)["text"]) before = job.read(0)["offset"] job.input("x" * 16384) job.input("y" * 16384) wait_for(lambda: job.read(0)["offset"] > before) if cancel: job.cancel() wait_for(lambda: job.state == "complete") self.assertEqual(job.exit_code, 130 if cancel else 124) self.assertIsNotNone(job.process.poll()) self.assertFalse(job._input) self.assertEqual(self.callback.call_count, 2) def test_queued_paste_reaches_a_slow_reader_in_order_without_losing_bytes(self): payload = ("Ž🙂abc" * 3500).encode() received = self.folder / "received" reader = ( "import os,pathlib; data=bytearray(); " f"size={len(payload)}; " "exec('while len(data)"](ws, "run-1") return ws def test_expired_login_cannot_start_and_has_zero_job_effects(self): for token, status in [("expired", 401), ("read-only", 403)]: with self.subTest(token=token), patch.object(jobs.subprocess, "Popen") as launch: response = self.ticket(token) self.assertEqual(response.status_code, status) self.attach(dict(resume=False, script_path=str(self.script), params=self.params())) launch.assert_not_called() self.assertFalse(jobs.list_jobs()) def test_expiry_during_backup_does_not_cancel_or_restart_and_fresh_login_resumes(self): ticket = self.ticket().json["ticket"] self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), ticket) job = jobs.get("run-1", 100) wait_for(lambda: self.events.exists()) denied = self.client.get("/api/lxc-update-jobs", headers={"Authorization": "Bearer expired"}) self.assertEqual(denied.status_code, 401) self.assertIsNone(job.stop_reason) self.assertEqual(self.client.get("/api/lxc-update-jobs", headers={"Authorization": "Bearer valid"}).json["jobs"][0]["id"], job.id) with patch.object(jobs.subprocess, "Popen") as launch: self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) launch.assert_not_called() self.finish(job) self.assertEqual(job.exit_code, 0) self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) self.callback.assert_called_once() complete = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) frames = [json.loads(c.args[0]) for c in complete.send.call_args_list] self.assertEqual(frames[-1], {"type": "script_complete", "exit_code": 0}) def test_answer_in_one_viewer_clears_the_question_for_all_viewers(self): self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), self.ticket().json["ticket"]) job = jobs.get("run-1", 100) identity = "fixture-" + os.urandom(8).hex() response = Path("/tmp/proxmenux_response_" + identity) self.addCleanup(lambda: response.unlink(missing_ok=True)) job.interaction = dict(type="yesno", id=identity, title="Question", message="Continue?") job.revision += 1 first = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"], [json.dumps(dict(type="interaction_response", id=identity, value="yes"))]) second = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) for ws in [first, second]: frames = [json.loads(c.args[0]) for c in ws.send.call_args_list] self.assertIn(dict(type="web_interaction", interaction=None), frames) self.assertEqual(response.read_text(), "yes") self.assertIsNone(job.stop_reason) self.callback.assert_not_called() self.finish(job) self.callback.assert_called_once() def test_ticket_is_single_use_and_invalid_resume_never_becomes_execution(self): ticket = self.ticket().json["ticket"] with patch.object(jobs.subprocess, "Popen") as launch: ws = self.attach(dict(resume=True, vmid=100), ticket) self.assertIn("job_unavailable", ws.send.call_args.args[0]) denied = self.attach(dict(resume=True, vmid=100), ticket) denied.close.assert_called_once() launch.assert_not_called() def test_expiry_during_update_then_two_viewers_cannot_duplicate_or_finalize_early(self): update_gate = self.folder / "update-done" self.script.write_text(self.script.read_text() + f"while [ ! -e {shlex.quote(str(update_gate))} ]; do sleep .02; done\n") self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), self.ticket().json["ticket"]) job = jobs.get("run-1", 100) self.gate.touch() wait_for(lambda: self.events.exists() and "update" in self.events.read_text()) for token, status in [("expired", 401), ("read-only", 403)]: self.assertEqual(self.client.get("/api/lxc-update-jobs", headers={ "Authorization": "Bearer " + token}).status_code, status) self.callback.assert_not_called() with patch.object(jobs.subprocess, "Popen") as spawn: for _ in range(2): self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) spawn.assert_not_called() self.assertIsNone(job.stop_reason) update_gate.touch() wait_for(lambda: job.state == "complete") self.callback.assert_called_once() self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) def test_original_interactive_route_still_terminates_its_process_on_disconnect(self): # Reproduce the original ownership bug using the untouched legacy route. # Only its allowed fixture directory and process observation are injected. nodes = [n for n in ast.parse((ROOT / "AppImage/scripts/flask_terminal_routes.py").read_text()).body if isinstance(n, ast.FunctionDef) and n.name == "script_websocket"] realpath = os.path.realpath os_view = SimpleNamespace(**vars(os)) os_view.path = SimpleNamespace(**vars(os.path)) os_view.path.realpath = lambda p: (str(self.folder) if p == "/usr/local/share/proxmenux/scripts" else realpath(p)) children = [] def spawn(*args, **kwargs): child = subprocess.Popen(*args, **kwargs) children.append(child) self.addCleanup(lambda: child.poll() is None and child.kill()) return child self.scope.update(os=os_view, subprocess=SimpleNamespace(Popen=spawn), tempfile=tempfile, pty=pty, fcntl=fcntl, select=select, base64=base64, threading=threading, _SAFE_ID_RE=re.compile(r"^[A-Za-z0-9_-]{1,64}$"), set_winsize=lambda *_: None) exec(compile(ast.Module(body=nodes, type_ignores=[]), 'actual-legacy-route', 'exec'), self.scope) ws = Mock() def receive(**_): if not children: return json.dumps(dict(script_path=str(self.script), params=self.params())) wait_for(lambda: self.events.exists()) raise ConnectionError("expired login reloaded browser") ws.receive.side_effect = receive with self.app.test_request_context('/ws/script/fixture?ticket='+self.ticket().json['ticket']): self.ws_routes['/ws/script/'](ws, 'fixture') self.assertIsNotNone(children[0].poll()) self.assertNotEqual(children[0].returncode, 0) self.assertEqual(self.events.read_text().splitlines(), ['backup']) if __name__ == "__main__": unittest.main()