fix: keep update viewers responsive under input backpressure

AppImage/components/script-terminal-modal.tsx

@@ -222,18 +222,30 @@ export function ScriptTerminalModal({
           try {
             const msg = JSON.parse(event.data)
             if (msg.type === "job_output") {
+              setIsWaitingNextInteraction(false)
+              if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
               if (msg.truncated) termRef.current?.writeln(t("vmLxc.updates.backgroundOutputTrimmed"))
               termRef.current?.write(msg.text)
               outputOffsetRef.current = msg.offset
               return
             }
             if (msg.type === "job_unavailable") {
+              setCurrentInteraction(null)
+              setIsWaitingNextInteraction(false)
+              if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
               termRef.current?.writeln(`\x1b[31m${msg.message}\x1b[0m`)
               completionReceivedRef.current = true
               setIsComplete(true)
               ws.close()
               return
             }
+            if (backgroundUpdate && msg.type === "web_interaction" && msg.interaction === null) {
+              setCurrentInteraction(null)
+              setInteractionInput("")
+              setIsWaitingNextInteraction(false)
+              if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
+              return
+            }
             if (msg.type === "web_interaction" && msg.interaction) {
               setIsWaitingNextInteraction(false)
               if (waitingTimeoutRef.current) {
@@ -254,6 +266,11 @@ export function ScriptTerminalModal({
               return
             }
             if (msg.type === "script_complete") {
+              if (backgroundUpdate) {
+                setCurrentInteraction(null)
+                setIsWaitingNextInteraction(false)
+                if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
+              }
               const exitCode = Number(msg.exit_code ?? 1)
               completionReceivedRef.current = true
               setIsComplete(true)
@@ -462,12 +479,17 @@ export function ScriptTerminalModal({
       try {
         const msg = JSON.parse(event.data)
         if (msg.type === "job_output") {
+          setIsWaitingNextInteraction(false)
+          if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
           if (msg.truncated) term.writeln(t("vmLxc.updates.backgroundOutputTrimmed"))
           term.write(msg.text)
           outputOffsetRef.current = msg.offset
           return
         }
         if (msg.type === "job_unavailable") {
+          setCurrentInteraction(null)
+          setIsWaitingNextInteraction(false)
+          if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
           term.writeln(`\x1b[31m${msg.message}\x1b[0m`)
           completionReceivedRef.current = true
           setIsComplete(true)
@@ -475,6 +497,13 @@ export function ScriptTerminalModal({
           return
         }
 
+        if (backgroundUpdate && msg.type === "web_interaction" && msg.interaction === null) {
+          setCurrentInteraction(null)
+          setInteractionInput("")
+          setIsWaitingNextInteraction(false)
+          if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
+          return
+        }
         if (msg.type === "web_interaction" && msg.interaction) {
           setIsWaitingNextInteraction(false)
           if (waitingTimeoutRef.current) {
@@ -496,6 +525,11 @@ export function ScriptTerminalModal({
           return
         }
         if (msg.type === "script_complete") {
+          if (backgroundUpdate) {
+            setCurrentInteraction(null)
+            setIsWaitingNextInteraction(false)
+            if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
+          }
           const exitCode = Number(msg.exit_code ?? 1)
           completionReceivedRef.current = true
           setIsComplete(true)

AppImage/scripts/flask_terminal_routes.py

@@ -171,9 +171,10 @@ def lxc_update_viewer(ws, run_id):
                 offset = view['offset']
             if view['revision'] != revision:
                 revision = view['revision']
-                if view['interaction']:
+                # A second viewer must also dismiss a question already
-                    ws.send(json.dumps({'type': 'web_interaction',
+                # answered elsewhere, or cleared when the job ends.
-                                        'interaction': view['interaction']}))
+                ws.send(json.dumps({'type': 'web_interaction',
+                                    'interaction': view['interaction']}))
             if view['state'] == 'complete':
                 ws.send(json.dumps({'type': 'script_complete', 'exit_code': view['exit_code']}))
                 break

AppImage/scripts/lxc_update_jobs.py

@@ -40,6 +40,7 @@ MAX_RUNTIME = 4 * 3600
 STOP_GRACE = 300
 RESULT_TTL = 6 * 3600
 MAX_OUTPUT = 256 * 1024
+MAX_INPUT = 64 * 1024
 MAX_JOBS = 32
 _lock = threading.RLock()
 _jobs = {}
@@ -91,11 +92,15 @@ class UpdateJob:
         self.process = None
         self.log_path = None
         self._interaction_buffer = ""
+        self._input = bytearray()
 
     def launch(self):
         master, slave = pty.openpty()
         self.master = master
         try:
+            # A viewer's paste must never block the worker's output, timeout
+            # or cancellation when the updater does not read its stdin.
+            os.set_blocking(master, False)
             fd, self.log_path = tempfile.mkstemp(prefix="proxmenux_update_", suffix=".log")
             os.close(fd)
             fcntl.ioctl(master, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 120, 0, 0))
@@ -158,8 +163,9 @@ class UpdateJob:
                     elif kind in ("input", "inputbox"):
                         value["default"] = base64.b64decode(parts[4]).decode()
                 with self.lock:
-                    self.interaction = value
+                    if not self.stop_reason:
-                    self.revision += 1
+                        self.interaction = value
+                        self.revision += 1
             except (ValueError, UnicodeError, IndexError):
                 continue
 
@@ -168,6 +174,9 @@ class UpdateJob:
             if self.process.poll() is not None or self.stop_reason:
                 return
             self.stop_reason, self.stop_at = reason, time.monotonic()
+            self._input.clear()
+            self.interaction = None
+            self.revision += 1
             try:
                 os.killpg(self.process.pid, signal.SIGTERM)
             except ProcessLookupError:
@@ -176,6 +185,25 @@ class UpdateJob:
     def cancel(self):
         self._signal("cancelled")
 
+    def _write_input(self):
+        # The descriptor is nonblocking, so even a full PTY cannot hold this
+        # lock. Retain unwritten bytes in order until the child can read them.
+        with self.lock:
+            if not self._input or self.stop_reason:
+                return
+            try:
+                written = os.write(self.master, self._input[:16384])
+            except BlockingIOError:
+                return
+            except OSError as exc:
+                if exc.errno != errno.EIO:
+                    raise
+                # The child can close its terminal and keep working. Losing
+                # stdin must not turn that update into a viewer failure.
+                self._input.clear()
+                return
+            del self._input[:written]
+
     def _run(self):
         decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
         eof, exited_at = False, None
@@ -196,10 +224,18 @@ class UpdateJob:
                         except ProcessLookupError:
                             pass
                     self._interactions(interactions)
-                    readable, _, _ = select.select([] if eof else [self.master], [], [], 0.05)
+                    with self.lock:
+                        pending_input = bool(self._input) and running and not self.stop_reason
+                    readable, writable, _ = select.select(
+                        [] if eof else [self.master],
+                        [self.master] if pending_input else [], [], 0.05)
+                    if writable:
+                        self._write_input()
                     if readable:
                         try:
                             data = os.read(self.master, 16384)
+                        except BlockingIOError:
+                            continue
                         except OSError as exc:
                             if exc.errno != errno.EIO:
                                 raise
@@ -232,6 +268,7 @@ class UpdateJob:
             self.exit_code = code
             self.state = "finalizing"
             self.interaction = None
+            self._input.clear()
             self.revision += 1
         try:
             self.callback(self.script, self.params, code, time.monotonic() - self.started)
@@ -266,7 +303,11 @@ class UpdateJob:
                 if message == "\x03":
                     self.cancel()
                 else:
-                    os.write(self.master, message.encode())
+                    data = message.encode()
+                    if len(self._input) + len(data) > MAX_INPUT:
+                        raise ValueError("Terminal input buffer is full")
+                    if not self.stop_reason:
+                        self._input.extend(data)
             elif message.get("type") == "cancel":
                 self.cancel()
             elif message.get("type") == "resize":

tests/test_lxc_update_jobs.cjs

@@ -32,7 +32,8 @@ function fixture(backgroundUpdate) {
     reconnectAttemptsRef: {current: 0}, reconnectTimeoutRef: {current: null},
     onCompleteRef: {current: code => finished.push(code)},
     setIsComplete: value => context.isComplete = value, setConnectionStatus() {},
-    setIsWaitingNextInteraction() {}, setCurrentInteraction: value => context.interaction = value,
+    setIsWaitingNextInteraction: value => context.waiting = value,
+    setCurrentInteraction: value => context.interaction = value, setInteractionInput() {},
     clearTimeout() {}, clearInterval() {}, setTimeout: fn => { fn(); return 1 },
     attemptReconnect: () => reconnects.push(true), t: key => key,
     getCompletionMessage: code => 'Result ' + code,
@@ -52,8 +53,18 @@ for (const handler of handlers.onmessage) {
   assert.equal(f.context.outputOffsetRef.current, 27)
   message({data: JSON.stringify({type: 'web_interaction', interaction: {id:'pending', type:'yesno'}})})
   assert.equal(f.context.interaction.id, 'pending', 'replayed interaction remains answerable')
+  f.context.waiting = true
+  message({data: JSON.stringify({type: 'web_interaction', interaction: null})})
+  assert.equal(f.context.interaction, null, 'answer in another viewer dismisses the stale question')
+  assert.equal(f.context.waiting, false)
+  f.context.waiting = true
+  message({data: JSON.stringify({type: 'job_output', text: 'working', offset: 34})})
+  assert.equal(f.context.waiting, false, 'ordinary progress clears the post-answer spinner')
+  f.context.interaction = {id: 'stale'}; f.context.waiting = true
   message({data: JSON.stringify({type: 'script_complete', exit_code: 3})})
   assert.deepEqual(f.finished, [3])
+  assert.equal(f.context.interaction, null, 'a completed job cannot retain an active question')
+  assert.equal(f.context.waiting, false, 'a completed job cannot retain a processing overlay')
   message({data: JSON.stringify({type: 'script_complete', exit_code: 0})})
   assert.deepEqual(f.finished, [3], 'replayed completion cannot finalize twice')
   const missing = fixture({runId: 'missing'})

tests/test_lxc_update_jobs.py

@@ -231,6 +231,140 @@ class JobTests(unittest.TestCase):
             files.assert_not_called()
         self.finish(job)
 
+    def test_large_paste_cannot_block_output_cancel_or_timeout(self):
+        self.script.write_text("stty raw -echo\necho ready\nwhile :; do echo progress; sleep .02; done\n")
+        for run, cancel in [("cancel-paste", True), ("timeout-paste", False)]:
+            with self.subTest(run=run), patch.object(jobs, "MAX_RUNTIME", .5), \
+                    patch.object(jobs, "STOP_GRACE", .1):
+                job = self.start(run)
+                wait_for(lambda: "ready" in job.read(0)["text"])
+                before = job.read(0)["offset"]
+                job.input("x" * 16384)
+                job.input("y" * 16384)
+                wait_for(lambda: job.read(0)["offset"] > before)
+                if cancel:
+                    job.cancel()
+                wait_for(lambda: job.state == "complete")
+                self.assertEqual(job.exit_code, 130 if cancel else 124)
+                self.assertIsNotNone(job.process.poll())
+                self.assertFalse(job._input)
+        self.assertEqual(self.callback.call_count, 2)
+
+    def test_queued_paste_reaches_a_slow_reader_in_order_without_losing_bytes(self):
+        payload = ("Ž🙂abc" * 3500).encode()
+        received = self.folder / "received"
+        reader = (
+            "import os,pathlib; data=bytearray(); "
+            f"size={len(payload)}; "
+            "exec('while len(data)<size:\\n data.extend(os.read(0,min(257,size-len(data))))'); "
+            f"pathlib.Path({str(received)!r}).write_bytes(data)"
+        )
+        self.script.write_text(
+            "stty raw -echo\necho ready\n"
+            f"while [ ! -f {shlex.quote(str(self.gate))} ]; do sleep .02; done\n"
+            + shlex.quote(sys.executable) + " -c " + shlex.quote(reader) + "\n")
+        job = self.start()
+        wait_for(lambda: "ready" in job.read(0)["text"])
+        text = payload.decode()
+        for offset in range(0, len(text), 8000):
+            job.input(text[offset:offset + 8000])
+        self.gate.touch()
+        wait_for(lambda: job.state == "complete")
+        self.assertEqual(job.exit_code, 0)
+        self.assertEqual(received.read_bytes(), payload)
+        self.callback.assert_called_once()
+
+    def test_full_input_queue_rejects_before_any_write_and_preserves_pending_data(self):
+        job = self.start()
+        with job.lock, patch.object(jobs.os, "write") as write:
+            job._input = bytearray(b"x" * jobs.MAX_INPUT)
+            before = bytes(job._input)
+            with self.assertRaisesRegex(ValueError, "buffer is full"):
+                job.input("overflow")
+            self.assertEqual(bytes(job._input), before)
+            write.assert_not_called()
+            job._input.clear()
+        self.assertIsNone(job.stop_reason)
+        self.finish(job)
+
+    def test_multibyte_output_crosses_pty_chunks_without_corrupting_replay(self):
+        payload = "Ž🙂abc" * 5000
+        command = "import os; os.write(1," + repr(payload) + ".encode())"
+        self.script.write_text(shlex.quote(sys.executable) + " -c " + shlex.quote(command) + "\n")
+        job = self.start()
+        wait_for(lambda: job.state == "complete")
+        self.assertEqual(job.read(0)["text"], payload)
+        self.assertEqual(job.read(len(payload) - 4)["text"], payload[-4:])
+
+    def test_capacity_rejection_preserves_running_jobs_and_evicts_only_completed(self):
+        with patch.object(jobs, "MAX_JOBS", 2):
+            first, second = self.start(), self.start("second", "101")
+            before = set(jobs._jobs)
+            with patch.object(jobs.subprocess, "Popen") as spawn:
+                with self.assertRaises(jobs.JobConflict):
+                    self.start("third", "102")
+                spawn.assert_not_called()
+            self.assertEqual(set(jobs._jobs), before)
+            with second.lock:
+                self.finish(first)
+            wait_for(lambda: second.state == "complete")
+            third = self.start("third", "102")
+            wait_for(lambda: third.state == "complete")
+            self.assertNotIn(first.id, jobs._jobs)
+            self.assertIn(second.id, jobs._jobs)
+            self.assertIn(third.id, jobs._jobs)
+
+    def test_finalization_keeps_the_guest_reserved_and_callback_runs_once(self):
+        entered, release = threading.Event(), threading.Event()
+        self.addCleanup(release.set)
+        def finalize(*_):
+            entered.set()
+            release.wait(3)
+        self.callback.side_effect = finalize
+        job = self.start()
+        self.gate.touch()
+        self.assertTrue(entered.wait(3))
+        self.assertEqual(job.state, "finalizing")
+        with patch.object(jobs.subprocess, "Popen") as spawn:
+            with self.assertRaises(jobs.JobConflict):
+                self.start("other")
+            self.assertIs(jobs.get(job.id, 100), job)
+            spawn.assert_not_called()
+        release.set()
+        wait_for(lambda: job.state == "complete")
+        self.callback.assert_called_once()
+
+    def test_worker_launch_failure_reaps_the_process_without_registering_a_job(self):
+        children = []
+        real_spawn = subprocess.Popen
+        def spawn(*args, **kwargs):
+            child = real_spawn(*args, **kwargs)
+            children.append(child)
+            return child
+        with patch.object(jobs.subprocess, "Popen", side_effect=spawn), \
+                patch.object(jobs.threading.Thread, "start", side_effect=RuntimeError("no worker")):
+            with self.assertRaisesRegex(RuntimeError, "no worker"):
+                self.start()
+        self.assertFalse(jobs.list_jobs())
+        self.assertEqual(len(children), 1)
+        self.assertIsNotNone(children[0].poll())
+        self.callback.assert_not_called()
+
+    def test_interaction_file_collision_is_not_overwritten_or_accepted(self):
+        job = self.start()
+        identity = "fixture-" + os.urandom(8).hex()
+        response = Path("/tmp/proxmenux_response_" + identity)
+        self.addCleanup(lambda: response.unlink(missing_ok=True))
+        response.symlink_to(self.events)
+        job.interaction = dict(type="input", id=identity)
+        wait_for(lambda: self.events.exists())
+        before = self.events.read_bytes() if self.events.exists() else None
+        with self.assertRaises(OSError):
+            job.input(dict(type="interaction_response", id=identity, value="do-not-write"))
+        self.assertEqual(job.interaction["id"], identity)
+        self.assertEqual(self.events.read_bytes() if self.events.exists() else None, before)
+        self.finish(job)
+
     def test_real_process_exit_calls_existing_backend_finalizer_without_viewer(self):
         source = ast.parse((ROOT / "AppImage/scripts/flask_server.py").read_text())
         node = next(n for n in source.body if isinstance(n, ast.FunctionDef)
@@ -414,6 +548,27 @@ class RouteTests(unittest.TestCase):
         frames = [json.loads(c.args[0]) for c in complete.send.call_args_list]
         self.assertEqual(frames[-1], {"type": "script_complete", "exit_code": 0})
 
+    def test_answer_in_one_viewer_clears_the_question_for_all_viewers(self):
+        self.attach(dict(resume=False, script_path=str(self.script), params=self.params()),
+                    self.ticket().json["ticket"])
+        job = jobs.get("run-1", 100)
+        identity = "fixture-" + os.urandom(8).hex()
+        response = Path("/tmp/proxmenux_response_" + identity)
+        self.addCleanup(lambda: response.unlink(missing_ok=True))
+        job.interaction = dict(type="yesno", id=identity, title="Question", message="Continue?")
+        job.revision += 1
+        first = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"],
+            [json.dumps(dict(type="interaction_response", id=identity, value="yes"))])
+        second = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"])
+        for ws in [first, second]:
+            frames = [json.loads(c.args[0]) for c in ws.send.call_args_list]
+            self.assertIn(dict(type="web_interaction", interaction=None), frames)
+        self.assertEqual(response.read_text(), "yes")
+        self.assertIsNone(job.stop_reason)
+        self.callback.assert_not_called()
+        self.finish(job)
+        self.callback.assert_called_once()
+
     def test_ticket_is_single_use_and_invalid_resume_never_becomes_execution(self):
         ticket = self.ticket().json["ticket"]
         with patch.object(jobs.subprocess, "Popen") as launch: