diff --git a/.github/scripts/run_offline_qualified.py b/.github/scripts/run_offline_qualified.py index e8bf3dd6..5880c6ba 100644 --- a/.github/scripts/run_offline_qualified.py +++ b/.github/scripts/run_offline_qualified.py @@ -13,6 +13,7 @@ import sys import tempfile PYTHON_FILES = ( + "tests/test_lxc_update_jobs.py", "tests/test_post_install_i18n.py", "tests/test_remaining_cli_i18n.py", "AppImage/scripts/tests/test_pbs_i18n.py", @@ -33,6 +34,7 @@ PYTHON_FILES = ( "AppImage/scripts/tests/test_health_persistence_recurrence.py", ) NODE_FILES = ( + "tests/test_lxc_update_jobs.cjs", "tests/test_api_import_boundaries.cjs", "tests/test_api_errors.cjs", "tests/test_optimization_labels.cjs", diff --git a/AppImage/components/lxc-update-jobs.tsx b/AppImage/components/lxc-update-jobs.tsx new file mode 100644 index 00000000..1591f87c --- /dev/null +++ b/AppImage/components/lxc-update-jobs.tsx @@ -0,0 +1,48 @@ +"use client" + +import { useEffect, useState } from "react" +import { Button } from "./ui/button" +import { fetchApi } from "../lib/api-config" +import { useT } from "../lib/i18n/provider" +import { viewedUpdateJobs, type LxcUpdateJob } from "../lib/lxc-update-jobs" + +export function LxcUpdateJobs({ activeRunId, onResume }: { + activeRunId?: string + onResume: (job: LxcUpdateJob) => void +}) { + const t = useT() + const [jobs, setJobs] = useState([]) + const [seen, setSeen] = useState([]) + useEffect(() => { + let closed = false + const read = () => { + void fetchApi<{ jobs: LxcUpdateJob[] }>("/api/lxc-update-jobs") + .then((data) => { if (!closed) setJobs(Array.isArray(data.jobs) ? data.jobs : []) }) + .catch(() => {}) // Auth recovery belongs to fetchApi, never to a job retry. + } + const viewed = () => setSeen(viewedUpdateJobs()) + viewed() + read() + const timer = setInterval(read, 5000) + window.addEventListener("proxmenux-update-jobs-viewed", viewed) + return () => { + closed = true + clearInterval(timer) + window.removeEventListener("proxmenux-update-jobs-viewed", viewed) + } + }, []) + const visible = jobs.filter((job) => job.id !== activeRunId + && (job.state !== "complete" || !seen.includes(job.id))) + if (!visible.length) return null + return
+ {visible.map((job) =>
+

{t(job.state === "complete" + ? "vmLxc.updates.backgroundFinished" : "vmLxc.updates.backgroundRunning", + { vmid: job.vmid, name: job.ct_name || `CT-${job.vmid}` })}

+ +
)} +
+} diff --git a/AppImage/components/script-terminal-modal.tsx b/AppImage/components/script-terminal-modal.tsx index db33d093..c1703c33 100644 --- a/AppImage/components/script-terminal-modal.tsx +++ b/AppImage/components/script-terminal-modal.tsx @@ -54,6 +54,8 @@ interface ScriptTerminalModalProps { completedSuccessfullyMessage?: string completedWithErrorMessage?: (exitCode: number) => string cancelledExitCode?: number + // Only LXC Updates opts into a separate server-owned job transport. + backgroundUpdate?: { runId: string; vmid: number; resume: boolean } // Optional callback fired when the script's WebSocket closes // (script_runner sends an exit code and then closes). Lets the // parent auto-dismiss the modal — used by host-backup's Restore @@ -72,6 +74,7 @@ export function ScriptTerminalModal({ completedSuccessfullyMessage, completedWithErrorMessage, cancelledExitCode, + backgroundUpdate, onComplete, }: ScriptTerminalModalProps) { const t = useT() @@ -81,6 +84,7 @@ export function ScriptTerminalModal({ // after dynamic imports resolve — captures the latest value without // re-binding the closure. const isOpenRef = useRef(false) + const openGenerationRef = useRef(0) const fitAddonRef = useRef(null) const sessionIdRef = useRef(Math.random().toString(36).substring(2, 8)) @@ -93,6 +97,7 @@ export function ScriptTerminalModal({ const reconnectAttemptsRef = useRef(0) const keepAliveIntervalRef = useRef(null) const completionReceivedRef = useRef(false) + const outputOffsetRef = useRef(0) const [isMobile, setIsMobile] = useState(false) const [isTablet, setIsTablet] = useState(false) @@ -130,7 +135,7 @@ export function ScriptTerminalModal({ }, [onComplete]) const attemptReconnect = useCallback(() => { - if (!isOpen || isComplete || reconnectAttemptsRef.current >= 3) { + if (!isOpenRef.current || isComplete || reconnectAttemptsRef.current >= 3) { return } @@ -154,7 +159,10 @@ export function ScriptTerminalModal({ const wsUrl = getScriptWebSocketUrl(sessionIdRef.current) // Single-use auth ticket appended as ?ticket=... — see lib/terminal-ws.ts. - const ws = new WebSocket(await getTicketedWsUrl(wsUrl)) + const generation = openGenerationRef.current + const ticketedUrl = await getTicketedWsUrl(wsUrl) + if (!isOpenRef.current || !termRef.current || generation !== openGenerationRef.current) return + const ws = new WebSocket(ticketedUrl) wsRef.current = ws ws.onopen = () => { @@ -170,9 +178,11 @@ export function ScriptTerminalModal({ } }, 30000) -const initMessage = { + const initMessage = { script_path: scriptPath, params: paramsRef.current, + ...(backgroundUpdate ? { resume: true, vmid: backgroundUpdate.vmid, + offset: outputOffsetRef.current } : {}), } ws.send(JSON.stringify(initMessage)) @@ -195,7 +205,7 @@ const initMessage = { // The PTY worker always emits this final line. Treat it as a // completion fallback because some WebSocket servers tear down the // connection before the following structured message is flushed. - const exitMatch = typeof event.data === "string" + const exitMatch = !backgroundUpdate && typeof event.data === "string" ? event.data.match(/\[Script exited with code (-?\d+)\]/) : null if (exitMatch) { @@ -211,6 +221,31 @@ const initMessage = { try { const msg = JSON.parse(event.data) + if (msg.type === "job_output") { + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + if (msg.truncated) termRef.current?.writeln(t("vmLxc.updates.backgroundOutputTrimmed")) + termRef.current?.write(msg.text) + outputOffsetRef.current = msg.offset + return + } + if (msg.type === "job_unavailable") { + setCurrentInteraction(null) + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + termRef.current?.writeln(`\x1b[31m${msg.message}\x1b[0m`) + completionReceivedRef.current = true + setIsComplete(true) + ws.close() + return + } + if (backgroundUpdate && msg.type === "web_interaction" && msg.interaction === null) { + setCurrentInteraction(null) + setInteractionInput("") + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + return + } if (msg.type === "web_interaction" && msg.interaction) { setIsWaitingNextInteraction(false) if (waitingTimeoutRef.current) { @@ -231,6 +266,11 @@ const initMessage = { return } if (msg.type === "script_complete") { + if (backgroundUpdate) { + setCurrentInteraction(null) + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + } const exitCode = Number(msg.exit_code ?? 1) completionReceivedRef.current = true setIsComplete(true) @@ -255,6 +295,7 @@ const initMessage = { } ws.onclose = (event) => { + if (!isOpenRef.current) return setConnectionStatus("offline") if (keepAliveIntervalRef.current) { clearInterval(keepAliveIntervalRef.current) @@ -266,13 +307,17 @@ const initMessage = { if (!isComplete && reconnectAttemptsRef.current < 3) { reconnectTimeoutRef.current = setTimeout(attemptReconnect, 2000) } else { - setIsComplete(true) - onCompleteRef.current?.(-1) + if (backgroundUpdate) { + termRef.current?.writeln(t("vmLxc.updates.backgroundDetached")) + } else { + setIsComplete(true) + onCompleteRef.current?.(-1) + } } } } }, 1000) - }, [isOpen, isComplete, scriptPath, getCompletionMessage, cancelledExitCode, t]) + }, [isOpen, isComplete, scriptPath, getCompletionMessage, cancelledExitCode, t, backgroundUpdate]) const sendKey = useCallback((key: string) => { if (!termRef.current) return @@ -300,12 +345,13 @@ const initMessage = { // otherwise we attach a Terminal to a stale ref and open a WS that // nobody reads. const wasOpenAtCall = isOpenRef.current + const generation = ++openGenerationRef.current const [TerminalClass, FitAddonClass] = await Promise.all([ import("xterm").then((mod) => mod.Terminal), import("xterm-addon-fit").then((mod) => mod.FitAddon), import("xterm/css/xterm.css"), ]) - if (!wasOpenAtCall || !isOpenRef.current) return + if (!wasOpenAtCall || !isOpenRef.current || generation !== openGenerationRef.current) return const fontSize = window.innerWidth < 768 ? 12 : 16 @@ -366,7 +412,9 @@ const initMessage = { const wsUrl = getScriptWebSocketUrl(sessionIdRef.current) // Single-use auth ticket appended as ?ticket=... — see lib/terminal-ws.ts. - const ws = new WebSocket(await getTicketedWsUrl(wsUrl)) + const ticketedUrl = await getTicketedWsUrl(wsUrl) + if (!isOpenRef.current || termRef.current !== term) return + const ws = new WebSocket(ticketedUrl) wsRef.current = ws ws.onopen = () => { @@ -384,6 +432,8 @@ const initMessage = { const initMessage = { script_path: scriptPath, params: paramsRef.current, + ...(backgroundUpdate ? { resume: backgroundUpdate.resume, + vmid: backgroundUpdate.vmid, offset: outputOffsetRef.current } : {}), } ws.send(JSON.stringify(initMessage)) @@ -412,7 +462,7 @@ const initMessage = { // See the reconnect handler above. The exit line is guaranteed to be // sent with the PTY output and is therefore a robust fallback when a // final JSON frame is lost during server-side socket teardown. - const exitMatch = typeof event.data === "string" + const exitMatch = !backgroundUpdate && typeof event.data === "string" ? event.data.match(/\[Script exited with code (-?\d+)\]/) : null if (exitMatch) { @@ -428,7 +478,32 @@ const initMessage = { try { const msg = JSON.parse(event.data) + if (msg.type === "job_output") { + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + if (msg.truncated) term.writeln(t("vmLxc.updates.backgroundOutputTrimmed")) + term.write(msg.text) + outputOffsetRef.current = msg.offset + return + } + if (msg.type === "job_unavailable") { + setCurrentInteraction(null) + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + term.writeln(`\x1b[31m${msg.message}\x1b[0m`) + completionReceivedRef.current = true + setIsComplete(true) + ws.close() + return + } + if (backgroundUpdate && msg.type === "web_interaction" && msg.interaction === null) { + setCurrentInteraction(null) + setInteractionInput("") + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + return + } if (msg.type === "web_interaction" && msg.interaction) { setIsWaitingNextInteraction(false) if (waitingTimeoutRef.current) { @@ -450,6 +525,11 @@ const initMessage = { return } if (msg.type === "script_complete") { + if (backgroundUpdate) { + setCurrentInteraction(null) + setIsWaitingNextInteraction(false) + if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) + } const exitCode = Number(msg.exit_code ?? 1) completionReceivedRef.current = true setIsComplete(true) @@ -478,6 +558,7 @@ const initMessage = { } ws.onclose = (event) => { + if (!isOpenRef.current) return setConnectionStatus("offline") if (!completionReceivedRef.current) { term.writeln(`\x1b[33m${t("scriptTerminal.connectionClosed")}\x1b[0m`) @@ -489,8 +570,13 @@ const initMessage = { } if (!completionReceivedRef.current && !isComplete) { - setIsComplete(true) - onCompleteRef.current?.(-1) + if (backgroundUpdate) { + term.writeln(t("vmLxc.updates.backgroundDetached")) + attemptReconnect() + } else { + setIsComplete(true) + onCompleteRef.current?.(-1) + } } } @@ -552,6 +638,7 @@ const initMessage = { if (isOpen) { initializeTerminal() } else { + openGenerationRef.current++ if (checkConnectionInterval.current) { clearInterval(checkConnectionInterval.current) } @@ -578,6 +665,7 @@ const initMessage = { sessionIdRef.current = Math.random().toString(36).substring(2, 8) reconnectAttemptsRef.current = 0 completionReceivedRef.current = false + outputOffsetRef.current = 0 setIsComplete(false) setInteractionInput("") setCurrentInteraction(null) @@ -586,6 +674,28 @@ const initMessage = { } }, [isOpen]) + useEffect(() => () => { + if (!backgroundUpdate) return // Transport mode is fixed for this instance. + // Conditional parents can unmount an open modal. Detach that viewer too; + // only the background job owns its process and completion result. + isOpenRef.current = false + openGenerationRef.current++ + for (const ref of [checkConnectionInterval, keepAliveIntervalRef]) { + if (ref.current) clearInterval(ref.current) + } + for (const ref of [waitingTimeoutRef, reconnectTimeoutRef]) { + if (ref.current) clearTimeout(ref.current) + } + if (wsRef.current) { + wsRef.current.onclose = null + wsRef.current.onmessage = null + wsRef.current.close() + wsRef.current = null + } + termRef.current?.dispose() + termRef.current = null + }, []) + useEffect(() => { const updateDeviceType = () => { const width = window.innerWidth @@ -641,8 +751,11 @@ const initMessage = { }, [isOpen, isComplete, attemptReconnect]) const getScriptWebSocketUrl = (sid: string): string => { + const route = backgroundUpdate + ? `/ws/lxc-update/${encodeURIComponent(backgroundUpdate.runId)}` + : `/ws/script/${sid}` if (typeof window === "undefined") { - return `ws://localhost:${API_PORT}/ws/script/${sid}` + return `ws://localhost:${API_PORT}${route}` } const { protocol, hostname, port } = window.location @@ -650,9 +763,9 @@ const initMessage = { const wsProtocol = protocol === "https:" ? "wss:" : "ws:" if (isStandardPort) { - return `${wsProtocol}//${hostname}/ws/script/${sid}` + return `${wsProtocol}//${hostname}${route}` } else { - return `${wsProtocol}//${hostname}:${API_PORT}/ws/script/${sid}` + return `${wsProtocol}//${hostname}:${API_PORT}${route}` } } @@ -664,6 +777,10 @@ const initMessage = { if (value === "cancel" || value === "") { setCurrentInteraction(null) setInteractionInput("") + if (backgroundUpdate) { + wsRef.current.send(JSON.stringify({ type: "cancel" })) + return + } handleCloseModal() return } diff --git a/AppImage/components/virtual-machines.tsx b/AppImage/components/virtual-machines.tsx index 966b05d4..18654320 100644 --- a/AppImage/components/virtual-machines.tsx +++ b/AppImage/components/virtual-machines.tsx @@ -22,6 +22,8 @@ import useSWR from "swr" import { MetricsView } from "./metrics-dialog" import { LxcTerminalModal } from "./lxc-terminal-modal" import { ScriptTerminalModal } from "./script-terminal-modal" +import { LxcUpdateJobs } from "./lxc-update-jobs" +import { markUpdateJobViewed, type LxcUpdateJob } from "../lib/lxc-update-jobs" import { OCI_CANCELLED_EXIT_CODE, shouldRefreshOciAfterCompletion } from "@/lib/script-completion" import { LxcAppPanel, ThemeAwareLogo, ociImageLabel } from "./lxc-app-panel" import { GuestBackupList } from "./guest-backup-list" @@ -1998,6 +2000,7 @@ export function VirtualMachines() { const [applyRestart, setApplyRestart] = useState(false) const [applyStartedAt, setApplyStartedAt] = useState(0) const [applyRunId, setApplyRunId] = useState("") + const [applyResume, setApplyResume] = useState(false) const [applyCheckedOs, setApplyCheckedOs] = useState(false) const [applyTargetIds, setApplyTargetIds] = useState([]) const [applyTargetLabels, setApplyTargetLabels] = useState([]) @@ -2639,6 +2642,7 @@ export function VirtualMachines() { setApplyVmid(vmid) setApplyTarget(target) setApplyRunId(runId) + setApplyResume(false) setApplyCheckedOs(opts?.checkedOs === true) setApplyTargetIds(opts?.targetIds?.length ? opts.targetIds : fallbackTargets) setApplyTargetLabels(opts?.targetLabels?.length ? opts.targetLabels : fallbackLabels) @@ -2659,27 +2663,37 @@ export function VirtualMachines() { setApplyStartedAt(Date.now()) setApplyOpen(true) } - const handleApplyComplete = async (exitCode = 0) => { + const resumeApplyJob = (job: LxcUpdateJob) => { + setApplyVmid(job.vmid) + setApplyTarget(job.target) + setApplyRunId(job.id) + setApplyCheckedOs(job.script_path.endsWith("/install_checked_os_updates.sh")) + setApplyResume(true) + setApplyStartedAt(job.created_at * 1000) + setApplyOpen(true) + } + const handleApplyComplete = async (exitCode = 0, notifyBackend = true) => { if (applyVmid == null) return const duration = Math.max(0, Math.round((Date.now() - applyStartedAt) / 1000)) - // The modal fires onComplete on any WS close (success or user cancel). - // Report the same run ID so the backend can collapse this compatibility - // callback with its authoritative process-completion hook. + // Legacy callers may report the same run ID to the compatibility route. + // Background Updates only refresh the view after the server's real exit. try { - await fetchApi(`/api/lxc-updates/${applyVmid}/applied`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - success: exitCode === 0, - run_id: applyRunId, - target: applyTarget, - requested_targets: applyTargetIds, - target_labels: applyTargetLabels, - duration_seconds: duration, - ct_name: selectedVM?.name || `CT-${applyVmid}`, - refresh_docker_inventory: applyDockerRefresh, - }), - }) + if (notifyBackend) { + await fetchApi(`/api/lxc-updates/${applyVmid}/applied`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + success: exitCode === 0, + run_id: applyRunId, + target: applyTarget, + requested_targets: applyTargetIds, + target_labels: applyTargetLabels, + duration_seconds: duration, + ct_name: selectedVM?.name || `CT-${applyVmid}`, + refresh_docker_inventory: applyDockerRefresh, + }), + }) + } } catch { // Non-fatal — the backend process hook owns finalization. } @@ -2697,6 +2711,12 @@ export function VirtualMachines() { // performs the requests (the data map is only a render seed), so no // invalidation is needed and there is no "Loading applications" flash. void fetchLxcApps(applyVmid) + // Recovery can be opened from the job banner while another CT is selected. + // Refresh the operated guest without changing that other guest's result UI. + if (selectedVMRef.current?.vmid !== applyVmid) { + void mutate() + return + } // Enter the "Comprobando resultado…" state on the Updates tab. // Baseline snapshot lets a downstream useEffect detect when the // SWR poll actually delivers the post-apply counts (as opposed @@ -7960,6 +7980,8 @@ export function VirtualMachines() { /> )} + + {/* LXC Update Apply Terminal — streams the apply_updates.sh script over WS/PTY (same wiring as the hardware installers). The backend finalizes on process exit; the close callback only @@ -7971,7 +7993,13 @@ export function VirtualMachines() { setApplyOpen(false) setApplyVmid(null) }} - onComplete={handleApplyComplete} + onComplete={(exitCode) => { + markUpdateJobViewed(applyRunId) + // The server-owned job is the sole completion authority. + void handleApplyComplete(exitCode ?? 1, false) + }} + backgroundUpdate={{ runId: applyRunId, vmid: applyVmid, resume: applyResume }} + cancelledExitCode={130} scriptPath={applyCheckedOs ? "/usr/local/share/proxmenux/scripts/lxc/install_checked_os_updates.sh" : "/usr/local/share/proxmenux/scripts/lxc/apply_updates.sh"} @@ -7981,13 +8009,7 @@ export function VirtualMachines() { completedWithErrorMessage={(exitCode) => t("vmLxc.updates.terminalCompletedWithError", { code: exitCode }) } - description={ - applyTarget === "os" - ? t("vmLxc.updates.terminalDescriptionOs") - : applyTarget === "app" - ? t("vmLxc.updates.terminalDescriptionApp") - : t("vmLxc.updates.terminalDescriptionBoth") - } + description={t("vmLxc.updates.backgroundDescription")} params={{ VMID: String(applyVmid), RUN_ID: applyRunId, diff --git a/AppImage/lib/lxc-update-jobs.ts b/AppImage/lib/lxc-update-jobs.ts new file mode 100644 index 00000000..6ad72d11 --- /dev/null +++ b/AppImage/lib/lxc-update-jobs.ts @@ -0,0 +1,31 @@ +/** Opaque viewer metadata only; updater commands and credentials stay server-side. */ +export type LxcUpdateJob = { + id: string + vmid: number + state: "running" | "finalizing" | "complete" + target: "os" | "app" | "both" + ct_name: string + script_path: string + created_at: number + exit_code: number | null +} + +const SEEN = "proxmenux-viewed-update-jobs" + +export function viewedUpdateJobs(): string[] { + try { + const value = JSON.parse(localStorage.getItem(SEEN) || "[]") + return Array.isArray(value) ? value.filter((id) => typeof id === "string").slice(-128) : [] + } catch { + return [] + } +} + +export function markUpdateJobViewed(id: string) { + try { + localStorage.setItem(SEEN, JSON.stringify([...new Set([...viewedUpdateJobs(), id])].slice(-128))) + window.dispatchEvent(new Event("proxmenux-update-jobs-viewed")) + } catch { + // Unavailable browser storage never controls or restarts server work. + } +} diff --git a/AppImage/messages/en/common.json b/AppImage/messages/en/common.json index aa97132a..bf230484 100644 --- a/AppImage/messages/en/common.json +++ b/AppImage/messages/en/common.json @@ -1551,6 +1551,12 @@ "installedVersionUnverified": "Installed version is unverified. Update availability could not be confirmed.", "versionStatusUnknown": "Update availability is unverified. Version information is unavailable or incomplete.", "terminalTitle": "Apply updates — CT {vmid}", + "backgroundRunning": "{name} (CT {vmid}): an update is running on the server.", + "backgroundOutputTrimmed": "Earlier output was trimmed; the update continues normally.", + "backgroundFinished": "{name} (CT {vmid}): an update has finished. View its result.", + "viewBackgroundJob": "View update", + "backgroundDetached": "The viewer disconnected. The update continues on the server; reconnect after logging in.", + "backgroundDescription": "Closing this window leaves the update running. Use Ctrl+C to cancel it.", "terminalDescriptionOs": "Applying OS package updates inside the container...", "terminalDescriptionApp": "Running the application updater inside the container...", "terminalDescriptionBoth": "Applying OS package and application updates inside the container...", diff --git a/AppImage/messages/sk/common.json b/AppImage/messages/sk/common.json index 2d9e219e..011ace36 100644 --- a/AppImage/messages/sk/common.json +++ b/AppImage/messages/sk/common.json @@ -1425,6 +1425,12 @@ "installedVersionUnverified": "Nainštalovaná verzia nie je overená. Dostupnosť aktualizácie sa nepodarilo potvrdiť.", "versionStatusUnknown": "Dostupnosť aktualizácie nie je overená. Údaje o verzii chýbajú alebo sú neúplné.", "terminalTitle": "Aktualizácia — CT {vmid}", + "backgroundRunning": "{name} (CT {vmid}): na serveri prebieha aktualizácia.", + "backgroundOutputTrimmed": "Starší výpis bol skrátený; aktualizácia pokračuje normálne.", + "backgroundFinished": "{name} (CT {vmid}): aktualizácia sa skončila. Pozrite si výsledok.", + "viewBackgroundJob": "Zobraziť aktualizáciu", + "backgroundDetached": "Spojenie s terminálom sa prerušilo. Aktualizácia na serveri pokračuje; po prihlásení sa znova pripojte.", + "backgroundDescription": "Zatvorením okna sa aktualizácia nezastaví. Zrušiť ju môžete cez Ctrl+C.", "terminalDescriptionOs": "Aktualizujem balíky systému v kontajneri...", "terminalDescriptionApp": "Spúšťam aktualizáciu aplikácie v kontajneri...", "terminalDescriptionBoth": "Aktualizujem systém aj aplikácie v kontajneri...", diff --git a/AppImage/scripts/build_appimage.sh b/AppImage/scripts/build_appimage.sh index 86eeb956..e578711b 100755 --- a/AppImage/scripts/build_appimage.sh +++ b/AppImage/scripts/build_appimage.sh @@ -148,6 +148,7 @@ cp "$SCRIPT_DIR/health_thresholds.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo " cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ managed_installs.py not found" cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/" +cp "$SCRIPT_DIR/lxc_update_jobs.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/local_app_icons.py" "$APP_DIR/usr/bin/" diff --git a/AppImage/scripts/flask_terminal_routes.py b/AppImage/scripts/flask_terminal_routes.py index 41c49721..45d38270 100644 --- a/AppImage/scripts/flask_terminal_routes.py +++ b/AppImage/scripts/flask_terminal_routes.py @@ -134,6 +134,72 @@ def _run_script_completion_hook(script_path, params, exit_code, duration_seconds except Exception as exc: print(f"[ProxMenux] script completion hook failed: {exc}", flush=True) + +@terminal_bp.route('/api/lxc-update-jobs', methods=['GET']) +@require_admin_scope +def lxc_update_jobs_status(): + """Read-only recovery after login. This never creates or restarts a job.""" + from lxc_update_jobs import list_jobs + return jsonify({'jobs': list_jobs()}) + + +@sock.route('/ws/lxc-update/') +def lxc_update_viewer(ws, run_id): + """An authenticated viewer, deliberately not the owner of the update.""" + if not _ws_auth_check(): + ws.close(reason=1008, message="Authentication required") + return + from lxc_update_jobs import get, start + try: + init = json.loads(ws.receive(timeout=10)) + if not isinstance(init, dict) or type(init.get('resume')) is not bool: + raise ValueError("Invalid update job request") + offset = init.get('offset', 0) + if type(offset) is not int or offset < 0: + raise ValueError("Invalid update output cursor") + if init['resume']: + job = get(run_id, init.get('vmid')) + else: + job = start(run_id, init.get('script_path'), init.get('params'), + _run_script_completion_hook) + revision = -1 + while True: + view = job.read(offset) + if view['text'] or view['offset'] != offset: + ws.send(json.dumps({'type': 'job_output', 'text': view['text'], + 'offset': view['offset'], 'truncated': view['truncated']})) + offset = view['offset'] + if view['revision'] != revision: + revision = view['revision'] + # A second viewer must also dismiss a question already + # answered elsewhere, or cleared when the job ends. + ws.send(json.dumps({'type': 'web_interaction', + 'interaction': view['interaction']})) + if view['state'] == 'complete': + ws.send(json.dumps({'type': 'script_complete', 'exit_code': view['exit_code']})) + break + data = ws.receive(timeout=0.2) + if data is None: + continue + try: + message = json.loads(data) + except (TypeError, ValueError): + message = data + if isinstance(message, dict) and message.get('type') == 'ping': + ws.send('{"type":"pong"}') + else: + job.input(message if isinstance(message, dict) else data) + except (ValueError, KeyError, OSError, TypeError) as exc: + try: + ws.send(json.dumps({'type': 'job_unavailable', 'message': str(exc)})) + except Exception: + pass + except Exception: + # Viewer loss, including auth expiry/reload, has no job side effects. + pass + # No process termination, PTY close or completion callback here. The job's + # worker alone owns those, even while no viewer is authenticated/connected. + @terminal_bp.route('/api/terminal/health', methods=['GET']) def terminal_health(): """Health check for terminal service""" diff --git a/AppImage/scripts/lxc_update_jobs.py b/AppImage/scripts/lxc_update_jobs.py new file mode 100644 index 00000000..8664559e --- /dev/null +++ b/AppImage/scripts/lxc_update_jobs.py @@ -0,0 +1,375 @@ +"""Browser-independent owners for explicitly confirmed LXC Updates runs. + +Only the two Updates entrypoints are allowed. Import/list/attach never runs +anything. Interactive /ws/script consumers retain their original lifecycle. +Jobs survive viewer loss, not a Monitor/host restart; missing resume IDs fail +closed. The update shell still owns the shared scheduler/control/check flock. +""" +import codecs +import errno +import json +import os +import pty +import re +import select +import signal +import struct +import subprocess +import tempfile +import termios +import fcntl +import threading +import time + +SCRIPTS = { + "/usr/local/share/proxmenux/scripts/lxc/apply_updates.sh", + "/usr/local/share/proxmenux/scripts/lxc/install_checked_os_updates.sh", +} +PARAMS = { + "VMID", "RUN_ID", "CT_NAME", "TARGET", "REQUESTED_TARGETS_JSON", + "TARGET_LABELS_JSON", "BACKUP", "BACKUP_STORAGE", "RESTART", + "UPDATE_COMMAND", "APP_NAME", "RUN_HELPER", "ALLOW_HELPER_WITH_CUSTOM", + "DOCKER_STANDALONE_TARGETS", "UPDATE_DOCKER_ENGINE", + "REFRESH_DOCKER_INVENTORY", +} +RUN_ID = re.compile(r"^[A-Za-z0-9_-]{1,128}$") +INTERACTION_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$") +MAX_RUNTIME = 4 * 3600 +# The shared runner restores a guest it started in its EXIT trap. Its package +# observation and `pct shutdown --timeout 60` need time after SIGTERM, too. +STOP_GRACE = 300 +RESULT_TTL = 6 * 3600 +MAX_OUTPUT = 256 * 1024 +MAX_INPUT = 64 * 1024 +MAX_JOBS = 32 +_lock = threading.RLock() +_jobs = {} + + +class JobConflict(ValueError): + pass + + +def _validate(run_id, script, params): + if not isinstance(run_id, str) or not RUN_ID.fullmatch(run_id): + raise ValueError("Invalid update job ID") + if script not in SCRIPTS or os.path.realpath(script) != script: + raise ValueError("Only the LXC Updates entrypoints support background runs") + if not isinstance(params, dict) or set(params) - PARAMS: + raise ValueError("Invalid update parameters") + if any(not isinstance(v, str) or len(v) > 65536 for v in params.values()): + raise ValueError("Invalid update parameter value") + if len(json.dumps(params)) > 128 * 1024: + raise ValueError("Update parameters are too large") + if params.get("RUN_ID") != run_id: + raise ValueError("Update run ID does not match") + vmid = params.get("VMID", "") + if not re.fullmatch(r"[1-9][0-9]{0,8}", vmid): + raise ValueError("Invalid container ID") + if params.get("TARGET") not in ("os", "app", "both"): + raise ValueError("Invalid update target") + return int(vmid) + + +class UpdateJob: + def __init__(self, run_id, script, params, callback): + self.id, self.script, self.params = run_id, script, dict(params) + self.vmid = int(params["VMID"]) + self.created_at = time.time() + self.started = time.monotonic() + self.finished_at = None + self.exit_code = None + self.state = "running" + self.output = "" + self.offset = 0 + self.interaction = None + self.revision = 0 + self.lock = threading.RLock() + self.callback = callback + self.stop_reason = None + self.stop_at = None + self.master = None + self.process = None + self.log_path = None + self._interaction_buffer = "" + self._input = bytearray() + + def launch(self): + master, slave = pty.openpty() + self.master = master + try: + # A viewer's paste must never block the worker's output, timeout + # or cancellation when the updater does not read its stdin. + os.set_blocking(master, False) + fd, self.log_path = tempfile.mkstemp(prefix="proxmenux_update_", suffix=".log") + os.close(fd) + fcntl.ioctl(master, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 120, 0, 0)) + env = dict(os.environ, **self.params, EXECUTION_MODE="web", + WEB_LOG=self.log_path, TERM="xterm-256color", PYTHONUNBUFFERED="1") + self.process = subprocess.Popen( + ["/bin/bash", self.script], stdin=slave, stdout=slave, stderr=slave, + start_new_session=True, env=env, + ) + except BaseException: + os.close(master) + self.master = None + if self.log_path: + os.unlink(self.log_path) + raise + finally: + os.close(slave) + try: + threading.Thread(target=self._run, daemon=True, + name=f"lxc-update-{self.id}").start() + except BaseException: + # A failed worker start must not leave an untracked updater alive. + try: + os.killpg(self.process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.process.wait() + os.close(master) + self.master = None + os.unlink(self.log_path) + raise + + def _append(self, text): + with self.lock: + self.output = (self.output + text)[-MAX_OUTPUT:] + self.offset += len(text) + + def _interactions(self, stream): + self._interaction_buffer += stream.read(16384) + if len(self._interaction_buffer) > 65536: + self._interaction_buffer = "" + return + lines = self._interaction_buffer.split("\n") + self._interaction_buffer = lines.pop() + for line in lines: + if not line.startswith("WEB_INTERACTION:"): + continue + try: + import base64 + parts = line.rstrip("\n")[16:].split(":", 4) + kind, identity, title, message = parts[:4] + if not INTERACTION_ID.fullmatch(identity): + continue + value = dict(type=kind, id=identity, + title=base64.b64decode(title).decode(), + message=base64.b64decode(message).decode()) + if len(parts) > 4: + if kind == "menu": + value["options"] = json.loads(parts[4]) + elif kind in ("input", "inputbox"): + value["default"] = base64.b64decode(parts[4]).decode() + with self.lock: + if not self.stop_reason: + self.interaction = value + self.revision += 1 + except (ValueError, UnicodeError, IndexError): + continue + + def _signal(self, reason): + with self.lock: + if self.process.poll() is not None or self.stop_reason: + return + self.stop_reason, self.stop_at = reason, time.monotonic() + self._input.clear() + self.interaction = None + self.revision += 1 + try: + os.killpg(self.process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + def cancel(self): + self._signal("cancelled") + + def _write_input(self): + # The descriptor is nonblocking, so even a full PTY cannot hold this + # lock. Retain unwritten bytes in order until the child can read them. + with self.lock: + if not self._input or self.stop_reason: + return + try: + written = os.write(self.master, self._input[:16384]) + except BlockingIOError: + return + except OSError as exc: + if exc.errno != errno.EIO: + raise + # The child can close its terminal and keep working. Losing + # stdin must not turn that update into a viewer failure. + self._input.clear() + return + del self._input[:written] + + def _run(self): + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + eof, exited_at = False, None + try: + with open(self.log_path, encoding="utf-8", errors="replace") as interactions: + while True: + now = time.monotonic() + running = self.process.poll() is None + if not running: + exited_at = exited_at or now + if eof or now - exited_at >= 1: + break + if time.monotonic() - self.started >= MAX_RUNTIME: + self._signal("timeout") + if running and self.stop_at and now - self.stop_at >= STOP_GRACE: + try: + os.killpg(self.process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self._interactions(interactions) + with self.lock: + pending_input = bool(self._input) and running and not self.stop_reason + readable, writable, _ = select.select( + [] if eof else [self.master], + [self.master] if pending_input else [], [], 0.05) + if writable: + self._write_input() + if readable: + try: + data = os.read(self.master, 16384) + except BlockingIOError: + continue + except OSError as exc: + if exc.errno != errno.EIO: + raise + data = b"" + if data: + self._append(decoder.decode(data)) + continue + eof = True + code = self.process.wait() + self._append(decoder.decode(b"", final=True)) + if self.stop_reason: + code = 124 if self.stop_reason == "timeout" else 130 + except BaseException as exc: + self._append(f"\r\nUpdate viewer failed: {type(exc).__name__}\r\n") + self._signal("viewer failure") + try: + self.process.wait(timeout=STOP_GRACE) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait() + code = 1 + finally: + os.close(self.master) + self.master = None + try: + os.unlink(self.log_path) + except FileNotFoundError: + pass + with self.lock: + self.exit_code = code + self.state = "finalizing" + self.interaction = None + self._input.clear() + self.revision += 1 + try: + self.callback(self.script, self.params, code, time.monotonic() - self.started) + finally: + with self.lock: + self.state = "complete" + self.finished_at = time.time() + self.revision += 1 + + def view(self): + with self.lock: + return dict(id=self.id, vmid=self.vmid, state=self.state, + target=self.params["TARGET"], ct_name=self.params.get("CT_NAME", ""), + created_at=self.created_at, finished_at=self.finished_at, + exit_code=self.exit_code, script_path=self.script) + + def read(self, offset): + with self.lock: + begin = self.offset - len(self.output) + cursor = max(begin, min(offset, self.offset)) + return dict(text=self.output[cursor - begin:], offset=self.offset, + truncated=offset < begin, interaction=self.interaction, + revision=self.revision, state=self.state, exit_code=self.exit_code) + + def input(self, message): + with self.lock: + if self.state != "running" or self.master is None: + return + if isinstance(message, str): + if len(message) > 16384: + raise ValueError("Terminal input is too large") + if message == "\x03": + self.cancel() + else: + data = message.encode() + if len(self._input) + len(data) > MAX_INPUT: + raise ValueError("Terminal input buffer is full") + if not self.stop_reason: + self._input.extend(data) + elif message.get("type") == "cancel": + self.cancel() + elif message.get("type") == "resize": + cols = max(20, min(400, int(message.get("cols", 120)))) + rows = max(5, min(200, int(message.get("rows", 30)))) + fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0)) + elif message.get("type") == "interaction_response": + identity, value = message.get("id"), message.get("value") + if (not self.interaction or identity != self.interaction["id"] + or not isinstance(value, str) or len(value) > 16384): + raise ValueError("No matching pending interaction") + path = f"/tmp/proxmenux_response_{identity}" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as response: + response.write(value) + self.interaction = None + self.revision += 1 + + +def start(run_id, script, params, callback): + vmid = _validate(run_id, script, params) + with _lock: + existing = _jobs.get(run_id) + if existing: + if existing.script != script or existing.params != params: + raise JobConflict("Update job ID already belongs to another request") + return existing + _prune() + if any(j.vmid == vmid and j.state != "complete" for j in _jobs.values()): + raise JobConflict("An update is already running for this container") + if len(_jobs) >= MAX_JOBS: + completed = [j for j in _jobs.values() if j.state == "complete"] + if not completed: + raise JobConflict("Too many active update jobs; try again later") + oldest = min(completed, key=lambda j: j.finished_at) + del _jobs[oldest.id] + job = UpdateJob(run_id, script, params, callback) + _jobs[run_id] = job + try: + job.launch() + except BaseException: + _jobs.pop(run_id, None) + raise + return job + + +def _prune(): + now = time.time() + for identity, job in list(_jobs.items()): + if job.finished_at and now - job.finished_at > RESULT_TTL: + del _jobs[identity] + + +def get(run_id, vmid): + with _lock: + job = _jobs.get(run_id) + if job is None or job.vmid != int(vmid): + raise KeyError("Update job is unavailable; it was not restarted") + return job + + +def list_jobs(): + with _lock: + _prune() + return [j.view() for j in sorted(_jobs.values(), key=lambda j: j.created_at, reverse=True)] diff --git a/tests/lxc_updates/test_observed_state_ui.cjs b/tests/lxc_updates/test_observed_state_ui.cjs index 4f2de690..8df5c18b 100644 --- a/tests/lxc_updates/test_observed_state_ui.cjs +++ b/tests/lxc_updates/test_observed_state_ui.cjs @@ -138,7 +138,7 @@ const context = { invalidateLxcApps: () => {}, setSelectedVM: value => { context.selectionChanged = value }, alert: value => { context.alerted = value }, } -for (const name of ["Vmid", "Target", "RunId", "TargetIds", "TargetLabels", "UpdateCommand", "AppName", +for (const name of ["Vmid", "Target", "RunId", "Resume", "TargetIds", "TargetLabels", "UpdateCommand", "AppName", "RunHelper", "AllowHelperWithCustom", "DockerStandaloneTargets", "DockerEngine", "DockerRefresh", "BackupStorage", "StartedAt", "Open", "CheckAndUpdate", "CheckedOs"]) context[`setApply${name}`] = value => { context[`apply${name}`] = value } @@ -205,6 +205,7 @@ for (const [target, options] of [ context.applyOpen = false ui.openApply(100, target, options) assert.equal(context.applyOpen, true, `explicit ${target} update opens the terminal directly`) + assert.equal(context.applyResume, false, 'an explicitly started update is distinguished from attach-only recovery') } assert.equal(confirmationCalls, 0, 'ordinary update entry points never open a browser confirmation') assert(!source.includes('checkAndUpdate'), 'the abandoned combined check/install action has no UI path') diff --git a/tests/test_lxc_update_jobs.cjs b/tests/test_lxc_update_jobs.cjs new file mode 100644 index 00000000..d15449ac --- /dev/null +++ b/tests/test_lxc_update_jobs.cjs @@ -0,0 +1,129 @@ +// Execute the actual modal handlers and Updates recovery wiring. +const assert = require('node:assert/strict') +const fs = require('node:fs'), path = require('node:path'), vm = require('node:vm') +const {loadTypeScript} = require('./ts-fixture.cjs') +const root = path.resolve(__dirname, '../AppImage') +const ts = require(path.join(root, 'node_modules/typescript')) +const source = fs.readFileSync(path.join(root, 'components/script-terminal-modal.tsx'), 'utf8') +const tree = ts.createSourceFile('modal.tsx', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX) +const handlers = {onmessage: [], onclose: []} +let unmount +function walk(node) { + if (ts.isBinaryExpression(node) && /^ws\.(onmessage|onclose)$/.test(node.left.getText(tree))) + handlers[node.left.name.text].push(node.right.getText(tree)) + if (ts.isCallExpression(node) && node.expression.getText(tree) === 'useEffect' + && node.arguments[0]?.getText(tree).includes('Transport mode is fixed')) + unmount = node.arguments[0].getText(tree) + ts.forEachChild(node, walk) +} +walk(tree) +assert.equal(handlers.onmessage.length, 2) +assert.equal(handlers.onclose.length, 2) +const completion = loadTypeScript('lib/script-completion.ts') +function fixture(backgroundUpdate) { + const output = [], finished = [], reconnects = [] + const term = {write: s => output.push(s), writeln: s => output.push(s)} + const ws = {readyState: 1, close() { this.readyState = 3 }} + const context = { + ...completion, backgroundUpdate, term, termRef: {current: term}, ws, WebSocket: {OPEN: 1}, + completionReceivedRef: {current: false}, outputOffsetRef: {current: 0}, + isOpenRef: {current: true}, isComplete: false, cancelledExitCode: 130, + waitingTimeoutRef: {current: null}, keepAliveIntervalRef: {current: null}, + reconnectAttemptsRef: {current: 0}, reconnectTimeoutRef: {current: null}, + onCompleteRef: {current: code => finished.push(code)}, + setIsComplete: value => context.isComplete = value, setConnectionStatus() {}, + setIsWaitingNextInteraction: value => context.waiting = value, + setCurrentInteraction: value => context.interaction = value, setInteractionInput() {}, + clearTimeout() {}, clearInterval() {}, setTimeout: fn => { fn(); return 1 }, + attemptReconnect: () => reconnects.push(true), t: key => key, + getCompletionMessage: code => 'Result ' + code, + } + return {context, output, finished, reconnects, ws} +} +function compile(code, context) { + return vm.runInNewContext(ts.transpileModule('(' + code + ')', {compilerOptions: { + target: ts.ScriptTarget.ES2020, module: ts.ModuleKind.CommonJS, + }}).outputText, context) +} +for (const handler of handlers.onmessage) { + const f = fixture({runId: 'existing', vmid: 100, resume: true}) + const message = compile(handler, f.context) + message({data: JSON.stringify({type: 'job_output', text: '[Script exited with code 0]', offset: 27})}) + assert.equal(f.finished.length, 0, 'text inside a job is never its authoritative result') + assert.equal(f.context.outputOffsetRef.current, 27) + message({data: JSON.stringify({type: 'web_interaction', interaction: {id:'pending', type:'yesno'}})}) + assert.equal(f.context.interaction.id, 'pending', 'replayed interaction remains answerable') + f.context.waiting = true + message({data: JSON.stringify({type: 'web_interaction', interaction: null})}) + assert.equal(f.context.interaction, null, 'answer in another viewer dismisses the stale question') + assert.equal(f.context.waiting, false) + f.context.waiting = true + message({data: JSON.stringify({type: 'job_output', text: 'working', offset: 34})}) + assert.equal(f.context.waiting, false, 'ordinary progress clears the post-answer spinner') + f.context.interaction = {id: 'stale'}; f.context.waiting = true + message({data: JSON.stringify({type: 'script_complete', exit_code: 3})}) + assert.deepEqual(f.finished, [3]) + assert.equal(f.context.interaction, null, 'a completed job cannot retain an active question') + assert.equal(f.context.waiting, false, 'a completed job cannot retain a processing overlay') + message({data: JSON.stringify({type: 'script_complete', exit_code: 0})}) + assert.deepEqual(f.finished, [3], 'replayed completion cannot finalize twice') + const missing = fixture({runId: 'missing'}) + compile(handler, missing.context)({data: JSON.stringify({type:'job_unavailable', message:'Unavailable; not restarted'})}) + assert.equal(missing.finished.length, 0, 'missing job is not success or a synthetic failure callback') + const legacy = fixture(undefined) + compile(handler, legacy.context)({data:'[Script exited with code 0]'}) + assert.deepEqual(legacy.finished, [0], 'legacy terminal exit fallback still works') +} +for (const handler of handlers.onclose) { + const f = fixture({runId: 'existing'}) + compile(handler, f.context)({}) + assert.equal(f.finished.length, 0, 'losing a viewer never reports update failure') + assert.equal(f.context.isComplete, false) + assert.equal(f.reconnects.length, 1) + const closed = fixture({runId:'existing'}); closed.context.isOpenRef.current = false + compile(handler, closed.context)({}) + assert.equal(closed.reconnects.length, 0, 'closing the view cannot reopen a socket') +} +assert.ok(unmount) +for (const backgroundUpdate of [undefined, {runId:'existing'}]) { + const f=fixture(backgroundUpdate), closes=[], sends=[], disposals=[] + const ref={current:{close:()=>closes.push(true),send:()=>sends.push(true)}} + Object.assign(f.context,{wsRef:ref,openGenerationRef:{current:1}, + checkConnectionInterval:{current:null},reconnectTimeoutRef:{current:null}}) + f.context.termRef.current.dispose=()=>disposals.push(true) + compile(unmount,f.context)()() + assert.equal(closes.length,backgroundUpdate?1:0) + assert.equal(sends.length,0,'unmount cannot send cancellation or another start') + assert.equal(f.finished.length,0) + assert.equal(disposals.length,backgroundUpdate?1:0) +} +const vms=fs.readFileSync(path.join(root,'components/virtual-machines.tsx'),'utf8') +const vmTree=ts.createSourceFile('vms.tsx',vms,ts.ScriptTarget.Latest,true,ts.ScriptKind.TSX) +let complete +function visit(n){if(ts.isVariableDeclaration(n)&&n.name.getText(vmTree)==='handleApplyComplete')complete=n.initializer.getText(vmTree);ts.forEachChild(n,visit)} +visit(vmTree) +async function checkCompletion(){ + for(const selected of [100,200]){ + const posts=[],states=[],refresh=[] + const context={applyVmid:100,applyStartedAt:Date.now(),selectedVM:{vmid:selected},selectedVMRef:{current:{vmid:selected}}, + applyRunId:'existing',applyTarget:'app',applyTargetIds:['app:test'],applyTargetLabels:['Test'],applyDockerRefresh:false, + vmModalCacheRef:{current:{details:new Map(),schedule:new Map()}},fetchApi:async(...args)=>posts.push(args), + fetchLxcApps:id=>refresh.push(id),mutate:()=>{},setUpdatesBaselineCount:v=>states.push(v),setUpdatesResult:v=>states.push(v),setUpdatesRefreshing:v=>states.push(v)} + await compile(complete,context)(0,false) + assert.equal(posts.length,0,'background completion never sends a second finalization POST') + assert.deepEqual(refresh,[100]) + assert.equal(states.length,selected===100?3:0,'another guest result UI is preserved') + } +} +const local = new Map(), events = [] +const storage = {getItem:k=>local.get(k), setItem:(k,v)=>local.set(k,v)} +const jobs = loadTypeScript('lib/lxc-update-jobs.ts', {localStorage:storage, + window:{dispatchEvent:e=>events.push(e)}, Event:class {constructor(type){this.type=type}}}) +local.set('proxmenux-viewed-update-jobs', '{') +assert.equal(jobs.viewedUpdateJobs().length, 0) +jobs.markUpdateJobViewed('opaque-run'); jobs.markUpdateJobViewed('opaque-run') +assert.equal(jobs.viewedUpdateJobs().length, 1) +assert.equal(events.length, 2) +assert.equal([...local.values()].join(''), '["opaque-run"]', 'storage has IDs, no commands or credentials') +console.log('PASS actual initial/reconnect handlers, auth-independent recovery metadata and legacy completion') +checkCompletion().catch(e=>{console.error(e);process.exitCode=1}) diff --git a/tests/test_lxc_update_jobs.py b/tests/test_lxc_update_jobs.py new file mode 100644 index 00000000..4f523424 --- /dev/null +++ b/tests/test_lxc_update_jobs.py @@ -0,0 +1,640 @@ +"""Real inert processes and actual Flask/ticket/socket handler boundaries.""" +import ast +import base64 +import io +import importlib.util +import json +import os +import pty +import re +import select +import fcntl +import struct +import termios +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import threading +import time +from types import ModuleType, SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from flask import Blueprint, Flask, jsonify, request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "AppImage/scripts")) +import lxc_update_jobs as jobs + + +def wait_for(predicate, timeout=6): + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError("Timed out waiting for fixture") + time.sleep(0.01) + + +class JobTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.folder = Path(self.temp.name) + self.script = self.folder / "apply.sh" + self.gate = self.folder / "backup-done" + self.events = self.folder / "events" + self.script.write_text( + f"echo backup-start; echo backup >> {shlex.quote(str(self.events))}\n" + f"while [ ! -e {shlex.quote(str(self.gate))} ]; do sleep .02; done\n" + f"echo update-finished; echo update >> {shlex.quote(str(self.events))}\n" + ) + self.allowed = patch.object(jobs, "SCRIPTS", {str(self.script)}) + self.allowed.start() + self.addCleanup(self.allowed.stop) + jobs._jobs.clear() + self.callback = Mock() + self.addCleanup(self.stop_jobs) + + def stop_jobs(self): + for job in list(jobs._jobs.values()): + if job.state != "complete": + job.cancel() + wait_for(lambda: job.state == "complete") + jobs._jobs.clear() + + def params(self, run="run-1", vmid="100"): + return dict(VMID=vmid, RUN_ID=run, TARGET="app", CT_NAME="fixture", BACKUP="1") + + def start(self, run="run-1", vmid="100"): + return jobs.start(run, str(self.script), self.params(run, vmid), self.callback) + + def finish(self, job): + self.gate.touch() + wait_for(lambda: job.state == "complete") + + def test_real_backup_and_update_continue_without_any_viewer(self): + job = self.start() + wait_for(lambda: self.events.exists()) + self.assertEqual(self.events.read_text().splitlines(), ["backup"]) + self.assertEqual(job.state, "running") + self.finish(job) + self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) + self.assertEqual(job.exit_code, 0) + self.callback.assert_called_once() + self.assertEqual(self.callback.call_args.args[:3], + (str(self.script), self.params(), 0)) + self.assertIn("update-finished", job.read(0)["text"]) + + def test_same_run_reconnect_is_idempotent_and_other_ct_cannot_attach(self): + job = self.start() + self.assertIs(self.start(), job) + with self.assertRaises(KeyError): + jobs.get(job.id, 200) + with self.assertRaises(KeyError): + jobs.get("lost-run", 100) + self.finish(job) + self.assertIs(jobs.get(job.id, 100), job) + self.assertIs(self.start(), job) + self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) + self.callback.assert_called_once() + + def test_different_run_for_same_ct_rejected_before_second_spawn(self): + job = self.start() + with patch.object(jobs.subprocess, "Popen") as spawn: + with self.assertRaises(jobs.JobConflict): + self.start("run-2") + spawn.assert_not_called() + self.finish(job) + other = self.start("run-2") + wait_for(lambda: other.state == "complete") + self.assertEqual(self.events.read_text().splitlines().count("update"), 2) + + def test_concurrent_same_id_starts_once(self): + calls, errors = [], [] + barrier = threading.Barrier(6) + def attempt(): + barrier.wait() + try: + calls.append(self.start()) + except Exception as exc: + errors.append(exc) + threads = [threading.Thread(target=attempt) for _ in range(6)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + self.assertFalse(errors) + self.assertEqual(len({id(job) for job in calls}), 1) + self.finish(calls[0]) + self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) + + def test_invalid_request_has_zero_process_and_zero_tempfile_effects(self): + bad = [ + ("../bad", str(self.script), self.params()), + ("run-1", "/bin/bash", self.params()), + ("run-1", str(self.script), dict(self.params(), LD_PRELOAD="/bad")), + ("run-1", str(self.script), dict(self.params(), VMID="0")), + ("run-1", str(self.script), dict(self.params(), RUN_ID="other")), + ("run-1", str(self.script), dict(self.params(), UPDATE_COMMAND=123)), + ("run-1", str(self.script), dict(self.params(), TARGET="unexpected")), + ] + with patch.object(jobs.subprocess, "Popen") as spawn, \ + patch.object(jobs.tempfile, "mkstemp") as files: + for identity, script, params in bad: + with self.subTest(params=params), self.assertRaises(ValueError): + jobs.start(identity, script, params, self.callback) + spawn.assert_not_called() + files.assert_not_called() + self.assertFalse(jobs.list_jobs()) + + def test_request_id_cannot_be_rebound_to_another_command(self): + job = self.start() + with patch.object(jobs.subprocess, "Popen") as spawn: + with self.assertRaises(jobs.JobConflict): + jobs.start("run-1", str(self.script), + dict(self.params(), UPDATE_COMMAND="different"), self.callback) + spawn.assert_not_called() + self.finish(job) + + def test_failure_and_launch_error_are_not_reported_as_success(self): + self.script.write_text("echo failed-backup; exit 3\n") + job = self.start() + wait_for(lambda: job.state == "complete") + self.assertEqual(job.exit_code, 3) + self.assertFalse(self.events.exists()) + self.callback.assert_called_once() + with patch.object(jobs.subprocess, "Popen", side_effect=OSError("cannot execute")): + with self.assertRaises(OSError): + self.start("run-2") + self.assertNotIn("run-2", jobs._jobs) + + def test_cancel_is_explicit_and_kills_the_inert_process_group(self): + job = self.start() + wait_for(lambda: self.events.exists()) + job.input({"type": "cancel"}) + wait_for(lambda: job.state == "complete") + self.assertEqual(job.exit_code, 130) + self.assertEqual(self.events.read_text().splitlines(), ["backup"]) + self.callback.assert_called_once() + + def test_timeout_is_bounded_even_if_the_child_closes_its_terminal(self): + self.script.write_text("exec /dev/null 2>&1\nwhile :; do sleep .02; done\n") + with patch.object(jobs, "MAX_RUNTIME", .15): + job = self.start() + wait_for(lambda: job.state == "complete") + self.assertEqual(job.exit_code, 124) + self.callback.assert_called_once() + + def test_cancel_waits_for_the_runners_original_state_cleanup(self): + # apply_updates.sh's EXIT trap may need a 60s package probe followed + # by `pct shutdown --timeout 60`. A two-second kill truncates that trap. + self.script.write_text( + f"trap 'sleep 2.3; echo restored >> {shlex.quote(str(self.events))}; exit 0' TERM\n" + + self.script.read_text()) + job = self.start() + wait_for(lambda: self.events.exists()) + job.cancel() + wait_for(lambda: job.state == 'complete') + self.assertEqual(job.exit_code, 130) + self.assertEqual(self.events.read_text().splitlines(), ['backup', 'restored']) + self.assertNotIn('update', self.events.read_text()) + self.callback.assert_called_once() + self.assertGreater(jobs.STOP_GRACE, 120) + + def test_timeout_still_reaps_a_process_that_ignores_termination(self): + self.script.write_text("trap '' TERM\necho ready\nwhile :; do sleep .02; done\n") + with patch.object(jobs, 'MAX_RUNTIME', .15), patch.object(jobs, 'STOP_GRACE', .1): + job = self.start() + wait_for(lambda: job.state == 'complete') + self.assertEqual(job.exit_code, 124) + self.assertIsNotNone(job.process.poll()) + self.callback.assert_called_once() + + def test_output_is_bounded_and_cursor_replay_is_read_only(self): + job = self.start() + wait_for(lambda: self.events.exists()) + with patch.object(jobs, "MAX_OUTPUT", 32): + job._append("x" * 100) + snap = job.read(0) + self.assertEqual(len(snap["text"]), 32) + self.assertTrue(snap["truncated"]) + self.assertEqual(job.read(snap["offset"])["text"], "") + self.finish(job) + + def test_unknown_interaction_response_never_writes_a_file(self): + job = self.start() + with patch.object(jobs.os, "open") as files: + with self.assertRaises(ValueError): + job.input({"type": "interaction_response", "id": "not-pending", "value": "yes"}) + files.assert_not_called() + self.finish(job) + + def test_large_paste_cannot_block_output_cancel_or_timeout(self): + self.script.write_text("stty raw -echo\necho ready\nwhile :; do echo progress; sleep .02; done\n") + for run, cancel in [("cancel-paste", True), ("timeout-paste", False)]: + with self.subTest(run=run), patch.object(jobs, "MAX_RUNTIME", .5), \ + patch.object(jobs, "STOP_GRACE", .1): + job = self.start(run) + wait_for(lambda: "ready" in job.read(0)["text"]) + before = job.read(0)["offset"] + job.input("x" * 16384) + job.input("y" * 16384) + wait_for(lambda: job.read(0)["offset"] > before) + if cancel: + job.cancel() + wait_for(lambda: job.state == "complete") + self.assertEqual(job.exit_code, 130 if cancel else 124) + self.assertIsNotNone(job.process.poll()) + self.assertFalse(job._input) + self.assertEqual(self.callback.call_count, 2) + + def test_queued_paste_reaches_a_slow_reader_in_order_without_losing_bytes(self): + payload = ("Ž🙂abc" * 3500).encode() + received = self.folder / "received" + reader = ( + "import os,pathlib; data=bytearray(); " + f"size={len(payload)}; " + "exec('while len(data)"](ws, "run-1") + return ws + + def test_expired_login_cannot_start_and_has_zero_job_effects(self): + for token, status in [("expired", 401), ("read-only", 403)]: + with self.subTest(token=token), patch.object(jobs.subprocess, "Popen") as launch: + response = self.ticket(token) + self.assertEqual(response.status_code, status) + self.attach(dict(resume=False, script_path=str(self.script), params=self.params())) + launch.assert_not_called() + self.assertFalse(jobs.list_jobs()) + + def test_expiry_during_backup_does_not_cancel_or_restart_and_fresh_login_resumes(self): + ticket = self.ticket().json["ticket"] + self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), ticket) + job = jobs.get("run-1", 100) + wait_for(lambda: self.events.exists()) + denied = self.client.get("/api/lxc-update-jobs", + headers={"Authorization": "Bearer expired"}) + self.assertEqual(denied.status_code, 401) + self.assertIsNone(job.stop_reason) + self.assertEqual(self.client.get("/api/lxc-update-jobs", + headers={"Authorization": "Bearer valid"}).json["jobs"][0]["id"], job.id) + with patch.object(jobs.subprocess, "Popen") as launch: + self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) + launch.assert_not_called() + self.finish(job) + self.assertEqual(job.exit_code, 0) + self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) + self.callback.assert_called_once() + complete = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) + frames = [json.loads(c.args[0]) for c in complete.send.call_args_list] + self.assertEqual(frames[-1], {"type": "script_complete", "exit_code": 0}) + + def test_answer_in_one_viewer_clears_the_question_for_all_viewers(self): + self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), + self.ticket().json["ticket"]) + job = jobs.get("run-1", 100) + identity = "fixture-" + os.urandom(8).hex() + response = Path("/tmp/proxmenux_response_" + identity) + self.addCleanup(lambda: response.unlink(missing_ok=True)) + job.interaction = dict(type="yesno", id=identity, title="Question", message="Continue?") + job.revision += 1 + first = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"], + [json.dumps(dict(type="interaction_response", id=identity, value="yes"))]) + second = self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) + for ws in [first, second]: + frames = [json.loads(c.args[0]) for c in ws.send.call_args_list] + self.assertIn(dict(type="web_interaction", interaction=None), frames) + self.assertEqual(response.read_text(), "yes") + self.assertIsNone(job.stop_reason) + self.callback.assert_not_called() + self.finish(job) + self.callback.assert_called_once() + + def test_ticket_is_single_use_and_invalid_resume_never_becomes_execution(self): + ticket = self.ticket().json["ticket"] + with patch.object(jobs.subprocess, "Popen") as launch: + ws = self.attach(dict(resume=True, vmid=100), ticket) + self.assertIn("job_unavailable", ws.send.call_args.args[0]) + denied = self.attach(dict(resume=True, vmid=100), ticket) + denied.close.assert_called_once() + launch.assert_not_called() + + def test_expiry_during_update_then_two_viewers_cannot_duplicate_or_finalize_early(self): + update_gate = self.folder / "update-done" + self.script.write_text(self.script.read_text() + + f"while [ ! -e {shlex.quote(str(update_gate))} ]; do sleep .02; done\n") + self.attach(dict(resume=False, script_path=str(self.script), params=self.params()), + self.ticket().json["ticket"]) + job = jobs.get("run-1", 100) + self.gate.touch() + wait_for(lambda: self.events.exists() and "update" in self.events.read_text()) + for token, status in [("expired", 401), ("read-only", 403)]: + self.assertEqual(self.client.get("/api/lxc-update-jobs", headers={ + "Authorization": "Bearer " + token}).status_code, status) + self.callback.assert_not_called() + with patch.object(jobs.subprocess, "Popen") as spawn: + for _ in range(2): + self.attach(dict(resume=True, vmid=100), self.ticket().json["ticket"]) + spawn.assert_not_called() + self.assertIsNone(job.stop_reason) + update_gate.touch() + wait_for(lambda: job.state == "complete") + self.callback.assert_called_once() + self.assertEqual(self.events.read_text().splitlines(), ["backup", "update"]) + + def test_original_interactive_route_still_terminates_its_process_on_disconnect(self): + # Reproduce the original ownership bug using the untouched legacy route. + # Only its allowed fixture directory and process observation are injected. + nodes = [n for n in ast.parse((ROOT / "AppImage/scripts/flask_terminal_routes.py").read_text()).body + if isinstance(n, ast.FunctionDef) and n.name == "script_websocket"] + realpath = os.path.realpath + os_view = SimpleNamespace(**vars(os)) + os_view.path = SimpleNamespace(**vars(os.path)) + os_view.path.realpath = lambda p: (str(self.folder) + if p == "/usr/local/share/proxmenux/scripts" else realpath(p)) + children = [] + def spawn(*args, **kwargs): + child = subprocess.Popen(*args, **kwargs) + children.append(child) + self.addCleanup(lambda: child.poll() is None and child.kill()) + return child + self.scope.update(os=os_view, subprocess=SimpleNamespace(Popen=spawn), + tempfile=tempfile, pty=pty, fcntl=fcntl, select=select, base64=base64, threading=threading, + _SAFE_ID_RE=re.compile(r"^[A-Za-z0-9_-]{1,64}$"), + set_winsize=lambda *_: None) + exec(compile(ast.Module(body=nodes, type_ignores=[]), 'actual-legacy-route', 'exec'), self.scope) + ws = Mock() + def receive(**_): + if not children: + return json.dumps(dict(script_path=str(self.script), params=self.params())) + wait_for(lambda: self.events.exists()) + raise ConnectionError("expired login reloaded browser") + ws.receive.side_effect = receive + with self.app.test_request_context('/ws/script/fixture?ticket='+self.ticket().json['ticket']): + self.ws_routes['/ws/script/'](ws, 'fixture') + self.assertIsNotNone(children[0].poll()) + self.assertNotEqual(children[0].returncode, 0) + self.assertEqual(self.events.read_text().splitlines(), ['backup']) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_oci_update_cancellation.cjs b/tests/test_oci_update_cancellation.cjs index b91e77e2..3113915f 100644 --- a/tests/test_oci_update_cancellation.cjs +++ b/tests/test_oci_update_cancellation.cjs @@ -23,7 +23,7 @@ function executable(node, file, scope) { function fixture(handler, locale, optedIn=true) { const writes=[], callbacks=[], state={closed:0}; const t=translator(locale), terminal={write:s=>writes.push(s),writeln:s=>writes.push(s)}; - const shared={...completion, cancelledExitCode:optedIn?6:undefined,t, + const shared={...completion, backgroundUpdate:undefined, isOpenRef:{current:true}, openGenerationRef:{current:1}, cancelledExitCode:optedIn?6:undefined,t, completedSuccessfullyMessage:undefined,completedWithErrorMessage:undefined}; const getCompletionMessage=executable(messageCallback,source,shared); const scope={...shared,getCompletionMessage,term:terminal,termRef:{current:terminal},