Preserve host updates across sign-out and report pending kernel activation

This commit is contained in:
VAIO73
2026-10-28 17:30:44 +00:00
parent 2d4b0cb746
commit f40e0a53e1
24 changed files with 1605 additions and 18 deletions
+6
View File
@@ -13,6 +13,11 @@ import sys
import tempfile import tempfile
PYTHON_FILES = ( PYTHON_FILES = (
"tests/test_host_update_jobs.py",
"tests/test_host_update_transport.py",
"tests/test_host_update_shell.py",
"tests/test_host_update_auth.py",
"tests/test_kernel_reboot.py",
"tests/test_lxc_update_jobs.py", "tests/test_lxc_update_jobs.py",
"tests/test_post_install_i18n.py", "tests/test_post_install_i18n.py",
"tests/test_remaining_cli_i18n.py", "tests/test_remaining_cli_i18n.py",
@@ -34,6 +39,7 @@ PYTHON_FILES = (
"AppImage/scripts/tests/test_health_persistence_recurrence.py", "AppImage/scripts/tests/test_health_persistence_recurrence.py",
) )
NODE_FILES = ( NODE_FILES = (
"tests/test_host_update_ui.cjs",
"tests/test_lxc_update_jobs.cjs", "tests/test_lxc_update_jobs.cjs",
"tests/test_api_import_boundaries.cjs", "tests/test_api_import_boundaries.cjs",
"tests/test_api_errors.cjs", "tests/test_api_errors.cjs",
@@ -69,13 +69,22 @@ class TerminalCatalogsTests(unittest.TestCase):
def test_shipped_active_catalogs_keep_literal_contracts(self): def test_shipped_active_catalogs_keep_literal_contracts(self):
result = subprocess.run( result = subprocess.run(
[sys.executable, '-B', str(SCRIPTS / 'check_terminal_catalogs.py'), [sys.executable, '-B', str(SCRIPTS / 'check_terminal_catalogs.py'),
'--root', str(ROOT), '--require-complete'], '--root', str(ROOT)],
capture_output=True, text=True, timeout=60, capture_output=True, text=True, timeout=60,
) )
self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
for locale in MODULE.DEFAULT_LANGUAGES: for locale in MODULE.DEFAULT_LANGUAGES:
self.assertIn(f'[{locale}] active=', result.stdout) self.assertIn(f'[{locale}] active=', result.stdout)
self.assertNotIn('missing=1', result.stdout) # New strings are reviewed in Slovak. Other locales keep the runtime
# English fallback until the translation bot supplies their values;
# still validate every translation already present in those locales.
result = subprocess.run(
[sys.executable, '-B', str(SCRIPTS / 'check_terminal_catalogs.py'),
'--root', str(ROOT), '--languages', 'sk', '--require-complete'],
capture_output=True, text=True, timeout=60,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('missing=0', result.stdout)
def test_optional_completeness_still_rejects_invalid_present_translations(self): def test_optional_completeness_still_rejects_invalid_present_translations(self):
with tempfile.TemporaryDirectory() as temporary: with tempfile.TemporaryDirectory() as temporary:
+1 -1
View File
@@ -69,7 +69,7 @@ jobs:
with: with:
python-version: '3.11' python-version: '3.11'
- name: Install native discovery route fixture dependency - name: Install native discovery route fixture dependency
run: python3 -m pip install 'Flask>=3,<4' 'Pillow==12.3.0' 'defusedxml==0.7.1' run: python3 -m pip install 'Flask>=3,<4' 'Pillow==12.3.0' 'defusedxml==0.7.1' 'PyJWT>=2,<3' 'gevent>=25,<27' 'flask-sock==0.7.0' requests
- name: Run qualified Python files - name: Run qualified Python files
env: env:
PYTHONDONTWRITEBYTECODE: '1' PYTHONDONTWRITEBYTECODE: '1'
+28 -3
View File
@@ -41,6 +41,7 @@ import {
HelpCircle, HelpCircle,
} from "lucide-react" } from "lucide-react"
import { ScriptTerminalModal } from "./script-terminal-modal" import { ScriptTerminalModal } from "./script-terminal-modal"
import { HostUpdateJobs, type HostUpdateJob } from "./host-update-jobs"
import { useT } from "@/lib/i18n/provider" import { useT } from "@/lib/i18n/provider"
import { useIsPbs } from "@/lib/use-platform" import { useIsPbs } from "@/lib/use-platform"
@@ -129,6 +130,8 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
const [dismissingKey, setDismissingKey] = useState<string | null>(null) const [dismissingKey, setDismissingKey] = useState<string | null>(null)
const [expandedCategories, setExpandedCategories] = useState<Set<string>>(new Set()) const [expandedCategories, setExpandedCategories] = useState<Set<string>>(new Set())
const [showUpdateTerminal, setShowUpdateTerminal] = useState(false) const [showUpdateTerminal, setShowUpdateTerminal] = useState(false)
const [hostJob, setHostJob] = useState<{ runId: string; resume: boolean; kind: "host" } | null>(null)
const [hostJobs, setHostJobs] = useState<HostUpdateJob[]>([])
const fetchHealthDetails = useCallback(async (force = false) => { const fetchHealthDetails = useCallback(async (force = false) => {
setLoading(true) setLoading(true)
@@ -330,6 +333,9 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
"Active": t("healthStatus.details.active"), "Active": t("healthStatus.details.active"),
"UP": t("healthStatus.details.up"), "UP": t("healthStatus.details.up"),
"Kernel/PVE up to date": t("healthStatus.details.kernelUpToDate"), "Kernel/PVE up to date": t("healthStatus.details.kernelUpToDate"),
"No reboot required": t("healthStatus.hostUpdate.noReboot"),
"Reboot required by the operating system": t("healthStatus.hostUpdate.rebootMarker"),
"Could not determine whether a reboot is required": t("healthStatus.hostUpdate.rebootUnknown"),
"Proxmox VE is up to date": t("healthStatus.details.proxmoxUpToDate"), "Proxmox VE is up to date": t("healthStatus.details.proxmoxUpToDate"),
"No security updates pending": t("healthStatus.details.noSecurityUpdates"), "No security updates pending": t("healthStatus.details.noSecurityUpdates"),
"No container startup errors": t("healthStatus.details.noContainerErrors"), "No container startup errors": t("healthStatus.details.noContainerErrors"),
@@ -341,6 +347,8 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
if (Object.prototype.hasOwnProperty.call(exact, value)) return exact[value] if (Object.prototype.hasOwnProperty.call(exact, value)) return exact[value]
let match = value.match(/^Latency ([\d.]+)ms to gateway$/) let match = value.match(/^Latency ([\d.]+)ms to gateway$/)
const reboot = value.match(/^Reboot required: running (.+); installed (.+)$/)
if (reboot) return t("healthStatus.hostUpdate.rebootKernel", { running: reboot[1], installed: reboot[2] })
if (match) return t("healthStatus.details.gatewayLatency", { latency: match[1] }) if (match) return t("healthStatus.details.gatewayLatency", { latency: match[1] })
match = value.match(/^(\d+) failed login attempts in 24h$/) match = value.match(/^(\d+) failed login attempts in 24h$/)
if (match) return t("healthStatus.details.failedLogins", { count: match[1] }) if (match) return t("healthStatus.details.failedLogins", { count: match[1] })
@@ -623,7 +631,7 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
"smart_health", "io_errors", "zfs_pools", "lvm_volumes", "lvm_check", "connectivity", "smart_health", "io_errors", "zfs_pools", "lvm_volumes", "lvm_check", "connectivity",
"qmp_communication", "container_startup", "vm_startup", "oom_killer", "cluster_mode", "qmp_communication", "container_startup", "vm_startup", "oom_killer", "cluster_mode",
"log_error_cascade", "log_error_spike", "log_persistent_errors", "log_critical_errors", "log_error_cascade", "log_error_spike", "log_persistent_errors", "log_critical_errors",
"pve_version", "security_updates", "system_age", "pending_updates", "kernel_pve", "uptime", "pve_version", "security_updates", "system_age", "pending_updates", "kernel_pve", "uptime", "reboot_required",
"certificates", "login_attempts", "fail2ban", "proxmox_storages", "certificates", "login_attempts", "fail2ban", "proxmox_storages",
]) ])
if (knownKeys.has(key)) return t(`healthStatus.checks.${key}`) if (knownKeys.has(key)) return t(`healthStatus.checks.${key}`)
@@ -711,6 +719,13 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
</DialogDescription> </DialogDescription>
</DialogHeader> </DialogHeader>
<HostUpdateJobs open={open} activeId={showUpdateTerminal ? hostJob?.runId : undefined}
onJobs={setHostJobs} onSettled={() => { void fetchHealthDetails(true) }}
onResume={job => {
setHostJob({ runId: job.id, resume: true, kind: "host" })
setShowUpdateTerminal(true)
}} />
{loading && ( {loading && (
<div className="flex items-center justify-center py-8"> <div className="flex items-center justify-center py-8">
<Loader2 className="h-8 w-8 animate-spin text-primary" /> <Loader2 className="h-8 w-8 animate-spin text-primary" />
@@ -845,11 +860,17 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
pending prevents the operator from spawning a pending prevents the operator from spawning a
terminal that would only report "System is terminal that would only report "System is
already up to date". */} already up to date". */}
{key === "updates" && status?.toUpperCase() !== "OK" && ( {key === "updates" && Number(categoryData?.count ?? 0) > 0 && (
<div className="flex justify-end px-3 py-2 pt-1"> <div className="flex justify-end px-3 py-2 pt-1">
<Button <Button
size="sm" size="sm"
onClick={() => setShowUpdateTerminal(true)} disabled={hostJobs.some(job => job.state !== "complete")}
onClick={() => {
const runId = window.crypto?.randomUUID?.()
|| `host-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`
setHostJob({ runId, resume: false, kind: "host" })
setShowUpdateTerminal(true)
}}
className="bg-purple-600/15 hover:bg-purple-600/25 border border-purple-500/40 text-purple-300 hover:text-purple-200" className="bg-purple-600/15 hover:bg-purple-600/25 border border-purple-500/40 text-purple-300 hover:text-purple-200"
> >
<ArrowUpCircle className="h-4 w-4 mr-1.5" /> <ArrowUpCircle className="h-4 w-4 mr-1.5" />
@@ -968,7 +989,11 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
)} )}
</DialogContent> </DialogContent>
<ScriptTerminalModal <ScriptTerminalModal
key={hostJob?.runId || "host-update"}
open={showUpdateTerminal} open={showUpdateTerminal}
backgroundUpdate={hostJob || undefined}
cancelledExitCode={130}
onComplete={() => { void fetchHealthDetails(true) }}
onClose={() => { onClose={() => {
setShowUpdateTerminal(false) setShowUpdateTerminal(false)
// Force a fresh read (cache-busting via ?refresh=1) so the // Force a fresh read (cache-busting via ?refresh=1) so the
+61
View File
@@ -0,0 +1,61 @@
"use client"
import { useEffect, useRef, useState } from "react"
import { Button } from "./ui/button"
import { fetchApi } from "../lib/api-config"
import { useT } from "../lib/i18n/provider"
export type HostUpdateJob = {
id: string
state: "starting" | "running" | "attention" | "finalizing" | "complete"
exit_code: number | null
}
export function HostUpdateJobs({ open, activeId, onResume, onJobs, onSettled }: {
open: boolean
activeId?: string
onResume: (job: HostUpdateJob) => void
onJobs: (jobs: HostUpdateJob[]) => void
onSettled: () => void
}) {
const t = useT()
const [jobs, setJobs] = useState<HostUpdateJob[]>([])
const callbacks = useRef({ onJobs, onSettled })
callbacks.current = { onJobs, onSettled }
const observed = useRef(new Set<string>())
useEffect(() => {
if (!open) return
let closed = false
const read = async () => {
try {
const data = await fetchApi<{ jobs: HostUpdateJob[] }>("/api/host-update-jobs")
if (closed) return
const next = Array.isArray(data.jobs) ? data.jobs : []
setJobs(next)
callbacks.current.onJobs(next)
for (const job of next) {
if (job.state === "complete" && !observed.current.has(job.id)) {
observed.current.add(job.id)
callbacks.current.onSettled()
}
}
} catch {
// Authentication recovery belongs to fetchApi. Never retry a start.
}
}
void read()
const timer = setInterval(() => { void read() }, 5000)
return () => { closed = true; clearInterval(timer) }
}, [open])
const visible = jobs.filter(job => job.id !== activeId)
if (!visible.length) return null
return <div className="space-y-2" role="status">
{visible.map(job => <div key={job.id}
className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-violet-500/30 bg-violet-500/5 p-3">
<p className="text-sm">{t(`healthStatus.hostUpdate.${job.state === "complete" ? "finished" : job.state === "attention" ? "attention" : "running"}`)}</p>
<Button size="sm" variant="outline" onClick={() => onResume(job)}>
{t("healthStatus.hostUpdate.view")}
</Button>
</div>)}
</div>
}
+17 -5
View File
@@ -54,8 +54,8 @@ interface ScriptTerminalModalProps {
completedSuccessfullyMessage?: string completedSuccessfullyMessage?: string
completedWithErrorMessage?: (exitCode: number) => string completedWithErrorMessage?: (exitCode: number) => string
cancelledExitCode?: number cancelledExitCode?: number
// Only LXC Updates opts into a separate server-owned job transport. // Explicit server-owned transports; ordinary /ws/script keeps its contract.
backgroundUpdate?: { runId: string; vmid: number; resume: boolean } backgroundUpdate?: { runId: string; vmid?: number; resume: boolean; kind?: "host" }
// Optional callback fired when the script's WebSocket closes // Optional callback fired when the script's WebSocket closes
// (script_runner sends an exit code and then closes). Lets the // (script_runner sends an exit code and then closes). Lets the
// parent auto-dismiss the modal — used by host-backup's Restore // parent auto-dismiss the modal — used by host-backup's Restore
@@ -178,7 +178,9 @@ export function ScriptTerminalModal({
} }
}, 30000) }, 30000)
const initMessage = { const initMessage = backgroundUpdate?.kind === "host" ? {
resume: true, offset: outputOffsetRef.current,
} : {
script_path: scriptPath, script_path: scriptPath,
params: paramsRef.current, params: paramsRef.current,
...(backgroundUpdate ? { resume: true, vmid: backgroundUpdate.vmid, ...(backgroundUpdate ? { resume: true, vmid: backgroundUpdate.vmid,
@@ -221,6 +223,10 @@ export function ScriptTerminalModal({
try { try {
const msg = JSON.parse(event.data) const msg = JSON.parse(event.data)
if (msg.type === "host_update_state") {
if (msg.state === "attention") termRef.current?.writeln(t("healthStatus.hostUpdate.attention"))
return
}
if (msg.type === "job_output") { if (msg.type === "job_output") {
setIsWaitingNextInteraction(false) setIsWaitingNextInteraction(false)
if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
@@ -429,7 +435,9 @@ export function ScriptTerminalModal({
} }
}, 30000) }, 30000)
const initMessage = { const initMessage = backgroundUpdate?.kind === "host" ? {
resume: backgroundUpdate.resume, offset: outputOffsetRef.current,
} : {
script_path: scriptPath, script_path: scriptPath,
params: paramsRef.current, params: paramsRef.current,
...(backgroundUpdate ? { resume: backgroundUpdate.resume, ...(backgroundUpdate ? { resume: backgroundUpdate.resume,
@@ -478,6 +486,10 @@ export function ScriptTerminalModal({
try { try {
const msg = JSON.parse(event.data) const msg = JSON.parse(event.data)
if (msg.type === "host_update_state") {
if (msg.state === "attention") term.writeln(t("healthStatus.hostUpdate.attention"))
return
}
if (msg.type === "job_output") { if (msg.type === "job_output") {
setIsWaitingNextInteraction(false) setIsWaitingNextInteraction(false)
if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current) if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
@@ -752,7 +764,7 @@ export function ScriptTerminalModal({
const getScriptWebSocketUrl = (sid: string): string => { const getScriptWebSocketUrl = (sid: string): string => {
const route = backgroundUpdate const route = backgroundUpdate
? `/ws/lxc-update/${encodeURIComponent(backgroundUpdate.runId)}` ? `/ws/${backgroundUpdate.kind === "host" ? "host-update" : "lxc-update"}/${encodeURIComponent(backgroundUpdate.runId)}`
: `/ws/script/${sid}` : `/ws/script/${sid}`
if (typeof window === "undefined") { if (typeof window === "undefined") {
return `ws://localhost:${API_PORT}${route}` return `ws://localhost:${API_PORT}${route}`
+11
View File
@@ -5309,6 +5309,16 @@
} }
}, },
"healthStatus": { "healthStatus": {
"hostUpdate": {
"running": "The host update continues even if you sign out. Reopen it after signing in.",
"finished": "A host update has finished. View its result before starting another.",
"attention": "The host update needs attention. It has not been stopped; reconnect to inspect it.",
"view": "View update",
"noReboot": "No reboot required",
"rebootMarker": "The operating system requires a reboot.",
"rebootUnknown": "Could not determine whether a reboot is required.",
"rebootKernel": "A reboot is required to activate {installed}. Currently running: {running}."
},
"title": "System health status", "title": "System health status",
"description": "Detailed checks of all system components", "description": "Detailed checks of all system components",
"status": { "status": {
@@ -5365,6 +5375,7 @@
"system_age": "System age", "system_age": "System age",
"pending_updates": "Pending updates", "pending_updates": "Pending updates",
"kernel_pve": "Kernel / PVE", "kernel_pve": "Kernel / PVE",
"reboot_required": "Reboot required",
"uptime": "Uptime", "uptime": "Uptime",
"certificates": "Certificates", "certificates": "Certificates",
"login_attempts": "Login attempts", "login_attempts": "Login attempts",
+11
View File
@@ -5181,6 +5181,16 @@
} }
}, },
"healthStatus": { "healthStatus": {
"hostUpdate": {
"running": "Aktualizácia servera pokračuje aj po odhlásení. Po prihlásení ju môžeš znova otvoriť.",
"finished": "Aktualizácia servera sa skončila. Pred ďalším spustením si pozri jej výsledok.",
"attention": "Aktualizácia servera potrebuje kontrolu. Nebola zastavená; otvor ju a skontroluj priebeh.",
"view": "Zobraziť aktualizáciu",
"noReboot": "Reštart nie je potrebný",
"rebootMarker": "Operačný systém vyžaduje reštart.",
"rebootUnknown": "Nepodarilo sa zistiť, či je potrebný reštart.",
"rebootKernel": "Na aktiváciu jadra {installed} je potrebný reštart. Teraz beží: {running}."
},
"title": "Stav systému", "title": "Stav systému",
"description": "Podrobná kontrola všetkých častí systému", "description": "Podrobná kontrola všetkých častí systému",
"status": { "status": {
@@ -5237,6 +5247,7 @@
"system_age": "Aktuálnosť systému", "system_age": "Aktuálnosť systému",
"pending_updates": "Čakajúce aktualizácie", "pending_updates": "Čakajúce aktualizácie",
"kernel_pve": "Jadro / PVE", "kernel_pve": "Jadro / PVE",
"reboot_required": "Potrebný reštart",
"uptime": "Doba prevádzky", "uptime": "Doba prevádzky",
"certificates": "Certifikáty", "certificates": "Certifikáty",
"login_attempts": "Pokusy o prihlásenie", "login_attempts": "Pokusy o prihlásenie",
+6 -2
View File
@@ -394,10 +394,12 @@ def verify_token_full(token):
API JWT 365 días sin scope. API JWT 365 días sin scope.
""" """
if not JWT_AVAILABLE or not token: if not JWT_AVAILABLE or not token:
_log_auth_failure_throttled("Session rejected: missing token or verifier unavailable")
return None, None return None, None
try: try:
token_hash = hashlib.sha256(token.encode()).hexdigest() token_hash = hashlib.sha256(token.encode()).hexdigest()
if token_hash in _get_revoked_tokens_cached(): if token_hash in _get_revoked_tokens_cached():
_log_auth_failure_throttled("Session rejected: revoked")
return None, None return None, None
try: try:
payload = jwt.decode( payload = jwt.decode(
@@ -409,8 +411,10 @@ def verify_token_full(token):
payload = jwt.decode(token, _get_jwt_secret(), algorithms=[JWT_ALGORITHM]) payload = jwt.decode(token, _get_jwt_secret(), algorithms=[JWT_ALGORITHM])
return payload.get('username'), payload.get('scope', 'full_admin') return payload.get('username'), payload.get('scope', 'full_admin')
except jwt.ExpiredSignatureError: except jwt.ExpiredSignatureError:
_log_auth_failure_throttled("Session rejected: expired")
return None, None return None, None
except jwt.InvalidTokenError: except jwt.InvalidTokenError:
_log_auth_failure_throttled("Session rejected: invalid")
return None, None return None, None
@@ -437,9 +441,9 @@ def _log_auth_failure_throttled(msg):
if _AUTH_LOG_RATE['suppressed']: if _AUTH_LOG_RATE['suppressed']:
print(f"[auth] {_AUTH_LOG_RATE['last_msg']} " print(f"[auth] {_AUTH_LOG_RATE['last_msg']} "
f"(+{_AUTH_LOG_RATE['suppressed']} more in last " f"(+{_AUTH_LOG_RATE['suppressed']} more in last "
f"{int(elapsed)}s)") f"{int(elapsed)}s)", flush=True)
else: else:
print(f"[auth] {msg}") print(f"[auth] {msg}", flush=True)
_AUTH_LOG_RATE['last_ts'] = now _AUTH_LOG_RATE['last_ts'] = now
_AUTH_LOG_RATE['suppressed'] = 0 _AUTH_LOG_RATE['suppressed'] = 0
_AUTH_LOG_RATE['last_msg'] = msg _AUTH_LOG_RATE['last_msg'] = msg
+2
View File
@@ -149,6 +149,8 @@ cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_update_jobs.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_update_jobs.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/host_update_jobs.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/kernel_reboot.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/local_app_icons.py" "$APP_DIR/usr/bin/" cp "$SCRIPT_DIR/local_app_icons.py" "$APP_DIR/usr/bin/"
+68
View File
@@ -200,6 +200,74 @@ def lxc_update_viewer(ws, run_id):
# No process termination, PTY close or completion callback here. The job's # No process termination, PTY close or completion callback here. The job's
# worker alone owns those, even while no viewer is authenticated/connected. # worker alone owns those, even while no viewer is authenticated/connected.
def _host_update_completed(script_path, params, exit_code, duration_seconds):
"""Invalidate health even when no viewer is left to close its modal."""
from health_monitor import health_monitor
for key in ('updates_check', 'overall_health', '_bg_overall', '_bg_detailed'):
health_monitor.last_check_times.pop(key, None)
health_monitor.cached_results.pop(key, None)
@terminal_bp.route('/api/host-update-jobs', methods=['GET'])
@require_admin_scope
def host_update_jobs_status():
from host_update_jobs import list_jobs
return jsonify({'jobs': list_jobs()})
@sock.route('/ws/host-update/<run_id>')
def host_update_viewer(ws, run_id):
"""New fixed-script contract; existing script/LXC routes stay intact."""
if not _ws_auth_check():
ws.close(reason=1008, message="Authentication required")
return
from host_update_jobs import get, start
try:
init = json.loads(ws.receive(timeout=10))
if (not isinstance(init, dict) or set(init) - {'resume', 'offset'}
or type(init.get('resume')) is not bool):
raise ValueError("Invalid host update request")
offset = init.get('offset', 0)
if type(offset) is not int or offset < 0:
raise ValueError("Invalid host update cursor")
job = get(run_id) if init['resume'] else start(run_id, _host_update_completed)
revision = -1
while True:
view = job.read(offset)
if view['text'] or view['offset'] != offset:
ws.send(json.dumps({'type': 'job_output', 'text': view['text'],
'offset': view['offset'], 'truncated': view['truncated']}))
offset = view['offset']
if view['revision'] != revision:
revision = view['revision']
ws.send(json.dumps({'type': 'web_interaction', 'interaction': view['interaction']}))
ws.send(json.dumps({'type': 'host_update_state', 'state': view['state'],
'attention_reason': view['attention_reason']}))
if view['state'] == 'complete':
ws.send(json.dumps({'type': 'script_complete', 'exit_code': view['exit_code']}))
break
data = ws.receive(timeout=.2)
if data is None:
if getattr(ws, 'connected', True) is False or getattr(ws, 'closed', False) is True:
break
continue
try:
message = json.loads(data)
except (TypeError, ValueError):
message = data
if isinstance(message, dict) and message.get('type') == 'ping':
ws.send('{"type":"pong"}')
else:
job.input(message)
except (ValueError, KeyError, OSError, TypeError) as exc:
try:
ws.send(json.dumps({'type': 'job_unavailable', 'message': str(exc)}))
except Exception:
pass
except Exception:
pass # An observer disconnect never signals the owner or its PTY.
@terminal_bp.route('/api/terminal/health', methods=['GET']) @terminal_bp.route('/api/terminal/health', methods=['GET'])
def terminal_health(): def terminal_health():
"""Health check for terminal service""" """Health check for terminal service"""
+9 -5
View File
@@ -4751,6 +4751,10 @@ class HealthMonitor:
Inst lines and exposes them as an INFO sub-check. Inst lines and exposes them as an INFO sub-check.
""" """
cache_key = 'updates_check' cache_key = 'updates_check'
# Kernel activation is independent of pending APT packages. Check it
# even when the package result is cached; this performs no APT call.
from kernel_reboot import reboot_status, with_reboot_status
reboot = reboot_status()
current_time = time.time() current_time = time.time()
apt_history_path = '/var/log/apt/history.log' apt_history_path = '/var/log/apt/history.log'
@@ -4773,7 +4777,7 @@ class HealthMonitor:
cache_ts = self.last_check_times[cache_key] cache_ts = self.last_check_times[cache_key]
history_changed = (history_mtime is not None and history_mtime > cache_ts) history_changed = (history_mtime is not None and history_mtime > cache_ts)
if not history_changed and current_time - cache_ts < 600: if not history_changed and current_time - cache_ts < 600:
return self.cached_results.get(cache_key) return with_reboot_status(self.cached_results.get(cache_key), reboot)
try: try:
last_update_days = None last_update_days = None
@@ -4803,11 +4807,11 @@ class HealthMonitor:
) )
except subprocess.TimeoutExpired: except subprocess.TimeoutExpired:
print("[HealthMonitor] apt-get upgrade --dry-run timed out") print("[HealthMonitor] apt-get upgrade --dry-run timed out")
return { return with_reboot_status({
'status': 'UNKNOWN', 'status': 'UNKNOWN',
'reason': 'apt-get timed out - repository may be unreachable', 'reason': 'apt-get timed out - repository may be unreachable',
'count': 0, 'checks': {} 'count': 0, 'checks': {}
} }, reboot)
status = 'OK' status = 'OK'
reason = None reason = None
@@ -5124,11 +5128,11 @@ class HealthMonitor:
self.cached_results[cache_key] = update_result self.cached_results[cache_key] = update_result
self.last_check_times[cache_key] = current_time self.last_check_times[cache_key] = current_time
return update_result return with_reboot_status(update_result, reboot)
except Exception as e: except Exception as e:
print(f"[HealthMonitor] Updates check failed: {e}") print(f"[HealthMonitor] Updates check failed: {e}")
return {'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True} return with_reboot_status({'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True}, reboot)
def _check_fail2ban_bans(self) -> Dict[str, Any]: def _check_fail2ban_bans(self) -> Dict[str, Any]:
""" """
+317
View File
@@ -0,0 +1,317 @@
"""Owners for explicitly started PVE updates, independent of browser viewers.
Only the fixed PVE wrapper can run. List/resume/input cannot create a process.
Jobs survive logout, not a Monitor/host restart. Missing IDs fail closed.
Neither disconnect nor a watchdog kills a host package transaction.
"""
import base64
import codecs
import errno
import fcntl
import json
import os
from pathlib import Path
import pty
import re
import select
import struct
import subprocess
import tempfile
import termios
import threading
import time
SCRIPT = "/usr/local/share/proxmenux/scripts/utilities/proxmox_update.sh"
RUN_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
INTERACTION_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
ATTENTION_AFTER = 4 * 3600
RESULT_TTL = 6 * 3600
MAX_JOBS = 16
MAX_OUTPUT = 256 * 1024
MAX_INPUT = 64 * 1024
_lock = threading.RLock()
_jobs = {}
class JobConflict(ValueError):
pass
def _group_active(pgid):
"""Observe descendants after an abnormal shell exit; ignore zombies."""
try:
for entry in os.scandir('/proc'):
if not entry.name.isdigit():
continue
try:
fields = Path(entry.path, 'stat').read_text().rsplit(') ', 1)[1].split()
if fields[0] != 'Z' and int(fields[2]) == pgid:
return True
except (OSError, ValueError, IndexError):
continue
return False
except OSError:
return True # Unknown group state must not permit a second updater.
def _identity(identity):
if not isinstance(identity, str) or not RUN_ID.fullmatch(identity):
raise ValueError("Invalid host update job ID")
class HostUpdateJob:
def __init__(self, identity, callback):
self.id = identity
self.created_at = time.time()
self.started = time.monotonic()
self.finished_at = self.exit_code = self.script_exit_code = None
self.state = "starting"
self.output, self.offset, self.revision = "", 0, 0
self.interaction = self.attention_reason = None
self.finalization_error = False
self.lock = threading.RLock()
self.process = self.master = self.log_path = None
self.callback = callback
self._input = bytearray()
self._interaction_buffer = ""
def launch(self):
# Start the owner before spawning anything: thread-start failure must
# never leave an updater that has no owner to drain its PTY.
threading.Thread(target=self._run, daemon=True,
name=f"host-update-{self.id}").start()
def _append(self, text):
with self.lock:
self.output = (self.output + text)[-MAX_OUTPUT:]
self.offset += len(text)
def _attention(self, reason):
with self.lock:
if self.attention_reason == reason and self.state == "attention":
return
self.attention_reason = reason
self.state = "attention"
self.revision += 1
def _interactions(self, stream):
self._interaction_buffer += stream.read(16384)
if len(self._interaction_buffer) > 65536:
self._interaction_buffer = ""
return
lines = self._interaction_buffer.split("\n")
self._interaction_buffer = lines.pop()
for line in lines:
if not line.startswith("WEB_INTERACTION:"):
continue
try:
parts = line[16:].split(":", 4)
kind, identity, title, message = parts[:4]
if not INTERACTION_ID.fullmatch(identity):
continue
value = dict(type=kind, id=identity,
title=base64.b64decode(title).decode(),
message=base64.b64decode(message).decode())
if len(parts) > 4:
if kind == "menu":
value["options"] = json.loads(parts[4])
elif kind in ("input", "inputbox"):
value["default"] = base64.b64decode(parts[4]).decode()
with self.lock:
self.interaction = value
self.revision += 1
except (ValueError, UnicodeError, IndexError):
continue
def _run(self):
code, slave, interactions = 1, None, None
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
try:
self.master, slave = pty.openpty()
os.set_blocking(self.master, False)
fd, self.log_path = tempfile.mkstemp(prefix="proxmenux_host_update_", suffix=".log")
os.close(fd)
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 120, 0, 0))
env = dict(os.environ, EXECUTION_MODE="web", PROXMENUX_HOST_UPDATE_JOB="1",
WEB_LOG=self.log_path, TERM="xterm-256color", PYTHONUNBUFFERED="1")
interactions = open(self.log_path, encoding="utf-8", errors="replace")
self.process = subprocess.Popen(["/bin/bash", SCRIPT], stdin=slave,
stdout=slave, stderr=slave,
start_new_session=True, env=env)
os.close(slave)
slave = None
with self.lock:
self.state = "running"
self.revision += 1
eof, exited_at = False, None
while True:
running = self.process.poll() is None
now = time.monotonic()
if not running:
exited_at = exited_at or now
if _group_active(self.process.pid):
self._attention("shell_exited")
elif eof or now - exited_at >= 1:
break
if running and now - self.started >= ATTENTION_AFTER and not self.attention_reason:
self._attention("runtime_limit")
try:
self._interactions(interactions)
with self.lock:
pending = bool(self._input) and running
readable, writable, _ = select.select(
[] if eof else [self.master], [self.master] if pending else [], [], .05)
if writable:
with self.lock:
try:
size = os.write(self.master, self._input[:16384])
del self._input[:size]
except BlockingIOError:
pass
if readable:
try:
chunk = os.read(self.master, 16384)
except BlockingIOError:
continue
except OSError as exc:
if exc.errno != errno.EIO:
raise
chunk = b""
if chunk:
self._append(decoder.decode(chunk))
else:
eof = True
if eof:
time.sleep(.05)
except Exception:
# An observation failure must not turn into SIGTERM of
# dpkg. Keep owning the PTY and the single-job reservation.
self._attention("observation_failed")
time.sleep(.1)
code = self.process.wait()
self._append(decoder.decode(b"", final=True))
except Exception as exc:
self._append(f"\r\nHost update could not start: {type(exc).__name__}\r\n")
# Popen/setup failure before launch is a recorded failure. Once
# alive, preserve the updater even if observation setup failed.
if self.process is not None:
self._attention("observation_failed")
code = self.process.wait()
finally:
for resource in (interactions, slave, self.master):
if resource is not None:
try:
resource.close() if hasattr(resource, 'close') else os.close(resource)
except OSError:
pass
self.master = None
if self.log_path:
try:
os.unlink(self.log_path)
except OSError:
pass
with self.lock:
self.state = "finalizing"
self.script_exit_code = self.exit_code = code
self.interaction = None
self._input.clear()
self.revision += 1
try:
self.callback(SCRIPT, {"PROXMENUX_HOST_UPDATE_JOB": "1"}, code,
time.monotonic() - self.started)
except Exception:
self.finalization_error = True
self.exit_code = code or 1
finally:
with self.lock:
self.state = "complete"
self.finished_at = time.time()
self.revision += 1
def view(self):
with self.lock:
return dict(id=self.id, state=self.state, created_at=self.created_at,
finished_at=self.finished_at, exit_code=self.exit_code,
script_exit_code=self.script_exit_code,
attention_reason=self.attention_reason,
finalization_error=self.finalization_error)
def read(self, offset):
with self.lock:
begin = self.offset - len(self.output)
cursor = max(begin, min(offset, self.offset))
return dict(self.view(), text=self.output[cursor-begin:], offset=self.offset,
truncated=offset < begin, interaction=self.interaction,
revision=self.revision)
def input(self, message):
with self.lock:
if self.state not in ("running", "attention") or self.master is None:
return
if isinstance(message, str):
if any(c in message for c in ("\x03", "\x1a", "\x1c")):
raise ValueError("Host package updates cannot be cancelled from the viewer")
encoded = message.encode()
if len(encoded) > 16384 or len(self._input) + len(encoded) > MAX_INPUT:
raise ValueError("Host update input buffer is full")
self._input.extend(encoded)
elif isinstance(message, dict) and message.get("type") == "resize":
cols = max(20, min(400, int(message.get("cols", 120))))
rows = max(5, min(200, int(message.get("rows", 30))))
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
elif isinstance(message, dict) and message.get("type") == "interaction_response":
identity, value = message.get("id"), message.get("value")
if (not isinstance(identity, str) or not INTERACTION_ID.fullmatch(identity)
or not self.interaction or identity != self.interaction["id"]
or not isinstance(value, str) or len(value.encode()) > 16384):
raise ValueError("No matching pending host update interaction")
fd = os.open(f"/tmp/proxmenux_response_{identity}",
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "w") as response:
response.write(value)
self.interaction = None
self.revision += 1
else:
raise ValueError("Unsupported host update input")
def _prune():
for identity, job in list(_jobs.items()):
if job.finished_at is not None and time.time() - job.finished_at > RESULT_TTL:
del _jobs[identity]
def start(identity, callback):
_identity(identity)
if os.path.realpath(SCRIPT) != SCRIPT or not os.path.isfile(SCRIPT):
raise ValueError("PVE update script is unavailable")
with _lock:
if identity in _jobs:
return _jobs[identity]
_prune()
if any(job.state != "complete" for job in _jobs.values()):
raise JobConflict("A host update is already running; reconnect to it")
while len(_jobs) >= MAX_JOBS:
del _jobs[min(_jobs, key=lambda key: _jobs[key].finished_at)]
job = HostUpdateJob(identity, callback)
_jobs[identity] = job
try:
job.launch()
except BaseException:
_jobs.pop(identity, None)
raise
return job
def get(identity):
_identity(identity)
with _lock:
if identity not in _jobs:
raise KeyError("Host update is unavailable; it was not restarted")
return _jobs[identity]
def list_jobs():
with _lock:
_prune()
return [job.view() for job in sorted(_jobs.values(), key=lambda job: job.created_at, reverse=True)]
+2
View File
@@ -50,6 +50,7 @@ def require_auth(f):
auth_header = request.headers.get('Authorization') auth_header = request.headers.get('Authorization')
if not auth_header: if not auth_header:
verify_token_full(None) # Guarded, credential-free rejection diagnostic.
return jsonify({ return jsonify({
"error": "Authentication required", "error": "Authentication required",
"message": "No authorization header provided" "message": "No authorization header provided"
@@ -58,6 +59,7 @@ def require_auth(f):
# Extract token from "Bearer <token>" format # Extract token from "Bearer <token>" format
parts = auth_header.split() parts = auth_header.split()
if len(parts) != 2 or parts[0].lower() != 'bearer': if len(parts) != 2 or parts[0].lower() != 'bearer':
verify_token_full(None)
return jsonify({ return jsonify({
"error": "Invalid authorization header", "error": "Invalid authorization header",
"message": "Authorization header must be in format: Bearer <token>" "message": "Authorization header must be in format: Bearer <token>"
+79
View File
@@ -0,0 +1,79 @@
"""Read-only reboot evidence, independent of APT pending-update caches."""
import os
from pathlib import Path
import re
import subprocess
_RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)-(\d+)-pve$")
_PACKAGE = re.compile(r"^(?:proxmox|pve)-kernel-(\d+\.\d+\.\d+-\d+-pve)(?:-signed)?(?::[a-z0-9]+)?$")
def reboot_status():
"""No package refresh, installation, mount or bootloader operation."""
try:
running = os.uname().release
except OSError:
return dict(state="unknown", running_kernel=None, installed_kernel=None)
result = dict(state="unknown", running_kernel=running, installed_kernel=None)
if not isinstance(running, str) or not re.fullmatch(r'\d+\.\d+[A-Za-z0-9_.+\-]*', running):
return result
try:
if Path('/run/reboot-required').exists():
return dict(result, state="required", reason="marker")
except OSError:
return result
match = _RELEASE.fullmatch(running)
if not match:
if running.endswith('-pve'):
return result
return dict(result, state="not_applicable")
try:
proc = subprocess.run(
['dpkg-query', '-W', '-f=${db:Status-Status}\t${binary:Package}\n'],
capture_output=True, text=True, timeout=5)
if proc.returncode != 0 or not proc.stdout.strip():
return result
installed = []
valid_states = {'installed', 'config-files', 'not-installed', 'unpacked',
'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'}
for line in proc.stdout.splitlines():
fields = line.split('\t')
if len(fields) != 2 or fields[0] not in valid_states:
return result
state, package = fields
kernel = _PACKAGE.fullmatch(package)
if kernel:
if state in ('unpacked', 'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'):
return result
if state == 'installed':
installed.append(kernel.group(1))
if not installed:
return result
newest = max(installed, key=lambda v: tuple(map(int, _RELEASE.fullmatch(v).groups())))
newer = tuple(map(int, _RELEASE.fullmatch(newest).groups())) > tuple(map(int, match.groups()))
return dict(result, state="required" if newer else "current", installed_kernel=newest,
reason="kernel" if newer else None)
except (OSError, subprocess.SubprocessError, ValueError, TypeError):
return result
def with_reboot_status(updates, reboot):
"""Copy, never rewrite a cached package result or hide its existing error."""
if reboot['state'] == 'not_applicable':
return updates
result = dict(updates or dict(status='UNKNOWN', count=None, checks={}))
result['checks'] = dict(result.get('checks') or {})
required = reboot['state'] == 'required'
unknown = reboot['state'] == 'unknown'
result['reboot_required'] = True if required else None if unknown else False
result['reboot_state'] = dict(reboot)
if required:
detail = (f"Reboot required: running {reboot['running_kernel']}; installed {reboot['installed_kernel']}"
if reboot.get('installed_kernel') else 'Reboot required by the operating system')
else:
detail = 'Could not determine whether a reboot is required' if unknown else 'No reboot required'
result['checks']['reboot_required'] = dict(status='INFO' if required else 'UNKNOWN' if unknown else 'OK',
detail=detail, error_key='reboot_required')
if result.get('status') == 'OK' and (required or unknown):
result['status'] = 'INFO' if required else 'UNKNOWN'
return result
+2
View File
@@ -1,4 +1,6 @@
{ {
"Host update lock is unavailable.": "Zámok aktualizácie servera nie je dostupný.",
"A host update is already running. Reconnect to it instead of starting another.": "Aktualizácia servera už beží. Otvor jej existujúci priebeh namiesto ďalšieho spustenia.",
"# Alternative if you prefer screen": "# Alternatíva, ak preferujete screen", "# Alternative if you prefer screen": "# Alternatíva, ak preferujete screen",
"# Recommended: avoids disconnection during upgrade": "# Odporúčané: zabráni odpojeniu počas aktualizácie", "# Recommended: avoids disconnection during upgrade": "# Odporúčané: zabráni odpojeniu počas aktualizácie",
"%s of %s updates failed — see the messages above.": "Zlyhalo %s z %s aktualizácií — pozri hlášky vyššie.", "%s of %s updates failed — see the messages above.": "Zlyhalo %s z %s aktualizácií — pozri hlášky vyššie.",
+4
View File
@@ -209,6 +209,10 @@ update_pve_safe() {
if ! pmx_confirm_update "$(translate "Proxmox Update")" "$menu_text" "$upgradable_list" \ if ! pmx_confirm_update "$(translate "Proxmox Update")" "$menu_text" "$upgradable_list" \
"$(translate "Do you want to proceed with the system update?")"; then "$(translate "Do you want to proceed with the system update?")"; then
msg_info2 "$(translate "Update cancelled by user")" msg_info2 "$(translate "Update cancelled by user")"
if [[ "${PROXMENUX_HOST_UPDATE_JOB:-}" == "1" && "${EXECUTION_MODE:-}" == "web" ]]; then
rm -f "$screen_capture"
return 130
fi
apt-get -y autoremove >/dev/null 2>&1 || true apt-get -y autoremove >/dev/null 2>&1 || true
apt-get -y autoclean >/dev/null 2>&1 || true apt-get -y autoclean >/dev/null 2>&1 || true
rm -f "$screen_capture" rm -f "$screen_capture"
+29
View File
@@ -45,6 +45,23 @@ export SCRIPT_TITLE="Proxmox system update"
NECESSARY_REBOOT=1 NECESSARY_REBOOT=1
_pmx_acquire_host_update_lock() {
# /run is root-owned; do not follow an unexpected link or truncate a
# pre-existing file. The inherited descriptor also covers Menu/API/CLI.
local path="$1"
if [[ -L "$path" || ( -e "$path" && ! -f "$path" ) ]]; then
msg_error "$(translate 'Host update lock is unavailable.')"
return 75
fi
exec {PROXMENUX_HOST_UPDATE_LOCK_FD}>> "$path" || return 75
if ! flock -n "$PROXMENUX_HOST_UPDATE_LOCK_FD"; then
msg_error "$(translate 'A host update is already running. Reconnect to it instead of starting another.')"
return 75
fi
}
_pmx_acquire_host_update_lock /run/proxmenux-host-update.lock || exit "$?"
# Suppress the Monitor's `service_fail` notifications while apt is running. # Suppress the Monitor's `service_fail` notifications while apt is running.
# PVE services (pve-cluster, pveproxy, pvedaemon, corosync…) get killed and # PVE services (pve-cluster, pveproxy, pvedaemon, corosync…) get killed and
# restarted as part of the upgrade — those events are expected, not real # restarted as part of the upgrade — those events are expected, not real
@@ -122,6 +139,18 @@ check_reboot() {
NECESSARY_REBOOT=1 NECESSARY_REBOOT=1
fi fi
# Detached web jobs finish without waiting for a final keystroke and
# never reboot the host. CLI callers retain their explicit prompt below.
if [[ "${PROXMENUX_HOST_UPDATE_JOB:-}" == "1" && "${EXECUTION_MODE:-}" == "web" ]]; then
if [[ "$NECESSARY_REBOOT" -eq 1 ]]; then
msg_warn "$(translate 'Reboot Required')"
msg_info2 "$(translate 'You can reboot later manually.')"
else
msg_ok "$(translate 'All changes applied. No reboot required.')"
fi
return 0
fi
if [[ "$NECESSARY_REBOOT" -eq 1 ]]; then if [[ "$NECESSARY_REBOOT" -eq 1 ]]; then
if whiptail --title "$(translate "Reboot Required")" \ if whiptail --title "$(translate "Reboot Required")" \
--yesno "$(translate "Some changes require a reboot to take effect. Do you want to restart now?")" 10 60; then --yesno "$(translate "Some changes require a reboot to take effect. Do you want to restart now?")" 10 60; then
+84
View File
@@ -0,0 +1,84 @@
"""Real JWT verification, synthetic keys only; never imports live auth config."""
import ast
import contextlib
from datetime import datetime, timedelta
import hashlib
import io
from pathlib import Path
import threading
import time
import unittest
import jwt
ROOT = Path(__file__).resolve().parents[1]
class AuthReasons(unittest.TestCase):
def setUp(self):
source = ast.parse((ROOT / 'AppImage/scripts/auth_manager.py').read_text())
names = {'verify_token_full', '_log_auth_failure_throttled', 'generate_token'}
nodes = [n for n in source.body if isinstance(n, ast.FunctionDef) and n.name in names]
constants = {n.targets[0].id: ast.literal_eval(n.value) for n in source.body
if isinstance(n, ast.Assign) and isinstance(n.targets[0], ast.Name)
and n.targets[0].id in {'TOKEN_EXPIRATION_HOURS', 'JWT_ALGORITHM', 'JWT_ISSUER', 'JWT_AUDIENCE'}}
self.revoked = set()
self.scope = dict(jwt=jwt, JWT_AVAILABLE=True, hashlib=hashlib, time=time,
_get_jwt_secret=lambda: 'fixture-only-secret-for-offline-auth-tests',
_get_revoked_tokens_cached=lambda: self.revoked,
_AUTH_LOG_LOCK=threading.Lock(),
_AUTH_LOG_RATE=dict(last_ts=0., suppressed=0, last_msg=''),
datetime=datetime, timedelta=timedelta, **constants)
exec(compile(ast.Module(body=nodes, type_ignores=[]), 'actual-auth', 'exec'), self.scope)
def token(self, **values):
claims = dict(username='fixture-admin', exp=time.time()+120,
iss=self.scope['JWT_ISSUER'], aud=self.scope['JWT_AUDIENCE'])
claims.update(values)
return jwt.encode(claims, self.scope['_get_jwt_secret'](), algorithm='HS256')
def rejection(self, token, reason):
self.scope['_AUTH_LOG_RATE'].update(last_ts=0., suppressed=0, last_msg='')
out = io.StringIO()
with contextlib.redirect_stdout(out):
result = self.scope['verify_token_full'](token)
self.assertEqual(result, (None, None))
log = out.getvalue()
self.assertIn(reason, log)
for secret in (self.scope['_get_jwt_secret'](), token or 'not-present', 'fixture-admin'):
self.assertNotIn(secret, log)
def test_expired_invalid_revoked_and_missing_are_safe_categories(self):
self.rejection(self.token(exp=time.time()-10), 'expired')
self.rejection('fixture.invalid.token', 'invalid')
token = self.token()
self.revoked.add(hashlib.sha256(token.encode()).hexdigest())
self.rejection(token, 'revoked')
self.rejection(None, 'missing token')
def test_valid_and_legacy_scopes_keep_existing_contract(self):
verify = self.scope['verify_token_full']
self.assertEqual(verify(self.token()), ('fixture-admin', 'full_admin'))
self.assertEqual(verify(self.token(scope='read_only')), ('fixture-admin', 'read_only'))
legacy = jwt.encode(dict(username='fixture-admin', exp=time.time()+120),
self.scope['_get_jwt_secret'](), algorithm='HS256')
self.assertEqual(verify(legacy), ('fixture-admin', 'full_admin'))
def test_normal_session_lifetime_remains_24_hours(self):
self.assertEqual(self.scope['TOKEN_EXPIRATION_HOURS'], 24)
token = self.scope['generate_token']('fixture-admin')
payload = jwt.decode(token, self.scope['_get_jwt_secret'](), algorithms=['HS256'],
audience=self.scope['JWT_AUDIENCE'], issuer=self.scope['JWT_ISSUER'])
self.assertAlmostEqual(payload['exp']-payload['iat'], 86400, delta=1)
def test_repeated_auth_failures_are_rate_limited_without_secrets(self):
out = io.StringIO()
with contextlib.redirect_stdout(out):
for _ in range(100):
self.scope['verify_token_full']('fixture.invalid.token')
self.assertEqual(out.getvalue().count('[auth]'), 1)
self.assertNotIn('fixture.invalid.token', out.getvalue())
if __name__ == '__main__':
unittest.main(verbosity=2)
+349
View File
@@ -0,0 +1,349 @@
"""Real inert host workers plus actual auth/ticket/viewer boundaries. No APT."""
import ast
import base64
import importlib.util
import io
import json
import os
from pathlib import Path
import shlex
import sys
import tempfile
import threading
import time
from types import ModuleType, SimpleNamespace
import unittest
from unittest.mock import Mock, patch
from flask import Blueprint, Flask, jsonify, request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'AppImage/scripts'))
import host_update_jobs as jobs
def wait(predicate, timeout=6):
limit = time.monotonic() + timeout
while not predicate():
if time.monotonic() >= limit:
raise AssertionError('Inert job did not reach the expected boundary')
time.sleep(.01)
class Jobs(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.folder = Path(self.tmp.name)
self.script, self.gate, self.events = [self.folder / n for n in ('host.sh', 'finish', 'events')]
self.script.write_text(f'echo started >> {shlex.quote(str(self.events))}\n'
f'while [ ! -e {shlex.quote(str(self.gate))} ]; do sleep .02; done\n'
f'echo final-step >> {shlex.quote(str(self.events))}\necho done\n')
self.fixed = patch.object(jobs, 'SCRIPT', str(self.script))
self.fixed.start()
self.addCleanup(self.fixed.stop)
jobs._jobs.clear()
self.callback = Mock()
self.addCleanup(self.finish_all)
def finish_all(self):
self.gate.touch()
for job in list(jobs._jobs.values()):
wait(lambda: job.state == 'complete')
jobs._jobs.clear()
def start(self, identity='job-1'):
return jobs.start(identity, self.callback)
def finish(self, job):
self.gate.touch()
wait(lambda: job.state == 'complete')
def test_real_worker_finishes_post_steps_without_any_viewer(self):
job = self.start()
wait(self.events.exists)
self.assertEqual(self.events.read_text().splitlines(), ['started'])
self.finish(job)
self.assertEqual(self.events.read_text().splitlines(), ['started', 'final-step'])
self.assertEqual(job.exit_code, 0)
self.callback.assert_called_once()
def test_same_id_is_idempotent_and_other_id_cannot_start(self):
first = self.start()
wait(self.events.exists)
with patch.object(jobs.subprocess, 'Popen') as spawn:
self.assertIs(self.start(), first)
with self.assertRaises(jobs.JobConflict):
self.start('other')
self.assertIs(jobs.get(first.id), first)
spawn.assert_not_called()
self.finish(first)
self.assertIs(self.start(), first)
def test_concurrent_start_has_one_process(self):
barrier, results = threading.Barrier(6), []
def run():
barrier.wait()
results.append(self.start())
workers = [threading.Thread(target=run) for _ in range(6)]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
self.assertEqual(len({id(job) for job in results}), 1)
self.finish(results[0])
self.assertEqual(self.events.read_text().splitlines().count('started'), 1)
def test_read_list_and_missing_resume_have_zero_effects(self):
with patch.object(jobs.subprocess, 'Popen') as spawn, patch.object(jobs.tempfile, 'mkstemp') as files:
self.assertEqual(jobs.list_jobs(), [])
for value in ('missing', '../bad', '', None):
with self.subTest(value=value), self.assertRaises((KeyError, ValueError)):
jobs.get(value)
spawn.assert_not_called()
files.assert_not_called()
def test_invalid_start_and_missing_script_spawn_nothing(self):
with patch.object(jobs.subprocess, 'Popen') as spawn, patch.object(jobs.tempfile, 'mkstemp') as files:
for value in ('../bad', 'x'*65, None):
with self.assertRaises(ValueError):
self.start(value)
with patch.object(jobs.os.path, 'isfile', return_value=False):
with self.assertRaises(ValueError):
self.start()
spawn.assert_not_called()
files.assert_not_called()
def test_owner_thread_failure_cannot_spawn_an_unowned_update(self):
with patch.object(jobs.threading.Thread, 'start', side_effect=RuntimeError('thread unavailable')), patch.object(jobs.subprocess, 'Popen') as spawn:
with self.assertRaises(RuntimeError):
self.start()
spawn.assert_not_called()
self.assertEqual(jobs.list_jobs(), [])
def test_popen_failure_is_recorded_not_success_or_automatic_retry(self):
with patch.object(jobs.subprocess, 'Popen', side_effect=OSError('fixture failure')) as spawn:
job = self.start()
wait(lambda: job.state == 'complete')
self.assertEqual(job.exit_code, 1)
self.assertIs(self.start(), job)
spawn.assert_called_once()
self.assertFalse(self.events.exists())
def test_runtime_limit_marks_attention_without_signal_or_second_update(self):
with patch.object(jobs, 'ATTENTION_AFTER', .05), patch.object(jobs.os, 'killpg') as signals:
job = self.start()
wait(lambda: job.state == 'attention')
self.assertIsNone(job.process.poll())
with self.assertRaises(jobs.JobConflict):
self.start('other')
signals.assert_not_called()
self.finish(job)
self.assertEqual(job.exit_code, 0)
def test_explicit_viewer_cancel_and_terminal_control_cannot_kill_host(self):
job = self.start()
wait(self.events.exists)
with patch.object(jobs.os, 'killpg') as signals:
for value in ({'type':'cancel'}, '\x03', 'paste\x1a', '\x1c'):
with self.assertRaises(ValueError):
job.input(value)
signals.assert_not_called()
self.finish(job)
def test_partial_interaction_replays_and_can_only_be_answered_once(self):
job = self.start()
wait(self.events.exists)
identity = 'host-fixture-' + os.urandom(8).hex()
target = Path('/tmp/proxmenux_response_' + identity)
self.addCleanup(lambda: target.unlink(missing_ok=True))
b64 = lambda s: base64.b64encode(s.encode()).decode()
text = f'WEB_INTERACTION:yesno:{identity}:{b64("Confirm")}:{b64("Question")}\n'
job._interactions(io.StringIO(text[:17]))
self.assertIsNone(job.interaction)
job._interactions(io.StringIO(text[17:]))
self.assertEqual(jobs.get(job.id).read(0)['interaction']['id'], identity)
job.input(dict(type='interaction_response', id=identity, value='yes'))
self.assertEqual(target.read_text(), 'yes')
self.assertEqual(target.stat().st_mode & 0o777, 0o600)
with patch.object(jobs.os, 'open') as write:
with self.assertRaises(ValueError):
job.input(dict(type='interaction_response', id=identity, value='yes'))
write.assert_not_called()
self.finish(job)
def test_symlink_response_is_not_overwritten(self):
job = self.start()
wait(self.events.exists)
identity = 'host-fixture-' + os.urandom(8).hex()
target = Path('/tmp/proxmenux_response_' + identity)
target.symlink_to(self.events)
self.addCleanup(lambda: target.unlink(missing_ok=True))
job.interaction = dict(id=identity)
before = self.events.read_bytes()
with self.assertRaises(OSError):
job.input(dict(type='interaction_response', id=identity, value='no'))
self.assertEqual(self.events.read_bytes(), before)
self.finish(job)
def test_finalizing_reserves_host_and_failure_is_not_hidden(self):
entered, release = threading.Event(), threading.Event()
self.addCleanup(release.set)
def finalize(*_):
entered.set()
release.wait(3)
raise RuntimeError('cache failure')
self.callback.side_effect = finalize
job = self.start()
self.gate.touch()
self.assertTrue(entered.wait(3))
with self.assertRaises(jobs.JobConflict):
self.start('other')
release.set()
wait(lambda: job.state == 'complete')
self.assertEqual(job.script_exit_code, 0)
self.assertEqual(job.exit_code, 1)
self.assertTrue(job.finalization_error)
self.callback.assert_called_once()
def test_failed_shell_with_surviving_child_remains_reserved(self):
self.script.write_text(
f'(trap "" HUP; echo child-ready >> {shlex.quote(str(self.events))}; '
f'while [ ! -e {shlex.quote(str(self.gate))} ]; do sleep .02; done; '
f'echo child-finished >> {shlex.quote(str(self.events))}) &\n'
f'while [ ! -s {shlex.quote(str(self.events))} ]; do sleep .01; done\nexit 7\n')
job = self.start()
wait(lambda: job.state == 'attention')
with self.assertRaises(jobs.JobConflict):
self.start('other')
self.gate.touch()
wait(lambda: job.state == 'complete')
self.assertEqual(job.exit_code, 7)
self.assertIn('child-finished', self.events.read_text())
class Routes(unittest.TestCase):
finish_all = Jobs.finish_all
finish = Jobs.finish
def setUp(self):
Jobs.setUp(self)
self.app = Flask(__name__)
auth = ModuleType('auth_manager')
auth.load_auth_config = lambda: {'enabled':True}
auth.verify_token = lambda token: 'fixture' if token=='valid' else None
auth.verify_token_full = lambda token: ('fixture','full_admin') if token=='valid' else ('fixture','read_only') if token=='read-only' else (None,None)
modules = patch.dict(sys.modules, {'auth_manager':auth})
modules.start()
self.addCleanup(modules.stop)
spec = importlib.util.spec_from_file_location('host_jwt_fixture',ROOT/'AppImage/scripts/jwt_middleware.py')
middleware = importlib.util.module_from_spec(spec)
spec.loader.exec_module(middleware)
self.ws_routes = {}
def route(path):
def register(fn):
self.ws_routes[path]=fn
return fn
return register
scope=dict(terminal_bp=Blueprint('host_fixture',__name__),sock=SimpleNamespace(route=route),
request=request,jsonify=jsonify,require_admin_scope=middleware.require_admin_scope,
_TERMINAL_TICKETS={},_TICKETS_LOCK=threading.Lock(),_TICKET_TTL=5,
_TICKET_MAX_INFLIGHT=256,time=time,secrets=__import__('secrets'),json=json,
_host_update_completed=self.callback)
names={'_issue_terminal_ticket','_consume_terminal_ticket','_ws_auth_check',
'issue_terminal_ticket_route','host_update_jobs_status','host_update_viewer'}
nodes=[n for n in ast.parse((ROOT/'AppImage/scripts/flask_terminal_routes.py').read_text()).body if isinstance(n,ast.FunctionDef) and n.name in names]
self.assertEqual(len(nodes),len(names))
exec(compile(ast.Module(body=nodes,type_ignores=[]),'actual-host-routes','exec'),scope)
self.app.register_blueprint(scope['terminal_bp'])
self.client=self.app.test_client()
def ticket(self, token='valid'):
return self.client.post('/api/terminal/ticket',headers={'Authorization':'Bearer '+token})
def attach(self, init, ticket=None, identity='job-1'):
frames=iter([json.dumps(init)])
ws=Mock()
def receive(**_):
try:
return next(frames)
except StopIteration:
raise ConnectionError('viewer signed out')
ws.receive.side_effect=receive
with self.app.test_request_context('/ws/host-update/'+identity+('?ticket='+ticket if ticket else '')):
self.ws_routes['/ws/host-update/<run_id>'](ws,identity)
return ws
def test_401_and_read_only_cannot_list_start_or_get_a_ticket(self):
for token,status in [('expired',401),('read-only',403)]:
with patch.object(jobs.subprocess,'Popen') as spawn:
self.assertEqual(self.ticket(token).status_code,status)
self.assertEqual(self.client.get('/api/host-update-jobs',headers={'Authorization':'Bearer '+token}).status_code,status)
self.attach(dict(resume=False))
spawn.assert_not_called()
self.assertEqual(jobs.list_jobs(),[])
def test_real_route_disconnect_401_then_relogin_replays_one_job(self):
self.attach(dict(resume=False),self.ticket().json['ticket'])
job=jobs.get('job-1')
wait(self.events.exists)
self.assertEqual(self.client.get('/api/host-update-jobs',headers={'Authorization':'Bearer expired'}).status_code,401)
self.assertEqual(self.client.get('/api/host-update-jobs',headers={'Authorization':'Bearer valid'}).json['jobs'][0]['id'],job.id)
with patch.object(jobs.subprocess,'Popen') as spawn:
self.attach(dict(resume=True),self.ticket().json['ticket'])
spawn.assert_not_called()
self.finish(job)
ws=self.attach(dict(resume=True),self.ticket().json['ticket'])
messages=[json.loads(call.args[0]) for call in ws.send.call_args_list]
self.assertEqual(messages[-1],dict(type='script_complete',exit_code=0))
self.assertEqual(self.events.read_text().splitlines(),['started','final-step'])
self.callback.assert_called_once()
def test_missing_resume_extra_params_and_boolean_cursor_spawn_nothing(self):
with patch.object(jobs.subprocess,'Popen') as spawn, patch.object(jobs.tempfile,'mkstemp') as files:
for init in (dict(resume=True),dict(resume=False,params={'LD_PRELOAD':'bad'}),dict(resume=False,offset=True),dict(resume='false')):
self.attach(init,self.ticket().json['ticket'])
spawn.assert_not_called()
files.assert_not_called()
def test_legacy_action_api_keeps_fixed_command_and_cancel_cannot_target_host_job(self):
# Execute the actual existing API functions, with only the systemd
# boundary replaced. They must keep their original unit/argv contract.
source=ast.parse((ROOT/'AppImage/scripts/flask_server.py').read_text())
names={'api_system_pve_update_run','api_system_pve_update_status','api_system_pve_update_cancel'}
nodes=[n for n in source.body if isinstance(n,ast.FunctionDef) and n.name in names]
app=Flask('legacy-host-action-fixture')
run=Mock(return_value=(True,None))
state=Mock(return_value={'status':'running'})
cancel=Mock()
scope=dict(app=app,require_admin_scope=lambda fn:fn,require_auth=lambda fn:fn,
os=os,jsonify=jsonify,_PVE_UPDATE_SCRIPT=str(self.script),
_action_run=run,_action_state=state,_action_cancel=cancel)
exec(compile(ast.Module(body=nodes,type_ignores=[]),'actual-legacy-api','exec'),scope)
client=app.test_client()
self.assertEqual(client.post('/api/system/pve-update/run').status_code,202)
run.assert_called_once_with('proxmenux-action-pve-update',['/bin/bash',str(self.script)],
description='ProxMenux action: Proxmox VE update')
self.assertEqual(client.get('/api/system/pve-update/status').status_code,200)
self.assertEqual(client.delete('/api/system/pve-update').status_code,200)
cancel.assert_called_once_with('proxmenux-action-pve-update')
self.assertEqual(jobs.list_jobs(),[])
def test_actual_host_finalizer_invalidates_health_without_viewer_or_package_calls(self):
from types import SimpleNamespace
keys=('updates_check','overall_health','_bg_overall','_bg_detailed')
cache=SimpleNamespace(last_check_times=dict.fromkeys((*keys,'unrelated'),123),
cached_results=dict.fromkeys((*keys,'unrelated'),{'status':'old'}))
module=ModuleType('health_monitor')
module.health_monitor=cache
node=next(n for n in ast.parse((ROOT/'AppImage/scripts/flask_terminal_routes.py').read_text()).body
if isinstance(n,ast.FunctionDef) and n.name=='_host_update_completed')
scope={}
exec(compile(ast.Module(body=[node],type_ignores=[]),'actual-finalizer','exec'),scope)
with patch.dict(sys.modules,{'health_monitor':module}),patch.object(jobs.subprocess,'Popen') as spawn:
scope['_host_update_completed'](jobs.SCRIPT,{},0,1)
spawn.assert_not_called()
self.assertEqual(cache.last_check_times,{'unrelated':123})
self.assertEqual(cache.cached_results,{'unrelated':{'status':'old'}})
if __name__=='__main__':
unittest.main(verbosity=2)
+204
View File
@@ -0,0 +1,204 @@
"""Actual wrapper + worker, with every machine effect replaced by local fixtures.
Only fixed filesystem locations are redirected. Package commands and reboot
are inert executables which record calls; no root, network or host update.
"""
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import unittest
ROOT = Path(__file__).resolve().parents[1]
class HostUpdateShell(unittest.TestCase):
def test_packaging_copies_required_modules_with_no_host_probe_on_import(self):
source = (ROOT / 'AppImage/scripts/build_appimage.sh').read_text().splitlines()
copies = [f'cp "$SCRIPT_DIR/{name}.py" "$APP_DIR/usr/bin/"'
for name in ('host_update_jobs', 'kernel_reboot')]
for line in copies:
self.assertIn(line, source)
app = self.folder / 'AppDir'
(app / 'usr/bin').mkdir(parents=True)
result = subprocess.run(['/bin/bash', '-c', '\n'.join(copies)],
env=dict(os.environ, SCRIPT_DIR=str(ROOT / 'AppImage/scripts'), APP_DIR=str(app)),
capture_output=True, text=True, timeout=3)
self.assertEqual(result.returncode, 0, result.stderr)
code = ('import sys; from unittest.mock import patch; sys.path.insert(0,sys.argv[1]); '
'guard=patch("subprocess.Popen",side_effect=AssertionError("forbidden probe")); '
'guard.start(); import host_update_jobs,kernel_reboot; '
'assert host_update_jobs.list_jobs()==[]')
result = subprocess.run([sys.executable, '-I', '-B', '-c', code, str(app / 'usr/bin')],
capture_output=True, text=True, timeout=3)
self.assertEqual(result.returncode, 0, result.stderr)
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix='host-update-shell-')
self.addCleanup(self.temp.cleanup)
self.folder = Path(self.temp.name)
self.bin = self.folder / 'bin'
self.bin.mkdir()
scripts = self.folder / 'scripts'
(scripts / 'utilities').mkdir(parents=True)
(scripts / 'global').mkdir()
self.calls = self.folder / 'calls'
self.calls.touch()
self.lock = self.folder / 'update.lock'
for name in ('utilities/proxmox_update.sh', 'global/update-pve-safe.sh'):
text = (ROOT / 'scripts' / name).read_text()
text = text.replace('/usr/local/share/proxmenux', str(self.folder))
text = text.replace('/run/proxmenux-host-update.lock', str(self.lock))
text = text.replace('/var/run/proxmenux-update-', str(self.folder / 'marker-'))
text = text.replace('/var/run/reboot-required', str(self.folder / 'reboot-required'))
text = text.replace('/var/log/proxmox-update-', str(self.folder / 'update-'))
text = text.replace('/tmp/proxmenux_screen_capture_', str(self.folder / 'capture-'))
(scripts / name).write_text(text)
(self.folder / 'utils.sh').write_text('''
load_language() { :; }
initialize_cache() { :; }
translate() { printf '%s' "$1"; }
msg_error() { echo "$*"; }
msg_warn() { echo "$*"; }
msg_info() { echo "$*"; }
msg_info2() { echo "$*"; }
msg_ok() { echo "$*"; }
msg_title() { echo "$*"; }
msg_success() { echo "$*"; }
show_proxmenux_logo() { :; }
LOCAL_SCRIPTS="$FIXTURE/scripts"
''')
(scripts / 'global/common-functions.sh').write_text('''
pmx_format_upgradable() { cat; }
pmx_dpkg_changes_since() { echo 'package changes fixture'; }
lvm_repair_check() { echo lvm-poststep >> "$CALLS"; }
pmx_rebuild_dkms_after_kernel() { echo dkms-poststep >> "$CALLS"; }
''')
(scripts / 'global/utils-install-functions.sh').write_text('ensure_repositories() { return 0; }\n')
commands = {
'pveversion': "echo 'pve-manager/9.2.24/test (running kernel: 7.0.14-25-pve)'",
'df': "printf 'Filesystem 1K-blocks Used Available Use%% Mounted\\nfixture 9999999 1 9999999 1%% /\\n'",
'uname': "echo 7.0.14-25-pve",
'dpkg-query': "printf 'install ok installed\\tproxmox-kernel-7.0.14-26-pve-signed\\n'",
'apt-cache': "echo 'Candidate: 9.2.24'",
'apt-get': 'echo "apt-get $*" >> "$CALLS"; exit 0',
'apt': '''echo "apt $*" >> "$CALLS"
if [[ "$1" == list ]]; then
printf 'Listing...\\npve-manager/stable 9.2.24 amd64 [upgradable from: 9.2.21]\\n'
else
touch "$FIXTURE/upgrade-started"
while [[ ! -e "$FIXTURE/release" ]]; do sleep .02; done
exit "${UPGRADE_EXIT:-0}"
fi''',
'dialog': 'echo dialog >> "$CALLS"; exit "${CONFIRM_EXIT:-0}"',
'whiptail': 'echo "whiptail $*" >> "$CALLS"; exit 1',
'reboot': 'echo FORBIDDEN-REBOOT >> "$CALLS"; exit 99',
'clear': ':',
}
for name, body in commands.items():
path = self.bin / name
path.write_text('#!/bin/bash\n' + body + '\n')
path.chmod(0o700)
self.env = dict(os.environ, PATH=str(self.bin)+':'+os.environ['PATH'],
FIXTURE=str(self.folder), CALLS=str(self.calls),
EXECUTION_MODE='web', PROXMENUX_HOST_UPDATE_JOB='1')
self.wrapper = scripts / 'utilities/proxmox_update.sh'
self.procs = []
self.addCleanup(self.finish)
def finish(self):
(self.folder / 'release').touch()
for proc in self.procs:
if proc.poll() is None:
proc.communicate('\n', timeout=8)
def start(self, **env):
proc = subprocess.Popen(['/bin/bash', str(self.wrapper)], env=dict(self.env, **env),
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True)
self.procs.append(proc)
return proc
def wait_upgrade(self):
until = time.monotonic() + 5
while not (self.folder / 'upgrade-started').exists():
self.assertLess(time.monotonic(), until, 'inert upgrade did not start')
time.sleep(.01)
def test_web_full_flow_finishes_poststeps_and_footer_without_reboot_or_keypress(self):
proc = self.start()
self.wait_upgrade()
(self.folder / 'release').touch()
output, _ = proc.communicate(timeout=8)
self.assertEqual(proc.returncode, 0, output)
calls = self.calls.read_text()
self.assertIn('lvm-poststep', calls)
self.assertIn('dkms-poststep', calls)
self.assertIn('apt-get -y autoremove', calls)
self.assertIn('apt-get -y autoclean', calls)
self.assertNotIn('FORBIDDEN-REBOOT', calls)
self.assertNotIn('whiptail', calls)
self.assertIn('Reboot Required', output)
self.assertIn('You can reboot later manually.', output)
self.assertFalse((self.folder / 'marker-in-progress').exists())
self.assertTrue((self.folder / 'marker-just-finished').exists())
log = next(self.folder.glob('update-*.log')).read_text()
self.assertIn('--- Packages changed (exit 0) ---', log)
self.assertIn('Finished:', log)
def test_cancel_before_upgrade_has_no_upgrade_cleanup_or_reboot(self):
proc = self.start(CONFIRM_EXIT='1')
output, _ = proc.communicate(timeout=5)
self.assertEqual(proc.returncode, 130, output)
calls = self.calls.read_text()
for forbidden in ('full-upgrade', 'autoremove', 'autoclean', 'REBOOT', 'poststep'):
self.assertNotIn(forbidden, calls)
def test_failure_keeps_failure_and_never_offers_reboot(self):
(self.folder / 'release').touch()
proc = self.start(UPGRADE_EXIT='1')
output, _ = proc.communicate(timeout=5)
self.assertEqual(proc.returncode, 1, output)
for forbidden in ('autoremove', 'autoclean', 'REBOOT', 'poststep', 'whiptail'):
self.assertNotIn(forbidden, self.calls.read_text())
def test_old_menu_cli_contract_retains_explicit_reboot_prompt(self):
(self.folder / 'release').touch()
proc = self.start(PROXMENUX_HOST_UPDATE_JOB='', EXECUTION_MODE='cli')
output, _ = proc.communicate('\n', timeout=5)
self.assertEqual(proc.returncode, 0, output)
self.assertIn('whiptail --title Reboot Required --yesno', self.calls.read_text())
self.assertNotIn('FORBIDDEN-REBOOT', self.calls.read_text())
def test_shared_lock_rejects_second_legacy_or_web_caller_before_any_package_call(self):
first = self.start()
self.wait_upgrade()
before = self.calls.read_text()
for mode in ('cli', 'web'):
second = self.start(EXECUTION_MODE=mode)
output, _ = second.communicate(timeout=3)
self.assertEqual(second.returncode, 75, output)
self.assertEqual(self.calls.read_text(), before)
self.assertIn('already running', output)
self.assertIsNone(first.poll())
def test_lock_does_not_follow_symlink_or_truncate_existing_file(self):
target = self.folder / 'protected'
target.write_text('preserve')
self.lock.symlink_to(target)
proc = self.start()
output, _ = proc.communicate(timeout=3)
self.assertEqual(proc.returncode, 75, output)
self.assertEqual(target.read_text(), 'preserve')
self.assertEqual(self.calls.read_text(), '')
self.lock.unlink()
self.lock.write_text('existing metadata')
proc = self.start(CONFIRM_EXIT='1')
proc.communicate(timeout=5)
self.assertEqual(self.lock.read_text(), 'existing metadata')
if __name__ == '__main__':
unittest.main(verbosity=2)
+107
View File
@@ -0,0 +1,107 @@
"""Real HTTP + WebSocket + PTY under the same gevent patch as Monitor.
Authentication boundaries use synthetic identities; full JWT verification is
covered separately. The sole executable is an inert temporary shell script.
"""
from pathlib import Path
import subprocess
import sys
import unittest
ROOT = Path(__file__).resolve().parents[1]
def wire_fixture():
import json
import shlex
import tempfile
import time
from types import ModuleType
import requests
from flask import Flask
from gevent.pywsgi import WSGIServer
from simple_websocket import Client
sys.path.insert(0, str(ROOT / 'AppImage/scripts'))
auth = ModuleType('auth_manager')
auth.load_auth_config = lambda: {'enabled': True}
auth.verify_token = lambda token: 'fixture' if token == 'valid' else None
auth.verify_token_full = lambda token: ('fixture', 'full_admin') if token == 'valid' else ('fixture', 'read_only') if token == 'read-only' else (None, None)
sys.modules['auth_manager'] = auth
import flask_terminal_routes as routes
import host_update_jobs as jobs
callbacks = []
routes._host_update_completed = lambda *args: callbacks.append(args)
app = Flask(__name__)
app.register_blueprint(routes.terminal_bp)
routes.sock.init_app(app)
server = WSGIServer(('127.0.0.1', 0), app, log=None)
server.start()
base = f'http://127.0.0.1:{server.server_port}'
headers = {'Authorization': 'Bearer valid'}
def wait(predicate):
end = time.monotonic() + 5
while not predicate():
assert time.monotonic() < end, 'fixture timeout'
time.sleep(.01)
def ticket():
response = requests.post(base + '/api/terminal/ticket', headers=headers, timeout=3)
assert response.status_code == 200
return response.json()['ticket']
def connect(identity, resume):
client = Client(base.replace('http:', 'ws:') + '/ws/host-update/' + identity + '?ticket=' + ticket())
client.send(json.dumps(dict(resume=resume, offset=0)))
return client
try:
with tempfile.TemporaryDirectory(prefix='host-update-wire-') as folder:
folder = Path(folder)
gate, events, script = [folder / n for n in ('release', 'events', 'update.sh')]
script.write_text(f'echo start >> {shlex.quote(str(events))}\n'
f'while [ ! -f {shlex.quote(str(gate))} ]; do sleep .02; done\n'
f'echo poststep >> {shlex.quote(str(events))}\necho completed\n')
jobs.SCRIPT = str(script)
viewer = connect('one-host-run', False)
wait(events.exists)
if viewer.connected:
viewer.close()
assert requests.get(base + '/api/host-update-jobs', headers={'Authorization': 'Bearer expired'}, timeout=3).status_code == 401
assert requests.post(base + '/api/terminal/ticket', headers={'Authorization': 'Bearer read-only'}, timeout=3).status_code == 403
listed = requests.get(base + '/api/host-update-jobs', headers=headers, timeout=3).json()['jobs']
assert listed[0]['id'] == 'one-host-run'
viewer = connect('one-host-run', True)
gate.touch()
frames = []
while not frames or frames[-1]['type'] != 'script_complete':
message = viewer.receive(timeout=5)
assert message is not None
frames.append(json.loads(message))
assert frames[-1]['exit_code'] == 0
assert events.read_text().splitlines() == ['start', 'poststep']
assert len(callbacks) == 1
assert 'completed' in ''.join(frame.get('text', '') for frame in frames)
if viewer.connected:
viewer.close()
viewer = connect('missing-host-run', True)
assert json.loads(viewer.receive(timeout=3))['type'] == 'job_unavailable'
if viewer.connected:
viewer.close()
assert len(jobs.list_jobs()) == 1
assert len(callbacks) == 1
finally:
server.stop(timeout=2)
print('PASS real gevent HTTP/WS disconnect, 401, relogin, resume, PTY poststep and missing-ID no restart')
class HostWire(unittest.TestCase):
def test_actual_gevent_transport_survives_viewer_loss(self):
command = ('from gevent import monkey; monkey.patch_all(); '
'import runpy,sys; runpy.run_path(sys.argv[1],run_name="__wire__")')
result = subprocess.run([sys.executable, '-I', '-B', '-c', command, str(Path(__file__).resolve())],
capture_output=True, text=True, timeout=25)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('PASS real gevent', result.stdout)
if __name__ == '__wire__':
wire_fixture()
elif __name__ == '__main__':
unittest.main(verbosity=2)
+63
View File
@@ -0,0 +1,63 @@
// Execute actual transport initializers, reconnect handlers and routing.
const assert = require('node:assert/strict')
const fs = require('node:fs'), path = require('node:path'), vm = require('node:vm')
const app = path.resolve(__dirname, '../AppImage')
const ts = require(path.join(app, 'node_modules/typescript'))
const text = fs.readFileSync(path.join(app, 'components/script-terminal-modal.tsx'), 'utf8')
const tree = ts.createSourceFile('modal.tsx', text, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX)
const opens = [], messages = [], closes = []
let route, cleanup
function walk(n) {
if (ts.isBinaryExpression(n) && n.left.getText(tree) === 'ws.onopen') opens.push(n.right.getText(tree))
if (ts.isBinaryExpression(n) && n.left.getText(tree) === 'ws.onmessage') messages.push(n.right.getText(tree))
if (ts.isBinaryExpression(n) && n.left.getText(tree) === 'ws.onclose') closes.push(n.right.getText(tree))
if (ts.isVariableDeclaration(n) && n.name.getText(tree) === 'getScriptWebSocketUrl') route = n.initializer.getText(tree)
if (ts.isCallExpression(n) && n.expression.getText(tree) === 'useEffect'
&& n.arguments[0]?.getText(tree).includes('Transport mode is fixed')) cleanup = n.arguments[0].getText(tree)
ts.forEachChild(n, walk)
}
walk(tree)
const compile = (code, scope) => vm.runInNewContext(ts.transpileModule('('+code+')', {
compilerOptions:{target:ts.ScriptTarget.ES2020,module:ts.ModuleKind.CommonJS},
}).outputText, scope)
assert.equal(opens.length, 2)
for (const [index, open] of opens.entries()) {
for (const resume of [false, true]) {
const sent = [], scope = {backgroundUpdate:{runId:'same-id',kind:'host',resume},
scriptPath:'/forbidden-client-path', paramsRef:{current:{FORBIDDEN:'client-env'}},
outputOffsetRef:{current:42}, setConnectionStatus(){}, reconnectAttemptsRef:{current:0},
keepAliveIntervalRef:{current:null}, clearInterval(){}, setInterval(){return 1}, setTimeout(){},
WebSocket:{OPEN:1}, ws:{readyState:1,send:s=>sent.push(JSON.parse(s))}}
compile(open, scope)()
assert.equal(sent.length,1)
assert.deepEqual(sent[0],{resume:index===0 ? true : resume, offset:42},
'reconnect must attach; explicit initial start is the sole resume=false message')
}
}
for (const backgroundUpdate of [{runId:'same-id',kind:'host'}, {runId:'lxc-id',vmid:101}, undefined]) {
const actual = compile(route,{backgroundUpdate,API_PORT:'8008'})('legacy-id')
assert.equal(actual,'ws://localhost:8008/ws/'+(backgroundUpdate?.kind==='host'?'host-update/same-id':backgroundUpdate?'lxc-update/lxc-id':'script/legacy-id'))
}
for (const close of closes) {
const sent=[], results=[], scope={backgroundUpdate:{runId:'same-id',kind:'host'},
ws:{send:s=>sent.push(s)}, isOpenRef:{current:true}, isComplete:false,
completionReceivedRef:{current:false}, keepAliveIntervalRef:{current:null},
reconnectAttemptsRef:{current:0},reconnectTimeoutRef:{current:null},setTimeout(fn){fn();return 1},clearTimeout(){},
setConnectionStatus(){}, clearInterval(){}, termRef:{current:{writeln(){}}},term:{writeln(){}},
attemptReconnect(){results.push('attach')}, onCompleteRef:{current:()=>results.push('complete')},t:s=>s}
compile(close,scope)({})
assert.deepEqual(results,['attach'])
assert.deepEqual(sent,[])
}
const sent=[], scope={backgroundUpdate:{runId:'same-id',kind:'host'},
openGenerationRef:{current:0}, isOpenRef:{current:true}, wsRef:{current:{close(){},send:x=>sent.push(x)}},
termRef:{current:{dispose(){}}}, clearTimeout(){},clearInterval(){},
waitingTimeoutRef:{current:null}, reconnectTimeoutRef:{current:null}, keepAliveIntervalRef:{current:null},checkConnectionInterval:{current:null}}
compile(cleanup,scope)()()
assert.deepEqual(sent,[],'unmount cannot send cancellation or restart')
const health=fs.readFileSync(path.join(app,'components/health-status-modal.tsx'),'utf8')
const en=JSON.parse(fs.readFileSync(path.join(app,'messages/en/common.json'),'utf8'))
const sk=JSON.parse(fs.readFileSync(path.join(app,'messages/sk/common.json'),'utf8'))
assert.deepEqual(Object.keys(en.healthStatus.hostUpdate).sort(),Object.keys(sk.healthStatus.hostUpdate).sort())
assert.ok(health.includes('Number(categoryData?.count ?? 0) > 0'),'reboot-only state has no Update Now action')
console.log('PASS host initial/resume transport, legacy/LXC routes, disconnect/unmount and EN/SK keys')
+134
View File
@@ -0,0 +1,134 @@
"""Real dpkg contract shape and actual cached HealthMonitor method. No APT."""
import ast
import copy
from pathlib import Path
import subprocess
import sys
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'AppImage/scripts'))
import kernel_reboot as kernel
def listing(*releases):
return '\n'.join('installed\tproxmox-kernel-'+v+'-signed' for v in releases)+'\n'
class Reboot(unittest.TestCase):
def setUp(self):
self.uname=patch.object(kernel.os,'uname',return_value=SimpleNamespace(release='7.0.14-25-pve'))
self.uname.start()
self.addCleanup(self.uname.stop)
self.marker=patch.object(Path,'exists',return_value=False)
self.marker.start()
self.addCleanup(self.marker.stop)
def check(self, output, rc=0):
with patch.object(kernel.subprocess,'run',return_value=subprocess.CompletedProcess([],rc,output,'')) as call:
result=kernel.reboot_status()
if call.called:
self.assertEqual(call.call_args.args[0][0],'dpkg-query')
self.assertNotIn('apt',call.call_args.args[0])
return result
def test_installed_26_running_25_requires_reboot_without_marker_or_pending_updates(self):
result=self.check(listing('7.0.14-25-pve','7.0.14-26-pve'))
self.assertEqual(result['state'],'required')
self.assertEqual(result['installed_kernel'],'7.0.14-26-pve')
def test_old_kernel_meta_and_removed_config_are_not_new_installed_kernel(self):
text=listing('7.0.14-25-pve','6.17.13-21-pve')+'installed\tproxmox-kernel-7.0\nconfig-files\tproxmox-kernel-7.0.14-26-pve-signed\n'
self.assertEqual(self.check(text)['state'],'current')
def test_reboot_into_new_kernel_clears_the_requirement(self):
with patch.object(kernel.os,'uname',return_value=SimpleNamespace(release='7.0.14-26-pve')):
self.assertEqual(self.check(listing('7.0.14-25-pve','7.0.14-26-pve'))['state'],'current')
def test_version_comparison_is_numeric(self):
self.assertEqual(self.check(listing('7.0.14-9-pve','7.0.14-100-pve'))['state'],'required')
def test_legacy_pve_package_and_arch_suffix_are_supported(self):
self.assertEqual(self.check('installed\tpve-kernel-7.0.14-26-pve:amd64\n')['state'],'required')
def test_missing_empty_malformed_and_partial_are_unknown_not_zero(self):
for text in ('','garbage','installed\tother-package\n','half-configured\tproxmox-kernel-7.0.14-26-pve-signed\n','bad-state\tfoo\n'):
with self.subTest(text=text):
self.assertEqual(self.check(text)['state'],'unknown')
self.assertEqual(self.check(listing('7.0.14-26-pve'),rc=1)['state'],'unknown')
def test_timeout_and_read_error_are_unknown(self):
for error in (subprocess.TimeoutExpired('dpkg-query',5),OSError('unreadable')):
with patch.object(kernel.subprocess,'run',side_effect=error):
self.assertEqual(kernel.reboot_status()['state'],'unknown')
def test_unavailable_or_invalid_uname_never_probes_packages(self):
with patch.object(kernel.os,'uname',side_effect=OSError('unknown')),patch.object(kernel.subprocess,'run') as packages:
self.assertEqual(kernel.reboot_status()['state'],'unknown')
packages.assert_not_called()
for release in ('','invalid','7.0.14--pve'):
with patch.object(kernel.os,'uname',return_value=SimpleNamespace(release=release)),patch.object(kernel.subprocess,'run') as packages:
self.assertEqual(kernel.reboot_status()['state'],'unknown')
packages.assert_not_called()
def test_supported_marker_never_requires_a_package_probe(self):
with patch.object(Path,'exists',return_value=True),patch.object(kernel.subprocess,'run') as packages:
self.assertEqual(kernel.reboot_status()['state'],'required')
packages.assert_not_called()
def test_non_pve_kernel_is_not_a_failed_pve_read(self):
with patch.object(kernel.os,'uname',return_value=SimpleNamespace(release='6.12.0-amd64')),patch.object(kernel.subprocess,'run') as packages:
self.assertEqual(kernel.reboot_status()['state'],'not_applicable')
packages.assert_not_called()
def test_overlay_does_not_mutate_cache_or_hide_existing_error(self):
for original in ('OK','UNKNOWN','WARNING','CRITICAL'):
cache=dict(status=original,count=0,checks={'pending_updates':dict(status='OK')})
before=copy.deepcopy(cache)
result=kernel.with_reboot_status(cache,dict(state='required',running_kernel='old',installed_kernel='new'))
self.assertEqual(cache,before)
self.assertTrue(result['reboot_required'])
self.assertEqual(result['status'],'INFO' if original=='OK' else original)
result=kernel.with_reboot_status(dict(status='OK',checks={}),dict(state='unknown',running_kernel='old',installed_kernel=None))
self.assertEqual(result['status'],'UNKNOWN')
self.assertIsNone(result['reboot_required'])
def test_actual_cached_health_path_refreshes_reboot_evidence_without_apt(self):
source=ast.parse((ROOT/'AppImage/scripts/health_monitor.py').read_text())
node=next(n for n in ast.walk(source) if isinstance(n,ast.FunctionDef) and n.name=='_check_updates')
node.decorator_list=[]
import __future__
scope=dict(os=kernel.os,time=time,subprocess=kernel.subprocess)
exec(compile(ast.Module(body=[node],type_ignores=[]),'actual-health','exec',flags=__future__.annotations.compiler_flag),scope)
for age in (0,599.999):
cache=dict(status='OK',count=0,checks={})
owner=SimpleNamespace(last_check_times={'updates_check':time.time()-age},cached_results={'updates_check':cache})
with patch.object(kernel.os.path,'exists',return_value=True),patch.object(kernel.os.path,'getmtime',return_value=0),patch.object(kernel.subprocess,'run',return_value=subprocess.CompletedProcess([],0,listing('7.0.14-26-pve'),'')) as probe:
result=scope['_check_updates'](owner)
self.assertEqual(result['status'],'INFO')
self.assertTrue(result['reboot_required'])
self.assertEqual(cache,dict(status='OK',count=0,checks={}))
self.assertEqual([call.args[0][0] for call in probe.call_args_list],['dpkg-query'])
def test_actual_health_failure_paths_keep_independent_reboot_evidence(self):
source=ast.parse((ROOT/'AppImage/scripts/health_monitor.py').read_text())
node=next(n for n in ast.walk(source) if isinstance(n,ast.FunctionDef) and n.name=='_check_updates')
import __future__
scope=dict(os=kernel.os,time=time,subprocess=kernel.subprocess)
exec(compile(ast.Module(body=[node],type_ignores=[]),'actual-health','exec',flags=__future__.annotations.compiler_flag),scope)
for age in (600,600.001):
for error in (subprocess.TimeoutExpired('apt-get',30),OSError('fixture-unavailable')):
owner=SimpleNamespace(last_check_times={'updates_check':time.time()-age},cached_results={})
def command(args, **kwargs):
if args[0]=='dpkg-query':
return subprocess.CompletedProcess(args,0,listing('7.0.14-26-pve'),'')
raise error
with patch.object(kernel.os.path,'exists',return_value=False),patch.object(kernel.subprocess,'run',side_effect=command) as probe:
result=scope['_check_updates'](owner)
self.assertEqual(result['status'],'UNKNOWN')
self.assertTrue(result['reboot_required'])
self.assertEqual([call.args[0][0] for call in probe.call_args_list],['dpkg-query','apt-get'])
if __name__=='__main__':
unittest.main(verbosity=2)