mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 10:52:22 +00:00
Preserve host updates across sign-out and report pending kernel activation
This commit is contained in:
@@ -41,6 +41,7 @@ import {
|
||||
HelpCircle,
|
||||
} from "lucide-react"
|
||||
import { ScriptTerminalModal } from "./script-terminal-modal"
|
||||
import { HostUpdateJobs, type HostUpdateJob } from "./host-update-jobs"
|
||||
import { useT } from "@/lib/i18n/provider"
|
||||
import { useIsPbs } from "@/lib/use-platform"
|
||||
|
||||
@@ -129,6 +130,8 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
const [dismissingKey, setDismissingKey] = useState<string | null>(null)
|
||||
const [expandedCategories, setExpandedCategories] = useState<Set<string>>(new Set())
|
||||
const [showUpdateTerminal, setShowUpdateTerminal] = useState(false)
|
||||
const [hostJob, setHostJob] = useState<{ runId: string; resume: boolean; kind: "host" } | null>(null)
|
||||
const [hostJobs, setHostJobs] = useState<HostUpdateJob[]>([])
|
||||
|
||||
const fetchHealthDetails = useCallback(async (force = false) => {
|
||||
setLoading(true)
|
||||
@@ -330,6 +333,9 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
"Active": t("healthStatus.details.active"),
|
||||
"UP": t("healthStatus.details.up"),
|
||||
"Kernel/PVE up to date": t("healthStatus.details.kernelUpToDate"),
|
||||
"No reboot required": t("healthStatus.hostUpdate.noReboot"),
|
||||
"Reboot required by the operating system": t("healthStatus.hostUpdate.rebootMarker"),
|
||||
"Could not determine whether a reboot is required": t("healthStatus.hostUpdate.rebootUnknown"),
|
||||
"Proxmox VE is up to date": t("healthStatus.details.proxmoxUpToDate"),
|
||||
"No security updates pending": t("healthStatus.details.noSecurityUpdates"),
|
||||
"No container startup errors": t("healthStatus.details.noContainerErrors"),
|
||||
@@ -341,6 +347,8 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
if (Object.prototype.hasOwnProperty.call(exact, value)) return exact[value]
|
||||
|
||||
let match = value.match(/^Latency ([\d.]+)ms to gateway$/)
|
||||
const reboot = value.match(/^Reboot required: running (.+); installed (.+)$/)
|
||||
if (reboot) return t("healthStatus.hostUpdate.rebootKernel", { running: reboot[1], installed: reboot[2] })
|
||||
if (match) return t("healthStatus.details.gatewayLatency", { latency: match[1] })
|
||||
match = value.match(/^(\d+) failed login attempts in 24h$/)
|
||||
if (match) return t("healthStatus.details.failedLogins", { count: match[1] })
|
||||
@@ -623,7 +631,7 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
"smart_health", "io_errors", "zfs_pools", "lvm_volumes", "lvm_check", "connectivity",
|
||||
"qmp_communication", "container_startup", "vm_startup", "oom_killer", "cluster_mode",
|
||||
"log_error_cascade", "log_error_spike", "log_persistent_errors", "log_critical_errors",
|
||||
"pve_version", "security_updates", "system_age", "pending_updates", "kernel_pve", "uptime",
|
||||
"pve_version", "security_updates", "system_age", "pending_updates", "kernel_pve", "uptime", "reboot_required",
|
||||
"certificates", "login_attempts", "fail2ban", "proxmox_storages",
|
||||
])
|
||||
if (knownKeys.has(key)) return t(`healthStatus.checks.${key}`)
|
||||
@@ -711,6 +719,13 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<HostUpdateJobs open={open} activeId={showUpdateTerminal ? hostJob?.runId : undefined}
|
||||
onJobs={setHostJobs} onSettled={() => { void fetchHealthDetails(true) }}
|
||||
onResume={job => {
|
||||
setHostJob({ runId: job.id, resume: true, kind: "host" })
|
||||
setShowUpdateTerminal(true)
|
||||
}} />
|
||||
|
||||
{loading && (
|
||||
<div className="flex items-center justify-center py-8">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-primary" />
|
||||
@@ -845,11 +860,17 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
pending prevents the operator from spawning a
|
||||
terminal that would only report "System is
|
||||
already up to date". */}
|
||||
{key === "updates" && status?.toUpperCase() !== "OK" && (
|
||||
{key === "updates" && Number(categoryData?.count ?? 0) > 0 && (
|
||||
<div className="flex justify-end px-3 py-2 pt-1">
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={() => setShowUpdateTerminal(true)}
|
||||
disabled={hostJobs.some(job => job.state !== "complete")}
|
||||
onClick={() => {
|
||||
const runId = window.crypto?.randomUUID?.()
|
||||
|| `host-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`
|
||||
setHostJob({ runId, resume: false, kind: "host" })
|
||||
setShowUpdateTerminal(true)
|
||||
}}
|
||||
className="bg-purple-600/15 hover:bg-purple-600/25 border border-purple-500/40 text-purple-300 hover:text-purple-200"
|
||||
>
|
||||
<ArrowUpCircle className="h-4 w-4 mr-1.5" />
|
||||
@@ -968,7 +989,11 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu
|
||||
)}
|
||||
</DialogContent>
|
||||
<ScriptTerminalModal
|
||||
key={hostJob?.runId || "host-update"}
|
||||
open={showUpdateTerminal}
|
||||
backgroundUpdate={hostJob || undefined}
|
||||
cancelledExitCode={130}
|
||||
onComplete={() => { void fetchHealthDetails(true) }}
|
||||
onClose={() => {
|
||||
setShowUpdateTerminal(false)
|
||||
// Force a fresh read (cache-busting via ?refresh=1) so the
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"use client"
|
||||
|
||||
import { useEffect, useRef, useState } from "react"
|
||||
import { Button } from "./ui/button"
|
||||
import { fetchApi } from "../lib/api-config"
|
||||
import { useT } from "../lib/i18n/provider"
|
||||
|
||||
export type HostUpdateJob = {
|
||||
id: string
|
||||
state: "starting" | "running" | "attention" | "finalizing" | "complete"
|
||||
exit_code: number | null
|
||||
}
|
||||
|
||||
export function HostUpdateJobs({ open, activeId, onResume, onJobs, onSettled }: {
|
||||
open: boolean
|
||||
activeId?: string
|
||||
onResume: (job: HostUpdateJob) => void
|
||||
onJobs: (jobs: HostUpdateJob[]) => void
|
||||
onSettled: () => void
|
||||
}) {
|
||||
const t = useT()
|
||||
const [jobs, setJobs] = useState<HostUpdateJob[]>([])
|
||||
const callbacks = useRef({ onJobs, onSettled })
|
||||
callbacks.current = { onJobs, onSettled }
|
||||
const observed = useRef(new Set<string>())
|
||||
useEffect(() => {
|
||||
if (!open) return
|
||||
let closed = false
|
||||
const read = async () => {
|
||||
try {
|
||||
const data = await fetchApi<{ jobs: HostUpdateJob[] }>("/api/host-update-jobs")
|
||||
if (closed) return
|
||||
const next = Array.isArray(data.jobs) ? data.jobs : []
|
||||
setJobs(next)
|
||||
callbacks.current.onJobs(next)
|
||||
for (const job of next) {
|
||||
if (job.state === "complete" && !observed.current.has(job.id)) {
|
||||
observed.current.add(job.id)
|
||||
callbacks.current.onSettled()
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Authentication recovery belongs to fetchApi. Never retry a start.
|
||||
}
|
||||
}
|
||||
void read()
|
||||
const timer = setInterval(() => { void read() }, 5000)
|
||||
return () => { closed = true; clearInterval(timer) }
|
||||
}, [open])
|
||||
const visible = jobs.filter(job => job.id !== activeId)
|
||||
if (!visible.length) return null
|
||||
return <div className="space-y-2" role="status">
|
||||
{visible.map(job => <div key={job.id}
|
||||
className="flex flex-wrap items-center justify-between gap-3 rounded-lg border border-violet-500/30 bg-violet-500/5 p-3">
|
||||
<p className="text-sm">{t(`healthStatus.hostUpdate.${job.state === "complete" ? "finished" : job.state === "attention" ? "attention" : "running"}`)}</p>
|
||||
<Button size="sm" variant="outline" onClick={() => onResume(job)}>
|
||||
{t("healthStatus.hostUpdate.view")}
|
||||
</Button>
|
||||
</div>)}
|
||||
</div>
|
||||
}
|
||||
@@ -54,8 +54,8 @@ interface ScriptTerminalModalProps {
|
||||
completedSuccessfullyMessage?: string
|
||||
completedWithErrorMessage?: (exitCode: number) => string
|
||||
cancelledExitCode?: number
|
||||
// Only LXC Updates opts into a separate server-owned job transport.
|
||||
backgroundUpdate?: { runId: string; vmid: number; resume: boolean }
|
||||
// Explicit server-owned transports; ordinary /ws/script keeps its contract.
|
||||
backgroundUpdate?: { runId: string; vmid?: number; resume: boolean; kind?: "host" }
|
||||
// Optional callback fired when the script's WebSocket closes
|
||||
// (script_runner sends an exit code and then closes). Lets the
|
||||
// parent auto-dismiss the modal — used by host-backup's Restore
|
||||
@@ -178,7 +178,9 @@ export function ScriptTerminalModal({
|
||||
}
|
||||
}, 30000)
|
||||
|
||||
const initMessage = {
|
||||
const initMessage = backgroundUpdate?.kind === "host" ? {
|
||||
resume: true, offset: outputOffsetRef.current,
|
||||
} : {
|
||||
script_path: scriptPath,
|
||||
params: paramsRef.current,
|
||||
...(backgroundUpdate ? { resume: true, vmid: backgroundUpdate.vmid,
|
||||
@@ -221,6 +223,10 @@ export function ScriptTerminalModal({
|
||||
|
||||
try {
|
||||
const msg = JSON.parse(event.data)
|
||||
if (msg.type === "host_update_state") {
|
||||
if (msg.state === "attention") termRef.current?.writeln(t("healthStatus.hostUpdate.attention"))
|
||||
return
|
||||
}
|
||||
if (msg.type === "job_output") {
|
||||
setIsWaitingNextInteraction(false)
|
||||
if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
|
||||
@@ -429,7 +435,9 @@ export function ScriptTerminalModal({
|
||||
}
|
||||
}, 30000)
|
||||
|
||||
const initMessage = {
|
||||
const initMessage = backgroundUpdate?.kind === "host" ? {
|
||||
resume: backgroundUpdate.resume, offset: outputOffsetRef.current,
|
||||
} : {
|
||||
script_path: scriptPath,
|
||||
params: paramsRef.current,
|
||||
...(backgroundUpdate ? { resume: backgroundUpdate.resume,
|
||||
@@ -478,6 +486,10 @@ export function ScriptTerminalModal({
|
||||
|
||||
try {
|
||||
const msg = JSON.parse(event.data)
|
||||
if (msg.type === "host_update_state") {
|
||||
if (msg.state === "attention") term.writeln(t("healthStatus.hostUpdate.attention"))
|
||||
return
|
||||
}
|
||||
if (msg.type === "job_output") {
|
||||
setIsWaitingNextInteraction(false)
|
||||
if (waitingTimeoutRef.current) clearTimeout(waitingTimeoutRef.current)
|
||||
@@ -752,7 +764,7 @@ export function ScriptTerminalModal({
|
||||
|
||||
const getScriptWebSocketUrl = (sid: string): string => {
|
||||
const route = backgroundUpdate
|
||||
? `/ws/lxc-update/${encodeURIComponent(backgroundUpdate.runId)}`
|
||||
? `/ws/${backgroundUpdate.kind === "host" ? "host-update" : "lxc-update"}/${encodeURIComponent(backgroundUpdate.runId)}`
|
||||
: `/ws/script/${sid}`
|
||||
if (typeof window === "undefined") {
|
||||
return `ws://localhost:${API_PORT}${route}`
|
||||
|
||||
@@ -5309,6 +5309,16 @@
|
||||
}
|
||||
},
|
||||
"healthStatus": {
|
||||
"hostUpdate": {
|
||||
"running": "The host update continues even if you sign out. Reopen it after signing in.",
|
||||
"finished": "A host update has finished. View its result before starting another.",
|
||||
"attention": "The host update needs attention. It has not been stopped; reconnect to inspect it.",
|
||||
"view": "View update",
|
||||
"noReboot": "No reboot required",
|
||||
"rebootMarker": "The operating system requires a reboot.",
|
||||
"rebootUnknown": "Could not determine whether a reboot is required.",
|
||||
"rebootKernel": "A reboot is required to activate {installed}. Currently running: {running}."
|
||||
},
|
||||
"title": "System health status",
|
||||
"description": "Detailed checks of all system components",
|
||||
"status": {
|
||||
@@ -5365,6 +5375,7 @@
|
||||
"system_age": "System age",
|
||||
"pending_updates": "Pending updates",
|
||||
"kernel_pve": "Kernel / PVE",
|
||||
"reboot_required": "Reboot required",
|
||||
"uptime": "Uptime",
|
||||
"certificates": "Certificates",
|
||||
"login_attempts": "Login attempts",
|
||||
|
||||
@@ -5181,6 +5181,16 @@
|
||||
}
|
||||
},
|
||||
"healthStatus": {
|
||||
"hostUpdate": {
|
||||
"running": "Aktualizácia servera pokračuje aj po odhlásení. Po prihlásení ju môžeš znova otvoriť.",
|
||||
"finished": "Aktualizácia servera sa skončila. Pred ďalším spustením si pozri jej výsledok.",
|
||||
"attention": "Aktualizácia servera potrebuje kontrolu. Nebola zastavená; otvor ju a skontroluj priebeh.",
|
||||
"view": "Zobraziť aktualizáciu",
|
||||
"noReboot": "Reštart nie je potrebný",
|
||||
"rebootMarker": "Operačný systém vyžaduje reštart.",
|
||||
"rebootUnknown": "Nepodarilo sa zistiť, či je potrebný reštart.",
|
||||
"rebootKernel": "Na aktiváciu jadra {installed} je potrebný reštart. Teraz beží: {running}."
|
||||
},
|
||||
"title": "Stav systému",
|
||||
"description": "Podrobná kontrola všetkých častí systému",
|
||||
"status": {
|
||||
@@ -5237,6 +5247,7 @@
|
||||
"system_age": "Aktuálnosť systému",
|
||||
"pending_updates": "Čakajúce aktualizácie",
|
||||
"kernel_pve": "Jadro / PVE",
|
||||
"reboot_required": "Potrebný reštart",
|
||||
"uptime": "Doba prevádzky",
|
||||
"certificates": "Certifikáty",
|
||||
"login_attempts": "Pokusy o prihlásenie",
|
||||
|
||||
@@ -394,10 +394,12 @@ def verify_token_full(token):
|
||||
API JWT 365 días sin scope.
|
||||
"""
|
||||
if not JWT_AVAILABLE or not token:
|
||||
_log_auth_failure_throttled("Session rejected: missing token or verifier unavailable")
|
||||
return None, None
|
||||
try:
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
if token_hash in _get_revoked_tokens_cached():
|
||||
_log_auth_failure_throttled("Session rejected: revoked")
|
||||
return None, None
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
@@ -409,8 +411,10 @@ def verify_token_full(token):
|
||||
payload = jwt.decode(token, _get_jwt_secret(), algorithms=[JWT_ALGORITHM])
|
||||
return payload.get('username'), payload.get('scope', 'full_admin')
|
||||
except jwt.ExpiredSignatureError:
|
||||
_log_auth_failure_throttled("Session rejected: expired")
|
||||
return None, None
|
||||
except jwt.InvalidTokenError:
|
||||
_log_auth_failure_throttled("Session rejected: invalid")
|
||||
return None, None
|
||||
|
||||
|
||||
@@ -437,9 +441,9 @@ def _log_auth_failure_throttled(msg):
|
||||
if _AUTH_LOG_RATE['suppressed']:
|
||||
print(f"[auth] {_AUTH_LOG_RATE['last_msg']} "
|
||||
f"(+{_AUTH_LOG_RATE['suppressed']} more in last "
|
||||
f"{int(elapsed)}s)")
|
||||
f"{int(elapsed)}s)", flush=True)
|
||||
else:
|
||||
print(f"[auth] {msg}")
|
||||
print(f"[auth] {msg}", flush=True)
|
||||
_AUTH_LOG_RATE['last_ts'] = now
|
||||
_AUTH_LOG_RATE['suppressed'] = 0
|
||||
_AUTH_LOG_RATE['last_msg'] = msg
|
||||
|
||||
@@ -149,6 +149,8 @@ cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
|
||||
cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_update_jobs.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/host_update_jobs.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/kernel_reboot.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/local_app_icons.py" "$APP_DIR/usr/bin/"
|
||||
|
||||
@@ -200,6 +200,74 @@ def lxc_update_viewer(ws, run_id):
|
||||
# No process termination, PTY close or completion callback here. The job's
|
||||
# worker alone owns those, even while no viewer is authenticated/connected.
|
||||
|
||||
def _host_update_completed(script_path, params, exit_code, duration_seconds):
|
||||
"""Invalidate health even when no viewer is left to close its modal."""
|
||||
from health_monitor import health_monitor
|
||||
for key in ('updates_check', 'overall_health', '_bg_overall', '_bg_detailed'):
|
||||
health_monitor.last_check_times.pop(key, None)
|
||||
health_monitor.cached_results.pop(key, None)
|
||||
|
||||
|
||||
@terminal_bp.route('/api/host-update-jobs', methods=['GET'])
|
||||
@require_admin_scope
|
||||
def host_update_jobs_status():
|
||||
from host_update_jobs import list_jobs
|
||||
return jsonify({'jobs': list_jobs()})
|
||||
|
||||
|
||||
@sock.route('/ws/host-update/<run_id>')
|
||||
def host_update_viewer(ws, run_id):
|
||||
"""New fixed-script contract; existing script/LXC routes stay intact."""
|
||||
if not _ws_auth_check():
|
||||
ws.close(reason=1008, message="Authentication required")
|
||||
return
|
||||
from host_update_jobs import get, start
|
||||
try:
|
||||
init = json.loads(ws.receive(timeout=10))
|
||||
if (not isinstance(init, dict) or set(init) - {'resume', 'offset'}
|
||||
or type(init.get('resume')) is not bool):
|
||||
raise ValueError("Invalid host update request")
|
||||
offset = init.get('offset', 0)
|
||||
if type(offset) is not int or offset < 0:
|
||||
raise ValueError("Invalid host update cursor")
|
||||
job = get(run_id) if init['resume'] else start(run_id, _host_update_completed)
|
||||
revision = -1
|
||||
while True:
|
||||
view = job.read(offset)
|
||||
if view['text'] or view['offset'] != offset:
|
||||
ws.send(json.dumps({'type': 'job_output', 'text': view['text'],
|
||||
'offset': view['offset'], 'truncated': view['truncated']}))
|
||||
offset = view['offset']
|
||||
if view['revision'] != revision:
|
||||
revision = view['revision']
|
||||
ws.send(json.dumps({'type': 'web_interaction', 'interaction': view['interaction']}))
|
||||
ws.send(json.dumps({'type': 'host_update_state', 'state': view['state'],
|
||||
'attention_reason': view['attention_reason']}))
|
||||
if view['state'] == 'complete':
|
||||
ws.send(json.dumps({'type': 'script_complete', 'exit_code': view['exit_code']}))
|
||||
break
|
||||
data = ws.receive(timeout=.2)
|
||||
if data is None:
|
||||
if getattr(ws, 'connected', True) is False or getattr(ws, 'closed', False) is True:
|
||||
break
|
||||
continue
|
||||
try:
|
||||
message = json.loads(data)
|
||||
except (TypeError, ValueError):
|
||||
message = data
|
||||
if isinstance(message, dict) and message.get('type') == 'ping':
|
||||
ws.send('{"type":"pong"}')
|
||||
else:
|
||||
job.input(message)
|
||||
except (ValueError, KeyError, OSError, TypeError) as exc:
|
||||
try:
|
||||
ws.send(json.dumps({'type': 'job_unavailable', 'message': str(exc)}))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass # An observer disconnect never signals the owner or its PTY.
|
||||
|
||||
|
||||
@terminal_bp.route('/api/terminal/health', methods=['GET'])
|
||||
def terminal_health():
|
||||
"""Health check for terminal service"""
|
||||
|
||||
@@ -4751,6 +4751,10 @@ class HealthMonitor:
|
||||
Inst lines and exposes them as an INFO sub-check.
|
||||
"""
|
||||
cache_key = 'updates_check'
|
||||
# Kernel activation is independent of pending APT packages. Check it
|
||||
# even when the package result is cached; this performs no APT call.
|
||||
from kernel_reboot import reboot_status, with_reboot_status
|
||||
reboot = reboot_status()
|
||||
current_time = time.time()
|
||||
apt_history_path = '/var/log/apt/history.log'
|
||||
|
||||
@@ -4773,7 +4777,7 @@ class HealthMonitor:
|
||||
cache_ts = self.last_check_times[cache_key]
|
||||
history_changed = (history_mtime is not None and history_mtime > cache_ts)
|
||||
if not history_changed and current_time - cache_ts < 600:
|
||||
return self.cached_results.get(cache_key)
|
||||
return with_reboot_status(self.cached_results.get(cache_key), reboot)
|
||||
|
||||
try:
|
||||
last_update_days = None
|
||||
@@ -4803,11 +4807,11 @@ class HealthMonitor:
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
print("[HealthMonitor] apt-get upgrade --dry-run timed out")
|
||||
return {
|
||||
return with_reboot_status({
|
||||
'status': 'UNKNOWN',
|
||||
'reason': 'apt-get timed out - repository may be unreachable',
|
||||
'count': 0, 'checks': {}
|
||||
}
|
||||
}, reboot)
|
||||
|
||||
status = 'OK'
|
||||
reason = None
|
||||
@@ -5124,11 +5128,11 @@ class HealthMonitor:
|
||||
|
||||
self.cached_results[cache_key] = update_result
|
||||
self.last_check_times[cache_key] = current_time
|
||||
return update_result
|
||||
return with_reboot_status(update_result, reboot)
|
||||
|
||||
except Exception as e:
|
||||
print(f"[HealthMonitor] Updates check failed: {e}")
|
||||
return {'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True}
|
||||
return with_reboot_status({'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True}, reboot)
|
||||
|
||||
def _check_fail2ban_bans(self) -> Dict[str, Any]:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
"""Owners for explicitly started PVE updates, independent of browser viewers.
|
||||
|
||||
Only the fixed PVE wrapper can run. List/resume/input cannot create a process.
|
||||
Jobs survive logout, not a Monitor/host restart. Missing IDs fail closed.
|
||||
Neither disconnect nor a watchdog kills a host package transaction.
|
||||
"""
|
||||
import base64
|
||||
import codecs
|
||||
import errno
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import termios
|
||||
import threading
|
||||
import time
|
||||
|
||||
SCRIPT = "/usr/local/share/proxmenux/scripts/utilities/proxmox_update.sh"
|
||||
RUN_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
|
||||
INTERACTION_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
|
||||
ATTENTION_AFTER = 4 * 3600
|
||||
RESULT_TTL = 6 * 3600
|
||||
MAX_JOBS = 16
|
||||
MAX_OUTPUT = 256 * 1024
|
||||
MAX_INPUT = 64 * 1024
|
||||
_lock = threading.RLock()
|
||||
_jobs = {}
|
||||
|
||||
|
||||
class JobConflict(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _group_active(pgid):
|
||||
"""Observe descendants after an abnormal shell exit; ignore zombies."""
|
||||
try:
|
||||
for entry in os.scandir('/proc'):
|
||||
if not entry.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
fields = Path(entry.path, 'stat').read_text().rsplit(') ', 1)[1].split()
|
||||
if fields[0] != 'Z' and int(fields[2]) == pgid:
|
||||
return True
|
||||
except (OSError, ValueError, IndexError):
|
||||
continue
|
||||
return False
|
||||
except OSError:
|
||||
return True # Unknown group state must not permit a second updater.
|
||||
|
||||
|
||||
def _identity(identity):
|
||||
if not isinstance(identity, str) or not RUN_ID.fullmatch(identity):
|
||||
raise ValueError("Invalid host update job ID")
|
||||
|
||||
|
||||
class HostUpdateJob:
|
||||
def __init__(self, identity, callback):
|
||||
self.id = identity
|
||||
self.created_at = time.time()
|
||||
self.started = time.monotonic()
|
||||
self.finished_at = self.exit_code = self.script_exit_code = None
|
||||
self.state = "starting"
|
||||
self.output, self.offset, self.revision = "", 0, 0
|
||||
self.interaction = self.attention_reason = None
|
||||
self.finalization_error = False
|
||||
self.lock = threading.RLock()
|
||||
self.process = self.master = self.log_path = None
|
||||
self.callback = callback
|
||||
self._input = bytearray()
|
||||
self._interaction_buffer = ""
|
||||
|
||||
def launch(self):
|
||||
# Start the owner before spawning anything: thread-start failure must
|
||||
# never leave an updater that has no owner to drain its PTY.
|
||||
threading.Thread(target=self._run, daemon=True,
|
||||
name=f"host-update-{self.id}").start()
|
||||
|
||||
def _append(self, text):
|
||||
with self.lock:
|
||||
self.output = (self.output + text)[-MAX_OUTPUT:]
|
||||
self.offset += len(text)
|
||||
|
||||
def _attention(self, reason):
|
||||
with self.lock:
|
||||
if self.attention_reason == reason and self.state == "attention":
|
||||
return
|
||||
self.attention_reason = reason
|
||||
self.state = "attention"
|
||||
self.revision += 1
|
||||
|
||||
def _interactions(self, stream):
|
||||
self._interaction_buffer += stream.read(16384)
|
||||
if len(self._interaction_buffer) > 65536:
|
||||
self._interaction_buffer = ""
|
||||
return
|
||||
lines = self._interaction_buffer.split("\n")
|
||||
self._interaction_buffer = lines.pop()
|
||||
for line in lines:
|
||||
if not line.startswith("WEB_INTERACTION:"):
|
||||
continue
|
||||
try:
|
||||
parts = line[16:].split(":", 4)
|
||||
kind, identity, title, message = parts[:4]
|
||||
if not INTERACTION_ID.fullmatch(identity):
|
||||
continue
|
||||
value = dict(type=kind, id=identity,
|
||||
title=base64.b64decode(title).decode(),
|
||||
message=base64.b64decode(message).decode())
|
||||
if len(parts) > 4:
|
||||
if kind == "menu":
|
||||
value["options"] = json.loads(parts[4])
|
||||
elif kind in ("input", "inputbox"):
|
||||
value["default"] = base64.b64decode(parts[4]).decode()
|
||||
with self.lock:
|
||||
self.interaction = value
|
||||
self.revision += 1
|
||||
except (ValueError, UnicodeError, IndexError):
|
||||
continue
|
||||
|
||||
def _run(self):
|
||||
code, slave, interactions = 1, None, None
|
||||
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
|
||||
try:
|
||||
self.master, slave = pty.openpty()
|
||||
os.set_blocking(self.master, False)
|
||||
fd, self.log_path = tempfile.mkstemp(prefix="proxmenux_host_update_", suffix=".log")
|
||||
os.close(fd)
|
||||
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 120, 0, 0))
|
||||
env = dict(os.environ, EXECUTION_MODE="web", PROXMENUX_HOST_UPDATE_JOB="1",
|
||||
WEB_LOG=self.log_path, TERM="xterm-256color", PYTHONUNBUFFERED="1")
|
||||
interactions = open(self.log_path, encoding="utf-8", errors="replace")
|
||||
self.process = subprocess.Popen(["/bin/bash", SCRIPT], stdin=slave,
|
||||
stdout=slave, stderr=slave,
|
||||
start_new_session=True, env=env)
|
||||
os.close(slave)
|
||||
slave = None
|
||||
with self.lock:
|
||||
self.state = "running"
|
||||
self.revision += 1
|
||||
eof, exited_at = False, None
|
||||
while True:
|
||||
running = self.process.poll() is None
|
||||
now = time.monotonic()
|
||||
if not running:
|
||||
exited_at = exited_at or now
|
||||
if _group_active(self.process.pid):
|
||||
self._attention("shell_exited")
|
||||
elif eof or now - exited_at >= 1:
|
||||
break
|
||||
if running and now - self.started >= ATTENTION_AFTER and not self.attention_reason:
|
||||
self._attention("runtime_limit")
|
||||
try:
|
||||
self._interactions(interactions)
|
||||
with self.lock:
|
||||
pending = bool(self._input) and running
|
||||
readable, writable, _ = select.select(
|
||||
[] if eof else [self.master], [self.master] if pending else [], [], .05)
|
||||
if writable:
|
||||
with self.lock:
|
||||
try:
|
||||
size = os.write(self.master, self._input[:16384])
|
||||
del self._input[:size]
|
||||
except BlockingIOError:
|
||||
pass
|
||||
if readable:
|
||||
try:
|
||||
chunk = os.read(self.master, 16384)
|
||||
except BlockingIOError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EIO:
|
||||
raise
|
||||
chunk = b""
|
||||
if chunk:
|
||||
self._append(decoder.decode(chunk))
|
||||
else:
|
||||
eof = True
|
||||
if eof:
|
||||
time.sleep(.05)
|
||||
except Exception:
|
||||
# An observation failure must not turn into SIGTERM of
|
||||
# dpkg. Keep owning the PTY and the single-job reservation.
|
||||
self._attention("observation_failed")
|
||||
time.sleep(.1)
|
||||
code = self.process.wait()
|
||||
self._append(decoder.decode(b"", final=True))
|
||||
except Exception as exc:
|
||||
self._append(f"\r\nHost update could not start: {type(exc).__name__}\r\n")
|
||||
# Popen/setup failure before launch is a recorded failure. Once
|
||||
# alive, preserve the updater even if observation setup failed.
|
||||
if self.process is not None:
|
||||
self._attention("observation_failed")
|
||||
code = self.process.wait()
|
||||
finally:
|
||||
for resource in (interactions, slave, self.master):
|
||||
if resource is not None:
|
||||
try:
|
||||
resource.close() if hasattr(resource, 'close') else os.close(resource)
|
||||
except OSError:
|
||||
pass
|
||||
self.master = None
|
||||
if self.log_path:
|
||||
try:
|
||||
os.unlink(self.log_path)
|
||||
except OSError:
|
||||
pass
|
||||
with self.lock:
|
||||
self.state = "finalizing"
|
||||
self.script_exit_code = self.exit_code = code
|
||||
self.interaction = None
|
||||
self._input.clear()
|
||||
self.revision += 1
|
||||
try:
|
||||
self.callback(SCRIPT, {"PROXMENUX_HOST_UPDATE_JOB": "1"}, code,
|
||||
time.monotonic() - self.started)
|
||||
except Exception:
|
||||
self.finalization_error = True
|
||||
self.exit_code = code or 1
|
||||
finally:
|
||||
with self.lock:
|
||||
self.state = "complete"
|
||||
self.finished_at = time.time()
|
||||
self.revision += 1
|
||||
|
||||
def view(self):
|
||||
with self.lock:
|
||||
return dict(id=self.id, state=self.state, created_at=self.created_at,
|
||||
finished_at=self.finished_at, exit_code=self.exit_code,
|
||||
script_exit_code=self.script_exit_code,
|
||||
attention_reason=self.attention_reason,
|
||||
finalization_error=self.finalization_error)
|
||||
|
||||
def read(self, offset):
|
||||
with self.lock:
|
||||
begin = self.offset - len(self.output)
|
||||
cursor = max(begin, min(offset, self.offset))
|
||||
return dict(self.view(), text=self.output[cursor-begin:], offset=self.offset,
|
||||
truncated=offset < begin, interaction=self.interaction,
|
||||
revision=self.revision)
|
||||
|
||||
def input(self, message):
|
||||
with self.lock:
|
||||
if self.state not in ("running", "attention") or self.master is None:
|
||||
return
|
||||
if isinstance(message, str):
|
||||
if any(c in message for c in ("\x03", "\x1a", "\x1c")):
|
||||
raise ValueError("Host package updates cannot be cancelled from the viewer")
|
||||
encoded = message.encode()
|
||||
if len(encoded) > 16384 or len(self._input) + len(encoded) > MAX_INPUT:
|
||||
raise ValueError("Host update input buffer is full")
|
||||
self._input.extend(encoded)
|
||||
elif isinstance(message, dict) and message.get("type") == "resize":
|
||||
cols = max(20, min(400, int(message.get("cols", 120))))
|
||||
rows = max(5, min(200, int(message.get("rows", 30))))
|
||||
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
|
||||
elif isinstance(message, dict) and message.get("type") == "interaction_response":
|
||||
identity, value = message.get("id"), message.get("value")
|
||||
if (not isinstance(identity, str) or not INTERACTION_ID.fullmatch(identity)
|
||||
or not self.interaction or identity != self.interaction["id"]
|
||||
or not isinstance(value, str) or len(value.encode()) > 16384):
|
||||
raise ValueError("No matching pending host update interaction")
|
||||
fd = os.open(f"/tmp/proxmenux_response_{identity}",
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd, "w") as response:
|
||||
response.write(value)
|
||||
self.interaction = None
|
||||
self.revision += 1
|
||||
else:
|
||||
raise ValueError("Unsupported host update input")
|
||||
|
||||
|
||||
def _prune():
|
||||
for identity, job in list(_jobs.items()):
|
||||
if job.finished_at is not None and time.time() - job.finished_at > RESULT_TTL:
|
||||
del _jobs[identity]
|
||||
|
||||
|
||||
def start(identity, callback):
|
||||
_identity(identity)
|
||||
if os.path.realpath(SCRIPT) != SCRIPT or not os.path.isfile(SCRIPT):
|
||||
raise ValueError("PVE update script is unavailable")
|
||||
with _lock:
|
||||
if identity in _jobs:
|
||||
return _jobs[identity]
|
||||
_prune()
|
||||
if any(job.state != "complete" for job in _jobs.values()):
|
||||
raise JobConflict("A host update is already running; reconnect to it")
|
||||
while len(_jobs) >= MAX_JOBS:
|
||||
del _jobs[min(_jobs, key=lambda key: _jobs[key].finished_at)]
|
||||
job = HostUpdateJob(identity, callback)
|
||||
_jobs[identity] = job
|
||||
try:
|
||||
job.launch()
|
||||
except BaseException:
|
||||
_jobs.pop(identity, None)
|
||||
raise
|
||||
return job
|
||||
|
||||
|
||||
def get(identity):
|
||||
_identity(identity)
|
||||
with _lock:
|
||||
if identity not in _jobs:
|
||||
raise KeyError("Host update is unavailable; it was not restarted")
|
||||
return _jobs[identity]
|
||||
|
||||
|
||||
def list_jobs():
|
||||
with _lock:
|
||||
_prune()
|
||||
return [job.view() for job in sorted(_jobs.values(), key=lambda job: job.created_at, reverse=True)]
|
||||
@@ -50,6 +50,7 @@ def require_auth(f):
|
||||
auth_header = request.headers.get('Authorization')
|
||||
|
||||
if not auth_header:
|
||||
verify_token_full(None) # Guarded, credential-free rejection diagnostic.
|
||||
return jsonify({
|
||||
"error": "Authentication required",
|
||||
"message": "No authorization header provided"
|
||||
@@ -58,6 +59,7 @@ def require_auth(f):
|
||||
# Extract token from "Bearer <token>" format
|
||||
parts = auth_header.split()
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer':
|
||||
verify_token_full(None)
|
||||
return jsonify({
|
||||
"error": "Invalid authorization header",
|
||||
"message": "Authorization header must be in format: Bearer <token>"
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Read-only reboot evidence, independent of APT pending-update caches."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
_RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)-(\d+)-pve$")
|
||||
_PACKAGE = re.compile(r"^(?:proxmox|pve)-kernel-(\d+\.\d+\.\d+-\d+-pve)(?:-signed)?(?::[a-z0-9]+)?$")
|
||||
|
||||
|
||||
def reboot_status():
|
||||
"""No package refresh, installation, mount or bootloader operation."""
|
||||
try:
|
||||
running = os.uname().release
|
||||
except OSError:
|
||||
return dict(state="unknown", running_kernel=None, installed_kernel=None)
|
||||
result = dict(state="unknown", running_kernel=running, installed_kernel=None)
|
||||
if not isinstance(running, str) or not re.fullmatch(r'\d+\.\d+[A-Za-z0-9_.+\-]*', running):
|
||||
return result
|
||||
try:
|
||||
if Path('/run/reboot-required').exists():
|
||||
return dict(result, state="required", reason="marker")
|
||||
except OSError:
|
||||
return result
|
||||
match = _RELEASE.fullmatch(running)
|
||||
if not match:
|
||||
if running.endswith('-pve'):
|
||||
return result
|
||||
return dict(result, state="not_applicable")
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
['dpkg-query', '-W', '-f=${db:Status-Status}\t${binary:Package}\n'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if proc.returncode != 0 or not proc.stdout.strip():
|
||||
return result
|
||||
installed = []
|
||||
valid_states = {'installed', 'config-files', 'not-installed', 'unpacked',
|
||||
'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'}
|
||||
for line in proc.stdout.splitlines():
|
||||
fields = line.split('\t')
|
||||
if len(fields) != 2 or fields[0] not in valid_states:
|
||||
return result
|
||||
state, package = fields
|
||||
kernel = _PACKAGE.fullmatch(package)
|
||||
if kernel:
|
||||
if state in ('unpacked', 'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'):
|
||||
return result
|
||||
if state == 'installed':
|
||||
installed.append(kernel.group(1))
|
||||
if not installed:
|
||||
return result
|
||||
newest = max(installed, key=lambda v: tuple(map(int, _RELEASE.fullmatch(v).groups())))
|
||||
newer = tuple(map(int, _RELEASE.fullmatch(newest).groups())) > tuple(map(int, match.groups()))
|
||||
return dict(result, state="required" if newer else "current", installed_kernel=newest,
|
||||
reason="kernel" if newer else None)
|
||||
except (OSError, subprocess.SubprocessError, ValueError, TypeError):
|
||||
return result
|
||||
|
||||
|
||||
def with_reboot_status(updates, reboot):
|
||||
"""Copy, never rewrite a cached package result or hide its existing error."""
|
||||
if reboot['state'] == 'not_applicable':
|
||||
return updates
|
||||
result = dict(updates or dict(status='UNKNOWN', count=None, checks={}))
|
||||
result['checks'] = dict(result.get('checks') or {})
|
||||
required = reboot['state'] == 'required'
|
||||
unknown = reboot['state'] == 'unknown'
|
||||
result['reboot_required'] = True if required else None if unknown else False
|
||||
result['reboot_state'] = dict(reboot)
|
||||
if required:
|
||||
detail = (f"Reboot required: running {reboot['running_kernel']}; installed {reboot['installed_kernel']}"
|
||||
if reboot.get('installed_kernel') else 'Reboot required by the operating system')
|
||||
else:
|
||||
detail = 'Could not determine whether a reboot is required' if unknown else 'No reboot required'
|
||||
result['checks']['reboot_required'] = dict(status='INFO' if required else 'UNKNOWN' if unknown else 'OK',
|
||||
detail=detail, error_key='reboot_required')
|
||||
if result.get('status') == 'OK' and (required or unknown):
|
||||
result['status'] = 'INFO' if required else 'UNKNOWN'
|
||||
return result
|
||||
Reference in New Issue
Block a user