Preserve host updates across sign-out and report pending kernel activation

This commit is contained in:
VAIO73
2026-10-28 17:30:44 +00:00
parent 2d4b0cb746
commit f40e0a53e1
24 changed files with 1605 additions and 18 deletions
+6 -2
View File
@@ -394,10 +394,12 @@ def verify_token_full(token):
API JWT 365 días sin scope.
"""
if not JWT_AVAILABLE or not token:
_log_auth_failure_throttled("Session rejected: missing token or verifier unavailable")
return None, None
try:
token_hash = hashlib.sha256(token.encode()).hexdigest()
if token_hash in _get_revoked_tokens_cached():
_log_auth_failure_throttled("Session rejected: revoked")
return None, None
try:
payload = jwt.decode(
@@ -409,8 +411,10 @@ def verify_token_full(token):
payload = jwt.decode(token, _get_jwt_secret(), algorithms=[JWT_ALGORITHM])
return payload.get('username'), payload.get('scope', 'full_admin')
except jwt.ExpiredSignatureError:
_log_auth_failure_throttled("Session rejected: expired")
return None, None
except jwt.InvalidTokenError:
_log_auth_failure_throttled("Session rejected: invalid")
return None, None
@@ -437,9 +441,9 @@ def _log_auth_failure_throttled(msg):
if _AUTH_LOG_RATE['suppressed']:
print(f"[auth] {_AUTH_LOG_RATE['last_msg']} "
f"(+{_AUTH_LOG_RATE['suppressed']} more in last "
f"{int(elapsed)}s)")
f"{int(elapsed)}s)", flush=True)
else:
print(f"[auth] {msg}")
print(f"[auth] {msg}", flush=True)
_AUTH_LOG_RATE['last_ts'] = now
_AUTH_LOG_RATE['suppressed'] = 0
_AUTH_LOG_RATE['last_msg'] = msg
+2
View File
@@ -149,6 +149,8 @@ cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_update_jobs.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/host_update_jobs.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/kernel_reboot.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/local_app_icons.py" "$APP_DIR/usr/bin/"
+68
View File
@@ -200,6 +200,74 @@ def lxc_update_viewer(ws, run_id):
# No process termination, PTY close or completion callback here. The job's
# worker alone owns those, even while no viewer is authenticated/connected.
def _host_update_completed(script_path, params, exit_code, duration_seconds):
"""Invalidate health even when no viewer is left to close its modal."""
from health_monitor import health_monitor
for key in ('updates_check', 'overall_health', '_bg_overall', '_bg_detailed'):
health_monitor.last_check_times.pop(key, None)
health_monitor.cached_results.pop(key, None)
@terminal_bp.route('/api/host-update-jobs', methods=['GET'])
@require_admin_scope
def host_update_jobs_status():
from host_update_jobs import list_jobs
return jsonify({'jobs': list_jobs()})
@sock.route('/ws/host-update/<run_id>')
def host_update_viewer(ws, run_id):
"""New fixed-script contract; existing script/LXC routes stay intact."""
if not _ws_auth_check():
ws.close(reason=1008, message="Authentication required")
return
from host_update_jobs import get, start
try:
init = json.loads(ws.receive(timeout=10))
if (not isinstance(init, dict) or set(init) - {'resume', 'offset'}
or type(init.get('resume')) is not bool):
raise ValueError("Invalid host update request")
offset = init.get('offset', 0)
if type(offset) is not int or offset < 0:
raise ValueError("Invalid host update cursor")
job = get(run_id) if init['resume'] else start(run_id, _host_update_completed)
revision = -1
while True:
view = job.read(offset)
if view['text'] or view['offset'] != offset:
ws.send(json.dumps({'type': 'job_output', 'text': view['text'],
'offset': view['offset'], 'truncated': view['truncated']}))
offset = view['offset']
if view['revision'] != revision:
revision = view['revision']
ws.send(json.dumps({'type': 'web_interaction', 'interaction': view['interaction']}))
ws.send(json.dumps({'type': 'host_update_state', 'state': view['state'],
'attention_reason': view['attention_reason']}))
if view['state'] == 'complete':
ws.send(json.dumps({'type': 'script_complete', 'exit_code': view['exit_code']}))
break
data = ws.receive(timeout=.2)
if data is None:
if getattr(ws, 'connected', True) is False or getattr(ws, 'closed', False) is True:
break
continue
try:
message = json.loads(data)
except (TypeError, ValueError):
message = data
if isinstance(message, dict) and message.get('type') == 'ping':
ws.send('{"type":"pong"}')
else:
job.input(message)
except (ValueError, KeyError, OSError, TypeError) as exc:
try:
ws.send(json.dumps({'type': 'job_unavailable', 'message': str(exc)}))
except Exception:
pass
except Exception:
pass # An observer disconnect never signals the owner or its PTY.
@terminal_bp.route('/api/terminal/health', methods=['GET'])
def terminal_health():
"""Health check for terminal service"""
+9 -5
View File
@@ -4751,6 +4751,10 @@ class HealthMonitor:
Inst lines and exposes them as an INFO sub-check.
"""
cache_key = 'updates_check'
# Kernel activation is independent of pending APT packages. Check it
# even when the package result is cached; this performs no APT call.
from kernel_reboot import reboot_status, with_reboot_status
reboot = reboot_status()
current_time = time.time()
apt_history_path = '/var/log/apt/history.log'
@@ -4773,7 +4777,7 @@ class HealthMonitor:
cache_ts = self.last_check_times[cache_key]
history_changed = (history_mtime is not None and history_mtime > cache_ts)
if not history_changed and current_time - cache_ts < 600:
return self.cached_results.get(cache_key)
return with_reboot_status(self.cached_results.get(cache_key), reboot)
try:
last_update_days = None
@@ -4803,11 +4807,11 @@ class HealthMonitor:
)
except subprocess.TimeoutExpired:
print("[HealthMonitor] apt-get upgrade --dry-run timed out")
return {
return with_reboot_status({
'status': 'UNKNOWN',
'reason': 'apt-get timed out - repository may be unreachable',
'count': 0, 'checks': {}
}
}, reboot)
status = 'OK'
reason = None
@@ -5124,11 +5128,11 @@ class HealthMonitor:
self.cached_results[cache_key] = update_result
self.last_check_times[cache_key] = current_time
return update_result
return with_reboot_status(update_result, reboot)
except Exception as e:
print(f"[HealthMonitor] Updates check failed: {e}")
return {'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True}
return with_reboot_status({'status': 'UNKNOWN', 'reason': f'Updates check unavailable: {str(e)}', 'count': 0, 'checks': {}, 'dismissable': True}, reboot)
def _check_fail2ban_bans(self) -> Dict[str, Any]:
"""
+317
View File
@@ -0,0 +1,317 @@
"""Owners for explicitly started PVE updates, independent of browser viewers.
Only the fixed PVE wrapper can run. List/resume/input cannot create a process.
Jobs survive logout, not a Monitor/host restart. Missing IDs fail closed.
Neither disconnect nor a watchdog kills a host package transaction.
"""
import base64
import codecs
import errno
import fcntl
import json
import os
from pathlib import Path
import pty
import re
import select
import struct
import subprocess
import tempfile
import termios
import threading
import time
SCRIPT = "/usr/local/share/proxmenux/scripts/utilities/proxmox_update.sh"
RUN_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
INTERACTION_ID = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
ATTENTION_AFTER = 4 * 3600
RESULT_TTL = 6 * 3600
MAX_JOBS = 16
MAX_OUTPUT = 256 * 1024
MAX_INPUT = 64 * 1024
_lock = threading.RLock()
_jobs = {}
class JobConflict(ValueError):
pass
def _group_active(pgid):
"""Observe descendants after an abnormal shell exit; ignore zombies."""
try:
for entry in os.scandir('/proc'):
if not entry.name.isdigit():
continue
try:
fields = Path(entry.path, 'stat').read_text().rsplit(') ', 1)[1].split()
if fields[0] != 'Z' and int(fields[2]) == pgid:
return True
except (OSError, ValueError, IndexError):
continue
return False
except OSError:
return True # Unknown group state must not permit a second updater.
def _identity(identity):
if not isinstance(identity, str) or not RUN_ID.fullmatch(identity):
raise ValueError("Invalid host update job ID")
class HostUpdateJob:
def __init__(self, identity, callback):
self.id = identity
self.created_at = time.time()
self.started = time.monotonic()
self.finished_at = self.exit_code = self.script_exit_code = None
self.state = "starting"
self.output, self.offset, self.revision = "", 0, 0
self.interaction = self.attention_reason = None
self.finalization_error = False
self.lock = threading.RLock()
self.process = self.master = self.log_path = None
self.callback = callback
self._input = bytearray()
self._interaction_buffer = ""
def launch(self):
# Start the owner before spawning anything: thread-start failure must
# never leave an updater that has no owner to drain its PTY.
threading.Thread(target=self._run, daemon=True,
name=f"host-update-{self.id}").start()
def _append(self, text):
with self.lock:
self.output = (self.output + text)[-MAX_OUTPUT:]
self.offset += len(text)
def _attention(self, reason):
with self.lock:
if self.attention_reason == reason and self.state == "attention":
return
self.attention_reason = reason
self.state = "attention"
self.revision += 1
def _interactions(self, stream):
self._interaction_buffer += stream.read(16384)
if len(self._interaction_buffer) > 65536:
self._interaction_buffer = ""
return
lines = self._interaction_buffer.split("\n")
self._interaction_buffer = lines.pop()
for line in lines:
if not line.startswith("WEB_INTERACTION:"):
continue
try:
parts = line[16:].split(":", 4)
kind, identity, title, message = parts[:4]
if not INTERACTION_ID.fullmatch(identity):
continue
value = dict(type=kind, id=identity,
title=base64.b64decode(title).decode(),
message=base64.b64decode(message).decode())
if len(parts) > 4:
if kind == "menu":
value["options"] = json.loads(parts[4])
elif kind in ("input", "inputbox"):
value["default"] = base64.b64decode(parts[4]).decode()
with self.lock:
self.interaction = value
self.revision += 1
except (ValueError, UnicodeError, IndexError):
continue
def _run(self):
code, slave, interactions = 1, None, None
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
try:
self.master, slave = pty.openpty()
os.set_blocking(self.master, False)
fd, self.log_path = tempfile.mkstemp(prefix="proxmenux_host_update_", suffix=".log")
os.close(fd)
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 120, 0, 0))
env = dict(os.environ, EXECUTION_MODE="web", PROXMENUX_HOST_UPDATE_JOB="1",
WEB_LOG=self.log_path, TERM="xterm-256color", PYTHONUNBUFFERED="1")
interactions = open(self.log_path, encoding="utf-8", errors="replace")
self.process = subprocess.Popen(["/bin/bash", SCRIPT], stdin=slave,
stdout=slave, stderr=slave,
start_new_session=True, env=env)
os.close(slave)
slave = None
with self.lock:
self.state = "running"
self.revision += 1
eof, exited_at = False, None
while True:
running = self.process.poll() is None
now = time.monotonic()
if not running:
exited_at = exited_at or now
if _group_active(self.process.pid):
self._attention("shell_exited")
elif eof or now - exited_at >= 1:
break
if running and now - self.started >= ATTENTION_AFTER and not self.attention_reason:
self._attention("runtime_limit")
try:
self._interactions(interactions)
with self.lock:
pending = bool(self._input) and running
readable, writable, _ = select.select(
[] if eof else [self.master], [self.master] if pending else [], [], .05)
if writable:
with self.lock:
try:
size = os.write(self.master, self._input[:16384])
del self._input[:size]
except BlockingIOError:
pass
if readable:
try:
chunk = os.read(self.master, 16384)
except BlockingIOError:
continue
except OSError as exc:
if exc.errno != errno.EIO:
raise
chunk = b""
if chunk:
self._append(decoder.decode(chunk))
else:
eof = True
if eof:
time.sleep(.05)
except Exception:
# An observation failure must not turn into SIGTERM of
# dpkg. Keep owning the PTY and the single-job reservation.
self._attention("observation_failed")
time.sleep(.1)
code = self.process.wait()
self._append(decoder.decode(b"", final=True))
except Exception as exc:
self._append(f"\r\nHost update could not start: {type(exc).__name__}\r\n")
# Popen/setup failure before launch is a recorded failure. Once
# alive, preserve the updater even if observation setup failed.
if self.process is not None:
self._attention("observation_failed")
code = self.process.wait()
finally:
for resource in (interactions, slave, self.master):
if resource is not None:
try:
resource.close() if hasattr(resource, 'close') else os.close(resource)
except OSError:
pass
self.master = None
if self.log_path:
try:
os.unlink(self.log_path)
except OSError:
pass
with self.lock:
self.state = "finalizing"
self.script_exit_code = self.exit_code = code
self.interaction = None
self._input.clear()
self.revision += 1
try:
self.callback(SCRIPT, {"PROXMENUX_HOST_UPDATE_JOB": "1"}, code,
time.monotonic() - self.started)
except Exception:
self.finalization_error = True
self.exit_code = code or 1
finally:
with self.lock:
self.state = "complete"
self.finished_at = time.time()
self.revision += 1
def view(self):
with self.lock:
return dict(id=self.id, state=self.state, created_at=self.created_at,
finished_at=self.finished_at, exit_code=self.exit_code,
script_exit_code=self.script_exit_code,
attention_reason=self.attention_reason,
finalization_error=self.finalization_error)
def read(self, offset):
with self.lock:
begin = self.offset - len(self.output)
cursor = max(begin, min(offset, self.offset))
return dict(self.view(), text=self.output[cursor-begin:], offset=self.offset,
truncated=offset < begin, interaction=self.interaction,
revision=self.revision)
def input(self, message):
with self.lock:
if self.state not in ("running", "attention") or self.master is None:
return
if isinstance(message, str):
if any(c in message for c in ("\x03", "\x1a", "\x1c")):
raise ValueError("Host package updates cannot be cancelled from the viewer")
encoded = message.encode()
if len(encoded) > 16384 or len(self._input) + len(encoded) > MAX_INPUT:
raise ValueError("Host update input buffer is full")
self._input.extend(encoded)
elif isinstance(message, dict) and message.get("type") == "resize":
cols = max(20, min(400, int(message.get("cols", 120))))
rows = max(5, min(200, int(message.get("rows", 30))))
fcntl.ioctl(self.master, termios.TIOCSWINSZ, struct.pack("HHHH", rows, cols, 0, 0))
elif isinstance(message, dict) and message.get("type") == "interaction_response":
identity, value = message.get("id"), message.get("value")
if (not isinstance(identity, str) or not INTERACTION_ID.fullmatch(identity)
or not self.interaction or identity != self.interaction["id"]
or not isinstance(value, str) or len(value.encode()) > 16384):
raise ValueError("No matching pending host update interaction")
fd = os.open(f"/tmp/proxmenux_response_{identity}",
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "w") as response:
response.write(value)
self.interaction = None
self.revision += 1
else:
raise ValueError("Unsupported host update input")
def _prune():
for identity, job in list(_jobs.items()):
if job.finished_at is not None and time.time() - job.finished_at > RESULT_TTL:
del _jobs[identity]
def start(identity, callback):
_identity(identity)
if os.path.realpath(SCRIPT) != SCRIPT or not os.path.isfile(SCRIPT):
raise ValueError("PVE update script is unavailable")
with _lock:
if identity in _jobs:
return _jobs[identity]
_prune()
if any(job.state != "complete" for job in _jobs.values()):
raise JobConflict("A host update is already running; reconnect to it")
while len(_jobs) >= MAX_JOBS:
del _jobs[min(_jobs, key=lambda key: _jobs[key].finished_at)]
job = HostUpdateJob(identity, callback)
_jobs[identity] = job
try:
job.launch()
except BaseException:
_jobs.pop(identity, None)
raise
return job
def get(identity):
_identity(identity)
with _lock:
if identity not in _jobs:
raise KeyError("Host update is unavailable; it was not restarted")
return _jobs[identity]
def list_jobs():
with _lock:
_prune()
return [job.view() for job in sorted(_jobs.values(), key=lambda job: job.created_at, reverse=True)]
+2
View File
@@ -50,6 +50,7 @@ def require_auth(f):
auth_header = request.headers.get('Authorization')
if not auth_header:
verify_token_full(None) # Guarded, credential-free rejection diagnostic.
return jsonify({
"error": "Authentication required",
"message": "No authorization header provided"
@@ -58,6 +59,7 @@ def require_auth(f):
# Extract token from "Bearer <token>" format
parts = auth_header.split()
if len(parts) != 2 or parts[0].lower() != 'bearer':
verify_token_full(None)
return jsonify({
"error": "Invalid authorization header",
"message": "Authorization header must be in format: Bearer <token>"
+79
View File
@@ -0,0 +1,79 @@
"""Read-only reboot evidence, independent of APT pending-update caches."""
import os
from pathlib import Path
import re
import subprocess
_RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)-(\d+)-pve$")
_PACKAGE = re.compile(r"^(?:proxmox|pve)-kernel-(\d+\.\d+\.\d+-\d+-pve)(?:-signed)?(?::[a-z0-9]+)?$")
def reboot_status():
"""No package refresh, installation, mount or bootloader operation."""
try:
running = os.uname().release
except OSError:
return dict(state="unknown", running_kernel=None, installed_kernel=None)
result = dict(state="unknown", running_kernel=running, installed_kernel=None)
if not isinstance(running, str) or not re.fullmatch(r'\d+\.\d+[A-Za-z0-9_.+\-]*', running):
return result
try:
if Path('/run/reboot-required').exists():
return dict(result, state="required", reason="marker")
except OSError:
return result
match = _RELEASE.fullmatch(running)
if not match:
if running.endswith('-pve'):
return result
return dict(result, state="not_applicable")
try:
proc = subprocess.run(
['dpkg-query', '-W', '-f=${db:Status-Status}\t${binary:Package}\n'],
capture_output=True, text=True, timeout=5)
if proc.returncode != 0 or not proc.stdout.strip():
return result
installed = []
valid_states = {'installed', 'config-files', 'not-installed', 'unpacked',
'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'}
for line in proc.stdout.splitlines():
fields = line.split('\t')
if len(fields) != 2 or fields[0] not in valid_states:
return result
state, package = fields
kernel = _PACKAGE.fullmatch(package)
if kernel:
if state in ('unpacked', 'half-configured', 'half-installed', 'triggers-awaited', 'triggers-pending'):
return result
if state == 'installed':
installed.append(kernel.group(1))
if not installed:
return result
newest = max(installed, key=lambda v: tuple(map(int, _RELEASE.fullmatch(v).groups())))
newer = tuple(map(int, _RELEASE.fullmatch(newest).groups())) > tuple(map(int, match.groups()))
return dict(result, state="required" if newer else "current", installed_kernel=newest,
reason="kernel" if newer else None)
except (OSError, subprocess.SubprocessError, ValueError, TypeError):
return result
def with_reboot_status(updates, reboot):
"""Copy, never rewrite a cached package result or hide its existing error."""
if reboot['state'] == 'not_applicable':
return updates
result = dict(updates or dict(status='UNKNOWN', count=None, checks={}))
result['checks'] = dict(result.get('checks') or {})
required = reboot['state'] == 'required'
unknown = reboot['state'] == 'unknown'
result['reboot_required'] = True if required else None if unknown else False
result['reboot_state'] = dict(reboot)
if required:
detail = (f"Reboot required: running {reboot['running_kernel']}; installed {reboot['installed_kernel']}"
if reboot.get('installed_kernel') else 'Reboot required by the operating system')
else:
detail = 'Could not determine whether a reboot is required' if unknown else 'No reboot required'
result['checks']['reboot_required'] = dict(status='INFO' if required else 'UNKNOWN' if unknown else 'OK',
detail=detail, error_key='reboot_required')
if result.get('status') == 'OK' and (required or unknown):
result['status'] = 'INFO' if required else 'UNKNOWN'
return result