fix: preserve update observations for stopped LXC guests

This commit is contained in:
VAIO73
2026-10-14 17:51:53 +00:00
parent ceb783bcf2
commit 3489dc0530
10 changed files with 1225 additions and 10 deletions
+2
View File
@@ -144,6 +144,8 @@ cp "$SCRIPT_DIR/smartctl_resolver.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
cp "$SCRIPT_DIR/disk_identity.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ disk_identity.py not found"
cp "$SCRIPT_DIR/health_thresholds.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ health_thresholds.py not found"
cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ managed_installs.py not found"
cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/lxc_apps.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ lxc_apps.py not found"
cp "$SCRIPT_DIR/custom_links.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ custom_links.py not found"
cp "$SCRIPT_DIR/recreate_docker_container.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ recreate_docker_container.py not found"
+32 -2
View File
@@ -14253,8 +14253,7 @@ def api_vm_apps_dismiss(vmid):
_LXC_APPLY_UPDATES_SCRIPT = "/usr/local/share/proxmenux/scripts/lxc/apply_updates.sh"
_LXC_UPDATE_RUN_ID_RE = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
_LXC_UPDATE_TARGET_RE = re.compile(r'^[A-Za-z0-9._:-]{1,180}$')
_lxc_update_finalization_lock = threading.RLock()
_lxc_update_finalizations: dict[tuple[int, str], dict] = {}
from lxc_update_finalization import lock as _lxc_update_finalization_lock, runs as _lxc_update_finalizations
_LXC_UPDATE_FINALIZATION_TTL = 6 * 3600
@@ -14547,6 +14546,37 @@ def _finalize_lxc_update(
verification_pending = _fast_guest_status(vmid, 'lxc') != 'running'
docker_inventory = None
if verification_pending and status in {'success', 'failure', 'partial'}:
try:
import managed_installs
import lxc_update_observation
observation = lxc_update_observation.read(vmid, safe_run_id)
verified = managed_installs.store_lxc_update_observation(vmid, observation)
print(f'[ProxMenux] stopped LXC verification: vmid={vmid} '
f'run_id={safe_run_id} handoff={observation is not None} '
f'verified={verified}', flush=True)
if not verified:
verification_errors.append('post-update OS verification unavailable')
apps_persisted = True
if observation is not None and 'apps' in observation:
import lxc_apps
try:
app_result = lxc_apps.store_installed_observations(vmid, observation['apps'])
verification_errors.extend(app_result['errors'])
if status == 'success':
lxc_apps.mark_manual_versions_unverified(
vmid, [item[4:] for item in requested if item.startswith('app:')])
_publish_guest_modal_cache_revision(vmid)
except Exception:
apps_persisted = False
verification_errors.append('post-update application persistence failed')
# Retain a rejected handoff for diagnosis rather than silently
# deleting the only evidence. Capture prunes old private files.
if observation is not None and verified and apps_persisted:
lxc_update_observation.discard(vmid, safe_run_id)
except Exception as exc:
verification_errors.append(f'post-update OS verification failed: {exc}')
if not verification_pending:
try:
import managed_installs
+106
View File
@@ -4503,6 +4503,112 @@ def emit_all_pending_updates() -> int:
return count
def _installed_observation_fingerprint(app: dict) -> str:
# Only version-probe configuration, never updater commands or credentials.
fields = ('installed_via', 'installed_regex', 'tag_regex') + _DETECTOR_FIELDS
return hashlib.sha256(json.dumps({k: app.get(k) for k in fields},
sort_keys=True).encode()).hexdigest()
def capture_installed_observations(vmid) -> list[dict]:
"""Probe registered automatic versions while a temporary CT still runs.
No shared writes, upstream requests, discovery or update commands. The
backend later merges this private handoff against the current sidecar.
Manual values and OCI/image registrations cannot be proven this way.
"""
with _cache_lock:
registrations = copy.deepcopy((_read_sidecar(vmid) or {}).get('apps') or [])
observations = []
for app in registrations:
if app.get('installed_via') not in set(_VALID_METHODS) - {'manual', 'oci_image'}:
continue
try:
version, error = detect_installed_version(vmid, app)
except Exception:
version, error = None, True
verified = (isinstance(version, str) and 0 < len(version) <= 256
and not re.search(r'[\x00-\x1f\x7f]', version) and not error)
observations.append({'id': app.get('id'),
'fingerprint': _installed_observation_fingerprint(app),
'checked_at': _now_iso(), 'verified': bool(verified),
'installed_version': version if verified else None})
return observations
def store_installed_observations(vmid, observations) -> dict:
"""Merge trusted captured versions without executing anything in the CT."""
errors = []
if not isinstance(observations, list):
return {'applied': 0, 'errors': ['post-update application observation unavailable']}
seen = set()
parsed = []
for item in observations:
try:
app_id = item['id']
fingerprint = item['fingerprint']
version = item['installed_version']
checked = datetime.datetime.fromisoformat(item['checked_at'].replace('Z', '+00:00'))
age = time.time() - checked.timestamp()
valid = (isinstance(app_id, str) and 0 < len(app_id) <= 128
and app_id not in seen and re.fullmatch(r'[a-zA-Z0-9._:-]+', app_id)
and isinstance(fingerprint, str) and re.fullmatch(r'[a-f0-9]{64}', fingerprint)
and checked.tzinfo is not None and 0 <= age <= 3600
and type(item['verified']) is bool
and ((item['verified'] and isinstance(version, str) and 0 < len(version) <= 256
and not re.search(r'[\x00-\x1f\x7f]', version))
or (not item['verified'] and version is None)))
if not valid:
raise ValueError('invalid observation')
seen.add(app_id)
parsed.append((item, checked))
except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
# Validate the whole payload before its first shared write.
return {'applied': 0, 'errors': ['invalid post-update application observation']}
if not parsed:
return {'applied': 0, 'errors': []}
with _cache_lock:
sidecar = _read_sidecar(vmid)
applied = 0
for item, checked in parsed:
app = _find_app(sidecar or {}, item['id'])
if not app or _installed_observation_fingerprint(app) != item['fingerprint']:
errors.append('application configuration changed after capture')
continue
state = dict(app.get('state') or {})
try:
current = datetime.datetime.fromisoformat((state.get('checked_at') or '').replace('Z', '+00:00'))
if current >= checked:
errors.append('newer application observation preserved')
continue
except (TypeError, ValueError):
pass
if item['verified']:
state['installed_version'] = item['installed_version']
state['update_available'] = (compare(item['installed_version'], state['latest_version'])
if state.get('latest_version') else None)
# An installed-version probe does not recheck the upstream;
# preserve an existing upstream error rather than hiding it.
if state.get('error') == 'post-update installed version could not be verified':
state['error'] = None
state.pop('last_known_installed_version', None)
else:
state['last_known_installed_version'] = (state.get('installed_version')
or state.get('last_known_installed_version'))
state['installed_version'] = None
state['update_available'] = None
state['error'] = 'post-update installed version could not be verified'
errors.append(state['error'])
state['checked_at'] = item['checked_at']
app['state'] = state
applied += 1
if applied:
sidecar['updated_at'] = _now_iso()
if not _write_sidecar(vmid, sidecar):
raise OSError('post-update application persistence failed')
return {'applied': applied, 'errors': errors}
def check_all(
vmid, force: bool = False, notify: bool = True,
) -> Optional[dict]:
@@ -0,0 +1,11 @@
"""One process-wide LXC completion state, independent of Flask import name.
AppRun executes flask_server as __main__, while collectors import it as
flask_server. Both completion paths must share this canonical module so
the terminal hook and compatibility HTTP callback cannot consume a run twice.
No guest actions, registry writes or background tasks occur on import.
"""
import threading
lock = threading.RLock()
runs: dict[tuple[int, str], dict] = {}
+180
View File
@@ -0,0 +1,180 @@
"""Trusted, run-scoped package observation captured before a temporary CT stops.
The runner only writes this private handoff, not the shared registry. The
Monitor consumes it under its existing registry lock after the process exits.
Never accepted as an HTTP payload and never refreshes package indexes.
"""
from __future__ import annotations
import argparse
import datetime
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import tempfile
import managed_installs
OBSERVATION_DIR = Path("/run/proxmenux/lxc-update-observations")
MAX_AGE = 3600
MAX_BYTES = 2 * 1024 * 1024
def _path(vmid: int, run_id: str) -> Path:
if int(vmid) <= 0 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", run_id):
raise ValueError("invalid LXC update identity")
return OBSERVATION_DIR / f"{int(vmid)}-{run_id}.json"
def _private_directory(*, create=True) -> None:
if create:
OBSERVATION_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
info = OBSERVATION_DIR.lstat()
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid()
or info.st_mode & 0o077):
raise OSError("unsafe LXC observation directory")
def capture(vmid: int, run_id: str, *, refresh_indexes: bool = False,
include_apps: bool = False) -> dict:
path = _path(vmid, run_id)
_private_directory()
# Remove only obsolete private handoffs left by terminated backend runs.
now = datetime.datetime.now(datetime.timezone.utc)
for old in OBSERVATION_DIR.iterdir():
info = old.lstat()
if (re.fullmatch(r"[1-9][0-9]*-[A-Za-z0-9._:-]{1,128}\.json", old.name)
and stat.S_ISREG(info.st_mode) and info.st_uid == os.geteuid()
and now.timestamp() - info.st_mtime > MAX_AGE):
old.unlink()
entry = {"id": f"lxc:{int(vmid)}", "type": "lxc", "_vmid": str(int(vmid))}
native_detected = False
try:
detected = managed_installs._detect_lxc_containers(only_vmid=int(vmid))
if not detected:
raise RuntimeError("running LXC detection unavailable or disabled")
entry = detected[0]
native_detected = not (entry.get('_is_oci') or entry.get('_managed_oci_app'))
if refresh_indexes:
# This is a separate, explicitly requested maintenance action,
# never the read-only button or the normal post-update capture.
if entry.get('_is_oci') or entry.get('_managed_oci_app'):
raise RuntimeError('OCI containers require their own updater')
family = managed_installs._probe_lxc_os(str(vmid))
commands = {'debian': ['apt-get', 'update', '--error-on=any'],
'ubuntu': ['apt-get', 'update', '--error-on=any'],
'alpine': ['apk', 'update']}
if family not in commands:
raise RuntimeError('unsupported or unknown current OS')
refreshed = subprocess.run([managed_installs._PCT_BIN, 'exec', str(vmid),
'--', *commands[family]], capture_output=True, text=True,
timeout=managed_installs._LXC_CACHE_REFRESH_TIMEOUT_SEC)
if refreshed.returncode != 0:
raise RuntimeError('package index refresh failed; no installation was attempted')
result = managed_installs._check_lxc_updates(entry, refresh_package_cache=False)
except Exception as exc:
entry["_os_family"] = None
result = {"available": False, "last_check": managed_installs._now_iso(),
"error": f"post-update package check failed: {exc}"}
observation = {"vmid": int(vmid), "run_id": run_id,
"captured_at": now.isoformat(), "entry": entry, "result": result}
# Only the post-install runner requests app probes. Check-only and the
# maintenance preflight keep their original OS-only contract; OCI never
# enters the generic app probe path, even when explicitly requested.
if include_apps and not refresh_indexes and native_detected:
try:
import lxc_apps
observation['apps'] = lxc_apps.capture_installed_observations(vmid)
except Exception:
observation['apps'] = None
fd, temporary = tempfile.mkstemp(prefix=".observation-", dir=OBSERVATION_DIR)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(observation, handle)
os.replace(temporary, path)
print(f"[ProxMenux] LXC observation captured: vmid={int(vmid)} "
f"run_id={run_id} os={entry.get('_os_family')} "
f"count={result.get('_count')} error={bool(result.get('error'))}", flush=True)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
return observation
def read(vmid: int, run_id: str) -> dict | None:
path = _path(vmid, run_id)
def reject(reason: str) -> None:
# Only identity and a fixed reason, never the guest's payload or secrets.
print(f"[ProxMenux] LXC observation rejected: vmid={int(vmid)} "
f"run_id={run_id} reason={reason}", flush=True)
return None
try:
_private_directory(create=False)
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(fd, "r", encoding="utf-8") as handle:
info = os.fstat(handle.fileno())
if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid()
or info.st_mode & 0o077 or info.st_size > MAX_BYTES):
return reject('unsafe-file')
observation = json.load(handle)
if not isinstance(observation, dict):
return reject('invalid-document')
captured = datetime.datetime.fromisoformat(observation["captured_at"])
age = (datetime.datetime.now(datetime.timezone.utc) - captured).total_seconds()
entry, result = observation["entry"], observation["result"]
if (observation["vmid"] != int(vmid) or observation["run_id"] != run_id
or not 0 <= age <= MAX_AGE or not isinstance(entry, dict)
or entry.get("id") != f"lxc:{int(vmid)}" or entry.get("type") != "lxc"
or str(entry.get("_vmid")) != str(int(vmid)) or not isinstance(result, dict)):
return reject('identity-age-or-shape')
if not result.get("error"):
# An incomplete result is unknown, not an implicitly successful zero.
is_oci = bool(entry.get('_is_oci') or entry.get('_managed_oci_app'))
if ((not is_oci and entry.get("_os_family") not in managed_installs._SUPPORTED_OS_FAMILIES)
or not isinstance(result.get("available"), bool)
or not result.get("last_check")
or any(type(result.get(key)) is not int or result[key] < 0
for key in ("_count", "_security_count"))
or not isinstance(result.get("_packages"), list)):
return reject('incomplete-success')
# OCI skips the generic OS probe, so no family is required. Its
# trusted result must nevertheless be the complete skipped zero.
if is_oci and (result['available'] or result['_count'] != 0
or result['_security_count'] != 0 or result['_packages']):
return reject('invalid-oci-skip')
print(f"[ProxMenux] LXC observation accepted: vmid={int(vmid)} "
f"run_id={run_id} count={result.get('_count')} "
f"error={bool(result.get('error'))}", flush=True)
return observation
except (OSError, ValueError, TypeError, KeyError, OverflowError) as exc:
return reject(type(exc).__name__)
def discard(vmid: int, run_id: str) -> None:
try:
_private_directory(create=False)
_path(vmid, run_id).unlink()
except FileNotFoundError:
pass
def main(argv=None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--vmid", required=True, type=int)
parser.add_argument("--run-id", required=True)
parser.add_argument('--include-apps', action='store_true',
help='Capture registered installed versions after an update')
args = parser.parse_args(argv)
try:
capture(args.vmid, args.run_id, include_apps=args.include_apps)
return 0
except Exception as exc:
print(f"Post-update package observation unavailable: {exc}")
return 1
if __name__ == "__main__":
raise SystemExit(main())
+84 -8
View File
@@ -520,10 +520,11 @@ def _lxc_updates_notification_enabled() -> bool:
return False
def _list_pve_lxcs() -> list[dict]:
def _list_pve_lxcs(*, strict: bool = False) -> list[dict]:
"""Return basic info per LXC on this node via ``pct list``. Each
item is ``{vmid, status, name}``. Empty list on any failure — never
raises so the detector caller can continue.
raises by default. Strict callers distinguish a failed read from a
verified empty inventory before removing persisted observations.
"""
try:
r = subprocess.run(
@@ -531,16 +532,27 @@ def _list_pve_lxcs() -> list[dict]:
capture_output=True, text=True, timeout=5,
)
except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
if strict:
raise RuntimeError('LXC inventory could not be read') from None
return []
if r.returncode != 0:
if strict:
raise RuntimeError('LXC inventory could not be read')
return []
lines = r.stdout.splitlines()
if strict and (not lines or not lines[0].strip().startswith('VMID') or 'Status' not in lines[0]):
raise RuntimeError('LXC inventory is incomplete')
out: list[dict] = []
for line in r.stdout.splitlines()[1:]: # skip header row
for line in lines[1:]: # skip header row
if not line.strip():
continue
# `pct list` columns: VMID Status Lock Name
# `Lock` is empty most of the time, so split max 4 ways
parts = line.split(None, 3)
if len(parts) < 2:
if strict:
raise RuntimeError('LXC inventory is incomplete')
continue
vmid = parts[0]
status = parts[1]
@@ -548,6 +560,8 @@ def _list_pve_lxcs() -> list[dict]:
# be the name itself if Lock was omitted by the formatter.
name = parts[-1] if len(parts) >= 3 else ""
if not vmid.isdigit():
if strict:
raise RuntimeError('LXC inventory is incomplete')
continue
out.append({"vmid": vmid, "status": status, "name": name})
return out
@@ -888,7 +902,7 @@ def _probe_lxc_os(vmid: str, *, raise_on_read_error: bool = False) -> Optional[s
def _detect_lxc_containers(only_vmid: Optional[int] = None) -> list[dict]:
"""Enumerate running Debian/Ubuntu CTs as registry entries.
"""Detect running supported CTs and retain known stopped CTs by presence.
OS detection is cached in the registry entry (`_os_family`), so the
expensive ``pct exec cat /etc/os-release`` only runs during detection
@@ -920,7 +934,7 @@ def _detect_lxc_containers(only_vmid: Optional[int] = None) -> list[dict]:
if isinstance(it, dict) and it.get("type") == "lxc"
}
cts = _list_pve_lxcs()
cts = _list_pve_lxcs(strict=True)
# Set of CTs currently managed by oci_manager (Secure Gateway etc).
# Their update path is the OCI app's own updater — we mark them so
# the LXC row in the UI redirects the user there instead of running
@@ -931,11 +945,17 @@ def _detect_lxc_containers(only_vmid: Optional[int] = None) -> list[dict]:
for ct in cts:
if only_vmid is not None and str(ct.get("vmid")) != str(int(only_vmid)):
continue
if ct["status"] != "running":
continue
vmid = ct["vmid"]
cid = f"lxc:{vmid}"
prior = existing_by_id.get(cid) or {}
if ct["status"] != "running":
# A stopped CT still exists. Only the full detection sweep keeps
# its registry identity; targeted refresh requires a running CT.
# Do not copy cached probe metadata back over a newer observation.
if only_vmid is None and prior:
out.append({'id': cid, 'type': 'lxc', '_vmid': vmid,
'name': ct.get('name') or prior.get('name') or f'CT-{vmid}'})
continue
# OCI-image marker is cached — the CT origin doesn't change
# over its lifetime, and reading the pct config file is cheap
@@ -1085,11 +1105,14 @@ def detect_and_register() -> dict:
Returns the new registry.
"""
discovered: dict[str, dict] = {}
failed_types: set[str] = set()
for detector in _DETECTORS:
try:
result = detector()
except Exception as e:
print(f"[ProxMenux] managed_installs detector {detector.__name__} failed: {e}")
if detector is _detect_lxc_containers:
failed_types.add('lxc')
continue
for entry in _normalise_detector_result(result):
if not entry.get("id"):
@@ -1115,7 +1138,7 @@ def detect_and_register() -> dict:
for it in items:
if not it.get("id") or it.get("removed_at"):
continue
if it["id"] not in discovered:
if it["id"] not in discovered and it.get('type') not in failed_types:
it["removed_at"] = now
reg["items"] = items
@@ -1810,9 +1833,20 @@ def check_for_updates(force: bool = False) -> list[dict]:
with _lock:
reg = _read_registry()
items = reg.get("items", [])
lxc_states: dict[str, str] = {}
if any(it.get('type') == 'lxc' and not it.get('removed_at') for it in items):
try:
lxc_states = {str(ct['vmid']): ct['status'] for ct in _list_pve_lxcs(strict=True)}
except RuntimeError:
# Do not overwrite an observation or attempt guest probes
# when presence could not be verified by the host.
print('[ProxMenux] LXC inventory unavailable; stored checks retained')
for it in items:
if it.get("removed_at"):
continue
if (it.get('type') == 'lxc'
and lxc_states.get(str(it.get('_vmid') or it.get('id', '').removeprefix('lxc:'))) != 'running'):
continue
checker = _CHECKERS.get(it.get("type"))
if not checker:
continue
@@ -1836,6 +1870,48 @@ def check_for_updates(force: bool = False) -> list[dict]:
return updates_available
def store_lxc_update_observation(vmid: int, observation: Optional[dict]) -> bool:
"""Store a trusted host-runner observation in the backend's registry lock.
Missing verification clears the old counts, never fabricates a fresh zero.
This does not start a guest or run a package-manager command.
"""
vmid = int(vmid)
now = _now_iso()
with _lock:
# A trusted targeted observation must never replace unrelated rows
# with an empty registry merely because the old file is unreadable.
try:
with open(_REGISTRY_PATH, encoding='utf-8') as handle:
existing = json.load(handle)
if (not isinstance(existing, dict) or not isinstance(existing.get('items'), list)
or any(not isinstance(item, dict) for item in existing['items'])):
raise OSError('could not verify existing LXC update registry')
except FileNotFoundError:
pass
reg = _read_registry()
items = reg.setdefault("items", [])
target = next((it for it in items if it.get("id") == f"lxc:{vmid}"
and it.get("type") == "lxc" and not it.get("removed_at")), None)
if target is None:
if observation is None:
return False
target = _new_detected_entry(observation["entry"], now)
items.append(target)
if observation is not None:
_merge_detected_entry(target, observation["entry"], now)
result = observation["result"]
else:
target["_os_family"] = None
result = {"available": False, "last_check": now,
"error": "post-update package verification unavailable while LXC is stopped"}
_store_update_result(target, result)
reg["version"] = _SCHEMA_VERSION
if not _write_registry(reg):
raise OSError("could not persist post-update LXC observation")
return not bool(result.get("error"))
def refresh_lxc(vmid: int, *, refresh_package_cache: bool = True) -> Optional[dict]:
"""Detect and refresh exactly one running LXC.