fix: separate LXC package checks from installation

AppImage/scripts/build_appimage.sh

@@ -146,6 +146,7 @@ cp "$SCRIPT_DIR/health_thresholds.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
 cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  managed_installs.py not found"
 cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
 cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
+cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/"
 cp "$SCRIPT_DIR/lxc_apps.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  lxc_apps.py not found"
 cp "$SCRIPT_DIR/custom_links.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  custom_links.py not found"
 cp "$SCRIPT_DIR/recreate_docker_container.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️  recreate_docker_container.py not found"

AppImage/scripts/flask_server.py

@@ -6728,6 +6728,11 @@ def _get_lxc_update_status_map() -> dict:
             'managed_oci_app': it.get('_managed_oci_app'),
             'os_family': it.get('_os_family'),
         }
+        if os.path.exists(f'/run/lock/proxmenux-lxc-check-invalid-{int(vmid)}'):
+            # Read-only projection: a failed/pending explicit check must not
+            # re-expose a former positive count after a persistence failure.
+            out[str(vmid)].update(available=False, count=0, packages=[],
+                error='Latest explicit package check is running or failed; check again')
     return out
 
 
@@ -13669,19 +13674,57 @@ def api_lxc_updates_check(vmid):
                             'retry_after': retry}), 429, {'Retry-After': str(retry)}
         _lxc_update_check_attempts[vmid] = now
         _lxc_update_check_inflight.add(vmid)
+    operation_lock = None
     try:
+        import lxc_check_job
+        operation_lock = lxc_check_job.acquire(vmid)
         item = managed_installs.refresh_lxc(vmid, refresh_package_cache=False)
         if item is None:
             return jsonify({'error': 'No supported running LXC detected'}), 409
         _vm_cache_invalidate(vmid, _lxc_os_info_cache)
         return jsonify({'vmid': vmid, 'update_check': _get_lxc_update_status_map().get(str(vmid))})
+    except BlockingIOError:
+        return jsonify({'error': 'Another container operation is running'}), 409
     except Exception as exc:
         return jsonify({'error': str(exc)}), 500
     finally:
+        if operation_lock is not None:
+            operation_lock.close()
         with _lxc_update_check_lock:
             _lxc_update_check_inflight.discard(vmid)
 
 
+@app.route('/api/vms/<int:vmid>/updates/refresh-check', methods=['POST'])
+@require_auth
+def api_lxc_updates_refresh_check(vmid):
+    """Explicit index refresh/check; never installs. Worker owns restoration."""
+    import lxc_check_job
+    try:
+        result, code = lxc_check_job.start(vmid)
+        if code == 429:
+            return jsonify({'error': 'Check already running or recently requested',
+                            'retry_after': 60}), 429, {'Retry-After': '60'}
+        return jsonify(result), code
+    except BlockingIOError:
+        return jsonify({'error': 'Another container operation is running'}), 409
+    except (OSError, RuntimeError, ValueError) as exc:
+        return jsonify({'error': str(exc)}), 409
+
+
+@app.route('/api/vms/<int:vmid>/updates/refresh-check/<job_id>', methods=['GET'])
+@require_auth
+def api_lxc_updates_refresh_check_status(vmid, job_id):
+    import lxc_check_job
+    result = lxc_check_job.get(vmid, job_id)
+    if result is None:
+        return jsonify({'error': 'Check result not found; check again'}), 404
+    if result['state'] != 'running':
+        _vm_cache_invalidate(vmid, _lxc_os_info_cache)
+        _pvesh_cache['cluster_resources_vm_time'] = 0
+        result['update_check'] = _get_lxc_update_status_map().get(str(vmid))
+    return jsonify(result)
+
+
 # ─── LXC Update Detection toggle ────────────────────────────────────────────
 # Dedicated toggle so the operator can opt out of the per-CT `pct exec apt
 # list --upgradable` scan entirely. The Notifications section keeps its own
@@ -14258,6 +14301,7 @@ def api_vm_apps_dismiss(vmid):
 # ─── LXC update finalization ────────────────────────────────────────────────
 
 _LXC_APPLY_UPDATES_SCRIPT = "/usr/local/share/proxmenux/scripts/lxc/apply_updates.sh"
+_LXC_INSTALL_CHECKED_OS_SCRIPT = "/usr/local/share/proxmenux/scripts/lxc/install_checked_os_updates.sh"
 _LXC_UPDATE_RUN_ID_RE = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
 _LXC_UPDATE_TARGET_RE = re.compile(r'^[A-Za-z0-9._:-]{1,180}$')
 from lxc_update_finalization import lock as _lxc_update_finalization_lock, runs as _lxc_update_finalizations
@@ -14711,7 +14755,9 @@ def _finalize_lxc_update(
 
 
 def _terminal_lxc_update_completed(*, script_path, params, exit_code, duration_seconds):
-    if os.path.realpath(script_path) != os.path.realpath(_LXC_APPLY_UPDATES_SCRIPT):
+    if os.path.realpath(script_path) not in {
+            os.path.realpath(_LXC_APPLY_UPDATES_SCRIPT),
+            os.path.realpath(_LXC_INSTALL_CHECKED_OS_SCRIPT)}:
         return
     run_id = _normalise_lxc_update_run_id(params.get('RUN_ID'), create=False)
     if not run_id:

AppImage/scripts/lxc_check_job.py

@@ -0,0 +1,188 @@
+"""Explicit check-only jobs; the shared updater lock protects each CT.
+
+Importing or reading a job never probes a guest. Only start() may launch work.
+Package observations are saved before restoring a temporarily started guest.
+"""
+import datetime
+import fcntl
+import json
+import os
+import re
+from pathlib import Path
+import subprocess
+import threading
+import time
+import uuid
+
+import managed_installs
+import lxc_update_observation
+
+LOCK_DIR = Path(os.environ.get('PROXMENUX_LOCK_DIR', '/run/lock'))
+CONF_DIR = Path('/etc/pve/lxc')
+OCI_REGISTRY = Path('/usr/local/share/proxmenux/oci/installed.json')
+_lock = threading.RLock()
+_jobs = {}
+_attempts = {}
+COOLDOWN = 60
+JOB_TTL = 3600
+
+
+def acquire(vmid):
+    path = LOCK_DIR / f'proxmenux-lxc-update-{int(vmid)}.lock'
+    handle = os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'r+')
+    try:
+        fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return handle
+    except BaseException:
+        handle.close()
+        raise
+
+
+def guard(vmid):
+    if not managed_installs._lxc_updates_detection_enabled():
+        raise RuntimeError('LXC update detection is disabled')
+    # Fail closed if current configuration cannot be read. Never infer OCI
+    # eligibility from an old OS hint or from a failed config read.
+    config = (CONF_DIR / f'{int(vmid)}.conf').read_text()
+    # The normal collector tolerates broken OCI metadata. An action must
+    # instead fail closed: an unreadable registry is not an empty registry.
+    if OCI_REGISTRY.exists():
+        registry = json.loads(OCI_REGISTRY.read_text())
+        instances = registry.get('instances')
+        if not isinstance(instances, dict) or any(not isinstance(item, dict) for item in instances.values()):
+            raise RuntimeError('OCI registry could not be verified')
+        if any(str(item.get('vmid')) == str(vmid) for item in instances.values()):
+            raise RuntimeError('OCI containers require their own updater')
+    if any(line.lstrip().startswith(managed_installs._OCI_LXC_MARKERS)
+           for line in config.splitlines()) or str(vmid) in managed_installs._get_oci_managed_vmids():
+        raise RuntimeError('OCI containers require their own updater')
+    # Retain the independent registered-OCI boundary if its collector is
+    # temporarily unavailable. These hints can only forbid an action.
+    if any(str(item.get('_vmid')) == str(vmid) and item.get('_managed_oci_app')
+           for item in managed_installs.get_active_items()):
+        raise RuntimeError('OCI containers require their own updater')
+    if any(line.startswith('lock:') for line in config.splitlines()):
+        raise RuntimeError('Container is locked by Proxmox')
+
+
+def command(vmid, *args, timeout=30):
+    result = subprocess.run([managed_installs._PCT_BIN, *args[:1], str(vmid), *args[1:]],
+                            capture_output=True, text=True, timeout=timeout)
+    if result.returncode:
+        raise RuntimeError(f'Container {args[0]} failed')
+    return result.stdout.strip()
+
+
+def status(vmid):
+    value = command(vmid, 'status')
+    if value not in ('status: running', 'status: stopped'):
+        raise RuntimeError('Container state could not be determined')
+    return value.split(': ', 1)[1]
+
+
+def checked_os(vmid):
+    """Installation precondition; called under the updater lock before start."""
+    guard(vmid)
+    if (LOCK_DIR / f'proxmenux-lxc-check-invalid-{int(vmid)}').exists():
+        raise RuntimeError('Latest check failed or is still running')
+    item = next((item for item in managed_installs.get_active_items()
+                 if item.get('id') == f'lxc:{int(vmid)}'), {})
+    observation = item.get('update_check') or {}
+    checked = datetime.datetime.fromisoformat(observation.get('last_check') or '')
+    if checked.tzinfo is None:
+        checked = checked.replace(tzinfo=datetime.timezone.utc)
+    age = (datetime.datetime.now(datetime.timezone.utc) - checked).total_seconds()
+    if (not 0 <= age < 172800 or observation.get('error')
+            or observation.get('available') is not True
+            or type(observation.get('_count')) is not int or observation['_count'] <= 0):
+        raise RuntimeError('Check OS updates successfully before installation')
+
+
+def run(vmid, job_id, reservation):
+    restore = False
+    error = None
+    invalid = LOCK_DIR / f'proxmenux-lxc-check-invalid-{int(vmid)}'
+    try:
+        invalid.touch(mode=0o600)
+        guard(vmid)
+        original = status(vmid)
+        if original == 'stopped':
+            # Set before start: a timed-out command can still have started CT.
+            restore = True
+            command(vmid, 'start', timeout=60)
+            deadline = time.monotonic() + 60
+            while True:
+                try:
+                    command(vmid, 'exec', '--', 'true', timeout=5)
+                    break
+                except (RuntimeError, subprocess.TimeoutExpired):
+                    if time.monotonic() >= deadline:
+                        raise RuntimeError('Container did not become ready')
+                    time.sleep(1)
+        observation = lxc_update_observation.capture(vmid, job_id, refresh_indexes=True)
+        if not managed_installs.store_lxc_update_observation(vmid, observation):
+            raise RuntimeError(observation['result'].get('error') or 'Package check failed')
+        lxc_update_observation.discard(vmid, job_id)
+    except Exception as exc:
+        error = str(exc)
+        # Preflight/start failures must revoke a previous positive observation.
+        try:
+            managed_installs.store_lxc_update_observation(vmid, None)
+        except Exception:
+            error += '; package result could not be persisted'
+    finally:
+        if restore:
+            try:
+                if status(vmid) != 'stopped':
+                    command(vmid, 'shutdown', '--timeout', '30', timeout=40)
+                if status(vmid) != 'stopped':
+                    raise RuntimeError('Container is still running')
+            except Exception:
+                try:
+                    command(vmid, 'stop', timeout=30)
+                    if status(vmid) != 'stopped':
+                        raise RuntimeError('Container is still running')
+                except Exception:
+                    error = (error + '; ' if error else '') + 'Original stopped state could not be restored'
+        if not error:
+            try:
+                invalid.unlink()
+            except OSError:
+                error = 'Successful check could not be published for installation'
+        reservation.close()
+        with _lock:
+            _jobs[job_id].update(state='failed' if error else 'complete', error=error)
+
+
+def start(vmid):
+    now = time.monotonic()
+    with _lock:
+        for key in list(_jobs):
+            if _jobs[key]['state'] != 'running' and now - _jobs[key]['created'] >= JOB_TTL:
+                del _jobs[key]
+        last = _attempts.get(vmid)
+        if (any(job['vmid'] == vmid and job['state'] == 'running' for job in _jobs.values())
+                or last is not None and now - last < COOLDOWN):
+            return None, 429
+        reservation = acquire(vmid)
+        try:
+            guard(vmid)
+            job_id = uuid.uuid4().hex
+            _jobs[job_id] = dict(vmid=vmid, state='running', error=None, created=now)
+            _attempts[vmid] = now
+            threading.Thread(target=run, args=(vmid, job_id, reservation), daemon=False).start()
+        except BaseException:
+            if 'job_id' in locals():
+                _jobs.pop(job_id, None)
+                _attempts.pop(vmid, None)
+            reservation.close()
+            raise
+        return dict(job_id=job_id, vmid=vmid, state='running'), 202
+
+
+def get(vmid, job_id):
+    with _lock:
+        job = _jobs.get(job_id)
+        if job is None or job['vmid'] != vmid:
+            return None
+        return {key: value for key, value in job.items() if key != 'created'}