fix: wait for VM power tasks before completing controls

AppImage/components/virtual-machines.tsx

@@ -28,6 +28,7 @@ import { AppUpdaterEditor, type AppUpdateMethod } from "./app-updater-editor"
 import { formatStorage } from "../lib/utils"
 import { formatNetworkTraffic, getNetworkUnit } from "../lib/format-network"
 import { fetchApi } from "../lib/api-config"
+import { waitForVmControlTask } from "../lib/vm-control"
 import { matchesVmSearch, normalizeVmSearchValue } from "../lib/vm-search"
 import { refreshCheckLxcUpdates, getLxcOsUpdateState, isLxcUpdateStale } from "../lib/lxc-update-observation"
 import DOMPurify from "dompurify"
@@ -872,6 +873,8 @@ export function VirtualMachines() {
   const lifecycleCacheRevisionsRef = useRef(new Map<number, number>())
   const [dockerInventoryRefreshingVmid, setDockerInventoryRefreshingVmid] = useState<number | null>(null)
   const [controlLoading, setControlLoading] = useState(false)
+  const controlRequestRef = useRef(false)
+  const [controlAction, setControlAction] = useState<{ vmid: number; action: string } | null>(null)
   // Destructive control confirmation. `Force Stop` and `Reboot` skip the OS
   // shutdown sequence and can corrupt running guests; gate them behind a
   // typed-VMID match prompt to prevent misclicks. See audit Tier 2 #17.
@@ -1797,12 +1800,16 @@ export function VirtualMachines() {
   }
 
   const handleVMControl = async (vmid: number, action: string) => {
+    if (controlRequestRef.current) return
+    controlRequestRef.current = true
+    setControlAction({ vmid, action })
     setControlLoading(true)
     try {
-      await fetchApi(`/api/vms/${vmid}/control`, {
+      const accepted = await fetchApi<{ task: string }>(`/api/vms/${vmid}/control`, {
         method: "POST",
         body: JSON.stringify({ action }),
       })
+      const completed = await waitForVmControlTask(vmid, action, accepted.task)
 
       // Any control action can change what the guest reports: a stop
       // hides runtime-only fields, a start may bring a new config that
@@ -1818,16 +1825,37 @@ export function VirtualMachines() {
       cache.firewall.delete(vmid)
       invalidateLxcApps(vmid)
 
-      mutate()
-      setSelectedVM(null)
-      setVMDetails(null)
+      // Publish the confirmed result as an SWR mutation before revalidating.
+      // SWR then discards list requests started before this completion.
+      try {
+        await mutate(current => current?.map(vm => vm.vmid === vmid
+          ? { ...vm, status: completed.guest_status as VMData["status"] } : vm), { revalidate: false })
+        await mutate()
+      } catch { /* Polling will retry; control already succeeded. */ }
+      // Keep the user's tab/scroll and last complete detail while revalidating.
+      // Never resurrect a modal the user closed, or overwrite another guest.
+      if (selectedVMRef.current?.vmid === vmid) {
+        setSelectedVM(current => current?.vmid === vmid
+          ? { ...current, status: completed.guest_status as VMData["status"] } : current)
+        try {
+          const details = await fetchApi<VMDetails>(`/api/vms/${vmid}`)
+          cache.details.set(vmid, details)
+          if (selectedVMRef.current?.vmid === vmid) setVMDetails(details)
+        } catch {
+          // The control succeeded; detail polling will retry independently.
+        }
+      }
     } catch (error) {
       console.error(`Failed to ${action} VM ${vmid}:`, error)
       // Same UX issue as handleCreateBackup: a silent console.error left the
       // user looking at a "Stop"/"Start" button that just never reacted.
-      const msg = error instanceof Error ? error.message : t("vmLxc.errors.unknown")
+      const msg = error instanceof Error
+        ? (error.message === "controlNotConfirmed" ? t("vmLxc.errors.controlNotConfirmed") : error.message)
+        : t("vmLxc.errors.unknown")
       alert(t("vmLxc.errors.controlFailed", { action, vmid, message: msg }))
     } finally {
+      controlRequestRef.current = false
+      setControlAction(null)
       setControlLoading(false)
     }
   }
@@ -7623,8 +7651,11 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                     tab we added. */}
                 {(() => {
                   const hasTerminal = selectedVM?.type === "lxc" && selectedVM?.status === "running"
+                  const pendingAction = controlAction?.vmid === selectedVM?.vmid ? controlAction?.action : null
+                  const controlLabel = (action: string, key: string) => pendingAction === action
+                    ? t(`vmLxc.actions.${action}Pending`) : t(key)
                   return (
-                    <div className={`grid gap-2 sm:gap-3 ${hasTerminal ? "grid-cols-4 lg:grid-cols-5" : "grid-cols-4"}`}>
+                    <div aria-busy={controlLoading} className={`grid gap-2 sm:gap-3 ${hasTerminal ? "grid-cols-4 lg:grid-cols-5" : "grid-cols-4"}`}>
                       {hasTerminal && (
                         <Button
                           className="w-full bg-zinc-600/20 border border-zinc-600/50 text-zinc-300 hover:bg-zinc-600/30 col-span-4 lg:col-span-1"
@@ -7638,19 +7669,19 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                         className="w-full bg-green-600/20 border border-green-600/50 text-green-400 hover:bg-green-600/30"
                         disabled={selectedVM?.status === "running" || controlLoading}
                         onClick={() => selectedVM && handleVMControl(selectedVM.vmid, "start")}
-                        title={t("vmLxc.actions.start")}
+                        title={controlLabel("start", "vmLxc.actions.start")}
                       >
-                        <Play className="h-4 w-4 sm:mr-2" />
-                        <span className="hidden sm:inline">{t("vmLxc.actions.start")}</span>
+                        {pendingAction === "start" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <Play className="h-4 w-4 sm:mr-2" />}
+                        <span className="hidden sm:inline">{controlLabel("start", "vmLxc.actions.start")}</span>
                       </Button>
                       <Button
                         className="w-full bg-blue-600/20 border border-blue-600/50 text-blue-400 hover:bg-blue-600/30"
                         disabled={selectedVM?.status !== "running" || controlLoading}
                         onClick={() => selectedVM && handleVMControl(selectedVM.vmid, "shutdown")}
-                        title={t("vmLxc.actions.shutdown")}
+                        title={controlLabel("shutdown", "vmLxc.actions.shutdown")}
                       >
-                        <Power className="h-4 w-4 sm:mr-2" />
-                        <span className="hidden sm:inline">{t("vmLxc.actions.shutdown")}</span>
+                        {pendingAction === "shutdown" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <Power className="h-4 w-4 sm:mr-2" />}
+                        <span className="hidden sm:inline">{controlLabel("shutdown", "vmLxc.actions.shutdown")}</span>
                       </Button>
                       <Button
                         className="w-full bg-blue-600/20 border border-blue-600/50 text-blue-400 hover:bg-blue-600/30"
@@ -7660,10 +7691,10 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                           vmid: selectedVM.vmid,
                           vmName: selectedVM.name,
                         })}
-                        title={t("vmLxc.actions.reboot")}
+                        title={controlLabel("reboot", "vmLxc.actions.reboot")}
                       >
-                        <RotateCcw className="h-4 w-4 sm:mr-2" />
-                        <span className="hidden sm:inline">{t("vmLxc.actions.reboot")}</span>
+                        {pendingAction === "reboot" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <RotateCcw className="h-4 w-4 sm:mr-2" />}
+                        <span className="hidden sm:inline">{controlLabel("reboot", "vmLxc.actions.reboot")}</span>
                       </Button>
                       <Button
                         className="w-full bg-red-600/20 border border-red-600/50 text-red-400 hover:bg-red-600/30"
@@ -7673,10 +7704,10 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
                           vmid: selectedVM.vmid,
                           vmName: selectedVM.name,
                         })}
-                        title={t("vmLxc.actions.forceStop")}
+                        title={controlLabel("stop", "vmLxc.actions.forceStop")}
                       >
-                        <StopCircle className="h-4 w-4 sm:mr-2" />
-                        <span className="hidden sm:inline">{t("vmLxc.actions.forceStop")}</span>
+                        {pendingAction === "stop" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <StopCircle className="h-4 w-4 sm:mr-2" />}
+                        <span className="hidden sm:inline">{controlLabel("stop", "vmLxc.actions.forceStop")}</span>
                       </Button>
                     </div>
                   )

AppImage/lib/vm-control.ts

@@ -0,0 +1,32 @@
+import { fetchApi } from './api-config'
+
+// A successful control POST accepts an asynchronous Proxmox task. In
+// particular, an unchanged "running" status cannot confirm a reboot.
+export async function waitForVmControlTask(vmid: number, action: string, task: string) {
+  if (!task) throw new Error('controlNotConfirmed')
+  const expected = action === 'shutdown' || action === 'stop' ? 'stopped' : 'running'
+  const deadline = Date.now() + 180_000
+  let readFailures = 0
+  while (Date.now() < deadline) {
+    let result: { status?: string; exitstatus?: string; guest_status?: string }
+    try {
+      result = await fetchApi(`/api/vms/${vmid}/control-task/${encodeURIComponent(task)}`, {
+        signal: AbortSignal.timeout(10_000),
+      })
+      readFailures = 0
+    } catch (error) {
+      if (++readFailures >= 3) throw new Error('controlNotConfirmed')
+      await new Promise(resolve => setTimeout(resolve, 1000))
+      continue
+    }
+    if (result?.status === 'stopped') {
+      if (!result.exitstatus) throw new Error('controlNotConfirmed')
+      if (result.exitstatus !== 'OK') throw new Error(result.exitstatus)
+      if (result.guest_status === expected) return result
+    } else if (result?.status !== 'running') {
+      throw new Error('controlNotConfirmed')
+    }
+    await new Promise(resolve => setTimeout(resolve, 1000))
+  }
+  throw new Error('controlNotConfirmed')
+}

AppImage/messages/en/common.json

@@ -1100,7 +1100,11 @@
       "shutdown": "Shutdown",
       "reboot": "Reboot",
       "forceStop": "Force Stop",
-      "working": "Working..."
+      "working": "Working...",
+      "startPending": "Starting...",
+      "shutdownPending": "Shutting down...",
+      "rebootPending": "Restarting...",
+      "stopPending": "Stopping..."
     },
     "confirm": {
       "forceStopTitle": "Force Stop",
@@ -1144,6 +1148,7 @@
       "unknown": "Unknown error",
       "backupStartFailed": "Failed to start backup: {message}",
       "controlFailed": "Failed to {action} VM {vmid}: {message}",
+      "controlNotConfirmed": "The operation has not been confirmed complete. Check the guest status before trying again.",
       "saveNotesFailed": "Error saving notes. Please try again.",
       "appNotFound": "This application is no longer available. Refresh the page and try again.",
       "saveCustomCommandFailed": "Could not save the custom update command: {message}",

AppImage/messages/sk/common.json

@@ -1100,7 +1100,11 @@
       "shutdown": "Vypnúť",
       "reboot": "Reštartovať",
       "forceStop": "Vynútiť vypnutie",
-      "working": "Pracujem..."
+      "working": "Pracujem...",
+      "startPending": "Spúšťa sa...",
+      "shutdownPending": "Vypína sa...",
+      "rebootPending": "Reštartuje sa...",
+      "stopPending": "Vypína sa..."
     },
     "confirm": {
       "forceStopTitle": "Vynútiť vypnutie",
@@ -1144,6 +1148,7 @@
       "unknown": "Neznáma chyba",
       "backupStartFailed": "Zálohu sa nepodarilo spustiť: {message}",
       "controlFailed": "Akcia {action} pre VM {vmid} zlyhala: {message}",
+      "controlNotConfirmed": "Dokončenie operácie sa zatiaľ nepodarilo potvrdiť. Pred ďalším pokusom skontrolujte stav VM alebo LXC.",
       "saveNotesFailed": "Poznámky sa nepodarilo uložiť. Skúste to znova.",
       "appNotFound": "Táto aplikácia už nie je dostupná. Obnovte stránku a skúste to znova.",
       "saveCustomCommandFailed": "Vlastný príkaz na aktualizáciu sa nepodarilo uložiť: {message}",

AppImage/scripts/flask_server.py

@@ -6847,10 +6847,15 @@ def get_proxmox_vms():
                         continue
 
                     vm_type = 'lxc' if resource.get('type') == 'lxc' else 'qemu'
+                    # The PVE summary can lag behind a completed LXC task.
+                    # Use the same direct local cgroup check as the detail
+                    # route; never resurrect a stopped CT from that summary.
+                    vm_status = (_fast_guest_status(resource.get('vmid'), vm_type)
+                                 if vm_type == 'lxc' else resource.get('status', 'unknown'))
                     vm_data = {
                         'vmid': resource.get('vmid'),
                         'name': resource.get('name', f"VM-{resource.get('vmid')}"),
-                        'status': resource.get('status', 'unknown'),
+                        'status': vm_status,
                         'type': vm_type,
                         'cpu': resource.get('cpu', 0),
                         'mem': resource.get('mem', 0),
@@ -6905,7 +6910,7 @@ def get_proxmox_vms():
                         # detail-aware clients.  The cache is still warmed
                         # at startup and invalidated by LXC lifecycle events.
                         vm_data['ips'] = []
-                        if resource.get('status') == 'running':
+                        if vm_data['status'] == 'running':
                             lxc_ip_info = _get_lxc_ip_info_cached(resource.get('vmid'))
                             lxc_ips = _normalise_lxc_ips(lxc_ip_info)
                             vm_data['ips'] = lxc_ips
@@ -15870,6 +15875,7 @@ def api_vm_config_set(vmid):
 @require_auth
 def api_vm_control(vmid):
     """Control VM/LXC (start, stop, shutdown, reboot)"""
+    operation_lock = None
     try:
         data = request.get_json()
         action = data.get('action')  # start, stop, shutdown, reboot
@@ -15892,6 +15898,13 @@ def api_vm_control(vmid):
             
             vm_type = 'lxc' if vm_info.get('type') == 'lxc' else 'qemu'
             node = vm_info.get('node', 'pve')
+            if vm_type == 'lxc':
+                import lxc_check_job
+                try:
+                    operation_lock = lxc_check_job.acquire(vmid)
+                except BlockingIOError:
+                    return jsonify({'error': 'Another container operation is running'}), 409
+                lxc_check_job.begin_control(operation_lock, node)
             
             # Execute action
             control_result = subprocess.run(
@@ -15906,13 +15919,22 @@ def api_vm_control(vmid):
                 # (lxc_ip_info, os_info) — a start/stop flips those, so
                 # drop the modal caches for this vmid.
                 _vm_cache_invalidate(vmid)
+                # Keep the legacy acceptance response; new clients also wait
+                # for this specific task instead of assuming it has finished.
+                task = (control_result.stdout or '').strip().strip('"')
+                if operation_lock is not None and task.startswith('UPID:'):
+                    lxc_check_job.retain_control(vmid, task, operation_lock)
+                    operation_lock = None
                 return jsonify({
                     'success': True,
                     'vmid': vmid,
                     'action': action,
+                    'task': task if task.startswith('UPID:') else None,
                     'message': f'Successfully executed {action} on {vm_info.get("name")}'
                 })
             else:
+                if operation_lock is not None:
+                    lxc_check_job.cancel_control(operation_lock)
                 # `pvesh` failed → fire the matching vm_fail / ct_fail
                 # notification so the user gets paged on their channels
                 # too, not just an in-dashboard alert. Previously this
@@ -15966,6 +15988,55 @@ def api_vm_control(vmid):
     except Exception as e:
         return jsonify({'error': str(e)}), 500
 
+    finally:
+        if operation_lock is not None:
+            operation_lock.close()
+
+@app.route('/api/vms/<int:vmid>/control-task/<path:task>', methods=['GET'])
+@require_auth
+def api_vm_control_task(vmid, task):
+    """Read only the power-control task bound to this guest and node."""
+    import re
+    import json
+    if not re.fullmatch(r'UPID:[\w.-]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:'
+                        r'(?:vz|qm)(?:start|stop|shutdown|reboot):[0-9]+:[\w.@-]+:', task):
+        return jsonify({'error': 'Invalid control task'}), 400
+    parts = task.split(':')
+    resource = next((item for item in get_cached_pvesh_cluster_resources_vm() or []
+                     if item.get('vmid') == vmid), None)
+    if resource is None:
+        return jsonify({'error': 'VM/LXC not found'}), 404
+    vm_type = 'lxc' if resource.get('type') == 'lxc' else 'qemu'
+    node = resource.get('node', 'pve')
+    prefix = 'vz' if vm_type == 'lxc' else 'qm'
+    if parts[1] != node or parts[6] != str(vmid) or not parts[5].startswith(prefix):
+        return jsonify({'error': 'Control task does not belong to this guest'}), 400
+    try:
+        result = subprocess.run(['pvesh', 'get', f'/nodes/{node}/tasks/{task}/status',
+                                 '--output-format', 'json'],
+                                capture_output=True, text=True, timeout=5)
+        if result.returncode != 0:
+            return jsonify({'error': 'Control task status unavailable'}), 502
+        status = json.loads(result.stdout)
+        if not isinstance(status, dict) or status.get('status') not in ('running', 'stopped'):
+            return jsonify({'error': 'Invalid control task status'}), 502
+        payload = {'status': status['status'], 'exitstatus': status.get('exitstatus')}
+        if status['status'] == 'stopped' and status.get('exitstatus') == 'OK':
+            current = subprocess.run(['pvesh', 'get', f'/nodes/{node}/{vm_type}/{vmid}/status/current',
+                                      '--output-format', 'json'],
+                                     capture_output=True, text=True, timeout=5)
+            if current.returncode != 0:
+                return jsonify({'error': 'Guest status unavailable'}), 502
+            guest = json.loads(current.stdout)
+            if not isinstance(guest, dict) or guest.get('status') not in ('running', 'stopped'):
+                return jsonify({'error': 'Invalid guest status'}), 502
+            payload['guest_status'] = guest['status']
+            _pvesh_cache['cluster_resources_vm_time'] = 0
+            _vm_cache_invalidate(vmid)
+        return jsonify(payload)
+    except (ValueError, subprocess.TimeoutExpired):
+        return jsonify({'error': 'Control task status unavailable'}), 502
+
 @app.route('/api/vms/<int:vmid>/description', methods=['PUT'])
 @app.route('/api/vms/<int:vmid>/config', methods=['PUT'])  # legacy alias
 @require_auth

AppImage/scripts/lxc_check_job.py

@@ -32,6 +32,7 @@ def acquire(vmid):
     handle = os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'r+')
     try:
         fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        check_control_task(vmid, handle)
         return handle
     except BaseException:
         handle.close()
@@ -85,6 +86,87 @@ def clear_invalid_marker(vmid, *, check_started_ns=None):
         handle.close()
 
 
+def _write_control(handle, record):
+    handle.seek(0)
+    if record is not None:
+        handle.write(json.dumps(record))
+    handle.flush()
+    # Do not erase the old reservation before the replacement was written.
+    # A partial/failed write stays non-empty and therefore fails closed.
+    handle.truncate(handle.tell())
+
+
+def begin_control(handle, node):
+    """Persist uncertainty before submission, including across Monitor restart."""
+    if not isinstance(node, str) or not re.fullmatch(r'[\w.-]+', node):
+        raise ValueError('Invalid control node')
+    _write_control(handle, {'node': node, 'task': None})
+
+
+def cancel_control(handle):
+    """Only a definite submission failure can clear an unconfirmed request."""
+    _write_control(handle, None)
+
+
+def check_control_task(vmid, handle=None):
+    """Called under the shared flock, also by the shell update runner.
+
+    No record means no probe. A restart releases OS file descriptors, but the
+    record still forbids a new operation until PVE confirms task termination.
+    Unknown submission/status cannot safely be treated as completion.
+    """
+    if handle is None:
+        with open(LOCK_DIR / f'proxmenux-lxc-update-{int(vmid)}.lock', 'r+') as stream:
+            return check_control_task(vmid, stream)
+    handle.seek(0)
+    raw = handle.read(1025)
+    if not raw:
+        return
+    try:
+        record = json.loads(raw)
+        node, task = record['node'], record['task']
+        if not isinstance(node, str) or not re.fullmatch(r'[\w.-]+', node):
+            raise ValueError('Invalid node')
+        if (not isinstance(task, str)
+                or not re.fullmatch(r'UPID:[\w.-]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:'
+                                    r'[0-9A-Fa-f]+:vz(?:start|stop|shutdown|reboot):'
+                                    r'[0-9]+:[\w.@!-]+:', task)
+                or task.split(':')[1] != node or task.split(':')[6] != str(vmid)):
+            raise ValueError('Unconfirmed control task')
+        result = subprocess.run(['pvesh', 'get', f'/nodes/{node}/tasks/{task}/status',
+                                '--output-format', 'json'],
+                               capture_output=True, text=True, timeout=5)
+        status = json.loads(result.stdout) if result.returncode == 0 else None
+        if (not isinstance(status, dict) or status.get('status') != 'stopped'
+                or not isinstance(status.get('exitstatus'), str) or not status['exitstatus']):
+            raise ValueError('Control task completion not confirmed')
+    except (ValueError, KeyError, TypeError, OSError, subprocess.TimeoutExpired) as exc:
+        raise BlockingIOError('Container control completion is not confirmed') from exc
+    _write_control(handle, None)
+
+
+def _wait_control(vmid, reservation):
+    try:
+        while True:
+            try:
+                check_control_task(vmid, reservation)
+                return
+            except BlockingIOError:
+                # A status timeout must not unlock a task that may still run.
+                time.sleep(2)
+    finally:
+        reservation.close()
+
+
+def retain_control(vmid, task, reservation):
+    reservation.seek(0)
+    record = json.loads(reservation.read(1025))
+    record['task'] = task
+    _write_control(reservation, record)
+    # Reservation ownership transfers only after Thread.start succeeds.
+    threading.Thread(target=_wait_control, args=(vmid, reservation), daemon=True).start()
+
+
 def guard(vmid):
     if not managed_installs._lxc_updates_detection_enabled():
         raise RuntimeError('LXC update detection is disabled')