Keep Monitor-started backups running across a Monitor restart

AppImage/scripts/flask_server.py

@@ -13040,77 +13040,117 @@ def api_create_backup(vmid):
         if pbs_change_detection and pbs_change_detection != 'default' and vm_type == 'lxc':
             cmd.extend(['--pbs-change-detection-mode', pbs_change_detection])
         
-        # Start vzdump detached from the Flask worker.
+        # Start vzdump in its own transient systemd unit. A child of this
-        # subprocess.run(timeout=60) SIGKILLs pvesh (and vzdump with it)
+        # service lives in its cgroup and is killed with it on any Monitor
-        # the moment a backup runs longer than 60s — a 100 GB VM easily
+        # restart (an update, a deploy), taking the backup down mid-upload.
-        # needs 2+ min and lands as `interrupted by signal`, leaving
+        # pvesh writes the task UPID to a file, read back here; the backup
-        # `.tar.dat` orphans and vzdumptmp dirs on disk. GH #295.
+        # then runs to completion on its own and the frontend polls
-        # Instead spawn pvesh in its own session so this endpoint can
+        # /api/task-log/<upid> for progress. GH #295 covers the older
-        # return as soon as the UPID is on stdout, and let the backup
+        # 60-second kill this also avoids.
-        # run to completion on its own. The frontend can then poll
-        # /api/task-log/<upid> for progress.
-        try:
-            proc = subprocess.Popen(
-                cmd,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                text=True,
-                start_new_session=True,
-            )
-        except Exception as e:
-            return jsonify({
-                'success': False,
-                'error': f'Failed to spawn pvesh: {e}',
-                'command': ' '.join(cmd),
-            }), 500
-
-        # Wait up to 10s for the UPID line (typically appears in <1s).
-        # If pvesh exits with error before that, surface the real cause.
-        upid = None
-        buf = ''
-        deadline = time.time() + 10.0
         upid_re = re.compile(
             r'UPID:[^:\s]+:[^:\s]+:[^:\s]+:[^:\s]+:vzdump:[^:\s]+:[^:\s]+:'
         )
-        while time.time() < deadline:
+        upid = None
-            remaining = max(0.05, deadline - time.time())
+        buf = ''
-            rlist, _, _ = select.select([proc.stdout], [], [], min(remaining, 1.0))
+        if shutil.which('systemd-run'):
-            if rlist:
+            run_dir = '/run/proxmenux'
-                chunk = proc.stdout.readline()
+            try:
-                if not chunk:
+                os.makedirs(run_dir, mode=0o700, exist_ok=True)
-                    break
+                now = time.time()
-                buf += chunk
+                for old in os.listdir(run_dir):
+                    old_path = os.path.join(run_dir, old)
+                    if old.startswith('vzdump-') and now - os.path.getmtime(old_path) > 86400:
+                        os.remove(old_path)
+            except OSError:
+                pass
+            unit = f'proxmenux-vzdump-{vmid}-{uuid.uuid4().hex[:8]}'
+            out_path = os.path.join(run_dir, f'vzdump-{unit}.out')
+            launch = subprocess.run(
+                ['systemd-run', f'--unit={unit}', '--collect', '--quiet',
+                 f'--description=ProxMenux backup of {vm_type.upper()} {vmid}',
+                 f'--property=StandardOutput=file:{out_path}', *cmd],
+                capture_output=True, text=True, timeout=15,
+            )
+            if launch.returncode != 0:
+                return jsonify({
+                    'success': False,
+                    'error': f'Failed to start the backup: {(launch.stderr or launch.stdout).strip()}',
+                    'command': ' '.join(cmd),
+                }), 500
+            deadline = time.time() + 10.0
+            while time.time() < deadline:
+                try:
+                    with open(out_path, encoding='utf-8', errors='replace') as handle:
+                        buf = handle.read()
+                except OSError:
+                    buf = ''
                 m = upid_re.search(buf)
                 if m:
                     upid = m.group(0).rstrip(':')
                     break
-            elif proc.poll() is not None:
+                state = subprocess.run(['systemctl', 'is-active', unit],
-                break
+                                       capture_output=True, text=True).stdout.strip()
-
+                if state not in ('active', 'activating'):
-        if upid is None and proc.poll() is not None and proc.returncode != 0:
+                    break
-            tail = buf.strip() or 'pvesh exited without producing a UPID'
+                time.sleep(0.3)
-            return jsonify({
+            if upid is None:
-                'success': False,
+                tail = buf.strip() or 'pvesh exited without producing a UPID'
-                'error': f'Backup failed: {tail}',
+                return jsonify({
-                'command': ' '.join(cmd),
+                    'success': False,
-            }), 500
+                    'error': f'Backup failed: {tail[-800:]}',
-
+                    'command': ' '.join(cmd),
-        # Drain stdout in the background so the pipe doesn't fill and
+                }), 500
-        # stall pvesh once vzdump starts emitting progress lines. We
+        else:
-        # don't wait() — the backup outlives this HTTP request; PVE
-        # cleans up the task itself.
-        def _drain(p):
             try:
-                for _ in iter(p.stdout.readline, ''):
+                proc = subprocess.Popen(
-                    pass
+                    cmd,
-            except Exception:
+                    stdout=subprocess.PIPE,
-                pass
+                    stderr=subprocess.STDOUT,
-            finally:
+                    text=True,
+                    start_new_session=True,
+                )
+            except Exception as e:
+                return jsonify({
+                    'success': False,
+                    'error': f'Failed to spawn pvesh: {e}',
+                    'command': ' '.join(cmd),
+                }), 500
+            deadline = time.time() + 10.0
+            while time.time() < deadline:
+                remaining = max(0.05, deadline - time.time())
+                rlist, _, _ = select.select([proc.stdout], [], [], min(remaining, 1.0))
+                if rlist:
+                    chunk = proc.stdout.readline()
+                    if not chunk:
+                        break
+                    buf += chunk
+                    m = upid_re.search(buf)
+                    if m:
+                        upid = m.group(0).rstrip(':')
+                        break
+                elif proc.poll() is not None:
+                    break
+            if upid is None and proc.poll() is not None and proc.returncode != 0:
+                tail = buf.strip() or 'pvesh exited without producing a UPID'
+                return jsonify({
+                    'success': False,
+                    'error': f'Backup failed: {tail}',
+                    'command': ' '.join(cmd),
+                }), 500
+
+            # Drain stdout so the pipe doesn't fill and stall pvesh.
+            def _drain(p):
                 try:
-                    p.stdout.close()
+                    for _ in iter(p.stdout.readline, ''):
+                        pass
                 except Exception:
                     pass
-        threading.Thread(target=_drain, args=(proc,), daemon=True).start()
+                finally:
+                    try:
+                        p.stdout.close()
+                    except Exception:
+                        pass
+            threading.Thread(target=_drain, args=(proc,), daemon=True).start()
 
         # New task started — the backups list will change soon (either
         # the new archive appears, or the task fails and something is
@@ -14644,6 +14684,7 @@ def api_health_unacknowledge():
 
         result = health_persistence.unacknowledge_error(error_key)
         # Invalidate caches so the next health fetch reflects the new state.
+        from health_monitor import health_monitor
         for ck in ['_bg_overall', '_bg_detailed', 'overall_health',
                    'storage_check', 'vms_check', 'logs_analysis',
                    'pve_services', 'updates_check', 'security_check',