mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 19:02:31 +00:00
Merge pull request #449 from Vaso73/fix/vm-control-task-completion
fix: wait for VM power tasks before completing controls
This commit is contained in:
@@ -28,6 +28,7 @@ import { AppUpdaterEditor, type AppUpdateMethod } from "./app-updater-editor"
|
||||
import { formatStorage } from "../lib/utils"
|
||||
import { formatNetworkTraffic, getNetworkUnit } from "../lib/format-network"
|
||||
import { fetchApi } from "../lib/api-config"
|
||||
import { waitForVmControlTask } from "../lib/vm-control"
|
||||
import { matchesVmSearch, normalizeVmSearchValue } from "../lib/vm-search"
|
||||
import { refreshCheckLxcUpdates, getLxcOsUpdateState, isLxcUpdateStale } from "../lib/lxc-update-observation"
|
||||
import DOMPurify from "dompurify"
|
||||
@@ -872,6 +873,8 @@ export function VirtualMachines() {
|
||||
const lifecycleCacheRevisionsRef = useRef(new Map<number, number>())
|
||||
const [dockerInventoryRefreshingVmid, setDockerInventoryRefreshingVmid] = useState<number | null>(null)
|
||||
const [controlLoading, setControlLoading] = useState(false)
|
||||
const controlRequestRef = useRef(false)
|
||||
const [controlAction, setControlAction] = useState<{ vmid: number; action: string } | null>(null)
|
||||
// Destructive control confirmation. `Force Stop` and `Reboot` skip the OS
|
||||
// shutdown sequence and can corrupt running guests; gate them behind a
|
||||
// typed-VMID match prompt to prevent misclicks. See audit Tier 2 #17.
|
||||
@@ -1797,12 +1800,16 @@ export function VirtualMachines() {
|
||||
}
|
||||
|
||||
const handleVMControl = async (vmid: number, action: string) => {
|
||||
if (controlRequestRef.current) return
|
||||
controlRequestRef.current = true
|
||||
setControlAction({ vmid, action })
|
||||
setControlLoading(true)
|
||||
try {
|
||||
await fetchApi(`/api/vms/${vmid}/control`, {
|
||||
const accepted = await fetchApi<{ task: string }>(`/api/vms/${vmid}/control`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ action }),
|
||||
})
|
||||
const completed = await waitForVmControlTask(vmid, action, accepted.task)
|
||||
|
||||
// Any control action can change what the guest reports: a stop
|
||||
// hides runtime-only fields, a start may bring a new config that
|
||||
@@ -1818,16 +1825,37 @@ export function VirtualMachines() {
|
||||
cache.firewall.delete(vmid)
|
||||
invalidateLxcApps(vmid)
|
||||
|
||||
mutate()
|
||||
setSelectedVM(null)
|
||||
setVMDetails(null)
|
||||
// Publish the confirmed result as an SWR mutation before revalidating.
|
||||
// SWR then discards list requests started before this completion.
|
||||
try {
|
||||
await mutate(current => current?.map(vm => vm.vmid === vmid
|
||||
? { ...vm, status: completed.guest_status as VMData["status"] } : vm), { revalidate: false })
|
||||
await mutate()
|
||||
} catch { /* Polling will retry; control already succeeded. */ }
|
||||
// Keep the user's tab/scroll and last complete detail while revalidating.
|
||||
// Never resurrect a modal the user closed, or overwrite another guest.
|
||||
if (selectedVMRef.current?.vmid === vmid) {
|
||||
setSelectedVM(current => current?.vmid === vmid
|
||||
? { ...current, status: completed.guest_status as VMData["status"] } : current)
|
||||
try {
|
||||
const details = await fetchApi<VMDetails>(`/api/vms/${vmid}`)
|
||||
cache.details.set(vmid, details)
|
||||
if (selectedVMRef.current?.vmid === vmid) setVMDetails(details)
|
||||
} catch {
|
||||
// The control succeeded; detail polling will retry independently.
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to ${action} VM ${vmid}:`, error)
|
||||
// Same UX issue as handleCreateBackup: a silent console.error left the
|
||||
// user looking at a "Stop"/"Start" button that just never reacted.
|
||||
const msg = error instanceof Error ? error.message : t("vmLxc.errors.unknown")
|
||||
const msg = error instanceof Error
|
||||
? (error.message === "controlNotConfirmed" ? t("vmLxc.errors.controlNotConfirmed") : error.message)
|
||||
: t("vmLxc.errors.unknown")
|
||||
alert(t("vmLxc.errors.controlFailed", { action, vmid, message: msg }))
|
||||
} finally {
|
||||
controlRequestRef.current = false
|
||||
setControlAction(null)
|
||||
setControlLoading(false)
|
||||
}
|
||||
}
|
||||
@@ -7623,8 +7651,11 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
|
||||
tab we added. */}
|
||||
{(() => {
|
||||
const hasTerminal = selectedVM?.type === "lxc" && selectedVM?.status === "running"
|
||||
const pendingAction = controlAction?.vmid === selectedVM?.vmid ? controlAction?.action : null
|
||||
const controlLabel = (action: string, key: string) => pendingAction === action
|
||||
? t(`vmLxc.actions.${action}Pending`) : t(key)
|
||||
return (
|
||||
<div className={`grid gap-2 sm:gap-3 ${hasTerminal ? "grid-cols-4 lg:grid-cols-5" : "grid-cols-4"}`}>
|
||||
<div aria-busy={controlLoading} className={`grid gap-2 sm:gap-3 ${hasTerminal ? "grid-cols-4 lg:grid-cols-5" : "grid-cols-4"}`}>
|
||||
{hasTerminal && (
|
||||
<Button
|
||||
className="w-full bg-zinc-600/20 border border-zinc-600/50 text-zinc-300 hover:bg-zinc-600/30 col-span-4 lg:col-span-1"
|
||||
@@ -7638,19 +7669,19 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
|
||||
className="w-full bg-green-600/20 border border-green-600/50 text-green-400 hover:bg-green-600/30"
|
||||
disabled={selectedVM?.status === "running" || controlLoading}
|
||||
onClick={() => selectedVM && handleVMControl(selectedVM.vmid, "start")}
|
||||
title={t("vmLxc.actions.start")}
|
||||
title={controlLabel("start", "vmLxc.actions.start")}
|
||||
>
|
||||
<Play className="h-4 w-4 sm:mr-2" />
|
||||
<span className="hidden sm:inline">{t("vmLxc.actions.start")}</span>
|
||||
{pendingAction === "start" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <Play className="h-4 w-4 sm:mr-2" />}
|
||||
<span className="hidden sm:inline">{controlLabel("start", "vmLxc.actions.start")}</span>
|
||||
</Button>
|
||||
<Button
|
||||
className="w-full bg-blue-600/20 border border-blue-600/50 text-blue-400 hover:bg-blue-600/30"
|
||||
disabled={selectedVM?.status !== "running" || controlLoading}
|
||||
onClick={() => selectedVM && handleVMControl(selectedVM.vmid, "shutdown")}
|
||||
title={t("vmLxc.actions.shutdown")}
|
||||
title={controlLabel("shutdown", "vmLxc.actions.shutdown")}
|
||||
>
|
||||
<Power className="h-4 w-4 sm:mr-2" />
|
||||
<span className="hidden sm:inline">{t("vmLxc.actions.shutdown")}</span>
|
||||
{pendingAction === "shutdown" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <Power className="h-4 w-4 sm:mr-2" />}
|
||||
<span className="hidden sm:inline">{controlLabel("shutdown", "vmLxc.actions.shutdown")}</span>
|
||||
</Button>
|
||||
<Button
|
||||
className="w-full bg-blue-600/20 border border-blue-600/50 text-blue-400 hover:bg-blue-600/30"
|
||||
@@ -7660,10 +7691,10 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
|
||||
vmid: selectedVM.vmid,
|
||||
vmName: selectedVM.name,
|
||||
})}
|
||||
title={t("vmLxc.actions.reboot")}
|
||||
title={controlLabel("reboot", "vmLxc.actions.reboot")}
|
||||
>
|
||||
<RotateCcw className="h-4 w-4 sm:mr-2" />
|
||||
<span className="hidden sm:inline">{t("vmLxc.actions.reboot")}</span>
|
||||
{pendingAction === "reboot" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <RotateCcw className="h-4 w-4 sm:mr-2" />}
|
||||
<span className="hidden sm:inline">{controlLabel("reboot", "vmLxc.actions.reboot")}</span>
|
||||
</Button>
|
||||
<Button
|
||||
className="w-full bg-red-600/20 border border-red-600/50 text-red-400 hover:bg-red-600/30"
|
||||
@@ -7673,10 +7704,10 @@ const handleDownloadLogs = async (vmid: number, vmName: string) => {
|
||||
vmid: selectedVM.vmid,
|
||||
vmName: selectedVM.name,
|
||||
})}
|
||||
title={t("vmLxc.actions.forceStop")}
|
||||
title={controlLabel("stop", "vmLxc.actions.forceStop")}
|
||||
>
|
||||
<StopCircle className="h-4 w-4 sm:mr-2" />
|
||||
<span className="hidden sm:inline">{t("vmLxc.actions.forceStop")}</span>
|
||||
{pendingAction === "stop" ? <Loader2 className="h-4 w-4 sm:mr-2 animate-spin" /> : <StopCircle className="h-4 w-4 sm:mr-2" />}
|
||||
<span className="hidden sm:inline">{controlLabel("stop", "vmLxc.actions.forceStop")}</span>
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { fetchApi } from './api-config'
|
||||
|
||||
// A successful control POST accepts an asynchronous Proxmox task. In
|
||||
// particular, an unchanged "running" status cannot confirm a reboot.
|
||||
export async function waitForVmControlTask(vmid: number, action: string, task: string) {
|
||||
if (!task) throw new Error('controlNotConfirmed')
|
||||
const expected = action === 'shutdown' || action === 'stop' ? 'stopped' : 'running'
|
||||
const deadline = Date.now() + 180_000
|
||||
let readFailures = 0
|
||||
while (Date.now() < deadline) {
|
||||
let result: { status?: string; exitstatus?: string; guest_status?: string }
|
||||
try {
|
||||
result = await fetchApi(`/api/vms/${vmid}/control-task/${encodeURIComponent(task)}`, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
})
|
||||
readFailures = 0
|
||||
} catch (error) {
|
||||
if (++readFailures >= 3) throw new Error('controlNotConfirmed')
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
continue
|
||||
}
|
||||
if (result?.status === 'stopped') {
|
||||
if (!result.exitstatus) throw new Error('controlNotConfirmed')
|
||||
if (result.exitstatus !== 'OK') throw new Error(result.exitstatus)
|
||||
if (result.guest_status === expected) return result
|
||||
} else if (result?.status !== 'running') {
|
||||
throw new Error('controlNotConfirmed')
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 1000))
|
||||
}
|
||||
throw new Error('controlNotConfirmed')
|
||||
}
|
||||
@@ -1100,7 +1100,11 @@
|
||||
"shutdown": "Shutdown",
|
||||
"reboot": "Reboot",
|
||||
"forceStop": "Force Stop",
|
||||
"working": "Working..."
|
||||
"working": "Working...",
|
||||
"startPending": "Starting...",
|
||||
"shutdownPending": "Shutting down...",
|
||||
"rebootPending": "Restarting...",
|
||||
"stopPending": "Stopping..."
|
||||
},
|
||||
"confirm": {
|
||||
"forceStopTitle": "Force Stop",
|
||||
@@ -1144,6 +1148,7 @@
|
||||
"unknown": "Unknown error",
|
||||
"backupStartFailed": "Failed to start backup: {message}",
|
||||
"controlFailed": "Failed to {action} VM {vmid}: {message}",
|
||||
"controlNotConfirmed": "The operation has not been confirmed complete. Check the guest status before trying again.",
|
||||
"saveNotesFailed": "Error saving notes. Please try again.",
|
||||
"appNotFound": "This application is no longer available. Refresh the page and try again.",
|
||||
"saveCustomCommandFailed": "Could not save the custom update command: {message}",
|
||||
|
||||
@@ -1100,7 +1100,11 @@
|
||||
"shutdown": "Vypnúť",
|
||||
"reboot": "Reštartovať",
|
||||
"forceStop": "Vynútiť vypnutie",
|
||||
"working": "Pracujem..."
|
||||
"working": "Pracujem...",
|
||||
"startPending": "Spúšťa sa...",
|
||||
"shutdownPending": "Vypína sa...",
|
||||
"rebootPending": "Reštartuje sa...",
|
||||
"stopPending": "Vypína sa..."
|
||||
},
|
||||
"confirm": {
|
||||
"forceStopTitle": "Vynútiť vypnutie",
|
||||
@@ -1144,6 +1148,7 @@
|
||||
"unknown": "Neznáma chyba",
|
||||
"backupStartFailed": "Zálohu sa nepodarilo spustiť: {message}",
|
||||
"controlFailed": "Akcia {action} pre VM {vmid} zlyhala: {message}",
|
||||
"controlNotConfirmed": "Dokončenie operácie sa zatiaľ nepodarilo potvrdiť. Pred ďalším pokusom skontrolujte stav VM alebo LXC.",
|
||||
"saveNotesFailed": "Poznámky sa nepodarilo uložiť. Skúste to znova.",
|
||||
"appNotFound": "Táto aplikácia už nie je dostupná. Obnovte stránku a skúste to znova.",
|
||||
"saveCustomCommandFailed": "Vlastný príkaz na aktualizáciu sa nepodarilo uložiť: {message}",
|
||||
|
||||
@@ -6847,10 +6847,15 @@ def get_proxmox_vms():
|
||||
continue
|
||||
|
||||
vm_type = 'lxc' if resource.get('type') == 'lxc' else 'qemu'
|
||||
# The PVE summary can lag behind a completed LXC task.
|
||||
# Use the same direct local cgroup check as the detail
|
||||
# route; never resurrect a stopped CT from that summary.
|
||||
vm_status = (_fast_guest_status(resource.get('vmid'), vm_type)
|
||||
if vm_type == 'lxc' else resource.get('status', 'unknown'))
|
||||
vm_data = {
|
||||
'vmid': resource.get('vmid'),
|
||||
'name': resource.get('name', f"VM-{resource.get('vmid')}"),
|
||||
'status': resource.get('status', 'unknown'),
|
||||
'status': vm_status,
|
||||
'type': vm_type,
|
||||
'cpu': resource.get('cpu', 0),
|
||||
'mem': resource.get('mem', 0),
|
||||
@@ -6905,7 +6910,7 @@ def get_proxmox_vms():
|
||||
# detail-aware clients. The cache is still warmed
|
||||
# at startup and invalidated by LXC lifecycle events.
|
||||
vm_data['ips'] = []
|
||||
if resource.get('status') == 'running':
|
||||
if vm_data['status'] == 'running':
|
||||
lxc_ip_info = _get_lxc_ip_info_cached(resource.get('vmid'))
|
||||
lxc_ips = _normalise_lxc_ips(lxc_ip_info)
|
||||
vm_data['ips'] = lxc_ips
|
||||
@@ -15870,6 +15875,8 @@ def api_vm_config_set(vmid):
|
||||
@require_auth
|
||||
def api_vm_control(vmid):
|
||||
"""Control VM/LXC (start, stop, shutdown, reboot)"""
|
||||
operation_lock = None
|
||||
task_identity_confirmed = False
|
||||
try:
|
||||
data = request.get_json()
|
||||
action = data.get('action') # start, stop, shutdown, reboot
|
||||
@@ -15892,6 +15899,13 @@ def api_vm_control(vmid):
|
||||
|
||||
vm_type = 'lxc' if vm_info.get('type') == 'lxc' else 'qemu'
|
||||
node = vm_info.get('node', 'pve')
|
||||
if vm_type == 'lxc':
|
||||
import lxc_check_job
|
||||
try:
|
||||
operation_lock = lxc_check_job.acquire(vmid)
|
||||
except BlockingIOError:
|
||||
return jsonify({'error': 'Another container operation is running'}), 409
|
||||
lxc_check_job.begin_control(operation_lock, node)
|
||||
|
||||
# Execute action
|
||||
control_result = subprocess.run(
|
||||
@@ -15899,6 +15913,37 @@ def api_vm_control(vmid):
|
||||
capture_output=True, text=True, timeout=30)
|
||||
|
||||
if control_result.returncode == 0:
|
||||
import re
|
||||
# pvesh can print progress lines before the task identity.
|
||||
# The final non-empty line is the only task associated with
|
||||
# this request; accepting a different log line would make the
|
||||
# frontend poll an unrelated guest operation.
|
||||
lines = [line.strip().strip('"') for line in
|
||||
(control_result.stdout or '').splitlines() if line.strip()]
|
||||
task = lines[-1] if lines else ''
|
||||
task_types = [f"{'vz' if vm_type == 'lxc' else 'qm'}{action}"]
|
||||
if action == 'start':
|
||||
task_types.append('hastart')
|
||||
elif action in ('stop', 'shutdown'):
|
||||
# Proxmox 9 routes HA shutdown through hastop for both
|
||||
# LXC and QEMU guests.
|
||||
task_types.append('hastop')
|
||||
task_pattern = (r'UPID:[\w.-]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:'
|
||||
r'[0-9A-Fa-f]+:(?:' + '|'.join(task_types) +
|
||||
rf'):{vmid}:[\w.@!-]+:')
|
||||
if not re.fullmatch(task_pattern, task):
|
||||
# A successful process exit without a verifiable task
|
||||
# cannot be polled. Do not leave a task-less control
|
||||
# marker that would block this CT forever.
|
||||
if operation_lock is not None:
|
||||
lxc_check_job.cancel_control(operation_lock)
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'vmid': vmid,
|
||||
'action': action,
|
||||
'error': 'Control task was not confirmed by Proxmox',
|
||||
}), 502
|
||||
task_identity_confirmed = True
|
||||
# Invalidate VM resources cache so the next /api/vms call
|
||||
# returns fresh status instead of the pre-action snapshot.
|
||||
_pvesh_cache['cluster_resources_vm_time'] = 0
|
||||
@@ -15906,13 +15951,21 @@ def api_vm_control(vmid):
|
||||
# (lxc_ip_info, os_info) — a start/stop flips those, so
|
||||
# drop the modal caches for this vmid.
|
||||
_vm_cache_invalidate(vmid)
|
||||
# The frontend waits for this exact task rather than assuming
|
||||
# that accepting the request means the action completed.
|
||||
if operation_lock is not None:
|
||||
lxc_check_job.retain_control(vmid, task, operation_lock)
|
||||
operation_lock = None
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'vmid': vmid,
|
||||
'action': action,
|
||||
'task': task,
|
||||
'message': f'Successfully executed {action} on {vm_info.get("name")}'
|
||||
})
|
||||
else:
|
||||
if operation_lock is not None:
|
||||
lxc_check_job.cancel_control(operation_lock)
|
||||
# `pvesh` failed → fire the matching vm_fail / ct_fail
|
||||
# notification so the user gets paged on their channels
|
||||
# too, not just an in-dashboard alert. Previously this
|
||||
@@ -15964,8 +16017,70 @@ def api_vm_control(vmid):
|
||||
else:
|
||||
return jsonify({'error': 'Failed to get VM details'}), 500
|
||||
except Exception as e:
|
||||
# A transport error happens before we have an identity that can be
|
||||
# recovered later. Leaving the pre-submission marker would otherwise
|
||||
# make the CT permanently busy after one timed-out pvesh invocation.
|
||||
if operation_lock is not None and not task_identity_confirmed:
|
||||
try:
|
||||
lxc_check_job.cancel_control(operation_lock)
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
finally:
|
||||
if operation_lock is not None:
|
||||
operation_lock.close()
|
||||
|
||||
@app.route('/api/vms/<int:vmid>/control-task/<path:task>', methods=['GET'])
|
||||
@require_auth
|
||||
def api_vm_control_task(vmid, task):
|
||||
"""Read only the power-control task bound to this guest and node."""
|
||||
import re
|
||||
import json
|
||||
if not re.fullmatch(r'UPID:[\w.-]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:'
|
||||
r'(?:(?:vz|qm)(?:start|stop|shutdown|reboot)|ha(?:start|stop)):'
|
||||
r'[0-9]+:[\w.@!-]+:', task):
|
||||
return jsonify({'error': 'Invalid control task'}), 400
|
||||
parts = task.split(':')
|
||||
resource = next((item for item in get_cached_pvesh_cluster_resources_vm() or []
|
||||
if item.get('vmid') == vmid), None)
|
||||
if resource is None:
|
||||
return jsonify({'error': 'VM/LXC not found'}), 404
|
||||
vm_type = 'lxc' if resource.get('type') == 'lxc' else 'qemu'
|
||||
node = resource.get('node', 'pve')
|
||||
prefix = 'vz' if vm_type == 'lxc' else 'qm'
|
||||
allowed_task_types = {f'{prefix}{action}' for action in
|
||||
('start', 'stop', 'shutdown', 'reboot')}
|
||||
allowed_task_types.update(('hastart', 'hastop'))
|
||||
if (parts[1] != node or parts[6] != str(vmid)
|
||||
or parts[5] not in allowed_task_types):
|
||||
return jsonify({'error': 'Control task does not belong to this guest'}), 400
|
||||
try:
|
||||
result = subprocess.run(['pvesh', 'get', f'/nodes/{node}/tasks/{task}/status',
|
||||
'--output-format', 'json'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if result.returncode != 0:
|
||||
return jsonify({'error': 'Control task status unavailable'}), 502
|
||||
status = json.loads(result.stdout)
|
||||
if not isinstance(status, dict) or status.get('status') not in ('running', 'stopped'):
|
||||
return jsonify({'error': 'Invalid control task status'}), 502
|
||||
payload = {'status': status['status'], 'exitstatus': status.get('exitstatus')}
|
||||
if status['status'] == 'stopped' and status.get('exitstatus') == 'OK':
|
||||
current = subprocess.run(['pvesh', 'get', f'/nodes/{node}/{vm_type}/{vmid}/status/current',
|
||||
'--output-format', 'json'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if current.returncode != 0:
|
||||
return jsonify({'error': 'Guest status unavailable'}), 502
|
||||
guest = json.loads(current.stdout)
|
||||
if not isinstance(guest, dict) or guest.get('status') not in ('running', 'stopped'):
|
||||
return jsonify({'error': 'Invalid guest status'}), 502
|
||||
payload['guest_status'] = guest['status']
|
||||
_pvesh_cache['cluster_resources_vm_time'] = 0
|
||||
_vm_cache_invalidate(vmid)
|
||||
return jsonify(payload)
|
||||
except (ValueError, subprocess.TimeoutExpired):
|
||||
return jsonify({'error': 'Control task status unavailable'}), 502
|
||||
|
||||
@app.route('/api/vms/<int:vmid>/description', methods=['PUT'])
|
||||
@app.route('/api/vms/<int:vmid>/config', methods=['PUT']) # legacy alias
|
||||
@require_auth
|
||||
|
||||
@@ -25,6 +25,7 @@ _jobs = {}
|
||||
_attempts = {}
|
||||
COOLDOWN = 60
|
||||
JOB_TTL = 3600
|
||||
CONTROL_TASK_WAIT_SECONDS = 180
|
||||
|
||||
|
||||
def acquire(vmid):
|
||||
@@ -32,6 +33,7 @@ def acquire(vmid):
|
||||
handle = os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600), 'r+')
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
check_control_task(vmid, handle)
|
||||
return handle
|
||||
except BaseException:
|
||||
handle.close()
|
||||
@@ -85,6 +87,92 @@ def clear_invalid_marker(vmid, *, check_started_ns=None):
|
||||
handle.close()
|
||||
|
||||
|
||||
def _write_control(handle, record):
|
||||
handle.seek(0)
|
||||
if record is not None:
|
||||
handle.write(json.dumps(record))
|
||||
handle.flush()
|
||||
# Do not erase the old reservation before the replacement was written.
|
||||
# A partial/failed write stays non-empty and therefore fails closed.
|
||||
handle.truncate(handle.tell())
|
||||
|
||||
|
||||
def begin_control(handle, node):
|
||||
"""Persist uncertainty before submission, including across Monitor restart."""
|
||||
if not isinstance(node, str) or not re.fullmatch(r'[\w.-]+', node):
|
||||
raise ValueError('Invalid control node')
|
||||
_write_control(handle, {'node': node, 'task': None})
|
||||
|
||||
|
||||
def cancel_control(handle):
|
||||
"""Clear a request for which no recoverable task identity was recorded."""
|
||||
_write_control(handle, None)
|
||||
|
||||
|
||||
def check_control_task(vmid, handle=None):
|
||||
"""Called under the shared flock, also by the shell update runner.
|
||||
|
||||
No record means no probe. A restart releases OS file descriptors, but the
|
||||
record still forbids a new operation until PVE confirms task termination.
|
||||
Unknown submission/status cannot safely be treated as completion.
|
||||
"""
|
||||
if handle is None:
|
||||
with open(LOCK_DIR / f'proxmenux-lxc-update-{int(vmid)}.lock', 'r+') as stream:
|
||||
return check_control_task(vmid, stream)
|
||||
handle.seek(0)
|
||||
raw = handle.read(1025)
|
||||
if not raw:
|
||||
return
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
node, task = record['node'], record['task']
|
||||
if not isinstance(node, str) or not re.fullmatch(r'[\w.-]+', node):
|
||||
raise ValueError('Invalid node')
|
||||
if (not isinstance(task, str)
|
||||
or not re.fullmatch(r'UPID:[\w.-]+:[0-9A-Fa-f]+:[0-9A-Fa-f]+:'
|
||||
r'[0-9A-Fa-f]+:(?:vz(?:start|stop|shutdown|reboot)'
|
||||
r'|ha(?:start|stop)):'
|
||||
r'[0-9]+:[\w.@!-]+:', task)
|
||||
or task.split(':')[1] != node or task.split(':')[6] != str(vmid)):
|
||||
raise ValueError('Unconfirmed control task')
|
||||
result = subprocess.run(['pvesh', 'get', f'/nodes/{node}/tasks/{task}/status',
|
||||
'--output-format', 'json'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
status = json.loads(result.stdout) if result.returncode == 0 else None
|
||||
if (not isinstance(status, dict) or status.get('status') != 'stopped'
|
||||
or not isinstance(status.get('exitstatus'), str) or not status['exitstatus']):
|
||||
raise ValueError('Control task completion not confirmed')
|
||||
except (ValueError, KeyError, TypeError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise BlockingIOError('Container control completion is not confirmed') from exc
|
||||
_write_control(handle, None)
|
||||
|
||||
|
||||
def _wait_control(vmid, reservation):
|
||||
deadline = time.monotonic() + CONTROL_TASK_WAIT_SECONDS
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
check_control_task(vmid, reservation)
|
||||
return
|
||||
except BlockingIOError:
|
||||
# Keep the known task record for recovery after this bounded
|
||||
# waiter exits; a later caller asks Proxmox to verify it.
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining > 0:
|
||||
time.sleep(min(2, remaining))
|
||||
finally:
|
||||
reservation.close()
|
||||
|
||||
|
||||
def retain_control(vmid, task, reservation):
|
||||
reservation.seek(0)
|
||||
record = json.loads(reservation.read(1025))
|
||||
record['task'] = task
|
||||
_write_control(reservation, record)
|
||||
# Reservation ownership transfers only after Thread.start succeeds.
|
||||
threading.Thread(target=_wait_control, args=(vmid, reservation), daemon=True).start()
|
||||
|
||||
|
||||
def guard(vmid):
|
||||
if not managed_installs._lxc_updates_detection_enabled():
|
||||
raise RuntimeError('LXC update detection is disabled')
|
||||
|
||||
@@ -36,6 +36,7 @@ class LxcVmIpInventoryTests(unittest.TestCase):
|
||||
"_get_lxc_app_watch_map": lambda: {},
|
||||
"_get_lxc_docker_inventory_map": lambda: {},
|
||||
"get_proxmox_node_name": lambda: "pve",
|
||||
"_fast_guest_status": lambda _vmid, _vm_type: "running",
|
||||
"_get_cached_vm_list_search_notes": lambda _resources, _node: {},
|
||||
"get_cached_vm_disk": lambda _vmid: None,
|
||||
"_guest_lifecycle_lock": threading.RLock(),
|
||||
|
||||
Reference in New Issue
Block a user