mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-07-27 10:58:23 +00:00
456 lines
20 KiB
Bash
Executable File
456 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ==========================================================
|
|
# ProxMenux RESTORE Debug Snapshot
|
|
# ==========================================================
|
|
# Author : MacRimi
|
|
# License : GPL-3.0
|
|
# ==========================================================
|
|
# Description:
|
|
# Single GitHub-pasteable block focused on RESTORE bug
|
|
# triage only. Each section corresponds to a class of
|
|
# restore failure we need to diagnose. Nothing else is
|
|
# included — no notifications data, no global guest counts,
|
|
# no system telemetry that doesn't directly explain a
|
|
# failed restore.
|
|
#
|
|
# Sections (and why each one is here):
|
|
# 1. Versions — reproducibility anchor.
|
|
# 2. GPU bind state — passthrough / VFIO bugs.
|
|
# 3. Components status — what ProxMenux thinks is installed.
|
|
# 4. Latest restore plan — rollback.json + plan.env + apply-list.
|
|
# 5. Pre-restore snapshot — what the restore overwrote.
|
|
# 6. VFIO config files — current state of every file the
|
|
# switch_gpu / restore flows touch
|
|
# (presence + content).
|
|
# 7. Boot args — /etc/kernel/cmdline vs /proc/cmdline,
|
|
# proxmox-boot-tool sync state.
|
|
# 8. dmesg boot binding — order vfio-pci ↔ nvidia loaded.
|
|
# 9. Restore service — systemd unit state + on-boot log.
|
|
# 10. Cluster post-boot log — initramfs regen + component reinstall.
|
|
# 11. Guests holding the GPU — VMs whose hostpci could keep the GPU
|
|
# reserved for vfio-pci.
|
|
# 12. Recent monitor errors — fail-fast clues from the service log.
|
|
#
|
|
# Usage:
|
|
# bash /usr/local/share/proxmenux/scripts/utilities/proxmenux_debug.sh > /tmp/debug.txt
|
|
# cat /tmp/debug.txt # copy-paste the whole block in the GitHub thread
|
|
#
|
|
# Output is redacted: hostnames, IPv4 addresses, MAC addresses
|
|
# and the obvious password=, passphrase=, token= patterns are
|
|
# replaced before printing.
|
|
# ==========================================================
|
|
set -u
|
|
|
|
# ── Redaction helpers ──────────────────────────────────────
|
|
_HOSTNAME_REAL="$(hostname 2>/dev/null || echo host)"
|
|
_redact() {
|
|
sed -E \
|
|
-e "s/${_HOSTNAME_REAL}/<host>/g" \
|
|
-e 's/([0-9]{1,3}\.){3}[0-9]{1,3}/<ip>/g' \
|
|
-e 's/([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}/<mac>/g' \
|
|
-e 's/(password|passphrase|secret|token|api[_-]?key)[[:space:]]*[:=][[:space:]]*"?[^"[:space:]]+"?/\1=<redacted>/gi'
|
|
}
|
|
_header() { echo; echo "── $1 ──"; }
|
|
_kv() { printf '%-22s %s\n' "$1:" "$2"; }
|
|
# Dump a config file inline only when it's small (≤2KB) and non-empty.
|
|
# Larger or absent files get a single status line so the section
|
|
# doesn't balloon with content we won't read anyway.
|
|
_dump_if_small() {
|
|
local f="$1"
|
|
if [ ! -e "$f" ]; then
|
|
echo " absent : $f"
|
|
return
|
|
fi
|
|
local sz
|
|
sz=$(stat -c%s "$f" 2>/dev/null || echo 0)
|
|
echo " present (${sz} bytes): $f"
|
|
if [ -s "$f" ] && [ "$sz" -lt 2048 ] 2>/dev/null; then
|
|
sed 's/^/ | /' "$f" | _redact
|
|
fi
|
|
}
|
|
|
|
echo "============================================================"
|
|
echo " ProxMenux RESTORE Debug Snapshot"
|
|
echo " Generated: $(date -Iseconds 2>/dev/null || date)"
|
|
echo "============================================================"
|
|
|
|
# ── 0. Quick status (auto-detected) ───────────────────────
|
|
# Runs a handful of well-known restore health checks and emits a
|
|
# colour-coded overall verdict. The point is that whoever pastes the
|
|
# snapshot in the GitHub thread, AND whoever reads it later, can both
|
|
# tell in five seconds whether the restore was clean or whether there
|
|
# is something to investigate — without scrolling through the rest of
|
|
# the report. Every signal here has a corresponding section below
|
|
# where the raw evidence lives.
|
|
declare -a _STATUS_LINES=()
|
|
_ERRORS=0
|
|
_WARNINGS=0
|
|
# Colour codes only when stdout is a real terminal — the operator
|
|
# almost always redirects this script to /tmp/proxmenux-debug.txt
|
|
# and pastes the file in GitHub, where raw `\033[…m` sequences
|
|
# would render as literal `[32m✓[0m` garbage.
|
|
if [ -t 1 ]; then
|
|
_C_OK=$'\033[32m'; _C_WARN=$'\033[33m'; _C_ERR=$'\033[31m'
|
|
_C_BOLD_OK=$'\033[1;32m'; _C_BOLD_WARN=$'\033[1;33m'; _C_BOLD_ERR=$'\033[1;31m'
|
|
_C_RESET=$'\033[0m'
|
|
else
|
|
_C_OK=''; _C_WARN=''; _C_ERR=''
|
|
_C_BOLD_OK=''; _C_BOLD_WARN=''; _C_BOLD_ERR=''
|
|
_C_RESET=''
|
|
fi
|
|
_ok() { _STATUS_LINES+=(" ${_C_OK}✓${_C_RESET} $1"); }
|
|
_warn() { _STATUS_LINES+=(" ${_C_WARN}⚠${_C_RESET} $1"); _WARNINGS=$((_WARNINGS + 1)); }
|
|
_err() { _STATUS_LINES+=(" ${_C_ERR}✗${_C_RESET} $1"); _ERRORS=$((_ERRORS + 1)); }
|
|
|
|
# Check A — NVIDIA GPUs binding state.
|
|
# We've hit several restore bugs where the GPU stayed bound to
|
|
# vfio-pci after a rollback-to-LXC. Flag any NVIDIA BDF stuck on
|
|
# vfio-pci so the operator notices before scrolling to section 2.
|
|
if lspci -d 10de: 2>/dev/null | grep -qiE 'VGA|3D'; then
|
|
stuck_vfio=0
|
|
for bdf in $(lspci -D -d 10de: 2>/dev/null | grep -iE 'VGA|3D' | awk '{print $1}'); do
|
|
drv=$(basename "$(readlink "/sys/bus/pci/devices/$bdf/driver" 2>/dev/null)" 2>/dev/null)
|
|
[ "$drv" = "vfio-pci" ] && stuck_vfio=$((stuck_vfio + 1))
|
|
done
|
|
if (( stuck_vfio > 0 )); then
|
|
_warn "$stuck_vfio NVIDIA GPU(s) bound to vfio-pci (intended only if you actively use passthrough)"
|
|
else
|
|
_ok "NVIDIA GPU(s) bound to the native driver"
|
|
fi
|
|
fi
|
|
|
|
# Check B — VFIO-related ProxMenux files lingering on the host.
|
|
# If present after a "rollback to host mode" restore, the next boot
|
|
# will re-bind the GPU to vfio-pci via the udev rule.
|
|
_vfio_present=0
|
|
for f in /etc/udev/rules.d/10-proxmenux-vfio-bind.rules \
|
|
/etc/modprobe.d/proxmenux-nvidia-vfio-blacklist.conf \
|
|
/etc/modprobe.d/nvidia-blacklist.conf \
|
|
/etc/proxmenux/vfio-bind.bdfs; do
|
|
[ -e "$f" ] && _vfio_present=$((_vfio_present + 1))
|
|
done
|
|
if (( _vfio_present > 0 )); then
|
|
_warn "$_vfio_present ProxMenux VFIO file(s) present on host (would re-bind GPU at next boot)"
|
|
else
|
|
_ok "No ProxMenux VFIO files present on host"
|
|
fi
|
|
|
|
# Note: cmdline / IOMMU drift was tried here as a check, but it
|
|
# is a system-config finding that's INDEPENDENT of whether the
|
|
# last restore succeeded. Including it in Quick status painted a
|
|
# perfectly clean restore in yellow and made operators think
|
|
# their restore had failed. The raw evidence still lives in
|
|
# section 7 for anyone investigating that specific issue.
|
|
|
|
# Check C — apply_cluster_postboot.sh actually finished.
|
|
_last_pb=$(ls -t /var/log/proxmenux/proxmenux-cluster-postboot-*.log 2>/dev/null | head -1)
|
|
if [ -n "$_last_pb" ]; then
|
|
if grep -q '=== Apply finished' "$_last_pb" 2>/dev/null; then
|
|
_ok "Last apply_cluster_postboot finished successfully"
|
|
else
|
|
_err "Last apply_cluster_postboot did NOT finish (crashed mid-apply or still running)"
|
|
fi
|
|
fi
|
|
|
|
# Check E — proxmenux-restore-onboot.service status.
|
|
if systemctl list-unit-files proxmenux-restore-onboot.service >/dev/null 2>&1; then
|
|
if systemctl is-failed --quiet proxmenux-restore-onboot.service 2>/dev/null; then
|
|
_err "proxmenux-restore-onboot.service is in failed state"
|
|
fi
|
|
fi
|
|
|
|
# Check F — rollback consistency: if the operator opted into
|
|
# destructive rollback AND the plan listed VMs to remove, those VMs
|
|
# should be gone now. If any survived, the rollback executor silently
|
|
# skipped them.
|
|
_latest=$(ls -t /var/lib/proxmenux/restore-pending/completed/ 2>/dev/null | head -1)
|
|
if [ -n "$_latest" ] && command -v jq >/dev/null 2>&1; then
|
|
_base="/var/lib/proxmenux/restore-pending/completed/$_latest"
|
|
_rb_ack=$(grep -E '^HB_ROLLBACK_EXECUTE=' "$_base/plan.env" 2>/dev/null | cut -d= -f2)
|
|
if [ "$_rb_ack" = "1" ] && [ -f "$_base/rollback.json" ]; then
|
|
_to_rm=$(jq -r '.vms_to_remove[]?' "$_base/rollback.json" 2>/dev/null)
|
|
_still=0
|
|
for _id in $_to_rm; do
|
|
[ -z "$_id" ] && continue
|
|
qm status "$_id" >/dev/null 2>&1 && _still=$((_still + 1))
|
|
done
|
|
if (( _still > 0 )); then
|
|
_err "Rollback was ACK'd but $_still VM(s) marked for removal are still present"
|
|
elif [ -n "$_to_rm" ]; then
|
|
_ok "Rollback executed: all VMs listed for removal are gone"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Render the section.
|
|
_header "0. Quick status (auto-detected)"
|
|
for _line in "${_STATUS_LINES[@]}"; do
|
|
printf '%s\n' "$_line"
|
|
done
|
|
echo
|
|
if (( _ERRORS > 0 )); then
|
|
printf ' %sOVERALL: 🔴 %d error(s), %d warning(s)%s — see sections below\n' \
|
|
"$_C_BOLD_ERR" "$_ERRORS" "$_WARNINGS" "$_C_RESET"
|
|
elif (( _WARNINGS > 0 )); then
|
|
printf ' %sOVERALL: 🟡 %d warning(s)%s — see sections below\n' \
|
|
"$_C_BOLD_WARN" "$_WARNINGS" "$_C_RESET"
|
|
else
|
|
printf ' %sOVERALL: 🟢 Restore state looks clean%s\n' \
|
|
"$_C_BOLD_OK" "$_C_RESET"
|
|
fi
|
|
|
|
# ── 1. Versions ────────────────────────────────────────────
|
|
_header "1. Versions"
|
|
PMX_VERSION="(unknown)"
|
|
if [ -f /usr/local/share/proxmenux/monitor-app/web/package.json ]; then
|
|
PMX_VERSION=$(grep -oE '"version":[[:space:]]*"[^"]*"' \
|
|
/usr/local/share/proxmenux/monitor-app/web/package.json \
|
|
| head -1 | sed -E 's/.*"version":[[:space:]]*"([^"]+)".*/\1/')
|
|
fi
|
|
_kv "ProxMenux" "$PMX_VERSION"
|
|
_kv "Proxmox VE" "$(command -v pveversion >/dev/null 2>&1 && pveversion 2>/dev/null | head -1)"
|
|
_kv "Kernel" "$(uname -r 2>/dev/null)"
|
|
_kv "Boot mode" "$([ -d /sys/firmware/efi ] && echo UEFI || echo BIOS)"
|
|
|
|
# ── 2. GPU bind state ─────────────────────────────────────
|
|
# Most restore bugs we've hit so far revolved around the GPU
|
|
# staying bound to vfio-pci when it should have returned to the
|
|
# nvidia driver (or vice-versa). Showing the driver, override and
|
|
# IOMMU group for every PCI display device tells us at a glance
|
|
# which side of that fence the host landed on.
|
|
_header "2. GPU PCI bind state"
|
|
if command -v lspci >/dev/null 2>&1; then
|
|
for bdf in $(lspci -D 2>/dev/null | grep -iE 'VGA compatible controller|3D controller|Display controller' | awk '{print $1}'); do
|
|
name=$(lspci -nn -s "$bdf" 2>/dev/null \
|
|
| sed -E 's/^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9]+ //' \
|
|
| sed -E 's/(VGA compatible controller|3D controller|Display controller) \[[0-9a-f]+\]:[[:space:]]*//I' \
|
|
| cut -c1-80)
|
|
drv=$(basename "$(readlink "/sys/bus/pci/devices/$bdf/driver" 2>/dev/null)" 2>/dev/null || echo "(none)")
|
|
override=$(cat "/sys/bus/pci/devices/$bdf/driver_override" 2>/dev/null || true)
|
|
iommu=$(basename "$(readlink "/sys/bus/pci/devices/$bdf/iommu_group" 2>/dev/null)" 2>/dev/null || echo "-")
|
|
echo " $bdf driver=$drv override=${override:-(unset)} iommu_group=$iommu"
|
|
echo " $name"
|
|
done
|
|
fi
|
|
# nvidia-smi gets its own short summary — confirms the driver
|
|
# can actually talk to the device, not just that it's bound.
|
|
if command -v nvidia-smi >/dev/null 2>&1; then
|
|
echo " nvidia-smi:"
|
|
nvidia-smi --query-gpu=name,driver_version --format=csv,noheader 2>&1 | sed 's/^/ /' | head -4
|
|
fi
|
|
|
|
# ── 3. ProxMenux components ───────────────────────────────
|
|
_header "3. ProxMenux components_status.json"
|
|
CSF=/usr/local/share/proxmenux/components_status.json
|
|
if [ -f "$CSF" ] && command -v jq >/dev/null 2>&1; then
|
|
jq -r 'to_entries[] | " \(.key): status=\(.value.status) version=\(.value.version // "-") patched=\(.value.patched // "-")"' "$CSF"
|
|
elif [ -f "$CSF" ]; then
|
|
sed 's/^/ /' "$CSF" | head -40
|
|
else
|
|
echo " (no components registered)"
|
|
fi
|
|
|
|
# ── 4. Latest restore plan ────────────────────────────────
|
|
# The rollback.json + plan.env + apply-on-boot.list of the
|
|
# most-recent restore tell us exactly what the operator asked
|
|
# for and what the system tried to do. Without these we can't
|
|
# tell "rollback flag never propagated" from "rollback flag
|
|
# propagated but executor skipped guests".
|
|
_header "4. Latest restore plan"
|
|
PEND=/var/lib/proxmenux/restore-pending
|
|
if [ -d "$PEND" ]; then
|
|
latest=$(ls -t "$PEND/completed" 2>/dev/null | head -1)
|
|
if [ -n "$latest" ]; then
|
|
base="$PEND/completed/$latest"
|
|
echo " pending dir: $base"
|
|
if [ -f "$base/plan.env" ]; then
|
|
echo " plan.env:"
|
|
sed 's/^/ /' "$base/plan.env"
|
|
fi
|
|
if [ -f "$base/rollback.json" ]; then
|
|
echo " rollback.json:"
|
|
sed 's/^/ /' "$base/rollback.json"
|
|
fi
|
|
if [ -f "$base/apply-on-boot.list" ]; then
|
|
count=$(wc -l < "$base/apply-on-boot.list" 2>/dev/null || echo 0)
|
|
# Drop the misleading "first 30" tag when the file already fits.
|
|
if [ "$count" -le 30 ]; then
|
|
echo " apply-on-boot.list (${count} paths):"
|
|
else
|
|
echo " apply-on-boot.list (${count} paths; first 30):"
|
|
fi
|
|
head -30 "$base/apply-on-boot.list" | sed 's/^/ /'
|
|
fi
|
|
else
|
|
echo " (no completed restores yet)"
|
|
fi
|
|
else
|
|
echo " (no pending-restore dir)"
|
|
fi
|
|
|
|
# ── 5. Pre-restore preservation ───────────────────────────
|
|
# `apply_pending_restore.sh` snapshots every file it's about to
|
|
# overwrite into /var/lib/proxmenux/pre-restore/<id>-onboot/. The
|
|
# list (paths only — we don't dump contents) tells us what the
|
|
# restore touched and lets us spot files that were ON the host
|
|
# before but NOT in the backup (additive-restore orphans).
|
|
_header "5. Pre-restore preservation (paths overwritten by the restore)"
|
|
PRE=/var/lib/proxmenux/pre-restore
|
|
if [ -d "$PRE" ]; then
|
|
latest_pre=$(ls -t "$PRE" 2>/dev/null | grep -E -- '-onboot$' | head -1)
|
|
if [ -n "$latest_pre" ]; then
|
|
echo " pre-restore dir: $PRE/$latest_pre"
|
|
find "$PRE/$latest_pre" -type f 2>/dev/null \
|
|
| sed "s|^$PRE/$latest_pre||" \
|
|
| sort | sed 's/^/ /'
|
|
else
|
|
echo " (no -onboot snapshot present — apply_pending_restore never ran)"
|
|
fi
|
|
else
|
|
echo " (no pre-restore dir)"
|
|
fi
|
|
|
|
# ── 6. VFIO / passthrough config files ────────────────────
|
|
# Every file the host-mode ↔ vfio-mode switch flow touches. We
|
|
# need both PRESENCE and CONTENT here, because "the file exists
|
|
# but is empty" and "the file has the wrong vendor IDs" are two
|
|
# different bug classes (we've hit both).
|
|
_header "6. VFIO / passthrough config files (current state)"
|
|
for f in \
|
|
/etc/proxmenux/vfio-bind.bdfs \
|
|
/etc/modules-load.d/nvidia-vfio.conf \
|
|
/etc/modules-load.d/nvidia-vfio.conf.proxmenux-disabled-vfio \
|
|
/etc/udev/rules.d/10-proxmenux-vfio-bind.rules \
|
|
/etc/udev/rules.d/70-nvidia.rules \
|
|
/etc/udev/rules.d/70-nvidia.rules.proxmenux-disabled \
|
|
/etc/modprobe.d/proxmenux-nvidia-vfio-blacklist.conf \
|
|
/etc/modprobe.d/nvidia-blacklist.conf \
|
|
/etc/modprobe.d/vfio.conf
|
|
do
|
|
_dump_if_small "$f"
|
|
done
|
|
|
|
# ── 7. Kernel boot args ───────────────────────────────────
|
|
# /etc/kernel/cmdline is what the operator (or our scripts)
|
|
# *configured*. /proc/cmdline is what the running kernel actually
|
|
# booted with. They diverge whenever someone edits cmdline and
|
|
# forgets proxmox-boot-tool refresh — and that drift breaks every
|
|
# downstream passthrough/intel_iommu assumption.
|
|
_header "7. Kernel boot args"
|
|
echo " /etc/kernel/cmdline (configured):"
|
|
[ -f /etc/kernel/cmdline ] && sed 's/^/ /' /etc/kernel/cmdline | _redact || echo " (not present — non-systemd-boot host)"
|
|
echo " /proc/cmdline (running):"
|
|
sed 's/^/ /' /proc/cmdline 2>/dev/null | _redact
|
|
# /etc/default/grub is the OTHER place kernel args can live. On
|
|
# systemd-boot hosts `proxmox-boot-tool refresh` reads cmdline +
|
|
# GRUB_CMDLINE_LINUX_DEFAULT and concatenates them — so a drift
|
|
# here is what causes "intel_iommu=on disappears at the next
|
|
# proxmox-boot-tool refresh after restore" bugs.
|
|
if [ -f /etc/default/grub ]; then
|
|
echo " /etc/default/grub GRUB_CMDLINE_LINUX*:"
|
|
grep -E '^GRUB_CMDLINE_LINUX(_DEFAULT)?=' /etc/default/grub 2>/dev/null \
|
|
| sed 's/^/ /' | _redact || echo " (no GRUB_CMDLINE_LINUX entries)"
|
|
fi
|
|
if command -v proxmox-boot-tool >/dev/null 2>&1; then
|
|
echo " proxmox-boot-tool status:"
|
|
proxmox-boot-tool status 2>&1 | grep -vE '^(System|Re-executing)' | sed 's/^/ /' | head -10
|
|
fi
|
|
|
|
# ── 8. dmesg — vfio + nvidia + GPU PCI bus ────────────────
|
|
# The early-boot trace. Tells us whether vfio-pci or nvidia
|
|
# claimed the PCI BDF FIRST. If we see `vfio-pci 0000:01:00.0`
|
|
# before `nvidia 0000:01:00.0`, that's the smoking gun for a
|
|
# restore that didn't clear the vfio bindings before reboot.
|
|
_header "8. dmesg — vfio / nvidia / GPU bus binding sequence"
|
|
dmesg 2>/dev/null | grep -iE 'vfio|nvidia|VFIO' | head -25 | _redact \
|
|
|| echo " (no matches in dmesg)"
|
|
|
|
# ── 9. Restore service status + on-boot log ───────────────
|
|
# systemd's own view of the on-boot apply (success/timeout/exit)
|
|
# plus the script's log file. The two together cover both
|
|
# "the script failed silently" and "the script ran but its log
|
|
# was truncated by a hung apply".
|
|
_header "9. proxmenux-restore-onboot.service — current boot"
|
|
journalctl -u proxmenux-restore-onboot.service -b 0 -n 20 --no-pager 2>/dev/null \
|
|
| _redact || echo " (no journal entries)"
|
|
echo
|
|
last_ob=$(ls -t /var/log/proxmenux/proxmenux-restore-onboot-*.log 2>/dev/null | head -1)
|
|
if [ -n "$last_ob" ]; then
|
|
echo " log file: $last_ob"
|
|
tail -30 "$last_ob" | sed 's/^/ /' | _redact
|
|
else
|
|
echo " (no restore-onboot log files)"
|
|
fi
|
|
|
|
# ── 10. Cluster post-boot log ─────────────────────────────
|
|
# Where update-initramfs / update-grub / component auto-reinstall
|
|
# actually run. Surface the whole tail so we see whether the
|
|
# postboot finished or stopped partway through.
|
|
_header "10. apply_cluster_postboot — last run (full log, redacted)"
|
|
last_pb=$(ls -t /var/log/proxmenux/proxmenux-cluster-postboot-*.log 2>/dev/null | head -1)
|
|
if [ -n "$last_pb" ]; then
|
|
echo " log file: $last_pb"
|
|
# Dump the WHOLE post-boot log when small enough — the operator
|
|
# almost always wants to see the header (`=== ProxMenux cluster
|
|
# post-boot apply at … ===`) AND the tail together. 50-line tail
|
|
# was eating the start of the log on every restore we've inspected.
|
|
sz=$(stat -c%s "$last_pb" 2>/dev/null || echo 0)
|
|
if [ "$sz" -lt 8192 ]; then
|
|
sed 's/^/ /' "$last_pb" | _redact
|
|
else
|
|
echo " (log > 8 KB — showing last 80 lines)"
|
|
tail -80 "$last_pb" | sed 's/^/ /' | _redact
|
|
fi
|
|
else
|
|
echo " (no cluster-postboot log files)"
|
|
fi
|
|
|
|
# ── 11. Guests holding the GPU ────────────────────────────
|
|
# If a VM with `hostpci` still exists after the restore, Proxmox
|
|
# will auto-rebind the matching PCI device to vfio-pci at the
|
|
# next boot — which makes the GPU look "stuck" in passthrough
|
|
# even though every config file looks clean. We only list IDs +
|
|
# BDFs, NOT full configs, to keep secrets out of the report.
|
|
_header "11. Guests with hostpci entries (would re-reserve the GPU)"
|
|
hits=0
|
|
if command -v qm >/dev/null 2>&1; then
|
|
for vmid in $(qm list 2>/dev/null | awk 'NR>1 {print $1}'); do
|
|
out=$(qm config "$vmid" 2>/dev/null | grep -iE '^hostpci[0-9]+:' || true)
|
|
if [ -n "$out" ]; then
|
|
echo " VM $vmid:"
|
|
echo "$out" | sed 's/^/ /'
|
|
hits=$((hits + 1))
|
|
fi
|
|
done
|
|
fi
|
|
if command -v pct >/dev/null 2>&1; then
|
|
for ctid in $(pct list 2>/dev/null | awk 'NR>1 {print $1}'); do
|
|
out=$(pct config "$ctid" 2>/dev/null | grep -iE 'mp[0-9]+:.*nvidia|dev[0-9]+:|lxc.cgroup2.devices' || true)
|
|
if [ -n "$out" ]; then
|
|
echo " CT $ctid:"
|
|
echo "$out" | sed 's/^/ /'
|
|
hits=$((hits + 1))
|
|
fi
|
|
done
|
|
fi
|
|
[ "$hits" = "0" ] && echo " (no guests hold passthrough entries)"
|
|
|
|
# ── 12. proxmenux-monitor errors (current boot) ───────────
|
|
# Any Python traceback or "Error/Exception" line that ended up
|
|
# in the service log this boot. Filtered so we don't drown the
|
|
# report in the normal startup noise.
|
|
_header "12. proxmenux-monitor.service — errors this boot"
|
|
err=$(journalctl -u proxmenux-monitor.service -b 0 --no-pager 2>/dev/null \
|
|
| grep -iE 'error|exception|traceback|failed' | tail -15)
|
|
if [ -n "$err" ]; then
|
|
echo "$err" | sed 's/^/ /' | _redact
|
|
else
|
|
echo " (no errors logged this boot)"
|
|
fi
|
|
|
|
echo
|
|
echo "============================================================"
|
|
echo " End of restore debug snapshot."
|
|
echo " Paste the whole block in the GitHub discussion thread."
|
|
echo "============================================================"
|