diff --git a/AppImage/ProxMenux-1.2.2.2-beta.AppImage b/AppImage/ProxMenux-1.2.2.2-beta.AppImage index 3cab0803..f2919c27 100755 Binary files a/AppImage/ProxMenux-1.2.2.2-beta.AppImage and b/AppImage/ProxMenux-1.2.2.2-beta.AppImage differ diff --git a/AppImage/ProxMenux-Monitor.AppImage.sha256 b/AppImage/ProxMenux-Monitor.AppImage.sha256 index 162d46ac..3f64fe6e 100644 --- a/AppImage/ProxMenux-Monitor.AppImage.sha256 +++ b/AppImage/ProxMenux-Monitor.AppImage.sha256 @@ -1 +1 @@ -79e28fc3a9f60ee3714c587d2dfd48d6d8e0a5b4407ec7a55258de40762ed5e4 ProxMenux-1.2.2.2-beta.AppImage +eae86502621e2ebb7d021b123d8161195e8c8e6441cf1a0e1c752fb206420e59 ProxMenux-1.2.2.2-beta.AppImage diff --git a/AppImage/components/host-backup.tsx b/AppImage/components/host-backup.tsx index 37929e4b..48547028 100644 --- a/AppImage/components/host-backup.tsx +++ b/AppImage/components/host-backup.tsx @@ -43,6 +43,7 @@ import { ShieldCheck, Info, ListTree, + History, } from "lucide-react" import { ScriptTerminalModal } from "./script-terminal-modal" import { fetchApi, getApiUrl } from "../lib/api-config" @@ -1244,40 +1245,50 @@ function InspectModal({ { if (!v) onClose() }}> - + {archive?.display_id} - {archive && ( - - {archive.source} - - )} - {remoteArc?.encrypted && ( - - - - )} {/* ── Body: same shape for the 3 backends ─────────────── */}
- {/* Snapshot info — uniform grid, with backend-specific - rows mixed in only when they carry data. */} + {/* Backup info — uniform grid, with backend-specific + rows mixed in only when they carry data. Backend + + encryption badges live here (instead of the header, + where they used to overlap the close button). */}
-
Snapshot
+
+
Backup
+
+ {archive && ( + + {archive.source} + + )} + {remoteArc?.encrypted && ( + + + + )} +
+
{/* Time + size — present for every backend. */} {archive && (archive.source === "pbs" || archive.source === "borg") && remoteArc ? ( <> -
Snapshot time: {formatMtime(remoteArc.backup_time)}
+
Backup time: {formatMtime(remoteArc.backup_time)}
{remoteArc.size_bytes > 0 &&
Size: {formatBytes(remoteArc.size_bytes)}
}
Repository: {remoteArc.repo_repository}
Repo name: {remoteArc.repo_name}
@@ -1303,7 +1314,7 @@ function InspectModal({ {/* PBS pxar files list — only PBS exposes this. */} {remoteArc?.files && remoteArc.files.length > 0 && (
-
Files in this snapshot
+
Files in this backup
    {remoteArc.files.map((f) => (
  • @@ -3219,6 +3230,7 @@ function CreateJobDialog({
-
@@ -6040,7 +6056,7 @@ function JobDetailModal({ or path string from forcing a horizontal scrollbar. */} - + {detail?.id ?? jobId} {detail && ( @@ -6048,6 +6064,18 @@ function JobDetailModal({ {detail.method} + {/* Encryption badge — mirrors the InspectModal one + so the operator sees the lock in both the summary + list and the job detail view. */} + {(detail.pbs_encrypt || (detail.borg_encrypt_mode && detail.borg_encrypt_mode !== "none")) && ( + + + + )} {detail.attached && ( attached @@ -6228,70 +6256,73 @@ function JobDetailModal({ )} {detail && ( -
- - - - + // Layout mirrors the InspectModal footer: primary action on + // the left in solid green (Run, matches Restore), secondary + // Edit next to it in blue outline, then state-changers on + // the right (Disable / Delete) — both outlined to read as + // less prominent than Run. +
+
+ + +
+
+ + +
)} @@ -6886,7 +6917,7 @@ function ArchiveContentsModal({ })()} {data.rollback_plan && ( - + )} @@ -6924,10 +6955,6 @@ function ArchiveContentsModal({
)} - -
- -
) @@ -7180,9 +7207,8 @@ function RollbackPlanView({ plan }: { plan: RollbackPlan }) {
{hasDestructive && (
-
- - Will be removed (created after the backup) +
+ Not in backup — will be deleted on rollback
{plan.vms_to_remove.length > 0 && (
@@ -7207,19 +7233,18 @@ function RollbackPlanView({ plan }: { plan: RollbackPlan }) { {hasRollback && (
-
- - Will be (re)applied from backup +
+ Configurations included in backup
{plan.vms_to_restore.length > 0 && (
- VMs: + VM configs: {plan.vms_to_restore.map((id) => {id})}
)} {plan.lxcs_to_restore.length > 0 && (
- LXCs: + LXC configs: {plan.lxcs_to_restore.map((id) => {id})}
)} @@ -7229,6 +7254,9 @@ function RollbackPlanView({ plan }: { plan: RollbackPlan }) { {plan.components_to_reinstall.map((c) => {c})}
)} +
+ Only the /etc/pve config is restored — disk images stay where they live. +
)}
diff --git a/AppImage/components/two-factor-setup.tsx b/AppImage/components/two-factor-setup.tsx index 15f649d6..a1325445 100644 --- a/AppImage/components/two-factor-setup.tsx +++ b/AppImage/components/two-factor-setup.tsx @@ -92,33 +92,59 @@ export function TwoFactorSetup({ open, onClose, onSuccess }: TwoFactorSetupProps const copyToClipboard = async (text: string, type: "secret" | "codes") => { let ok = false - // Preferred path (HTTPS / localhost). On plain HTTP the Promise rejects, - // so we catch and fall through to the textarea fallback. + // Path 1: modern Clipboard API. Only works on HTTPS / localhost. try { - if (navigator.clipboard && window.isSecureContext) { + if (navigator.clipboard?.writeText) { await navigator.clipboard.writeText(text) ok = true } } catch { - // fall through to execCommand fallback + // fall through } + // Path 2: legacy execCommand. Picky — some browsers (iOS Safari + // especially) refuse to copy from an element placed off-screen + // (`left: -9999px`), which is the previous version's mistake. + // Keep the textarea inside the viewport but visually invisible. if (!ok) { + const textarea = document.createElement("textarea") + textarea.value = text + textarea.style.position = "fixed" + textarea.style.top = "0" + textarea.style.left = "0" + textarea.style.width = "2em" + textarea.style.height = "2em" + textarea.style.padding = "0" + textarea.style.border = "none" + textarea.style.outline = "none" + textarea.style.boxShadow = "none" + textarea.style.background = "transparent" + textarea.style.opacity = "0" + textarea.setAttribute("readonly", "") + textarea.setAttribute("aria-hidden", "true") + document.body.appendChild(textarea) try { - const textarea = document.createElement("textarea") - textarea.value = text - textarea.style.position = "fixed" - textarea.style.left = "-9999px" - textarea.style.top = "-9999px" - textarea.style.opacity = "0" - textarea.readOnly = true - document.body.appendChild(textarea) textarea.focus() textarea.select() + textarea.setSelectionRange(0, text.length) ok = document.execCommand("copy") - document.body.removeChild(textarea) } catch { ok = false + } finally { + document.body.removeChild(textarea) + } + } + + // Path 3: last-resort window.prompt — ugly but unblockable. The + // user can select+copy from the prompt manually. This guarantees + // they can finish the 2FA setup even on plain-HTTP Monitor where + // both the Clipboard API and execCommand may be locked down. + if (!ok) { + try { + window.prompt("Copy this value:", text) + ok = true + } catch { + // ignore } } diff --git a/AppImage/scripts/health_monitor.py b/AppImage/scripts/health_monitor.py index e4e47c89..bd4137f1 100644 --- a/AppImage/scripts/health_monitor.py +++ b/AppImage/scripts/health_monitor.py @@ -1219,6 +1219,16 @@ class HealthMonitor: WARNING_MIN_SAMPLES = 25 # ~250s of sustained elevated CPU RECOVERY_MIN_SAMPLES = 10 # ~100s of recovery + # Build the `details` payload the cpu_high notification + # template (notification_templates.py) expects: `value` + # (already-formatted percent), `cores` (CPU count) and a + # human-readable `details` line. The previous payload only + # carried `cpu_percent`/`duration`, which left the template + # placeholders blank — the user got "High CPU usage — %" + # and "CPU usage has reached % on cores." with no values. + cpu_count = os.cpu_count() or 1 + value_str = f"{cpu_percent:.0f}" + if len(critical_samples) >= CRITICAL_MIN_SAMPLES: # Calculate actual duration from oldest to newest sample oldest = min(s['time'] for s in critical_samples) @@ -1231,7 +1241,13 @@ class HealthMonitor: category='cpu', severity='CRITICAL', reason=reason, - details={'cpu_percent': cpu_percent, 'duration': actual_duration} + details={ + 'value': value_str, + 'cores': cpu_count, + 'details': f'Sustained for {actual_duration}s above {self.CPU_CRITICAL}%.', + 'cpu_percent': cpu_percent, + 'duration': actual_duration, + }, ) elif len(warning_samples) >= WARNING_MIN_SAMPLES and len(recovery_samples) < RECOVERY_MIN_SAMPLES: oldest = min(s['time'] for s in warning_samples) @@ -1244,7 +1260,13 @@ class HealthMonitor: category='cpu', severity='WARNING', reason=reason, - details={'cpu_percent': cpu_percent, 'duration': actual_duration} + details={ + 'value': value_str, + 'cores': cpu_count, + 'details': f'Sustained for {actual_duration}s above {self.CPU_WARNING}%.', + 'cpu_percent': cpu_percent, + 'duration': actual_duration, + }, ) else: status = 'OK' diff --git a/AppImage/scripts/notification_manager.py b/AppImage/scripts/notification_manager.py index 3d9f6cd2..8bafaf11 100644 --- a/AppImage/scripts/notification_manager.py +++ b/AppImage/scripts/notification_manager.py @@ -1533,7 +1533,22 @@ class NotificationManager: def _flush_digest_for_channel(self, ch_name: str, channel: Any, now: datetime) -> None: """Read pending rows for the channel, render a grouped summary, - send it, and delete the buffer entries on success.""" + send it, and delete the buffer entries on success. + + Every path through here records a `digest` entry in the + notification history (success or fail, empty buffer or not) and + bumps `_stats['total_sent']` / `total_errors` accordingly — the + rest of the notification system does this in + `_dispatch_to_channels`, and skipping it here was the reason the + operator's `/api/notifications/history` and `total_sent` counter + showed zero digest entries even when the schedule was firing + (issue #233). + """ + host = _hostname(self._config) + summary_title = ( + f"{host}: 24h summary ({now.strftime('%Y-%m-%d %H:%M')})" + ) + try: conn = sqlite3.connect(str(DB_PATH), timeout=10) conn.execute('PRAGMA journal_mode=WAL') @@ -1547,6 +1562,12 @@ class NotificationManager: conn.close() except Exception as e: print(f"[NotificationManager] digest read failed for {ch_name}: {e}") + self._record_history( + 'digest', ch_name, summary_title, + f'digest read failed: {e}', 'INFO', + False, str(e), 'digest_scheduler', + ) + self._stats['total_errors'] += 1 return # Mark `last_at` even if there's nothing to send — otherwise an @@ -1554,24 +1575,46 @@ class NotificationManager: self._save_setting(f'{ch_name}.digest_last_at', now.isoformat()) if not rows: + # Empty digest: don't ping the channel (no point in sending + # "nothing to report"), but DO log the run in history so the + # operator can see the schedule fired. Without this the digest + # looked dead silent when in fact it was working — there was + # just nothing INFO non-exempt to summarize. + self._record_history( + 'digest', ch_name, summary_title, + 'No INFO events buffered for this digest window.', + 'INFO', True, '', 'digest_scheduler', + ) return - host = _hostname(self._config) - summary_title = ( - f"{host}: 24h summary ({now.strftime('%Y-%m-%d %H:%M')})" - ) summary_body = self._compose_digest_body(rows) + result: dict = {'success': False, 'error': ''} try: - channel.send(summary_title, summary_body, severity='INFO', - data={'_digest': True, '_count': len(rows)}) + result = channel.send(summary_title, summary_body, severity='INFO', + data={'_digest': True, '_count': len(rows)}) or result except Exception as e: print(f"[NotificationManager] digest send failed for " f"{ch_name}: {e}") - return + result = {'success': False, 'error': str(e)} + + if result.get('success'): + self._stats['total_sent'] += 1 + self._stats['last_sent_at'] = datetime.now().isoformat() + else: + self._stats['total_errors'] += 1 + self._record_history( + 'digest', ch_name, summary_title, summary_body, 'INFO', + result.get('success', False), result.get('error', '') or '', + 'digest_scheduler', + ) # Delete only after a successful send so a transient failure - # doesn't lose the day's data. + # doesn't lose the day's data. The pre-fix version deleted + # unconditionally as long as `channel.send` didn't raise — a + # silent `{'success': False}` would still wipe the buffer. + if not result.get('success'): + return try: ids = [r[0] for r in rows] conn = sqlite3.connect(str(DB_PATH), timeout=10) diff --git a/scripts/global/share-common.func b/scripts/global/share-common.func index c135785e..02a9297e 100644 --- a/scripts/global/share-common.func +++ b/scripts/global/share-common.func @@ -26,7 +26,11 @@ ensure_repositories() { if (( pve_version >= 9 )); then # ===== PVE 9 (Debian 13 - trixie) ===== - # proxmox.sources (no-subscription) - create if missing + # proxmox.sources (no-subscription) - create if missing. + # chmod 0644 explicit on every new .sources file: under the default + # root umask 0027 the redirect would land at 0640, which the PVE 9 + # webgui's repository manager treats as unparseable and silently + # hides the source — issue #230. if [[ ! -f /etc/apt/sources.list.d/proxmox.sources ]]; then cat > /etc/apt/sources.list.d/proxmox.sources <<'EOF' Enabled: true @@ -36,6 +40,7 @@ Suites: trixie Components: pve-no-subscription Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/proxmox.sources need_update=true fi @@ -54,6 +59,7 @@ Suites: trixie-security Components: main contrib non-free-firmware Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/debian.sources need_update=true fi diff --git a/scripts/global/update-pve9_2.sh b/scripts/global/update-pve9_2.sh index d5e61bbc..58d9e85f 100644 --- a/scripts/global/update-pve9_2.sh +++ b/scripts/global/update-pve9_2.sh @@ -129,6 +129,7 @@ Suites: ${TARGET_CODENAME} Components: pve-no-subscription Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/proxmox.sources msg_ok "$(translate "Proxmox VE 9.x no-subscription repository created")" | tee -a "$screen_capture" changes_made=true @@ -146,6 +147,7 @@ Suites: ${TARGET_CODENAME}-security Components: main contrib non-free non-free-firmware Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/debian.sources msg_ok "$(translate "Debian repositories configured for $TARGET_CODENAME")" diff --git a/scripts/global/utils-install-functions.sh b/scripts/global/utils-install-functions.sh index 58c0eb89..7dcfefc3 100644 --- a/scripts/global/utils-install-functions.sh +++ b/scripts/global/utils-install-functions.sh @@ -52,6 +52,10 @@ ensure_repositories() { if (( pve_version >= 9 )); then # ===== PVE 9 (Debian 13 - trixie) ===== + # Force 0644 (world-readable) on every .sources file we drop. + # Under the default root umask 0027 the redirect would land at + # 0640, which the PVE 9 webgui's repository manager treats as + # unparseable and silently hides the source — issue #230. if [[ ! -f /etc/apt/sources.list.d/proxmox.sources ]]; then cat > /etc/apt/sources.list.d/proxmox.sources <<'EOF' Enabled: true @@ -61,6 +65,7 @@ Suites: trixie Components: pve-no-subscription Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/proxmox.sources need_update=true fi @@ -78,6 +83,7 @@ Suites: trixie-security Components: main contrib non-free-firmware Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/debian.sources need_update=true fi diff --git a/scripts/post_install/customizable_post_install.sh b/scripts/post_install/customizable_post_install.sh index 086fd472..d453741d 100644 --- a/scripts/post_install/customizable_post_install.sh +++ b/scripts/post_install/customizable_post_install.sh @@ -988,6 +988,7 @@ Suites: ${target_codename} Components: no-subscription Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg EOF + chmod 0644 /etc/apt/sources.list.d/ceph.sources msg_ok "$(translate "Ceph repository configured for PVE 9")" else diff --git a/scripts/utilities/proxmenux_debug.sh b/scripts/utilities/proxmenux_debug.sh new file mode 100755 index 00000000..4489bf17 --- /dev/null +++ b/scripts/utilities/proxmenux_debug.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# ========================================================== +# ProxMenux Debug Snapshot +# ========================================================== +# Author : MacRimi +# License : GPL-3.0 +# ========================================================== +# Description: +# Emits a single GitHub-pasteable text block describing the +# current host + ProxMenux state, intended for collaborator +# bug reports. The operator runs this after reproducing an +# issue (especially in the host-backup / restore flow) and +# pastes the output verbatim in the discussion thread. +# +# Design rules: +# • read-only — never modifies anything on the host. +# • no network round-trip — everything is local. +# • redacts hostnames, public IPs, MAC addresses and +# credentials from the captured output before printing. +# • single self-contained script, no external deps beyond +# standard Proxmox tooling (jq optional). +# +# Usage: +# bash /usr/local/share/proxmenux/scripts/utilities/proxmenux_debug.sh +# # or capture to a file then upload: +# bash .../proxmenux_debug.sh > /tmp/proxmenux-debug.txt +# ========================================================== +set -u + +# ── Redaction helpers ────────────────────────────────────── +_HOSTNAME_REAL="$(hostname 2>/dev/null || echo host)" +_HOSTNAME_TOKEN="" + +# Replace the real hostname plus common public/PII patterns +# in any captured output. Keeps the report shareable. +_redact() { + sed -E \ + -e "s/${_HOSTNAME_REAL}/${_HOSTNAME_TOKEN}/g" \ + -e 's/([0-9]{1,3}\.){3}[0-9]{1,3}//g' \ + -e 's/([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}//g' \ + -e 's/(password|passphrase|secret|token|api[_-]?key)[[:space:]]*[:=][[:space:]]*"?[^"[:space:]]+"?/\1=/gi' +} + +_header() { echo; echo "── $1 ──"; } +_kv() { printf '%-22s %s\n' "$1:" "$2"; } +_hash() { + [ -f "$1" ] && sha256sum "$1" 2>/dev/null | awk -v p="$1" '{print substr($1,1,16)"… "p}' \ + || echo "(missing) $1" +} + +# ── Banner ───────────────────────────────────────────────── +echo "============================================================" +echo " ProxMenux Debug Snapshot" +echo " Generated: $(date -Iseconds 2>/dev/null || date)" +echo "============================================================" + +# ── ProxMenux + Proxmox + Hardware ───────────────────────── +_header "Versions" +PMX_VERSION="(unknown)" +if [ -f /usr/local/share/proxmenux/monitor-app/web/package.json ]; then + PMX_VERSION=$(grep -oE '"version":[[:space:]]*"[^"]*"' \ + /usr/local/share/proxmenux/monitor-app/web/package.json \ + | head -1 | sed -E 's/.*"version":[[:space:]]*"([^"]+)".*/\1/') +fi +PVE_VERSION="(not a Proxmox host)" +command -v pveversion >/dev/null 2>&1 && PVE_VERSION=$(pveversion 2>/dev/null | head -1) +_kv "ProxMenux" "$PMX_VERSION" +_kv "Proxmox VE" "$PVE_VERSION" +_kv "Kernel" "$(uname -r 2>/dev/null)" +_kv "Distro" "$(grep -E '^PRETTY_NAME=' /etc/os-release 2>/dev/null | cut -d= -f2 | tr -d '\"')" +_kv "Uptime" "$(uptime -p 2>/dev/null || uptime)" + +_header "Hardware" +_kv "CPU model" "$(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed 's/^ //')" +_kv "CPU cores" "$(nproc 2>/dev/null)" +_kv "RAM total" "$(grep -m1 MemTotal /proc/meminfo 2>/dev/null | awk '{printf "%.1f GB\n", $2/1024/1024}')" +_kv "Boot mode" "$([ -d /sys/firmware/efi ] && echo UEFI || echo BIOS)" + +if command -v lspci >/dev/null 2>&1; then + _kv "NVIDIA GPU" "$(lspci -nn 2>/dev/null | grep -iE 'nvidia.*(vga|3d)' | sed -E 's/^[0-9a-f]{2}:[0-9a-f]{2}\.[0-9]+ //' | head -1 | cut -c1-90)" + _kv "AMD GPU" "$(lspci -nn 2>/dev/null | grep -iE 'amd.*(vga|3d)|advanced micro.*(vga|3d)' | sed -E 's/^[0-9a-f]{2}:[0-9a-f]{2}\.[0-9]+ //' | head -1 | cut -c1-90)" + _kv "Intel iGPU" "$(lspci -nn 2>/dev/null | grep -iE 'intel.*(vga|3d|display)' | sed -E 's/^[0-9a-f]{2}:[0-9a-f]{2}\.[0-9]+ //' | head -1 | cut -c1-90)" +fi + +# Show whichever NVIDIA devices live on this host along with the +# driver they're currently bound to — the single most useful piece +# of info for any GPU passthrough / restore bug. +if lspci -d 10de: 2>/dev/null | grep -q .; then + _header "NVIDIA bind state" + while IFS= read -r bdf; do + [ -z "$bdf" ] && continue + drv=$(basename "$(readlink "/sys/bus/pci/devices/$bdf/driver" 2>/dev/null)" 2>/dev/null || echo "(none)") + override=$(cat "/sys/bus/pci/devices/$bdf/driver_override" 2>/dev/null || true) + echo " $bdf driver=$drv override=${override:-(unset)}" + done < <(lspci -D -d 10de: 2>/dev/null | awk '{print $1}') + echo + echo " nvidia-smi:" + if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader 2>&1 | sed 's/^/ /' | head -4 + else + echo " nvidia-smi not installed" + fi +fi + +# ── Components ───────────────────────────────────────────── +_header "ProxMenux components_status.json" +CSF=/usr/local/share/proxmenux/components_status.json +if [ -f "$CSF" ] && command -v jq >/dev/null 2>&1; then + jq -r 'to_entries[] | " \(.key): status=\(.value.status) version=\(.value.version // "-") patched=\(.value.patched // "-")"' "$CSF" +elif [ -f "$CSF" ]; then + sed -E 's/[[:space:]]+/ /g' "$CSF" | head -40 +else + echo " (no components registered)" +fi + +# ── Script integrity ─────────────────────────────────────── +_header "Script SHA-256 (first 16 chars · for drift detection)" +for f in \ + /usr/local/share/proxmenux/scripts/backup_restore/backup_host.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/lib_host_backup_common.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/run_scheduled_backup.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/apply_pending_restore.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/apply_cluster_postboot.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/restore/monitor_apply.sh \ + /usr/local/share/proxmenux/scripts/backup_restore/restore/compute_rollback_plan.sh \ + /usr/local/share/proxmenux/scripts/gpu_tpu/nvidia_installer.sh \ + /usr/local/share/proxmenux/scripts/gpu_tpu/switch_gpu_mode.sh \ + /usr/local/share/proxmenux/scripts/gpu_tpu/switch_gpu_mode_direct.sh \ + /usr/local/share/proxmenux/scripts/global/utils-install-functions.sh +do + echo " $(_hash "$f")" +done + +# ── Backup / restore state ──────────────────────────────── +_header "Backup jobs" +JOBS_DIR=/var/lib/proxmenux/backup-jobs +if [ -d "$JOBS_DIR" ]; then + for env_f in "$JOBS_DIR"/*.env; do + [ -f "$env_f" ] || continue + name=$(basename "$env_f" .env) + backend=$(grep -E '^BACKEND=' "$env_f" 2>/dev/null | cut -d= -f2) + mode=$(grep -E '^PROFILE_MODE=' "$env_f" 2>/dev/null | cut -d= -f2) + enabled=$(grep -E '^ENABLED=' "$env_f" 2>/dev/null | cut -d= -f2) + manual=$(grep -E '^MANUAL_RUN=' "$env_f" 2>/dev/null | cut -d= -f2) + status_f="/var/log/proxmenux/backup-jobs/${name}-last.status" + result=$(grep -E '^RESULT=' "$status_f" 2>/dev/null | cut -d= -f2) + echo " $name backend=$backend profile=$mode enabled=${enabled:-?} manual=${manual:-0} last=${result:-(no runs)}" + done | sort -u +else + echo " (no jobs dir)" +fi + +_header "Pending restore" +PEND=/var/lib/proxmenux/restore-pending +if [ -d "$PEND" ]; then + current=$(readlink -f "$PEND/current" 2>/dev/null) + [ -n "$current" ] && echo " current → $current" + echo " pending dirs:" + ls -1t "$PEND" 2>/dev/null | grep -vE '^current$|^completed$' | sed 's/^/ /' + echo " completed (last 3):" + ls -1t "$PEND/completed" 2>/dev/null | head -3 | sed 's/^/ /' + # Surface the latest rollback.json if present — this is the + # single most useful piece of info for restore bugs (it shows + # what the rollback plan computed before the operator clicked). + latest_completed=$(ls -t "$PEND/completed" 2>/dev/null | head -1) + if [ -n "$latest_completed" ] && [ -f "$PEND/completed/$latest_completed/rollback.json" ]; then + echo " latest rollback.json:" + sed 's/^/ /' "$PEND/completed/$latest_completed/rollback.json" + echo + if [ -f "$PEND/completed/$latest_completed/plan.env" ]; then + echo " latest plan.env:" + sed 's/^/ /' "$PEND/completed/$latest_completed/plan.env" + fi + fi +else + echo " (no pending restore dir)" +fi + +# ── VFIO / passthrough artefacts ────────────────────────── +_header "VFIO / passthrough artefacts on host" +for f in \ + /etc/modprobe.d/proxmenux-nvidia-vfio-blacklist.conf \ + /etc/modprobe.d/nvidia-blacklist.conf \ + /etc/modprobe.d/vfio.conf \ + /etc/udev/rules.d/10-proxmenux-vfio-bind.rules \ + /etc/modules-load.d/nvidia-vfio.conf \ + /etc/proxmenux/vfio-bind.bdfs +do + if [ -e "$f" ]; then + sz=$(stat -c%s "$f" 2>/dev/null || echo "?") + echo " present (${sz} bytes): $f" + else + echo " absent : $f" + fi +done + +# ── Recent logs (last 30 lines each) ────────────────────── +_header "Latest cluster-postboot log (last 30 lines)" +last_pb=$(ls -t /var/log/proxmenux/proxmenux-cluster-postboot-*.log 2>/dev/null | head -1) +[ -n "$last_pb" ] && tail -30 "$last_pb" | _redact || echo " (no cluster-postboot logs)" + +_header "Latest restore-onboot log (last 30 lines)" +last_ob=$(ls -t /var/log/proxmenux/proxmenux-restore-onboot-*.log 2>/dev/null | head -1) +[ -n "$last_ob" ] && tail -30 "$last_ob" | _redact || echo " (no restore-onboot logs)" + +_header "Latest backup runner log (last 30 lines)" +last_bj=$(ls -t /var/log/proxmenux/backup-jobs/*.log 2>/dev/null | head -1) +[ -n "$last_bj" ] && tail -30 "$last_bj" | _redact || echo " (no backup-job logs)" + +_header "proxmenux-monitor.service (last 20 lines, errors only)" +journalctl -u proxmenux-monitor.service -n 200 --no-pager 2>/dev/null \ + | grep -iE 'error|exception|traceback|failed' \ + | tail -20 | _redact \ + || echo " (no errors)" + +# ── Notifications config + last events ───────────────────── +_header "Notifications status" +NOTIF_DB=/usr/local/share/proxmenux/monitor.db +if [ -f "$NOTIF_DB" ] && command -v sqlite3 >/dev/null 2>&1; then + echo " channels configured:" + sqlite3 "$NOTIF_DB" \ + "SELECT ' '||setting_key||' = '||substr(setting_value,1,20)||'…' \ + FROM user_settings \ + WHERE setting_key LIKE 'notifications.%enabled' \ + OR setting_key LIKE 'notifications.%digest%' \ + OR setting_key LIKE 'notifications.quiet_hours%' \ + OR setting_key LIKE '%.digest_last_at';" 2>/dev/null | _redact || echo " (sqlite query failed)" + echo " history (last 5):" + sqlite3 -separator ' | ' "$NOTIF_DB" \ + "SELECT sent_at, channel, event_type, substr(title,1,40) FROM notification_history ORDER BY sent_at DESC LIMIT 5;" \ + 2>/dev/null | sed 's/^/ /' | _redact || echo " (no history)" +else + echo " (no monitor.db or sqlite3 missing)" +fi + +# ── Guests (count only — config-level data stays private) ── +_header "Guests on this host (count only)" +qm_count=$(qm list 2>/dev/null | awk 'NR>1' | wc -l) +pct_count=$(pct list 2>/dev/null | awk 'NR>1' | wc -l) +_kv "VMs" "$qm_count" +_kv "LXCs" "$pct_count" + +echo +echo "============================================================" +echo " End of debug snapshot. Paste the above in the GitHub thread." +echo "============================================================"