feat: support PBS hosts and remote PBS monitoring

This commit is contained in:
Jonatan
2026-10-16 21:46:26 +02:00
parent 3000c1cd66
commit 511bfe52ea
26 changed files with 1355 additions and 15 deletions
+3
View File
@@ -1115,6 +1115,9 @@ def _detect_proxmox_certificate_paths():
return PROXMOX_CUSTOM_CERT_PATH, PROXMOX_CUSTOM_KEY_PATH
if os.path.isfile(PROXMOX_CERT_PATH) and os.path.isfile(PROXMOX_KEY_PATH):
return PROXMOX_CERT_PATH, PROXMOX_KEY_PATH
pbs_cert, pbs_key = "/etc/proxmox-backup/proxy.pem", "/etc/proxmox-backup/proxy.key"
if os.path.isfile(pbs_cert) and os.path.isfile(pbs_key):
return pbs_cert, pbs_key
return "", ""
+3
View File
@@ -160,6 +160,9 @@ cp "$SCRIPT_DIR/flask_terminal_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || ech
cp "$SCRIPT_DIR/hardware_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ hardware_monitor.py not found"
cp "$SCRIPT_DIR/temperature_sensor_resolver.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ temperature_sensor_resolver.py not found"
cp "$SCRIPT_DIR/proxmox_storage_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ proxmox_storage_monitor.py not found"
cp "$SCRIPT_DIR/pbs_manager.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/flask_pbs_routes.py" "$APP_DIR/usr/bin/"
cp "$APPIMAGE_ROOT/../scripts/global/proxmox_platform.py" "$APP_DIR/usr/bin/"
cp "$SCRIPT_DIR/flask_script_runner.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_script_runner.py not found"
cp "$SCRIPT_DIR/security_manager.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ security_manager.py not found"
cp "$SCRIPT_DIR/flask_security_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_security_routes.py not found"
+71
View File
@@ -0,0 +1,71 @@
"""Read-only PBS status and administrative connection management."""
from flask import Blueprint, jsonify, request
import auth_manager
import pbs_manager as pbs
from jwt_middleware import require_auth, require_admin_scope
pbs_bp = Blueprint("pbs", __name__)
@pbs_bp.after_request
def no_cache(response):
response.headers["Cache-Control"] = "no-store"
return response
@pbs_bp.route("/api/platform")
@require_auth
def platform():
return jsonify(pbs.get_platform())
@pbs_bp.route("/api/pbs/servers")
@require_auth
def servers():
try:
return jsonify(servers=pbs.public_servers(), local_available=pbs.is_pbs_host())
except pbs.PBSError as exc:
return jsonify(error=str(exc)), 500
def _protected_configuration():
local = auth_manager.load_auth_config()
return local.get("enabled") and not local.get("declined") and local.get("password_hash")
@pbs_bp.route("/api/pbs/servers", methods=["POST"])
@require_admin_scope
def save_server():
if not _protected_configuration():
return jsonify(error="Enable local authentication before saving remote PBS credentials."), 409
try:
identifier = pbs.save_server(request.get_json(silent=True))
return jsonify(success=True, id=identifier, servers=pbs.public_servers())
except pbs.PBSError as exc:
return jsonify(error=str(exc)), 400
except OSError:
return jsonify(error="Could not save PBS connection."), 500
@pbs_bp.route("/api/pbs/servers/<identifier>", methods=["DELETE"])
@require_admin_scope
def delete_server(identifier):
if not _protected_configuration():
return jsonify(error="Enable local authentication before changing PBS connections."), 409
try:
pbs.delete_server(identifier)
return jsonify(success=True)
except pbs.PBSError as exc:
return jsonify(error=str(exc)), 400
except OSError:
return jsonify(error="Could not remove PBS connection."), 500
@pbs_bp.route("/api/pbs/status")
@require_auth
def status():
try:
return jsonify(pbs.get_status(request.args.get("server", "local")))
except pbs.PBSError as exc:
return jsonify(error=str(exc)), 502
+6
View File
@@ -90,6 +90,8 @@ from flask_terminal_routes import ( # noqa: E402
from flask_health_routes import health_bp # noqa: E402
from flask_auth_routes import auth_bp # noqa: E402
from flask_proxmenux_routes import proxmenux_bp # noqa: E402
from flask_pbs_routes import pbs_bp # noqa: E402
from pbs_manager import get_platform, is_pbs_host # noqa: E402
from flask_security_routes import security_bp # noqa: E402
from flask_notification_routes import notification_bp # noqa: E402
from flask_oci_routes import oci_bp # noqa: E402
@@ -193,6 +195,7 @@ post_install_versions.scan_at_startup()
app.register_blueprint(auth_bp)
app.register_blueprint(health_bp)
app.register_blueprint(proxmenux_bp)
app.register_blueprint(pbs_bp)
app.register_blueprint(security_bp)
app.register_blueprint(notification_bp)
app.register_blueprint(oci_bp)
@@ -3474,6 +3477,8 @@ def get_coral_info():
def get_proxmox_version():
"""Get Proxmox version if available. Cached for 6 hours."""
global _system_info_cache
if is_pbs_host():
return get_platform().get('version')
now = time.time()
if _system_info_cache['proxmox_version'] is not None and \
@@ -9863,6 +9868,7 @@ def api_system():
'cpu_cores': cpu_cores,
'cpu_threads': cpu_threads,
'proxmox_version': proxmox_version,
'platform': get_platform(),
'kernel_version': kernel_version,
'available_updates': available_updates
})
+36 -8
View File
@@ -51,6 +51,7 @@ DEBUG_PERF = False
# ─── Startup Grace Period ────────────────────────────────────────────────────
# Import centralized startup grace management for consistent behavior
import startup_grace
from pbs_manager import get_platform, PBS_SERVICES
def _is_startup_health_grace() -> bool:
"""Check if we're within the startup health grace period (5 min).
@@ -466,6 +467,7 @@ class HealthMonitor:
def __init__(self):
"""Initialize health monitor with state tracking"""
self.platform_type = get_platform().get("product")
self.state_history = defaultdict(list)
self.last_check_times = {}
self.cached_results = {}
@@ -919,9 +921,9 @@ class HealthMonitor:
_perf_log("services", (time.time() - _t) * 1000)
details['services'] = services_status
if services_status['status'] == 'CRITICAL':
critical_issues.append(f"PVE Services: {services_status.get('reason', 'Service failure')}")
critical_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service failure')}")
elif services_status['status'] == 'WARNING':
warning_issues.append(f"PVE Services: {services_status.get('reason', 'Service issue')}")
warning_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service issue')}")
# Priority 1.5: Proxmox Storage Check (External Module)
_t = time.time()
@@ -2347,6 +2349,8 @@ class HealthMonitor:
# The new _check_proxmox_storage function handles this logic better.
def _check_proxmox_storages(self) -> Dict[str, Any]:
"""Check Proxmox-specific storages (only report problems)"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
storages = {}
try:
@@ -3515,6 +3519,8 @@ class HealthMonitor:
Optimized VM/CT check - detects qmp failures and startup errors from logs.
Improved detection of container and VM errors from journalctl.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
try:
# First: auto-resolve any persisted VM/CT errors where the guest
# is now running. This clears stale "Failed to start" / QMP
@@ -3678,6 +3684,8 @@ class HealthMonitor:
Check VMs/CTs with persistent error tracking.
Errors persist until VM starts or 48h elapsed.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
try:
issues = []
vm_details = {}
@@ -3919,8 +3927,18 @@ class HealthMonitor:
"""
try:
# Build service list: base PVE services + corosync if clustered
services_to_check = list(self.PVE_SERVICES)
is_cluster = os.path.exists('/etc/corosync/corosync.conf')
pbs_host = getattr(self, "platform_type", "pve") == "pbs"
service_prefix = "pbs" if pbs_host else "pve"
product_name = "PBS" if pbs_host else "PVE"
services_to_check = list(PBS_SERVICES) if pbs_host else list(self.PVE_SERVICES)
if pbs_host:
# Old Monitor versions incorrectly treated absent PVE services
# as PBS failures; remove only those known inapplicable errors.
for service in self.PVE_SERVICES:
old_key = f'pve_service_{service}'
if health_persistence.is_error_active(old_key):
health_persistence.clear_error(old_key)
is_cluster = not pbs_host and os.path.exists('/etc/corosync/corosync.conf')
if is_cluster and 'corosync' not in services_to_check:
services_to_check.append('corosync')
@@ -3952,14 +3970,14 @@ class HealthMonitor:
# cleanup loop). Audit Tier 5 — Health stack.
for svc in services_to_check:
if svc not in failed_services:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
if health_persistence.is_error_active(error_key):
health_persistence.clear_error(error_key)
# Build checks dict with status per service
checks = {}
for svc in services_to_check:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
if svc in failed_services:
state = service_details.get(svc, 'inactive')
checks[svc] = {
@@ -3987,12 +4005,12 @@ class HealthMonitor:
# Record each failed service in persistence, respecting dismiss
active_failed = []
for svc in failed_services:
error_key = f'pve_service_{svc}'
error_key = f'{service_prefix}_service_{svc}'
rec_result = health_persistence.record_error(
error_key=error_key,
category='pve_services',
severity='CRITICAL',
reason=f'PVE service {svc} is {service_details.get(svc, "inactive")}',
reason=f'{product_name} service {svc} is {service_details.get(svc, "inactive")}',
details={'service': svc, 'state': service_details.get(svc, 'inactive')}
)
if rec_result and rec_result.get('type') == 'skipped_acknowledged':
@@ -5816,6 +5834,8 @@ class HealthMonitor:
- No persistent errors are recorded
This prevents false positives when NFS/PBS/remote storage is still mounting.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not PROXMOX_STORAGE_AVAILABLE:
return None
@@ -6205,6 +6225,8 @@ class HealthMonitor:
Notification cooldown is the standard 1h via the storage
category, so a CT teetering at 96% won't spam the channel.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
# Cheap pre-check: bail out when there are no running LXCs. On
# nodes without CTs, the previous version still paid ~860ms per
# cycle for a `pvesh /cluster/resources` call that produced no
@@ -6365,6 +6387,8 @@ class HealthMonitor:
path, so a slow / dead guest agent can't stretch the health
cycle.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
try:
import flask_server # deferred — avoids circular import
resources = flask_server.get_cached_pvesh_cluster_resources_vm() or []
@@ -6524,6 +6548,8 @@ class HealthMonitor:
check (already covered) because those are usually large data
mounts whose fill rate is driven by user content, not log spam.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not MOUNT_MONITOR_AVAILABLE:
return None
try:
@@ -6627,6 +6653,8 @@ class HealthMonitor:
check via _check_filesystem. Re-alerting on the same condition
through two channels just clutters the inbox.
"""
if getattr(self, "platform_type", "pve") == "pbs":
return None
if not PROXMOX_STORAGE_AVAILABLE:
return None
try:
+288
View File
@@ -0,0 +1,288 @@
"""Local PBS maintenance information and remote, read-only PBS monitoring."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import hmac
import http.client
import importlib.util
import json
import os
from pathlib import Path
import re
import secrets
import shutil
import ssl
import subprocess
import tempfile
import threading
import time
from urllib.parse import urlsplit
import auth_manager
try:
from proxmox_platform import detect_platform
except ModuleNotFoundError:
# Development checkout; AppImage bundles the canonical shared detector.
_path = Path(__file__).resolve().parents[2] / "scripts/global/proxmox_platform.py"
_spec = importlib.util.spec_from_file_location("proxmox_platform", _path)
_platform_module = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_platform_module)
detect_platform = _platform_module.detect_platform
_lock = threading.RLock()
_platform_cache = (0, {})
_status_cache = {}
PBS_SERVICES = ("proxmox-backup", "proxmox-backup-proxy")
MAX_RESPONSE = 4 * 1024 * 1024
class PBSError(ValueError):
pass
def get_platform():
global _platform_cache
with _lock:
if _platform_cache[0] <= time.monotonic():
_platform_cache = (time.monotonic() + 60, detect_platform())
return dict(_platform_cache[1])
def is_pbs_host():
return get_platform().get("product") == "pbs"
def _servers_path():
return auth_manager.CONFIG_DIR / "pbs_servers.json"
def _load_servers():
try:
data = json.loads(_servers_path().read_text())
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
raise ValueError()
return data
except FileNotFoundError:
return []
except (OSError, ValueError) as exc:
raise PBSError("Could not read the PBS server configuration.") from exc
def _save_servers(servers):
auth_manager.ensure_config_dir()
descriptor, temporary = tempfile.mkstemp(prefix=".pbs-servers-", dir=auth_manager.CONFIG_DIR)
try:
with os.fdopen(descriptor, "w") as handle:
os.fchmod(handle.fileno(), 0o600)
json.dump(servers, handle, indent=2)
os.replace(temporary, _servers_path())
finally:
if os.path.exists(temporary):
os.unlink(temporary)
_status_cache.clear()
def public_servers():
fields = ("id", "name", "url", "token_id", "fingerprint")
with _lock:
return [{**{key: row.get(key, "") for key in fields}, "has_token": bool(row.get("token_secret"))}
for row in _load_servers()]
def _validate_server(data, previous=None):
if not isinstance(data, dict):
raise PBSError("Invalid PBS server configuration.")
previous = previous or {}
row = {field: data.get(field, previous.get(field, ""))
for field in ("name", "url", "token_id", "token_secret", "fingerprint")}
if any(not isinstance(value, str) or len(value) > 4096 for value in row.values()):
raise PBSError("Invalid PBS connection field.")
row["name"] = row["name"].strip()
row["url"] = row["url"].strip().rstrip("/")
try:
parts = urlsplit(row["url"])
parts.port
if (parts.scheme != "https" or not parts.hostname or parts.username or parts.password
or parts.query or parts.fragment):
raise ValueError()
except ValueError as exc:
raise PBSError("Use an HTTPS PBS URL without embedded credentials, query or fragment.") from exc
if not row["name"] or len(row["name"]) > 128:
raise PBSError("A server name of at most 128 characters is required.")
row["token_id"] = row["token_id"].strip()
if not re.fullmatch(r"[^\s:!]+@[^\s:!]+![^\s:!]+", row["token_id"]):
raise PBSError("Enter a PBS API token ID in user@realm!token format.")
if (not row["token_secret"] or not row["token_secret"].isascii()
or any(char.isspace() or ord(char) < 32 for char in row["token_secret"])):
raise PBSError("A valid API token secret is required.")
if not row["token_id"].isascii():
raise PBSError("Invalid API token ID.")
fingerprint = row["fingerprint"].strip().lower().replace(":", "")
if fingerprint and not re.fullmatch("[a-f0-9]{64}", fingerprint):
raise PBSError("The certificate fingerprint must be SHA256 (64 hex characters).")
row["fingerprint"] = fingerprint
return row
def _remote_get(server, endpoint):
"""Validate the TLS peer BEFORE transmitting the PBS token (no redirects)."""
parts = urlsplit(server["url"])
fingerprint = server.get("fingerprint")
if fingerprint:
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE # Authenticated by the explicit pin below.
else:
context = ssl.create_default_context()
connection = http.client.HTTPSConnection(parts.hostname, parts.port or 443,
timeout=5, context=context)
try:
connection.connect()
if fingerprint:
actual = hashlib.sha256(connection.sock.getpeercert(binary_form=True)).hexdigest()
if not hmac.compare_digest(actual, fingerprint):
raise PBSError("PBS certificate fingerprint does not match. No credentials were sent.")
path = parts.path.rstrip("/") + "/api2/json" + endpoint
connection.request("GET", path, headers={
"Accept": "application/json",
"Authorization": "PBSAPIToken=" + server["token_id"] + ":" + server["token_secret"],
})
response = connection.getresponse()
content = response.read(MAX_RESPONSE + 1)
if response.status in (401, 403):
raise PBSError("PBS rejected the API token or its permissions. Check both the user and token ACLs.")
if response.status != 200 or len(content) > MAX_RESPONSE:
raise PBSError("PBS returned an invalid response. Redirects are not followed.")
document = json.loads(content)
if not isinstance(document, dict) or "data" not in document:
raise PBSError("PBS returned an invalid JSON document.")
return document["data"]
except PBSError:
raise
except (OSError, http.client.HTTPException, ValueError) as exc:
raise PBSError("Could not connect to PBS over verified HTTPS. Check the address, certificate and service.") from exc
finally:
connection.close()
def save_server(data):
identifier = data.get("id") if isinstance(data, dict) else None
if identifier is not None and not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", str(identifier)):
raise PBSError("Invalid PBS server ID.")
with _lock:
previous = next((row for row in _load_servers() if row["id"] == identifier), None)
row = _validate_server(data, previous)
# /version may be public; validate credentials on the protected status API.
usage = _remote_get(row, "/status/datastore-usage")
if not isinstance(usage, list):
raise PBSError("PBS returned an invalid datastore list.")
row.update(id=identifier or "pbs-" + secrets.token_hex(6), revision=secrets.token_hex(16))
with _lock:
servers = _load_servers()
if previous is None and len(servers) >= 32:
raise PBSError("At most 32 PBS connections may be configured.")
servers = [existing for existing in servers if existing["id"] != row["id"]]
_save_servers([*servers, row])
return row["id"]
def delete_server(identifier):
with _lock:
servers = _load_servers()
remaining = [row for row in servers if row["id"] != identifier]
if len(remaining) == len(servers):
raise PBSError("PBS server not found.")
_save_servers(remaining)
def _cli(*arguments):
try:
result = subprocess.run(["proxmox-backup-manager", *arguments, "--output-format", "json"],
check=True, text=True, capture_output=True, timeout=8)
data = json.loads(result.stdout)
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
raise ValueError()
return data
except (OSError, subprocess.SubprocessError, ValueError) as exc:
raise PBSError("Could not read PBS " + " ".join(arguments[:2]) + ". Check its service and permissions.") from exc
def _local_datastores():
stores = _cli("datastore", "list")
result = []
for store in stores:
row = {"name": store.get("name"), "path": store.get("path"),
"maintenance_mode": store.get("maintenance-mode"), "total": None, "used": None, "avail": None}
path = store.get("path")
# A removable datastore directory may survive unmounting. Do not report
# its underlying root filesystem as the missing datastore's capacity.
inactive = store.get("backing-device") and not (path and os.path.ismount(path))
if not path or inactive or store.get("maintenance-mode"):
row["error"] = "Datastore is offline, in maintenance or has no mounted backing device."
else:
try:
capacity = shutil.disk_usage(path)
row.update(total=capacity.total, used=capacity.used, avail=capacity.free)
except OSError:
row["error"] = "Datastore filesystem is not available."
result.append(row)
return result
def _local_services():
result = {}
for service in PBS_SERVICES:
try:
check = subprocess.run(["systemctl", "is-active", service], capture_output=True, text=True, timeout=3)
result[service] = check.stdout.strip() if check.returncode == 0 else "inactive"
except (OSError, subprocess.SubprocessError):
result[service] = "unknown"
return result
def get_status(identifier):
with _lock:
cached = _status_cache.get(identifier)
if cached and cached[0] > time.monotonic():
return cached[1]
errors = {}
if identifier == "local":
if not is_pbs_host():
raise PBSError("The local host is not a PBS server.")
name, version = "Local PBS", get_platform().get("version")
services = _local_services()
operations = {"datastores": _local_datastores,
"tasks": lambda: _cli("task", "list", "--all", "true", "--limit", "20"),
"sync": lambda: _cli("sync-job", "list"),
"verify": lambda: _cli("verify-job", "list"),
"prune": lambda: _cli("prune-job", "list")}
else:
with _lock:
server = next((row for row in _load_servers() if row["id"] == identifier), None)
if not server:
raise PBSError("PBS server not found.")
metadata = _remote_get(server, "/version")
name, version = server["name"], metadata.get("version") if isinstance(metadata, dict) else None
services = {}
endpoints = {"datastores": "/status/datastore-usage", "tasks": "/nodes/localhost/tasks?limit=20",
"sync": "/admin/sync", "verify": "/admin/verify", "prune": "/admin/prune"}
operations = {key: (lambda endpoint=path: _remote_get(server, endpoint)) for key, path in endpoints.items()}
result = {"server": identifier, "name": name, "version": version, "services": services,
"fetched_at": int(time.time()), "errors": errors}
with ThreadPoolExecutor(max_workers=5) as executor:
futures = {key: executor.submit(operation) for key, operation in operations.items()}
for key, future in futures.items():
try:
data = future.result()
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
raise PBSError("PBS returned an invalid list.")
result[key] = [{**row, "name": row.get("store", row.get("name"))} for row in data] if key == "datastores" else data
except PBSError as exc:
result[key] = None # Unknown/forbidden is never presented as zero.
errors[key] = str(exc)
with _lock:
if len(_status_cache) >= 33:
_status_cache.clear()
_status_cache[identifier] = (time.monotonic() + 15, result)
return result
+289
View File
@@ -0,0 +1,289 @@
"""PBS platform, HTTPS protocol, authorization and isolated AppImage regressions."""
import ast
from datetime import datetime, timedelta, timezone
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import ipaddress
import json
from pathlib import Path
import shlex
import ssl
import subprocess
import sys
import tempfile
import threading
from types import SimpleNamespace
import unittest
from unittest import mock
ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT / "AppImage/scripts"))
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from flask import Flask
import jwt
import auth_manager
import pbs_manager as pbs
from flask_pbs_routes import pbs_bp
class PlatformTests(unittest.TestCase):
def test_supported_platforms_and_partial_dpkg_failure(self):
module = pbs.detect_platform.__globals__
with tempfile.TemporaryDirectory() as directory:
release = Path(directory) / "os-release"
for package, version, suite, product in (
("pve-manager", "8.4.15", "bookworm", "pve"),
("pve-manager", "9.2.1", "trixie", "pve"),
("proxmox-backup-server", "3.4.0-1", "bookworm", "pbs"),
("proxmox-backup-server", "4.2.5-1", "trixie", "pbs"),
):
release.write_text(f'VERSION_CODENAME="{suite}"\n')
output = f"{package}\tinstall ok installed\t{version}\n"
with self.subTest(product=product, version=version), \
mock.patch.dict(module, {"OS_RELEASE": release}), \
mock.patch.object(subprocess, "run", return_value=SimpleNamespace(stdout=output, returncode=1)):
detected = pbs.detect_platform()
self.assertEqual(detected["product"], product)
self.assertEqual(detected["version"], version)
self.assertTrue(detected["supported"])
def test_client_only_and_removed_server_are_not_pbs(self):
for output in ("proxmox-backup-client\tinstall ok installed\t4.2.5\n",
"proxmox-backup-server\tdeinstall ok config-files\t4.2.5\n", ""):
with self.subTest(output=output), mock.patch.object(subprocess, "run", return_value=SimpleNamespace(stdout=output)):
self.assertEqual(pbs.detect_platform()["product"], "unknown")
def test_pbs_utilities_preserve_sources_and_never_call_pve_policy(self):
helper = ROOT / "scripts/global/repository-functions.sh"
for supported in (True, False):
harness = f'''
source {shlex.quote(str(helper))}
pmx_platform_field() {{ if [[ "$1" == product ]]; then echo pbs; else echo {str(supported).lower()}; fi; }}
msg_info() {{ :; }}
msg_error() {{ :; }}
translate() {{ printf '%s' "$1"; }}
pveversion() {{ echo 'FORBIDDEN: pveversion'; return 90; }}
repository_policy() {{ echo 'FORBIDDEN: repository mutation'; return 90; }}
apt-get() {{ echo 'FORBIDDEN: apt'; return 90; }}
ensure_repositories
'''
result = subprocess.run(["bash"], input=harness, text=True, capture_output=True)
self.assertNotIn("FORBIDDEN", result.stdout + result.stderr)
self.assertEqual(result.returncode, 0 if supported else 1)
class PBSManagerTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name)
patch = mock.patch.multiple(auth_manager, CONFIG_DIR=self.root, AUTH_CONFIG_FILE=self.root / "auth.json")
patch.start(); self.addCleanup(patch.stop)
auth_manager.save_auth_config({"enabled": True, "username": "admin", "password_hash": "configured",
"jwt_secret": "s" * 48, "api_tokens": [], "revoked_tokens": []})
auth_manager._invalidate_revoked_cache()
pbs._status_cache.clear()
self.server = {"name": "Backup", "url": "https://pbs.test:8007", "token_id": "audit@pbs!monitor",
"token_secret": "fixture-token-secret", "fingerprint": "a" * 64}
app = Flask(__name__); app.register_blueprint(pbs_bp)
self.client = app.test_client()
self.headers = {"Authorization": "Bearer " + auth_manager.generate_token("admin")}
def test_save_redacts_secret_and_probes_authenticated_endpoint(self):
with mock.patch.object(pbs, "_remote_get", return_value=[]) as request:
identifier = pbs.save_server(self.server)
request.assert_called_once()
self.assertEqual(request.call_args.args[1], "/status/datastore-usage")
self.assertEqual((self.root / "pbs_servers.json").stat().st_mode & 0o777, 0o600)
public = pbs.public_servers()
self.assertEqual(public[0]["id"], identifier)
self.assertNotIn("fixture-token-secret", json.dumps(public))
self.assertTrue(public[0]["has_token"])
with mock.patch.object(pbs, "_remote_get", return_value=[]):
pbs.save_server({"id": identifier, "name": "Renamed"})
self.assertEqual(pbs._load_servers()[0]["token_secret"], "fixture-token-secret")
pbs.delete_server(identifier)
self.assertEqual(pbs.public_servers(), [])
def test_failed_credentials_or_invalid_config_do_not_replace_saved_connection(self):
with mock.patch.object(pbs, "_remote_get", return_value=[]):
identifier = pbs.save_server(self.server)
original = (self.root / "pbs_servers.json").read_bytes()
with mock.patch.object(pbs, "_remote_get", side_effect=pbs.PBSError("Rejected token")):
with self.assertRaises(pbs.PBSError):
pbs.save_server({**self.server, "id": identifier, "token_secret": "wrong"})
for changes in ({"url": "http://pbs.test"}, {"url": "https://user:pass@pbs.test"},
{"fingerprint": "bad-pin"}, {"token_id": "not-a-token"},
{"token_secret": "line\nbreak"}):
with self.subTest(changes=changes), self.assertRaises(pbs.PBSError):
pbs.save_server({**self.server, **changes, "id": identifier})
self.assertEqual((self.root / "pbs_servers.json").read_bytes(), original)
def test_remote_partial_failure_is_unknown_instead_of_zero(self):
def remote(server, endpoint):
if endpoint == "/version": return {"version": "4.2.5"}
if endpoint == "/status/datastore-usage": return [{"store": "backup", "total": 100, "used": 30, "avail": 70}]
if endpoint == "/admin/verify": raise pbs.PBSError("Permission denied")
return []
with mock.patch.object(pbs, "_remote_get", side_effect=remote) as request:
identifier = pbs.save_server(self.server)
result = pbs.get_status(identifier)
count = request.call_count
self.assertEqual(pbs.get_status(identifier), result)
self.assertEqual(request.call_count, count)
self.assertEqual(result["datastores"][0]["name"], "backup")
self.assertIsNone(result["verify"])
self.assertIn("verify", result["errors"])
def test_unmounted_removable_datastore_has_no_fabricated_capacity(self):
with mock.patch.object(pbs, "_cli", return_value=[{"name": "removable", "path": "/not-mounted", "backing-device": "uuid"}]), \
mock.patch.object(pbs.os.path, "ismount", return_value=False), \
mock.patch.object(pbs.shutil, "disk_usage") as usage:
result = pbs._local_datastores()
usage.assert_not_called()
self.assertIsNone(result[0]["total"])
self.assertIn("error", result[0])
def test_local_status_reads_pbs_commands_and_filesystem_capacity(self):
def run(command, **kwargs):
self.assertNotIn("shell", kwargs)
self.assertEqual(command[0], "proxmox-backup-manager")
self.assertEqual(command[-2:], ["--output-format", "json"])
data = [{"name": "backup", "path": str(self.root)}] if command[1:3] == ["datastore", "list"] else []
return SimpleNamespace(stdout=json.dumps(data), returncode=0)
with mock.patch.object(pbs, "get_platform", return_value={"product": "pbs", "version": "4.2.5"}), \
mock.patch.object(pbs, "_local_services", return_value={key: "active" for key in pbs.PBS_SERVICES}), \
mock.patch.object(pbs.subprocess, "run", side_effect=run), \
mock.patch.object(pbs.shutil, "disk_usage", return_value=SimpleNamespace(total=100, used=30, free=70)):
result = pbs.get_status("local")
self.assertEqual(result["version"], "4.2.5")
self.assertEqual(result["errors"], {})
self.assertEqual(result["datastores"][0]["used"], 30)
self.assertEqual(set(result["services"]), set(pbs.PBS_SERVICES))
def test_connection_api_requires_full_admin_and_does_not_return_secret(self):
self.assertEqual(self.client.get("/api/pbs/servers").status_code, 401)
with mock.patch.object(pbs, "_remote_get", return_value=[]):
response = self.client.post("/api/pbs/servers", headers=self.headers, json=self.server)
self.assertEqual(response.status_code, 200)
self.assertNotIn("fixture-token-secret", response.text)
payload = jwt.decode(self.headers["Authorization"].split()[1], "s" * 48,
algorithms=["HS256"], audience=auth_manager.JWT_AUDIENCE)
payload["scope"] = "read_only"
read_only = {"Authorization": "Bearer " + jwt.encode(payload, "s" * 48, algorithm="HS256")}
self.assertEqual(self.client.post("/api/pbs/servers", headers=read_only, json=self.server).status_code, 403)
self.assertEqual(self.client.get("/api/pbs/servers", headers=read_only).status_code, 200)
auth_manager.save_auth_config({"enabled": False})
self.assertEqual(self.client.post("/api/pbs/servers", json=self.server).status_code, 409)
def test_health_uses_pbs_services_and_skips_pve_only_checks(self):
source = ast.parse((ROOT / "AppImage/scripts/health_monitor.py").read_text())
names = ("_check_pve_services", "_check_vms_cts_optimized", "_check_vms_cts_with_persistence",
"_check_proxmox_storages", "_check_proxmox_storage", "_check_lxc_disk_usage",
"_check_vm_disk_usage", "_check_lxc_mount_capacity", "_check_pve_storage_capacity")
functions = [node for node in ast.walk(source) if isinstance(node, ast.FunctionDef) and node.name in names]
persistence = mock.Mock(); persistence.is_error_active.return_value = False
run = mock.Mock(return_value=SimpleNamespace(returncode=0, stdout="active\n"))
namespace = {"Dict": dict, "Any": object, "Optional": __import__('typing').Optional,
"PBS_SERVICES": pbs.PBS_SERVICES, "health_persistence": persistence,
"subprocess": SimpleNamespace(run=run), "os": __import__('os')}
exec(compile(ast.Module(body=functions, type_ignores=[]), '<health-pbs-test>', 'exec'), namespace)
instance = SimpleNamespace(platform_type="pbs", PVE_SERVICES=["pveproxy", "pvedaemon", "pvestatd", "pve-cluster"])
services = namespace["_check_pve_services"](instance)
self.assertEqual(services["status"], "OK")
self.assertEqual([call.args[0][2] for call in run.call_args_list], list(pbs.PBS_SERVICES))
for name in names[1:]:
result = namespace[name](instance)
self.assertTrue(result is None or result.get("not_applicable"))
def test_appimage_copies_and_imports_pbs_dependencies_outside_source_tree(self):
builder = (ROOT / "AppImage/scripts/build_appimage.sh").read_text()
modules = ("pbs_manager", "flask_pbs_routes", "auth_manager", "jwt_middleware")
copies = []
for name in modules:
matches = [line for line in builder.splitlines() if line.startswith(f'cp "$SCRIPT_DIR/{name}.py"')]
self.assertEqual(len(matches), 1, name)
copies.extend(matches)
copies.extend(line for line in builder.splitlines() if line.startswith('cp "$APPIMAGE_ROOT/../scripts/global/proxmox_platform.py"'))
self.assertEqual(len(copies), len(modules) + 1)
app = self.root / 'AppDir'; (app / 'usr/bin').mkdir(parents=True)
q = shlex.quote
shell = f'APP_DIR={q(str(app))}\nSCRIPT_DIR={q(str(ROOT / "AppImage/scripts"))}\nAPPIMAGE_ROOT={q(str(ROOT / "AppImage"))}\n' + '\n'.join(copies)
subprocess.run(['bash', '-e', '-c', shell], check=True)
code = f'import sys; sys.path.insert(0, {str(app / "usr/bin")!r}); import pbs_manager, flask_pbs_routes, proxmox_platform'
subprocess.run([sys.executable, '-I', '-B', '-c', code], cwd=self.root, check=True, capture_output=True)
class TLSTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temporary = tempfile.TemporaryDirectory()
root = Path(cls.temporary.name)
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
certificate = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject).public_key(key.public_key())
.serial_number(x509.random_serial_number()).not_valid_before(datetime.now(timezone.utc) - timedelta(minutes=1))
.not_valid_after(datetime.now(timezone.utc) + timedelta(hours=1))
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
.sign(key, hashes.SHA256()))
cert_path, key_path = root / 'server.pem', root / 'server.key'
cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
key_path.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()))
cls.fingerprint = hashlib.sha256(certificate.public_bytes(serialization.Encoding.DER)).hexdigest()
cls.requests = []
cls.response_status = 200
class Handler(BaseHTTPRequestHandler):
def handle(self):
try:
super().handle()
except ConnectionResetError:
pass # A rejected TLS pin closes before any HTTP request.
def do_GET(self):
cls.requests.append((self.path, self.headers.get("Authorization")))
body = json.dumps({"data": []}).encode()
self.send_response(cls.response_status)
self.send_header('Content-Length', str(len(body)))
self.end_headers(); self.wfile.write(body)
def log_message(self, *args): pass
cls.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER); context.load_cert_chain(cert_path, key_path)
cls.server.socket = context.wrap_socket(cls.server.socket, server_side=True)
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True); cls.thread.start()
@classmethod
def tearDownClass(cls):
cls.server.shutdown(); cls.server.server_close(); cls.thread.join(timeout=5); cls.temporary.cleanup()
def setUp(self):
self.requests.clear(); type(self).response_status = 200
self.connection = {"url": f'https://127.0.0.1:{self.server.server_port}',
"fingerprint": self.fingerprint, "token_id": "audit@pbs!monitor", "token_secret": "fake-secret"}
def test_real_tls_matching_pin_sends_correct_pbs_authorization(self):
self.assertEqual(pbs._remote_get(self.connection, '/status/datastore-usage'), [])
self.assertEqual(self.requests, [('/api2/json/status/datastore-usage', 'PBSAPIToken=audit@pbs!monitor:fake-secret')])
def test_bad_pin_and_untrusted_ca_send_no_http_credentials(self):
for fingerprint in ('0' * 64, ''):
with self.subTest(fingerprint=fingerprint), self.assertRaises(pbs.PBSError):
pbs._remote_get({**self.connection, "fingerprint": fingerprint}, '/status/datastore-usage')
self.assertEqual(self.requests, [])
def test_bad_credentials_and_redirects_are_rejected(self):
for status in (401, 403, 302):
type(self).response_status = status
with self.subTest(status=status), self.assertRaises(pbs.PBSError):
pbs._remote_get(self.connection, '/status/datastore-usage')
if __name__ == '__main__':
unittest.main()