mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-01 10:22:08 +00:00
feat: support PBS hosts and remote PBS monitoring
This commit is contained in:
@@ -1115,6 +1115,9 @@ def _detect_proxmox_certificate_paths():
|
||||
return PROXMOX_CUSTOM_CERT_PATH, PROXMOX_CUSTOM_KEY_PATH
|
||||
if os.path.isfile(PROXMOX_CERT_PATH) and os.path.isfile(PROXMOX_KEY_PATH):
|
||||
return PROXMOX_CERT_PATH, PROXMOX_KEY_PATH
|
||||
pbs_cert, pbs_key = "/etc/proxmox-backup/proxy.pem", "/etc/proxmox-backup/proxy.key"
|
||||
if os.path.isfile(pbs_cert) and os.path.isfile(pbs_key):
|
||||
return pbs_cert, pbs_key
|
||||
return "", ""
|
||||
|
||||
|
||||
|
||||
@@ -160,6 +160,9 @@ cp "$SCRIPT_DIR/flask_terminal_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || ech
|
||||
cp "$SCRIPT_DIR/hardware_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ hardware_monitor.py not found"
|
||||
cp "$SCRIPT_DIR/temperature_sensor_resolver.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ temperature_sensor_resolver.py not found"
|
||||
cp "$SCRIPT_DIR/proxmox_storage_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ proxmox_storage_monitor.py not found"
|
||||
cp "$SCRIPT_DIR/pbs_manager.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/flask_pbs_routes.py" "$APP_DIR/usr/bin/"
|
||||
cp "$APPIMAGE_ROOT/../scripts/global/proxmox_platform.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/flask_script_runner.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_script_runner.py not found"
|
||||
cp "$SCRIPT_DIR/security_manager.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ security_manager.py not found"
|
||||
cp "$SCRIPT_DIR/flask_security_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_security_routes.py not found"
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Read-only PBS status and administrative connection management."""
|
||||
|
||||
from flask import Blueprint, jsonify, request
|
||||
import auth_manager
|
||||
import pbs_manager as pbs
|
||||
from jwt_middleware import require_auth, require_admin_scope
|
||||
|
||||
pbs_bp = Blueprint("pbs", __name__)
|
||||
|
||||
|
||||
@pbs_bp.after_request
|
||||
def no_cache(response):
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
|
||||
@pbs_bp.route("/api/platform")
|
||||
@require_auth
|
||||
def platform():
|
||||
return jsonify(pbs.get_platform())
|
||||
|
||||
|
||||
@pbs_bp.route("/api/pbs/servers")
|
||||
@require_auth
|
||||
def servers():
|
||||
try:
|
||||
return jsonify(servers=pbs.public_servers(), local_available=pbs.is_pbs_host())
|
||||
except pbs.PBSError as exc:
|
||||
return jsonify(error=str(exc)), 500
|
||||
|
||||
|
||||
def _protected_configuration():
|
||||
local = auth_manager.load_auth_config()
|
||||
return local.get("enabled") and not local.get("declined") and local.get("password_hash")
|
||||
|
||||
|
||||
@pbs_bp.route("/api/pbs/servers", methods=["POST"])
|
||||
@require_admin_scope
|
||||
def save_server():
|
||||
if not _protected_configuration():
|
||||
return jsonify(error="Enable local authentication before saving remote PBS credentials."), 409
|
||||
try:
|
||||
identifier = pbs.save_server(request.get_json(silent=True))
|
||||
return jsonify(success=True, id=identifier, servers=pbs.public_servers())
|
||||
except pbs.PBSError as exc:
|
||||
return jsonify(error=str(exc)), 400
|
||||
except OSError:
|
||||
return jsonify(error="Could not save PBS connection."), 500
|
||||
|
||||
|
||||
@pbs_bp.route("/api/pbs/servers/<identifier>", methods=["DELETE"])
|
||||
@require_admin_scope
|
||||
def delete_server(identifier):
|
||||
if not _protected_configuration():
|
||||
return jsonify(error="Enable local authentication before changing PBS connections."), 409
|
||||
try:
|
||||
pbs.delete_server(identifier)
|
||||
return jsonify(success=True)
|
||||
except pbs.PBSError as exc:
|
||||
return jsonify(error=str(exc)), 400
|
||||
except OSError:
|
||||
return jsonify(error="Could not remove PBS connection."), 500
|
||||
|
||||
|
||||
@pbs_bp.route("/api/pbs/status")
|
||||
@require_auth
|
||||
def status():
|
||||
try:
|
||||
return jsonify(pbs.get_status(request.args.get("server", "local")))
|
||||
except pbs.PBSError as exc:
|
||||
return jsonify(error=str(exc)), 502
|
||||
@@ -90,6 +90,8 @@ from flask_terminal_routes import ( # noqa: E402
|
||||
from flask_health_routes import health_bp # noqa: E402
|
||||
from flask_auth_routes import auth_bp # noqa: E402
|
||||
from flask_proxmenux_routes import proxmenux_bp # noqa: E402
|
||||
from flask_pbs_routes import pbs_bp # noqa: E402
|
||||
from pbs_manager import get_platform, is_pbs_host # noqa: E402
|
||||
from flask_security_routes import security_bp # noqa: E402
|
||||
from flask_notification_routes import notification_bp # noqa: E402
|
||||
from flask_oci_routes import oci_bp # noqa: E402
|
||||
@@ -193,6 +195,7 @@ post_install_versions.scan_at_startup()
|
||||
app.register_blueprint(auth_bp)
|
||||
app.register_blueprint(health_bp)
|
||||
app.register_blueprint(proxmenux_bp)
|
||||
app.register_blueprint(pbs_bp)
|
||||
app.register_blueprint(security_bp)
|
||||
app.register_blueprint(notification_bp)
|
||||
app.register_blueprint(oci_bp)
|
||||
@@ -3474,6 +3477,8 @@ def get_coral_info():
|
||||
def get_proxmox_version():
|
||||
"""Get Proxmox version if available. Cached for 6 hours."""
|
||||
global _system_info_cache
|
||||
if is_pbs_host():
|
||||
return get_platform().get('version')
|
||||
|
||||
now = time.time()
|
||||
if _system_info_cache['proxmox_version'] is not None and \
|
||||
@@ -9863,6 +9868,7 @@ def api_system():
|
||||
'cpu_cores': cpu_cores,
|
||||
'cpu_threads': cpu_threads,
|
||||
'proxmox_version': proxmox_version,
|
||||
'platform': get_platform(),
|
||||
'kernel_version': kernel_version,
|
||||
'available_updates': available_updates
|
||||
})
|
||||
|
||||
@@ -51,6 +51,7 @@ DEBUG_PERF = False
|
||||
# ─── Startup Grace Period ────────────────────────────────────────────────────
|
||||
# Import centralized startup grace management for consistent behavior
|
||||
import startup_grace
|
||||
from pbs_manager import get_platform, PBS_SERVICES
|
||||
|
||||
def _is_startup_health_grace() -> bool:
|
||||
"""Check if we're within the startup health grace period (5 min).
|
||||
@@ -466,6 +467,7 @@ class HealthMonitor:
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize health monitor with state tracking"""
|
||||
self.platform_type = get_platform().get("product")
|
||||
self.state_history = defaultdict(list)
|
||||
self.last_check_times = {}
|
||||
self.cached_results = {}
|
||||
@@ -919,9 +921,9 @@ class HealthMonitor:
|
||||
_perf_log("services", (time.time() - _t) * 1000)
|
||||
details['services'] = services_status
|
||||
if services_status['status'] == 'CRITICAL':
|
||||
critical_issues.append(f"PVE Services: {services_status.get('reason', 'Service failure')}")
|
||||
critical_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service failure')}")
|
||||
elif services_status['status'] == 'WARNING':
|
||||
warning_issues.append(f"PVE Services: {services_status.get('reason', 'Service issue')}")
|
||||
warning_issues.append(f"Proxmox Services: {services_status.get('reason', 'Service issue')}")
|
||||
|
||||
# Priority 1.5: Proxmox Storage Check (External Module)
|
||||
_t = time.time()
|
||||
@@ -2347,6 +2349,8 @@ class HealthMonitor:
|
||||
# The new _check_proxmox_storage function handles this logic better.
|
||||
def _check_proxmox_storages(self) -> Dict[str, Any]:
|
||||
"""Check Proxmox-specific storages (only report problems)"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
storages = {}
|
||||
|
||||
try:
|
||||
@@ -3515,6 +3519,8 @@ class HealthMonitor:
|
||||
Optimized VM/CT check - detects qmp failures and startup errors from logs.
|
||||
Improved detection of container and VM errors from journalctl.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
try:
|
||||
# First: auto-resolve any persisted VM/CT errors where the guest
|
||||
# is now running. This clears stale "Failed to start" / QMP
|
||||
@@ -3678,6 +3684,8 @@ class HealthMonitor:
|
||||
Check VMs/CTs with persistent error tracking.
|
||||
Errors persist until VM starts or 48h elapsed.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return {'status': 'OK', 'reason': 'Not applicable on PBS', 'checks': {}, 'not_applicable': True}
|
||||
try:
|
||||
issues = []
|
||||
vm_details = {}
|
||||
@@ -3919,8 +3927,18 @@ class HealthMonitor:
|
||||
"""
|
||||
try:
|
||||
# Build service list: base PVE services + corosync if clustered
|
||||
services_to_check = list(self.PVE_SERVICES)
|
||||
is_cluster = os.path.exists('/etc/corosync/corosync.conf')
|
||||
pbs_host = getattr(self, "platform_type", "pve") == "pbs"
|
||||
service_prefix = "pbs" if pbs_host else "pve"
|
||||
product_name = "PBS" if pbs_host else "PVE"
|
||||
services_to_check = list(PBS_SERVICES) if pbs_host else list(self.PVE_SERVICES)
|
||||
if pbs_host:
|
||||
# Old Monitor versions incorrectly treated absent PVE services
|
||||
# as PBS failures; remove only those known inapplicable errors.
|
||||
for service in self.PVE_SERVICES:
|
||||
old_key = f'pve_service_{service}'
|
||||
if health_persistence.is_error_active(old_key):
|
||||
health_persistence.clear_error(old_key)
|
||||
is_cluster = not pbs_host and os.path.exists('/etc/corosync/corosync.conf')
|
||||
if is_cluster and 'corosync' not in services_to_check:
|
||||
services_to_check.append('corosync')
|
||||
|
||||
@@ -3952,14 +3970,14 @@ class HealthMonitor:
|
||||
# cleanup loop). Audit Tier 5 — Health stack.
|
||||
for svc in services_to_check:
|
||||
if svc not in failed_services:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
if health_persistence.is_error_active(error_key):
|
||||
health_persistence.clear_error(error_key)
|
||||
|
||||
# Build checks dict with status per service
|
||||
checks = {}
|
||||
for svc in services_to_check:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
if svc in failed_services:
|
||||
state = service_details.get(svc, 'inactive')
|
||||
checks[svc] = {
|
||||
@@ -3987,12 +4005,12 @@ class HealthMonitor:
|
||||
# Record each failed service in persistence, respecting dismiss
|
||||
active_failed = []
|
||||
for svc in failed_services:
|
||||
error_key = f'pve_service_{svc}'
|
||||
error_key = f'{service_prefix}_service_{svc}'
|
||||
rec_result = health_persistence.record_error(
|
||||
error_key=error_key,
|
||||
category='pve_services',
|
||||
severity='CRITICAL',
|
||||
reason=f'PVE service {svc} is {service_details.get(svc, "inactive")}',
|
||||
reason=f'{product_name} service {svc} is {service_details.get(svc, "inactive")}',
|
||||
details={'service': svc, 'state': service_details.get(svc, 'inactive')}
|
||||
)
|
||||
if rec_result and rec_result.get('type') == 'skipped_acknowledged':
|
||||
@@ -5816,6 +5834,8 @@ class HealthMonitor:
|
||||
- No persistent errors are recorded
|
||||
This prevents false positives when NFS/PBS/remote storage is still mounting.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not PROXMOX_STORAGE_AVAILABLE:
|
||||
return None
|
||||
|
||||
@@ -6205,6 +6225,8 @@ class HealthMonitor:
|
||||
Notification cooldown is the standard 1h via the storage
|
||||
category, so a CT teetering at 96% won't spam the channel.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
# Cheap pre-check: bail out when there are no running LXCs. On
|
||||
# nodes without CTs, the previous version still paid ~860ms per
|
||||
# cycle for a `pvesh /cluster/resources` call that produced no
|
||||
@@ -6365,6 +6387,8 @@ class HealthMonitor:
|
||||
path, so a slow / dead guest agent can't stretch the health
|
||||
cycle.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
try:
|
||||
import flask_server # deferred — avoids circular import
|
||||
resources = flask_server.get_cached_pvesh_cluster_resources_vm() or []
|
||||
@@ -6524,6 +6548,8 @@ class HealthMonitor:
|
||||
check (already covered) because those are usually large data
|
||||
mounts whose fill rate is driven by user content, not log spam.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not MOUNT_MONITOR_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
@@ -6627,6 +6653,8 @@ class HealthMonitor:
|
||||
check via _check_filesystem. Re-alerting on the same condition
|
||||
through two channels just clutters the inbox.
|
||||
"""
|
||||
if getattr(self, "platform_type", "pve") == "pbs":
|
||||
return None
|
||||
if not PROXMOX_STORAGE_AVAILABLE:
|
||||
return None
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Local PBS maintenance information and remote, read-only PBS monitoring."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import auth_manager
|
||||
|
||||
try:
|
||||
from proxmox_platform import detect_platform
|
||||
except ModuleNotFoundError:
|
||||
# Development checkout; AppImage bundles the canonical shared detector.
|
||||
_path = Path(__file__).resolve().parents[2] / "scripts/global/proxmox_platform.py"
|
||||
_spec = importlib.util.spec_from_file_location("proxmox_platform", _path)
|
||||
_platform_module = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(_platform_module)
|
||||
detect_platform = _platform_module.detect_platform
|
||||
|
||||
_lock = threading.RLock()
|
||||
_platform_cache = (0, {})
|
||||
_status_cache = {}
|
||||
PBS_SERVICES = ("proxmox-backup", "proxmox-backup-proxy")
|
||||
MAX_RESPONSE = 4 * 1024 * 1024
|
||||
|
||||
|
||||
class PBSError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def get_platform():
|
||||
global _platform_cache
|
||||
with _lock:
|
||||
if _platform_cache[0] <= time.monotonic():
|
||||
_platform_cache = (time.monotonic() + 60, detect_platform())
|
||||
return dict(_platform_cache[1])
|
||||
|
||||
|
||||
def is_pbs_host():
|
||||
return get_platform().get("product") == "pbs"
|
||||
|
||||
|
||||
def _servers_path():
|
||||
return auth_manager.CONFIG_DIR / "pbs_servers.json"
|
||||
|
||||
|
||||
def _load_servers():
|
||||
try:
|
||||
data = json.loads(_servers_path().read_text())
|
||||
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
|
||||
raise ValueError()
|
||||
return data
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
except (OSError, ValueError) as exc:
|
||||
raise PBSError("Could not read the PBS server configuration.") from exc
|
||||
|
||||
|
||||
def _save_servers(servers):
|
||||
auth_manager.ensure_config_dir()
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".pbs-servers-", dir=auth_manager.CONFIG_DIR)
|
||||
try:
|
||||
with os.fdopen(descriptor, "w") as handle:
|
||||
os.fchmod(handle.fileno(), 0o600)
|
||||
json.dump(servers, handle, indent=2)
|
||||
os.replace(temporary, _servers_path())
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
_status_cache.clear()
|
||||
|
||||
|
||||
def public_servers():
|
||||
fields = ("id", "name", "url", "token_id", "fingerprint")
|
||||
with _lock:
|
||||
return [{**{key: row.get(key, "") for key in fields}, "has_token": bool(row.get("token_secret"))}
|
||||
for row in _load_servers()]
|
||||
|
||||
|
||||
def _validate_server(data, previous=None):
|
||||
if not isinstance(data, dict):
|
||||
raise PBSError("Invalid PBS server configuration.")
|
||||
previous = previous or {}
|
||||
row = {field: data.get(field, previous.get(field, ""))
|
||||
for field in ("name", "url", "token_id", "token_secret", "fingerprint")}
|
||||
if any(not isinstance(value, str) or len(value) > 4096 for value in row.values()):
|
||||
raise PBSError("Invalid PBS connection field.")
|
||||
row["name"] = row["name"].strip()
|
||||
row["url"] = row["url"].strip().rstrip("/")
|
||||
try:
|
||||
parts = urlsplit(row["url"])
|
||||
parts.port
|
||||
if (parts.scheme != "https" or not parts.hostname or parts.username or parts.password
|
||||
or parts.query or parts.fragment):
|
||||
raise ValueError()
|
||||
except ValueError as exc:
|
||||
raise PBSError("Use an HTTPS PBS URL without embedded credentials, query or fragment.") from exc
|
||||
if not row["name"] or len(row["name"]) > 128:
|
||||
raise PBSError("A server name of at most 128 characters is required.")
|
||||
row["token_id"] = row["token_id"].strip()
|
||||
if not re.fullmatch(r"[^\s:!]+@[^\s:!]+![^\s:!]+", row["token_id"]):
|
||||
raise PBSError("Enter a PBS API token ID in user@realm!token format.")
|
||||
if (not row["token_secret"] or not row["token_secret"].isascii()
|
||||
or any(char.isspace() or ord(char) < 32 for char in row["token_secret"])):
|
||||
raise PBSError("A valid API token secret is required.")
|
||||
if not row["token_id"].isascii():
|
||||
raise PBSError("Invalid API token ID.")
|
||||
fingerprint = row["fingerprint"].strip().lower().replace(":", "")
|
||||
if fingerprint and not re.fullmatch("[a-f0-9]{64}", fingerprint):
|
||||
raise PBSError("The certificate fingerprint must be SHA256 (64 hex characters).")
|
||||
row["fingerprint"] = fingerprint
|
||||
return row
|
||||
|
||||
|
||||
def _remote_get(server, endpoint):
|
||||
"""Validate the TLS peer BEFORE transmitting the PBS token (no redirects)."""
|
||||
parts = urlsplit(server["url"])
|
||||
fingerprint = server.get("fingerprint")
|
||||
if fingerprint:
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE # Authenticated by the explicit pin below.
|
||||
else:
|
||||
context = ssl.create_default_context()
|
||||
connection = http.client.HTTPSConnection(parts.hostname, parts.port or 443,
|
||||
timeout=5, context=context)
|
||||
try:
|
||||
connection.connect()
|
||||
if fingerprint:
|
||||
actual = hashlib.sha256(connection.sock.getpeercert(binary_form=True)).hexdigest()
|
||||
if not hmac.compare_digest(actual, fingerprint):
|
||||
raise PBSError("PBS certificate fingerprint does not match. No credentials were sent.")
|
||||
path = parts.path.rstrip("/") + "/api2/json" + endpoint
|
||||
connection.request("GET", path, headers={
|
||||
"Accept": "application/json",
|
||||
"Authorization": "PBSAPIToken=" + server["token_id"] + ":" + server["token_secret"],
|
||||
})
|
||||
response = connection.getresponse()
|
||||
content = response.read(MAX_RESPONSE + 1)
|
||||
if response.status in (401, 403):
|
||||
raise PBSError("PBS rejected the API token or its permissions. Check both the user and token ACLs.")
|
||||
if response.status != 200 or len(content) > MAX_RESPONSE:
|
||||
raise PBSError("PBS returned an invalid response. Redirects are not followed.")
|
||||
document = json.loads(content)
|
||||
if not isinstance(document, dict) or "data" not in document:
|
||||
raise PBSError("PBS returned an invalid JSON document.")
|
||||
return document["data"]
|
||||
except PBSError:
|
||||
raise
|
||||
except (OSError, http.client.HTTPException, ValueError) as exc:
|
||||
raise PBSError("Could not connect to PBS over verified HTTPS. Check the address, certificate and service.") from exc
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def save_server(data):
|
||||
identifier = data.get("id") if isinstance(data, dict) else None
|
||||
if identifier is not None and not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", str(identifier)):
|
||||
raise PBSError("Invalid PBS server ID.")
|
||||
with _lock:
|
||||
previous = next((row for row in _load_servers() if row["id"] == identifier), None)
|
||||
row = _validate_server(data, previous)
|
||||
# /version may be public; validate credentials on the protected status API.
|
||||
usage = _remote_get(row, "/status/datastore-usage")
|
||||
if not isinstance(usage, list):
|
||||
raise PBSError("PBS returned an invalid datastore list.")
|
||||
row.update(id=identifier or "pbs-" + secrets.token_hex(6), revision=secrets.token_hex(16))
|
||||
with _lock:
|
||||
servers = _load_servers()
|
||||
if previous is None and len(servers) >= 32:
|
||||
raise PBSError("At most 32 PBS connections may be configured.")
|
||||
servers = [existing for existing in servers if existing["id"] != row["id"]]
|
||||
_save_servers([*servers, row])
|
||||
return row["id"]
|
||||
|
||||
|
||||
def delete_server(identifier):
|
||||
with _lock:
|
||||
servers = _load_servers()
|
||||
remaining = [row for row in servers if row["id"] != identifier]
|
||||
if len(remaining) == len(servers):
|
||||
raise PBSError("PBS server not found.")
|
||||
_save_servers(remaining)
|
||||
|
||||
|
||||
def _cli(*arguments):
|
||||
try:
|
||||
result = subprocess.run(["proxmox-backup-manager", *arguments, "--output-format", "json"],
|
||||
check=True, text=True, capture_output=True, timeout=8)
|
||||
data = json.loads(result.stdout)
|
||||
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
|
||||
raise ValueError()
|
||||
return data
|
||||
except (OSError, subprocess.SubprocessError, ValueError) as exc:
|
||||
raise PBSError("Could not read PBS " + " ".join(arguments[:2]) + ". Check its service and permissions.") from exc
|
||||
|
||||
|
||||
def _local_datastores():
|
||||
stores = _cli("datastore", "list")
|
||||
result = []
|
||||
for store in stores:
|
||||
row = {"name": store.get("name"), "path": store.get("path"),
|
||||
"maintenance_mode": store.get("maintenance-mode"), "total": None, "used": None, "avail": None}
|
||||
path = store.get("path")
|
||||
# A removable datastore directory may survive unmounting. Do not report
|
||||
# its underlying root filesystem as the missing datastore's capacity.
|
||||
inactive = store.get("backing-device") and not (path and os.path.ismount(path))
|
||||
if not path or inactive or store.get("maintenance-mode"):
|
||||
row["error"] = "Datastore is offline, in maintenance or has no mounted backing device."
|
||||
else:
|
||||
try:
|
||||
capacity = shutil.disk_usage(path)
|
||||
row.update(total=capacity.total, used=capacity.used, avail=capacity.free)
|
||||
except OSError:
|
||||
row["error"] = "Datastore filesystem is not available."
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def _local_services():
|
||||
result = {}
|
||||
for service in PBS_SERVICES:
|
||||
try:
|
||||
check = subprocess.run(["systemctl", "is-active", service], capture_output=True, text=True, timeout=3)
|
||||
result[service] = check.stdout.strip() if check.returncode == 0 else "inactive"
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
result[service] = "unknown"
|
||||
return result
|
||||
|
||||
|
||||
def get_status(identifier):
|
||||
with _lock:
|
||||
cached = _status_cache.get(identifier)
|
||||
if cached and cached[0] > time.monotonic():
|
||||
return cached[1]
|
||||
errors = {}
|
||||
if identifier == "local":
|
||||
if not is_pbs_host():
|
||||
raise PBSError("The local host is not a PBS server.")
|
||||
name, version = "Local PBS", get_platform().get("version")
|
||||
services = _local_services()
|
||||
operations = {"datastores": _local_datastores,
|
||||
"tasks": lambda: _cli("task", "list", "--all", "true", "--limit", "20"),
|
||||
"sync": lambda: _cli("sync-job", "list"),
|
||||
"verify": lambda: _cli("verify-job", "list"),
|
||||
"prune": lambda: _cli("prune-job", "list")}
|
||||
else:
|
||||
with _lock:
|
||||
server = next((row for row in _load_servers() if row["id"] == identifier), None)
|
||||
if not server:
|
||||
raise PBSError("PBS server not found.")
|
||||
metadata = _remote_get(server, "/version")
|
||||
name, version = server["name"], metadata.get("version") if isinstance(metadata, dict) else None
|
||||
services = {}
|
||||
endpoints = {"datastores": "/status/datastore-usage", "tasks": "/nodes/localhost/tasks?limit=20",
|
||||
"sync": "/admin/sync", "verify": "/admin/verify", "prune": "/admin/prune"}
|
||||
operations = {key: (lambda endpoint=path: _remote_get(server, endpoint)) for key, path in endpoints.items()}
|
||||
result = {"server": identifier, "name": name, "version": version, "services": services,
|
||||
"fetched_at": int(time.time()), "errors": errors}
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
futures = {key: executor.submit(operation) for key, operation in operations.items()}
|
||||
for key, future in futures.items():
|
||||
try:
|
||||
data = future.result()
|
||||
if not isinstance(data, list) or not all(isinstance(row, dict) for row in data):
|
||||
raise PBSError("PBS returned an invalid list.")
|
||||
result[key] = [{**row, "name": row.get("store", row.get("name"))} for row in data] if key == "datastores" else data
|
||||
except PBSError as exc:
|
||||
result[key] = None # Unknown/forbidden is never presented as zero.
|
||||
errors[key] = str(exc)
|
||||
with _lock:
|
||||
if len(_status_cache) >= 33:
|
||||
_status_cache.clear()
|
||||
_status_cache[identifier] = (time.monotonic() + 15, result)
|
||||
return result
|
||||
@@ -0,0 +1,289 @@
|
||||
"""PBS platform, HTTPS protocol, authorization and isolated AppImage regressions."""
|
||||
|
||||
import ast
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import hashlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import ipaddress
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
sys.path.insert(0, str(ROOT / "AppImage/scripts"))
|
||||
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
from flask import Flask
|
||||
import jwt
|
||||
import auth_manager
|
||||
import pbs_manager as pbs
|
||||
from flask_pbs_routes import pbs_bp
|
||||
|
||||
|
||||
class PlatformTests(unittest.TestCase):
|
||||
def test_supported_platforms_and_partial_dpkg_failure(self):
|
||||
module = pbs.detect_platform.__globals__
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
release = Path(directory) / "os-release"
|
||||
for package, version, suite, product in (
|
||||
("pve-manager", "8.4.15", "bookworm", "pve"),
|
||||
("pve-manager", "9.2.1", "trixie", "pve"),
|
||||
("proxmox-backup-server", "3.4.0-1", "bookworm", "pbs"),
|
||||
("proxmox-backup-server", "4.2.5-1", "trixie", "pbs"),
|
||||
):
|
||||
release.write_text(f'VERSION_CODENAME="{suite}"\n')
|
||||
output = f"{package}\tinstall ok installed\t{version}\n"
|
||||
with self.subTest(product=product, version=version), \
|
||||
mock.patch.dict(module, {"OS_RELEASE": release}), \
|
||||
mock.patch.object(subprocess, "run", return_value=SimpleNamespace(stdout=output, returncode=1)):
|
||||
detected = pbs.detect_platform()
|
||||
self.assertEqual(detected["product"], product)
|
||||
self.assertEqual(detected["version"], version)
|
||||
self.assertTrue(detected["supported"])
|
||||
|
||||
def test_client_only_and_removed_server_are_not_pbs(self):
|
||||
for output in ("proxmox-backup-client\tinstall ok installed\t4.2.5\n",
|
||||
"proxmox-backup-server\tdeinstall ok config-files\t4.2.5\n", ""):
|
||||
with self.subTest(output=output), mock.patch.object(subprocess, "run", return_value=SimpleNamespace(stdout=output)):
|
||||
self.assertEqual(pbs.detect_platform()["product"], "unknown")
|
||||
|
||||
def test_pbs_utilities_preserve_sources_and_never_call_pve_policy(self):
|
||||
helper = ROOT / "scripts/global/repository-functions.sh"
|
||||
for supported in (True, False):
|
||||
harness = f'''
|
||||
source {shlex.quote(str(helper))}
|
||||
pmx_platform_field() {{ if [[ "$1" == product ]]; then echo pbs; else echo {str(supported).lower()}; fi; }}
|
||||
msg_info() {{ :; }}
|
||||
msg_error() {{ :; }}
|
||||
translate() {{ printf '%s' "$1"; }}
|
||||
pveversion() {{ echo 'FORBIDDEN: pveversion'; return 90; }}
|
||||
repository_policy() {{ echo 'FORBIDDEN: repository mutation'; return 90; }}
|
||||
apt-get() {{ echo 'FORBIDDEN: apt'; return 90; }}
|
||||
ensure_repositories
|
||||
'''
|
||||
result = subprocess.run(["bash"], input=harness, text=True, capture_output=True)
|
||||
self.assertNotIn("FORBIDDEN", result.stdout + result.stderr)
|
||||
self.assertEqual(result.returncode, 0 if supported else 1)
|
||||
|
||||
|
||||
class PBSManagerTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
patch = mock.patch.multiple(auth_manager, CONFIG_DIR=self.root, AUTH_CONFIG_FILE=self.root / "auth.json")
|
||||
patch.start(); self.addCleanup(patch.stop)
|
||||
auth_manager.save_auth_config({"enabled": True, "username": "admin", "password_hash": "configured",
|
||||
"jwt_secret": "s" * 48, "api_tokens": [], "revoked_tokens": []})
|
||||
auth_manager._invalidate_revoked_cache()
|
||||
pbs._status_cache.clear()
|
||||
self.server = {"name": "Backup", "url": "https://pbs.test:8007", "token_id": "audit@pbs!monitor",
|
||||
"token_secret": "fixture-token-secret", "fingerprint": "a" * 64}
|
||||
app = Flask(__name__); app.register_blueprint(pbs_bp)
|
||||
self.client = app.test_client()
|
||||
self.headers = {"Authorization": "Bearer " + auth_manager.generate_token("admin")}
|
||||
|
||||
def test_save_redacts_secret_and_probes_authenticated_endpoint(self):
|
||||
with mock.patch.object(pbs, "_remote_get", return_value=[]) as request:
|
||||
identifier = pbs.save_server(self.server)
|
||||
request.assert_called_once()
|
||||
self.assertEqual(request.call_args.args[1], "/status/datastore-usage")
|
||||
self.assertEqual((self.root / "pbs_servers.json").stat().st_mode & 0o777, 0o600)
|
||||
public = pbs.public_servers()
|
||||
self.assertEqual(public[0]["id"], identifier)
|
||||
self.assertNotIn("fixture-token-secret", json.dumps(public))
|
||||
self.assertTrue(public[0]["has_token"])
|
||||
with mock.patch.object(pbs, "_remote_get", return_value=[]):
|
||||
pbs.save_server({"id": identifier, "name": "Renamed"})
|
||||
self.assertEqual(pbs._load_servers()[0]["token_secret"], "fixture-token-secret")
|
||||
pbs.delete_server(identifier)
|
||||
self.assertEqual(pbs.public_servers(), [])
|
||||
|
||||
def test_failed_credentials_or_invalid_config_do_not_replace_saved_connection(self):
|
||||
with mock.patch.object(pbs, "_remote_get", return_value=[]):
|
||||
identifier = pbs.save_server(self.server)
|
||||
original = (self.root / "pbs_servers.json").read_bytes()
|
||||
with mock.patch.object(pbs, "_remote_get", side_effect=pbs.PBSError("Rejected token")):
|
||||
with self.assertRaises(pbs.PBSError):
|
||||
pbs.save_server({**self.server, "id": identifier, "token_secret": "wrong"})
|
||||
for changes in ({"url": "http://pbs.test"}, {"url": "https://user:pass@pbs.test"},
|
||||
{"fingerprint": "bad-pin"}, {"token_id": "not-a-token"},
|
||||
{"token_secret": "line\nbreak"}):
|
||||
with self.subTest(changes=changes), self.assertRaises(pbs.PBSError):
|
||||
pbs.save_server({**self.server, **changes, "id": identifier})
|
||||
self.assertEqual((self.root / "pbs_servers.json").read_bytes(), original)
|
||||
|
||||
def test_remote_partial_failure_is_unknown_instead_of_zero(self):
|
||||
def remote(server, endpoint):
|
||||
if endpoint == "/version": return {"version": "4.2.5"}
|
||||
if endpoint == "/status/datastore-usage": return [{"store": "backup", "total": 100, "used": 30, "avail": 70}]
|
||||
if endpoint == "/admin/verify": raise pbs.PBSError("Permission denied")
|
||||
return []
|
||||
with mock.patch.object(pbs, "_remote_get", side_effect=remote) as request:
|
||||
identifier = pbs.save_server(self.server)
|
||||
result = pbs.get_status(identifier)
|
||||
count = request.call_count
|
||||
self.assertEqual(pbs.get_status(identifier), result)
|
||||
self.assertEqual(request.call_count, count)
|
||||
self.assertEqual(result["datastores"][0]["name"], "backup")
|
||||
self.assertIsNone(result["verify"])
|
||||
self.assertIn("verify", result["errors"])
|
||||
|
||||
def test_unmounted_removable_datastore_has_no_fabricated_capacity(self):
|
||||
with mock.patch.object(pbs, "_cli", return_value=[{"name": "removable", "path": "/not-mounted", "backing-device": "uuid"}]), \
|
||||
mock.patch.object(pbs.os.path, "ismount", return_value=False), \
|
||||
mock.patch.object(pbs.shutil, "disk_usage") as usage:
|
||||
result = pbs._local_datastores()
|
||||
usage.assert_not_called()
|
||||
self.assertIsNone(result[0]["total"])
|
||||
self.assertIn("error", result[0])
|
||||
|
||||
def test_local_status_reads_pbs_commands_and_filesystem_capacity(self):
|
||||
def run(command, **kwargs):
|
||||
self.assertNotIn("shell", kwargs)
|
||||
self.assertEqual(command[0], "proxmox-backup-manager")
|
||||
self.assertEqual(command[-2:], ["--output-format", "json"])
|
||||
data = [{"name": "backup", "path": str(self.root)}] if command[1:3] == ["datastore", "list"] else []
|
||||
return SimpleNamespace(stdout=json.dumps(data), returncode=0)
|
||||
with mock.patch.object(pbs, "get_platform", return_value={"product": "pbs", "version": "4.2.5"}), \
|
||||
mock.patch.object(pbs, "_local_services", return_value={key: "active" for key in pbs.PBS_SERVICES}), \
|
||||
mock.patch.object(pbs.subprocess, "run", side_effect=run), \
|
||||
mock.patch.object(pbs.shutil, "disk_usage", return_value=SimpleNamespace(total=100, used=30, free=70)):
|
||||
result = pbs.get_status("local")
|
||||
self.assertEqual(result["version"], "4.2.5")
|
||||
self.assertEqual(result["errors"], {})
|
||||
self.assertEqual(result["datastores"][0]["used"], 30)
|
||||
self.assertEqual(set(result["services"]), set(pbs.PBS_SERVICES))
|
||||
|
||||
def test_connection_api_requires_full_admin_and_does_not_return_secret(self):
|
||||
self.assertEqual(self.client.get("/api/pbs/servers").status_code, 401)
|
||||
with mock.patch.object(pbs, "_remote_get", return_value=[]):
|
||||
response = self.client.post("/api/pbs/servers", headers=self.headers, json=self.server)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertNotIn("fixture-token-secret", response.text)
|
||||
payload = jwt.decode(self.headers["Authorization"].split()[1], "s" * 48,
|
||||
algorithms=["HS256"], audience=auth_manager.JWT_AUDIENCE)
|
||||
payload["scope"] = "read_only"
|
||||
read_only = {"Authorization": "Bearer " + jwt.encode(payload, "s" * 48, algorithm="HS256")}
|
||||
self.assertEqual(self.client.post("/api/pbs/servers", headers=read_only, json=self.server).status_code, 403)
|
||||
self.assertEqual(self.client.get("/api/pbs/servers", headers=read_only).status_code, 200)
|
||||
auth_manager.save_auth_config({"enabled": False})
|
||||
self.assertEqual(self.client.post("/api/pbs/servers", json=self.server).status_code, 409)
|
||||
|
||||
def test_health_uses_pbs_services_and_skips_pve_only_checks(self):
|
||||
source = ast.parse((ROOT / "AppImage/scripts/health_monitor.py").read_text())
|
||||
names = ("_check_pve_services", "_check_vms_cts_optimized", "_check_vms_cts_with_persistence",
|
||||
"_check_proxmox_storages", "_check_proxmox_storage", "_check_lxc_disk_usage",
|
||||
"_check_vm_disk_usage", "_check_lxc_mount_capacity", "_check_pve_storage_capacity")
|
||||
functions = [node for node in ast.walk(source) if isinstance(node, ast.FunctionDef) and node.name in names]
|
||||
persistence = mock.Mock(); persistence.is_error_active.return_value = False
|
||||
run = mock.Mock(return_value=SimpleNamespace(returncode=0, stdout="active\n"))
|
||||
namespace = {"Dict": dict, "Any": object, "Optional": __import__('typing').Optional,
|
||||
"PBS_SERVICES": pbs.PBS_SERVICES, "health_persistence": persistence,
|
||||
"subprocess": SimpleNamespace(run=run), "os": __import__('os')}
|
||||
exec(compile(ast.Module(body=functions, type_ignores=[]), '<health-pbs-test>', 'exec'), namespace)
|
||||
instance = SimpleNamespace(platform_type="pbs", PVE_SERVICES=["pveproxy", "pvedaemon", "pvestatd", "pve-cluster"])
|
||||
services = namespace["_check_pve_services"](instance)
|
||||
self.assertEqual(services["status"], "OK")
|
||||
self.assertEqual([call.args[0][2] for call in run.call_args_list], list(pbs.PBS_SERVICES))
|
||||
for name in names[1:]:
|
||||
result = namespace[name](instance)
|
||||
self.assertTrue(result is None or result.get("not_applicable"))
|
||||
|
||||
def test_appimage_copies_and_imports_pbs_dependencies_outside_source_tree(self):
|
||||
builder = (ROOT / "AppImage/scripts/build_appimage.sh").read_text()
|
||||
modules = ("pbs_manager", "flask_pbs_routes", "auth_manager", "jwt_middleware")
|
||||
copies = []
|
||||
for name in modules:
|
||||
matches = [line for line in builder.splitlines() if line.startswith(f'cp "$SCRIPT_DIR/{name}.py"')]
|
||||
self.assertEqual(len(matches), 1, name)
|
||||
copies.extend(matches)
|
||||
copies.extend(line for line in builder.splitlines() if line.startswith('cp "$APPIMAGE_ROOT/../scripts/global/proxmox_platform.py"'))
|
||||
self.assertEqual(len(copies), len(modules) + 1)
|
||||
app = self.root / 'AppDir'; (app / 'usr/bin').mkdir(parents=True)
|
||||
q = shlex.quote
|
||||
shell = f'APP_DIR={q(str(app))}\nSCRIPT_DIR={q(str(ROOT / "AppImage/scripts"))}\nAPPIMAGE_ROOT={q(str(ROOT / "AppImage"))}\n' + '\n'.join(copies)
|
||||
subprocess.run(['bash', '-e', '-c', shell], check=True)
|
||||
code = f'import sys; sys.path.insert(0, {str(app / "usr/bin")!r}); import pbs_manager, flask_pbs_routes, proxmox_platform'
|
||||
subprocess.run([sys.executable, '-I', '-B', '-c', code], cwd=self.root, check=True, capture_output=True)
|
||||
|
||||
|
||||
class TLSTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.temporary = tempfile.TemporaryDirectory()
|
||||
root = Path(cls.temporary.name)
|
||||
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
|
||||
certificate = (x509.CertificateBuilder().subject_name(subject).issuer_name(subject).public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number()).not_valid_before(datetime.now(timezone.utc) - timedelta(minutes=1))
|
||||
.not_valid_after(datetime.now(timezone.utc) + timedelta(hours=1))
|
||||
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
|
||||
.sign(key, hashes.SHA256()))
|
||||
cert_path, key_path = root / 'server.pem', root / 'server.key'
|
||||
cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()))
|
||||
cls.fingerprint = hashlib.sha256(certificate.public_bytes(serialization.Encoding.DER)).hexdigest()
|
||||
cls.requests = []
|
||||
cls.response_status = 200
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def handle(self):
|
||||
try:
|
||||
super().handle()
|
||||
except ConnectionResetError:
|
||||
pass # A rejected TLS pin closes before any HTTP request.
|
||||
|
||||
def do_GET(self):
|
||||
cls.requests.append((self.path, self.headers.get("Authorization")))
|
||||
body = json.dumps({"data": []}).encode()
|
||||
self.send_response(cls.response_status)
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.end_headers(); self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args): pass
|
||||
|
||||
cls.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER); context.load_cert_chain(cert_path, key_path)
|
||||
cls.server.socket = context.wrap_socket(cls.server.socket, server_side=True)
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True); cls.thread.start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown(); cls.server.server_close(); cls.thread.join(timeout=5); cls.temporary.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
self.requests.clear(); type(self).response_status = 200
|
||||
self.connection = {"url": f'https://127.0.0.1:{self.server.server_port}',
|
||||
"fingerprint": self.fingerprint, "token_id": "audit@pbs!monitor", "token_secret": "fake-secret"}
|
||||
|
||||
def test_real_tls_matching_pin_sends_correct_pbs_authorization(self):
|
||||
self.assertEqual(pbs._remote_get(self.connection, '/status/datastore-usage'), [])
|
||||
self.assertEqual(self.requests, [('/api2/json/status/datastore-usage', 'PBSAPIToken=audit@pbs!monitor:fake-secret')])
|
||||
|
||||
def test_bad_pin_and_untrusted_ca_send_no_http_credentials(self):
|
||||
for fingerprint in ('0' * 64, ''):
|
||||
with self.subTest(fingerprint=fingerprint), self.assertRaises(pbs.PBSError):
|
||||
pbs._remote_get({**self.connection, "fingerprint": fingerprint}, '/status/datastore-usage')
|
||||
self.assertEqual(self.requests, [])
|
||||
|
||||
def test_bad_credentials_and_redirects_are_rejected(self):
|
||||
for status in (401, 403, 302):
|
||||
type(self).response_status = status
|
||||
with self.subTest(status=status), self.assertRaises(pbs.PBSError):
|
||||
pbs._remote_get(self.connection, '/status/datastore-usage')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user