mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 19:02:31 +00:00
fix(monitor): write Proxmox's notification settings only while /etc/pve is mounted
- Nothing is written under /etc/pve unless it is the mounted cluster filesystem; a Monitor that starts while pve-cluster is being restarted waits for it and registers its webhook afterwards. - The Monitor service is ordered after pve-cluster.service. - The settings are written only when they have to change. The copies taken before a change are kept in /usr/local/share/proxmenux/backups/pve-notifications, and the ones left in /etc/pve by earlier versions are moved there. - Uninstalling the Monitor removes its target and matcher from Proxmox. Reported in #427.
This commit is contained in:
@@ -895,6 +895,20 @@ def _pve_webhook_url() -> str:
|
|||||||
_PVE_WEBHOOK_URL = _pve_webhook_url()
|
_PVE_WEBHOOK_URL = _pve_webhook_url()
|
||||||
|
|
||||||
|
|
||||||
|
_PVE_NOT_MOUNTED = 'The Proxmox cluster filesystem (/etc/pve) is not mounted'
|
||||||
|
|
||||||
|
|
||||||
|
def _pve_cluster_fs_mounted() -> bool:
|
||||||
|
"""Whether /etc/pve is the Proxmox cluster filesystem.
|
||||||
|
|
||||||
|
While its service is stopped or restarting, /etc/pve is an empty
|
||||||
|
directory of the root disk. A file written there is not a configuration
|
||||||
|
file: it keeps the service from mounting the filesystem again.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
return os.path.ismount(os.path.dirname(_PVE_NOTIFICATIONS_CFG))
|
||||||
|
|
||||||
|
|
||||||
def _pve_read_file(path):
|
def _pve_read_file(path):
|
||||||
"""Read file, return (content, error). Content is '' if missing."""
|
"""Read file, return (content, error). Content is '' if missing."""
|
||||||
try:
|
try:
|
||||||
@@ -909,26 +923,47 @@ def _pve_read_file(path):
|
|||||||
|
|
||||||
|
|
||||||
_PVE_BACKUPS_KEPT = 3
|
_PVE_BACKUPS_KEPT = 3
|
||||||
|
# Copies of Proxmox's notification settings, taken before changing them. They
|
||||||
|
# are kept with ProxMenux, not next to the files: /etc/pve belongs to Proxmox
|
||||||
|
# and is replicated to every node of a cluster.
|
||||||
|
_PVE_BACKUP_DIR = '/usr/local/share/proxmenux/backups/pve-notifications'
|
||||||
|
_PVE_BACKUP_MARK = '.proxmenux_backup_'
|
||||||
|
|
||||||
|
|
||||||
|
def _pve_backup_prefix(path):
|
||||||
|
"""Name the copies of a file are kept under. The private file carries
|
||||||
|
its own name, as both are called notifications.cfg."""
|
||||||
|
import os
|
||||||
|
name = os.path.basename(path)
|
||||||
|
return ('priv-' if path == _PVE_PRIV_CFG else '') + name + _PVE_BACKUP_MARK
|
||||||
|
|
||||||
|
|
||||||
|
def _pve_backups(path):
|
||||||
|
"""The copies kept of a file, newest first."""
|
||||||
|
import os
|
||||||
|
prefix = _pve_backup_prefix(path)
|
||||||
|
try:
|
||||||
|
names = os.listdir(_PVE_BACKUP_DIR)
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
found = [os.path.join(_PVE_BACKUP_DIR, name) for name in names if name.startswith(prefix)]
|
||||||
|
return sorted(found, key=os.path.getmtime, reverse=True)
|
||||||
|
|
||||||
|
|
||||||
def _pve_backup_file(path):
|
def _pve_backup_file(path):
|
||||||
"""Create timestamped backup if file exists. Never fails fatally.
|
"""Keep a timestamped copy of the file if it exists. Never fails fatally.
|
||||||
|
|
||||||
The backups live next to the file, inside the cluster filesystem, whose
|
A copy identical to the newest one is not written again and only the
|
||||||
size is limited: a copy identical to the newest one is not written again
|
newest few are kept. Restore uses the newest.
|
||||||
and only the newest few are kept. Restore uses the newest.
|
|
||||||
"""
|
"""
|
||||||
import os, shutil
|
import os, shutil
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
try:
|
try:
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
return
|
return
|
||||||
directory, name = os.path.split(path)
|
os.makedirs(_PVE_BACKUP_DIR, mode=0o700, exist_ok=True)
|
||||||
prefix = f"{name}.proxmenux_backup_"
|
os.chmod(_PVE_BACKUP_DIR, 0o700)
|
||||||
existing = sorted(
|
existing = _pve_backups(path)
|
||||||
(os.path.join(directory, f) for f in os.listdir(directory) if f.startswith(prefix)),
|
|
||||||
key=os.path.getmtime, reverse=True,
|
|
||||||
)
|
|
||||||
with open(path, 'rb') as f:
|
with open(path, 'rb') as f:
|
||||||
current = f.read()
|
current = f.read()
|
||||||
newest_matches = False
|
newest_matches = False
|
||||||
@@ -940,16 +975,48 @@ def _pve_backup_file(path):
|
|||||||
pass
|
pass
|
||||||
if not newest_matches:
|
if not newest_matches:
|
||||||
ts = datetime.now().strftime('%Y%m%d_%H%M%S')
|
ts = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||||
shutil.copy2(path, os.path.join(directory, prefix + ts))
|
destination = os.path.join(_PVE_BACKUP_DIR, _pve_backup_prefix(path) + ts)
|
||||||
existing = sorted(
|
shutil.copy2(path, destination)
|
||||||
(os.path.join(directory, f) for f in os.listdir(directory) if f.startswith(prefix)),
|
# The private file holds the secrets of every notification target.
|
||||||
key=os.path.getmtime, reverse=True,
|
os.chmod(destination, 0o600)
|
||||||
)
|
_pve_prune_backups(path)
|
||||||
for stale in existing[_PVE_BACKUPS_KEPT:]:
|
except Exception:
|
||||||
try:
|
pass
|
||||||
os.remove(stale)
|
|
||||||
except OSError:
|
|
||||||
pass
|
def _pve_prune_backups(path):
|
||||||
|
"""Keep only the newest copies of a file."""
|
||||||
|
import os
|
||||||
|
for stale in _pve_backups(path)[_PVE_BACKUPS_KEPT:]:
|
||||||
|
try:
|
||||||
|
os.remove(stale)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _pve_collect_legacy_backups(path):
|
||||||
|
"""Earlier versions left the copies next to the file, inside /etc/pve.
|
||||||
|
The newest ones are moved with the rest and none is left there."""
|
||||||
|
import os, shutil
|
||||||
|
try:
|
||||||
|
directory, name = os.path.split(path)
|
||||||
|
legacy = sorted(
|
||||||
|
(os.path.join(directory, f) for f in os.listdir(directory) if f.startswith(name + _PVE_BACKUP_MARK)),
|
||||||
|
key=os.path.getmtime, reverse=True,
|
||||||
|
)
|
||||||
|
if not legacy:
|
||||||
|
return
|
||||||
|
os.makedirs(_PVE_BACKUP_DIR, mode=0o700, exist_ok=True)
|
||||||
|
os.chmod(_PVE_BACKUP_DIR, 0o700)
|
||||||
|
for position, source in enumerate(legacy):
|
||||||
|
if position < _PVE_BACKUPS_KEPT:
|
||||||
|
stamp = os.path.basename(source)[len(name + _PVE_BACKUP_MARK):]
|
||||||
|
destination = os.path.join(_PVE_BACKUP_DIR, _pve_backup_prefix(path) + stamp)
|
||||||
|
if not os.path.exists(destination):
|
||||||
|
shutil.copy2(source, destination)
|
||||||
|
os.chmod(destination, 0o600)
|
||||||
|
os.remove(source)
|
||||||
|
_pve_prune_backups(path)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -1091,6 +1158,11 @@ def setup_pve_webhook_core() -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
if not _pve_cluster_fs_mounted():
|
||||||
|
result['error'] = _PVE_NOT_MOUNTED
|
||||||
|
result['waiting'] = True
|
||||||
|
return result
|
||||||
|
|
||||||
# ── Step 1: Ensure webhook secret exists (for our own internal use) ──
|
# ── Step 1: Ensure webhook secret exists (for our own internal use) ──
|
||||||
secret = notification_manager.get_webhook_secret()
|
secret = notification_manager.get_webhook_secret()
|
||||||
if not secret:
|
if not secret:
|
||||||
@@ -1109,11 +1181,10 @@ def setup_pve_webhook_core() -> dict:
|
|||||||
if err:
|
if err:
|
||||||
priv_text = None
|
priv_text = None
|
||||||
|
|
||||||
# ── Step 4: Create backups before ANY modification ──
|
# ── Step 4: Backups are taken below, only of a file about to change ──
|
||||||
_pve_backup_file(_PVE_NOTIFICATIONS_CFG)
|
_pve_collect_legacy_backups(_PVE_NOTIFICATIONS_CFG)
|
||||||
if priv_text is not None:
|
_pve_collect_legacy_backups(_PVE_PRIV_CFG)
|
||||||
_pve_backup_file(_PVE_PRIV_CFG)
|
|
||||||
|
|
||||||
# ── Step 5: Remove any previous proxmenux blocks from BOTH files ──
|
# ── Step 5: Remove any previous proxmenux blocks from BOTH files ──
|
||||||
cleaned_cfg = _pve_remove_our_blocks(cfg_text, _PVE_OUR_HEADERS)
|
cleaned_cfg = _pve_remove_our_blocks(cfg_text, _PVE_OUR_HEADERS)
|
||||||
|
|
||||||
@@ -1161,10 +1232,12 @@ def setup_pve_webhook_core() -> dict:
|
|||||||
|
|
||||||
new_cfg = cleaned_cfg + endpoint_block + '\n' + matcher_block
|
new_cfg = cleaned_cfg + endpoint_block + '\n' + matcher_block
|
||||||
|
|
||||||
# ── Step 8: Write main config ──
|
# ── Step 8: Write main config, when it is not already as wanted ──
|
||||||
try:
|
try:
|
||||||
with open(_PVE_NOTIFICATIONS_CFG, 'w') as f:
|
if new_cfg != cfg_text:
|
||||||
f.write(new_cfg)
|
_pve_backup_file(_PVE_NOTIFICATIONS_CFG)
|
||||||
|
with open(_PVE_NOTIFICATIONS_CFG, 'w') as f:
|
||||||
|
f.write(new_cfg)
|
||||||
except PermissionError:
|
except PermissionError:
|
||||||
result['error'] = f'Permission denied writing {_PVE_NOTIFICATIONS_CFG}'
|
result['error'] = f'Permission denied writing {_PVE_NOTIFICATIONS_CFG}'
|
||||||
result['fallback_commands'] = _build_webhook_fallback()
|
result['fallback_commands'] = _build_webhook_fallback()
|
||||||
@@ -1214,8 +1287,11 @@ def setup_pve_webhook_core() -> dict:
|
|||||||
new_priv = priv_block
|
new_priv = priv_block
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(_PVE_PRIV_CFG, 'w') as f:
|
if new_priv != priv_text:
|
||||||
f.write(new_priv)
|
if priv_text is not None:
|
||||||
|
_pve_backup_file(_PVE_PRIV_CFG)
|
||||||
|
with open(_PVE_PRIV_CFG, 'w') as f:
|
||||||
|
f.write(new_priv)
|
||||||
except PermissionError:
|
except PermissionError:
|
||||||
result['error'] = f'Permission denied writing {_PVE_PRIV_CFG}'
|
result['error'] = f'Permission denied writing {_PVE_PRIV_CFG}'
|
||||||
result['fallback_commands'] = _build_webhook_fallback()
|
result['fallback_commands'] = _build_webhook_fallback()
|
||||||
@@ -1238,6 +1314,17 @@ def setup_pve_webhook_core() -> dict:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def setup_pve_webhook_when_mounted(timeout=900, interval=5) -> dict:
|
||||||
|
"""Configure the webhook once /etc/pve is mounted. For the Monitor that
|
||||||
|
starts while the cluster service is still starting or being restarted."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while not _pve_cluster_fs_mounted():
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
return {'configured': False, 'error': _PVE_NOT_MOUNTED, 'waiting': True}
|
||||||
|
time.sleep(interval)
|
||||||
|
return setup_pve_webhook_core()
|
||||||
|
|
||||||
|
|
||||||
@notification_bp.route('/api/notifications/proxmox/setup-webhook', methods=['POST'])
|
@notification_bp.route('/api/notifications/proxmox/setup-webhook', methods=['POST'])
|
||||||
@require_auth
|
@require_auth
|
||||||
def setup_proxmox_webhook():
|
def setup_proxmox_webhook():
|
||||||
@@ -1254,6 +1341,10 @@ def cleanup_pve_webhook_core() -> dict:
|
|||||||
result = {'cleaned': False, 'error': None}
|
result = {'cleaned': False, 'error': None}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
if not _pve_cluster_fs_mounted():
|
||||||
|
result['error'] = _PVE_NOT_MOUNTED
|
||||||
|
return result
|
||||||
|
|
||||||
# Read both files
|
# Read both files
|
||||||
cfg_text, err = _pve_read_file(_PVE_NOTIFICATIONS_CFG)
|
cfg_text, err = _pve_read_file(_PVE_NOTIFICATIONS_CFG)
|
||||||
if err:
|
if err:
|
||||||
@@ -1336,10 +1427,6 @@ def read_pve_notification_cfg():
|
|||||||
'priv_cfg': '/etc/pve/priv/notifications.cfg',
|
'priv_cfg': '/etc/pve/priv/notifications.cfg',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Also look for any backups we created
|
|
||||||
backup_dir = '/etc/pve'
|
|
||||||
priv_backup_dir = '/etc/pve/priv'
|
|
||||||
|
|
||||||
result = {}
|
result = {}
|
||||||
for key, path in files.items():
|
for key, path in files.items():
|
||||||
try:
|
try:
|
||||||
@@ -1357,24 +1444,18 @@ def read_pve_notification_cfg():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
result[key] = {'path': path, 'content': None, 'size': 0, 'error': str(e)}
|
result[key] = {'path': path, 'content': None, 'size': 0, 'error': str(e)}
|
||||||
|
|
||||||
# Find backups
|
# The copies kept of both files
|
||||||
backups = []
|
backups = []
|
||||||
for d in [backup_dir, priv_backup_dir]:
|
for fpath in _pve_backups(_PVE_NOTIFICATIONS_CFG) + _pve_backups(_PVE_PRIV_CFG):
|
||||||
try:
|
try:
|
||||||
for fname in sorted(os.listdir(d)):
|
with open(fpath, 'r') as f:
|
||||||
if 'proxmenux_backup' in fname:
|
backups.append({
|
||||||
fpath = os.path.join(d, fname)
|
'path': fpath,
|
||||||
try:
|
'content': f.read(),
|
||||||
with open(fpath, 'r') as f:
|
'size': os.path.getsize(fpath),
|
||||||
backups.append({
|
})
|
||||||
'path': fpath,
|
|
||||||
'content': f.read(),
|
|
||||||
'size': os.path.getsize(fpath),
|
|
||||||
})
|
|
||||||
except Exception:
|
|
||||||
backups.append({'path': fpath, 'content': None, 'error': 'read_failed'})
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
backups.append({'path': fpath, 'content': None, 'error': 'read_failed'})
|
||||||
|
|
||||||
result['backups'] = backups
|
result['backups'] = backups
|
||||||
return jsonify(result), 200
|
return jsonify(result), 200
|
||||||
@@ -1391,35 +1472,26 @@ def restore_pve_notification_cfg():
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
files_to_restore = {
|
|
||||||
'/etc/pve': '/etc/pve/notifications.cfg',
|
|
||||||
'/etc/pve/priv': '/etc/pve/priv/notifications.cfg',
|
|
||||||
}
|
|
||||||
|
|
||||||
restored = []
|
restored = []
|
||||||
errors = []
|
errors = []
|
||||||
|
|
||||||
for search_dir, target_path in files_to_restore.items():
|
|
||||||
try:
|
|
||||||
# Pick the most recent backup by mtime, not lexicographic name.
|
|
||||||
# An attacker (or accidental rename) with a write primitive
|
|
||||||
# could craft `notifications.cfg.proxmenux_backup_99999999_999999`
|
|
||||||
# and have it sort first, hijacking the restore. mtime tracks
|
|
||||||
# the actual file age so renamed/touched files don't fool us.
|
|
||||||
# Audit Tier 3.1 — restore-cfg sort lexicográfico.
|
|
||||||
candidates = [
|
|
||||||
f for f in os.listdir(search_dir)
|
|
||||||
if 'proxmenux_backup' in f and f.startswith('notifications.cfg')
|
|
||||||
]
|
|
||||||
|
|
||||||
|
if not _pve_cluster_fs_mounted():
|
||||||
|
return jsonify({'restored': [], 'errors': [{'target': '/etc/pve', 'error': _PVE_NOT_MOUNTED}],
|
||||||
|
'success': False}), 200
|
||||||
|
|
||||||
|
for target_path in (_PVE_NOTIFICATIONS_CFG, _PVE_PRIV_CFG):
|
||||||
|
try:
|
||||||
|
_pve_collect_legacy_backups(target_path)
|
||||||
|
# The newest by modification time: a name can be crafted to sort
|
||||||
|
# first, a file's age cannot.
|
||||||
|
candidates = _pve_backups(target_path)
|
||||||
if candidates:
|
if candidates:
|
||||||
candidates.sort(
|
# Content only: the cluster filesystem sets its own permissions.
|
||||||
key=lambda f: os.path.getmtime(os.path.join(search_dir, f)),
|
with open(candidates[0], 'rb') as source:
|
||||||
reverse=True,
|
content = source.read()
|
||||||
)
|
with open(target_path, 'wb') as target:
|
||||||
backup_path = os.path.join(search_dir, candidates[0])
|
target.write(content)
|
||||||
shutil.copy2(backup_path, target_path)
|
restored.append({'target': target_path, 'from_backup': candidates[0]})
|
||||||
restored.append({'target': target_path, 'from_backup': backup_path})
|
|
||||||
else:
|
else:
|
||||||
errors.append({'target': target_path, 'error': 'no_backup_found'})
|
errors.append({'target': target_path, 'error': 'no_backup_found'})
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -1141,6 +1141,11 @@ class NotificationManager:
|
|||||||
wh_result = setup_pve_webhook_core()
|
wh_result = setup_pve_webhook_core()
|
||||||
if wh_result.get('configured'):
|
if wh_result.get('configured'):
|
||||||
print("[NotificationManager] PVE webhook configured OK.")
|
print("[NotificationManager] PVE webhook configured OK.")
|
||||||
|
elif wh_result.get('waiting'):
|
||||||
|
# /etc/pve is not mounted yet: nothing is written until it is.
|
||||||
|
print("[NotificationManager] PVE webhook: waiting for /etc/pve to be mounted.")
|
||||||
|
threading.Thread(target=self._configure_pve_webhook_when_mounted, daemon=True,
|
||||||
|
name='pve-webhook-setup').start()
|
||||||
elif wh_result.get('error'):
|
elif wh_result.get('error'):
|
||||||
print(f"[NotificationManager] PVE webhook warning: {wh_result['error']}")
|
print(f"[NotificationManager] PVE webhook warning: {wh_result['error']}")
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@@ -2285,6 +2290,18 @@ class NotificationManager:
|
|||||||
'auth_fail',
|
'auth_fail',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _configure_pve_webhook_when_mounted(self):
|
||||||
|
"""Configure the PVE webhook once the cluster filesystem is back."""
|
||||||
|
try:
|
||||||
|
from flask_notification_routes import setup_pve_webhook_when_mounted
|
||||||
|
result = setup_pve_webhook_when_mounted()
|
||||||
|
if result.get('configured'):
|
||||||
|
print("[NotificationManager] PVE webhook configured OK.")
|
||||||
|
elif result.get('error'):
|
||||||
|
print(f"[NotificationManager] PVE webhook warning: {result['error']}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[NotificationManager] PVE webhook setup error: {e}")
|
||||||
|
|
||||||
def _reset_cooldowns_on_start(self):
|
def _reset_cooldowns_on_start(self):
|
||||||
"""Clear DB rows in notification_last_sent for the curated set of
|
"""Clear DB rows in notification_last_sent for the curated set of
|
||||||
event types listed in `_EVENT_TYPES_RESET_ON_START`.
|
event types listed in `_EVENT_TYPES_RESET_ON_START`.
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Proxmox's notification files are only written while /etc/pve is the
|
||||||
|
mounted cluster filesystem, and only when their content has to change."""
|
||||||
|
import ast
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
SCRIPTS = Path(__file__).resolve().parents[1]
|
||||||
|
SOURCE = SCRIPTS / 'flask_notification_routes.py'
|
||||||
|
NAMES = {'_pve_cluster_fs_mounted', '_pve_read_file', '_pve_backup_file', '_pve_prune_backups', '_pve_backup_prefix',
|
||||||
|
'_pve_backups', '_pve_collect_legacy_backups', 'restore_pve_notification_cfg', '_pve_remove_our_blocks', '_pve_webhook_url',
|
||||||
|
'_build_webhook_fallback', 'setup_pve_webhook_core', 'cleanup_pve_webhook_core', 'setup_pve_webhook_when_mounted'}
|
||||||
|
ASSIGNED = {'_PVE_ENDPOINT_ID', '_PVE_MATCHER_ID', '_PVE_OUR_HEADERS', '_PVE_NOT_MOUNTED', '_PVE_BACKUPS_KEPT',
|
||||||
|
'_PVE_HEADER_RE', 'WEBHOOK_LOOPBACK_PORT', '_PVE_BACKUP_MARK'}
|
||||||
|
|
||||||
|
|
||||||
|
class FakeManager:
|
||||||
|
def __init__(self):
|
||||||
|
self.secret = 'c2VjcmV0LXRva2Vu'
|
||||||
|
|
||||||
|
def get_webhook_secret(self):
|
||||||
|
return self.secret
|
||||||
|
|
||||||
|
def _save_setting(self, _key, value):
|
||||||
|
self.secret = value
|
||||||
|
|
||||||
|
|
||||||
|
def load(directory):
|
||||||
|
"""The functions under test, pointed at a directory standing for /etc/pve."""
|
||||||
|
body = []
|
||||||
|
for node in ast.parse(SOURCE.read_text()).body:
|
||||||
|
if isinstance(node, ast.FunctionDef) and node.name in NAMES:
|
||||||
|
node.decorator_list = []
|
||||||
|
body.append(node)
|
||||||
|
elif isinstance(node, ast.Assign) and any(getattr(t, 'id', None) in ASSIGNED for t in node.targets):
|
||||||
|
body.append(node)
|
||||||
|
elif isinstance(node, ast.Import) and any(a.asname == '_re_pve_cfg' for a in node.names):
|
||||||
|
body.append(node)
|
||||||
|
import time
|
||||||
|
scope = {'notification_manager': FakeManager(), 'time': time, 'jsonify': lambda value: value,
|
||||||
|
'_PVE_NOTIFICATIONS_CFG': str(directory / 'notifications.cfg'),
|
||||||
|
'_PVE_PRIV_CFG': str(directory / 'priv' / 'notifications.cfg'),
|
||||||
|
'_PVE_BACKUP_DIR': str(directory.parent / 'proxmenux' / 'backups' / 'pve-notifications')}
|
||||||
|
exec(compile(ast.fix_missing_locations(ast.Module(body=body, type_ignores=[])), str(SOURCE), 'exec'), scope)
|
||||||
|
return scope
|
||||||
|
|
||||||
|
|
||||||
|
class PveNotificationWriteTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
self.pve = Path(tmp.name) / 'pve'
|
||||||
|
self.pve.mkdir()
|
||||||
|
self.scope = load(self.pve)
|
||||||
|
self.kept = Path(self.scope['_PVE_BACKUP_DIR'])
|
||||||
|
self.mounted = True
|
||||||
|
patcher = mock.patch('os.path.ismount', side_effect=lambda path: self.mounted and path == str(self.pve))
|
||||||
|
patcher.start()
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
def files(self):
|
||||||
|
return sorted(str(p.relative_to(self.pve)) for p in self.pve.rglob('*') if p.is_file())
|
||||||
|
|
||||||
|
def test_nothing_is_written_while_the_cluster_filesystem_is_down(self):
|
||||||
|
# The bare directory of the root disk: empty, and it has to stay so.
|
||||||
|
self.mounted = False
|
||||||
|
result = self.scope['setup_pve_webhook_core']()
|
||||||
|
self.assertFalse(result['configured'])
|
||||||
|
self.assertTrue(result['waiting'])
|
||||||
|
self.assertEqual(self.scope['cleanup_pve_webhook_core']()['error'], self.scope['_PVE_NOT_MOUNTED'])
|
||||||
|
self.assertEqual(list(self.pve.iterdir()), [])
|
||||||
|
|
||||||
|
def test_the_webhook_is_written_once_and_not_again(self):
|
||||||
|
(self.pve / 'priv').mkdir()
|
||||||
|
existing = 'sendmail: mail-to-root\n\tcomment Send mails to root@pam\n\tmailto-user root@pam\n'
|
||||||
|
(self.pve / 'notifications.cfg').write_text(existing)
|
||||||
|
(self.pve / 'priv' / 'notifications.cfg').write_text('')
|
||||||
|
self.assertTrue(self.scope['setup_pve_webhook_core']()['configured'])
|
||||||
|
written = (self.pve / 'notifications.cfg').read_text()
|
||||||
|
self.assertTrue(written.startswith(existing))
|
||||||
|
self.assertIn('webhook: proxmenux-webhook\n', written)
|
||||||
|
self.assertIn('matcher: proxmenux-default\n', written)
|
||||||
|
first = {name: (self.pve / name).stat().st_mtime_ns for name in self.files()}
|
||||||
|
# The copies taken before the change are kept with ProxMenux, not in /etc/pve.
|
||||||
|
self.assertEqual(sorted(first), ['notifications.cfg', 'priv/notifications.cfg'])
|
||||||
|
copies = sorted(p.name.split('.proxmenux_backup_')[0] for p in self.kept.iterdir())
|
||||||
|
self.assertEqual(copies, ['notifications.cfg', 'priv-notifications.cfg'])
|
||||||
|
self.assertEqual({oct(p.stat().st_mode & 0o777) for p in self.kept.iterdir()}, {'0o600'})
|
||||||
|
self.assertEqual(oct(self.kept.stat().st_mode & 0o777), '0o700')
|
||||||
|
# A second start finds both files as wanted: no write, no new backup.
|
||||||
|
os.utime(self.pve / 'notifications.cfg', ns=(1, 1))
|
||||||
|
os.utime(self.pve / 'priv' / 'notifications.cfg', ns=(1, 1))
|
||||||
|
self.assertTrue(self.scope['setup_pve_webhook_core']()['configured'])
|
||||||
|
self.assertEqual(self.files(), sorted(first))
|
||||||
|
self.assertEqual((self.pve / 'notifications.cfg').stat().st_mtime_ns, 1)
|
||||||
|
self.assertEqual((self.pve / 'priv' / 'notifications.cfg').stat().st_mtime_ns, 1)
|
||||||
|
|
||||||
|
def test_copies_left_in_etc_pve_by_an_earlier_version_are_collected_without_a_write(self):
|
||||||
|
(self.pve / 'priv').mkdir()
|
||||||
|
(self.pve / 'priv' / 'notifications.cfg').write_text('')
|
||||||
|
(self.pve / 'notifications.cfg').write_text('')
|
||||||
|
self.scope['setup_pve_webhook_core']()
|
||||||
|
for item in self.kept.iterdir():
|
||||||
|
item.unlink()
|
||||||
|
for index in range(40):
|
||||||
|
for folder in (self.pve, self.pve / 'priv'):
|
||||||
|
stale = folder / f'notifications.cfg.proxmenux_backup_20260101_{index:06d}'
|
||||||
|
stale.write_text(f'old {index}')
|
||||||
|
os.utime(stale, ns=(index * 10 ** 9, index * 10 ** 9))
|
||||||
|
current = (self.pve / 'notifications.cfg').read_text()
|
||||||
|
self.assertTrue(self.scope['setup_pve_webhook_core']()['configured'])
|
||||||
|
self.assertEqual(self.files(), ['notifications.cfg', 'priv/notifications.cfg'])
|
||||||
|
self.assertEqual((self.pve / 'notifications.cfg').read_text(), current)
|
||||||
|
kept = sorted(p.name for p in self.kept.iterdir())
|
||||||
|
self.assertEqual(kept, [f'notifications.cfg.proxmenux_backup_20260101_{i:06d}' for i in (37, 38, 39)]
|
||||||
|
+ [f'priv-notifications.cfg.proxmenux_backup_20260101_{i:06d}' for i in (37, 38, 39)])
|
||||||
|
self.assertEqual((self.kept / kept[2]).read_text(), 'old 39')
|
||||||
|
|
||||||
|
def test_restore_takes_the_newest_copy_of_each_file(self):
|
||||||
|
(self.pve / 'priv').mkdir()
|
||||||
|
(self.pve / 'notifications.cfg').write_text('before\n')
|
||||||
|
(self.pve / 'priv' / 'notifications.cfg').write_text('private before\n')
|
||||||
|
self.scope['setup_pve_webhook_core']()
|
||||||
|
answer, _status = self.scope['restore_pve_notification_cfg']()
|
||||||
|
self.assertTrue(answer['success'])
|
||||||
|
self.assertEqual((self.pve / 'notifications.cfg').read_text(), 'before\n')
|
||||||
|
self.assertEqual((self.pve / 'priv' / 'notifications.cfg').read_text(), 'private before\n')
|
||||||
|
self.mounted = False
|
||||||
|
answer, _status = self.scope['restore_pve_notification_cfg']()
|
||||||
|
self.assertFalse(answer['success'])
|
||||||
|
|
||||||
|
def test_uninstalling_the_monitor_removes_its_target_from_proxmox(self):
|
||||||
|
menu = (SCRIPTS.parents[1] / 'scripts/menus/config_menu.sh').read_text()
|
||||||
|
body = menu[menu.index('remove_monitor_notification_target() {'):menu.index('uninstall_proxmenux_monitor() {')]
|
||||||
|
self.assertIn('mountpoint -q /etc/pve || return 0', body)
|
||||||
|
# The rule first: Proxmox does not delete a target that a rule still uses.
|
||||||
|
self.assertLess(body.index('matchers/proxmenux-default'), body.index('endpoints/webhook/proxmenux-webhook'))
|
||||||
|
self.assertIn('/etc/pve/priv/notifications.cfg.proxmenux_backup_*', body)
|
||||||
|
uninstall = menu[menu.index('uninstall_proxmenux_monitor() {'):menu.index('check_monitor_status() {')]
|
||||||
|
self.assertIn('remove_monitor_notification_target', uninstall)
|
||||||
|
|
||||||
|
def test_a_start_during_a_restart_waits_for_the_mount(self):
|
||||||
|
self.mounted = False
|
||||||
|
(self.pve / 'priv').mkdir()
|
||||||
|
|
||||||
|
def back(_seconds):
|
||||||
|
self.mounted = True
|
||||||
|
with mock.patch('time.sleep', side_effect=back):
|
||||||
|
result = self.scope['setup_pve_webhook_when_mounted'](timeout=60, interval=1)
|
||||||
|
self.assertTrue(result['configured'])
|
||||||
|
self.assertIn('webhook: proxmenux-webhook', (self.pve / 'notifications.cfg').read_text())
|
||||||
|
|
||||||
|
def test_the_wait_ends_without_writing(self):
|
||||||
|
self.mounted = False
|
||||||
|
result = self.scope['setup_pve_webhook_when_mounted'](timeout=0, interval=0)
|
||||||
|
self.assertTrue(result['waiting'])
|
||||||
|
self.assertEqual(list(self.pve.iterdir()), [])
|
||||||
|
|
||||||
|
def test_the_monitor_starts_after_the_cluster_service(self):
|
||||||
|
root = SCRIPTS.parents[1]
|
||||||
|
for name in ('install_proxmenux.sh', 'install_proxmenux_beta.sh', 'scripts/menus/config_menu.sh'):
|
||||||
|
text = (root / name).read_text()
|
||||||
|
self.assertIn('After=network.target pve-cluster.service', text, name)
|
||||||
|
self.assertNotIn('After=network.target\n\n[Service]\nType=simple\nUser=root\nWorkingDirectory=$MONITOR', text, name)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -623,7 +623,7 @@ _proxmenux_rewrite_monitor_unit_for_apprun() {
|
|||||||
cat > "$MONITOR_SERVICE_FILE" << EOF
|
cat > "$MONITOR_SERVICE_FILE" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=ProxMenux Monitor - Web Dashboard
|
Description=ProxMenux Monitor - Web Dashboard
|
||||||
After=network.target
|
After=network.target pve-cluster.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
@@ -655,7 +655,7 @@ create_monitor_service() {
|
|||||||
cat > "$MONITOR_SERVICE_FILE" << EOF
|
cat > "$MONITOR_SERVICE_FILE" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=ProxMenux Monitor - Web Dashboard
|
Description=ProxMenux Monitor - Web Dashboard
|
||||||
After=network.target
|
After=network.target pve-cluster.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
|
|||||||
@@ -496,7 +496,7 @@ update_monitor_service() {
|
|||||||
cat > "$MONITOR_SERVICE_FILE" << EOF
|
cat > "$MONITOR_SERVICE_FILE" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=ProxMenux Monitor - Web Dashboard (Beta)
|
Description=ProxMenux Monitor - Web Dashboard (Beta)
|
||||||
After=network.target
|
After=network.target pve-cluster.service
|
||||||
Before=shutdown.target reboot.target halt.target
|
Before=shutdown.target reboot.target halt.target
|
||||||
Conflicts=shutdown.target reboot.target halt.target
|
Conflicts=shutdown.target reboot.target halt.target
|
||||||
|
|
||||||
@@ -535,7 +535,7 @@ create_monitor_service() {
|
|||||||
cat > "$MONITOR_SERVICE_FILE" << EOF
|
cat > "$MONITOR_SERVICE_FILE" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=ProxMenux Monitor - Web Dashboard (Beta)
|
Description=ProxMenux Monitor - Web Dashboard (Beta)
|
||||||
After=network.target
|
After=network.target pve-cluster.service
|
||||||
Before=shutdown.target reboot.target halt.target
|
Before=shutdown.target reboot.target halt.target
|
||||||
Conflicts=shutdown.target reboot.target halt.target
|
Conflicts=shutdown.target reboot.target halt.target
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,17 @@ BACKTITLE="$(translate "$BACKTITLE")"
|
|||||||
|
|
||||||
# ==========================================================
|
# ==========================================================
|
||||||
|
|
||||||
|
# Proxmox would keep sending every notification to a Monitor that is gone.
|
||||||
|
# Its own interface removes the target and the rule that feeds it; the copies
|
||||||
|
# of the settings that earlier versions left in /etc/pve are removed too.
|
||||||
|
remove_monitor_notification_target() {
|
||||||
|
mountpoint -q /etc/pve || return 0
|
||||||
|
pvesh delete /cluster/notifications/matchers/proxmenux-default > /dev/null 2>&1 || true
|
||||||
|
pvesh delete /cluster/notifications/endpoints/webhook/proxmenux-webhook > /dev/null 2>&1 || true
|
||||||
|
rm -f /etc/pve/notifications.cfg.proxmenux_backup_* /etc/pve/priv/notifications.cfg.proxmenux_backup_*
|
||||||
|
echo " - Notification target removed from Proxmox"
|
||||||
|
}
|
||||||
|
|
||||||
uninstall_proxmenux_monitor() {
|
uninstall_proxmenux_monitor() {
|
||||||
|
|
||||||
# 1. Stop service if it is running
|
# 1. Stop service if it is running
|
||||||
@@ -81,6 +92,8 @@ uninstall_proxmenux_monitor() {
|
|||||||
echo " - Service is not running (ok)"
|
echo " - Service is not running (ok)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
remove_monitor_notification_target
|
||||||
|
|
||||||
# 2. Disable service if enabled
|
# 2. Disable service if enabled
|
||||||
if systemctl is-enabled --quiet "${MONITOR_SERVICE}"; then
|
if systemctl is-enabled --quiet "${MONITOR_SERVICE}"; then
|
||||||
echo " - Disabling service..."
|
echo " - Disabling service..."
|
||||||
@@ -213,7 +226,7 @@ normalize_stable_monitor_service() {
|
|||||||
cat > "$MONITOR_UNIT_FILE" << EOF
|
cat > "$MONITOR_UNIT_FILE" << EOF
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=ProxMenux Monitor - Web Dashboard
|
Description=ProxMenux Monitor - Web Dashboard
|
||||||
After=network.target
|
After=network.target pve-cluster.service
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
Type=simple
|
||||||
|
|||||||
@@ -342,7 +342,7 @@
|
|||||||
"practiceBody": "The PVE setup writes the target as <code>http://127.0.0.1:8008</code>, so PVE-emitted notifications always go through the loopback path with rate-limit-only security. The remote-caller path with the shared secret is opt-in for custom integrations — point an external service at <code>https://<monitor-host>:<port>/api/notifications/webhook</code> and supply the <code>X-Webhook-Secret</code> header to use it.",
|
"practiceBody": "The PVE setup writes the target as <code>http://127.0.0.1:8008</code>, so PVE-emitted notifications always go through the loopback path with rate-limit-only security. The remote-caller path with the shared secret is opt-in for custom integrations — point an external service at <code>https://<monitor-host>:<port>/api/notifications/webhook</code> and supply the <code>X-Webhook-Secret</code> header to use it.",
|
||||||
"actionsIntro": "The Monitor manages this target through three actions on the Settings tab:",
|
"actionsIntro": "The Monitor manages this target through three actions on the Settings tab:",
|
||||||
"actionsItems": [
|
"actionsItems": [
|
||||||
"<strong>Setup</strong> — runs automatically when you enable Notifications. Creates the entry in <code>/etc/pve/notifications.cfg</code> after backing up the current file.",
|
"<strong>Setup</strong> — runs automatically when you enable Notifications. Creates the entry in <code>/etc/pve/notifications.cfg</code>. The file is written only when its content has to change, and a copy of the previous one is kept in <code>/usr/local/share/proxmenux/backups/pve-notifications/</code> (the three newest). Nothing is written while <code>/etc/pve</code> is not mounted: a Monitor that starts during a restart of <code>pve-cluster</code> registers the entry once it is back.",
|
||||||
"<strong>Cleanup</strong> — removes the entry. The previous backup of the file is kept.",
|
"<strong>Cleanup</strong> — removes the entry. The previous backup of the file is kept.",
|
||||||
"<strong>Read config</strong> — shows the current targets and matchers as PVE sees them. This is how you confirm the Monitor's entry is the one firing when PVE has multiple notification routes configured."
|
"<strong>Read config</strong> — shows the current targets and matchers as PVE sees them. This is how you confirm the Monitor's entry is the one firing when PVE has multiple notification routes configured."
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
"items": [
|
"items": [
|
||||||
"<strong>Confirmation dialog.</strong> Yes/no — “Are you sure you want to uninstall ProxMenux?”",
|
"<strong>Confirmation dialog.</strong> Yes/no — “Are you sure you want to uninstall ProxMenux?”",
|
||||||
"<strong>Dependency removal checklist.</strong> Pre-selected list depends on install type (see table below). All boxes default to OFF — you opt in to removing each dependency.",
|
"<strong>Dependency removal checklist.</strong> Pre-selected list depends on install type (see table below). All boxes default to OFF — you opt in to removing each dependency.",
|
||||||
"<strong>Removal runs with a progress bar.</strong> Removes the venv (Translation install only), the <code>menu</code> launcher, the <code>/usr/local/share/proxmenux/</code> tree, the ProxMenux Monitor service if present, the dependencies you ticked, and finally restores the system file backups (see below).",
|
"<strong>Removal runs with a progress bar.</strong> Removes the venv (Translation install only), the <code>menu</code> launcher, the <code>/usr/local/share/proxmenux/</code> tree, the ProxMenux Monitor service if present together with the notification target it registered in Proxmox VE (<code>proxmenux-webhook</code> and its matcher), the dependencies you ticked, and finally restores the system file backups (see below).",
|
||||||
"<strong>Completion dialog.</strong> Lists removed dependencies and exits."
|
"<strong>Completion dialog.</strong> Lists removed dependencies and exits."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -342,7 +342,7 @@
|
|||||||
"practiceBody": "El setup de PVE escribe el destino como <code>http://127.0.0.1:8008</code>, así que las notificaciones emitidas por PVE siempre pasan por el camino loopback con seguridad solo rate-limit. El camino de llamador remoto con shared secret es opt-in para integraciones personalizadas — apunta un servicio externo a <code>https://<host-monitor>:<port>/api/notifications/webhook</code> y suministra la cabecera <code>X-Webhook-Secret</code> para usarlo.",
|
"practiceBody": "El setup de PVE escribe el destino como <code>http://127.0.0.1:8008</code>, así que las notificaciones emitidas por PVE siempre pasan por el camino loopback con seguridad solo rate-limit. El camino de llamador remoto con shared secret es opt-in para integraciones personalizadas — apunta un servicio externo a <code>https://<host-monitor>:<port>/api/notifications/webhook</code> y suministra la cabecera <code>X-Webhook-Secret</code> para usarlo.",
|
||||||
"actionsIntro": "El Monitor gestiona este destino a través de tres acciones en la pestaña Settings:",
|
"actionsIntro": "El Monitor gestiona este destino a través de tres acciones en la pestaña Settings:",
|
||||||
"actionsItems": [
|
"actionsItems": [
|
||||||
"<strong>Setup</strong> — corre automáticamente cuando activas Notifications. Crea la entrada en <code>/etc/pve/notifications.cfg</code> tras hacer backup del archivo actual.",
|
"<strong>Setup</strong> — corre automáticamente cuando activas Notifications. Crea la entrada en <code>/etc/pve/notifications.cfg</code>. El archivo solo se escribe cuando su contenido tiene que cambiar, y se guarda un backup del anterior en <code>/usr/local/share/proxmenux/backups/pve-notifications/</code> (los tres más recientes). No se escribe nada mientras <code>/etc/pve</code> no está montado: un Monitor que arranca durante un reinicio de <code>pve-cluster</code> registra la entrada cuando vuelve.",
|
||||||
"<strong>Cleanup</strong> — elimina la entrada. El backup previo del archivo se conserva.",
|
"<strong>Cleanup</strong> — elimina la entrada. El backup previo del archivo se conserva.",
|
||||||
"<strong>Read config</strong> — muestra los destinos y matchers actuales como PVE los ve. Así es como confirmas que la entrada del Monitor es la que dispara cuando PVE tiene múltiples rutas de notificación configuradas."
|
"<strong>Read config</strong> — muestra los destinos y matchers actuales como PVE los ve. Así es como confirmas que la entrada del Monitor es la que dispara cuando PVE tiene múltiples rutas de notificación configuradas."
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -19,7 +19,7 @@
|
|||||||
"items": [
|
"items": [
|
||||||
"<strong>Diálogo de confirmación.</strong> Sí/no — \"¿Seguro que quieres desinstalar ProxMenux?\"",
|
"<strong>Diálogo de confirmación.</strong> Sí/no — \"¿Seguro que quieres desinstalar ProxMenux?\"",
|
||||||
"<strong>Checklist de eliminación de dependencias.</strong> La lista preseleccionada depende del tipo de instalación (mira la tabla de abajo). Todas las casillas por defecto en OFF — te apuntas a eliminar cada dependencia.",
|
"<strong>Checklist de eliminación de dependencias.</strong> La lista preseleccionada depende del tipo de instalación (mira la tabla de abajo). Todas las casillas por defecto en OFF — te apuntas a eliminar cada dependencia.",
|
||||||
"<strong>La eliminación corre con una barra de progreso.</strong> Elimina el venv (solo instalación con traducción), el lanzador <code>menu</code>, el árbol <code>/usr/local/share/proxmenux/</code>, el servicio ProxMenux Monitor si está presente, las dependencias que marcaste y, por último, restaura los backups de archivos del sistema (mira abajo).",
|
"<strong>La eliminación corre con una barra de progreso.</strong> Elimina el venv (solo instalación con traducción), el lanzador <code>menu</code>, el árbol <code>/usr/local/share/proxmenux/</code>, el servicio ProxMenux Monitor si está presente junto con el destino de notificaciones que registró en Proxmox VE (<code>proxmenux-webhook</code> y su matcher), las dependencias que marcaste y, por último, restaura los backups de archivos del sistema (mira abajo).",
|
||||||
"<strong>Diálogo de finalización.</strong> Lista las dependencias eliminadas y sale."
|
"<strong>Diálogo de finalización.</strong> Lista las dependencias eliminadas y sale."
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in New Issue
Block a user