replication failure notifications

This commit is contained in:
MacRimi
2026-09-02 16:28:03 +02:00
parent d10acbc895
commit e7b69dae91
15 changed files with 544 additions and 12 deletions
+48
View File
@@ -13747,6 +13747,54 @@ def api_apps_catalog():
return jsonify({'error': str(e)}), 500
@app.route('/api/apps/github-token', methods=['GET'])
@require_auth
def api_apps_github_token_status():
"""Return whether the optional GitHub API token is configured.
The token itself is deliberately never returned to the client.
"""
try:
if not notification_manager._config:
notification_manager._load_config()
value = notification_manager._config.get('github_pat', '')
return jsonify({'configured': bool(isinstance(value, str) and value.strip())})
except Exception as e:
return jsonify({'error': str(e)}), 500
@app.route('/api/apps/github-token', methods=['PUT'])
@require_admin_scope
def api_apps_github_token_save():
"""Store the GitHub API token in the existing encrypted settings store."""
payload = request.get_json(silent=True) or {}
token = payload.get('token')
if not isinstance(token, str):
return jsonify({'error': 'token must be a string'}), 400
token = token.strip()
if not token:
return jsonify({'error': 'token is required'}), 400
if len(token) > 512:
return jsonify({'error': 'token exceeds the 512 character limit'}), 400
if any(ch.isspace() or ord(ch) < 33 or ord(ch) == 127 for ch in token):
return jsonify({'error': 'token contains whitespace or control characters'}), 400
result = notification_manager.save_settings({'github_pat': token})
if not result.get('success'):
return jsonify({'error': result.get('error', 'failed to save token')}), 500
return jsonify({'success': True, 'configured': True})
@app.route('/api/apps/github-token', methods=['DELETE'])
@require_admin_scope
def api_apps_github_token_remove():
"""Clear the optional GitHub API token without exposing its old value."""
result = notification_manager.save_settings({'github_pat': ''})
if not result.get('success'):
return jsonify({'error': result.get('error', 'failed to remove token')}), 500
return jsonify({'success': True, 'configured': False})
@app.route('/api/lxc-apps/dockerhub-tag-preview', methods=['POST'])
@require_auth
def api_lxc_apps_dockerhub_tag_preview():
+8 -11
View File
@@ -1285,16 +1285,13 @@ def _select_working_hint_detector(vmid, hint: dict) -> tuple[dict, Optional[str]
def _github_pat() -> Optional[str]:
try:
from notification_manager import notification_manager
pat = notification_manager._config.get("github_pat") if notification_manager._config else None
if not pat:
return None
try:
from notification_manager import decrypt_sensitive_value
if isinstance(pat, str) and pat.startswith("encrypted:"):
return decrypt_sensitive_value(pat)
except Exception:
pass
return pat if isinstance(pat, str) else None
# The notification manager owns the shared encrypted settings store.
# During very early calls its runtime cache may not have been loaded
# yet, so initialise it before reading the optional GitHub token.
if not notification_manager._config:
notification_manager._load_config()
pat = notification_manager._config.get("github_pat")
return pat.strip() if isinstance(pat, str) and pat.strip() else None
except Exception:
return None
@@ -1365,7 +1362,7 @@ def _fetch_github_latest_details(config: dict) -> tuple[Optional[str], Optional[
if e.code == 403:
remaining = e.headers.get("X-RateLimit-Remaining", "1")
if remaining == "0":
return None, "github rate limited — configure a PAT in Settings", None
return None, "github rate limited — configure a PAT in Settings → GitHub API", None
return None, "github rejected the request (403)", None
return None, f"github error {e.code}", None
except (urllib.error.URLError, TimeoutError, OSError) as e:
+78
View File
@@ -4147,6 +4147,17 @@ class ProxmoxHookWatcher:
'job_id': pve_job_id,
}
if pve_type == 'replication':
replication = self._extract_replication_context(
fields, title, message
)
data.update(replication)
entity_id = (
replication.get('job_id')
or replication.get('vmid')
or entity_id
)
# `system_problem` is the generic fallback of `_classify_pve` for
# unknown/empty pve_type. Without a populated `reason`, the template
# renders "Reason: " (empty) and `_summarize_event` falls back to
@@ -4274,6 +4285,73 @@ class ProxmoxHookWatcher:
self._queue.put(event)
return {'accepted': True, 'event_type': event_type, 'event_id': event.event_id}
def _extract_replication_context(self, fields: dict, title: str,
message: str) -> dict:
"""Map a native PVE replication notice to template fields."""
raw_job_id = fields.get('job-id') or fields.get('job_id') or ''
job_id = str(raw_job_id).strip()
if not re.fullmatch(r'\d+(?:-\d+)?', job_id):
combined = f'{title or ""}\n{message or ""}'
match = re.search(
r'\breplication(?:\s+job)?(?:\s*:\s*|\s+)'
r'[\'\"]?(\d+(?:-\d+)?)\b',
combined,
re.IGNORECASE,
)
if not match:
match = re.search(r'\b(\d+-\d+)\b', combined)
job_id = match.group(1) if match else ''
vmid_match = re.fullmatch(r'(\d+)(?:-\d+)?', job_id)
vmid = vmid_match.group(1) if vmid_match else ''
vmname = self._resolve_replication_guest_name(vmid)
if not vmname and vmid:
vmname = 'VM/CT'
target = str(
fields.get('job-target') or fields.get('target') or ''
).strip()
if not target:
target_match = re.search(
r'\bwith\s+target\s+[\'\"]([^\'\"\n]+)[\'\"]',
message or '',
re.IGNORECASE,
)
if target_match:
target = target_match.group(1).strip()
reason_match = re.search(
r'^\s*Error:\s*(.*?)\s*\Z',
message or '',
re.IGNORECASE | re.MULTILINE | re.DOTALL,
)
reason = reason_match.group(1).strip() if reason_match else ''
if not reason:
reason = (message or title or '').strip()
return {
'job_id': job_id,
'vmid': vmid,
'vmname': vmname,
'target_node': target,
'reason': reason,
}
@staticmethod
def _resolve_replication_guest_name(vmid: str) -> str:
"""Resolve the replicated guest name from the cluster config."""
if not vmid or not vmid.isdigit():
return ''
for base in ('/etc/pve/qemu-server', '/etc/pve/lxc'):
try:
with open(f'{base}/{vmid}.conf', encoding='utf-8') as config:
for line in config:
if line.startswith(('name:', 'hostname:')):
return line.split(':', 1)[1].strip()
except OSError:
continue
return ''
def _classify_pve(self, pve_type: str, severity: str,
title: str, message: str) -> tuple:
+4
View File
@@ -64,6 +64,10 @@ ENCRYPTION_KEY_FILE = Path('/usr/local/share/proxmenux/.notification_key')
# Keys that contain sensitive data and should be encrypted
SENSITIVE_KEYS = {
# Optional GitHub API token used by the LXC app version tracker.
# It lives in the shared settings store so it benefits from the same
# ENC2 encryption and never needs a second secrets file.
'github_pat',
'ai_api_key', # Legacy - kept for migration
'ai_api_key_groq',
'ai_api_key_gemini',
@@ -0,0 +1,106 @@
import sys
import unittest
from pathlib import Path
from queue import Queue
from unittest import mock
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
if str(SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPTS_DIR))
import notification_events # noqa: E402
import notification_templates # noqa: E402
class ReplicationWebhookTests(unittest.TestCase):
def _process(self, payload, guest_name='fileserver'):
watcher = notification_events.ProxmoxHookWatcher(Queue())
with mock.patch.object(
watcher,
'_resolve_replication_guest_name',
return_value=guest_name,
), mock.patch.object(
notification_events,
'capture_journal_context',
return_value='',
):
result = watcher.process_webhook(payload)
return result, watcher._queue.get_nowait()
def test_structured_job_id_populates_template_fields(self):
reason = 'command zfs error: cannot open pool\nremote side unavailable'
result, event = self._process({
'title': "Replication Job: '100-0' failed",
'message': (
"Replication job '100-0' with target 'pve02' and schedule "
"'*/15' failed!\n\n"
"Last successful sync: 2026-09-02 15:00:00\n"
"Next sync try: 2026-09-02 15:30:00\n"
"Failure count: 1\n\n"
f"Error:\n{reason}"
),
'severity': 'error',
'fields': {
'type': 'replication',
'hostname': 'pve01',
'job-id': '100-0',
},
})
self.assertTrue(result['accepted'])
self.assertEqual(event.event_type, 'replication_fail')
self.assertEqual(event.entity_id, '100-0')
self.assertEqual(event.data['job_id'], '100-0')
self.assertEqual(event.data['vmid'], '100')
self.assertEqual(event.data['vmname'], 'fileserver')
self.assertEqual(event.data['target_node'], 'pve02')
self.assertEqual(event.data['reason'], reason)
rendered = notification_templates.render_template(
event.event_type,
event.data,
)
self.assertIn('fileserver (100)', rendered['title'])
self.assertIn('ID: 100', rendered['body_text'])
self.assertIn(reason, rendered['body_text'])
def test_title_and_message_are_used_when_job_id_field_is_missing(self):
_, event = self._process({
'title': "Replication Job: '212-3' failed",
'message': (
"Replication job '212-3' with target 'pve03' failed!\n\n"
"Error: storage 'replica-zfs' is not available"
),
'severity': 'error',
'fields': {'type': 'replication', 'hostname': 'pve01'},
}, guest_name='')
self.assertEqual(event.entity_id, '212-3')
self.assertEqual(event.data['vmid'], '212')
self.assertEqual(event.data['vmname'], 'VM/CT')
self.assertEqual(event.data['target_node'], 'pve03')
self.assertEqual(
event.data['reason'],
"storage 'replica-zfs' is not available",
)
def test_missing_error_block_never_renders_an_empty_reason(self):
message = "Replication job '300-0' failed unexpectedly"
_, event = self._process({
'title': "Replication Job: '300-0' failed",
'message': message,
'severity': 'error',
'fields': {'type': 'replication', 'hostname': 'pve01'},
})
self.assertEqual(event.data['reason'], message)
rendered = notification_templates.render_template(
event.event_type,
event.data,
)
self.assertIn(f'Reason: {message}', rendered['body_text'])
if __name__ == '__main__':
unittest.main()