mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-09-14 18:56:52 +00:00
replication failure notifications
This commit is contained in:
@@ -13747,6 +13747,54 @@ def api_apps_catalog():
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/apps/github-token', methods=['GET'])
|
||||
@require_auth
|
||||
def api_apps_github_token_status():
|
||||
"""Return whether the optional GitHub API token is configured.
|
||||
|
||||
The token itself is deliberately never returned to the client.
|
||||
"""
|
||||
try:
|
||||
if not notification_manager._config:
|
||||
notification_manager._load_config()
|
||||
value = notification_manager._config.get('github_pat', '')
|
||||
return jsonify({'configured': bool(isinstance(value, str) and value.strip())})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/apps/github-token', methods=['PUT'])
|
||||
@require_admin_scope
|
||||
def api_apps_github_token_save():
|
||||
"""Store the GitHub API token in the existing encrypted settings store."""
|
||||
payload = request.get_json(silent=True) or {}
|
||||
token = payload.get('token')
|
||||
if not isinstance(token, str):
|
||||
return jsonify({'error': 'token must be a string'}), 400
|
||||
token = token.strip()
|
||||
if not token:
|
||||
return jsonify({'error': 'token is required'}), 400
|
||||
if len(token) > 512:
|
||||
return jsonify({'error': 'token exceeds the 512 character limit'}), 400
|
||||
if any(ch.isspace() or ord(ch) < 33 or ord(ch) == 127 for ch in token):
|
||||
return jsonify({'error': 'token contains whitespace or control characters'}), 400
|
||||
|
||||
result = notification_manager.save_settings({'github_pat': token})
|
||||
if not result.get('success'):
|
||||
return jsonify({'error': result.get('error', 'failed to save token')}), 500
|
||||
return jsonify({'success': True, 'configured': True})
|
||||
|
||||
|
||||
@app.route('/api/apps/github-token', methods=['DELETE'])
|
||||
@require_admin_scope
|
||||
def api_apps_github_token_remove():
|
||||
"""Clear the optional GitHub API token without exposing its old value."""
|
||||
result = notification_manager.save_settings({'github_pat': ''})
|
||||
if not result.get('success'):
|
||||
return jsonify({'error': result.get('error', 'failed to remove token')}), 500
|
||||
return jsonify({'success': True, 'configured': False})
|
||||
|
||||
|
||||
@app.route('/api/lxc-apps/dockerhub-tag-preview', methods=['POST'])
|
||||
@require_auth
|
||||
def api_lxc_apps_dockerhub_tag_preview():
|
||||
|
||||
@@ -1285,16 +1285,13 @@ def _select_working_hint_detector(vmid, hint: dict) -> tuple[dict, Optional[str]
|
||||
def _github_pat() -> Optional[str]:
|
||||
try:
|
||||
from notification_manager import notification_manager
|
||||
pat = notification_manager._config.get("github_pat") if notification_manager._config else None
|
||||
if not pat:
|
||||
return None
|
||||
try:
|
||||
from notification_manager import decrypt_sensitive_value
|
||||
if isinstance(pat, str) and pat.startswith("encrypted:"):
|
||||
return decrypt_sensitive_value(pat)
|
||||
except Exception:
|
||||
pass
|
||||
return pat if isinstance(pat, str) else None
|
||||
# The notification manager owns the shared encrypted settings store.
|
||||
# During very early calls its runtime cache may not have been loaded
|
||||
# yet, so initialise it before reading the optional GitHub token.
|
||||
if not notification_manager._config:
|
||||
notification_manager._load_config()
|
||||
pat = notification_manager._config.get("github_pat")
|
||||
return pat.strip() if isinstance(pat, str) and pat.strip() else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -1365,7 +1362,7 @@ def _fetch_github_latest_details(config: dict) -> tuple[Optional[str], Optional[
|
||||
if e.code == 403:
|
||||
remaining = e.headers.get("X-RateLimit-Remaining", "1")
|
||||
if remaining == "0":
|
||||
return None, "github rate limited — configure a PAT in Settings", None
|
||||
return None, "github rate limited — configure a PAT in Settings → GitHub API", None
|
||||
return None, "github rejected the request (403)", None
|
||||
return None, f"github error {e.code}", None
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
|
||||
@@ -4147,6 +4147,17 @@ class ProxmoxHookWatcher:
|
||||
'job_id': pve_job_id,
|
||||
}
|
||||
|
||||
if pve_type == 'replication':
|
||||
replication = self._extract_replication_context(
|
||||
fields, title, message
|
||||
)
|
||||
data.update(replication)
|
||||
entity_id = (
|
||||
replication.get('job_id')
|
||||
or replication.get('vmid')
|
||||
or entity_id
|
||||
)
|
||||
|
||||
# `system_problem` is the generic fallback of `_classify_pve` for
|
||||
# unknown/empty pve_type. Without a populated `reason`, the template
|
||||
# renders "Reason: " (empty) and `_summarize_event` falls back to
|
||||
@@ -4274,6 +4285,73 @@ class ProxmoxHookWatcher:
|
||||
|
||||
self._queue.put(event)
|
||||
return {'accepted': True, 'event_type': event_type, 'event_id': event.event_id}
|
||||
|
||||
def _extract_replication_context(self, fields: dict, title: str,
|
||||
message: str) -> dict:
|
||||
"""Map a native PVE replication notice to template fields."""
|
||||
raw_job_id = fields.get('job-id') or fields.get('job_id') or ''
|
||||
job_id = str(raw_job_id).strip()
|
||||
if not re.fullmatch(r'\d+(?:-\d+)?', job_id):
|
||||
combined = f'{title or ""}\n{message or ""}'
|
||||
match = re.search(
|
||||
r'\breplication(?:\s+job)?(?:\s*:\s*|\s+)'
|
||||
r'[\'\"]?(\d+(?:-\d+)?)\b',
|
||||
combined,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if not match:
|
||||
match = re.search(r'\b(\d+-\d+)\b', combined)
|
||||
job_id = match.group(1) if match else ''
|
||||
|
||||
vmid_match = re.fullmatch(r'(\d+)(?:-\d+)?', job_id)
|
||||
vmid = vmid_match.group(1) if vmid_match else ''
|
||||
vmname = self._resolve_replication_guest_name(vmid)
|
||||
if not vmname and vmid:
|
||||
vmname = 'VM/CT'
|
||||
|
||||
target = str(
|
||||
fields.get('job-target') or fields.get('target') or ''
|
||||
).strip()
|
||||
if not target:
|
||||
target_match = re.search(
|
||||
r'\bwith\s+target\s+[\'\"]([^\'\"\n]+)[\'\"]',
|
||||
message or '',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
if target_match:
|
||||
target = target_match.group(1).strip()
|
||||
|
||||
reason_match = re.search(
|
||||
r'^\s*Error:\s*(.*?)\s*\Z',
|
||||
message or '',
|
||||
re.IGNORECASE | re.MULTILINE | re.DOTALL,
|
||||
)
|
||||
reason = reason_match.group(1).strip() if reason_match else ''
|
||||
if not reason:
|
||||
reason = (message or title or '').strip()
|
||||
|
||||
return {
|
||||
'job_id': job_id,
|
||||
'vmid': vmid,
|
||||
'vmname': vmname,
|
||||
'target_node': target,
|
||||
'reason': reason,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_replication_guest_name(vmid: str) -> str:
|
||||
"""Resolve the replicated guest name from the cluster config."""
|
||||
if not vmid or not vmid.isdigit():
|
||||
return ''
|
||||
for base in ('/etc/pve/qemu-server', '/etc/pve/lxc'):
|
||||
try:
|
||||
with open(f'{base}/{vmid}.conf', encoding='utf-8') as config:
|
||||
for line in config:
|
||||
if line.startswith(('name:', 'hostname:')):
|
||||
return line.split(':', 1)[1].strip()
|
||||
except OSError:
|
||||
continue
|
||||
return ''
|
||||
|
||||
def _classify_pve(self, pve_type: str, severity: str,
|
||||
title: str, message: str) -> tuple:
|
||||
|
||||
@@ -64,6 +64,10 @@ ENCRYPTION_KEY_FILE = Path('/usr/local/share/proxmenux/.notification_key')
|
||||
|
||||
# Keys that contain sensitive data and should be encrypted
|
||||
SENSITIVE_KEYS = {
|
||||
# Optional GitHub API token used by the LXC app version tracker.
|
||||
# It lives in the shared settings store so it benefits from the same
|
||||
# ENC2 encryption and never needs a second secrets file.
|
||||
'github_pat',
|
||||
'ai_api_key', # Legacy - kept for migration
|
||||
'ai_api_key_groq',
|
||||
'ai_api_key_gemini',
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from queue import Queue
|
||||
from unittest import mock
|
||||
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import notification_events # noqa: E402
|
||||
import notification_templates # noqa: E402
|
||||
|
||||
|
||||
class ReplicationWebhookTests(unittest.TestCase):
|
||||
def _process(self, payload, guest_name='fileserver'):
|
||||
watcher = notification_events.ProxmoxHookWatcher(Queue())
|
||||
with mock.patch.object(
|
||||
watcher,
|
||||
'_resolve_replication_guest_name',
|
||||
return_value=guest_name,
|
||||
), mock.patch.object(
|
||||
notification_events,
|
||||
'capture_journal_context',
|
||||
return_value='',
|
||||
):
|
||||
result = watcher.process_webhook(payload)
|
||||
return result, watcher._queue.get_nowait()
|
||||
|
||||
def test_structured_job_id_populates_template_fields(self):
|
||||
reason = 'command zfs error: cannot open pool\nremote side unavailable'
|
||||
result, event = self._process({
|
||||
'title': "Replication Job: '100-0' failed",
|
||||
'message': (
|
||||
"Replication job '100-0' with target 'pve02' and schedule "
|
||||
"'*/15' failed!\n\n"
|
||||
"Last successful sync: 2026-09-02 15:00:00\n"
|
||||
"Next sync try: 2026-09-02 15:30:00\n"
|
||||
"Failure count: 1\n\n"
|
||||
f"Error:\n{reason}"
|
||||
),
|
||||
'severity': 'error',
|
||||
'fields': {
|
||||
'type': 'replication',
|
||||
'hostname': 'pve01',
|
||||
'job-id': '100-0',
|
||||
},
|
||||
})
|
||||
|
||||
self.assertTrue(result['accepted'])
|
||||
self.assertEqual(event.event_type, 'replication_fail')
|
||||
self.assertEqual(event.entity_id, '100-0')
|
||||
self.assertEqual(event.data['job_id'], '100-0')
|
||||
self.assertEqual(event.data['vmid'], '100')
|
||||
self.assertEqual(event.data['vmname'], 'fileserver')
|
||||
self.assertEqual(event.data['target_node'], 'pve02')
|
||||
self.assertEqual(event.data['reason'], reason)
|
||||
|
||||
rendered = notification_templates.render_template(
|
||||
event.event_type,
|
||||
event.data,
|
||||
)
|
||||
self.assertIn('fileserver (100)', rendered['title'])
|
||||
self.assertIn('ID: 100', rendered['body_text'])
|
||||
self.assertIn(reason, rendered['body_text'])
|
||||
|
||||
def test_title_and_message_are_used_when_job_id_field_is_missing(self):
|
||||
_, event = self._process({
|
||||
'title': "Replication Job: '212-3' failed",
|
||||
'message': (
|
||||
"Replication job '212-3' with target 'pve03' failed!\n\n"
|
||||
"Error: storage 'replica-zfs' is not available"
|
||||
),
|
||||
'severity': 'error',
|
||||
'fields': {'type': 'replication', 'hostname': 'pve01'},
|
||||
}, guest_name='')
|
||||
|
||||
self.assertEqual(event.entity_id, '212-3')
|
||||
self.assertEqual(event.data['vmid'], '212')
|
||||
self.assertEqual(event.data['vmname'], 'VM/CT')
|
||||
self.assertEqual(event.data['target_node'], 'pve03')
|
||||
self.assertEqual(
|
||||
event.data['reason'],
|
||||
"storage 'replica-zfs' is not available",
|
||||
)
|
||||
|
||||
def test_missing_error_block_never_renders_an_empty_reason(self):
|
||||
message = "Replication job '300-0' failed unexpectedly"
|
||||
_, event = self._process({
|
||||
'title': "Replication Job: '300-0' failed",
|
||||
'message': message,
|
||||
'severity': 'error',
|
||||
'fields': {'type': 'replication', 'hostname': 'pve01'},
|
||||
})
|
||||
|
||||
self.assertEqual(event.data['reason'], message)
|
||||
rendered = notification_templates.render_template(
|
||||
event.event_type,
|
||||
event.data,
|
||||
)
|
||||
self.assertIn(f'Reason: {message}', rendered['body_text'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user