mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-08 22:46:41 +00:00
fix(notifications): scope outcome improvements to backups
This commit is contained in:
martino
@@ -131,7 +131,6 @@ cp "$SCRIPT_DIR/auth_manager.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️
|
||||
cp "$SCRIPT_DIR/jwt_middleware.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ jwt_middleware.py not found"
|
||||
cp "$SCRIPT_DIR/health_monitor.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ health_monitor.py not found"
|
||||
cp "$SCRIPT_DIR/health_persistence.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ health_persistence.py not found"
|
||||
cp "$SCRIPT_DIR/health_recovery.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/flask_health_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_health_routes.py not found"
|
||||
cp "$SCRIPT_DIR/flask_proxmenux_routes.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ flask_proxmenux_routes.py not found"
|
||||
cp "$SCRIPT_DIR/post_install_versions.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ post_install_versions.py not found"
|
||||
|
||||
@@ -1427,8 +1427,6 @@ class HealthMonitor:
|
||||
'details': f'Sustained for {actual_duration}s above {self.CPU_CRITICAL}%.',
|
||||
'cpu_percent': cpu_percent,
|
||||
'duration': actual_duration,
|
||||
'cpu_policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
|
||||
'recovery': self.CPU_RECOVERY},
|
||||
},
|
||||
)
|
||||
elif len(warning_samples) >= WARNING_MIN_SAMPLES and len(recovery_samples) < RECOVERY_MIN_SAMPLES:
|
||||
@@ -1448,32 +1446,13 @@ class HealthMonitor:
|
||||
'details': f'Sustained for {actual_duration}s above {self.CPU_WARNING}%.',
|
||||
'cpu_percent': cpu_percent,
|
||||
'duration': actual_duration,
|
||||
'cpu_policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
|
||||
'recovery': self.CPU_RECOVERY},
|
||||
},
|
||||
)
|
||||
else:
|
||||
status = 'OK'
|
||||
reason = None
|
||||
# CPU is normal - auto-resolve any existing CPU errors
|
||||
evidence = None
|
||||
# Presentation proof is stricter than operational hysteresis:
|
||||
# a supported warning can be below the fixed recovery cutoff.
|
||||
from health_recovery import _finite_number
|
||||
criterion = min(self.CPU_WARNING, self.CPU_RECOVERY)
|
||||
normal_samples = [entry for entry in self.state_history[state_key]
|
||||
if _finite_number(entry['value']) and 0 <= entry['value'] < criterion
|
||||
and _finite_number(entry['time'])
|
||||
and 0 <= current_time - entry['time'] <= self.CPU_RECOVERY_DURATION]
|
||||
if (_finite_number(cpu_percent) and 0 <= cpu_percent < criterion
|
||||
and len(normal_samples) >= RECOVERY_MIN_SAMPLES):
|
||||
evidence = {'check': 'cpu_usage', 'checked_at': current_time,
|
||||
'value': cpu_percent, 'normal_samples': len(normal_samples),
|
||||
'max_sample': max(entry['value'] for entry in normal_samples),
|
||||
'policy': {'warning': self.CPU_WARNING, 'critical': self.CPU_CRITICAL,
|
||||
'recovery': self.CPU_RECOVERY}}
|
||||
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal',
|
||||
check_evidence=evidence)
|
||||
health_persistence.resolve_error('cpu_usage', 'CPU usage returned to normal')
|
||||
|
||||
temp_status = self._check_cpu_temperature()
|
||||
|
||||
@@ -3921,7 +3900,6 @@ class HealthMonitor:
|
||||
|
||||
failed_services = []
|
||||
service_details = {}
|
||||
active_evidence = {}
|
||||
|
||||
for service in services_to_check:
|
||||
try:
|
||||
@@ -3936,10 +3914,6 @@ class HealthMonitor:
|
||||
if result.returncode != 0 or status != 'active':
|
||||
failed_services.append(service)
|
||||
service_details[service] = status or 'inactive'
|
||||
else:
|
||||
active_evidence[service] = {'check': f'pve_service_{service}',
|
||||
'checked_at': time.time(), 'service': service, 'state': status,
|
||||
'returncode': result.returncode}
|
||||
except Exception:
|
||||
failed_services.append(service)
|
||||
service_details[service] = 'error'
|
||||
@@ -3954,7 +3928,7 @@ class HealthMonitor:
|
||||
if svc not in failed_services:
|
||||
error_key = f'pve_service_{svc}'
|
||||
if health_persistence.is_error_active(error_key):
|
||||
health_persistence.clear_error(error_key, check_evidence=active_evidence.get(svc))
|
||||
health_persistence.clear_error(error_key)
|
||||
|
||||
# Build checks dict with status per service
|
||||
checks = {}
|
||||
|
||||
@@ -600,7 +600,7 @@ class HealthPersistence:
|
||||
|
||||
cursor.execute('''
|
||||
SELECT id, acknowledged, resolved_at, category, severity, first_seen,
|
||||
notification_sent, suppression_hours, acknowledged_at, details
|
||||
notification_sent, suppression_hours, acknowledged_at
|
||||
FROM errors WHERE error_key = ?
|
||||
''', (error_key,))
|
||||
existing = cursor.fetchone()
|
||||
@@ -609,7 +609,7 @@ class HealthPersistence:
|
||||
|
||||
if existing:
|
||||
(err_id, ack, resolved_at, old_cat, old_severity, first_seen,
|
||||
notif_sent, stored_suppression, acknowledged_at, old_details_json) = existing
|
||||
notif_sent, stored_suppression, acknowledged_at) = existing
|
||||
|
||||
if ack == 1:
|
||||
# SAFETY OVERRIDE: Critical CPU temperature ALWAYS re-triggers
|
||||
@@ -680,18 +680,6 @@ class HealthPersistence:
|
||||
conn.commit()
|
||||
return event_info
|
||||
|
||||
# Original CPU policy is immutable for this row's incident.
|
||||
# Never upgrade a legacy row from later/current settings.
|
||||
if error_key == 'cpu_usage':
|
||||
try:
|
||||
old_details = json.loads(old_details_json or '{}')
|
||||
except (ValueError, TypeError):
|
||||
old_details = {}
|
||||
details = dict(details) if isinstance(details, dict) else {}
|
||||
details.pop('cpu_policy', None)
|
||||
if isinstance(old_details, dict) and 'cpu_policy' in old_details:
|
||||
details['cpu_policy'] = old_details['cpu_policy']
|
||||
details_json = json.dumps(details)
|
||||
# Not acknowledged - update existing active error
|
||||
cursor.execute('''
|
||||
UPDATE errors
|
||||
@@ -756,12 +744,12 @@ class HealthPersistence:
|
||||
|
||||
return event_info
|
||||
|
||||
def resolve_error(self, error_key: str, reason: str = 'auto-resolved', *, check_evidence=None):
|
||||
def resolve_error(self, error_key: str, reason: str = 'auto-resolved'):
|
||||
"""Mark an error as resolved"""
|
||||
with self._db_lock:
|
||||
return self._resolve_error_impl(error_key, reason, check_evidence=check_evidence)
|
||||
return self._resolve_error_impl(error_key, reason)
|
||||
|
||||
def _resolve_error_impl(self, error_key, reason, *, check_evidence=None):
|
||||
def _resolve_error_impl(self, error_key, reason):
|
||||
with self._db_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
now = datetime.now().isoformat()
|
||||
@@ -788,7 +776,7 @@ class HealthPersistence:
|
||||
# was created — otherwise "Storage 'Tuxis' unavailable"
|
||||
# comes back as "Resolved - Storage" with no identity.
|
||||
cursor.execute(
|
||||
'SELECT details, id, first_seen, resolved_at FROM errors WHERE error_key = ? ORDER BY id DESC LIMIT 1',
|
||||
'SELECT details FROM errors WHERE error_key = ? ORDER BY id DESC LIMIT 1',
|
||||
(error_key,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
@@ -798,71 +786,14 @@ class HealthPersistence:
|
||||
stored_details = json.loads(row[0])
|
||||
except Exception:
|
||||
stored_details = None
|
||||
# Legacy/mismatched original policy cannot certify normality.
|
||||
if error_key == 'cpu_usage' and (not isinstance(stored_details, dict)
|
||||
or not isinstance(check_evidence, dict)
|
||||
or check_evidence.get('policy') != stored_details.get('cpu_policy')
|
||||
or not stored_details.get('cpu_policy')):
|
||||
check_evidence = None
|
||||
self._record_event(cursor, 'resolved', error_key, {
|
||||
'reason': reason,
|
||||
# Only explicit current-check callers attach this proof.
|
||||
# Generic resolve/cleanup remains neutral.
|
||||
'check_evidence': check_evidence,
|
||||
'incident': {'id': row[1], 'first_seen': row[2], 'resolved_at': row[3]},
|
||||
'entity': self._entity_from_details(stored_details),
|
||||
'details': stored_details or {},
|
||||
})
|
||||
|
||||
conn.commit()
|
||||
|
||||
def get_recovery_evidence(self, error_key: str, first_seen: str):
|
||||
"""Return fresh same-incident native check proof, never absence of errors.
|
||||
|
||||
Host CPU and exact per-service active checks carry provenance. Other
|
||||
checks, generic clears, excluded/deleted records and legacy events stay
|
||||
neutral until they have equivalent per-condition provenance.
|
||||
"""
|
||||
if not isinstance(error_key, str) or not first_seen or not (
|
||||
error_key == 'cpu_usage' or error_key.startswith('pve_service_')):
|
||||
return None
|
||||
try:
|
||||
# One SQLite statement is one consistent row/ack/closure snapshot.
|
||||
# Latest native observation/closure by durable event id: a later
|
||||
# abnormal record supersedes proof even when a resolved row is
|
||||
# reused and wall-clock time moves backward. No-op clears create
|
||||
# no event, so they do not invalidate a genuine closure.
|
||||
with self._db_lock, self._db_connection() as conn:
|
||||
row = conn.execute('''
|
||||
SELECT e.first_seen, e.last_seen, e.resolved_at, e.acknowledged,
|
||||
e.id, v.timestamp, v.data
|
||||
FROM errors e JOIN events v ON v.id = (
|
||||
SELECT id FROM events WHERE error_key = e.error_key
|
||||
AND event_type IN ('resolved', 'cleared', 'new', 'updated', 'escalated')
|
||||
ORDER BY id DESC LIMIT 1
|
||||
) WHERE e.error_key = ?
|
||||
''', (error_key,)).fetchone()
|
||||
if not row or row[0] != first_seen or not row[2] or row[3]:
|
||||
return None
|
||||
event_data = json.loads(row[6])
|
||||
if event_data.get('incident') != {'id': row[4], 'first_seen': row[0], 'resolved_at': row[2]}:
|
||||
return None
|
||||
proof = event_data.get('check_evidence')
|
||||
from health_recovery import valid_check_evidence
|
||||
if not valid_check_evidence(error_key, proof, now=datetime.now().timestamp()):
|
||||
return None
|
||||
if error_key == 'cpu_usage' and proof.get('policy') != event_data.get('details', {}).get('cpu_policy'):
|
||||
return None
|
||||
checked = float(proof['checked_at'])
|
||||
last_seen = datetime.fromisoformat(row[1]).timestamp()
|
||||
resolved = datetime.fromisoformat(row[2]).timestamp()
|
||||
recorded = datetime.fromisoformat(row[5]).timestamp()
|
||||
if not last_seen <= checked <= resolved <= recorded:
|
||||
return None
|
||||
return proof
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
return None
|
||||
|
||||
def is_error_active(self, error_key: str, category: Optional[str] = None) -> bool:
|
||||
"""
|
||||
Check if an error is currently active OR suppressed (dismissed but within suppression period).
|
||||
@@ -928,7 +859,7 @@ class HealthPersistence:
|
||||
|
||||
return False
|
||||
|
||||
def clear_error(self, error_key: str, *, check_evidence=None):
|
||||
def clear_error(self, error_key: str):
|
||||
"""
|
||||
Remove/resolve a specific error immediately.
|
||||
Used when the condition that caused the error no longer exists
|
||||
@@ -950,7 +881,7 @@ class HealthPersistence:
|
||||
|
||||
# Check if this error was acknowledged (dismissed)
|
||||
cursor.execute('''
|
||||
SELECT acknowledged, id, first_seen FROM errors WHERE error_key = ?
|
||||
SELECT acknowledged FROM errors WHERE error_key = ?
|
||||
''', (error_key,))
|
||||
row = cursor.fetchone()
|
||||
|
||||
@@ -969,10 +900,7 @@ class HealthPersistence:
|
||||
''', (now, error_key))
|
||||
|
||||
if cursor.rowcount > 0:
|
||||
self._record_event(cursor, 'cleared', error_key, {
|
||||
'reason': 'condition_resolved', 'check_evidence': check_evidence,
|
||||
'incident': {'id': row[1], 'first_seen': row[2], 'resolved_at': now},
|
||||
})
|
||||
self._record_event(cursor, 'cleared', error_key, {'reason': 'condition_resolved'})
|
||||
|
||||
conn.commit()
|
||||
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
"""Bounded recovery metadata admission, without importing monitor singletons.
|
||||
|
||||
Native persistence additionally binds the exact incident and closure. Manual
|
||||
notifications remain authenticated caller assertions: shape validation cannot
|
||||
establish that an asserted measurement actually happened.
|
||||
"""
|
||||
import math
|
||||
import time
|
||||
from typing import TypeGuard
|
||||
|
||||
|
||||
def _finite_number(value) -> TypeGuard[int | float]:
|
||||
try:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
||||
except (OverflowError, ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def valid_check_evidence(error_key, proof, *, now=None):
|
||||
"""Validate a supported measurement contract, not its external authenticity."""
|
||||
if not isinstance(proof, dict) or proof.get('check') != error_key:
|
||||
return False
|
||||
checked = proof.get('checked_at')
|
||||
now = time.time() if now is None else now
|
||||
if not _finite_number(checked) or not _finite_number(now) or not 0 <= now-checked <= 7200:
|
||||
return False
|
||||
if error_key == 'cpu_usage':
|
||||
policy = proof.get('policy')
|
||||
if not isinstance(policy, dict) or set(policy) != {'warning', 'critical', 'recovery'}:
|
||||
return False
|
||||
if not all(_finite_number(v) and 1 <= v <= 100 for v in policy.values()):
|
||||
return False
|
||||
if policy['warning'] > policy['critical']:
|
||||
return False
|
||||
value, maximum, count = proof.get('value'), proof.get('max_sample'), proof.get('normal_samples')
|
||||
return (_finite_number(value) and _finite_number(maximum)
|
||||
and 0 <= value <= maximum < min(policy['warning'], policy['recovery'])
|
||||
and isinstance(count, int) and not isinstance(count, bool) and count >= 10)
|
||||
if isinstance(error_key, str) and error_key.startswith('pve_service_'):
|
||||
service = error_key[len('pve_service_'):]
|
||||
return (bool(service) and proof.get('service') == service
|
||||
and proof.get('state') == 'active'
|
||||
and type(proof.get('returncode')) is int and proof['returncode'] == 0)
|
||||
return False
|
||||
|
||||
|
||||
def presents_recovery(data):
|
||||
"""One presentation predicate shared by template, icon and email badge."""
|
||||
return (isinstance(data, dict) and data.get('recovery_outcome') == 'resolved'
|
||||
and data.get('is_recovery') is True
|
||||
and valid_check_evidence(data.get('error_key'), data.get('check_evidence')))
|
||||
@@ -1037,15 +1037,7 @@ class EmailChannel(NotificationChannel):
|
||||
|
||||
# Determine group for section header
|
||||
event_type = data.get('_event_type', '')
|
||||
if event_type == 'error_resolved':
|
||||
from health_recovery import presents_recovery
|
||||
if presents_recovery(data):
|
||||
sev.update(self._SEV_STYLE['OK'])
|
||||
sev['label'] = _runtime_notification_text('healthRecovery.status', data)
|
||||
else:
|
||||
sev.update(self._SEV_DEFAULT)
|
||||
sev['label'] = _runtime_text('email.severity.observation', data)
|
||||
elif event_type == 'backup_complete':
|
||||
if event_type == 'backup_complete':
|
||||
outcome = data.get('backup_outcome')
|
||||
if outcome == 'confirmed':
|
||||
sev.update(self._SEV_STYLE['OK'])
|
||||
@@ -1064,13 +1056,12 @@ class EmailChannel(NotificationChannel):
|
||||
# Scoped inline mail-compatible wrapping for authoritative raw-context
|
||||
# bodies, including restore bodies released from quiet hours.
|
||||
backup_email = event_type in {'backup_complete', 'backup_fail'}
|
||||
wrap_body = (event_type in {'temp_high', 'system_restore_completed', 'error_resolved'}
|
||||
wrap_body = (event_type in {'temp_high', 'system_restore_completed'}
|
||||
or backup_email or data.get('_restore_summary') or data.get('_backup_summary'))
|
||||
temp_cell_wrap = 'word-wrap:break-word;overflow-wrap:break-word;word-break:break-word;' if wrap_body else ''
|
||||
temp_table_layout = 'table-layout:fixed;' if wrap_body else ''
|
||||
# Recovery exposes the same literal host context as backup notices.
|
||||
# Keep wrapping event-scoped; unrelated mail remains byte-identical.
|
||||
context_email = backup_email or event_type == 'error_resolved'
|
||||
context_email = backup_email
|
||||
backup_title_wrap = temp_cell_wrap if context_email else ''
|
||||
backup_metadata_layout = 'table-layout:fixed;' if context_email else ''
|
||||
section_label = _runtime_text(f'email.groups.{group}', data)
|
||||
@@ -1106,12 +1097,11 @@ class EmailChannel(NotificationChannel):
|
||||
# A metadata-only/manual body may be generic. Keep actionable raw
|
||||
# context once, without restoring duplicated inventory metadata.
|
||||
reason = data.get('reason', '')
|
||||
if reason and len(reason) <= 80 and reason not in body:
|
||||
if backup_email and reason and len(reason) <= 80 and reason not in body:
|
||||
detail_rows.append((html_mod.escape(_runtime_text('email.fields.reason', data)),
|
||||
html_mod.escape(reason)))
|
||||
|
||||
if event_type in {'system_restore_completed', 'error_resolved'} or data.get('_restore_summary'):
|
||||
# Observation age/disappearance must not become a green OK row.
|
||||
if event_type == 'system_restore_completed' or data.get('_restore_summary'):
|
||||
# The endpoint's warnings_block and task counts live in the
|
||||
# localized body, not the generic services Event row.
|
||||
detail_rows = [('', html_mod.escape(line if data.get('_quiet_hours_summary') else line.strip()))
|
||||
@@ -1150,7 +1140,7 @@ class EmailChannel(NotificationChannel):
|
||||
reason = data.get('reason', '')
|
||||
reason_html = ''
|
||||
if reason and len(reason) > 80 and not (
|
||||
(event_type in {'temp_high', 'error_resolved'} or backup_email) and reason in body):
|
||||
(event_type == 'temp_high' or backup_email) and reason in body):
|
||||
reason_html = f'''
|
||||
<div style="margin:16px 0 0;padding:12px 16px;border:1px solid #d1d5db;border-radius:6px;">
|
||||
<p style="margin:0 0 4px;font-size:11px;font-weight:600;color:#374151;text-transform:uppercase;letter-spacing:0.05em;">{_runtime_text('email.details', data)}</p>
|
||||
|
||||
@@ -3223,13 +3223,6 @@ class PollingCollector:
|
||||
self._last_notified.pop(key, None)
|
||||
continue
|
||||
|
||||
# Disappearance is not recovery. Only same-incident proof written
|
||||
# by a successful existing native check can certify normality.
|
||||
try:
|
||||
recovery_evidence = health_persistence.get_recovery_evidence(key, first_seen)
|
||||
except Exception:
|
||||
recovery_evidence = None
|
||||
|
||||
# Calculate duration
|
||||
duration = ''
|
||||
if first_seen:
|
||||
@@ -3261,7 +3254,7 @@ class PollingCollector:
|
||||
reason_lines = (reason or '').split('\n')
|
||||
reason_summary = reason_lines[0] if reason_lines else ''
|
||||
|
||||
# Keep the earlier device context without asserting recovery.
|
||||
# Try to extract device info for a clean "Device: xxx (recovered)" line
|
||||
device_line = ''
|
||||
for line in reason_lines:
|
||||
if 'Device:' in line or 'Device not currently' in line or '/dev/' in line:
|
||||
@@ -3274,15 +3267,12 @@ class PollingCollector:
|
||||
break
|
||||
|
||||
if reason_summary and device_line:
|
||||
clean_reason = f'{reason_summary}\n{device_line} (no longer reported)'
|
||||
clean_reason = f'{reason_summary}\n{device_line} (recovered)'
|
||||
elif reason_summary:
|
||||
clean_reason = f'{reason_summary} (no longer reported)'
|
||||
clean_reason = f'{reason_summary} (recovered)'
|
||||
else:
|
||||
clean_reason = 'Condition no longer reported'
|
||||
clean_reason = 'Condition resolved'
|
||||
|
||||
if recovery_evidence:
|
||||
clean_reason = reason_summary
|
||||
|
||||
# `original_severity` must match what the user actually saw
|
||||
# in the most-recent notification for this error, not the
|
||||
# latest DB severity. See `_notified_severity` docstring at
|
||||
@@ -3309,9 +3299,7 @@ class PollingCollector:
|
||||
'original_severity': original_severity,
|
||||
'first_seen': first_seen,
|
||||
'duration': duration_label,
|
||||
'is_recovery': bool(recovery_evidence),
|
||||
'recovery_outcome': 'resolved' if recovery_evidence else 'no_longer_reported',
|
||||
'check_evidence': recovery_evidence,
|
||||
'is_recovery': True,
|
||||
}
|
||||
# Spread the original details blob so the resolved notification
|
||||
# can use the same {storage_name}/{vm_name}/{device} placeholders
|
||||
|
||||
@@ -2012,7 +2012,9 @@ class NotificationManager:
|
||||
'digest.quietTitle', language, hostname=host, count=len(rows),
|
||||
)
|
||||
use_icons = self._config.get(f'{ch_name}.rich_format', 'false') == 'true'
|
||||
summary_body = self._compose_digest_body(rows, use_icons=use_icons, quiet_release=True)
|
||||
quiet_details = any(row[1] in ('backup_complete', 'backup_fail', 'system_restore_completed')
|
||||
for row in rows)
|
||||
summary_body = self._compose_digest_body(rows, use_icons=use_icons, quiet_release=quiet_details)
|
||||
|
||||
result: dict = {'success': False, 'error': ''}
|
||||
try:
|
||||
@@ -2498,7 +2500,7 @@ class NotificationManager:
|
||||
runtime_data.setdefault('_notification_language', self._notification_language())
|
||||
# Match queued dispatch's presentation context for these outcome
|
||||
# notices; this does not alter event/severity or direct-send policy.
|
||||
if event_type in ('backup_complete', 'backup_fail', 'error_resolved', 'system_restore_completed'):
|
||||
if event_type in ('backup_complete', 'backup_fail', 'system_restore_completed'):
|
||||
runtime_data['_event_type'] = event_type
|
||||
runtime_data['_group'] = TEMPLATES[event_type].get('group', 'other')
|
||||
|
||||
|
||||
@@ -817,10 +817,10 @@ TEMPLATES = {
|
||||
# `{entity}` is populated by health_persistence.resolve_error()
|
||||
# (via _entity_from_details) and by PollingCollector's spread of
|
||||
# the original details blob. When absent, _SafeDict elides the
|
||||
# placeholder and the title collapses back to "No longer reported - <cat>"
|
||||
# placeholder and the title collapses back to "Resolved - <cat>"
|
||||
# without a trailing dash.
|
||||
'title': '{hostname}: No longer reported - {category}{entity_suffix}',
|
||||
'body': 'The {category} issue is no longer in active health records.\n{reason}\n\U0001F6A6 Previous severity: {original_severity}\n\u23F1\uFE0F Time since first observation: {duration}',
|
||||
'title': '{hostname}: Resolved - {category}{entity_suffix}',
|
||||
'body': 'The {category} issue has been resolved.\n{reason}\n\U0001F6A6 Previous severity: {original_severity}\n\u23F1\uFE0F Duration: {duration}',
|
||||
'label': 'Recovery notification',
|
||||
'group': 'health',
|
||||
'default_enabled': True,
|
||||
@@ -1037,8 +1037,8 @@ TEMPLATES = {
|
||||
'default_enabled': False,
|
||||
},
|
||||
'backup_complete': {
|
||||
'title': '{hostname}: Backup outcome unconfirmed',
|
||||
'body': 'The backup outcome could not be confirmed from this notice.',
|
||||
'title': '{hostname} → {storage}: Backup complete — {vmname} ({vmid})',
|
||||
'body': 'Backup of {vmname} (ID: {vmid}) completed successfully on {storage}.\nSize: {size}',
|
||||
'label': 'Backup complete',
|
||||
'group': 'backup',
|
||||
'default_enabled': True,
|
||||
@@ -1308,7 +1308,8 @@ TEMPLATES = {
|
||||
'Stale node dirs removed: {stale_nodes}\n'
|
||||
'Components reinstalled: {components}\n'
|
||||
'Duration: {duration}\n'
|
||||
'{warnings_block}'
|
||||
'{warnings_block}\n'
|
||||
'The node is now fully ready to use.'
|
||||
),
|
||||
'label': 'Host restore completed',
|
||||
'group': 'services',
|
||||
@@ -1894,6 +1895,10 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
template[field] = localized
|
||||
backup_title_target = ''
|
||||
if event_type == 'backup_complete':
|
||||
template['title'] = runtime_message('backup.unconfirmedTitle', language,
|
||||
hostname=data.get('hostname') or _get_hostname()) or (
|
||||
str(data.get('hostname') or _get_hostname()) + ': Backup outcome unconfirmed')
|
||||
template['body'] = runtime_message('backup.unconfirmedBody', language) or 'The backup outcome is not confirmed.'
|
||||
outcome = data.get('backup_outcome')
|
||||
if outcome == 'confirmed':
|
||||
template['title'] = runtime_message('backup.confirmedTitle', language,
|
||||
@@ -1961,7 +1966,7 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
'log_file': '',
|
||||
}
|
||||
variables.update(data)
|
||||
if event_type == 'backup_fail' or (event_type == 'backup_complete' and data.get('backup_outcome') in ('confirmed', 'completed_with_warnings', 'failed')):
|
||||
if event_type in ('backup_fail', 'backup_complete'):
|
||||
# The provider has already substituted raw Display Names. Insert the
|
||||
# resolved title as a value, never reinterpret its literal braces.
|
||||
variables['_backup_title'] = template['title']
|
||||
@@ -2070,13 +2075,6 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
return ''
|
||||
|
||||
safe_vars = _SafeDict(variables)
|
||||
if event_type == 'error_resolved':
|
||||
from health_recovery import presents_recovery
|
||||
if presents_recovery(data):
|
||||
safe_vars['_health_title'] = runtime_message('healthRecovery.title', language, **variables)
|
||||
safe_vars['_health_body'] = runtime_message('healthRecovery.body', language, **variables)
|
||||
template['title'] = '{_health_title}'
|
||||
template['body'] = '{_health_body}'
|
||||
try:
|
||||
title = template['title'].format_map(safe_vars)
|
||||
except (ValueError, IndexError):
|
||||
@@ -2152,7 +2150,7 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
key = ('backup.errorBody' if data.get('backup_outcome') == 'failed'
|
||||
else 'backup.warningBody' if data.get('backup_outcome') == 'completed_with_warnings'
|
||||
else 'backup.unconfirmedBody')
|
||||
body_text = runtime_message(key, language) + '\n' + body_text
|
||||
body_text = (runtime_message(key, language) or template['body']) + '\n' + body_text
|
||||
elif event_type == 'system_mail' and pve_message:
|
||||
# System mail -- use PVE message directly (mail bounce, cron, smartd)
|
||||
body_text = pve_message.strip()[:1000]
|
||||
@@ -2181,14 +2179,19 @@ def render_template(event_type: str, data: Dict[str, Any],
|
||||
if event_type in ('backup_complete', 'backup_fail') and (
|
||||
event_type == 'backup_fail' or data.get('backup_outcome') == 'failed'):
|
||||
source_subject = str(data.get('pve_title') or '').strip()
|
||||
native_failure = re.fullmatch(
|
||||
r'vzdump backup status \([^\r\n]*\): backup failed(?::\s*(.*))?',
|
||||
source_subject, re.IGNORECASE)
|
||||
guest_context = (_parse_vzdump_message(str(pve_message or '')) or {}).get('vms')
|
||||
if guest_context:
|
||||
# Native single-line job errors live only in the subject. Retain
|
||||
# that cause, not the redundant job/host envelope or generic count.
|
||||
if native_failure:
|
||||
# Before the first guest, too, only the native cause is diagnostic;
|
||||
# the original host/job envelope is not display-name context.
|
||||
source_subject = (native_failure.group(1) or '').strip()
|
||||
elif guest_context:
|
||||
cause = re.search(r'\bbackup failed:\s*(.+)', source_subject, re.IGNORECASE)
|
||||
source_subject = cause.group(1).strip() if cause else ''
|
||||
if source_subject.lower() == 'multiple problems':
|
||||
source_subject = ''
|
||||
if source_subject.lower() == 'multiple problems':
|
||||
source_subject = ''
|
||||
if source_subject and source_subject not in {line.strip() for line in body_text.splitlines()}:
|
||||
# Reserve the subject-equivalent diagnostic BEFORE the cap. Finding
|
||||
# it in uncapped logs is not enough: that late line could be omitted.
|
||||
@@ -2374,14 +2377,14 @@ EVENT_EMOJI = {
|
||||
'system_startup': '\U0001F680', # rocket (startup)
|
||||
'system_shutdown': '\u23FB\uFE0F', # power symbol (Unicode)
|
||||
'system_reboot': '\U0001F504',
|
||||
'system_restore_completed': '\U0001F4CB', # post-restore task report (boot may have warnings)
|
||||
'system_restore_completed': '✅', # check mark
|
||||
'system_problem': '\u26A0\uFE0F',
|
||||
'kernel_warning': '\u26A0\uFE0F',
|
||||
'service_fail': '\u274C',
|
||||
'oom_kill': '\U0001F4A3', # bomb
|
||||
# Health
|
||||
'new_error': '\U0001F198', # SOS
|
||||
'error_resolved': '\U0001F4CB', # no longer active in health records, not proven recovery
|
||||
'error_resolved': '\u2705',
|
||||
'error_escalated': '\U0001F53A', # red triangle up
|
||||
'health_degraded': '\u26A0\uFE0F',
|
||||
'health_persistent': '\U0001F4CB', # clipboard
|
||||
@@ -2533,10 +2536,6 @@ def enrich_with_emojis(event_type: str, title: str, body: str,
|
||||
severity = data.get('severity', 'INFO')
|
||||
|
||||
icon = EVENT_EMOJI.get(event_type) or CATEGORY_EMOJI.get(group) or SEVERITY_ICONS.get(severity, '')
|
||||
if event_type == 'error_resolved':
|
||||
from health_recovery import presents_recovery
|
||||
if presents_recovery(data):
|
||||
icon = '✅'
|
||||
if event_type == 'backup_complete':
|
||||
icon = {
|
||||
'confirmed': '💾✅', 'completed_with_warnings': '💾⚠️', 'failed': '💾❌',
|
||||
|
||||
@@ -50,10 +50,6 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
self.assertIsInstance(templates[event_type][field], str)
|
||||
self.assertTrue(templates[event_type][field])
|
||||
if field in source:
|
||||
# Upstream Slovak backup title/body still belong to
|
||||
# the pre-outcome schema; runtime falls back to EN.
|
||||
if language == 'sk' and event_type == 'backup_complete' and field != 'label':
|
||||
continue
|
||||
self.assertEqual(
|
||||
_placeholders(templates[event_type][field]),
|
||||
_placeholders(source[field]),
|
||||
@@ -74,10 +70,9 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
en = flatten(self.catalogs["en"])
|
||||
pending_slovak = {"backup.confirmedTitle", "backup.confirmedBody",
|
||||
"backup.errorTitle", "backup.errorBody", "backup.unconfirmedBody",
|
||||
"channels.email.severity.observation", "channels.email.status.unconfirmed",
|
||||
"backup.unconfirmedTitle", "channels.email.status.unconfirmed",
|
||||
"backup.warningTitle", "backup.warningBody", "backup.diagnosticsOmitted",
|
||||
"channels.email.status.completed_with_warnings",
|
||||
"healthRecovery.title", "healthRecovery.body", "healthRecovery.status"}
|
||||
"channels.email.status.completed_with_warnings"}
|
||||
for language, catalog in self.catalogs.items():
|
||||
translated = flatten(catalog)
|
||||
if language == 'sk':
|
||||
@@ -90,9 +85,6 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
for key in translated:
|
||||
self.assertIsInstance(translated[key], str, f"{language}:{key}")
|
||||
self.assertTrue(translated[key].strip(), f"{language}:{key}")
|
||||
if language == 'sk' and key in ('templates.backup_complete.title',
|
||||
'templates.backup_complete.body'):
|
||||
continue # exact upstream SK, superseded only at render time
|
||||
self.assertEqual(_placeholders(translated[key]), _placeholders(en[key]), f"{language}:{key}")
|
||||
|
||||
def test_notification_language_ui_keys_exist_in_both_catalogs(self):
|
||||
@@ -132,6 +124,11 @@ class RuntimeCatalogTests(unittest.TestCase):
|
||||
with mock.patch.object(notification_templates, "_get_hostname", return_value="HOST-ŽILINA"):
|
||||
for event_type in notification_templates.TEMPLATES:
|
||||
event_values = dict(values)
|
||||
if event_type == "backup_complete":
|
||||
# Legacy catalog fields keep their completion meaning;
|
||||
# the new unknown-outcome keys intentionally omit guest data.
|
||||
# Exercise retained metadata on the explicit confirmed path.
|
||||
event_values["backup_outcome"] = "confirmed"
|
||||
if event_type == "temp_high":
|
||||
# Temperature is a measured numeric contract; arbitrary
|
||||
# DYNAMIC_VALUE is correctly rejected by its fallback.
|
||||
|
||||
Reference in New Issue
Block a user