mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-08 22:46:41 +00:00
Fix principal cause caps and bind measured recovery to incident policy
This commit is contained in:
martino
@@ -0,0 +1,86 @@
|
||||
"""Pinned, inert recovery review. No operational module imports or threads.
|
||||
Run with python3 -S under bwrap --unshare-net; DBs and export use TMPDIR.
|
||||
"""
|
||||
import ast, contextlib, datetime, hashlib, io, json, os, pathlib, sqlite3, subprocess, sys, tarfile, tempfile, threading, types, typing
|
||||
from unittest.mock import patch
|
||||
REPO=pathlib.Path('/home/martino/projects/proxmox/notification-maintainer-followup')
|
||||
BASE=datetime.datetime(2026,9,30,20,0,0).timestamp()
|
||||
class Clock(datetime.datetime):
|
||||
epoch=BASE
|
||||
tick=0.0
|
||||
@classmethod
|
||||
def now(cls,tz=None):
|
||||
value=cls.fromtimestamp(cls.epoch,tz)
|
||||
cls.epoch+=cls.tick
|
||||
return value
|
||||
TIME=types.SimpleNamespace(time=lambda:Clock.epoch)
|
||||
|
||||
def extract(path,name,owner,ns):
|
||||
tree=ast.parse(path.read_text())
|
||||
nodes=tree.body if owner is None else next(n.body for n in tree.body if isinstance(n,ast.ClassDef) and n.name==owner)
|
||||
node=next(n for n in nodes if isinstance(n,(ast.FunctionDef,ast.AsyncFunctionDef)) and n.name==name)
|
||||
node.decorator_list=[]
|
||||
exec(compile(ast.Module(body=[node],type_ignores=[]),str(path),'exec'),ns)
|
||||
return ns[name]
|
||||
|
||||
from notification_fixture import SCRIPTS as scripts
|
||||
pns=dict(vars(typing),datetime=Clock,timedelta=datetime.timedelta,json=json,sqlite3=sqlite3,contextmanager=contextlib.contextmanager,re=__import__('re'),_re_disk_base=__import__('re'))
|
||||
pns['disk_base_name']=extract(scripts/'health_persistence.py','disk_base_name',None,pns)
|
||||
methods={n:extract(scripts/'health_persistence.py',n,'HealthPersistence',pns) for n in ('_get_conn','_db_connection','_init_database','record_error','_record_error_impl','resolve_error','_resolve_error_impl','get_recovery_evidence','_record_event','_entity_from_details','clear_error','get_active_errors','is_error_active','is_error_acknowledged','_get_setting_impl','get_setting','set_setting','acknowledge_error','_acknowledge_error_impl','get_excluded_interface_names')}
|
||||
def make_store(directory):
|
||||
s=types.SimpleNamespace(db_path=pathlib.Path(directory)/'health.sqlite',_db_lock=threading.RLock(),DEFAULT_SUPPRESSION_HOURS=24,CATEGORY_SETTING_MAP={})
|
||||
for n,f in methods.items():
|
||||
if n=='_entity_from_details':setattr(s,n,f)
|
||||
elif n=='_db_connection':setattr(s,n,types.MethodType(contextlib.contextmanager(f),s))
|
||||
else:setattr(s,n,types.MethodType(f,s))
|
||||
s._init_database()
|
||||
return s
|
||||
def sql(s,query,args=()):
|
||||
with s._db_connection() as c:
|
||||
data=c.execute(query,args).fetchall();c.commit();return data
|
||||
def record(s,key='cpu_usage',category='cpu',reason='CPU high',details=None):
|
||||
Clock.epoch=BASE-600
|
||||
with patch.dict(sys.modules,{'os':types.SimpleNamespace(path=types.SimpleNamespace(exists=lambda p:True))}):s.record_error(key,category,'WARNING',reason,details)
|
||||
Clock.epoch=BASE-10
|
||||
with patch.dict(sys.modules,{'os':types.SimpleNamespace(path=types.SimpleNamespace(exists=lambda p:True))}):s.record_error(key,category,'WARNING',reason,details)
|
||||
Clock.epoch=BASE
|
||||
assert s.get_active_errors()
|
||||
return sql(s,'SELECT first_seen FROM errors WHERE error_key=?',(key,))[0][0]
|
||||
def cpu(s,current=20,history=None,warning=85,critical=95):
|
||||
ns=dict(vars(typing),time=TIME,os=types.SimpleNamespace(cpu_count=lambda:4),health_persistence=s,psutil=types.SimpleNamespace(cpu_percent=lambda **kw:current,cpu_count=lambda:4))
|
||||
fn=extract(scripts/'health_monitor.py','_check_cpu_with_hysteresis','HealthMonitor',ns)
|
||||
if history is None:history=[{'value':20,'time':BASE-i*10} for i in range(1,11)]
|
||||
target=types.SimpleNamespace(state_history={'cpu_usage':list(history)},CPU_WARNING=85,CPU_CRITICAL=95,CPU_RECOVERY=75,CPU_WARNING_DURATION=300,CPU_CRITICAL_DURATION=300,CPU_RECOVERY_DURATION=120,_check_cpu_temperature=lambda:None)
|
||||
refresh=extract(scripts/'health_monitor.py','_refresh_thresholds','HealthMonitor',{})
|
||||
with patch.dict(sys.modules,{'health_thresholds':types.SimpleNamespace(get=lambda section,key:({'warning':warning,'critical':critical}.get(key) if section=='cpu' else None))}):refresh(target)
|
||||
return fn(target)
|
||||
def poll(s,first,key='cpu_usage',category='cpu',reason='CPU high',details=None,first_done=True,foreign=False,restored=False):
|
||||
events=[]
|
||||
meta={'category':category,'reason':reason,'severity':'WARNING','first_seen':first,'details':details}
|
||||
c=types.SimpleNamespace(_hostname='node-a',_ENTITY_MAP={'cpu':('node',''),'pve_services':('node',''),'network':('node','')},_first_poll_done=first_done,_known_errors={key:meta},_notified_severity={key:'WARNING'},_last_notified={key:BASE-1},SAME_ERROR_COOLDOWN=86400,_get_cooldown_from_db=lambda *a:BASE-1,_queue=types.SimpleNamespace(put=events.append),_guest_storage_error_is_now_foreign=lambda *a:foreign,_save_known_errors_meta=lambda:None)
|
||||
ns=dict(vars(typing),time=TIME,json=json,re=__import__('re'),NotificationEvent=lambda *a,**kw:types.SimpleNamespace(event_type=a[0],severity=a[1],data=a[2],**kw),startup_grace=types.SimpleNamespace(should_suppress_category=lambda *a:False))
|
||||
c._guest_storage_error_is_now_foreign=extract(scripts/'notification_events.py','_guest_storage_error_is_now_foreign','PollingCollector',ns)
|
||||
fn=extract(scripts/'notification_events.py','_check_persistent_health','PollingCollector',ns)
|
||||
with patch.dict(sys.modules,{'health_persistence':types.SimpleNamespace(health_persistence=s),'flask_server':types.SimpleNamespace(get_proxmox_node_name=lambda:'node-a',get_cached_pvesh_cluster_resources_vm=lambda:[{'vmid':100,'type':'lxc','node':'node-b' if foreign else 'node-a'}]),'datetime':types.SimpleNamespace(**{**vars(datetime),'datetime':Clock})}):
|
||||
if restored:
|
||||
c._KNOWN_ERRORS_SETTING_KEY='pollingcollector_known_errors_v1'
|
||||
s.set_setting(c._KNOWN_ERRORS_SETTING_KEY,json.dumps(c._known_errors));c._known_errors={}
|
||||
extract(scripts/'notification_events.py','_load_known_errors_meta','PollingCollector',ns)(c)
|
||||
c._first_poll_done=bool(c._known_errors)
|
||||
fn(c)
|
||||
return [e.data for e in events],c
|
||||
def service(store, rc=0, stdout='active\n', raised=False, services=('pvedaemon',), clustered=False):
|
||||
calls=[]
|
||||
def run(argv, **kw):
|
||||
calls.append((argv, kw))
|
||||
assert argv[:2] == ['systemctl', 'is-active']
|
||||
if raised: raise TimeoutError('inert timeout')
|
||||
return types.SimpleNamespace(returncode=rc, stdout=stdout)
|
||||
ns=dict(vars(typing),time=TIME,os=types.SimpleNamespace(path=types.SimpleNamespace(exists=lambda p:clustered)),subprocess=types.SimpleNamespace(run=run),health_persistence=store)
|
||||
result=extract(scripts/'health_monitor.py','_check_pve_services','HealthMonitor',ns)(types.SimpleNamespace(PVE_SERVICES=list(services)))
|
||||
return result,calls
|
||||
|
||||
@contextlib.contextmanager
|
||||
def case():
|
||||
Clock.epoch=BASE
|
||||
with tempfile.TemporaryDirectory(prefix='recovery-review-db-',dir=os.environ['TMPDIR']) as d:yield make_store(d)
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Native initializer, measurement methods, SQL writers/readers and collector."""
|
||||
import unittest
|
||||
from notification_recovery_fixture import case, Clock, BASE, cpu, sql, poll
|
||||
|
||||
class RecoveryCorrectionTests(unittest.TestCase):
|
||||
def test_supported_low_warning_current_violation_is_neutral(self):
|
||||
with case() as store:
|
||||
Clock.epoch = BASE - 600
|
||||
initial = cpu(store, 80, [{'value':80, 'time':Clock.epoch-i*5} for i in range(1,26)], warning=50)
|
||||
self.assertEqual(initial['status'], 'WARNING')
|
||||
first = sql(store, 'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch = BASE
|
||||
result = cpu(store, 60, [{'value':60, 'time':BASE-i*5} for i in range(1,26)], warning=50)
|
||||
events, _ = poll(store, first, reason=initial['reason'])
|
||||
# Preserve operational clear/hysteresis behavior, not its factual claim.
|
||||
self.assertEqual(result['status'], 'OK')
|
||||
self.assertFalse(events[0]['is_recovery'], events)
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage', first))
|
||||
|
||||
def test_original_policy_survives_repeated_native_updates(self):
|
||||
import json
|
||||
for later_warning in (85, 40):
|
||||
with self.subTest(later_warning=later_warning), case() as store:
|
||||
Clock.epoch = BASE - 600
|
||||
cpu(store, 80, [{'value':80, 'time':Clock.epoch-i*5} for i in range(1,26)], warning=50)
|
||||
first = sql(store, 'SELECT first_seen FROM errors')[0][0]
|
||||
for step in (400, 200):
|
||||
Clock.epoch = BASE-step
|
||||
cpu(store, 90, [{'value':90, 'time':Clock.epoch-i*5} for i in range(1,26)], warning=later_warning)
|
||||
original = json.loads(sql(store, 'SELECT details FROM errors')[0][0])['cpu_policy']
|
||||
self.assertEqual(original['warning'], 50)
|
||||
Clock.epoch = BASE
|
||||
cpu(store, 20, warning=later_warning)
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage', first))
|
||||
self.assertFalse(poll(store, first)[0][0]['is_recovery'])
|
||||
|
||||
def test_clock_rollback_new_row_generic_clear_cannot_inherit_old_proof(self):
|
||||
for rollback, reuse_first in ((True,False),(False,False),(True,True)):
|
||||
with self.subTest(rollback=rollback,reuse_first=reuse_first),case() as store:
|
||||
Clock.epoch = BASE-600
|
||||
cpu(store, 90, [{'value':90, 'time':Clock.epoch-i*5} for i in range(1,26)])
|
||||
first = sql(store, 'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch = BASE; Clock.tick = .001
|
||||
try: cpu(store)
|
||||
finally: Clock.tick = 0
|
||||
self.assertTrue(store.get_recovery_evidence('cpu_usage', first))
|
||||
store.acknowledge_error('cpu_usage', suppression_hours=-1)
|
||||
store.clear_error('cpu_usage')
|
||||
Clock.epoch = BASE-100 if rollback else BASE+1
|
||||
store.record_error('cpu_usage','cpu','WARNING','new incident after clock step',{})
|
||||
second = sql(store, 'SELECT first_seen FROM errors')[0][0]
|
||||
self.assertNotEqual(first, second)
|
||||
if reuse_first:
|
||||
# Restored malformed snapshot with reused wall-clock identity;
|
||||
# native row id and latest closure still prevent replay.
|
||||
sql(store,'UPDATE errors SET first_seen=?',(first,));second=first
|
||||
Clock.epoch = BASE+.0005 if rollback else BASE+2
|
||||
store.clear_error('cpu_usage'); Clock.epoch = BASE+3
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage', second))
|
||||
self.assertFalse(poll(store, second)[0][0]['is_recovery'])
|
||||
|
||||
def test_malformed_native_and_manual_proof_is_neutral_at_all_consumers(self):
|
||||
import copy, json, time
|
||||
from unittest.mock import patch
|
||||
from notification_fixture import LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
with case() as store:
|
||||
Clock.epoch = BASE-600
|
||||
cpu(store, 90, [{'value':90,'time':Clock.epoch-i*5} for i in range(1,26)])
|
||||
first = sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch = BASE; cpu(store)
|
||||
event_data = json.loads(sql(store,"SELECT data FROM events WHERE event_type='resolved'")[0][0])
|
||||
data = poll(store,first)[0][0]
|
||||
for field, bad in [('checked_at','bad'),('checked_at',True),('checked_at',float('inf')),
|
||||
('checked_at',10**400),('value',60),('max_sample',float('nan')),
|
||||
('normal_samples',True),('normal_samples',9),('checked_at',BASE+1),('checked_at',BASE-7201),
|
||||
('policy',{'warning':False,'critical':95,'recovery':75}),
|
||||
('policy',{'warning':96,'critical':95,'recovery':75}),
|
||||
('policy',{'warning':85,'critical':95}),
|
||||
('policy',{'warning':85,'critical':95,'recovery':float('nan')}),
|
||||
('policy',{'warning':85,'critical':95,'recovery':75,'extra':0})]:
|
||||
broken = copy.deepcopy(event_data)
|
||||
broken['check_evidence'][field] = bad
|
||||
if field == 'value': broken['check_evidence']['policy']['warning'] = 50
|
||||
sql(store,"UPDATE events SET data=? WHERE event_type='resolved'",(json.dumps(broken),))
|
||||
with self.subTest(field=field,bad=str(bad)),patch('health_recovery.time.time',return_value=BASE):
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage',first))
|
||||
for language in LANGUAGES:
|
||||
for manual in (False,True):
|
||||
result = deliver('error_resolved',{**data,'check_evidence':broken['check_evidence']},'OK',language,manual=manual)
|
||||
self.assertNotIn('background:#f0fdf4;', result['html'])
|
||||
# Caller content is trusted, not authenticated native proof; even
|
||||
# well-shaped assertions require a valid time/type/numeric contract.
|
||||
for proof in ({'check':'cpu_usage'}, {'check':'cpu_usage','checked_at':time.time()+1}):
|
||||
self.assertNotIn('background:#f0fdf4;', deliver('error_resolved',{**data,'check_evidence':proof},'OK',manual=True)['html'])
|
||||
|
||||
def test_exact_service_active_native_clear_reaches_recovery_consumers(self):
|
||||
from unittest.mock import patch
|
||||
from notification_recovery_fixture import service
|
||||
from notification_fixture import LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
with case() as store:
|
||||
Clock.epoch = BASE-600
|
||||
service(store,3,'inactive\n')
|
||||
first = sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch = BASE
|
||||
result, calls = service(store)
|
||||
self.assertEqual(result['status'],'OK')
|
||||
self.assertEqual(calls,[(['systemctl','is-active','pvedaemon'], {'capture_output':True,'text':True,'timeout':2})])
|
||||
proof = store.get_recovery_evidence('pve_service_pvedaemon',first)
|
||||
self.assertTrue(proof)
|
||||
data = poll(store,first,'pve_service_pvedaemon','pve_services','PVE service pvedaemon is inactive',details={'service':'pvedaemon'})[0][0]
|
||||
self.assertTrue(data['is_recovery'])
|
||||
with patch('health_recovery.time.time',return_value=BASE):
|
||||
for language in LANGUAGES:
|
||||
for manual in (False,True):
|
||||
rendered = deliver('error_resolved',data,'OK',language,manual=manual)
|
||||
self.assertIn('background:#f0fdf4;',rendered['html'])
|
||||
self.assertIn('pvedaemon', rendered['text'])
|
||||
|
||||
def test_cpu_positive_default_low_policy_and_neutral_history_controls(self):
|
||||
from notification_recovery_fixture import record
|
||||
for warning,current,history,legacy,expected in (
|
||||
(85,20,None,False,True), (50,20,None,False,True),
|
||||
(85,20,None,True,False), (85,99,[],False,False),
|
||||
(85,20,[],False,False),
|
||||
(85,20,[{'value':20,'time':BASE+i*5} for i in range(1,10)],False,False),
|
||||
(85,20,[{'value':20,'time':BASE-121-i} for i in range(12)],False,False),
|
||||
(85,99,None,False,False),
|
||||
(85,float('nan'),None,False,False), (85,float('inf'),None,False,False),
|
||||
(85,True,None,False,False), (85,10**400,None,False,False)):
|
||||
with self.subTest(warning=warning,current=str(current),legacy=legacy), case() as store:
|
||||
if legacy: first = record(store)
|
||||
else:
|
||||
Clock.epoch=BASE-600
|
||||
cpu(store,90,[{'value':90,'time':Clock.epoch-i*5} for i in range(1,26)],warning=warning)
|
||||
first=sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch=BASE; cpu(store,current,history,warning=warning)
|
||||
self.assertEqual(bool(store.get_recovery_evidence('cpu_usage',first)),expected)
|
||||
|
||||
def test_service_unavailable_removed_overall_ok_and_ack_controls(self):
|
||||
from notification_recovery_fixture import service, record
|
||||
for rc,stdout,raised in ((3,'inactive\n',False),(4,'unknown\n',False),(0,'active extra\n',False),(1,'active\n',False),(0,'',True)):
|
||||
with self.subTest(rc=rc,stdout=stdout,raised=raised),case() as store:
|
||||
Clock.epoch=BASE-600; service(store,3,'inactive\n')
|
||||
first=sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch=BASE; service(store,rc,stdout,raised)
|
||||
self.assertIsNone(store.get_recovery_evidence('pve_service_pvedaemon',first))
|
||||
self.assertFalse(poll(store,first,'pve_service_pvedaemon','pve_services')[0])
|
||||
for removed in ('pvedaemon','corosync'):
|
||||
with self.subTest(removed=removed),case() as store:
|
||||
first=record(store,'pve_service_'+removed,'pve_services','service inactive',{'service':removed})
|
||||
result,calls=service(store,services=(),clustered=False)
|
||||
self.assertEqual(result['status'],'OK'); self.assertEqual(calls,[])
|
||||
store.clear_error('pve_service_'+removed)
|
||||
self.assertFalse(poll(store,first,'pve_service_'+removed,'pve_services')[0][0]['is_recovery'])
|
||||
with case() as store:
|
||||
first=record(store,'pve_service_corosync','pve_services','corosync inactive',{'service':'corosync'})
|
||||
result,calls=service(store,services=('pvedaemon',),clustered=False)
|
||||
self.assertEqual(result['status'],'OK')
|
||||
self.assertEqual([c[0][-1] for c in calls],['pvedaemon'])
|
||||
self.assertTrue(store.is_error_active('pve_service_corosync'))
|
||||
self.assertIsNone(store.get_recovery_evidence('pve_service_corosync',first))
|
||||
with case() as store:
|
||||
first=record(store,'pve_service_pvedaemon','pve_services','service inactive')
|
||||
store.acknowledge_error('pve_service_pvedaemon',suppression_hours=-1)
|
||||
store.clear_error('pve_service_pvedaemon',check_evidence={'check':'pve_service_pvedaemon','checked_at':BASE,'service':'pvedaemon','state':'active','returncode':0})
|
||||
self.assertEqual(sql(store,'SELECT id FROM errors'),[])
|
||||
self.assertIsNone(store.get_recovery_evidence('pve_service_pvedaemon',first))
|
||||
|
||||
def test_native_binding_latest_closure_rollbacks_and_consistent_ack_read(self):
|
||||
import contextlib, json, sqlite3
|
||||
for mutation in ('row_id','first_seen','closure','last_seen','latest_clear','latest_resolve','ack','event_insert_failure','ack_before_join'):
|
||||
with self.subTest(mutation=mutation),case() as store:
|
||||
Clock.epoch=BASE-600
|
||||
cpu(store,90,[{'value':90,'time':Clock.epoch-i*5} for i in range(1,26)])
|
||||
first=sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch=BASE
|
||||
if mutation=='event_insert_failure':
|
||||
sql(store,"CREATE TRIGGER fail_resolve BEFORE INSERT ON events WHEN NEW.event_type='resolved' BEGIN SELECT RAISE(ABORT,'fixture'); END")
|
||||
self.assertEqual(cpu(store)['status'],'UNKNOWN')
|
||||
self.assertIsNone(sql(store,'SELECT resolved_at FROM errors')[0][0])
|
||||
continue
|
||||
cpu(store); self.assertTrue(store.get_recovery_evidence('cpu_usage',first))
|
||||
if mutation in ('row_id','first_seen','closure'):
|
||||
data=json.loads(sql(store,"SELECT data FROM events WHERE event_type='resolved'")[0][0])
|
||||
field={'row_id':'id','first_seen':'first_seen','closure':'resolved_at'}[mutation]
|
||||
data['incident'][field]='wrong'
|
||||
sql(store,"UPDATE events SET data=? WHERE event_type='resolved'",(json.dumps(data),))
|
||||
elif mutation=='last_seen': sql(store,'UPDATE errors SET last_seen=?',(Clock.fromtimestamp(BASE+1).isoformat(),))
|
||||
elif mutation.startswith('latest_'):
|
||||
sql(store,"INSERT INTO events(event_type,error_key,timestamp,data) VALUES(?,'cpu_usage',?,'{}')",('cleared' if mutation=='latest_clear' else 'resolved',Clock.now().isoformat()))
|
||||
elif mutation=='ack': store.acknowledge_error('cpu_usage',suppression_hours=-1)
|
||||
else:
|
||||
original=store._db_connection; triggered=[]
|
||||
class Proxy:
|
||||
def __init__(self,connection):self.connection=connection
|
||||
def __getattr__(self,name):return getattr(self.connection,name)
|
||||
def execute(self,query,args=()):
|
||||
if 'FROM errors e JOIN events' in query and not triggered:
|
||||
triggered.append(True)
|
||||
store.acknowledge_error('cpu_usage',suppression_hours=-1)
|
||||
return self.connection.execute(query,args)
|
||||
@contextlib.contextmanager
|
||||
def interleaved(**kw):
|
||||
with original(**kw) as conn: yield Proxy(conn)
|
||||
store._db_connection=interleaved
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage',first))
|
||||
if mutation=='ack_before_join': self.assertTrue(triggered)
|
||||
with case() as store:
|
||||
sql(store,"INSERT INTO errors(error_key,category,severity,reason,first_seen,last_seen) VALUES('cpu_usage','cpu','WARNING','fixture','x','x')")
|
||||
with self.assertRaises(sqlite3.IntegrityError):
|
||||
sql(store,"INSERT INTO errors(error_key,category,severity,reason,first_seen,last_seen) VALUES('cpu_usage','cpu','WARNING','duplicate','x','x')")
|
||||
|
||||
def test_malformed_history_declines_proof_without_changing_operational_clear(self):
|
||||
with case() as store:
|
||||
Clock.epoch=BASE-600
|
||||
cpu(store,90,[{'value':90,'time':Clock.epoch-i*5} for i in range(1,26)])
|
||||
first=sql(store,'SELECT first_seen FROM errors')[0][0]
|
||||
Clock.epoch=BASE
|
||||
result=cpu(store,20,[{'value':10**400,'time':BASE-i*5} for i in range(1,10)])
|
||||
self.assertEqual(result['status'],'OK')
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage',first))
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
@@ -1,100 +1,68 @@
|
||||
"""Fresh existing-check provenance; extracted consumers, real disposable SQLite."""
|
||||
import contextlib
|
||||
import datetime
|
||||
"""Fresh existing-check provenance; native initializer and disposable SQLite."""
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import time
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from notification_fixture import extract, SCRIPTS, templates, LANGUAGES
|
||||
from notification_fixture import templates, LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
from notification_recovery_fixture import case, Clock, BASE, cpu, sql, poll
|
||||
|
||||
|
||||
def original_cpu(store):
|
||||
Clock.epoch = BASE-600
|
||||
result = cpu(store, 90, [{'value':90,'time':Clock.epoch-i*5} for i in range(1,26)])
|
||||
assert result['status'] == 'WARNING'
|
||||
Clock.epoch = BASE
|
||||
return sql(store, 'SELECT first_seen FROM errors')[0][0]
|
||||
|
||||
|
||||
class RecoveryEvidenceTests(unittest.TestCase):
|
||||
def test_cpu_success_provenance_is_persisted_only_after_normal_samples(self):
|
||||
events=[]
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
db=scratch+'/health.sqlite'
|
||||
conn=sqlite3.connect(db)
|
||||
conn.execute('CREATE TABLE errors(id INTEGER PRIMARY KEY,error_key TEXT,details TEXT,resolved_at TEXT,resolution_type TEXT,resolution_reason TEXT)')
|
||||
conn.execute("INSERT INTO errors(error_key,details) VALUES ('cpu_usage','{}')")
|
||||
conn.commit();conn.close()
|
||||
@contextlib.contextmanager
|
||||
def connection():
|
||||
c=sqlite3.connect(db)
|
||||
try:yield c
|
||||
finally:c.close()
|
||||
ns={'datetime':datetime.datetime,'json':json}
|
||||
resolve=extract(SCRIPTS/'health_persistence.py','_resolve_error_impl','HealthPersistence',ns)
|
||||
store=types.SimpleNamespace(_db_connection=connection,_entity_from_details=lambda details:'',_record_event=lambda cursor,kind,key,data:events.append(data))
|
||||
store.resolve_error=lambda key,reason,**kw:resolve(store,key,reason,**kw)
|
||||
ns={'Dict':dict,'Any':object,'os':os,'time':time,'health_persistence':store,'psutil':types.SimpleNamespace(cpu_percent=lambda **kw:20,cpu_count=lambda:4)}
|
||||
check=extract(SCRIPTS/'health_monitor.py','_check_cpu_with_hysteresis','HealthMonitor',ns)
|
||||
target=types.SimpleNamespace(state_history={'cpu_usage':[{'value':20,'time':time.time()-i*10} for i in range(10)]},CPU_CRITICAL=95,CPU_WARNING=85,CPU_RECOVERY=75,CPU_CRITICAL_DURATION=300,CPU_WARNING_DURATION=300,CPU_RECOVERY_DURATION=120,_check_cpu_temperature=lambda:None)
|
||||
result=check(target)
|
||||
self.assertEqual(result['status'],'OK')
|
||||
self.assertTrue(events[-1].get('check_evidence'),events)
|
||||
proof=events[-1]['check_evidence']
|
||||
self.assertEqual(proof['check'],'cpu_usage')
|
||||
self.assertGreaterEqual(proof['checked_at'],time.time()-5)
|
||||
# Existing generic resolve callers (cleanup/exclusion) get no proof.
|
||||
conn=sqlite3.connect(db);conn.execute('UPDATE errors SET resolved_at=NULL');conn.commit();conn.close()
|
||||
resolve(store,'cpu_usage','No longer present')
|
||||
self.assertFalse(events[-1].get('check_evidence'))
|
||||
with case() as store:
|
||||
first = original_cpu(store)
|
||||
self.assertEqual(cpu(store)['status'], 'OK')
|
||||
proof = store.get_recovery_evidence('cpu_usage', first)
|
||||
self.assertTrue(proof)
|
||||
self.assertEqual(proof['check'], 'cpu_usage')
|
||||
self.assertEqual(proof['checked_at'], BASE)
|
||||
# A generic closure never gains proof; use another actual native row.
|
||||
store.record_error('pve_service_test','pve_services','CRITICAL','inactive')
|
||||
store.resolve_error('pve_service_test','No longer present')
|
||||
self.assertFalse(json.loads(sql(store,"SELECT data FROM events ORDER BY id DESC LIMIT 1")[0][0]).get('check_evidence'))
|
||||
|
||||
def test_recovery_query_requires_fresh_same_incident_proof(self):
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
db=scratch+'/health.sqlite'
|
||||
conn=sqlite3.connect(db)
|
||||
conn.execute('CREATE TABLE errors(id INTEGER PRIMARY KEY,error_key TEXT,first_seen TEXT,last_seen TEXT,resolved_at TEXT,acknowledged INTEGER)')
|
||||
conn.execute('CREATE TABLE events(id INTEGER PRIMARY KEY,event_type TEXT,error_key TEXT,timestamp TEXT,data TEXT)')
|
||||
now=datetime.datetime.now(); first=(now-datetime.timedelta(minutes=10)).isoformat(); last=(now-datetime.timedelta(minutes=1)).isoformat(); resolved=now.isoformat()
|
||||
proof={'check':'cpu_usage','checked_at':now.timestamp()}
|
||||
conn.execute('INSERT INTO errors VALUES(1,?,?,?,?,0)',('cpu_usage',first,last,resolved))
|
||||
conn.execute('INSERT INTO events VALUES(1,?,?,?,?)',('resolved','cpu_usage',resolved,json.dumps({'check_evidence':proof})))
|
||||
conn.commit();conn.close()
|
||||
@contextlib.contextmanager
|
||||
def connection(**kwargs):
|
||||
c=sqlite3.connect(db)
|
||||
try:yield c
|
||||
finally:c.close()
|
||||
ns={'datetime':datetime.datetime,'json':json,'time':time}
|
||||
tree=(SCRIPTS/'health_persistence.py').read_text()
|
||||
query=extract(SCRIPTS/'health_persistence.py','get_recovery_evidence','HealthPersistence',ns) if 'def get_recovery_evidence(' in tree else lambda *args:None
|
||||
store=types.SimpleNamespace(_db_connection=connection)
|
||||
self.assertEqual(query(store,'cpu_usage',first),proof)
|
||||
self.assertIsNone(query(store,'cpu_usage','different incident'))
|
||||
for field,value in [('acknowledged',1),('resolved_at',None),('last_seen',(now+datetime.timedelta(seconds=1)).isoformat())]:
|
||||
conn=sqlite3.connect(db);conn.execute(f'UPDATE errors SET {field}=?',(value,));conn.commit();conn.close()
|
||||
self.assertIsNone(query(store,'cpu_usage',first))
|
||||
conn=sqlite3.connect(db);conn.execute('UPDATE errors SET acknowledged=0,resolved_at=?,last_seen=?',(resolved,last));conn.commit();conn.close()
|
||||
for bad in (None,{'check':'cpu_usage','checked_at':now.timestamp()-7201},{'check':'storage_removed','checked_at':now.timestamp()},{'check':'cpu_usage','checked_at':float('inf')},{'check':'cpu_usage','checked_at':10**400}):
|
||||
conn=sqlite3.connect(db);conn.execute('UPDATE events SET data=?',(json.dumps({'check_evidence':bad}),));conn.commit();conn.close()
|
||||
self.assertIsNone(query(store,'cpu_usage',first))
|
||||
with case() as store:
|
||||
first = original_cpu(store); cpu(store)
|
||||
proof = store.get_recovery_evidence('cpu_usage',first)
|
||||
self.assertTrue(proof)
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage','different incident'))
|
||||
saved = sql(store,'SELECT last_seen,resolved_at FROM errors')[0]
|
||||
for field,value in [('acknowledged',1),('resolved_at',None),('last_seen',Clock.fromtimestamp(BASE+1).isoformat())]:
|
||||
sql(store,f'UPDATE errors SET {field}=?',(value,))
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage',first))
|
||||
sql(store,'UPDATE errors SET acknowledged=0,last_seen=?,resolved_at=?',saved)
|
||||
data = json.loads(sql(store,"SELECT data FROM events WHERE event_type='resolved'")[0][0])
|
||||
for bad in (None,{'check':'cpu_usage','checked_at':BASE-7201}, {'check':'storage_removed','checked_at':BASE}, {'check':'cpu_usage','checked_at':float('inf')}, {'check':'cpu_usage','checked_at':10**400}):
|
||||
sql(store,"UPDATE events SET data=? WHERE event_type='resolved'",(json.dumps({**data,'check_evidence':bad}),))
|
||||
self.assertIsNone(store.get_recovery_evidence('cpu_usage',first))
|
||||
|
||||
def test_poller_and_all_consumers_distinguish_proven_recovery_from_disappearance(self):
|
||||
import sys
|
||||
ns={'time':time,'json':json,'Dict':dict,'NotificationEvent':lambda *a,**kw:types.SimpleNamespace(event_type=a[0],severity=a[1],data=a[2])}
|
||||
poll=extract(SCRIPTS/'notification_events.py','_check_persistent_health','PollingCollector',ns)
|
||||
for proof in (None,{'check':'cpu_usage','checked_at':time.time()}):
|
||||
events=[]
|
||||
store=types.SimpleNamespace(get_active_errors=lambda:[],is_error_acknowledged=lambda key:False,get_recovery_evidence=lambda *a:proof)
|
||||
collector=types.SimpleNamespace(_hostname='node-a',_ENTITY_MAP={'cpu':('node','')},_first_poll_done=True,_known_errors={'cpu_usage':{'category':'cpu','reason':'CPU high','severity':'WARNING','first_seen':'2026-09-30T00:00:00'}},_notified_severity={'cpu_usage':'WARNING'},_last_notified={'cpu_usage':1},_queue=types.SimpleNamespace(put=events.append),_guest_storage_error_is_now_foreign=lambda *a:False,_save_known_errors_meta=lambda:None)
|
||||
with patch.dict(sys.modules,{'health_persistence':types.SimpleNamespace(health_persistence=store)}):poll(collector)
|
||||
self.assertEqual(len(events),1)
|
||||
event=events[0]
|
||||
self.assertEqual(event.data.get('recovery_outcome'),'resolved' if proof else 'no_longer_reported')
|
||||
self.assertEqual(event.data['is_recovery'],bool(proof))
|
||||
for lang in LANGUAGES:
|
||||
for manual in (False,True):
|
||||
result=deliver(event.event_type,event.data,event.severity,lang,manual=manual)
|
||||
if proof:
|
||||
self.assertIn(templates.runtime_message('healthRecovery.title',lang,hostname='node-a',category='cpu',entity_suffix=''),result['title'])
|
||||
self.assertIn('background:#f0fdf4;',result['html'])
|
||||
else:self.assertNotIn('background:#f0fdf4;',result['html'])
|
||||
self.assertEqual(result['text'].count(event.data['reason']),1)
|
||||
for proved in (False,True):
|
||||
with case() as store:
|
||||
first = original_cpu(store)
|
||||
if proved: cpu(store)
|
||||
else: store.resolve_error('cpu_usage','No longer present')
|
||||
data = poll(store,first,reason='CPU high')[0][0]
|
||||
self.assertEqual(data['is_recovery'],proved)
|
||||
self.assertEqual(data['recovery_outcome'],'resolved' if proved else 'no_longer_reported')
|
||||
with patch('health_recovery.time.time',return_value=BASE):
|
||||
for lang in LANGUAGES:
|
||||
for manual in (False,True):
|
||||
result = deliver('error_resolved',data,'OK',lang,manual=manual)
|
||||
self.assertEqual('background:#f0fdf4;' in result['html'],proved)
|
||||
if proved:
|
||||
self.assertIn(templates.runtime_message('healthRecovery.title',lang,hostname='node-a',category='cpu',entity_suffix=''),result['title'])
|
||||
self.assertEqual(result['text'].count(data['reason']),1)
|
||||
|
||||
def test_manual_recovery_flag_alone_is_not_authoritative_evidence(self):
|
||||
data={'hostname':'node-a','category':'cpu','reason':'Observation disappeared','duration':'1h','original_severity':'WARNING','recovery_outcome':'resolved'}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Review regressions at actual consumer seams; no operational imports."""
|
||||
import unittest
|
||||
from notification_fixture import receive, LANGUAGES
|
||||
from notification_final_fixture import deliver
|
||||
|
||||
class ReviewCorrectionTests(unittest.TestCase):
|
||||
def test_subject_equivalent_late_error_is_reserved_before_cap(self):
|
||||
cause = 'job-end hook denied'
|
||||
# Frozen native Perl notifier/log-reader output; Rust table source-modeled.
|
||||
raw = '\nDetails\n=======\nVMID Name Status Time Size Filename \n100 web err 1m 1s 0 B null \n\nTotal running time: 1m 1s\nTotal size: 0 B\n\nLogs\n====\nvzdump --all 1 --storage PBS --mode snapshot\n\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 0\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 1\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 2\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 3\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 4\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 5\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 6\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 7\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 8\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 9\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 10\n100: 2026-09-29 17:00:00 ERROR: earlier diagnostic 11\n100: 2026-09-29 17:00:00 ERROR: job-end hook denied\n\n\n'
|
||||
event = receive(raw, 'error', 'vzdump backup status (raw-host): backup failed: ' + cause)
|
||||
for language in LANGUAGES:
|
||||
for manual in (False, True):
|
||||
with self.subTest(language=language, manual=manual):
|
||||
result = deliver(event.event_type, {**event.data, 'hostname':'alias {rack.location}'}, event.severity, language, manual=manual)
|
||||
self.assertEqual(result['text'].count(cause), 1)
|
||||
self.assertNotIn('raw-host', result['text'])
|
||||
diagnostics = [line for line in result['body'].splitlines() if 'ERROR:' in line]
|
||||
self.assertLessEqual(len(diagnostics), 8)
|
||||
self.assertLessEqual(len('\n'.join(diagnostics)), 1024)
|
||||
self.assertTrue(all(len(line) <= 512 for line in diagnostics))
|
||||
self.assertEqual(result['data']['pve_message'], raw)
|
||||
|
||||
if __name__ == '__main__': unittest.main()
|
||||
Reference in New Issue
Block a user