mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-08 22:46:41 +00:00
feat(oci): watchdog that restarts an application when it stops on its own
- New service that starts again an OCI application that stopped on its own, such as Frigate after Save & Restart. A stop or shutdown asked by the user, a backup, a migration and a ProxMenux operation are never undone. - The installer asks it in both modes and proposes what the recipe declares; it is changed from the management menu or with the Watchdog switch of the container modal in the Monitor. - Notifications when the watchdog restarts an application and when it keeps stopping. - The service is recorded in the change journal, and the feature is documented.
This commit is contained in:
MacRimi
@@ -0,0 +1,209 @@
|
||||
"""The watchdog starts again what stopped on its own, and nothing else."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_watchdog as watchdog
|
||||
|
||||
ACTIVE = """UPID:amd:001C34BC:01A809CF:6AC55103:vzshutdown:109:root@pam: 1 6AC55105 OK
|
||||
UPID:amd:001C3370:01A80568:6AC550F8:vzstart:109:root@pam: 1 6AC550FA OK
|
||||
UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:108:root@pam: 1 6AC550F1 unable to stop: timeout
|
||||
UPID:amd:001C3999:01A80999:6AC55200:vzdump:110:root@pam: 0
|
||||
"""
|
||||
INDEX = """UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:109:root@pam: 6AC550F1 OK
|
||||
UPID:amd:001C3370:01A80568:6AC550F8:stopall::root@pam: 6AC55300 unable to stop every guest
|
||||
"""
|
||||
|
||||
|
||||
def never():
|
||||
raise AssertionError("not expected to be asked")
|
||||
|
||||
|
||||
class Watch:
|
||||
"""One container seen through the decisions of the watchdog."""
|
||||
def __init__(self):
|
||||
self.state = watchdog.new_state()
|
||||
|
||||
def look(self, running, now, busy=False, requested=False):
|
||||
return watchdog.decide(self.state, running, now, lambda: busy, lambda seen: requested)
|
||||
|
||||
|
||||
class TaskList(unittest.TestCase):
|
||||
def test_both_lists_are_read_with_their_own_columns(self):
|
||||
self.assertEqual(watchdog.parse_tasks(ACTIVE, True), [
|
||||
("vzshutdown", "109", 0x6AC55105), ("vzstart", "109", 0x6AC550FA),
|
||||
("vzstop", "108", 0x6AC550F1), ("vzdump", "110", None)])
|
||||
self.assertEqual(watchdog.parse_tasks(INDEX, False), [("vzstop", "109", 0x6AC550F1), ("stopall", "", 0x6AC55300)])
|
||||
|
||||
def test_a_request_counts_when_it_ended_after_the_container_was_last_seen_running(self):
|
||||
tasks = watchdog.parse_tasks(ACTIVE, True)
|
||||
self.assertTrue(watchdog.stop_requested(tasks, 109, 0x6AC55104))
|
||||
self.assertFalse(watchdog.stop_requested(tasks, 109, 0x6AC55110))
|
||||
self.assertFalse(watchdog.stop_requested(tasks, 111, 0x6AC55000))
|
||||
|
||||
def test_a_running_request_and_a_stop_of_every_guest_count(self):
|
||||
self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(ACTIVE, True), 110, 0x6AC55900))
|
||||
self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(INDEX, False), 555, 0x6AC552F0))
|
||||
|
||||
def test_starting_a_container_is_not_a_request_to_stop_it(self):
|
||||
self.assertFalse(watchdog.stop_requested([("vzstart", "109", None)], 109, 0))
|
||||
|
||||
|
||||
class Decisions(unittest.TestCase):
|
||||
def test_a_container_never_seen_running_is_left_alone(self):
|
||||
watch = Watch()
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 100, never, never))
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))
|
||||
|
||||
def test_a_container_that_stops_on_its_own_is_started_again(self):
|
||||
watch = Watch()
|
||||
watch.look(True, 100)
|
||||
self.assertEqual(watch.look(False, 110), "start")
|
||||
self.assertEqual(watch.state["attempts"], 1)
|
||||
|
||||
def test_a_requested_stop_is_never_undone(self):
|
||||
watch = Watch()
|
||||
watch.look(True, 100)
|
||||
self.assertIsNone(watch.look(False, 110, requested=True))
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 9000, never, never))
|
||||
watch.look(True, 9100)
|
||||
self.assertEqual(watch.look(False, 9110), "start")
|
||||
|
||||
def test_nothing_is_decided_while_something_works_on_the_container(self):
|
||||
watch = Watch()
|
||||
watch.look(True, 100)
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 110, lambda: True, never))
|
||||
self.assertIsNone(watch.look(False, 120, requested=True))
|
||||
watch.look(True, 300)
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 310, lambda: True, never))
|
||||
self.assertEqual(watch.look(False, 320), "start")
|
||||
|
||||
def test_a_stop_asked_after_a_restart_is_respected(self):
|
||||
watch = Watch()
|
||||
watch.look(True, 100)
|
||||
self.assertEqual(watch.look(False, 110), "start")
|
||||
watch.look(True, 120)
|
||||
self.assertIsNone(watch.look(False, 130, requested=True))
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))
|
||||
|
||||
def test_an_application_that_keeps_stopping_is_tried_with_longer_waits_and_then_left(self):
|
||||
watch, now, actions = Watch(), 100, []
|
||||
watch.look(True, now)
|
||||
for _ in range(200):
|
||||
now += 10
|
||||
action = watch.look(False, now)
|
||||
if action:
|
||||
actions.append((action, now))
|
||||
self.assertEqual([name for name, _ in actions], ["start"] * 5 + ["give-up"])
|
||||
waits = [later - earlier for (_, earlier), (_, later) in zip(actions, actions[1:])]
|
||||
self.assertEqual(waits, [30, 60, 120, 300, 300])
|
||||
self.assertIsNone(watchdog.decide(watch.state, False, now + 9000, never, never))
|
||||
|
||||
def test_an_application_that_restarts_itself_now_and_then_is_always_started_at_once(self):
|
||||
watch, now = Watch(), 100
|
||||
watch.look(True, now)
|
||||
for _ in range(12):
|
||||
now += 10
|
||||
self.assertEqual(watch.look(False, now), "start")
|
||||
for _ in range(4):
|
||||
now += 10
|
||||
watch.look(True, now)
|
||||
self.assertEqual(watch.state["attempts"], 0)
|
||||
|
||||
def test_a_restart_is_told_once_in_a_while(self):
|
||||
state = watchdog.new_state()
|
||||
self.assertTrue(watchdog.notice_due(state, 1000))
|
||||
self.assertFalse(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY - 1))
|
||||
self.assertTrue(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY))
|
||||
|
||||
def test_running_for_a_while_forgets_the_earlier_attempts(self):
|
||||
watch = Watch()
|
||||
watch.look(True, 100)
|
||||
self.assertEqual(watch.look(False, 110), "start")
|
||||
watch.look(True, 120)
|
||||
watch.look(True, 120 + watchdog.STABLE)
|
||||
self.assertEqual(watch.state["attempts"], 0)
|
||||
self.assertEqual(watch.look(False, 400), "start")
|
||||
self.assertEqual(watch.state["attempts"], 1)
|
||||
|
||||
|
||||
class Registry(unittest.TestCase):
|
||||
def test_only_installed_applications_that_asked_for_it_are_watched(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
records = {
|
||||
101: {"status": "installed", "deployment": {"watchdog": True},
|
||||
"template": {"catalog_ui": {"title": {"en_US": "Jellyfin"}}}},
|
||||
102: {"status": "installed", "deployment": {"watchdog": False}},
|
||||
103: {"status": "installed", "deployment": {}},
|
||||
104: {"status": "updating", "deployment": {"watchdog": True}},
|
||||
105: {"status": "installed", "deployment": {"watchdog": True}, "pending_stack_transaction": "/journal"},
|
||||
106: {"status": "installed", "deployment": {"watchdog": True}},
|
||||
}
|
||||
for vmid, record in records.items():
|
||||
(root / str(vmid)).mkdir()
|
||||
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({"vmid": vmid, **record}))
|
||||
(root / "107").mkdir()
|
||||
(root / "107" / "oci-compose.json").write_text("{broken")
|
||||
self.assertEqual(watchdog.watched(root), {101: "Jellyfin", 106: "CT 106"})
|
||||
|
||||
|
||||
class Start(unittest.TestCase):
|
||||
def test_the_container_is_started_outside_the_service(self):
|
||||
from unittest.mock import patch
|
||||
with patch.object(watchdog.subprocess, "run") as run:
|
||||
run.return_value.returncode = 0
|
||||
self.assertTrue(watchdog.start(109))
|
||||
run.return_value.returncode = 255
|
||||
self.assertFalse(watchdog.start(109))
|
||||
self.assertEqual(run.call_args.args[0], ["systemd-run", "--scope", "--quiet", "--collect", "pct", "start", "109"])
|
||||
|
||||
|
||||
class Switch(unittest.TestCase):
|
||||
"""Turning the watchdog on or off for a whole application."""
|
||||
def registry(self, folder):
|
||||
root = Path(folder)
|
||||
ids = {101: "11111111-1111-4111-8111-111111111111", 102: "22222222-2222-4222-8222-222222222222",
|
||||
103: "33333333-3333-4333-8333-333333333333"}
|
||||
stack = {"members": [{"vmid": 101}, {"vmid": 102}]}
|
||||
records = {101: {"stack": stack, "stack_member": {"primary_vmid": 101}},
|
||||
102: {"stack_member": {"primary_vmid": 101}}, 103: {}}
|
||||
for vmid, extra in records.items():
|
||||
(root / str(vmid)).mkdir()
|
||||
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({
|
||||
"schema_version": 1, "vmid": vmid, "installation_id": ids[vmid], "status": "installed",
|
||||
"deployment": {"onboot": True}, **extra}))
|
||||
return root
|
||||
|
||||
def flags(self, root):
|
||||
return {int(path.parent.name): json.loads(path.read_text())["deployment"].get("watchdog")
|
||||
for path in root.glob("*/oci-compose.json")}
|
||||
|
||||
def test_every_container_of_the_application_takes_the_choice_and_keeps_the_rest_of_its_record(self):
|
||||
from unittest.mock import patch
|
||||
import oci_carried_record
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = self.registry(folder)
|
||||
with patch.object(watchdog, "ensure_service") as service, \
|
||||
patch.object(oci_carried_record, "carry", return_value="carried") as carry:
|
||||
self.assertEqual(watchdog.set_watchdog(root, 102, True), [101, 102])
|
||||
self.assertEqual(self.flags(root), {101: True, 102: True, 103: None})
|
||||
service.assert_called_once_with()
|
||||
self.assertEqual([call.args[1] for call in carry.call_args_list], [101, 102])
|
||||
self.assertEqual(watchdog.set_watchdog(root, 103, True), [103])
|
||||
self.assertEqual(watchdog.set_watchdog(root, 101, False), [101, 102])
|
||||
self.assertEqual(self.flags(root), {101: False, 102: False, 103: True})
|
||||
self.assertEqual(service.call_count, 2)
|
||||
record = json.loads((root / "101" / "oci-compose.json").read_text())
|
||||
self.assertEqual(record["deployment"]["onboot"], True)
|
||||
self.assertEqual(record["stack"]["members"], [{"vmid": 101}, {"vmid": 102}])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user