mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-08 22:46:41 +00:00
MacRimi
41ae752da1
- New service that starts again an OCI application that stopped on its own, such as Frigate after Save & Restart. A stop or shutdown asked by the user, a backup, a migration and a ProxMenux operation are never undone. - The installer asks it in both modes and proposes what the recipe declares; it is changed from the management menu or with the Watchdog switch of the container modal in the Monitor. - Notifications when the watchdog restarts an application and when it keeps stopping. - The service is recorded in the change journal, and the feature is documented.
210 lines
9.8 KiB
Python
210 lines
9.8 KiB
Python
"""The watchdog starts again what stopped on its own, and nothing else."""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "remote"))
|
|
|
|
import oci_watchdog as watchdog
|
|
|
|
ACTIVE = """UPID:amd:001C34BC:01A809CF:6AC55103:vzshutdown:109:root@pam: 1 6AC55105 OK
|
|
UPID:amd:001C3370:01A80568:6AC550F8:vzstart:109:root@pam: 1 6AC550FA OK
|
|
UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:108:root@pam: 1 6AC550F1 unable to stop: timeout
|
|
UPID:amd:001C3999:01A80999:6AC55200:vzdump:110:root@pam: 0
|
|
"""
|
|
INDEX = """UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:109:root@pam: 6AC550F1 OK
|
|
UPID:amd:001C3370:01A80568:6AC550F8:stopall::root@pam: 6AC55300 unable to stop every guest
|
|
"""
|
|
|
|
|
|
def never():
|
|
raise AssertionError("not expected to be asked")
|
|
|
|
|
|
class Watch:
|
|
"""One container seen through the decisions of the watchdog."""
|
|
def __init__(self):
|
|
self.state = watchdog.new_state()
|
|
|
|
def look(self, running, now, busy=False, requested=False):
|
|
return watchdog.decide(self.state, running, now, lambda: busy, lambda seen: requested)
|
|
|
|
|
|
class TaskList(unittest.TestCase):
|
|
def test_both_lists_are_read_with_their_own_columns(self):
|
|
self.assertEqual(watchdog.parse_tasks(ACTIVE, True), [
|
|
("vzshutdown", "109", 0x6AC55105), ("vzstart", "109", 0x6AC550FA),
|
|
("vzstop", "108", 0x6AC550F1), ("vzdump", "110", None)])
|
|
self.assertEqual(watchdog.parse_tasks(INDEX, False), [("vzstop", "109", 0x6AC550F1), ("stopall", "", 0x6AC55300)])
|
|
|
|
def test_a_request_counts_when_it_ended_after_the_container_was_last_seen_running(self):
|
|
tasks = watchdog.parse_tasks(ACTIVE, True)
|
|
self.assertTrue(watchdog.stop_requested(tasks, 109, 0x6AC55104))
|
|
self.assertFalse(watchdog.stop_requested(tasks, 109, 0x6AC55110))
|
|
self.assertFalse(watchdog.stop_requested(tasks, 111, 0x6AC55000))
|
|
|
|
def test_a_running_request_and_a_stop_of_every_guest_count(self):
|
|
self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(ACTIVE, True), 110, 0x6AC55900))
|
|
self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(INDEX, False), 555, 0x6AC552F0))
|
|
|
|
def test_starting_a_container_is_not_a_request_to_stop_it(self):
|
|
self.assertFalse(watchdog.stop_requested([("vzstart", "109", None)], 109, 0))
|
|
|
|
|
|
class Decisions(unittest.TestCase):
|
|
def test_a_container_never_seen_running_is_left_alone(self):
|
|
watch = Watch()
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 100, never, never))
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))
|
|
|
|
def test_a_container_that_stops_on_its_own_is_started_again(self):
|
|
watch = Watch()
|
|
watch.look(True, 100)
|
|
self.assertEqual(watch.look(False, 110), "start")
|
|
self.assertEqual(watch.state["attempts"], 1)
|
|
|
|
def test_a_requested_stop_is_never_undone(self):
|
|
watch = Watch()
|
|
watch.look(True, 100)
|
|
self.assertIsNone(watch.look(False, 110, requested=True))
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 9000, never, never))
|
|
watch.look(True, 9100)
|
|
self.assertEqual(watch.look(False, 9110), "start")
|
|
|
|
def test_nothing_is_decided_while_something_works_on_the_container(self):
|
|
watch = Watch()
|
|
watch.look(True, 100)
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 110, lambda: True, never))
|
|
self.assertIsNone(watch.look(False, 120, requested=True))
|
|
watch.look(True, 300)
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 310, lambda: True, never))
|
|
self.assertEqual(watch.look(False, 320), "start")
|
|
|
|
def test_a_stop_asked_after_a_restart_is_respected(self):
|
|
watch = Watch()
|
|
watch.look(True, 100)
|
|
self.assertEqual(watch.look(False, 110), "start")
|
|
watch.look(True, 120)
|
|
self.assertIsNone(watch.look(False, 130, requested=True))
|
|
self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))
|
|
|
|
def test_an_application_that_keeps_stopping_is_tried_with_longer_waits_and_then_left(self):
|
|
watch, now, actions = Watch(), 100, []
|
|
watch.look(True, now)
|
|
for _ in range(200):
|
|
now += 10
|
|
action = watch.look(False, now)
|
|
if action:
|
|
actions.append((action, now))
|
|
self.assertEqual([name for name, _ in actions], ["start"] * 5 + ["give-up"])
|
|
waits = [later - earlier for (_, earlier), (_, later) in zip(actions, actions[1:])]
|
|
self.assertEqual(waits, [30, 60, 120, 300, 300])
|
|
self.assertIsNone(watchdog.decide(watch.state, False, now + 9000, never, never))
|
|
|
|
def test_an_application_that_restarts_itself_now_and_then_is_always_started_at_once(self):
|
|
watch, now = Watch(), 100
|
|
watch.look(True, now)
|
|
for _ in range(12):
|
|
now += 10
|
|
self.assertEqual(watch.look(False, now), "start")
|
|
for _ in range(4):
|
|
now += 10
|
|
watch.look(True, now)
|
|
self.assertEqual(watch.state["attempts"], 0)
|
|
|
|
def test_a_restart_is_told_once_in_a_while(self):
|
|
state = watchdog.new_state()
|
|
self.assertTrue(watchdog.notice_due(state, 1000))
|
|
self.assertFalse(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY - 1))
|
|
self.assertTrue(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY))
|
|
|
|
def test_running_for_a_while_forgets_the_earlier_attempts(self):
|
|
watch = Watch()
|
|
watch.look(True, 100)
|
|
self.assertEqual(watch.look(False, 110), "start")
|
|
watch.look(True, 120)
|
|
watch.look(True, 120 + watchdog.STABLE)
|
|
self.assertEqual(watch.state["attempts"], 0)
|
|
self.assertEqual(watch.look(False, 400), "start")
|
|
self.assertEqual(watch.state["attempts"], 1)
|
|
|
|
|
|
class Registry(unittest.TestCase):
|
|
def test_only_installed_applications_that_asked_for_it_are_watched(self):
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = Path(folder)
|
|
records = {
|
|
101: {"status": "installed", "deployment": {"watchdog": True},
|
|
"template": {"catalog_ui": {"title": {"en_US": "Jellyfin"}}}},
|
|
102: {"status": "installed", "deployment": {"watchdog": False}},
|
|
103: {"status": "installed", "deployment": {}},
|
|
104: {"status": "updating", "deployment": {"watchdog": True}},
|
|
105: {"status": "installed", "deployment": {"watchdog": True}, "pending_stack_transaction": "/journal"},
|
|
106: {"status": "installed", "deployment": {"watchdog": True}},
|
|
}
|
|
for vmid, record in records.items():
|
|
(root / str(vmid)).mkdir()
|
|
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({"vmid": vmid, **record}))
|
|
(root / "107").mkdir()
|
|
(root / "107" / "oci-compose.json").write_text("{broken")
|
|
self.assertEqual(watchdog.watched(root), {101: "Jellyfin", 106: "CT 106"})
|
|
|
|
|
|
class Start(unittest.TestCase):
|
|
def test_the_container_is_started_outside_the_service(self):
|
|
from unittest.mock import patch
|
|
with patch.object(watchdog.subprocess, "run") as run:
|
|
run.return_value.returncode = 0
|
|
self.assertTrue(watchdog.start(109))
|
|
run.return_value.returncode = 255
|
|
self.assertFalse(watchdog.start(109))
|
|
self.assertEqual(run.call_args.args[0], ["systemd-run", "--scope", "--quiet", "--collect", "pct", "start", "109"])
|
|
|
|
|
|
class Switch(unittest.TestCase):
|
|
"""Turning the watchdog on or off for a whole application."""
|
|
def registry(self, folder):
|
|
root = Path(folder)
|
|
ids = {101: "11111111-1111-4111-8111-111111111111", 102: "22222222-2222-4222-8222-222222222222",
|
|
103: "33333333-3333-4333-8333-333333333333"}
|
|
stack = {"members": [{"vmid": 101}, {"vmid": 102}]}
|
|
records = {101: {"stack": stack, "stack_member": {"primary_vmid": 101}},
|
|
102: {"stack_member": {"primary_vmid": 101}}, 103: {}}
|
|
for vmid, extra in records.items():
|
|
(root / str(vmid)).mkdir()
|
|
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({
|
|
"schema_version": 1, "vmid": vmid, "installation_id": ids[vmid], "status": "installed",
|
|
"deployment": {"onboot": True}, **extra}))
|
|
return root
|
|
|
|
def flags(self, root):
|
|
return {int(path.parent.name): json.loads(path.read_text())["deployment"].get("watchdog")
|
|
for path in root.glob("*/oci-compose.json")}
|
|
|
|
def test_every_container_of_the_application_takes_the_choice_and_keeps_the_rest_of_its_record(self):
|
|
from unittest.mock import patch
|
|
import oci_carried_record
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root = self.registry(folder)
|
|
with patch.object(watchdog, "ensure_service") as service, \
|
|
patch.object(oci_carried_record, "carry", return_value="carried") as carry:
|
|
self.assertEqual(watchdog.set_watchdog(root, 102, True), [101, 102])
|
|
self.assertEqual(self.flags(root), {101: True, 102: True, 103: None})
|
|
service.assert_called_once_with()
|
|
self.assertEqual([call.args[1] for call in carry.call_args_list], [101, 102])
|
|
self.assertEqual(watchdog.set_watchdog(root, 103, True), [103])
|
|
self.assertEqual(watchdog.set_watchdog(root, 101, False), [101, 102])
|
|
self.assertEqual(self.flags(root), {101: False, 102: False, 103: True})
|
|
self.assertEqual(service.call_count, 2)
|
|
record = json.loads((root / "101" / "oci-compose.json").read_text())
|
|
self.assertEqual(record["deployment"]["onboot"], True)
|
|
self.assertEqual(record["stack"]["members"], [{"vmid": 101}, {"vmid": 102}])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|