ProxMenux/oci/tests/test_watchdog.py

"""The watchdog starts again what stopped on its own, and nothing else."""

import json
from pathlib import Path
import sys
import tempfile
import unittest

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "remote"))

import oci_watchdog as watchdog

ACTIVE = """UPID:amd:001C34BC:01A809CF:6AC55103:vzshutdown:109:root@pam: 1 6AC55105 OK
UPID:amd:001C3370:01A80568:6AC550F8:vzstart:109:root@pam: 1 6AC550FA OK
UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:108:root@pam: 1 6AC550F1 unable to stop: timeout
UPID:amd:001C3999:01A80999:6AC55200:vzdump:110:root@pam: 0
"""
INDEX = """UPID:amd:001C32AC:01A80211:6AC550EF:vzstop:109:root@pam: 6AC550F1 OK
UPID:amd:001C3370:01A80568:6AC550F8:stopall::root@pam: 6AC55300 unable to stop every guest
"""


def never():
    raise AssertionError("not expected to be asked")


class Watch:
    """One container seen through the decisions of the watchdog."""
    def __init__(self):
        self.state = watchdog.new_state()

    def look(self, running, now, busy=False, requested=False):
        return watchdog.decide(self.state, running, now, lambda: busy, lambda seen: requested)


class TaskList(unittest.TestCase):
    def test_both_lists_are_read_with_their_own_columns(self):
        self.assertEqual(watchdog.parse_tasks(ACTIVE, True), [
            ("vzshutdown", "109", 0x6AC55105), ("vzstart", "109", 0x6AC550FA),
            ("vzstop", "108", 0x6AC550F1), ("vzdump", "110", None)])
        self.assertEqual(watchdog.parse_tasks(INDEX, False), [("vzstop", "109", 0x6AC550F1), ("stopall", "", 0x6AC55300)])

    def test_a_request_counts_when_it_ended_after_the_container_was_last_seen_running(self):
        tasks = watchdog.parse_tasks(ACTIVE, True)
        self.assertTrue(watchdog.stop_requested(tasks, 109, 0x6AC55104))
        self.assertFalse(watchdog.stop_requested(tasks, 109, 0x6AC55110))
        self.assertFalse(watchdog.stop_requested(tasks, 111, 0x6AC55000))

    def test_a_running_request_and_a_stop_of_every_guest_count(self):
        self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(ACTIVE, True), 110, 0x6AC55900))
        self.assertTrue(watchdog.stop_requested(watchdog.parse_tasks(INDEX, False), 555, 0x6AC552F0))

    def test_starting_a_container_is_not_a_request_to_stop_it(self):
        self.assertFalse(watchdog.stop_requested([("vzstart", "109", None)], 109, 0))


class Decisions(unittest.TestCase):
    def test_a_container_never_seen_running_is_left_alone(self):
        watch = Watch()
        self.assertIsNone(watchdog.decide(watch.state, False, 100, never, never))
        self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))

    def test_a_container_that_stops_on_its_own_is_started_again(self):
        watch = Watch()
        watch.look(True, 100)
        self.assertEqual(watch.look(False, 110), "start")
        self.assertEqual(watch.state["attempts"], 1)

    def test_a_requested_stop_is_never_undone(self):
        watch = Watch()
        watch.look(True, 100)
        self.assertIsNone(watch.look(False, 110, requested=True))
        self.assertIsNone(watchdog.decide(watch.state, False, 9000, never, never))
        watch.look(True, 9100)
        self.assertEqual(watch.look(False, 9110), "start")

    def test_nothing_is_decided_while_something_works_on_the_container(self):
        watch = Watch()
        watch.look(True, 100)
        self.assertIsNone(watchdog.decide(watch.state, False, 110, lambda: True, never))
        self.assertIsNone(watch.look(False, 120, requested=True))
        watch.look(True, 300)
        self.assertIsNone(watchdog.decide(watch.state, False, 310, lambda: True, never))
        self.assertEqual(watch.look(False, 320), "start")

    def test_a_stop_asked_after_a_restart_is_respected(self):
        watch = Watch()
        watch.look(True, 100)
        self.assertEqual(watch.look(False, 110), "start")
        watch.look(True, 120)
        self.assertIsNone(watch.look(False, 130, requested=True))
        self.assertIsNone(watchdog.decide(watch.state, False, 5000, never, never))

    def test_an_application_that_keeps_stopping_is_tried_with_longer_waits_and_then_left(self):
        watch, now, actions = Watch(), 100, []
        watch.look(True, now)
        for _ in range(200):
            now += 10
            action = watch.look(False, now)
            if action:
                actions.append((action, now))
        self.assertEqual([name for name, _ in actions], ["start"] * 5 + ["give-up"])
        waits = [later - earlier for (_, earlier), (_, later) in zip(actions, actions[1:])]
        self.assertEqual(waits, [30, 60, 120, 300, 300])
        self.assertIsNone(watchdog.decide(watch.state, False, now + 9000, never, never))

    def test_an_application_that_restarts_itself_now_and_then_is_always_started_at_once(self):
        watch, now = Watch(), 100
        watch.look(True, now)
        for _ in range(12):
            now += 10
            self.assertEqual(watch.look(False, now), "start")
            for _ in range(4):
                now += 10
                watch.look(True, now)
        self.assertEqual(watch.state["attempts"], 0)

    def test_a_restart_is_told_once_in_a_while(self):
        state = watchdog.new_state()
        self.assertTrue(watchdog.notice_due(state, 1000))
        self.assertFalse(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY - 1))
        self.assertTrue(watchdog.notice_due(state, 1000 + watchdog.NOTICE_EVERY))

    def test_running_for_a_while_forgets_the_earlier_attempts(self):
        watch = Watch()
        watch.look(True, 100)
        self.assertEqual(watch.look(False, 110), "start")
        watch.look(True, 120)
        watch.look(True, 120 + watchdog.STABLE)
        self.assertEqual(watch.state["attempts"], 0)
        self.assertEqual(watch.look(False, 400), "start")
        self.assertEqual(watch.state["attempts"], 1)


class Registry(unittest.TestCase):
    def test_only_installed_applications_that_asked_for_it_are_watched(self):
        with tempfile.TemporaryDirectory() as folder:
            root = Path(folder)
            records = {
                101: {"status": "installed", "deployment": {"watchdog": True},
                      "template": {"catalog_ui": {"title": {"en_US": "Jellyfin"}}}},
                102: {"status": "installed", "deployment": {"watchdog": False}},
                103: {"status": "installed", "deployment": {}},
                104: {"status": "updating", "deployment": {"watchdog": True}},
                105: {"status": "installed", "deployment": {"watchdog": True}, "pending_stack_transaction": "/journal"},
                106: {"status": "installed", "deployment": {"watchdog": True}},
            }
            for vmid, record in records.items():
                (root / str(vmid)).mkdir()
                (root / str(vmid) / "oci-compose.json").write_text(json.dumps({"vmid": vmid, **record}))
            (root / "107").mkdir()
            (root / "107" / "oci-compose.json").write_text("{broken")
            self.assertEqual(watchdog.watched(root), {101: "Jellyfin", 106: "CT 106"})


class Start(unittest.TestCase):
    def test_the_container_is_started_outside_the_service(self):
        from unittest.mock import patch
        with patch.object(watchdog.subprocess, "run") as run:
            run.return_value.returncode = 0
            self.assertTrue(watchdog.start(109))
            run.return_value.returncode = 255
            self.assertFalse(watchdog.start(109))
        self.assertEqual(run.call_args.args[0], ["systemd-run", "--scope", "--quiet", "--collect", "pct", "start", "109"])


class Switch(unittest.TestCase):
    """Turning the watchdog on or off for a whole application."""
    def registry(self, folder):
        root = Path(folder)
        ids = {101: "11111111-1111-4111-8111-111111111111", 102: "22222222-2222-4222-8222-222222222222",
               103: "33333333-3333-4333-8333-333333333333"}
        stack = {"members": [{"vmid": 101}, {"vmid": 102}]}
        records = {101: {"stack": stack, "stack_member": {"primary_vmid": 101}},
                   102: {"stack_member": {"primary_vmid": 101}}, 103: {}}
        for vmid, extra in records.items():
            (root / str(vmid)).mkdir()
            (root / str(vmid) / "oci-compose.json").write_text(json.dumps({
                "schema_version": 1, "vmid": vmid, "installation_id": ids[vmid], "status": "installed",
                "deployment": {"onboot": True}, **extra}))
        return root

    def flags(self, root):
        return {int(path.parent.name): json.loads(path.read_text())["deployment"].get("watchdog")
                for path in root.glob("*/oci-compose.json")}

    def test_every_container_of_the_application_takes_the_choice_and_keeps_the_rest_of_its_record(self):
        from unittest.mock import patch
        import oci_carried_record
        with tempfile.TemporaryDirectory() as folder:
            root = self.registry(folder)
            with patch.object(watchdog, "ensure_service") as service, \
                    patch.object(oci_carried_record, "carry", return_value="carried") as carry:
                self.assertEqual(watchdog.set_watchdog(root, 102, True), [101, 102])
                self.assertEqual(self.flags(root), {101: True, 102: True, 103: None})
                service.assert_called_once_with()
                self.assertEqual([call.args[1] for call in carry.call_args_list], [101, 102])
                self.assertEqual(watchdog.set_watchdog(root, 103, True), [103])
                self.assertEqual(watchdog.set_watchdog(root, 101, False), [101, 102])
                self.assertEqual(self.flags(root), {101: False, 102: False, 103: True})
                self.assertEqual(service.call_count, 2)
            record = json.loads((root / "101" / "oci-compose.json").read_text())
            self.assertEqual(record["deployment"]["onboot"], True)
            self.assertEqual(record["stack"]["members"], [{"vmid": 101}, {"vmid": 102}])


if __name__ == "__main__":
    unittest.main()