mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 19:02:31 +00:00
Merge develop into OCI input review
This commit is contained in:
Jonatan
@@ -0,0 +1,52 @@
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_runtime_settings
|
||||
|
||||
|
||||
class PowerDNSRecursorProfileTest(unittest.TestCase):
|
||||
def test_dns_port_is_available_to_the_non_root_image_only_inside_its_unprivileged_lxc(self):
|
||||
curated = json.loads((ROOT / "catalog/curated/pdns-recursor.json").read_text())
|
||||
app = json.loads((ROOT / "catalog/apps/pdns-recursor.json").read_text())
|
||||
|
||||
expected_sysctls = [{
|
||||
"name": "net.ipv4.ip_unprivileged_port_start",
|
||||
"value": "53",
|
||||
}]
|
||||
self.assertEqual(
|
||||
curated["proxmox"]["installer_profile"]["security"]["sysctls"],
|
||||
expected_sysctls,
|
||||
)
|
||||
self.assertEqual(
|
||||
app["proxmox"]["installer_profile"]["security"]["sysctls"],
|
||||
expected_sysctls,
|
||||
)
|
||||
self.assertTrue(curated["proxmox"]["defaults"]["unprivileged"])
|
||||
self.assertNotIn(
|
||||
"required_capabilities",
|
||||
curated["proxmox"]["installer_profile"].get("security", {}),
|
||||
)
|
||||
self.assertEqual(
|
||||
oci_runtime_settings.sysctl_content({"security": {"sysctls": expected_sysctls}}),
|
||||
"lxc.sysctl.net.ipv4.ip_unprivileged_port_start = 53\n",
|
||||
)
|
||||
self.assertEqual(
|
||||
oci_runtime_settings.include_path(111),
|
||||
Path("/etc/pve/proxmenux/111.sysctls"),
|
||||
)
|
||||
installer = (ROOT / "remote/install_oci.sh").read_text()
|
||||
self.assertIn('SYSCTL_INCLUDE="/etc/pve/proxmenux/${VMID}.sysctls"', installer)
|
||||
self.assertIn('set_lxc_directive "lxc.include" "$SYSCTL_INCLUDE"', installer)
|
||||
self.assertEqual(curated["compatibility"]["validated"], False)
|
||||
self.assertEqual(curated["validation"]["clean_install"], "pending")
|
||||
self.assertEqual(curated, app)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -51,7 +51,6 @@ class RecreationCatalogTests(unittest.TestCase):
|
||||
|
||||
def test_curated_only_index_entry_has_internal_template_id(self):
|
||||
catalog = Catalog(ROOT)
|
||||
self.assertFalse((ROOT / 'catalog' / 'apps' / 'it-tools.json').exists())
|
||||
item = next(item for item in catalog._curated_items({}) if item['id'] == 'it-tools')
|
||||
self.assertEqual(item['template_id'], 'image-it-tools')
|
||||
self.assertIsNone(item['template'])
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from proxmenux_oci.catalog import Catalog
|
||||
from proxmenux_oci.installer import DEFAULT_MODE, build_deployment
|
||||
|
||||
|
||||
class RecordingUI:
|
||||
def __init__(self):
|
||||
self.asked = []
|
||||
|
||||
def ask(self, text, default=None, required=True):
|
||||
self.asked.append(text)
|
||||
return default if default not in (None, "") else "fixture-value"
|
||||
|
||||
def password(self, text, required=True):
|
||||
self.asked.append(text)
|
||||
return "fixture-password"
|
||||
|
||||
def choose(self, text, options, default=None):
|
||||
self.asked.append(text)
|
||||
return default
|
||||
|
||||
def confirm(self, text, default=False):
|
||||
self.asked.append(text)
|
||||
return default
|
||||
|
||||
def checklist(self, text, options, default=None):
|
||||
self.asked.append(text)
|
||||
return default
|
||||
|
||||
def info(self, text):
|
||||
pass
|
||||
|
||||
def message(self, text):
|
||||
pass
|
||||
|
||||
|
||||
def _addresses(ui, bridge, names, *args):
|
||||
return {name: "dhcp" for name in names}, None
|
||||
|
||||
|
||||
class RequiredGeneratedEnvironmentTests(unittest.TestCase):
|
||||
@patch("proxmenux_oci.i18n.language", return_value="en")
|
||||
@patch("proxmenux_oci.installer.host.storages", return_value=[
|
||||
{"storage": "local-lvm", "type": "lvmthin", "avail": 100 * 2**30}
|
||||
])
|
||||
@patch("proxmenux_oci.installer.host.bridges", return_value=[
|
||||
{"iface": "vmbr0", "cidr": "192.0.2.10/24"}
|
||||
])
|
||||
@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
|
||||
@patch("proxmenux_oci.installer.host.gpus", return_value={"intel": [], "amd": [], "nvidia": False})
|
||||
@patch("proxmenux_oci.installer.access.ask_ipv4", return_value=("dhcp", None))
|
||||
@patch("proxmenux_oci.installer.access.ask_addresses", side_effect=_addresses)
|
||||
@patch("proxmenux_oci.installer.secrets.token_hex", return_value="a1" * 24)
|
||||
def test_database_secrets_are_generated_without_a_prompt_or_static_value(self, token_hex, *_):
|
||||
requirements = {
|
||||
"mysql": "MYSQL_ROOT_PASSWORD",
|
||||
"clickhouse": "CLICKHOUSE_PASSWORD",
|
||||
"typesense": "TYPESENSE_API_KEY",
|
||||
}
|
||||
catalog = Catalog(ROOT)
|
||||
|
||||
for app, name in requirements.items():
|
||||
with self.subTest(app=app):
|
||||
ui = RecordingUI()
|
||||
template = catalog.compose(app)
|
||||
plan = build_deployment(template, ui, DEFAULT_MODE)
|
||||
variable = next(item for item in plan["environment"] if item["name"] == name)
|
||||
|
||||
self.assertTrue(variable["sensitive"])
|
||||
self.assertRegex(variable["value"], re.compile(r"^[0-9a-f]{48}$"))
|
||||
self.assertNotIn(name, " ".join(ui.asked))
|
||||
contract = next(item for item in template["container_contract"]["environment"]
|
||||
if item["name"] == name)
|
||||
self.assertNotIn("prompt_user", contract)
|
||||
token_hex.assert_called_with(24)
|
||||
token_hex.reset_mock()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,49 @@
|
||||
"""rqlite uses its image default command and persists the default data directory."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class RqliteProfileTests(unittest.TestCase):
|
||||
def test_default_startup_persists_default_data_directory(self):
|
||||
curated = json.loads((ROOT / "catalog/curated/rqlite.json").read_text())
|
||||
app = json.loads((ROOT / "catalog/apps/rqlite.json").read_text())
|
||||
self.assertEqual(curated, app)
|
||||
|
||||
# The official image's default `run` entrypoint starts rqlited and
|
||||
# uses DATA_DIR=/rqlite/file/data. Avoid an override: positional
|
||||
# non-flag arguments are executed literally by its entrypoint.
|
||||
compose_service = curated["compose_stack"]["services"][0]["compose"]
|
||||
original_compose = json.loads(curated["container_contract"]["original_compose"])
|
||||
self.assertNotIn("runtime", curated["proxmox"]["installer_profile"])
|
||||
image_policy = curated["proxmox"]["image_metadata_policy"]
|
||||
self.assertEqual(image_policy["entrypoint"], "import-from-oci-image")
|
||||
self.assertEqual(image_policy["cmd"], "import-from-oci-image")
|
||||
for config in (compose_service, original_compose["services"]["rqlite"]):
|
||||
self.assertNotIn("command", config)
|
||||
|
||||
image = curated["container_contract"]["image"]
|
||||
self.assertEqual(image["reference"], "ghcr.io/rqlite/rqlite:latest")
|
||||
self.assertEqual(image["registry"], "ghcr.io")
|
||||
self.assertEqual(image["repository"], "ghcr.io/rqlite/rqlite")
|
||||
self.assertEqual(original_compose["services"]["rqlite"]["image"], image["reference"])
|
||||
|
||||
mounts = [
|
||||
volume["container_path"]
|
||||
for volume in curated["container_contract"]["volumes"]
|
||||
]
|
||||
self.assertIn("/rqlite/file", mounts)
|
||||
self.assertTrue(
|
||||
any(
|
||||
path == "/rqlite/file/data"
|
||||
or "/rqlite/file/data".startswith(path.rstrip("/") + "/")
|
||||
for path in mounts
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+168
-2
@@ -2,9 +2,11 @@
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
@@ -148,10 +150,26 @@ class Registry(unittest.TestCase):
|
||||
}
|
||||
for vmid, record in records.items():
|
||||
(root / str(vmid)).mkdir()
|
||||
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({"vmid": vmid, **record}))
|
||||
(root / str(vmid) / "oci-compose.json").write_text(json.dumps({
|
||||
"schema_version": 1, "vmid": vmid,
|
||||
"installation_id": "11111111-1111-4111-8111-111111111111", **record}))
|
||||
(root / "107").mkdir()
|
||||
(root / "107" / "oci-compose.json").write_text("{broken")
|
||||
self.assertEqual(watchdog.watched(root), {101: "Jellyfin", 106: "CT 106"})
|
||||
self.assertEqual(watchdog.watched(root), {
|
||||
101: ("Jellyfin", "11111111-1111-4111-8111-111111111111"),
|
||||
106: ("CT 106", "11111111-1111-4111-8111-111111111111")})
|
||||
|
||||
def test_a_record_without_a_valid_installation_identity_is_not_watched(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
path = root / "109" / "oci-compose.json"
|
||||
path.parent.mkdir()
|
||||
for identity in (None, "", "not-a-uuid", 123, []):
|
||||
with self.subTest(identity=identity):
|
||||
path.write_text(json.dumps({
|
||||
"schema_version": 1, "vmid": 109, "status": "installed",
|
||||
"installation_id": identity, "deployment": {"watchdog": True}}))
|
||||
self.assertEqual(watchdog.watched(root), {})
|
||||
|
||||
|
||||
class Start(unittest.TestCase):
|
||||
@@ -165,6 +183,154 @@ class Start(unittest.TestCase):
|
||||
self.assertEqual(run.call_args.args[0], ["systemd-run", "--scope", "--quiet", "--collect", "pct", "start", "109"])
|
||||
|
||||
|
||||
class InstallationIdentity(unittest.TestCase):
|
||||
"""A recycled VMID is not the application the watchdog saw running."""
|
||||
VMID = 109
|
||||
ORIGINAL = "11111111-1111-4111-8111-111111111111"
|
||||
REPLACEMENT = "22222222-2222-4222-8222-222222222222"
|
||||
|
||||
def setUp(self):
|
||||
folder = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(folder.cleanup)
|
||||
self.base = Path(folder.name)
|
||||
self.registry = self.base / "instances"
|
||||
self.configs = self.base / "lxc"
|
||||
self.cgroups = self.base / "cgroups"
|
||||
for path in (self.registry, self.configs, self.cgroups):
|
||||
path.mkdir()
|
||||
for target, value in (("CONFIGS", self.configs), ("CGROUPS", self.cgroups)):
|
||||
patcher = patch.object(watchdog, target, value)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
patcher = patch.object(watchdog.instances, "ROOT", self.registry)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
for name, target, kwargs in (
|
||||
("start", watchdog, {"return_value": True}),
|
||||
("is_busy", watchdog, {"return_value": False}),
|
||||
("read_tasks", watchdog, {"return_value": []}),
|
||||
("notify", watchdog.oci_operation_notice, {}),
|
||||
("run", watchdog.subprocess, {"side_effect": self.command})):
|
||||
patcher = patch.object(target, name, **kwargs)
|
||||
setattr(self, name, patcher.start())
|
||||
self.addCleanup(patcher.stop)
|
||||
self.states = {}
|
||||
self.record()
|
||||
self.config(self.ORIGINAL)
|
||||
|
||||
def record(self, identity=None, **extra):
|
||||
path = self.registry / str(self.VMID) / "oci-compose.json"
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_text(json.dumps({
|
||||
"schema_version": 1, "vmid": self.VMID,
|
||||
"installation_id": identity or self.ORIGINAL,
|
||||
"status": "installed", "deployment": {"watchdog": True},
|
||||
"template": {"catalog_ui": {"title": {"en_US": "Example"}}}, **extra}))
|
||||
|
||||
def config(self, identity):
|
||||
self.config_text(f"description: proxmenux-instance={identity}\narch: amd64\n")
|
||||
|
||||
def config_text(self, text):
|
||||
(self.configs / f"{self.VMID}.conf").write_text(text)
|
||||
|
||||
def command(self, argv, **kwargs):
|
||||
self.assertEqual(argv, ["pct", "config", str(self.VMID)])
|
||||
return subprocess.CompletedProcess(argv, 0, (self.configs / f"{self.VMID}.conf").read_bytes(), b"")
|
||||
|
||||
def seen_running(self):
|
||||
(self.cgroups / str(self.VMID)).mkdir()
|
||||
watchdog.look(self.states, 100)
|
||||
(self.cgroups / str(self.VMID)).rmdir()
|
||||
|
||||
def assert_left_alone(self):
|
||||
watchdog.look(self.states, 110)
|
||||
self.start.assert_not_called()
|
||||
self.notify.assert_not_called()
|
||||
|
||||
def test_transaction_holder_is_not_started_for_a_stale_application_record(self):
|
||||
self.seen_running()
|
||||
self.config_text("description: proxmenux-transaction=example-update\nhostname: oci-data-holder\n")
|
||||
self.assert_left_alone()
|
||||
|
||||
def test_another_installation_at_the_same_vmid_is_not_started(self):
|
||||
self.seen_running()
|
||||
self.config(self.REPLACEMENT)
|
||||
self.assert_left_alone()
|
||||
|
||||
def test_a_marker_in_old_snapshot_notes_does_not_own_the_live_guest(self):
|
||||
self.seen_running()
|
||||
self.config_text("description: unrelated container\n\n[old]\n"
|
||||
f"description: proxmenux-instance={self.ORIGINAL}\n")
|
||||
self.assert_left_alone()
|
||||
|
||||
def test_an_unreadable_live_configuration_is_not_started(self):
|
||||
self.seen_running()
|
||||
self.run.side_effect = None
|
||||
self.run.return_value = subprocess.CompletedProcess([], 2, b"", b"missing config")
|
||||
self.assert_left_alone()
|
||||
self.run.side_effect = self.command
|
||||
watchdog.look(self.states, 111)
|
||||
self.start.assert_called_once_with(self.VMID)
|
||||
self.assertEqual(self.states[self.VMID]["attempts"], 1)
|
||||
|
||||
def test_a_removed_guest_drops_its_running_history_without_a_pve_query(self):
|
||||
self.seen_running()
|
||||
(self.configs / f"{self.VMID}.conf").unlink()
|
||||
self.assert_left_alone()
|
||||
self.run.assert_not_called()
|
||||
self.assertNotIn(self.VMID, self.states)
|
||||
|
||||
def test_a_configuration_timeout_is_retried_without_consuming_an_attempt(self):
|
||||
self.seen_running()
|
||||
self.run.side_effect = subprocess.TimeoutExpired("pct", 30)
|
||||
self.assert_left_alone()
|
||||
self.run.side_effect = self.command
|
||||
watchdog.look(self.states, 111)
|
||||
self.start.assert_called_once_with(self.VMID)
|
||||
self.assertEqual(self.states[self.VMID]["attempts"], 1)
|
||||
|
||||
def test_a_new_installation_does_not_inherit_running_history(self):
|
||||
self.seen_running()
|
||||
self.record(self.REPLACEMENT)
|
||||
self.config(self.REPLACEMENT)
|
||||
self.assert_left_alone()
|
||||
(self.cgroups / str(self.VMID)).mkdir()
|
||||
watchdog.look(self.states, 120)
|
||||
(self.cgroups / str(self.VMID)).rmdir()
|
||||
watchdog.look(self.states, 130)
|
||||
self.start.assert_called_once_with(self.VMID)
|
||||
|
||||
def test_a_transaction_started_after_the_scan_is_respected(self):
|
||||
self.seen_running()
|
||||
|
||||
def updating(vmid):
|
||||
self.record(status="updating", pending_transaction="/journal")
|
||||
return False
|
||||
|
||||
with patch.object(watchdog, "is_running", side_effect=updating):
|
||||
self.assert_left_alone()
|
||||
|
||||
def test_a_busy_registry_does_not_start_or_consume_a_restart_attempt(self):
|
||||
self.seen_running()
|
||||
with watchdog.instances.locked(self.registry):
|
||||
self.assert_left_alone()
|
||||
watchdog.look(self.states, 111)
|
||||
self.start.assert_called_once_with(self.VMID)
|
||||
self.assertEqual(self.states[self.VMID]["attempts"], 1)
|
||||
|
||||
def test_the_same_installation_is_still_restarted_after_a_crash(self):
|
||||
self.seen_running()
|
||||
watchdog.look(self.states, 110)
|
||||
self.start.assert_called_once_with(self.VMID)
|
||||
self.notify.assert_called_once()
|
||||
|
||||
def test_a_recycled_vmid_does_not_receive_a_crash_loop_alert(self):
|
||||
self.seen_running()
|
||||
self.states[self.VMID].update(attempts=len(watchdog.DELAYS), retry_at=0)
|
||||
self.config(self.REPLACEMENT)
|
||||
self.assert_left_alone()
|
||||
|
||||
|
||||
class Switch(unittest.TestCase):
|
||||
"""Turning the watchdog on or off for a whole application."""
|
||||
def registry(self, folder):
|
||||
|
||||
Reference in New Issue
Block a user