mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-10-16 02:22:04 +00:00
feat(oci): recover applications after a Proxmox reinstall, cluster records and AMD GPU profiles
The installation record of an OCI application travels with its container: a copy inside the container and another in /etc/pve, written together with the one kept on the host. A container restored on a newly installed Proxmox, restored with another ID or moved to another node of a cluster is recognised and registered again, with its private network, hookscript, Rclone mount, host firewall rule and NVIDIA runtime. The Monitor offers the same recovery from the Updates tab. AMD GPUs are offered by generation. A GPU the ROCm image supports takes the profile as it is; one of a supported family (Radeon 680M, 780M) is an experimental option that asks for confirmation and is never proposed; an older one is not offered. The GPU is checked with a real inference before the installation accepts it. Recreate changes what runs recognition in an installed Immich, between the CPU and a GPU of the host. Updates: - A failed update that is restored and checked removes its temporary container and the disks of the failed attempt. - Every container volume is part of the backups, so Jellyfin, Plex and Hugo update with their default installation. - An image published with a Docker-format manifest is recognised by its layers and build time and updates. - The Proxmox notes of a multi-container application link to its LAN address.
This commit is contained in:
MacRimi
@@ -0,0 +1,81 @@
|
||||
"""The record of an installation travels inside its container, so a backup
|
||||
restored on another host still carries it."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_carried_record as carried
|
||||
|
||||
|
||||
def bundle(**extra):
|
||||
return {"schema_version": 1, "kind": carried.KIND, "node": "pve", "vmid": 120,
|
||||
"record": {"vmid": 120, "installation_id": "488ed3cb-1145-477a-b90f-c46777d69fb2"}, **extra}
|
||||
|
||||
|
||||
class CarriedRecordTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.rootfs = Path(tmp.name) / "rootfs"
|
||||
self.rootfs.mkdir()
|
||||
self.outside = Path(tmp.name) / "outside"
|
||||
self.outside.mkdir()
|
||||
self.owner = (os.getuid(), os.getgid())
|
||||
|
||||
def test_the_copy_is_private_and_read_back_as_written(self):
|
||||
carried.write_copy(self.rootfs, bundle(), self.owner)
|
||||
path = self.rootfs / carried.DIRECTORY / carried.NAME
|
||||
self.assertEqual(path.stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(path.parent.stat().st_mode & 0o777, 0o700)
|
||||
self.assertEqual(carried.read_copy(self.rootfs), bundle())
|
||||
self.assertEqual(os.listdir(path.parent), [carried.NAME])
|
||||
|
||||
def test_a_newer_record_replaces_the_copy(self):
|
||||
carried.write_copy(self.rootfs, bundle(), self.owner)
|
||||
carried.write_copy(self.rootfs, bundle(node="other"), self.owner)
|
||||
self.assertEqual(carried.read_copy(self.rootfs)["node"], "other")
|
||||
|
||||
def test_a_link_left_in_place_of_the_folder_is_not_followed(self):
|
||||
(self.rootfs / carried.DIRECTORY).symlink_to(self.outside)
|
||||
with self.assertRaises(OSError):
|
||||
carried.write_copy(self.rootfs, bundle(), self.owner)
|
||||
self.assertEqual(os.listdir(self.outside), [])
|
||||
(self.outside / carried.NAME).write_text(json.dumps(bundle()))
|
||||
self.assertIsNone(carried.read_copy(self.rootfs))
|
||||
|
||||
def test_a_link_left_in_place_of_the_copy_is_not_followed(self):
|
||||
(self.outside / "secret").write_text(json.dumps(bundle()))
|
||||
(self.rootfs / carried.DIRECTORY).mkdir()
|
||||
(self.rootfs / carried.DIRECTORY / carried.NAME).symlink_to(self.outside / "secret")
|
||||
self.assertIsNone(carried.read_copy(self.rootfs))
|
||||
|
||||
def test_what_is_not_a_copy_is_not_read(self):
|
||||
self.assertIsNone(carried.read_copy(self.rootfs))
|
||||
folder = self.rootfs / carried.DIRECTORY
|
||||
folder.mkdir()
|
||||
for content in ("not json", json.dumps([1]), json.dumps(bundle(kind="other")),
|
||||
json.dumps(bundle(schema_version=2)), json.dumps({**bundle(), "record": "text"})):
|
||||
(folder / carried.NAME).write_text(content)
|
||||
self.assertIsNone(carried.read_copy(self.rootfs), content)
|
||||
|
||||
def test_the_time_it_was_saved_does_not_make_a_copy_different(self):
|
||||
self.assertEqual(carried.digest(bundle(saved_at="2026-10-03")), carried.digest(bundle(saved_at="2026-10-04")))
|
||||
self.assertEqual(carried.digest(bundle(generation="a")), carried.digest(bundle(generation="b")))
|
||||
self.assertNotEqual(carried.digest(bundle()), carried.digest(bundle(node="other")))
|
||||
|
||||
def test_the_owner_is_root_of_the_container(self):
|
||||
self.assertEqual(carried.mapped_root("arch: amd64\nunprivileged: 1\n"), (100000, 100000))
|
||||
self.assertEqual(carried.mapped_root("arch: amd64\n"), (0, 0))
|
||||
custom = "unprivileged: 1\nlxc.idmap: u 0 200000 65536\nlxc.idmap: g 0 300000 65536\nlxc.idmap: u 1000 1000 1\n"
|
||||
self.assertEqual(carried.mapped_root(custom), (200000, 300000))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""A container volume is always part of the backups: an update restores its
|
||||
application from one, and refuses a volume that would be left out."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
|
||||
def volumes_without_backup(value, found=None):
|
||||
"""Container paths a template would create as a container volume left out of the backups."""
|
||||
found = [] if found is None else found
|
||||
if isinstance(value, dict):
|
||||
managed = value.get("managed_volume")
|
||||
if (isinstance(managed, dict) and managed.get("backup") is not True
|
||||
and "managed-volume" in (value.get("installation_choice") or [])):
|
||||
found.append(value.get("container_path"))
|
||||
if value.get("mode") == "managed-volume" and value.get("backup") is False:
|
||||
found.append(value.get("container_path"))
|
||||
for item in value.values():
|
||||
volumes_without_backup(item, found)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
volumes_without_backup(item, found)
|
||||
return found
|
||||
|
||||
|
||||
class ContainerVolumeBackupTests(unittest.TestCase):
|
||||
def test_no_application_of_the_catalog_creates_a_volume_left_out_of_the_backups(self):
|
||||
for folder in ("apps", "curated"):
|
||||
for path in sorted((ROOT / "catalog" / folder).glob("*.json")):
|
||||
template = json.loads(path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(volumes_without_backup(template), [], f"{folder}/{path.name}")
|
||||
|
||||
def test_a_converted_compose_backs_up_every_volume_it_creates(self):
|
||||
from proxmenux_oci import converter
|
||||
contract = converter._mount_contract(["/srv/app/config:/config", "/srv/app/cache:/cache",
|
||||
"/srv/app/transcode:/transcode", "/srv/app/tmp:/tmp"], set())
|
||||
self.assertEqual([item["container_path"] for item in contract], ["/config", "/cache", "/transcode", "/tmp"])
|
||||
self.assertTrue(all(item["managed_volume"]["backup"] is True for item in contract))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -53,3 +53,146 @@ class FrigateAmdProfileTests(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class FrigateAmdGenerationTests(unittest.TestCase):
|
||||
"""The ROCm image of Frigate carries the kernels of RDNA2 and newer GPUs:
|
||||
on an older one, such as the integrated Vega of a Ryzen 5000U, its
|
||||
detector aborts, so the profile is not offered there."""
|
||||
|
||||
def profile(self):
|
||||
profiles = Catalog(ROOT).compose("frigate")["proxmox"]["installer_profile"]["hardware_acceleration"]["profiles"]
|
||||
return next(item for item in profiles if item["id"] == "rocm")
|
||||
|
||||
def usable(self, target):
|
||||
from proxmenux_oci.installer import _profile_usable
|
||||
found = {"intel": [], "amd": ["/dev/dri/renderD128"], "nvidia": False, "amd_gfx_target": target}
|
||||
with patch("proxmenux_oci.installer.Path.is_char_device", return_value=True):
|
||||
return _profile_usable(self.profile(), found)
|
||||
|
||||
def test_the_profile_is_offered_on_the_generations_the_image_supports(self):
|
||||
self.assertIn(100300, self.profile()["amd_gfx_targets"])
|
||||
self.assertTrue(self.usable(100300))
|
||||
self.assertTrue(self.usable(110501))
|
||||
self.assertFalse(self.usable(90012))
|
||||
self.assertFalse(self.usable(100100))
|
||||
# The other GPUs of a family it supports are offered as experimental.
|
||||
self.assertTrue(self.usable(100305))
|
||||
self.assertTrue(self.usable(110003))
|
||||
# A driver that does not say the generation does not hide the profile.
|
||||
self.assertTrue(self.usable(None))
|
||||
|
||||
def test_the_generation_is_read_from_the_compute_driver(self):
|
||||
import tempfile
|
||||
from proxmenux_oci import host
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
nodes = Path(directory) / "sys/class/kfd/kfd/topology/nodes"
|
||||
for index, value in enumerate((0, 90012)):
|
||||
(nodes / str(index)).mkdir(parents=True)
|
||||
(nodes / str(index) / "properties").write_text(f"simd_count 32\ngfx_target_version {value}\ndevice_id 5708\n")
|
||||
self.assertEqual(host.amd_gfx_target(Path(directory)), 90012)
|
||||
self.assertIsNone(host.amd_gfx_target(Path(directory) / "none"))
|
||||
self.assertEqual([host.gfx_name(value) for value in (90012, 100300, 110501, 90010, 90402)],
|
||||
["gfx90c", "gfx1030", "gfx1151", "gfx90a", "gfx942"])
|
||||
|
||||
def test_recognition_with_rocm_needs_a_supported_generation(self):
|
||||
from proxmenux_oci import host
|
||||
with patch("proxmenux_oci.host.Path.is_char_device", return_value=True), \
|
||||
patch("proxmenux_oci.host.storages", return_value=[]):
|
||||
for target, expected in ((90012, "generation"), (100100, "generation"), (100305, None),
|
||||
(110003, None), (110501, None), (90010, None), (None, None)):
|
||||
with patch("proxmenux_oci.host.amd_gfx_target", return_value=target):
|
||||
self.assertEqual(host.rocm_blocker("local-lvm"), expected, target)
|
||||
|
||||
def test_how_each_generation_runs_rocm(self):
|
||||
from proxmenux_oci import host
|
||||
expected = {100300: ("native", None), 110501: ("native", None), 90010: ("native", None),
|
||||
100305: ("experimental", "10.3.0"), 100302: ("experimental", "10.3.0"),
|
||||
110003: ("experimental", "11.0.0"), 90012: (None, None), 100100: (None, None),
|
||||
110502: (None, None), None: ("native", None)}
|
||||
for target, (support, override) in expected.items():
|
||||
self.assertEqual((host.rocm_support(target), host.rocm_override(target)), (support, override), target)
|
||||
# An image with fewer kernels supports fewer generations.
|
||||
self.assertIsNone(host.rocm_support(110003, (100300,)))
|
||||
|
||||
def test_the_gpu_is_named_as_its_owner_knows_it(self):
|
||||
from proxmenux_oci import host
|
||||
from types import SimpleNamespace
|
||||
listed = ("35:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] "
|
||||
"Rembrandt [Radeon 680M] [1002:1681] (rev c7)\n")
|
||||
plain = "04:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Lucienne [1002:164c] (rev c1)\n"
|
||||
for output, target, name in ((listed, 100305, "Radeon 680M (gfx1035)"), (plain, 90012, "Lucienne (gfx90c)"),
|
||||
("", 100300, "AMD GPU (gfx1030)"), ("", None, "AMD GPU")):
|
||||
with patch("proxmenux_oci.host.subprocess.run", return_value=SimpleNamespace(stdout=output)), \
|
||||
patch("proxmenux_oci.host.amd_gfx_target", return_value=target):
|
||||
self.assertEqual(host.amd_gpu_name(), name)
|
||||
|
||||
|
||||
@patch("proxmenux_oci.i18n.language", return_value="en")
|
||||
@patch("proxmenux_oci.installer.host.storages", side_effect=storages)
|
||||
@patch("proxmenux_oci.installer.host.bridges", return_value=[{"iface": "vmbr0", "cidr": "192.0.2.10/24"}])
|
||||
@patch("proxmenux_oci.installer.host.timezone", return_value="Europe/Madrid")
|
||||
@patch("proxmenux_oci.installer.Path.is_char_device", return_value=True)
|
||||
@patch("proxmenux_oci.installer.host.amd_gpu_name", return_value="Radeon 680M (gfx1035)")
|
||||
class FrigateExperimentalRocmTests(unittest.TestCase):
|
||||
"""On a GPU ROCm does not support officially the profile is offered as
|
||||
experimental, explained before the larger image is downloaded, and never
|
||||
taken without a yes."""
|
||||
|
||||
WARNING = "ROCm does not support this GPU"
|
||||
|
||||
class UI(RecordingUI):
|
||||
def __init__(self, answers, accept):
|
||||
super().__init__(answers)
|
||||
self.labels, self.messages, self.accept, self.asked_warning = {}, [], accept, 0
|
||||
|
||||
def choose(self, text, options, default=None):
|
||||
self.labels[text] = dict(options)
|
||||
return super().choose(text, options, default)
|
||||
|
||||
def confirm(self, text, default=False):
|
||||
if FrigateExperimentalRocmTests.WARNING in text:
|
||||
self.asked_warning += 1
|
||||
return self.accept
|
||||
return default
|
||||
|
||||
def message(self, text, title=None):
|
||||
self.messages.append(text)
|
||||
|
||||
def build(self, target, answers, accept=True):
|
||||
found = {"intel": [], "amd": ["/dev/dri/renderD128"], "nvidia": False, "amd_gfx_target": target}
|
||||
ui = self.UI(answers, accept)
|
||||
with patch("proxmenux_oci.installer.host.gpus", return_value=found):
|
||||
plan = build_deployment(Catalog(ROOT).compose("frigate"), ui, DEFAULT_MODE)
|
||||
return ui, plan
|
||||
|
||||
def override(self, plan):
|
||||
return [item["value"] for item in plan["environment"] if item["name"] == "HSA_OVERRIDE_GFX_VERSION"]
|
||||
|
||||
def test_the_profile_is_marked_and_confirmed_and_sets_the_generation(self, *_):
|
||||
ui, plan = self.build(100305, {PROMPT: "rocm"})
|
||||
self.assertIn("experimental on this GPU", ui.labels[PROMPT]["rocm"])
|
||||
self.assertEqual(ui.asked_warning, 1)
|
||||
self.assertEqual(plan["hardware_profile"], "rocm")
|
||||
self.assertEqual(self.override(plan), ["10.3.0"])
|
||||
|
||||
def test_a_supported_gpu_has_no_mark_and_no_question(self, *_):
|
||||
ui, plan = self.build(100300, {PROMPT: "rocm"})
|
||||
self.assertNotIn("experimental", ui.labels[PROMPT]["rocm"])
|
||||
self.assertEqual((ui.asked_warning, self.override(plan)), (0, []))
|
||||
|
||||
def test_without_a_yes_the_menu_is_asked_again(self, *_):
|
||||
answers = iter(["rocm", "vaapi"])
|
||||
found = {"intel": [], "amd": ["/dev/dri/renderD128"], "nvidia": False, "amd_gfx_target": 100305}
|
||||
ui = self.UI({}, False)
|
||||
ui.choose = lambda text, options, default=None: next(answers) if text == PROMPT else default
|
||||
with patch("proxmenux_oci.installer.host.gpus", return_value=found):
|
||||
plan = build_deployment(Catalog(ROOT).compose("frigate"), ui, DEFAULT_MODE)
|
||||
self.assertEqual(plan["hardware_profile"], "vaapi")
|
||||
self.assertEqual(self.override(plan), [])
|
||||
|
||||
def test_a_gpu_the_image_cannot_use_is_not_offered_and_the_reason_is_said(self, *_):
|
||||
with patch("proxmenux_oci.installer.host.amd_gpu_name", return_value="Lucienne (gfx90c)"):
|
||||
ui, plan = self.build(90012, {})
|
||||
self.assertNotIn("rocm", ui.labels[PROMPT])
|
||||
self.assertTrue(any("Lucienne (gfx90c)" in message and "not offered" in message for message in ui.messages))
|
||||
|
||||
@@ -91,14 +91,57 @@ class ImmichAccelerationTests(unittest.TestCase):
|
||||
self.assertIn("No usable GPU", ui.messages[0])
|
||||
self.assertEqual(result, ("cpu", "cpu"), mode)
|
||||
|
||||
def test_an_amd_host_that_cannot_run_rocm_says_so_and_recognises_on_the_cpu(self, *_):
|
||||
for blocker, text in (("kfd", "/dev/kfd"), ("space", "40 GB")):
|
||||
with patch("proxmenux_oci.installer.host.rocm_blocker", return_value=blocker):
|
||||
ui, result, _ = self.plan(AMD, DEFAULT_MODE, "amd")
|
||||
def test_an_amd_host_that_cannot_run_rocm_offers_video_only_and_says_why(self, *_):
|
||||
for blocker, text in (("kfd", "/dev/kfd"), ("space", "40 GB"), ("generation", "no support for the AMD GPU")):
|
||||
with patch("proxmenux_oci.installer.host.rocm_blocker", return_value=blocker), \
|
||||
patch("proxmenux_oci.installer.host.amd_gpu_name", return_value="Lucienne (gfx90c)"):
|
||||
ui, result, _ = self.plan(AMD, DEFAULT_MODE)
|
||||
self.assertEqual(ui.options[PROMPT], ["cpu", "amd-video"], blocker)
|
||||
self.assertEqual(ui.defaults[PROMPT], "amd-video", blocker)
|
||||
self.assertEqual(result, ("vaapi", "cpu"), blocker)
|
||||
self.assertTrue(any(text in message and "Recognition runs on the CPU." in message
|
||||
self.assertTrue(any(text in message and "Recognition is not offered" in message
|
||||
for message in ui.messages), blocker)
|
||||
|
||||
def test_a_gpu_rocm_supports_is_proposed_whole(self, *_):
|
||||
for target in (100300, 110501, None):
|
||||
ui, result, plan = self.plan(dict(AMD, amd_gfx_target=target), DEFAULT_MODE)
|
||||
self.assertEqual(ui.defaults[PROMPT], "amd", target)
|
||||
self.assertEqual(result, ("vaapi", "rocm"), target)
|
||||
self.assertIsNone(plan["machine_learning"]["gfx_override"], target)
|
||||
|
||||
def test_a_gpu_rocm_does_not_support_officially_is_offered_as_experimental(self, *_):
|
||||
gpus = dict(AMD, amd_gfx_target=100305)
|
||||
with patch("proxmenux_oci.installer.host.amd_gpu_name", return_value="Radeon 680M (gfx1035)"):
|
||||
# Never the proposal: left alone, recognition stays on the CPU.
|
||||
ui, result, _ = self.plan(gpus, DEFAULT_MODE)
|
||||
self.assertEqual(ui.options[PROMPT], ["cpu", "amd", "amd-video", "amd-ml"])
|
||||
self.assertEqual(ui.defaults[PROMPT], "amd-video")
|
||||
self.assertEqual(result, ("vaapi", "cpu"))
|
||||
# Chosen and confirmed, it is installed with the generation of its family.
|
||||
ui = OptionsUI({PROMPT: "amd"})
|
||||
ui.confirm = lambda text, default=False: True if "ROCm does not support this GPU" in text else default
|
||||
with patch("proxmenux_oci.installer.host.gpus", return_value=gpus):
|
||||
plan = build_deployment(self.template, ui, DEFAULT_MODE)
|
||||
self.assertEqual(plan["machine_learning"]["acceleration"], "rocm")
|
||||
self.assertEqual(plan["machine_learning"]["gfx_override"], "10.3.0")
|
||||
# Declined, the menu is asked again.
|
||||
answers = iter(["amd", "amd-video"])
|
||||
ui = OptionsUI()
|
||||
ui.choose = lambda text, options, default=None: next(answers) if text == PROMPT else default
|
||||
with patch("proxmenux_oci.installer.host.gpus", return_value=gpus):
|
||||
plan = build_deployment(self.template, ui, DEFAULT_MODE)
|
||||
self.assertEqual(plan["machine_learning"]["acceleration"], "cpu")
|
||||
self.assertEqual(plan["video_transcoding"]["acceleration"], "vaapi")
|
||||
|
||||
def test_the_780m_family_is_presented_as_its_generation(self, *_):
|
||||
gpus = dict(AMD, amd_gfx_target=110003)
|
||||
ui = OptionsUI({PROMPT: "amd-ml"})
|
||||
ui.confirm = lambda text, default=False: True if "ROCm does not support this GPU" in text else default
|
||||
with patch("proxmenux_oci.installer.host.gpus", return_value=gpus), \
|
||||
patch("proxmenux_oci.installer.host.amd_gpu_name", return_value="Radeon 780M (gfx1103)"):
|
||||
plan = build_deployment(self.template, ui, DEFAULT_MODE)
|
||||
self.assertEqual(plan["machine_learning"]["gfx_override"], "11.0.0")
|
||||
|
||||
def test_machine_learning_gets_four_cores_and_at_least_four_gigabytes(self, *_):
|
||||
_, _, plan = self.plan(BOTH, DEFAULT_MODE, "cpu")
|
||||
self.assertEqual(plan["machine_learning"]["resources"]["cores"], 4)
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""The recognition of an installed Immich moves between the CPU and a GPU of
|
||||
the host without reinstalling: the machine learning container takes the image
|
||||
and the devices of the new choice."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
import oci_immich_recognition as recognition
|
||||
import oci_rocm_check
|
||||
|
||||
CONFIG = """arch: amd64
|
||||
cores: 4
|
||||
dev0: path=/dev/dri/renderD128,gid=993,mode=0660
|
||||
dev1: path=/dev/kfd,gid=993,mode=0660
|
||||
memory: 8192
|
||||
lxc.environment.runtime: IMMICH_PORT=3003
|
||||
lxc.environment.runtime: HSA_OVERRIDE_GFX_VERSION=10.3.0
|
||||
lxc.environment.runtime: HSA_USE_SVM=0
|
||||
lxc.environment: NVIDIA_VISIBLE_DEVICES=all
|
||||
lxc.hook.mount: /usr/local/lib/proxmenux/oci/nvidia-mount-abc.sh
|
||||
lxc.signal.halt: SIGTERM
|
||||
|
||||
[snapshot]
|
||||
arch: amd64
|
||||
lxc.environment.runtime: HSA_USE_SVM=0
|
||||
"""
|
||||
|
||||
|
||||
class ConfigurationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.path = Path(tmp.name) / "101.conf"
|
||||
self.path.write_text(CONFIG)
|
||||
patcher = patch.object(recognition, "conf", return_value=self.path)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_what_the_previous_choice_left_is_taken_away(self):
|
||||
deleted = []
|
||||
with patch.object(recognition.oci_stack_modify, "entries",
|
||||
return_value={"dev0": "path=/dev/dri/renderD128,gid=993", "dev1": "path=/dev/kfd,gid=993",
|
||||
"dev2": "path=/dev/ttyUSB0,gid=20"}), \
|
||||
patch.object(recognition, "run", side_effect=lambda *command: deleted.append(command[-1])):
|
||||
recognition.strip_gpu(101)
|
||||
self.assertEqual(deleted, ["dev0", "dev1"])
|
||||
text = self.path.read_text()
|
||||
current, _, snapshot = text.partition("\n[")
|
||||
for gone in ("HSA_OVERRIDE", "HSA_USE_SVM", "NVIDIA_VISIBLE_DEVICES", "nvidia-mount"):
|
||||
self.assertNotIn(gone, current)
|
||||
self.assertIn("lxc.environment.runtime: IMMICH_PORT=3003\n", current)
|
||||
self.assertIn("lxc.signal.halt: SIGTERM\n", current)
|
||||
# A snapshot is history: it is not rewritten.
|
||||
self.assertEqual(snapshot, CONFIG.partition("\n[")[2])
|
||||
|
||||
def test_new_settings_go_before_the_snapshots(self):
|
||||
recognition.append_lines(101, ["lxc.environment.runtime: HSA_OVERRIDE_GFX_VERSION=11.0.0"])
|
||||
current, _, snapshot = self.path.read_text().partition("\n[")
|
||||
self.assertTrue(current.endswith("lxc.environment.runtime: HSA_OVERRIDE_GFX_VERSION=11.0.0\n"))
|
||||
self.assertEqual(snapshot, CONFIG.partition("\n[")[2])
|
||||
|
||||
|
||||
class RecordTests(unittest.TestCase):
|
||||
RECORD = {"deployment": {"machine_learning": {
|
||||
"acceleration": "openvino", "render_device": "/dev/dri/renderD128", "model_cache_size_gb": 8,
|
||||
"resources": {"cores": 4, "memory_mb": 8192, "swap_mb": 1024, "cpu_allocation": "quota"}}}}
|
||||
|
||||
def test_the_record_says_what_runs_recognition_and_with_what_resources(self):
|
||||
cpu = recognition.recorded(self.RECORD, "cpu", None, None)
|
||||
self.assertEqual((cpu["acceleration"], cpu["render_device"], cpu["gfx_override"]), ("cpu", None, None))
|
||||
self.assertEqual((cpu["resources"]["memory_mb"], cpu["resources"]["cpu_allocation"]), (4096, "cpuset"))
|
||||
self.assertEqual((cpu["model_cache_size_gb"], cpu["resources"]["swap_mb"]), (8, 1024))
|
||||
rocm = recognition.recorded(self.RECORD, "rocm", "/dev/dri/renderD129", "10.3.0")
|
||||
self.assertEqual((rocm["acceleration"], rocm["render_device"], rocm["gfx_override"]),
|
||||
("rocm", "/dev/dri/renderD129", "10.3.0"))
|
||||
self.assertEqual(rocm["resources"]["memory_mb"], 8192)
|
||||
self.assertEqual(recognition.recorded(self.RECORD, "openvino", "/dev/dri/renderD128", None)["resources"]
|
||||
["cpu_allocation"], "quota")
|
||||
# The record it was read from is not changed.
|
||||
self.assertEqual(self.RECORD["deployment"]["machine_learning"]["acceleration"], "openvino")
|
||||
|
||||
def test_the_record_declares_the_variables_of_the_new_choice(self):
|
||||
base = [{"name": "IMMICH_PORT", "value": "3003"}]
|
||||
names = lambda found: [(entry["name"], entry["value"]) for entry in found]
|
||||
nvidia = recognition.declared(base, "cuda", None)
|
||||
self.assertEqual(names(nvidia), [("IMMICH_PORT", "3003"), ("NVIDIA_DRIVER_CAPABILITIES", "compute,utility")])
|
||||
amd = recognition.declared(nvidia, "rocm", "10.3.0")
|
||||
self.assertEqual(names(amd), [("IMMICH_PORT", "3003"), ("HSA_OVERRIDE_GFX_VERSION", "10.3.0"),
|
||||
("HSA_USE_SVM", "0")])
|
||||
self.assertEqual(names(recognition.declared(nvidia, "rocm", None)), names(base))
|
||||
self.assertEqual(names(recognition.declared(amd, "cpu", None)), names(base))
|
||||
self.assertEqual(names(recognition.declared(amd, "openvino", None)), names(base))
|
||||
|
||||
def test_the_record_declares_the_resources_of_the_new_choice(self):
|
||||
intel = {"cores": 4, "memory_mb": 8192, "swap_mb": 1024, "cpu_allocation": "quota"}
|
||||
self.assertEqual(recognition.resources(intel, "cpu"), {"cores": 4, "memory_mb": 4096, "swap_mb": 1024})
|
||||
self.assertEqual(recognition.resources(intel, "cuda"), {"cores": 4, "memory_mb": 8192, "swap_mb": 1024})
|
||||
self.assertEqual(recognition.resources(recognition.resources(intel, "cpu"), "openvino"), intel)
|
||||
self.assertEqual(intel["cpu_allocation"], "quota")
|
||||
|
||||
def test_a_record_rebuilt_by_an_update_describes_the_new_container(self):
|
||||
def record(**plan):
|
||||
return {"vmid": 101, "template": {"proxmox": {"installer_profile": {"id": "ml", "cpu_allocation": "quota"}}},
|
||||
"deployment": {"environment": [], "rootfs": {"storage": "local-lvm", "size_gb": 12},
|
||||
"resources": {"cores": 4, "memory_mb": 8192, "cpu_allocation": "quota"}, **plan}}
|
||||
|
||||
with patch.object(recognition, "settings", return_value={"rootfs": "local-lvm:vm-101-disk-0,size=40G"}):
|
||||
rebuilt = record()
|
||||
recognition.declare(rebuilt, "rocm", "11.0.0")
|
||||
self.assertEqual(rebuilt["deployment"]["rootfs"]["size_gb"], 40)
|
||||
self.assertEqual(rebuilt["deployment"]["resources"], {"cores": 4, "memory_mb": 8192})
|
||||
self.assertEqual(rebuilt["template"]["proxmox"]["installer_profile"], {"id": "ml"})
|
||||
self.assertEqual([entry["name"] for entry in rebuilt["deployment"]["environment"]],
|
||||
["HSA_OVERRIDE_GFX_VERSION", "HSA_USE_SVM"])
|
||||
recognition.declare(rebuilt, "openvino", None)
|
||||
self.assertEqual(rebuilt["template"]["proxmox"]["installer_profile"]["cpu_allocation"], "quota")
|
||||
self.assertEqual(rebuilt["deployment"]["environment"], [])
|
||||
# A record still described by its native configuration is read from it.
|
||||
native = record(native_config="arch: amd64")
|
||||
recognition.declare(native, "rocm", "11.0.0")
|
||||
self.assertEqual(native, record(native_config="arch: amd64"))
|
||||
|
||||
def test_a_choice_this_host_cannot_serve_is_refused_before_anything_changes(self):
|
||||
with self.assertRaises(ValueError):
|
||||
recognition.validate("vulkan", None, None)
|
||||
with self.assertRaises(ValueError):
|
||||
recognition.validate("rocm", "/dev/dri/renderD999", None)
|
||||
with self.assertRaises(ValueError):
|
||||
recognition.validate("openvino", "/etc/passwd", None)
|
||||
with self.assertRaises(ValueError):
|
||||
recognition.validate("cpu", None, "10.3.0")
|
||||
recognition.validate("cpu", None, None)
|
||||
|
||||
def test_only_an_immich_of_proxmenux_is_changed(self):
|
||||
def member(vmid, adapter, role):
|
||||
return {"vmid": vmid, "deployment": {"replay_profile": {"adapter": adapter, "role": role}}}
|
||||
|
||||
def records(roles, adapter="install_immich_stack.sh"):
|
||||
found = {vmid: member(vmid, adapter, role) for vmid, role in roles.items()}
|
||||
found[100]["stack"] = {"members": [{"vmid": vmid} for vmid in roles]}
|
||||
return found
|
||||
|
||||
immich = records({100: "server", 101: "machine-learning", 102: "database", 103: "valkey"})
|
||||
with patch.object(recognition.instances, "read", side_effect=lambda root, vmid: immich[vmid]):
|
||||
_, found = recognition.members(Path("/nonexistent"), 100)
|
||||
self.assertEqual(found["machine-learning"]["vmid"], 101)
|
||||
for other in (records({100: "application", 101: "database"}, "install_tandoor_stack.sh"),
|
||||
records({100: "server", 101: "machine-learning"})):
|
||||
with patch.object(recognition.instances, "read", side_effect=lambda root, vmid: other[vmid]):
|
||||
with self.assertRaises(ValueError):
|
||||
recognition.members(Path("/nonexistent"), 100)
|
||||
|
||||
|
||||
class RevertTests(unittest.TestCase):
|
||||
def test_the_previous_choice_is_what_the_record_says(self):
|
||||
record = {"deployment": {"machine_learning": {"acceleration": "rocm", "render_device": "/dev/dri/renderD128",
|
||||
"gfx_override": "10.3.0"}}}
|
||||
self.assertEqual(recognition.chosen(record), ("rocm", "/dev/dri/renderD128", "10.3.0"))
|
||||
self.assertEqual(recognition.chosen({"deployment": {}}), ("cpu", None, None))
|
||||
|
||||
def test_a_failed_change_puts_the_previous_choice_back_the_way_it_was_given(self):
|
||||
calls = []
|
||||
with patch.object(recognition.oci_stack_modify, "is_running", return_value=True), \
|
||||
patch.object(recognition, "stop", side_effect=lambda vmid: calls.append(("stop", vmid))), \
|
||||
patch.object(recognition, "apply", side_effect=lambda *arguments: calls.append(("apply",) + arguments[2:])), \
|
||||
patch.object(recognition.subprocess, "run", side_effect=lambda command, **_: calls.append(tuple(command[:3]))):
|
||||
recognition.revert(Path("/nonexistent"), 100, 101, {"cpu": "image"}, ("cpu", None, None), True)
|
||||
self.assertEqual(calls, [("stop", 101), ("apply", 101, {"cpu": "image"}, "cpu", None, None),
|
||||
("pct", "start", "101")])
|
||||
|
||||
def test_a_revert_that_fails_does_not_hide_the_first_error(self):
|
||||
with patch.object(recognition.oci_stack_modify, "is_running", return_value=False), \
|
||||
patch.object(recognition, "apply", side_effect=RuntimeError("pct set: locked")), \
|
||||
patch.object(recognition, "msg_warn") as warned, \
|
||||
patch.object(recognition.subprocess, "run") as started:
|
||||
recognition.revert(Path("/nonexistent"), 100, 101, {}, ("cpu", None, None), True)
|
||||
self.assertIn("pct set: locked", warned.call_args[0][0])
|
||||
started.assert_not_called()
|
||||
|
||||
|
||||
class RocmProbeTests(unittest.TestCase):
|
||||
def test_the_probe_runs_with_what_the_container_tells_rocm(self):
|
||||
config = ("arch: amd64\nlxc.environment.runtime: PATH=/opt/venv/bin\n"
|
||||
"lxc.environment.runtime: HSA_OVERRIDE_GFX_VERSION=10.3.0\nlxc.environment.runtime: HSA_USE_SVM=0\n"
|
||||
"lxc.environment.runtime: HSA_BAD=1; reboot\n")
|
||||
self.assertEqual(oci_rocm_check.variables(config), ["HSA_OVERRIDE_GFX_VERSION=10.3.0", "HSA_USE_SVM=0"])
|
||||
self.assertEqual(oci_rocm_check.variables("arch: amd64\n"), [])
|
||||
listed = "arch: amd64\nenv: PATH=/opt/venv/bin\0HSA_OVERRIDE_GFX_VERSION=11.0.0\0HSA_USE_SVM=0\0HOME=/root\n"
|
||||
self.assertEqual(oci_rocm_check.variables(listed), ["HSA_OVERRIDE_GFX_VERSION=11.0.0", "HSA_USE_SVM=0"])
|
||||
|
||||
|
||||
def protobuf_fields(data):
|
||||
"""The fields of a protobuf message as (number, value) pairs."""
|
||||
position = 0
|
||||
|
||||
def varint():
|
||||
nonlocal position
|
||||
value = shift = 0
|
||||
while True:
|
||||
byte = data[position]
|
||||
position += 1
|
||||
value |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
if not byte & 0x80:
|
||||
return value
|
||||
|
||||
while position < len(data):
|
||||
key = varint()
|
||||
number, wire = key >> 3, key & 7
|
||||
if wire == 0:
|
||||
yield number, varint()
|
||||
elif wire == 2:
|
||||
size = varint()
|
||||
yield number, data[position:position + size]
|
||||
position += size
|
||||
else:
|
||||
position += 4 if wire == 5 else 8
|
||||
|
||||
|
||||
def packed_numbers(data):
|
||||
numbers, value, shift = [], 0, 0
|
||||
for byte in data:
|
||||
value |= (byte & 0x7F) << shift
|
||||
shift += 7
|
||||
if not byte & 0x80:
|
||||
numbers.append(value)
|
||||
value = shift = 0
|
||||
return numbers
|
||||
|
||||
|
||||
class RocmProbeModelTests(unittest.TestCase):
|
||||
def test_the_embedded_model_has_layers_that_fit_each_other(self):
|
||||
import base64
|
||||
model = base64.b64decode(oci_rocm_check.MODEL, validate=True)
|
||||
graph = next(value for number, value in protobuf_fields(model) if number == 7)
|
||||
shapes = {}
|
||||
for number, tensor in protobuf_fields(graph):
|
||||
if number != 5:
|
||||
continue
|
||||
dims, name = [], None
|
||||
for field, value in protobuf_fields(tensor):
|
||||
if field == 1:
|
||||
dims += packed_numbers(value) if isinstance(value, bytes) else [value]
|
||||
elif field == 8:
|
||||
name = value.decode()
|
||||
shapes[name] = dims
|
||||
# The dense layer takes one value per channel of the convolution.
|
||||
self.assertEqual(shapes["w"], [4, 3, 3, 3])
|
||||
self.assertEqual(shapes["fw"], [shapes["w"][0], 2])
|
||||
self.assertEqual((shapes["b"], shapes["fb"]), ([4], [2]))
|
||||
|
||||
|
||||
class ChoicesTests(unittest.TestCase):
|
||||
def choices(self, found, blocker=None):
|
||||
from proxmenux_oci import stack_recreation
|
||||
with patch("proxmenux_oci.host.gpus", return_value=found), \
|
||||
patch("proxmenux_oci.host.rocm_blocker", return_value=blocker):
|
||||
return stack_recreation.recognition_choices("local-lvm")
|
||||
|
||||
def test_the_choices_are_what_the_host_has(self):
|
||||
node = "/dev/dri/renderD128"
|
||||
tags = lambda found, blocker=None: [tag for tag, _, _ in self.choices(found, blocker)]
|
||||
self.assertEqual(tags({"intel": [], "amd": [], "nvidia": False}), ["cpu"])
|
||||
self.assertEqual(tags({"intel": [node], "amd": [], "nvidia": True}), ["cpu", "openvino", "cuda"])
|
||||
self.assertEqual(tags({"intel": [], "amd": [node], "nvidia": False, "amd_gfx_target": 100300}), ["cpu", "rocm"])
|
||||
self.assertEqual(tags({"intel": [], "amd": [node], "nvidia": False, "amd_gfx_target": 90012}, "generation"), ["cpu"])
|
||||
|
||||
def test_an_amd_gpu_rocm_does_not_support_officially_is_marked(self):
|
||||
found = {"intel": [], "amd": ["/dev/dri/renderD128"], "nvidia": False, "amd_gfx_target": 100305}
|
||||
tag, label, details = self.choices(found)[-1]
|
||||
self.assertEqual(tag, "rocm")
|
||||
self.assertIn("experimental", label)
|
||||
self.assertEqual((details["experimental"], details["override"], details["render"]),
|
||||
(True, "10.3.0", "/dev/dri/renderD128"))
|
||||
native = self.choices(dict(found, amd_gfx_target=110501))[-1]
|
||||
self.assertEqual((native[2]["experimental"], native[2]["override"]), (False, None))
|
||||
self.assertNotIn("experimental", native[1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -32,6 +32,8 @@ class RemovalCleanupTests(unittest.TestCase):
|
||||
self.snippets.mkdir()
|
||||
self.apps = self.root / "apps"
|
||||
self.apps.mkdir()
|
||||
self.records = self.root / "cluster-records"
|
||||
self.records.mkdir()
|
||||
self.cluster = self.root / "proxmenux"
|
||||
self.cluster.mkdir()
|
||||
self.legacy = self.root / "legacy"
|
||||
@@ -41,6 +43,7 @@ class RemovalCleanupTests(unittest.TestCase):
|
||||
patch.object(oci_remove, "CLUSTER_NODES", self.nodes),
|
||||
patch.object(oci_remove, "SNIPPETS", self.snippets),
|
||||
patch.object(oci_remove, "MONITOR_APPS", self.apps),
|
||||
patch.object(oci_remove, "CLUSTER_RECORDS", self.records),
|
||||
patch.object(oci_remove, "HOST_MONITOR_INCLUDES", self.host_monitor),
|
||||
patch.object(runtime_settings, "include_path", lambda vmid: self.cluster / f"{vmid}.sysctls"),
|
||||
patch.object(runtime_settings, "legacy_include_path", lambda vmid: self.legacy / f"{vmid}.proxmenux-sysctls"),
|
||||
@@ -67,6 +70,12 @@ class RemovalCleanupTests(unittest.TestCase):
|
||||
self.assertFalse((self.apps / "113.json").exists())
|
||||
self.assertEqual(json.loads((self.apps / ".oci-dismissed.json").read_text()), {"112": "b"})
|
||||
|
||||
def test_the_copy_of_the_record_in_the_cluster_goes_with_the_container(self):
|
||||
(self.records / "120.json").write_text("{}")
|
||||
(self.records / "121.json").write_text("{}")
|
||||
oci_remove.remove_host_state(120)
|
||||
self.assertEqual([path.name for path in self.records.iterdir()], ["121.json"])
|
||||
|
||||
def test_a_published_view_with_content_is_kept(self):
|
||||
shared = self.root / "shared"
|
||||
(shared / "rw/drive").mkdir(parents=True)
|
||||
|
||||
@@ -0,0 +1,550 @@
|
||||
"""A container restored from a backup is found, checked against what this
|
||||
host has, and registered again only when its whole application can be."""
|
||||
|
||||
import copy
|
||||
import ipaddress
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_carried_record as carried
|
||||
import oci_instances as instances
|
||||
import oci_restore_recovery as recovery
|
||||
|
||||
APP = "83e373f2-5ccc-4548-9b89-22d0956d1c77"
|
||||
DB = "488ed3cb-1145-477a-b90f-c46777d69fb2"
|
||||
|
||||
|
||||
def saved(identity, hostname, extra=""):
|
||||
return f"#<div>Tandoor</div>\n#<!-- proxmenux-instance={identity} -->\narch: amd64\nhostname: {hostname}\n{extra}"
|
||||
|
||||
|
||||
def config(identity, extra=""):
|
||||
return (f"arch: amd64\ndescription: <!-- proxmenux-instance={identity} -->\nhostname: app\nmemory: 2048\n"
|
||||
f"rootfs: local-lvm:vm-119-disk-0,size=8G\nunprivileged: 1\n{extra}")
|
||||
|
||||
|
||||
CONSOLE = ("lxc.console.logfile: /var/log/proxmenux/oci/{0}.console.log\n"
|
||||
"lxc.hook.pre-start: /bin/sh -c 'mkdir -p /var/log/proxmenux/oci; "
|
||||
"test -x /usr/local/share/proxmenux/oci/engine/remote/oci_console_mark.sh "
|
||||
"&& /usr/local/share/proxmenux/oci/engine/remote/oci_console_mark.sh {0}; exit 0'\n")
|
||||
|
||||
|
||||
def record(vmid, identity, **extra):
|
||||
return {"schema_version": 1, "vmid": vmid, "installation_id": identity, "status": "installed",
|
||||
"template": {"catalog_ui": {"title": {"en_US": "Tandoor Recipes"}}},
|
||||
"deployment": {"vmid": vmid, "hostname": "app", "rootfs": {"storage": "local-lvm", "size_gb": 8}},
|
||||
"observed": {"config": config(identity), "config_sha256": "old", "api_config_sha256": "old",
|
||||
"image": {"manifest_digest": "sha256:1"}, "archive_path": "/gone.tar",
|
||||
"resolved_registry_digest": "sha256:1"}, **extra}
|
||||
|
||||
|
||||
def entry(vmid, identity, value, text=None):
|
||||
return {"vmid": vmid, "installation_id": identity, "hostname": f"ct{vmid}", "problem": None,
|
||||
"config": text if text is not None else config(identity),
|
||||
"copy": {"schema_version": 1, "kind": carried.KIND, "node": "old", "vmid": vmid, "record": value}}
|
||||
|
||||
|
||||
class DiscoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.root = Path(tmp.name)
|
||||
|
||||
def register(self, vmid, identity):
|
||||
with patch.object(instances, "private_directory", lambda path: path.mkdir(parents=True, exist_ok=True)):
|
||||
instances.write(instances.location(self.root, vmid), record(vmid, identity))
|
||||
|
||||
def pending(self, guests):
|
||||
with patch.object(recovery, "local_guests", return_value=guests):
|
||||
return [row["vmid"] for row in recovery.pending(self.root)]
|
||||
|
||||
def test_the_mark_is_read_from_the_saved_configuration(self):
|
||||
self.assertEqual(recovery.marker(saved(APP, "app")), APP)
|
||||
self.assertIsNone(recovery.marker("arch: amd64\nhostname: plain\n"))
|
||||
|
||||
def test_a_marked_container_without_record_waits_for_recovery(self):
|
||||
guests = {119: saved(APP, "app"), 120: saved(DB, "db"), 100: "arch: amd64\nhostname: plain\n"}
|
||||
self.assertEqual(self.pending(guests), [119, 120])
|
||||
self.register(119, APP)
|
||||
self.assertEqual(self.pending(guests), [120])
|
||||
|
||||
def test_the_record_of_another_installation_does_not_count(self):
|
||||
self.register(119, DB)
|
||||
self.assertEqual(self.pending({119: saved(APP, "app")}), [119])
|
||||
|
||||
def test_a_container_that_carries_no_record_is_not_offered_again(self):
|
||||
guests = {119: saved(APP, "app"), 120: saved(DB, "db")}
|
||||
with patch.object(instances, "private_directory", lambda path: path.mkdir(parents=True, exist_ok=True)), \
|
||||
patch.object(recovery, "local_guests", return_value=guests):
|
||||
recovery.set_unrecoverable(self.root, [{"vmid": 119, "installation_id": APP}])
|
||||
self.assertEqual(self.pending(guests), [120])
|
||||
# Another installation restored on that ID is a new case.
|
||||
self.assertEqual(self.pending({119: saved(DB, "other")}), [119])
|
||||
|
||||
def test_a_clone_of_a_registered_container_is_left_alone(self):
|
||||
self.register(119, APP)
|
||||
self.assertEqual(self.pending({119: saved(APP, "app"), 130: saved(APP, "copy")}), [])
|
||||
|
||||
|
||||
class ApplicationTests(unittest.TestCase):
|
||||
def stack(self):
|
||||
database = record(120, DB, stack_member={"stack_id": APP, "primary_vmid": 119, "name": "database"})
|
||||
application = record(119, APP, stack_member={"stack_id": APP, "primary_vmid": 119, "name": "application"})
|
||||
application["stack"] = {"id": APP, "template": {"catalog_ui": {"title": {"en_US": "Tandoor Recipes"}}},
|
||||
"members": [copy.deepcopy(application), copy.deepcopy(database)]}
|
||||
return application, database
|
||||
|
||||
def plans(self, found, guests, known=None):
|
||||
with patch.object(recovery, "local_guests", return_value=guests), \
|
||||
patch.object(recovery, "registered_identities", return_value=known or {}):
|
||||
return recovery.applications(Path("/nonexistent"), found)
|
||||
|
||||
def test_the_containers_of_one_application_are_recovered_together(self):
|
||||
application, database = self.stack()
|
||||
plans = self.plans([entry(120, DB, database), entry(119, APP, application)],
|
||||
{119: saved(APP, "app"), 120: saved(DB, "db")})
|
||||
self.assertEqual(len(plans), 1)
|
||||
self.assertEqual((plans[0]["title"], plans[0]["primary"], plans[0]["blockers"]), ("Tandoor Recipes", 119, []))
|
||||
self.assertEqual([m["state"] for m in plans[0]["members"]], ["restored", "restored"])
|
||||
|
||||
def test_an_application_with_a_container_missing_is_not_recovered(self):
|
||||
application, _ = self.stack()
|
||||
plans = self.plans([entry(119, APP, application)], {119: saved(APP, "app")})
|
||||
self.assertEqual(len(plans[0]["blockers"]), 1)
|
||||
self.assertIn("CT 120 (database)", plans[0]["blockers"][0])
|
||||
|
||||
def test_containers_restored_with_other_ids_keep_their_application(self):
|
||||
application, database = self.stack()
|
||||
application["stack"]["deployment"] = {"base_vmid": 119, "services": [
|
||||
{"vmid": 119, "name": "application"},
|
||||
{"vmid": 120, "name": "database", "healthcheck": {"timeout_seconds": 120}}]}
|
||||
application["deployment"]["create_arguments"] = ["119", "local:vztmpl/app.tar"]
|
||||
application["observed"]["config"] = config(APP, CONSOLE.format(119))
|
||||
found = [entry(140, APP, application), entry(135, DB, database)]
|
||||
found[0]["copy"]["stack_contract"] = {"schema": 1, "dependencies": [
|
||||
{"vmid": 120, "label": "PostgreSQL", "healthcheck": {"type": "running", "timeout_seconds": 120}}]}
|
||||
plans = self.plans(found, {140: saved(APP, "app"), 135: saved(DB, "db")})
|
||||
plan = plans[0]
|
||||
self.assertEqual((plan["blockers"], plan["renumbered"], plan["primary"]), ([], {119: 140, 120: 135}, 140))
|
||||
self.assertEqual([m["vmid"] for m in plan["members"]], [140, 135])
|
||||
main, other = (e["copy"]["record"] for e in plan["restored"])
|
||||
self.assertEqual((main["vmid"], main["deployment"]["vmid"], main["stack_member"]["primary_vmid"]), (140, 140, 140))
|
||||
self.assertEqual((other["vmid"], other["stack_member"]["primary_vmid"]), (135, 140))
|
||||
self.assertEqual([m["vmid"] for m in main["stack"]["members"]], [140, 135])
|
||||
self.assertEqual(main["stack"]["deployment"]["base_vmid"], 140)
|
||||
services = main["stack"]["deployment"]["services"]
|
||||
self.assertEqual([s["vmid"] for s in services], [140, 135])
|
||||
# A number that is not an ID stays: 120 seconds is not CT 120.
|
||||
self.assertEqual(services[1]["healthcheck"]["timeout_seconds"], 120)
|
||||
contract = plan["restored"][0]["copy"]["stack_contract"]["dependencies"][0]
|
||||
self.assertEqual((contract["vmid"], contract["healthcheck"]["timeout_seconds"]), (135, 120))
|
||||
self.assertEqual(main["deployment"]["create_arguments"][0], "140")
|
||||
self.assertIn("/var/log/proxmenux/oci/140.console.log", main["observed"]["config"])
|
||||
self.assertIn("oci_console_mark.sh 140; exit 0", main["observed"]["config"])
|
||||
self.assertNotIn("119", main["observed"]["config"].replace("vm-119-disk", ""))
|
||||
self.assertEqual(plan["restored"][0]["recorded"], 119)
|
||||
|
||||
def test_the_lines_of_a_configuration_that_carry_the_id(self):
|
||||
text = config(APP, CONSOLE.format(119) + "lxc.include: /etc/pve/proxmenux/119.sysctls\nmemory: 119\n")
|
||||
result = recovery.renumbered_lines(text, 119, 140)
|
||||
self.assertIn("lxc.console.logfile: /var/log/proxmenux/oci/140.console.log\n", result)
|
||||
self.assertIn("oci_console_mark.sh 140; exit 0'\n", result)
|
||||
self.assertIn("lxc.include: /etc/pve/proxmenux/140.sysctls\n", result)
|
||||
# Proxmox renames the volumes; anything else that happens to read 119 stays.
|
||||
self.assertIn("rootfs: local-lvm:vm-119-disk-0,size=8G\n", result)
|
||||
self.assertIn("memory: 119\n", result)
|
||||
self.assertTrue(result.endswith("\n"))
|
||||
legacy = recovery.renumbered_lines("lxc.include: /etc/pve/lxc/119.proxmenux-sysctls\n", 119, 140)
|
||||
self.assertEqual(legacy, "lxc.include: /etc/pve/proxmenux/140.sysctls\n")
|
||||
|
||||
def test_a_container_without_its_copy_stops_the_recovery(self):
|
||||
broken = dict(entry(119, APP, record(119, APP)), copy=None, problem="no copy")
|
||||
plans = self.plans([broken], {119: saved(APP, "app")})
|
||||
self.assertEqual(plans[0]["blockers"], ["CT 119: no copy"])
|
||||
|
||||
|
||||
class NetworkTests(unittest.TestCase):
|
||||
def check(self, text, links=("lo", "vmbr0"), addresses=None, routes=(), defined=None, guests=None):
|
||||
plan = {"restored": [entry(119, APP, record(119, APP), text)], "members": [{"vmid": 119}],
|
||||
"blockers": [], "notes": [], "bridges": {}}
|
||||
with patch.object(recovery, "links", return_value=set(links)), \
|
||||
patch.object(recovery, "host_addresses", return_value=addresses or {}), \
|
||||
patch.object(recovery, "routed_networks", return_value=list(routes)), \
|
||||
patch.object(recovery, "defined_network", return_value=defined), \
|
||||
patch.object(recovery, "local_guests", return_value=guests or {}):
|
||||
recovery.check_networks(plan)
|
||||
return plan
|
||||
|
||||
PRIVATE = config(APP, "net0: name=eth0,bridge=vmbr0,ip=dhcp\nnet1: name=eth1,bridge=vmbr11,ip=10.77.1.40/24\n")
|
||||
|
||||
def test_a_missing_private_network_is_created_with_the_same_subnet(self):
|
||||
plan = self.check(self.PRIVATE)
|
||||
self.assertEqual(plan["blockers"], [])
|
||||
self.assertEqual(plan["bridges"], {"vmbr11": ipaddress.ip_network("10.77.1.0/24")})
|
||||
|
||||
def test_a_private_network_already_here_is_left_as_it_is(self):
|
||||
plan = self.check(self.PRIVATE, links=("vmbr0", "vmbr11"),
|
||||
addresses={"vmbr11": [ipaddress.ip_interface("10.77.1.1/24")]})
|
||||
self.assertEqual((plan["blockers"], plan["bridges"]), ([], {}))
|
||||
|
||||
def test_the_subnet_in_use_on_this_host_cancels_the_recovery(self):
|
||||
plan = self.check(self.PRIVATE, links=("vmbr0", "vmbr20"),
|
||||
addresses={"vmbr20": [ipaddress.ip_interface("10.77.1.1/24")]})
|
||||
self.assertEqual(len(plan["blockers"]), 1)
|
||||
self.assertIn("10.77.1.0/24", plan["blockers"][0])
|
||||
routed = self.check(self.PRIVATE, routes=[("vmbr0", ipaddress.ip_network("10.0.0.0/8"))])
|
||||
self.assertEqual(len(routed["blockers"]), 1)
|
||||
|
||||
def test_the_bridge_name_used_for_another_network_cancels_the_recovery(self):
|
||||
plan = self.check(self.PRIVATE, links=("vmbr0", "vmbr11"),
|
||||
addresses={"vmbr11": [ipaddress.ip_interface("192.168.5.1/24")]})
|
||||
self.assertEqual(len(plan["blockers"]), 1)
|
||||
defined = self.check(self.PRIVATE, defined=ipaddress.ip_network("192.168.5.0/24"))
|
||||
self.assertEqual(len(defined["blockers"]), 1)
|
||||
|
||||
def test_an_address_taken_by_another_container_cancels_the_recovery(self):
|
||||
plan = self.check(self.PRIVATE, guests={200: "net0: name=eth0,bridge=vmbr30,ip=10.77.1.40/24\n"})
|
||||
self.assertTrue(any("10.77.1.40" in line for line in plan["blockers"]))
|
||||
|
||||
def test_a_missing_bridge_of_the_local_network_is_not_invented(self):
|
||||
plan = self.check(config(APP, "net0: name=eth0,bridge=vmbr5,ip=dhcp\n"))
|
||||
self.assertEqual(len(plan["blockers"]), 1)
|
||||
self.assertEqual(plan["bridges"], {})
|
||||
|
||||
|
||||
class RecordTests(unittest.TestCase):
|
||||
def test_what_a_restore_changes(self):
|
||||
related = recovery.restore_related
|
||||
self.assertTrue(related("rootfs", "local-lvm:vm-119-disk-0,size=8G", "tank:subvol-119-disk-0,size=8G"))
|
||||
self.assertTrue(related("mp0", "local-lvm:vm-119-disk-1,mp=/data,backup=1,size=2G",
|
||||
"tank:subvol-119-disk-1,mp=/data,backup=1,size=2G"))
|
||||
self.assertFalse(related("mp0", "local-lvm:vm-119-disk-1,mp=/data,backup=1,size=2G",
|
||||
"local-lvm:vm-119-disk-1,mp=/other,backup=1,size=2G"))
|
||||
self.assertFalse(related("mp0", "/srv/a,mp=/data", "/srv/b,mp=/data"))
|
||||
self.assertTrue(related("net0", "name=eth0,bridge=vmbr0,hwaddr=AA:AA,ip=dhcp", "name=eth0,bridge=vmbr0,hwaddr=BB:BB,ip=dhcp"))
|
||||
self.assertFalse(related("net0", "name=eth0,bridge=vmbr0,ip=dhcp", "name=eth0,bridge=vmbr1,ip=dhcp"))
|
||||
self.assertTrue(related("hookscript", "local:snippets/a.sh", "nas:snippets/a.sh"))
|
||||
self.assertTrue(related("dev0", "path=/dev/dri/renderD128", None))
|
||||
self.assertTrue(related("mp1", "/srv/media,mp=/media", None))
|
||||
self.assertFalse(related("mp1", "local-lvm:vm-119-disk-1,mp=/data,backup=1,size=2G", None))
|
||||
self.assertFalse(related("memory", "2048", "4096"))
|
||||
self.assertFalse(related("mp2", None, "/etc,mp=/host"))
|
||||
|
||||
def standalone(self, text, value=None):
|
||||
value = value or record(119, APP)
|
||||
observed = {"config": text, "config_sha256": "new", "api_config_sha256": "new"}
|
||||
with patch.object(instances, "observe", return_value=observed) as observe, \
|
||||
patch.object(recovery, "capture_sources", return_value={}), \
|
||||
patch.object(recovery, "capture_gpu_devices", return_value={}):
|
||||
return recovery.standalone_record(entry(119, APP, value, text)), observe
|
||||
|
||||
def test_volumes_on_another_storage_become_the_reference(self):
|
||||
text = config(APP).replace("local-lvm:vm-119-disk-0", "tank:subvol-119-disk-0")
|
||||
result, observe = self.standalone(text)
|
||||
observe.assert_called_once()
|
||||
self.assertEqual(result["observed"]["config"], text)
|
||||
self.assertEqual(result["deployment"]["rootfs"]["storage"], "tank")
|
||||
|
||||
def test_a_change_made_outside_stays_a_difference(self):
|
||||
text = config(APP).replace("local-lvm:vm-119-disk-0", "tank:subvol-119-disk-0").replace("memory: 2048", "memory: 4096")
|
||||
result, observe = self.standalone(text)
|
||||
observe.assert_not_called()
|
||||
self.assertIn("memory: 2048", result["observed"]["config"])
|
||||
self.assertIn("rootfs: tank:subvol-119-disk-0,size=8G", result["observed"]["config"])
|
||||
self.assertNotIn("api_config_sha256", result["observed"])
|
||||
self.assertEqual(result["deployment"]["rootfs"]["storage"], "tank")
|
||||
|
||||
def test_a_single_container_restored_with_another_id_is_registered_under_it(self):
|
||||
value = record(119, APP)
|
||||
value["observed"]["config"] = config(APP, CONSOLE.format(119))
|
||||
found = entry(141, APP, value, config(APP, CONSOLE.format(141)).replace("vm-119-disk-0", "vm-141-disk-0"))
|
||||
with patch.object(recovery, "local_guests", return_value={141: saved(APP, "app")}), \
|
||||
patch.object(recovery, "registered_identities", return_value={}):
|
||||
plan = recovery.applications(Path("/nonexistent"), [found])[0]
|
||||
self.assertEqual((plan["blockers"], plan["renumbered"]), ([], {119: 141}))
|
||||
observed = {"config": found["config"], "config_sha256": "new"}
|
||||
with patch.object(instances, "observe", return_value=observed) as observe:
|
||||
result = recovery.standalone_record(plan["restored"][0])
|
||||
observe.assert_called_once()
|
||||
self.assertEqual((result["vmid"], result["deployment"]["vmid"]), (141, 141))
|
||||
|
||||
def test_the_copy_cannot_add_a_host_directory_or_a_device(self):
|
||||
value = record(119, APP)
|
||||
value["deployment"]["mounts"] = [
|
||||
{"type": "host-bind", "source": "/etc", "container_path": "/host"},
|
||||
{"type": "host-bind", "source": "/srv/media", "container_path": "/media"},
|
||||
{"type": "managed-volume", "source": "local-lvm", "container_path": "/data"}]
|
||||
value["deployment"]["devices"] = [{"kind": "character-device", "host_path": "/dev/kvm"},
|
||||
{"kind": "character-device", "host_path": "/dev/dri/renderD128"}]
|
||||
extra = "mp0: /srv/media,mp=/media,backup=0\nmp1: tank:subvol-119-disk-1,mp=/data,backup=1,size=2G\ndev0: path=/dev/dri/renderD128,gid=104\n"
|
||||
value["observed"]["config"] = config(APP, extra.replace("tank:subvol", "local-lvm:vm"))
|
||||
result, _ = self.standalone(config(APP, extra), value)
|
||||
self.assertEqual([m["source"] for m in result["deployment"]["mounts"]], ["/srv/media", "tank"])
|
||||
self.assertEqual([d["host_path"] for d in result["deployment"]["devices"]], ["/dev/dri/renderD128"])
|
||||
|
||||
def test_the_start_order_must_name_containers_of_the_application(self):
|
||||
contract = {"schema": 1, "stack": "tandoor", "dependencies": [
|
||||
{"vmid": 120, "label": "PostgreSQL",
|
||||
"healthcheck": {"type": "exec", "timeout_seconds": 120, "argv": ["pg_isready"]}}]}
|
||||
self.assertTrue(recovery.valid_contract(contract, {120}))
|
||||
self.assertFalse(recovery.valid_contract(contract, {121}))
|
||||
self.assertFalse(recovery.valid_contract(None, {120}))
|
||||
http = copy.deepcopy(contract)
|
||||
http["dependencies"][0]["healthcheck"] = {"type": "http", "timeout_seconds": 60, "url": "http://10.77.1.41:8080/health"}
|
||||
self.assertTrue(recovery.valid_contract(http, {120}))
|
||||
http["dependencies"][0]["healthcheck"]["url"] = "http://192.168.0.1/admin"
|
||||
self.assertFalse(recovery.valid_contract(http, {120}))
|
||||
|
||||
|
||||
class RcloneMountTests(unittest.TestCase):
|
||||
MOUNT = {"mount_name": "drive", "shared_mount_root": "/mnt/oci-shared/remotes",
|
||||
"shared_mount_read_only_root": "/mnt/oci-shared/remotes-ro", "shared_mount_root_parent": "/mnt/oci-shared"}
|
||||
|
||||
def test_a_mount_inside_its_common_root_is_published_again(self):
|
||||
self.assertEqual(recovery.rclone_mount(dict(self.MOUNT, extra="ignored")), self.MOUNT)
|
||||
|
||||
def test_a_mount_the_host_cannot_publish_safely_is_left_to_the_user(self):
|
||||
for change in ({"mount_name": "../etc"}, {"mount_name": "a b"}, {"shared_mount_root": "/srv/other"},
|
||||
{"shared_mount_root_parent": "/etc", "shared_mount_root": "/etc/remotes",
|
||||
"shared_mount_read_only_root": "/etc/remotes-ro"},
|
||||
{"shared_mount_root": "/mnt/oci-shared/../../etc"}, {"shared_mount_root_parent": "/"},
|
||||
{"shared_mount_read_only_root": 5}):
|
||||
self.assertIsNone(recovery.rclone_mount({**self.MOUNT, **change}), change)
|
||||
self.assertIsNone(recovery.rclone_mount(None))
|
||||
|
||||
def test_the_hookscript_of_the_mount_follows_the_new_id(self):
|
||||
text = "hookscript: local:snippets/proxmenux-rclone-119-fuse-hook.sh\nmemory: 119\n"
|
||||
self.assertEqual(recovery.renumbered_lines(text, 119, 143),
|
||||
"hookscript: local:snippets/proxmenux-rclone-143-fuse-hook.sh\nmemory: 119\n")
|
||||
other = "hookscript: local:snippets/proxmenux-stack-dependencies.sh\n"
|
||||
self.assertEqual(recovery.renumbered_lines(other, 119, 143), other)
|
||||
|
||||
def test_the_parameters_are_read_from_the_hookscript_of_the_host(self):
|
||||
hook = ("inside=/data/mounts/drive\npublished=/mnt/oci-shared/remotes/drive\n"
|
||||
"published_ro=/mnt/oci-shared/remotes-ro/drive\n if ! mountpoint -q /mnt/oci-shared; then\n")
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
(Path(tmp.name) / "proxmenux-rclone-119-fuse-hook.sh").write_text(hook)
|
||||
config = "arch: amd64\nhookscript: local:snippets/proxmenux-rclone-119-fuse-hook.sh\n"
|
||||
with patch.object(carried, "SNIPPETS", Path(tmp.name)):
|
||||
self.assertEqual(carried.rclone_mount(config), self.MOUNT)
|
||||
self.assertIsNone(carried.rclone_mount("arch: amd64\n"))
|
||||
self.assertIsNone(carried.rclone_mount(config.replace("119", "120")))
|
||||
|
||||
|
||||
class NvidiaStaticTests(unittest.TestCase):
|
||||
"""A privileged container names every file of the NVIDIA driver of its host."""
|
||||
|
||||
@staticmethod
|
||||
def inventory(version, uuid):
|
||||
library = f"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.{version}"
|
||||
return {"gpus": [f"Quadro P1000, {uuid}, {version}"], "toolkit_version": ["cli-version: 1", "lib-version: 1"],
|
||||
"devices": {"/dev/nvidia0": {"source": "/dev/nvidia0", "uid": 0, "gid": 0, "mode": 0o666,
|
||||
"major": 195, "minor": 0}},
|
||||
"files": {library: {"source": library, "uid": 0, "gid": 0, "mode": 0o644, "size": 1, "sha256": "x"}},
|
||||
"links": {}}
|
||||
|
||||
def test_the_driver_files_are_named_again_for_the_host_the_container_is_on(self):
|
||||
import oci_nvidia_runtime as runtime
|
||||
old, new = self.inventory("550.1", "GPU-aaa"), self.inventory("560.2", "GPU-aaa")
|
||||
text = ("arch: amd64\ndev0: path=/dev/nvidia0,mode=0666,gid=0,deny-write=0\n"
|
||||
"lxc.mount.entry: /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.550.1 "
|
||||
"usr/lib/x86_64-linux-gnu/libnvidia-ml.so.550.1 none ro,bind,create=file 0 0\n").encode()
|
||||
plan = runtime.refresh_plan(text, old, new)
|
||||
self.assertTrue(plan["changed"])
|
||||
self.assertIn(b"libnvidia-ml.so.560.2 usr/lib/x86_64-linux-gnu/libnvidia-ml.so.560.2", plan["config"])
|
||||
self.assertNotIn(b"550.1", plan["config"])
|
||||
other = self.inventory("560.2", "GPU-bbb")
|
||||
with self.assertRaises(ValueError):
|
||||
runtime.refresh_plan(text, old, other)
|
||||
self.assertTrue(runtime.refresh_plan(text, old, other, same_gpu=False)["changed"])
|
||||
self.assertFalse(runtime.refresh_plan(text, old, old)["changed"])
|
||||
|
||||
|
||||
class ReturnedContainerTests(unittest.TestCase):
|
||||
"""A container that came back from another node, an older backup or a
|
||||
snapshot carries the record that describes it."""
|
||||
|
||||
def copy(self, value, generation):
|
||||
return {"schema_version": 1, "kind": carried.KIND, "node": "other", "vmid": 119,
|
||||
"generation": generation, "record": value}
|
||||
|
||||
def test_a_copy_this_host_did_not_write_with_another_record_wins(self):
|
||||
host, older = record(119, APP), record(119, APP)
|
||||
older["deployment"]["hostname"] = "before-the-update"
|
||||
stamp = {"digest": "x", "generation": "ours"}
|
||||
self.assertTrue(carried.superseded(host, self.copy(older, "theirs"), stamp))
|
||||
# Written by this host: the record changed here afterwards.
|
||||
self.assertFalse(carried.superseded(host, self.copy(older, "ours"), stamp))
|
||||
# Came back without changes: the same record.
|
||||
self.assertFalse(carried.superseded(host, self.copy(record(119, APP), "theirs"), stamp))
|
||||
# Nothing was written by this host yet, or the copy is of another installation.
|
||||
self.assertFalse(carried.superseded(host, self.copy(older, "theirs"), {}))
|
||||
self.assertFalse(carried.superseded(host, self.copy(record(119, DB), "theirs"), stamp))
|
||||
|
||||
def test_the_mark_of_the_last_copy_is_kept_on_the_host(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
path = Path(tmp.name) / carried.STAMP
|
||||
self.assertEqual(carried.read_stamp(path), {})
|
||||
path.write_text("abc\n")
|
||||
self.assertEqual(carried.read_stamp(path), {"digest": "abc"})
|
||||
path.write_text('{"digest": "abc", "generation": "g1"}\n')
|
||||
self.assertEqual(carried.read_stamp(path), {"digest": "abc", "generation": "g1"})
|
||||
|
||||
|
||||
class ClusterCopyTests(unittest.TestCase):
|
||||
"""Every node of a cluster reads the copy kept in /etc/pve."""
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
self.root = Path(tmp.name)
|
||||
patcher = patch.object(carried, "CLUSTER", self.root / "cluster")
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def copy(self, value, generation="g1", node="pve1", saved_at="2026-10-03T10:00:00+00:00"):
|
||||
return {"schema_version": 1, "kind": carried.KIND, "node": node, "vmid": value["vmid"],
|
||||
"generation": generation, "saved_at": saved_at, "record": value}
|
||||
|
||||
def test_the_copy_is_written_and_read_back(self):
|
||||
self.assertIsNone(carried.read_cluster(119))
|
||||
self.assertTrue(carried.write_cluster(119, self.copy(record(119, APP))))
|
||||
self.assertEqual(carried.read_cluster(119)["record"]["installation_id"], APP)
|
||||
(carried.CLUSTER / "120.json").write_text("not json")
|
||||
self.assertIsNone(carried.read_cluster(120))
|
||||
carried.remove_cluster(119)
|
||||
self.assertIsNone(carried.read_cluster(119))
|
||||
|
||||
def test_it_follows_the_copy_the_container_carries(self):
|
||||
value = self.copy(record(119, APP))
|
||||
carried.keep_cluster(119, value, "g1")
|
||||
written = (carried.CLUSTER / "119.json").stat().st_mtime_ns
|
||||
carried.keep_cluster(119, value, "g1")
|
||||
self.assertEqual((carried.CLUSTER / "119.json").stat().st_mtime_ns, written)
|
||||
carried.keep_cluster(119, value, "g2")
|
||||
self.assertEqual(carried.read_cluster(119)["generation"], "g2")
|
||||
|
||||
def test_a_record_changed_on_another_node_replaces_the_one_of_this_host(self):
|
||||
path = self.root / "oci-compose.json"
|
||||
path.write_text("{}")
|
||||
os.utime(path, (1_000_000_000, 1_000_000_000))
|
||||
host, changed = record(119, APP), record(119, APP)
|
||||
changed["deployment"]["hostname"] = "updated-on-the-other-node"
|
||||
self.assertTrue(carried.replaced_elsewhere(host, path, self.copy(changed, node="pve2"), "pve1"))
|
||||
# Written by this node, the same record, another installation or an older copy.
|
||||
self.assertFalse(carried.replaced_elsewhere(host, path, self.copy(changed, node="pve1"), "pve1"))
|
||||
self.assertFalse(carried.replaced_elsewhere(host, path, self.copy(record(119, APP), node="pve2"), "pve1"))
|
||||
self.assertFalse(carried.replaced_elsewhere(host, path, self.copy(record(119, DB), node="pve2"), "pve1"))
|
||||
old = self.copy(changed, node="pve2", saved_at="1999-01-01T00:00:00+00:00")
|
||||
self.assertFalse(carried.replaced_elsewhere(host, path, old, "pve1"))
|
||||
self.assertFalse(carried.replaced_elsewhere(host, path, None, "pve1"))
|
||||
|
||||
def examine(self, inside, shared):
|
||||
if shared is not None:
|
||||
carried.write_cluster(119, shared)
|
||||
done = type("Done", (), {"returncode": 0, "stdout": config(APP)})()
|
||||
|
||||
class Root:
|
||||
def __enter__(self): return Path("/rootfs")
|
||||
def __exit__(self, *_): return False
|
||||
|
||||
with patch.object(recovery, "run", return_value=done), \
|
||||
patch.object(carried, "container_root", return_value=Root()), \
|
||||
patch.object(carried, "read_copy", return_value=inside):
|
||||
return recovery.examine({"vmid": 119, "installation_id": APP, "hostname": "app"})
|
||||
|
||||
def test_the_same_copy_in_the_cluster_is_taken_without_asking(self):
|
||||
value = self.copy(record(119, APP), "g1")
|
||||
result = self.examine(dict(value), value)
|
||||
self.assertTrue(result["trusted"])
|
||||
self.assertIsNone(result["problem"])
|
||||
|
||||
def test_a_container_of_another_moment_is_described_by_its_own_copy(self):
|
||||
older = record(119, APP)
|
||||
older["deployment"]["hostname"] = "before"
|
||||
result = self.examine(self.copy(older, "g0"), self.copy(record(119, APP), "g1"))
|
||||
self.assertFalse(result["trusted"])
|
||||
self.assertEqual(result["copy"]["record"]["deployment"]["hostname"], "before")
|
||||
|
||||
def test_a_container_without_its_copy_takes_the_one_of_the_cluster_and_asks(self):
|
||||
result = self.examine(None, self.copy(record(119, APP), "g1"))
|
||||
self.assertFalse(result["trusted"])
|
||||
self.assertIsNone(result["problem"])
|
||||
self.assertIsNotNone(result["copy"])
|
||||
carried.remove_cluster(119)
|
||||
nothing = self.examine(None, None)
|
||||
self.assertTrue(nothing["unrecoverable"])
|
||||
|
||||
def test_the_copy_of_another_installation_is_not_used(self):
|
||||
carried.remove_cluster(119)
|
||||
result = self.examine(self.copy(record(119, APP), "g1"), self.copy(record(119, DB), "g1"))
|
||||
self.assertFalse(result["trusted"])
|
||||
self.assertEqual(result["copy"]["record"]["installation_id"], APP)
|
||||
|
||||
def test_only_what_needs_no_question_is_registered_on_its_own(self):
|
||||
def plan(**extra):
|
||||
item = dict(entry(119, APP, record(119, APP)), trusted=True)
|
||||
item.update(extra.pop("entry", {}))
|
||||
return {"blockers": [], "bridges": {}, "renumbered": {}, "hold": False, "restored": [item], **extra}
|
||||
self.assertTrue(recovery.automatic(plan()))
|
||||
self.assertFalse(recovery.automatic(plan(entry={"trusted": False})))
|
||||
self.assertFalse(recovery.automatic(plan(bridges={"vmbr11": "10.77.1.0/24"})))
|
||||
self.assertFalse(recovery.automatic(plan(blockers=["missing"])))
|
||||
self.assertFalse(recovery.automatic(plan(renumbered={119: 140})))
|
||||
self.assertFalse(recovery.automatic(plan(entry={"firewall": {"port": 19999}})))
|
||||
self.assertFalse(recovery.automatic(plan(entry={"contract": {"schema": 1}})))
|
||||
|
||||
|
||||
class RegistrationTests(unittest.TestCase):
|
||||
def test_no_record_is_left_when_the_application_cannot_be_registered_whole(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
database = record(120, DB, stack_member={"stack_id": APP, "primary_vmid": 119, "name": "database"})
|
||||
application = record(119, APP, stack_member={"stack_id": APP, "primary_vmid": 119, "name": "application"})
|
||||
plan = {"restored": [entry(119, APP, application), entry(120, DB, database)]}
|
||||
|
||||
def refuse(_root, vmid):
|
||||
if vmid == 120:
|
||||
raise ValueError("cannot be updated")
|
||||
|
||||
with patch.object(instances, "private_directory", lambda path: path.mkdir(parents=True, exist_ok=True)), \
|
||||
patch.object(recovery.oci_stack_modify, "register", side_effect=refuse):
|
||||
with self.assertRaises(ValueError):
|
||||
recovery.register(root, plan)
|
||||
self.assertFalse(instances.location(root, 119).exists())
|
||||
self.assertFalse(instances.location(root, 120).exists())
|
||||
|
||||
|
||||
def test_the_plan_each_container_was_installed_with_is_left_as_it_is_now(self):
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(tmp.cleanup)
|
||||
root = Path(tmp.name)
|
||||
database = record(150, DB, stack_member={"stack_id": APP, "primary_vmid": 151, "name": "database"})
|
||||
database["deployment"]["mounts"] = [{"source": "local-lvm:vm-150-disk-1"}]
|
||||
application = record(151, APP, stack_member={"stack_id": APP, "primary_vmid": 151, "name": "application"})
|
||||
application["stack"] = {"id": APP, "members": [], "deployment": {"services": [
|
||||
{"name": "database", "vmid": 150, "deployment": {"mounts": [{"source": "local-lvm:vm-120-disk-1"}]}},
|
||||
{"name": "application", "vmid": 151, "deployment": {"create_arguments": ["119"]}},
|
||||
{"name": "check", "vmid": 150, "healthcheck": {"type": "running"}}]}}
|
||||
with patch.object(instances, "private_directory", lambda path: path.mkdir(parents=True, exist_ok=True)):
|
||||
instances.write(instances.location(root, 150), database)
|
||||
instances.write(instances.location(root, 151), application)
|
||||
recovery.refresh_service_plans(root, [151, 150])
|
||||
services = instances.read(root, 151)["stack"]["deployment"]["services"]
|
||||
self.assertEqual(services[0]["deployment"]["mounts"], [{"source": "local-lvm:vm-150-disk-1"}])
|
||||
self.assertEqual(services[1]["deployment"], application["deployment"])
|
||||
self.assertNotIn("deployment", services[2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""An image is the same in the registry and in the archive made from it,
|
||||
whichever format the registry publishes its manifest in."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
from oci_installation_state import same_image
|
||||
|
||||
REGISTRY = "sha256:" + "a" * 64
|
||||
ARCHIVE = "sha256:" + "b" * 64
|
||||
LAYERS = ["sha256:" + "c" * 64, "sha256:" + "d" * 64]
|
||||
BUILT = "2026-09-15T01:13:55.019799592Z"
|
||||
|
||||
|
||||
class SameImageTests(unittest.TestCase):
|
||||
def test_an_oci_manifest_keeps_its_digest(self):
|
||||
self.assertTrue(same_image({"manifest_digest": REGISTRY}, {"manifest_digest": REGISTRY}))
|
||||
|
||||
def test_a_docker_manifest_is_recognised_by_its_layers_and_when_it_was_built(self):
|
||||
self.assertTrue(same_image({"manifest_digest": REGISTRY, "layers": LAYERS, "created": BUILT},
|
||||
{"manifest_digest": ARCHIVE, "layers": list(LAYERS), "created": BUILT}))
|
||||
|
||||
def test_another_image_is_not_the_same(self):
|
||||
candidate = {"manifest_digest": REGISTRY, "layers": LAYERS, "created": BUILT}
|
||||
self.assertFalse(same_image(candidate, {"manifest_digest": ARCHIVE, "layers": LAYERS[:1], "created": BUILT}))
|
||||
self.assertFalse(same_image(candidate, {"manifest_digest": ARCHIVE, "layers": LAYERS[::-1], "created": BUILT}))
|
||||
# The same layers rebuilt with another configuration are another image.
|
||||
self.assertFalse(same_image(candidate, {"manifest_digest": ARCHIVE, "layers": LAYERS,
|
||||
"created": "2026-10-01T00:00:00Z"}))
|
||||
|
||||
def test_what_is_missing_never_matches(self):
|
||||
self.assertFalse(same_image({"manifest_digest": REGISTRY}, {"manifest_digest": ARCHIVE}))
|
||||
self.assertFalse(same_image({"manifest_digest": REGISTRY, "layers": [], "created": BUILT},
|
||||
{"manifest_digest": ARCHIVE, "layers": [], "created": BUILT}))
|
||||
self.assertFalse(same_image({"manifest_digest": REGISTRY, "layers": LAYERS, "created": None},
|
||||
{"manifest_digest": ARCHIVE, "layers": LAYERS, "created": None}))
|
||||
# A record saved before the layers were kept is compared by its digest alone.
|
||||
self.assertFalse(same_image({"manifest_digest": REGISTRY, "layers": LAYERS, "created": BUILT},
|
||||
{"manifest_digest": ARCHIVE}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""The notes Proxmox shows for a multi-container application link to the
|
||||
address it answers on from the local network, not to its private leg."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_native_stack as stack
|
||||
|
||||
TWO_LEGS = "net0: name=eth0,bridge=vmbr0,ip=dhcp,type=veth\nnet1: name=eth1,bridge=vmbr10,ip=10.77.0.10/24,type=veth\n"
|
||||
PRIVATE_ONLY = "net0: name=eth0,bridge=vmbr10,ip=10.77.0.12/24,type=veth\n"
|
||||
|
||||
|
||||
class AccessAddressTests(unittest.TestCase):
|
||||
def address(self, config, *answers, wait=20):
|
||||
listed = iter(answers)
|
||||
|
||||
def run(command, **_):
|
||||
if command[0] == "pct":
|
||||
return SimpleNamespace(stdout=config)
|
||||
return SimpleNamespace(stdout=next(listed))
|
||||
|
||||
with patch.object(stack.subprocess, "run", side_effect=run), patch.object(stack.time, "sleep"):
|
||||
return stack.access_address(101, wait)
|
||||
|
||||
def test_the_address_of_the_local_network_is_used(self):
|
||||
self.assertEqual(self.address(TWO_LEGS, "10.77.0.10\n192.168.0.44\n"), "192.168.0.44")
|
||||
self.assertEqual(self.address(TWO_LEGS, "192.168.0.44\n10.77.0.10\n"), "192.168.0.44")
|
||||
|
||||
def test_a_lease_that_has_not_arrived_is_waited_for(self):
|
||||
self.assertEqual(self.address(TWO_LEGS, "10.77.0.10\n", "10.77.0.10\n192.168.0.44\n"), "192.168.0.44")
|
||||
|
||||
def test_a_container_with_only_its_private_leg_keeps_that_address(self):
|
||||
self.assertEqual(self.address(PRIVATE_ONLY, "10.77.0.12\n"), "10.77.0.12")
|
||||
|
||||
def test_a_container_without_address_has_none(self):
|
||||
self.assertEqual(self.address(PRIVATE_ONLY, "\n"), "")
|
||||
self.assertEqual(self.address(TWO_LEGS, "10.77.0.10\n", wait=0), "10.77.0.10")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""The temporary container that holds the data of an application during an
|
||||
update is removed when the operation ends: empty after a commit, and with the
|
||||
disks of the failed attempt after a verified recovery."""
|
||||
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "remote"))
|
||||
|
||||
import oci_instance_transaction as transaction
|
||||
|
||||
EMPTY = b"arch: amd64\nhostname: oci-data-holder\nrootfs: local-lvm:vm-104-disk-0,size=8G\n"
|
||||
HOLDING = EMPTY + b"mp0: local-lvm:vm-104-disk-1,mp=/transaction-retained/bd021f9e,backup=1,size=32G\n"
|
||||
STATE = {"id": "4dfbfd9223a440c1b07a0334b08fc4fd", "stage": 104}
|
||||
|
||||
|
||||
class StageReleaseTests(unittest.TestCase):
|
||||
def release(self, config, **arguments):
|
||||
calls = []
|
||||
with patch.object(transaction, "owned", return_value=config) as owned, \
|
||||
patch.object(transaction, "stop", side_effect=lambda vmid: calls.append(("stop", vmid))), \
|
||||
patch.object(transaction, "run", side_effect=lambda *command: calls.append(command)):
|
||||
transaction.release_stage(STATE, **arguments)
|
||||
owned.assert_called_once_with(104, "proxmenux-transaction=" + STATE["id"])
|
||||
return calls
|
||||
|
||||
def test_after_a_commit_the_empty_container_is_removed(self):
|
||||
self.assertEqual(self.release(EMPTY), [("pct", "destroy", "104")])
|
||||
|
||||
def test_a_container_that_still_holds_a_disk_is_kept(self):
|
||||
self.assertEqual(self.release(HOLDING), [])
|
||||
self.assertEqual(self.release(EMPTY + b"unused0: local-lvm:vm-104-disk-2\n"), [])
|
||||
|
||||
def test_after_a_verified_recovery_it_is_removed_with_what_it_holds(self):
|
||||
self.assertEqual(self.release(HOLDING, discard=True), [("stop", 104), ("pct", "destroy", "104")])
|
||||
|
||||
def test_a_container_of_another_operation_is_never_removed(self):
|
||||
calls = []
|
||||
with patch.object(transaction, "owned", side_effect=ValueError("not ours")), \
|
||||
patch.object(transaction, "run", side_effect=lambda *command: calls.append(command)):
|
||||
transaction.release_stage(STATE, discard=True)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_an_operation_without_a_temporary_container_does_nothing(self):
|
||||
with patch.object(transaction, "owned") as owned:
|
||||
transaction.release_stage({"id": "x"}, discard=True)
|
||||
owned.assert_not_called()
|
||||
|
||||
def test_a_removal_that_fails_does_not_undo_the_recovery(self):
|
||||
with patch.object(transaction, "owned", return_value=HOLDING), \
|
||||
patch.object(transaction, "stop"), \
|
||||
patch.object(transaction, "run", side_effect=RuntimeError("pct failed with exit code 255")), \
|
||||
patch.object(transaction, "log") as logged:
|
||||
transaction.discard_stage(STATE)
|
||||
self.assertIn("pct failed", logged.call_args[0][0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user