Mirrors/ProxMenux
1
0
Fork 0
mirror of https://github.com/MacRimi/ProxMenux.git synced 2026-11-01 18:32:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(post-install): rebuild Log2RAM with the journal and task logs kept on disk

Browse Source
This commit is contained in:
MacRimi
2026-10-27 19:48:26 +01:00
parent ac9b8c2705
commit 4cefc0484b
57 changed files with 992 additions and 1065 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/tests/test_log2ram_journal_sync.py
-105
View File
@@ -1,105 +0,0 @@
"""Log2RAM syncs only the archived journal files, so the active file it found
on disk when it was installed stays there and is restored at every boot.
journald then continues an old sequence and deletes its newest archives first.
The post-install makes the sync remove that file from the disk copy."""
from pathlib import Path
import re
import shutil
import stat
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[3]
SHARED = ROOT / "scripts/global/utils-install-functions.sh"
FLOWS = {"auto_post_install.sh": "install_log2ram_auto", "customizable_post_install.sh": "configure_log2ram"}
# The sync of Log2RAM in its journald-aware mode, as its project ships it.
JOURNALD_AWARE = '''#!/usr/bin/env bash
sync_to_disk() {
is_safe
local optional_params=()
if journald_logrotate; then
optional_params+=("--include=journal/*/*@*.journal")
optional_params+=("--exclude=journal/*/*")
fi
rsync -aXv --no-acls --delete-after "${optional_params[@]}" "${RAM_LOG}"/ "${HDD_LOG}"/
}
'''
NOT_AWARE = '''#!/usr/bin/env bash
sync_to_disk() {
rsync -aXv --delete-after "${RAM_LOG}"/ "${HDD_LOG}"/
}
'''
def gnu_sed():
return shutil.which("bash") and subprocess.run(["sed", "--version"], capture_output=True).returncode == 0
@unittest.skipUnless(gnu_sed(), "bash and GNU sed are needed")
class Log2ramJournalSyncTests(unittest.TestCase):
def correct(self, folder, content, times=1):
script = Path(folder) / "log2ram"
script.write_text(content)
script.chmod(0o755)
calls = "; ".join(['pmx_log2ram_drop_stale_journal "$1"; echo "rc=$?"'] * times)
# As the change journal does, the new content is read whole before it is written.
write = 'pmx_write_file() { local content; content="$(cat)"; printf "%s\\n" "$content" > "$1"; }'
run = subprocess.run(["bash", "-c", f'source "{SHARED}"; {write}; {calls}',
"bash", str(script)], capture_output=True, text=True, check=False)
return script, run.stdout.split()
def test_the_sync_removes_what_it_does_not_copy(self):
with tempfile.TemporaryDirectory() as folder:
script, results = self.correct(folder, JOURNALD_AWARE)
lines = script.read_text().splitlines()
position = lines.index(' optional_params+=("--exclude=journal/*/*")')
self.assertEqual(lines[position + 1], ' optional_params+=("--delete-excluded")')
self.assertEqual(lines[position + 2], " fi")
self.assertEqual(results, ["rc=0"])
self.assertEqual(subprocess.run(["bash", "-n", str(script)]).returncode, 0)
self.assertTrue(script.stat().st_mode & stat.S_IXUSR)
self.assertEqual(Path(str(script) + ".proxmenux.bak").read_text(), JOURNALD_AWARE)
def test_it_is_applied_once(self):
with tempfile.TemporaryDirectory() as folder:
script, results = self.correct(folder, JOURNALD_AWARE, times=2)
self.assertEqual(results, ["rc=0", "rc=1"])
self.assertEqual(script.read_text().count("--delete-excluded"), 1)
def test_a_sync_that_copies_every_file_is_left_as_it_is(self):
with tempfile.TemporaryDirectory() as folder:
script, results = self.correct(folder, NOT_AWARE)
self.assertEqual(results, ["rc=1"])
self.assertEqual(script.read_text(), NOT_AWARE)
self.assertFalse(Path(str(script) + ".proxmenux.bak").exists())
def test_both_flows_apply_it_to_new_and_existing_installations(self):
for name, function in FLOWS.items():
with self.subTest(flow=name):
text = (ROOT / "scripts/post_install" / name).read_text()
self.assertEqual(text.count("pmx_log2ram_drop_stale_journal"), 2)
# An existing installation is told that it needs a reboot.
self.assertIn('if pmx_log2ram_drop_stale_journal "$log2ram_bin"; then\n NECESSARY_REBOOT=1', text)
# The version is what offers the update to existing installations.
version = re.search(rf'{function}\(\) {{\n local FUNC_VERSION="([0-9.]+)"', text)
self.assertIsNotNone(version)
self.assertGreaterEqual(tuple(map(int, version.group(1).split("."))), (1, 7))
def test_rotated_logs_are_compressed_at_once_on_a_tmpfs_var_log(self):
# A rotated log kept uncompressed filled the 128M Log2RAM of a PBS.
for name, function in FLOWS.items():
with self.subTest(flow=name):
text = (ROOT / "scripts/post_install" / name).read_text()
self.assertNotIn("delaycompress", text)
version = re.search(rf'{function}\(\) {{\n local FUNC_VERSION="([0-9.]+)"', text)
self.assertGreaterEqual(tuple(map(int, version.group(1).split("."))), (1, 8))
if __name__ == "__main__":
unittest.main()
.github/scripts/tests/test_log2ram_setup.py
+125
View File
@@ -0,0 +1,125 @@
"""Log2RAM keeps /var/log in RAM while the journal and the task logs of
Proxmox VE and Proxmox Backup Server stay on disk behind a link. An update keeps the size
and interval the host already has."""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[3]
KEEP = ROOT / "scripts/global/log2ram_keep_on_disk.sh"
SETUP = ROOT / "scripts/global/log2ram_setup.sh"
@unittest.skipUnless(shutil.which("bash"), "bash is needed")
class KeepOnDiskTests(unittest.TestCase):
def setUp(self):
folder = tempfile.TemporaryDirectory()
self.addCleanup(folder.cleanup)
self.root = Path(folder.name)
self.log = self.root / "var/log"
(self.log / "journal/abc").mkdir(parents=True)
(self.log / "journal/abc/system.journal").write_text("entries")
(self.log / "pveproxy").mkdir()
(self.log / "pveproxy/access.log").write_text("GET /")
def run_script(self, revert=False):
env = dict(os.environ, PMX_L2R_ROOT=str(self.root), PMX_L2R_REVERT="1" if revert else "0")
return subprocess.run(["bash", str(KEEP)], env=env, capture_output=True, text=True)
def test_the_journal_moves_to_disk_behind_a_link(self):
self.assertEqual(self.run_script().returncode, 0)
journal = self.log / "journal"
self.assertTrue(journal.is_symlink())
self.assertEqual(os.readlink(journal), "/var/hdd.journal")
self.assertEqual((self.root / "var/hdd.journal/abc/system.journal").read_text(), "entries")
self.assertTrue((self.log / "pveproxy/access.log").exists())
self.assertFalse((self.root / "var/hdd.proxmox-backup").exists())
def test_running_again_changes_nothing(self):
self.run_script()
self.assertEqual(self.run_script().returncode, 0)
self.assertEqual(os.readlink(self.log / "journal"), "/var/hdd.journal")
self.assertEqual(sorted(p.name for p in (self.root / "var/hdd.journal").iterdir()), ["abc"])
def test_the_task_logs_of_backup_server_stay_on_disk_too(self):
(self.log / "proxmox-backup/tasks").mkdir(parents=True)
(self.log / "proxmox-backup/tasks/active").write_text("task")
self.run_script()
self.assertEqual(os.readlink(self.log / "proxmox-backup"), "/var/hdd.proxmox-backup")
self.assertEqual((self.root / "var/hdd.proxmox-backup/tasks/active").read_text(), "task")
def test_files_written_back_to_the_folder_join_the_copy_on_disk(self):
self.run_script()
(self.log / "journal").unlink()
(self.log / "journal/def").mkdir(parents=True)
(self.log / "journal/def/system.journal").write_text("later")
self.run_script()
disk = self.root / "var/hdd.journal"
self.assertEqual(sorted(p.name for p in disk.iterdir()), ["abc", "def"])
self.assertTrue((self.log / "journal").is_symlink())
def test_a_host_without_a_journal_folder_gets_one_on_disk(self):
shutil.rmtree(self.log / "journal")
self.run_script()
self.assertTrue((self.root / "var/hdd.journal").is_dir())
self.assertTrue((self.log / "journal").is_symlink())
def test_the_task_logs_of_proxmox_ve_stay_on_disk(self):
(self.log / "pve/tasks").mkdir(parents=True)
(self.log / "pve/tasks/index").write_text("UPID")
self.run_script()
self.assertEqual(os.readlink(self.log / "pve"), "/var/hdd.pve")
self.assertEqual((self.root / "var/hdd.pve/tasks/index").read_text(), "UPID")
def test_removing_log2ram_puts_the_folders_back(self):
(self.log / "proxmox-backup").mkdir()
self.run_script()
self.assertEqual(self.run_script(revert=True).returncode, 0)
for name in ("journal", "proxmox-backup"):
self.assertFalse((self.log / name).is_symlink(), name)
self.assertTrue((self.log / name).is_dir(), name)
self.assertFalse((self.root / f"var/hdd.{name}").exists(), name)
self.assertEqual((self.log / "journal/abc/system.journal").read_text(), "entries")
@unittest.skipUnless(shutil.which("bash"), "bash is needed")
class KeptSettingsTests(unittest.TestCase):
def read(self, function, files):
with tempfile.TemporaryDirectory() as folder:
script = SETUP.read_text()
for path, content in files.items():
target = Path(folder) / path.lstrip("/")
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content)
script = script.replace(path, str(target))
library = Path(folder) / "setup.sh"
library.write_text(script)
result = subprocess.run(["bash", "-c", f'source "{library}"; {function}'],
capture_output=True, text=True)
return result.stdout.strip()
def test_the_size_comes_from_the_installed_configuration(self):
self.assertEqual(self.read("pmx_log2ram_current_size", {"/etc/log2ram.conf": "SIZE=256M\n"}), "256")
def test_the_interval_comes_from_the_timer_of_this_version(self):
timer = "[Timer]\nOnCalendar=\nOnCalendar=*-*-* 00/6:55:00\n"
self.assertEqual(self.read("pmx_log2ram_current_hours",
{"/etc/systemd/system/log2ram-daily.timer.d/proxmenux.conf": timer}), "6")
def test_the_interval_of_an_earlier_version_is_kept(self):
cron = "MAILTO=\"\"\n0 */3 * * * root /usr/local/bin/log2ram write >/dev/null 2>&1\n"
self.assertEqual(self.read("pmx_log2ram_current_hours", {"/etc/cron.d/log2ram": cron}), "3")
def test_the_release_is_pinned_to_a_verified_commit(self):
text = SETUP.read_text()
self.assertIn('PMX_L2R_TAG="1.7.2"', text)
self.assertIn('PMX_L2R_COMMIT="f171449530f8984374c8f69fff3c92af813a694b"', text)
self.assertNotIn("install.sh", text.split("_pmx_log2ram_install_release() {", 1)[1].split("\n}\n", 1)[0])
if __name__ == "__main__":
unittest.main()