diff --git a/.github/ISSUE_TEMPLATE/oci-validation.yml b/.github/ISSUE_TEMPLATE/oci-validation.yml new file mode 100644 index 00000000..f5e9fdce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/oci-validation.yml @@ -0,0 +1,105 @@ +name: OCI validation report +description: Report a real test of an application from the OCI manager catalog. +title: "OCI validation: " +labels: ["oci-validation"] +body: + - type: markdown + attributes: + value: | + One report describes one application and the scenario you tested. Once it is reviewed, the application is recorded as verified: it shows with a ✓ in the OCI installer and is listed in [oci/VALIDATION.md](https://github.com/MacRimi/ProxMenux/blob/develop/oci/VALIDATION.md) with your GitHub user, the date and a link to this report. + + Reports that an application does not work are just as useful: they tell others what to expect and what needs fixing. + - type: input + id: application + attributes: + label: Application + description: Name or ID as it appears in the OCI catalog, for example "Glances" or "glances". + validations: + required: true + - type: input + id: version + attributes: + label: Image version + description: Version or tag shown by the installer. + validations: + required: true + - type: input + id: digest + attributes: + label: Image digest + description: Shown in the App tab of the container in ProxMenux Monitor, after the build date — for example "2026-09-06 · b1f339cf". The full sha256 digest is also accepted. + placeholder: b1f339cf + validations: + required: true + - type: input + id: proxmox + attributes: + label: Proxmox VE version + placeholder: "9.1" + validations: + required: true + - type: dropdown + id: install + attributes: + label: Install mode + options: + - Default configuration + - Advanced configuration + validations: + required: true + - type: dropdown + id: storage + attributes: + label: Data storage + options: + - Volume on Proxmox storage + - Host directory + - Both + - No persistent data + validations: + required: true + - type: dropdown + id: network + attributes: + label: Network + options: + - DHCP + - Static address + validations: + required: true + - type: dropdown + id: gpu + attributes: + label: GPU + options: + - No GPU + - Intel + - AMD + - NVIDIA + validations: + required: true + - type: checkboxes + id: checks + attributes: + label: What was checked + options: + - label: Installs cleanly + - label: The application responds (web interface or service) + - label: Keeps working after a container restart + - label: Survives a Proxmox backup and restore + - label: An image update keeps the data + - type: dropdown + id: result + attributes: + label: Result + options: + - Works + - Works with problems + - Does not work + validations: + required: true + - type: textarea + id: notes + attributes: + label: Notes + description: Anything worth knowing. If something failed, what you saw and how to reproduce it. diff --git a/.github/oci-validation-reviewers b/.github/oci-validation-reviewers new file mode 100644 index 00000000..c8109716 --- /dev/null +++ b/.github/oci-validation-reviewers @@ -0,0 +1,5 @@ +# GitHub users who can validate an OCI validation report by commenting +# /validated on it. A reviewer does not validate their own report. +MacRimi +Vaso73 +f3rs3n diff --git a/.github/scripts/oci_validation.py b/.github/scripts/oci_validation.py new file mode 100644 index 00000000..b63b75ba --- /dev/null +++ b/.github/scripts/oci_validation.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""OCI validation record: renders oci/VALIDATION.md from verification.json, +checks the record, checks who a pull request credits, and records a reviewed +validation report.""" +import argparse +import datetime +import json +import os +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[2] +VERIFICATION = ROOT / "oci/catalog/verification.json" +INDEX = ROOT / "oci/catalog/index.json" +OUTPUT = ROOT / "oci/VALIDATION.md" +REVIEWERS = ROOT / ".github/oci-validation-reviewers" + +REPO = "https://github.com/MacRimi/ProxMenux" +BOARD = "https://github.com/users/MacRimi/projects/1" +FORM = f"{REPO}/issues/new?template=oci-validation.yml" +DISCUSSION = f"{REPO}/discussions/360" +MAINTAINERS = {"macrimi"} +LAB = "laboratory-validated" + +USER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}$") +REPORT_RE = re.compile(r"^https://github\.com/MacRimi/ProxMenux/" + r"(issues/[0-9]+|discussions/[0-9]+(#discussioncomment-[0-9]+)?)$") +ENTRY_KEYS = {"status", "community_tested"} +COMMUNITY_KEYS = {"by", "date", "report", "digest", "scenario"} +DIGEST_RE = re.compile(r"^[a-f0-9]{8,64}$") +# Form fields that describe the tested scenario, in the order they are read. +SCENARIO_FIELDS = ("Install mode", "Data storage", "Network", "GPU") +NO_RESPONSE = "_No response_" + + +def load(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def dump(data): + return json.dumps(data, indent=2, ensure_ascii=False) + "\n" + + +def applications(index): + return {item["id"]: item for item in index.get("applications", [])} + + +def lab_validated(entry, item): + # Same rule as the catalog: an application that cannot be installed is not verified. + return entry.get("status") == LAB and bool(item.get("automatic_install_candidate")) + + +def community(entry): + tested = entry.get("community_tested") + return tested if isinstance(tested, dict) and tested.get("by") else None + + +def cell(text): + return str(text).replace("|", "\\|").strip() + + +def render(verification, index): + apps = applications(index) + entries = {app: entry for app, entry in verification.get("applications", {}).items() + if app in apps and isinstance(entry, dict)} + lab = sorted((app for app, entry in entries.items() if lab_validated(entry, apps[app])), + key=lambda app: str(apps[app].get("title") or app).casefold()) + tested = sorted((app for app, entry in entries.items() if community(entry)), + key=lambda app: str(apps[app].get("title") or app).casefold()) + visible = sum(1 for item in apps.values() if not item.get("hidden")) + verified = len(set(lab) | set(tested)) + lines = [ + "# OCI application validation", + "", + "", + "", + f"{verified} of {visible} applications in the OCI catalog have been tested for real: " + f"{len(lab)} in the ProxMenux lab and {len(tested)} by the community. " + "The OCI installer shows them as verified, with a ✓ in the lists.", + "", + "Each test describes the scenario that was run and names the image it ran on. It does not cover every possible " + "configuration of the application, and a newer image has not been tested until someone reports it.", + "", + "## Taking part", + "", + "Any application that is not listed here is open for testing.", + "", + f"1. Check the [ProxMenux Roadmap]({BOARD}) to see which applications someone is already testing. " + "With access to the board, create a card for the application and move it to In progress while you test it.", + f"2. Test the application with the OCI manager and fill in the [OCI validation report]({FORM}). " + "The report records the image and the exact scenario: install mode, storage, network and GPU.", + "3. A reviewer reads the report and comments `/validated` on it. The application is then added to this list " + "with your GitHub user and shown as verified in the OCI installer, and the report is closed.", + "", + "Reviewers are listed in [.github/oci-validation-reviewers](../.github/oci-validation-reviewers), and a " + "reviewer does not validate their own report. A validation can also be added with a pull request that adds " + "the entry to `oci/catalog/verification.json` and regenerates this file with " + "`python3 .github/scripts/oci_validation.py render`; CI checks that the entry credits the author of the pull request.", + "", + "An application that cannot be validated for a reason outside the tester's hands — hardware nobody has, something " + "only Docker provides, a fix still pending in ProxMenux — moves to Blocked on the board, with a line saying why " + "and what would make it worth trying again.", + "", + f"Questions and ideas about OCI testing go in [discussion #360]({DISCUSSION}).", + "", + "## Tested by the community", + "", + ] + if tested: + lines += ["| Application | Tested by | Date | Image | Scenario | Report |", "|---|---|---|---|---|---|"] + for app in tested: + entry = community(entries[app]) + report = f"[Report]({entry['report']})" if entry.get("report") else "—" + image = f"`{entry['digest'][:12]}`" if entry.get("digest") else "—" + lines.append(f"| {cell(apps[app].get('title') or app)} | [@{entry['by']}](https://github.com/{entry['by']}) " + f"| {entry.get('date') or '—'} | {image} | {cell(entry.get('scenario') or '—')} | {report} |") + else: + lines.append("No application has been tested by the community yet.") + lines += ["", "## Tested in the ProxMenux lab", ""] + if lab: + lines += ["| Application | Category |", "|---|---|"] + for app in lab: + lines.append(f"| {cell(apps[app].get('title') or app)} | {cell(apps[app].get('category_label') or '—')} |") + else: + lines.append("No application has been tested in the ProxMenux lab yet.") + return "\n".join(lines) + "\n" + + +def problems(verification, index, rendered=None, today=None): + today = today or datetime.date.today() + apps = applications(index) + found = [] + entries = verification.get("applications") if isinstance(verification, dict) else None + if not isinstance(entries, dict): + return ["verification.json must hold an \"applications\" object"] + for app, entry in entries.items(): + where = f"verification.json: {app}" + if app not in apps: + found.append(f"{where}: not an application of the OCI catalog") + if not isinstance(entry, dict) or not entry: + found.append(f"{where}: the entry must be a non-empty object") + continue + if set(entry) - ENTRY_KEYS: + found.append(f"{where}: unknown keys {sorted(set(entry) - ENTRY_KEYS)}") + if "status" in entry and entry["status"] != LAB: + found.append(f"{where}: status can only be \"{LAB}\"") + if "community_tested" not in entry: + continue + tested = entry["community_tested"] + if not isinstance(tested, dict): + found.append(f"{where}: community_tested must be an object") + continue + if set(tested) - COMMUNITY_KEYS: + found.append(f"{where}: unknown community_tested keys {sorted(set(tested) - COMMUNITY_KEYS)}") + if not USER_RE.fullmatch(str(tested.get("by", ""))): + found.append(f"{where}: \"by\" must be a GitHub user name") + try: + date = datetime.date.fromisoformat(str(tested.get("date", ""))) + if date > today + datetime.timedelta(days=1): + found.append(f"{where}: the date is in the future") + except ValueError: + found.append(f"{where}: \"date\" must be YYYY-MM-DD") + if "report" in tested and not REPORT_RE.fullmatch(str(tested["report"])): + found.append(f"{where}: \"report\" must link to an issue or discussion of MacRimi/ProxMenux") + if "digest" in tested and not DIGEST_RE.fullmatch(str(tested["digest"])): + found.append(f"{where}: \"digest\" must be 8 to 64 lowercase hex characters of the image digest") + if "scenario" in tested and (not isinstance(tested["scenario"], str) or len(tested["scenario"]) > 300): + found.append(f"{where}: \"scenario\" must be text of at most 300 characters") + if rendered is not None and rendered != render(verification, index): + found.append("oci/VALIDATION.md is out of date: run python3 .github/scripts/oci_validation.py render") + return found + + +def author_problems(base, head, author): + """A pull request only adds or changes community tests credited to its author.""" + if author.casefold() in MAINTAINERS: + return [] + found = [] + before = base.get("applications", {}) if isinstance(base, dict) else {} + after = head.get("applications", {}) + for app in sorted(set(before) | set(after)): + old = before.get(app) if isinstance(before.get(app), dict) else {} + new = after.get(app) if isinstance(after.get(app), dict) else {} + if old.get("status") != new.get("status"): + found.append(f"{app}: the ProxMenux lab status is changed by the maintainer only") + if old.get("community_tested") == new.get("community_tested"): + continue + credited = community(new) + if not credited: + found.append(f"{app}: a community test is removed by the maintainer only") + elif str(credited["by"]).casefold() != author.casefold(): + found.append(f"{app}: credits @{credited['by']}, but the pull request is from @{author}") + return found + + +def parse_form(body): + """Issue form answers, keyed by the field label.""" + fields, label, lines = {}, None, [] + for line in (body or "").replace("\r\n", "\n").split("\n"): + if line.startswith("### "): + if label is not None: + fields[label] = "\n".join(lines).strip() + label, lines = line[4:].strip(), [] + elif label is not None: + lines.append(line) + if label is not None: + fields[label] = "\n".join(lines).strip() + return {key: ("" if value == NO_RESPONSE else value) for key, value in fields.items()} + + +def resolve(name, index): + wanted = " ".join(name.split()).casefold() + matches = [item["id"] for item in index.get("applications", []) + if wanted in (str(item["id"]).casefold(), " ".join(str(item.get("title") or "").split()).casefold())] + return matches[0] if len(matches) == 1 else None + + +def image_digest(text): + """The digest as the Monitor shows it (b1f339cf) or in full (sha256:b1f3...).""" + value = str(text or "").strip().lower() + value = value.split("·")[-1].strip() + value = value[7:] if value.startswith("sha256:") else value + return value if DIGEST_RE.fullmatch(value) else None + + +def scenario(fields): + parts = [fields.get(name, "") for name in SCENARIO_FIELDS] + if fields.get("Proxmox VE version"): + parts.append(f"Proxmox VE {fields['Proxmox VE version']}") + if fields.get("Image version"): + parts.append(f"image {fields['Image version']}") + text = " · ".join(part.strip() for part in parts if part and part.strip()) + if fields.get("Result") == "Works with problems": + text = f"Works with problems: {text}" + return text[:300] + + +def reviewers(text): + """User names of the reviewer list, one per line; # starts a comment.""" + names = {line.split("#", 1)[0].strip() for line in text.splitlines()} + return {name.casefold() for name in names if USER_RE.fullmatch(name)} + + +def record(event, verification, index, today, allowed=frozenset()): + """Returns (verification, app id, message); raises ValueError with the reason to refuse. + + A label can only be added by someone with write access to the repository. + A /validated comment is accepted from the reviewer list, never from the + report's own author unless it is the maintainer.""" + issue = event.get("issue") or {} + labels = {label.get("name") for label in issue.get("labels", [])} + if "oci-validation" not in labels: + raise ValueError("This issue is not an OCI validation report.") + author = str((issue.get("user") or {}).get("login", "")) + report = str(issue.get("html_url", "")) + if not USER_RE.fullmatch(author) or not REPORT_RE.fullmatch(report): + raise ValueError("The report author or link could not be read.") + reviewer = str(((event.get("comment") or {}).get("user") or {}).get("login", "")) + if event.get("comment") is not None: + if reviewer.casefold() not in allowed: + raise ValueError(f"@{reviewer} is not in the list of OCI validation reviewers " + "(.github/oci-validation-reviewers), so the report is not recorded.") + if reviewer.casefold() == author.casefold() and reviewer.casefold() not in MAINTAINERS: + raise ValueError("A report is validated by a reviewer other than its author.") + fields = parse_form(issue.get("body")) + name = fields.get("Application", "") + app = resolve(name, index) + if app is None: + raise ValueError(f"\"{name}\" does not match one application of the OCI catalog. " + "Edit the Application field with its name or ID as shown in the catalog, then add the label again.") + if fields.get("Result") == "Does not work": + raise ValueError("The report says the application does not work, so it is not recorded as verified.") + digest = image_digest(fields.get("Image digest")) + if digest is None: + raise ValueError("The Image digest field does not hold an image digest. Edit it with the digest shown in the " + "App tab of the Monitor, for example b1f339cf, then add the label again.") + data = json.loads(json.dumps(verification)) + entries = data.setdefault("applications", {}) + entry = entries.setdefault(app, {}) + previous = community(entry) + entry["community_tested"] = {"by": author, "date": today.isoformat(), "report": report, + "digest": digest, "scenario": scenario(fields)} + data["applications"] = dict(sorted(entries.items())) + title = applications(index)[app].get("title") or app + validated = f" Validated by @{reviewer}." if reviewer else "" + message = (f"Thank you, @{author}! {title} is now recorded as verified.{validated} " + f"It shows with a ✓ in the OCI installer and is listed in [oci/VALIDATION.md]({REPO}/blob/develop/oci/VALIDATION.md).") + if previous and previous.get("by") != author: + message += f" This report replaces the previous one from @{previous['by']}." + return data, app, message + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("render", help="Write oci/VALIDATION.md") + commands.add_parser("check", help="Check verification.json and that oci/VALIDATION.md is current") + authors = commands.add_parser("authors", help="Check who a pull request credits") + authors.add_argument("--base", required=True, help="verification.json of the base branch; may be missing") + authors.add_argument("--author", required=True) + rec = commands.add_parser("record", help="Record a reviewed validation report") + rec.add_argument("--event", required=True) + rec.add_argument("--message", required=True, help="File that receives the reply for the issue") + args = parser.parse_args(argv) + + verification, index = load(VERIFICATION), load(INDEX) + if args.command == "render": + OUTPUT.write_text(render(verification, index), encoding="utf-8") + return 0 + if args.command == "check": + rendered = OUTPUT.read_text(encoding="utf-8") if OUTPUT.exists() else "" + found = problems(verification, index, rendered) + elif args.command == "authors": + base = load(args.base) if Path(args.base).exists() and Path(args.base).stat().st_size else {} + found = author_problems(base, verification, args.author) + else: + try: + allowed = reviewers(REVIEWERS.read_text(encoding="utf-8")) if REVIEWERS.exists() else frozenset() + data, app, message = record(load(args.event), verification, index, datetime.date.today(), allowed) + except ValueError as error: + Path(args.message).write_text(str(error) + "\n", encoding="utf-8") + return 2 + VERIFICATION.write_text(dump(data), encoding="utf-8") + OUTPUT.write_text(render(data, index), encoding="utf-8") + Path(args.message).write_text(message + "\n", encoding="utf-8") + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"app={app}\n") + return 0 + for problem in found: + print(f"::error::{problem}") + return 1 if found else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/tests/test_oci_validation.py b/.github/scripts/tests/test_oci_validation.py new file mode 100644 index 00000000..cc3b77d6 --- /dev/null +++ b/.github/scripts/tests/test_oci_validation.py @@ -0,0 +1,231 @@ +"""OCI validation record: generated list, record checks, credited authors and the report bot.""" +import datetime +import importlib.util +import json +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location("oci_validation", ROOT / ".github/scripts/oci_validation.py") +validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validation) + +TODAY = datetime.date(2026, 9, 28) +REPORT = "https://github.com/MacRimi/ProxMenux/issues/400" +INDEX = {"applications": [ + {"id": "glances", "title": "Glances", "category_label": "Monitoring", "automatic_install_candidate": True}, + {"id": "2fauth", "title": "2FAuth", "category_label": "Security", "automatic_install_candidate": True}, + {"id": "legacy", "title": "Legacy | App", "category_label": "Other", "automatic_install_candidate": False}, + {"id": "hidden", "title": "Hidden", "hidden": True}, +]} + + +def form(**answers): + fields = {"Application": "Glances", "Image version": "4.3", "Image digest": "2026-09-06 · B1F339CF", + "Proxmox VE version": "9.1", "Install mode": "Default configuration", + "Data storage": "Volume on Proxmox storage", "Network": "DHCP", "GPU": "No GPU", + "Result": "Works", "Notes": "_No response_"} + fields.update(answers) + return "\n\n".join(f"### {label}\n\n{value}" for label, value in fields.items()) + + +def event(body=None, labels=("oci-validation", "validated"), user="Tester-1"): + return {"issue": {"number": 400, "html_url": REPORT, "user": {"login": user}, "body": form() if body is None else body, + "labels": [{"name": name} for name in labels]}} + + +class Render(unittest.TestCase): + def test_lists_community_and_installable_lab_entries(self): + record = {"applications": { + "glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27", "report": REPORT}}, + "2fauth": {"status": "laboratory-validated"}, + "legacy": {"status": "laboratory-validated"}}} + text = validation.render(record, INDEX) + self.assertIn("2 of 3 applications", text) + self.assertIn("| Glances | [@Vaso73](https://github.com/Vaso73) | 2026-09-27 | — | — | [Report](" + REPORT + ") |", text) + self.assertIn("| 2FAuth | Security |", text) + # An application that cannot be installed is not listed as verified. + self.assertNotIn("Legacy", text) + + def test_empty_record_and_escaped_titles(self): + text = validation.render({"applications": {}}, INDEX) + self.assertIn("No application has been tested by the community yet.", text) + record = {"applications": {"legacy": {"community_tested": {"by": "a", "date": "2026-09-27"}}}} + self.assertIn("| Legacy \\| App |", validation.render(record, INDEX)) + + def test_repository_record_is_valid_and_current(self): + record = validation.load(validation.VERIFICATION) + index = validation.load(validation.INDEX) + self.assertEqual(validation.problems(record, index, validation.OUTPUT.read_text(encoding="utf-8")), []) + + +class Problems(unittest.TestCase): + def check(self, entry): + return validation.problems({"applications": {"glances": entry}}, INDEX, today=TODAY) + + def test_valid_entries(self): + self.assertEqual(self.check({"status": "laboratory-validated"}), []) + self.assertEqual(self.check({"community_tested": {"by": "Vaso73", "date": "2026-09-27", "report": REPORT}}), []) + self.assertEqual(self.check({"community_tested": {"by": "Vaso73", "date": "2026-09-27", + "report": "https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868"}}), []) + + def test_rejected_entries(self): + cases = [ + ({"status": "works"}, "status can only be"), + ({"community_tested": {"by": "bad user", "date": "2026-09-27"}}, "GitHub user name"), + ({"community_tested": {"by": "a", "date": "27/09/2026"}}, "YYYY-MM-DD"), + ({"community_tested": {"by": "a", "date": "2027-01-01"}}, "in the future"), + ({"community_tested": {"by": "a", "date": "2026-09-27", "report": "https://example.com/x"}}, "must link"), + ({"community_tested": {"by": "a", "date": "2026-09-27", "extra": 1}}, "unknown community_tested keys"), + ({"verified": True}, "unknown keys"), + ({}, "non-empty object"), + ] + for entry, message in cases: + with self.subTest(entry=entry): + self.assertTrue(any(message in problem for problem in self.check(entry)), self.check(entry)) + self.assertTrue(validation.problems({"applications": {"nope": {"status": "laboratory-validated"}}}, INDEX)) + + def test_stale_list_is_reported(self): + record = {"applications": {"2fauth": {"status": "laboratory-validated"}}} + self.assertTrue(any("out of date" in p for p in validation.problems(record, INDEX, "old", TODAY))) + + +class Authors(unittest.TestCase): + base = {"applications": {"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27"}}, + "2fauth": {"status": "laboratory-validated"}}} + + def head(self, **changes): + data = json.loads(json.dumps(self.base)) + data["applications"].update(changes) + return data + + def test_author_adds_own_test(self): + head = self.head(pocketbase={"community_tested": {"by": "f3rs3n", "date": "2026-09-28"}}) + self.assertEqual(validation.author_problems(self.base, head, "F3rs3n"), []) + + def test_crediting_someone_else_is_refused(self): + head = self.head(pocketbase={"community_tested": {"by": "Vaso73", "date": "2026-09-28"}}) + self.assertEqual(validation.author_problems(self.base, head, "f3rs3n"), + ["pocketbase: credits @Vaso73, but the pull request is from @f3rs3n"]) + + def test_lab_status_and_removals_are_for_the_maintainer(self): + head = self.head(glances={}, pocketbase={"status": "laboratory-validated"}) + found = validation.author_problems(self.base, head, "f3rs3n") + self.assertIn("glances: a community test is removed by the maintainer only", found) + self.assertIn("pocketbase: the ProxMenux lab status is changed by the maintainer only", found) + self.assertEqual(validation.author_problems(self.base, head, "MacRimi"), []) + + def test_missing_base_file(self): + head = self.head(pocketbase={"community_tested": {"by": "f3rs3n", "date": "2026-09-28"}}) + self.assertEqual(validation.author_problems({}, {"applications": {"pocketbase": head["applications"]["pocketbase"]}}, + "f3rs3n"), []) + + +class Record(unittest.TestCase): + def test_parse_form_reads_github_issue_form_body(self): + fields = validation.parse_form(form(Notes="Line one\r\nLine two")) + self.assertEqual(fields["Application"], "Glances") + self.assertEqual(fields["Notes"], "Line one\nLine two") + self.assertEqual(validation.parse_form(form())["Notes"], "") + + def test_records_the_report_author(self): + data, app, message = validation.record(event(), {"applications": {"glances": {"status": "laboratory-validated"}}}, + INDEX, TODAY) + self.assertEqual(app, "glances") + self.assertEqual(data["applications"]["glances"], {"status": "laboratory-validated", "community_tested": { + "by": "Tester-1", "date": "2026-09-28", "report": REPORT, "digest": "b1f339cf", + "scenario": "Default configuration · Volume on Proxmox storage · DHCP · No GPU · Proxmox VE 9.1 · image 4.3"}}) + self.assertIn("@Tester-1", message) + self.assertEqual(validation.problems(data, INDEX, today=TODAY), []) + + def test_matches_id_or_title_and_replaces_older_test(self): + record = {"applications": {"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27"}}}} + data, app, message = validation.record(event(form(Application=" glances ")), record, INDEX, TODAY) + self.assertEqual(data["applications"]["glances"]["community_tested"]["by"], "Tester-1") + self.assertIn("replaces the previous one from @Vaso73", message) + self.assertEqual(record["applications"]["glances"]["community_tested"]["by"], "Vaso73") + + def test_refusals(self): + for case, message in [ + (event(form(Application="Unknown")), "does not match"), + (event(form(Result="Does not work")), "does not work"), + (event(labels=("validated",)), "not an OCI validation report"), + (event(user="bad user"), "could not be read"), + (event(form(**{"Image digest": "latest"})), "does not hold an image digest"), + ]: + with self.subTest(message=message): + with self.assertRaisesRegex(ValueError, message): + validation.record(case, {"applications": {}}, INDEX, TODAY) + + +class ReviewerComment(unittest.TestCase): + allowed = frozenset({"macrimi", "vaso73", "f3rs3n"}) + + def comment(self, reviewer, author="Tester-1"): + data = event(labels=("oci-validation",), user=author) + data["comment"] = {"user": {"login": reviewer}, "body": "/validated"} + return data + + def test_listed_reviewer_validates_and_is_named(self): + data, app, message = validation.record(self.comment("Vaso73"), {"applications": {}}, INDEX, TODAY, self.allowed) + self.assertEqual(data["applications"]["glances"]["community_tested"]["by"], "Tester-1") + self.assertIn("Validated by @Vaso73.", message) + + def test_unlisted_reviewer_and_own_report_are_refused(self): + with self.assertRaisesRegex(ValueError, "not in the list of OCI validation reviewers"): + validation.record(self.comment("someone"), {"applications": {}}, INDEX, TODAY, self.allowed) + with self.assertRaisesRegex(ValueError, "other than its author"): + validation.record(self.comment("f3rs3n", author="f3rs3n"), {"applications": {}}, INDEX, TODAY, self.allowed) + + def test_maintainer_may_validate_own_report_and_label_needs_no_list(self): + validation.record(self.comment("MacRimi", author="MacRimi"), {"applications": {}}, INDEX, TODAY, self.allowed) + validation.record(event(), {"applications": {}}, INDEX, TODAY) + + def test_reviewer_list_file(self): + self.assertEqual(validation.reviewers("# comment\nMacRimi\n Vaso73 # tester\nbad name\n"), + {"macrimi", "vaso73"}) + listed = validation.reviewers(validation.REVIEWERS.read_text(encoding="utf-8")) + self.assertEqual(listed, self.allowed) + + +class ImageAndScenario(unittest.TestCase): + def test_digest_as_the_monitor_shows_it_or_in_full(self): + full = "sha256:" + "ab" * 32 + self.assertEqual(validation.image_digest("2026-09-06 · b1f339cf"), "b1f339cf") + self.assertEqual(validation.image_digest(full.upper()), "ab" * 32) + for value in ("", "b1f3", "latest", "sha256:xyz12345"): + self.assertIsNone(validation.image_digest(value)) + + def test_scenario_names_problems_and_skips_empty_fields(self): + fields = validation.parse_form(form(Result="Works with problems", GPU="_No response_")) + self.assertEqual(validation.scenario(fields), "Works with problems: Default configuration · " + "Volume on Proxmox storage · DHCP · Proxmox VE 9.1 · image 4.3") + + def test_list_shows_image_and_scenario(self): + record = {"applications": {"glances": {"community_tested": { + "by": "Vaso73", "date": "2026-09-27", "digest": "b1f339cf08ae", "scenario": "Default | DHCP"}}}} + self.assertIn("| `b1f339cf08ae` | Default \\| DHCP |", validation.render(record, INDEX)) + + def test_bad_digest_and_long_scenario_are_rejected(self): + found = validation.problems({"applications": {"glances": {"community_tested": { + "by": "a", "date": "2026-09-27", "digest": "XYZ", "scenario": "x" * 301}}}}, INDEX, today=TODAY) + self.assertTrue(any("digest" in f for f in found) and any("scenario" in f for f in found), found) + + +class IssueForm(unittest.TestCase): + def test_form_labels_match_the_bot(self): + text = (ROOT / ".github/ISSUE_TEMPLATE/oci-validation.yml").read_text(encoding="utf-8") + labels = re.findall(r"^\s+label: (.+)$", text, re.M) + self.assertIn("Application", labels) + self.assertIn("Result", labels) + self.assertIn("Image digest", labels) + for name in validation.SCENARIO_FIELDS: + self.assertIn(name, labels) + self.assertIn("- Does not work", text) + self.assertIn('labels: ["oci-validation"]', text) + self.assertIn("template=oci-validation.yml", validation.FORM) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/oci-validation-label.yml b/.github/workflows/oci-validation-label.yml new file mode 100644 index 00000000..a7e889a5 --- /dev/null +++ b/.github/workflows/oci-validation-label.yml @@ -0,0 +1,25 @@ +name: Label OCI validation pull requests + +# A pull request that changes verification.json gets the "oci-validation" +# label, which adds it to the ProxMenux Roadmap. The pull request's code is +# never checked out: only its list of changed files is read. + +on: + pull_request_target: + types: [opened, reopened, synchronize] + paths: + - 'oci/catalog/verification.json' + +permissions: + issues: write + pull-requests: write + +jobs: + label: + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - env: + GH_TOKEN: ${{ github.token }} + NUMBER: ${{ github.event.pull_request.number }} + run: gh api "repos/$GITHUB_REPOSITORY/issues/$NUMBER/labels" -f "labels[]=oci-validation" --silent diff --git a/.github/workflows/oci-validation-record.yml b/.github/workflows/oci-validation-record.yml new file mode 100644 index 00000000..8fd7eb82 --- /dev/null +++ b/.github/workflows/oci-validation-record.yml @@ -0,0 +1,86 @@ +name: Record OCI validation + +# When an OCI validation report is validated, the report's author is recorded +# in oci/catalog/verification.json on develop and oci/VALIDATION.md is +# regenerated. A report is validated by the "validated" label, which only users +# with write access can add, or by a /validated comment from a user listed in +# .github/oci-validation-reviewers, checked by the script. The issue body and +# the comment are read from the event file, never interpolated into a shell. + +on: + issues: + types: [labeled] + issue_comment: + types: [created] + +concurrency: + group: oci-validation-record + cancel-in-progress: false + +jobs: + record: + if: >- + contains(github.event.issue.labels.*.name, 'oci-validation') && ( + (github.event_name == 'issues' && github.event.label.name == 'validated') || + (github.event_name == 'issue_comment' && !github.event.issue.pull_request && + startsWith(github.event.comment.body, '/validated'))) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + issues: write + steps: + - uses: actions/checkout@v4 + with: + ref: develop + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Record the report + id: record + run: | + set +e + python3 .github/scripts/oci_validation.py record --event "$GITHUB_EVENT_PATH" --message "$RUNNER_TEMP/reply.md" + echo "code=$?" >> "$GITHUB_OUTPUT" + - name: Commit + push + if: steps.record.outputs.code == '0' + env: + APP: ${{ steps.record.outputs.app }} + NUMBER: ${{ github.event.issue.number }} + run: | + git config user.name "ProxMenuxBot" + git config user.email "bot@proxmenux.local" + git add oci/catalog/verification.json oci/VALIDATION.md + git commit -m "chore(oci): record the validation of $APP (#$NUMBER)" + for attempt in 1 2 3 4 5; do + git fetch origin develop + if git rebase origin/develop && git push origin HEAD:develop; then + exit 0 + fi + git rebase --abort 2>/dev/null || true + sleep $(( attempt * 3 )) + done + echo "push failed after 5 attempts" + exit 1 + - name: Reply and close + if: steps.record.outputs.code == '0' + env: + GH_TOKEN: ${{ github.token }} + NUMBER: ${{ github.event.issue.number }} + run: | + gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/reply.md" + gh issue close "$NUMBER" --repo "$GITHUB_REPOSITORY" --reason completed + - name: Explain why it was not recorded + if: steps.record.outputs.code == '2' + env: + GH_TOKEN: ${{ github.token }} + NUMBER: ${{ github.event.issue.number }} + run: | + gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/reply.md" + if [ "$GITHUB_EVENT_NAME" = issues ]; then + gh api -X DELETE "repos/$GITHUB_REPOSITORY/issues/$NUMBER/labels/validated" --silent + fi + - name: Fail on an unexpected error + if: steps.record.outputs.code != '0' && steps.record.outputs.code != '2' + run: exit 1 diff --git a/.github/workflows/oci-validation.yml b/.github/workflows/oci-validation.yml new file mode 100644 index 00000000..5a780479 --- /dev/null +++ b/.github/workflows/oci-validation.yml @@ -0,0 +1,49 @@ +name: OCI validation record + +# verification.json is the record of OCI applications tested for real, and +# oci/VALIDATION.md is generated from it. A pull request only adds or changes +# the community tests credited to its own author. + +on: + pull_request: + paths: + - 'oci/catalog/verification.json' + - 'oci/catalog/index.json' + - 'oci/VALIDATION.md' + - '.github/scripts/oci_validation.py' + - '.github/workflows/oci-validation.yml' + push: + branches: [main, develop] + paths: + - 'oci/catalog/verification.json' + - 'oci/catalog/index.json' + - 'oci/VALIDATION.md' + - '.github/scripts/oci_validation.py' + - '.github/workflows/oci-validation.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Check the record and oci/VALIDATION.md + run: python3 .github/scripts/oci_validation.py check + - name: Check who the pull request credits + if: github.event_name == 'pull_request' + env: + AUTHOR: ${{ github.event.pull_request.user.login }} + BASE: ${{ github.event.pull_request.base.sha }} + run: | + git show "$BASE:oci/catalog/verification.json" > "$RUNNER_TEMP/base.json" 2>/dev/null || : > "$RUNNER_TEMP/base.json" + python3 .github/scripts/oci_validation.py authors --base "$RUNNER_TEMP/base.json" --author "$AUTHOR" diff --git a/oci/README.md b/oci/README.md index b31755c6..3de39a7f 100644 --- a/oci/README.md +++ b/oci/README.md @@ -228,10 +228,17 @@ Generated templates start as `generated-unvalidated`. Promotion requires: 6. Review of every platform adaptation and unsupported feature. Real tests are recorded in `catalog/verification.json`, which the catalog -applies on top of the generated templates and the overlays, so a regeneration -keeps them: `"status": "laboratory-validated"` for an application tested in -the ProxMenux lab, or `"community_tested": {"by": "", "date": -""}` for one tested by the community. +reads when it loads and applies on top of the generated templates and the +overlays, so a new entry shows without regenerating and a regeneration keeps +it: `"status": "laboratory-validated"` for an application tested in the +ProxMenux lab, or `"community_tested": {"by": "", "date": +"", "report": ""}` for one tested by the +community. Both show as verified in the OCI installer. + +[VALIDATION.md](VALIDATION.md) lists the verified applications and explains +how to take part. It is generated from `catalog/verification.json` with +`python3 .github/scripts/oci_validation.py render`, and CI checks that it is +current. The mini changelog comes from the LinuxServer README `Versions` section. At installation, the architecture-specific registry digest and image labels are diff --git a/oci/VALIDATION.md b/oci/VALIDATION.md new file mode 100644 index 00000000..8363b23e --- /dev/null +++ b/oci/VALIDATION.md @@ -0,0 +1,69 @@ +# OCI application validation + + + +38 of 335 applications in the OCI catalog have been tested for real: 36 in the ProxMenux lab and 2 by the community. The OCI installer shows them as verified, with a ✓ in the lists. + +Each test describes the scenario that was run and names the image it ran on. It does not cover every possible configuration of the application, and a newer image has not been tested until someone reports it. + +## Taking part + +Any application that is not listed here is open for testing. + +1. Check the [ProxMenux Roadmap](https://github.com/users/MacRimi/projects/1) to see which applications someone is already testing. With access to the board, create a card for the application and move it to In progress while you test it. +2. Test the application with the OCI manager and fill in the [OCI validation report](https://github.com/MacRimi/ProxMenux/issues/new?template=oci-validation.yml). The report records the image and the exact scenario: install mode, storage, network and GPU. +3. A reviewer reads the report and comments `/validated` on it. The application is then added to this list with your GitHub user and shown as verified in the OCI installer, and the report is closed. + +Reviewers are listed in [.github/oci-validation-reviewers](../.github/oci-validation-reviewers), and a reviewer does not validate their own report. A validation can also be added with a pull request that adds the entry to `oci/catalog/verification.json` and regenerates this file with `python3 .github/scripts/oci_validation.py render`; CI checks that the entry credits the author of the pull request. + +An application that cannot be validated for a reason outside the tester's hands — hardware nobody has, something only Docker provides, a fix still pending in ProxMenux — moves to Blocked on the board, with a line saying why and what would make it worth trying again. + +Questions and ideas about OCI testing go in [discussion #360](https://github.com/MacRimi/ProxMenux/discussions/360). + +## Tested by the community + +| Application | Tested by | Date | Image | Scenario | Report | +|---|---|---|---|---|---| +| Glances | [@Vaso73](https://github.com/Vaso73) | 2026-09-27 | — | — | [Report](https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868) | +| PocketBase | [@Vaso73](https://github.com/Vaso73) | 2026-09-27 | — | — | [Report](https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868) | + +## Tested in the ProxMenux lab + +| Application | Category | +|---|---| +| 2FAuth | Authentication & Security | +| Adguardhome Sync | Adblock & DNS | +| Alby Hub ✨ | Finance & Budgeting | +| Alist | Productivity & Workflows | +| aMule | Files & Downloads | +| CodeProject.AI Server | AI | +| CopyParty | Files & Downloads | +| Crafty | Gaming & Leisure | +| Ddclient | Adblock & DNS | +| Duplicati | Backup & Recovery | +| Etherpad | Documents & Notes | +| FileBrowser Quantum | Tools | +| FlareSolverr | *Arr Suite | +| Flexget | *Arr Suite | +| Frigate | NVR & Cameras | +| Grafana | Monitoring & Analytics | +| Immich | Media | +| JDownloader | Files & Downloads | +| Jenkins CI/CD | AI / Coding & Dev-Tools | +| Linkwarden | Documents & Notes | +| Memos | Documents & Notes | +| MineOS | Gaming & Leisure | +| Motioneye | NVR & Cameras | +| Nextcloud | Productivity & Workflows | +| OpenList | Productivity & Workflows | +| Openssh Server | Remote Access & VPN | +| Paperless-ngx | Productivity & Workflows | +| Phpmyadmin | Databases | +| Qbittorrent | Files & Downloads | +| Rclone WebUI | Backup & Recovery | +| Real-Debrid Torrent Client | Files & Downloads | +| SnapOtter | Media & Streaming | +| Thelounge | Communication & Community | +| Trilium | Documents & Notes | +| Wireguard | Remote Access & VPN | +| WireGuard Easy | Remote Access & VPN | diff --git a/oci/catalog/verification.json b/oci/catalog/verification.json index 62dbfb59..b6c6a90c 100644 --- a/oci/catalog/verification.json +++ b/oci/catalog/verification.json @@ -1,5 +1,5 @@ { - "_comment": "Applications tested for real. \"status\": \"laboratory-validated\" when tested in the ProxMenux lab; \"community_tested\" with the tester's GitHub user and the date when tested by the community. Applied on top of the generated templates and the overlays, so a catalog regeneration keeps it.", + "_comment": "Applications tested for real. \"status\": \"laboratory-validated\" when tested in the ProxMenux lab; \"community_tested\" with the tester's GitHub user, the date and a link to the report when tested by the community. Read when the catalog loads and applied on top of the generated templates and the overlays, so a new entry shows without regenerating and a regeneration keeps it. oci/VALIDATION.md is generated from this file.", "applications": { "2fauth": { "status": "laboratory-validated" @@ -49,7 +49,8 @@ "glances": { "community_tested": { "by": "Vaso73", - "date": "2026-09-27" + "date": "2026-09-27", + "report": "https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868" } }, "grafana": { @@ -94,7 +95,8 @@ "pocketbase": { "community_tested": { "by": "Vaso73", - "date": "2026-09-27" + "date": "2026-09-27", + "report": "https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868" } }, "qbittorrent": { diff --git a/oci/schemas/oci-template.schema.json b/oci/schemas/oci-template.schema.json index 1e75f7f1..0b9b47fd 100644 --- a/oci/schemas/oci-template.schema.json +++ b/oci/schemas/oci-template.schema.json @@ -30,7 +30,9 @@ "properties": { "by": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9-]{0,38}$"}, "date": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, - "report": {"type": "string", "pattern": "^https://github\\.com/MacRimi/ProxMenux/(issues/[0-9]+|discussions/[0-9]+(#discussioncomment-[0-9]+)?)$"} + "report": {"type": "string", "pattern": "^https://github\\.com/MacRimi/ProxMenux/(issues/[0-9]+|discussions/[0-9]+(#discussioncomment-[0-9]+)?)$"}, + "digest": {"type": "string", "pattern": "^[a-f0-9]{8,64}$"}, + "scenario": {"type": "string", "maxLength": 300} } }, "catalog_ui": { diff --git a/oci/src/proxmenux_oci/catalog.py b/oci/src/proxmenux_oci/catalog.py index 313f7d58..e43cd8ae 100644 --- a/oci/src/proxmenux_oci/catalog.py +++ b/oci/src/proxmenux_oci/catalog.py @@ -341,6 +341,7 @@ class Catalog: return self.sync_index() payload = json.loads(self.index_path.read_text(encoding="utf-8")) self._enrich_index_from_templates(payload) + self._apply_verification_to_index(payload) return payload def categories(self) -> dict[str, Any]: @@ -816,13 +817,29 @@ class Catalog: if isinstance(entry.get("community_tested"), dict): template["community_tested"] = dict(entry["community_tested"]) + def _apply_verification_to_index(self, payload: dict[str, Any]) -> None: + """verification.json is read at load time, so a new entry shows without + regenerating the templates.""" + path = self.catalog_dir / "verification.json" + try: + entries = json.loads(path.read_text(encoding="utf-8")).get("applications", {}) + except (OSError, json.JSONDecodeError, AttributeError): + return + for item in payload.get("applications", []): + entry = entries.get(item.get("id")) + if not isinstance(entry, dict): + entry = {} + if entry.get("status") == "laboratory-validated" and item.get("automatic_install_candidate"): + item["template_status"] = "laboratory-validated" + self._index_community_tested(item, entry) + @staticmethod def _index_community_tested(item: dict[str, Any], template: dict[str, Any]) -> None: """Who tested the application for real outside the ProxMenux lab, and - when, as recorded in its overlay.""" + when, as recorded in verification.json.""" tested = template.get("community_tested") if isinstance(tested, dict) and tested.get("by"): - item["community_tested"] = {"by": str(tested["by"]), "date": str(tested.get("date") or "")} + item["community_tested"] = {key: str(tested[key]) for key in ("by", "date", "report") if tested.get(key)} else: item.pop("community_tested", None) diff --git a/oci/tests/test_verification_at_load.py b/oci/tests/test_verification_at_load.py new file mode 100644 index 00000000..962783ce --- /dev/null +++ b/oci/tests/test_verification_at_load.py @@ -0,0 +1,54 @@ +"""verification.json applies when the catalog loads, without regenerating templates.""" + +import json +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from proxmenux_oci import cli +from proxmenux_oci.catalog import Catalog + + +class VerificationAtLoadTests(unittest.TestCase): + def catalog(self, applications, verification): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name) + (root / "catalog").mkdir() + files = {"index.json": {"applications": applications}, + "categories.json": {"applications": {}, "labels": {}}, + "verification.json": {"applications": verification}} + for name, data in files.items(): + (root / "catalog" / name).write_text(json.dumps(data), encoding="utf-8") + return {item["id"]: item for item in Catalog(root).load_index()["applications"]} + + def test_new_entries_show_as_verified(self): + items = self.catalog( + [{"id": "glances", "automatic_install_candidate": True}, + {"id": "2fauth", "automatic_install_candidate": True}, + {"id": "legacy", "automatic_install_candidate": False}, + {"id": "plex", "automatic_install_candidate": True}], + {"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27", + "report": "https://github.com/MacRimi/ProxMenux/issues/400"}}, + "2fauth": {"status": "laboratory-validated"}, + "legacy": {"status": "laboratory-validated"}}) + self.assertTrue(cli.is_tested(items["glances"])) + self.assertEqual(items["glances"]["community_tested"]["report"], + "https://github.com/MacRimi/ProxMenux/issues/400") + self.assertTrue(cli.is_tested(items["2fauth"])) + self.assertFalse(cli.is_tested(items["legacy"])) + self.assertFalse(cli.is_tested(items["plex"])) + + def test_removed_community_entry_no_longer_shows(self): + items = self.catalog([{"id": "glances", "automatic_install_candidate": True, + "community_tested": {"by": "Vaso73", "date": "2026-09-27"}}], {}) + self.assertNotIn("community_tested", items["glances"]) + self.assertFalse(cli.is_tested(items["glances"])) + + +if __name__ == "__main__": + unittest.main()