mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-09-29 18:16:43 +00:00
feat(oci): community validation record, report form and generated list
This commit is contained in:
MacRimi
@@ -0,0 +1,105 @@
|
||||
name: OCI validation report
|
||||
description: Report a real test of an application from the OCI manager catalog.
|
||||
title: "OCI validation: "
|
||||
labels: ["oci-validation"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
One report describes one application and the scenario you tested. Once it is reviewed, the application is recorded as verified: it shows with a ✓ in the OCI installer and is listed in [oci/VALIDATION.md](https://github.com/MacRimi/ProxMenux/blob/develop/oci/VALIDATION.md) with your GitHub user, the date and a link to this report.
|
||||
|
||||
Reports that an application does not work are just as useful: they tell others what to expect and what needs fixing.
|
||||
- type: input
|
||||
id: application
|
||||
attributes:
|
||||
label: Application
|
||||
description: Name or ID as it appears in the OCI catalog, for example "Glances" or "glances".
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Image version
|
||||
description: Version or tag shown by the installer.
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: digest
|
||||
attributes:
|
||||
label: Image digest
|
||||
description: Shown in the App tab of the container in ProxMenux Monitor, after the build date — for example "2026-09-06 · b1f339cf". The full sha256 digest is also accepted.
|
||||
placeholder: b1f339cf
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: proxmox
|
||||
attributes:
|
||||
label: Proxmox VE version
|
||||
placeholder: "9.1"
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: install
|
||||
attributes:
|
||||
label: Install mode
|
||||
options:
|
||||
- Default configuration
|
||||
- Advanced configuration
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: storage
|
||||
attributes:
|
||||
label: Data storage
|
||||
options:
|
||||
- Volume on Proxmox storage
|
||||
- Host directory
|
||||
- Both
|
||||
- No persistent data
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: network
|
||||
attributes:
|
||||
label: Network
|
||||
options:
|
||||
- DHCP
|
||||
- Static address
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: gpu
|
||||
attributes:
|
||||
label: GPU
|
||||
options:
|
||||
- No GPU
|
||||
- Intel
|
||||
- AMD
|
||||
- NVIDIA
|
||||
validations:
|
||||
required: true
|
||||
- type: checkboxes
|
||||
id: checks
|
||||
attributes:
|
||||
label: What was checked
|
||||
options:
|
||||
- label: Installs cleanly
|
||||
- label: The application responds (web interface or service)
|
||||
- label: Keeps working after a container restart
|
||||
- label: Survives a Proxmox backup and restore
|
||||
- label: An image update keeps the data
|
||||
- type: dropdown
|
||||
id: result
|
||||
attributes:
|
||||
label: Result
|
||||
options:
|
||||
- Works
|
||||
- Works with problems
|
||||
- Does not work
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: notes
|
||||
attributes:
|
||||
label: Notes
|
||||
description: Anything worth knowing. If something failed, what you saw and how to reproduce it.
|
||||
@@ -0,0 +1,5 @@
|
||||
# GitHub users who can validate an OCI validation report by commenting
|
||||
# /validated on it. A reviewer does not validate their own report.
|
||||
MacRimi
|
||||
Vaso73
|
||||
f3rs3n
|
||||
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
"""OCI validation record: renders oci/VALIDATION.md from verification.json,
|
||||
checks the record, checks who a pull request credits, and records a reviewed
|
||||
validation report."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VERIFICATION = ROOT / "oci/catalog/verification.json"
|
||||
INDEX = ROOT / "oci/catalog/index.json"
|
||||
OUTPUT = ROOT / "oci/VALIDATION.md"
|
||||
REVIEWERS = ROOT / ".github/oci-validation-reviewers"
|
||||
|
||||
REPO = "https://github.com/MacRimi/ProxMenux"
|
||||
BOARD = "https://github.com/users/MacRimi/projects/1"
|
||||
FORM = f"{REPO}/issues/new?template=oci-validation.yml"
|
||||
DISCUSSION = f"{REPO}/discussions/360"
|
||||
MAINTAINERS = {"macrimi"}
|
||||
LAB = "laboratory-validated"
|
||||
|
||||
USER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9-]{0,38}$")
|
||||
REPORT_RE = re.compile(r"^https://github\.com/MacRimi/ProxMenux/"
|
||||
r"(issues/[0-9]+|discussions/[0-9]+(#discussioncomment-[0-9]+)?)$")
|
||||
ENTRY_KEYS = {"status", "community_tested"}
|
||||
COMMUNITY_KEYS = {"by", "date", "report", "digest", "scenario"}
|
||||
DIGEST_RE = re.compile(r"^[a-f0-9]{8,64}$")
|
||||
# Form fields that describe the tested scenario, in the order they are read.
|
||||
SCENARIO_FIELDS = ("Install mode", "Data storage", "Network", "GPU")
|
||||
NO_RESPONSE = "_No response_"
|
||||
|
||||
|
||||
def load(path):
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def dump(data):
|
||||
return json.dumps(data, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def applications(index):
|
||||
return {item["id"]: item for item in index.get("applications", [])}
|
||||
|
||||
|
||||
def lab_validated(entry, item):
|
||||
# Same rule as the catalog: an application that cannot be installed is not verified.
|
||||
return entry.get("status") == LAB and bool(item.get("automatic_install_candidate"))
|
||||
|
||||
|
||||
def community(entry):
|
||||
tested = entry.get("community_tested")
|
||||
return tested if isinstance(tested, dict) and tested.get("by") else None
|
||||
|
||||
|
||||
def cell(text):
|
||||
return str(text).replace("|", "\\|").strip()
|
||||
|
||||
|
||||
def render(verification, index):
|
||||
apps = applications(index)
|
||||
entries = {app: entry for app, entry in verification.get("applications", {}).items()
|
||||
if app in apps and isinstance(entry, dict)}
|
||||
lab = sorted((app for app, entry in entries.items() if lab_validated(entry, apps[app])),
|
||||
key=lambda app: str(apps[app].get("title") or app).casefold())
|
||||
tested = sorted((app for app, entry in entries.items() if community(entry)),
|
||||
key=lambda app: str(apps[app].get("title") or app).casefold())
|
||||
visible = sum(1 for item in apps.values() if not item.get("hidden"))
|
||||
verified = len(set(lab) | set(tested))
|
||||
lines = [
|
||||
"# OCI application validation",
|
||||
"",
|
||||
"<!-- Generated from oci/catalog/verification.json by .github/scripts/oci_validation.py. Do not edit by hand. -->",
|
||||
"",
|
||||
f"{verified} of {visible} applications in the OCI catalog have been tested for real: "
|
||||
f"{len(lab)} in the ProxMenux lab and {len(tested)} by the community. "
|
||||
"The OCI installer shows them as verified, with a ✓ in the lists.",
|
||||
"",
|
||||
"Each test describes the scenario that was run and names the image it ran on. It does not cover every possible "
|
||||
"configuration of the application, and a newer image has not been tested until someone reports it.",
|
||||
"",
|
||||
"## Taking part",
|
||||
"",
|
||||
"Any application that is not listed here is open for testing.",
|
||||
"",
|
||||
f"1. Check the [ProxMenux Roadmap]({BOARD}) to see which applications someone is already testing. "
|
||||
"With access to the board, create a card for the application and move it to In progress while you test it.",
|
||||
f"2. Test the application with the OCI manager and fill in the [OCI validation report]({FORM}). "
|
||||
"The report records the image and the exact scenario: install mode, storage, network and GPU.",
|
||||
"3. A reviewer reads the report and comments `/validated` on it. The application is then added to this list "
|
||||
"with your GitHub user and shown as verified in the OCI installer, and the report is closed.",
|
||||
"",
|
||||
"Reviewers are listed in [.github/oci-validation-reviewers](../.github/oci-validation-reviewers), and a "
|
||||
"reviewer does not validate their own report. A validation can also be added with a pull request that adds "
|
||||
"the entry to `oci/catalog/verification.json` and regenerates this file with "
|
||||
"`python3 .github/scripts/oci_validation.py render`; CI checks that the entry credits the author of the pull request.",
|
||||
"",
|
||||
"An application that cannot be validated for a reason outside the tester's hands — hardware nobody has, something "
|
||||
"only Docker provides, a fix still pending in ProxMenux — moves to Blocked on the board, with a line saying why "
|
||||
"and what would make it worth trying again.",
|
||||
"",
|
||||
f"Questions and ideas about OCI testing go in [discussion #360]({DISCUSSION}).",
|
||||
"",
|
||||
"## Tested by the community",
|
||||
"",
|
||||
]
|
||||
if tested:
|
||||
lines += ["| Application | Tested by | Date | Image | Scenario | Report |", "|---|---|---|---|---|---|"]
|
||||
for app in tested:
|
||||
entry = community(entries[app])
|
||||
report = f"[Report]({entry['report']})" if entry.get("report") else "—"
|
||||
image = f"`{entry['digest'][:12]}`" if entry.get("digest") else "—"
|
||||
lines.append(f"| {cell(apps[app].get('title') or app)} | [@{entry['by']}](https://github.com/{entry['by']}) "
|
||||
f"| {entry.get('date') or '—'} | {image} | {cell(entry.get('scenario') or '—')} | {report} |")
|
||||
else:
|
||||
lines.append("No application has been tested by the community yet.")
|
||||
lines += ["", "## Tested in the ProxMenux lab", ""]
|
||||
if lab:
|
||||
lines += ["| Application | Category |", "|---|---|"]
|
||||
for app in lab:
|
||||
lines.append(f"| {cell(apps[app].get('title') or app)} | {cell(apps[app].get('category_label') or '—')} |")
|
||||
else:
|
||||
lines.append("No application has been tested in the ProxMenux lab yet.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def problems(verification, index, rendered=None, today=None):
|
||||
today = today or datetime.date.today()
|
||||
apps = applications(index)
|
||||
found = []
|
||||
entries = verification.get("applications") if isinstance(verification, dict) else None
|
||||
if not isinstance(entries, dict):
|
||||
return ["verification.json must hold an \"applications\" object"]
|
||||
for app, entry in entries.items():
|
||||
where = f"verification.json: {app}"
|
||||
if app not in apps:
|
||||
found.append(f"{where}: not an application of the OCI catalog")
|
||||
if not isinstance(entry, dict) or not entry:
|
||||
found.append(f"{where}: the entry must be a non-empty object")
|
||||
continue
|
||||
if set(entry) - ENTRY_KEYS:
|
||||
found.append(f"{where}: unknown keys {sorted(set(entry) - ENTRY_KEYS)}")
|
||||
if "status" in entry and entry["status"] != LAB:
|
||||
found.append(f"{where}: status can only be \"{LAB}\"")
|
||||
if "community_tested" not in entry:
|
||||
continue
|
||||
tested = entry["community_tested"]
|
||||
if not isinstance(tested, dict):
|
||||
found.append(f"{where}: community_tested must be an object")
|
||||
continue
|
||||
if set(tested) - COMMUNITY_KEYS:
|
||||
found.append(f"{where}: unknown community_tested keys {sorted(set(tested) - COMMUNITY_KEYS)}")
|
||||
if not USER_RE.fullmatch(str(tested.get("by", ""))):
|
||||
found.append(f"{where}: \"by\" must be a GitHub user name")
|
||||
try:
|
||||
date = datetime.date.fromisoformat(str(tested.get("date", "")))
|
||||
if date > today + datetime.timedelta(days=1):
|
||||
found.append(f"{where}: the date is in the future")
|
||||
except ValueError:
|
||||
found.append(f"{where}: \"date\" must be YYYY-MM-DD")
|
||||
if "report" in tested and not REPORT_RE.fullmatch(str(tested["report"])):
|
||||
found.append(f"{where}: \"report\" must link to an issue or discussion of MacRimi/ProxMenux")
|
||||
if "digest" in tested and not DIGEST_RE.fullmatch(str(tested["digest"])):
|
||||
found.append(f"{where}: \"digest\" must be 8 to 64 lowercase hex characters of the image digest")
|
||||
if "scenario" in tested and (not isinstance(tested["scenario"], str) or len(tested["scenario"]) > 300):
|
||||
found.append(f"{where}: \"scenario\" must be text of at most 300 characters")
|
||||
if rendered is not None and rendered != render(verification, index):
|
||||
found.append("oci/VALIDATION.md is out of date: run python3 .github/scripts/oci_validation.py render")
|
||||
return found
|
||||
|
||||
|
||||
def author_problems(base, head, author):
|
||||
"""A pull request only adds or changes community tests credited to its author."""
|
||||
if author.casefold() in MAINTAINERS:
|
||||
return []
|
||||
found = []
|
||||
before = base.get("applications", {}) if isinstance(base, dict) else {}
|
||||
after = head.get("applications", {})
|
||||
for app in sorted(set(before) | set(after)):
|
||||
old = before.get(app) if isinstance(before.get(app), dict) else {}
|
||||
new = after.get(app) if isinstance(after.get(app), dict) else {}
|
||||
if old.get("status") != new.get("status"):
|
||||
found.append(f"{app}: the ProxMenux lab status is changed by the maintainer only")
|
||||
if old.get("community_tested") == new.get("community_tested"):
|
||||
continue
|
||||
credited = community(new)
|
||||
if not credited:
|
||||
found.append(f"{app}: a community test is removed by the maintainer only")
|
||||
elif str(credited["by"]).casefold() != author.casefold():
|
||||
found.append(f"{app}: credits @{credited['by']}, but the pull request is from @{author}")
|
||||
return found
|
||||
|
||||
|
||||
def parse_form(body):
|
||||
"""Issue form answers, keyed by the field label."""
|
||||
fields, label, lines = {}, None, []
|
||||
for line in (body or "").replace("\r\n", "\n").split("\n"):
|
||||
if line.startswith("### "):
|
||||
if label is not None:
|
||||
fields[label] = "\n".join(lines).strip()
|
||||
label, lines = line[4:].strip(), []
|
||||
elif label is not None:
|
||||
lines.append(line)
|
||||
if label is not None:
|
||||
fields[label] = "\n".join(lines).strip()
|
||||
return {key: ("" if value == NO_RESPONSE else value) for key, value in fields.items()}
|
||||
|
||||
|
||||
def resolve(name, index):
|
||||
wanted = " ".join(name.split()).casefold()
|
||||
matches = [item["id"] for item in index.get("applications", [])
|
||||
if wanted in (str(item["id"]).casefold(), " ".join(str(item.get("title") or "").split()).casefold())]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def image_digest(text):
|
||||
"""The digest as the Monitor shows it (b1f339cf) or in full (sha256:b1f3...)."""
|
||||
value = str(text or "").strip().lower()
|
||||
value = value.split("·")[-1].strip()
|
||||
value = value[7:] if value.startswith("sha256:") else value
|
||||
return value if DIGEST_RE.fullmatch(value) else None
|
||||
|
||||
|
||||
def scenario(fields):
|
||||
parts = [fields.get(name, "") for name in SCENARIO_FIELDS]
|
||||
if fields.get("Proxmox VE version"):
|
||||
parts.append(f"Proxmox VE {fields['Proxmox VE version']}")
|
||||
if fields.get("Image version"):
|
||||
parts.append(f"image {fields['Image version']}")
|
||||
text = " · ".join(part.strip() for part in parts if part and part.strip())
|
||||
if fields.get("Result") == "Works with problems":
|
||||
text = f"Works with problems: {text}"
|
||||
return text[:300]
|
||||
|
||||
|
||||
def reviewers(text):
|
||||
"""User names of the reviewer list, one per line; # starts a comment."""
|
||||
names = {line.split("#", 1)[0].strip() for line in text.splitlines()}
|
||||
return {name.casefold() for name in names if USER_RE.fullmatch(name)}
|
||||
|
||||
|
||||
def record(event, verification, index, today, allowed=frozenset()):
|
||||
"""Returns (verification, app id, message); raises ValueError with the reason to refuse.
|
||||
|
||||
A label can only be added by someone with write access to the repository.
|
||||
A /validated comment is accepted from the reviewer list, never from the
|
||||
report's own author unless it is the maintainer."""
|
||||
issue = event.get("issue") or {}
|
||||
labels = {label.get("name") for label in issue.get("labels", [])}
|
||||
if "oci-validation" not in labels:
|
||||
raise ValueError("This issue is not an OCI validation report.")
|
||||
author = str((issue.get("user") or {}).get("login", ""))
|
||||
report = str(issue.get("html_url", ""))
|
||||
if not USER_RE.fullmatch(author) or not REPORT_RE.fullmatch(report):
|
||||
raise ValueError("The report author or link could not be read.")
|
||||
reviewer = str(((event.get("comment") or {}).get("user") or {}).get("login", ""))
|
||||
if event.get("comment") is not None:
|
||||
if reviewer.casefold() not in allowed:
|
||||
raise ValueError(f"@{reviewer} is not in the list of OCI validation reviewers "
|
||||
"(.github/oci-validation-reviewers), so the report is not recorded.")
|
||||
if reviewer.casefold() == author.casefold() and reviewer.casefold() not in MAINTAINERS:
|
||||
raise ValueError("A report is validated by a reviewer other than its author.")
|
||||
fields = parse_form(issue.get("body"))
|
||||
name = fields.get("Application", "")
|
||||
app = resolve(name, index)
|
||||
if app is None:
|
||||
raise ValueError(f"\"{name}\" does not match one application of the OCI catalog. "
|
||||
"Edit the Application field with its name or ID as shown in the catalog, then add the label again.")
|
||||
if fields.get("Result") == "Does not work":
|
||||
raise ValueError("The report says the application does not work, so it is not recorded as verified.")
|
||||
digest = image_digest(fields.get("Image digest"))
|
||||
if digest is None:
|
||||
raise ValueError("The Image digest field does not hold an image digest. Edit it with the digest shown in the "
|
||||
"App tab of the Monitor, for example b1f339cf, then add the label again.")
|
||||
data = json.loads(json.dumps(verification))
|
||||
entries = data.setdefault("applications", {})
|
||||
entry = entries.setdefault(app, {})
|
||||
previous = community(entry)
|
||||
entry["community_tested"] = {"by": author, "date": today.isoformat(), "report": report,
|
||||
"digest": digest, "scenario": scenario(fields)}
|
||||
data["applications"] = dict(sorted(entries.items()))
|
||||
title = applications(index)[app].get("title") or app
|
||||
validated = f" Validated by @{reviewer}." if reviewer else ""
|
||||
message = (f"Thank you, @{author}! {title} is now recorded as verified.{validated} "
|
||||
f"It shows with a ✓ in the OCI installer and is listed in [oci/VALIDATION.md]({REPO}/blob/develop/oci/VALIDATION.md).")
|
||||
if previous and previous.get("by") != author:
|
||||
message += f" This report replaces the previous one from @{previous['by']}."
|
||||
return data, app, message
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("render", help="Write oci/VALIDATION.md")
|
||||
commands.add_parser("check", help="Check verification.json and that oci/VALIDATION.md is current")
|
||||
authors = commands.add_parser("authors", help="Check who a pull request credits")
|
||||
authors.add_argument("--base", required=True, help="verification.json of the base branch; may be missing")
|
||||
authors.add_argument("--author", required=True)
|
||||
rec = commands.add_parser("record", help="Record a reviewed validation report")
|
||||
rec.add_argument("--event", required=True)
|
||||
rec.add_argument("--message", required=True, help="File that receives the reply for the issue")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
verification, index = load(VERIFICATION), load(INDEX)
|
||||
if args.command == "render":
|
||||
OUTPUT.write_text(render(verification, index), encoding="utf-8")
|
||||
return 0
|
||||
if args.command == "check":
|
||||
rendered = OUTPUT.read_text(encoding="utf-8") if OUTPUT.exists() else ""
|
||||
found = problems(verification, index, rendered)
|
||||
elif args.command == "authors":
|
||||
base = load(args.base) if Path(args.base).exists() and Path(args.base).stat().st_size else {}
|
||||
found = author_problems(base, verification, args.author)
|
||||
else:
|
||||
try:
|
||||
allowed = reviewers(REVIEWERS.read_text(encoding="utf-8")) if REVIEWERS.exists() else frozenset()
|
||||
data, app, message = record(load(args.event), verification, index, datetime.date.today(), allowed)
|
||||
except ValueError as error:
|
||||
Path(args.message).write_text(str(error) + "\n", encoding="utf-8")
|
||||
return 2
|
||||
VERIFICATION.write_text(dump(data), encoding="utf-8")
|
||||
OUTPUT.write_text(render(data, index), encoding="utf-8")
|
||||
Path(args.message).write_text(message + "\n", encoding="utf-8")
|
||||
if os.environ.get("GITHUB_OUTPUT"):
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
|
||||
output.write(f"app={app}\n")
|
||||
return 0
|
||||
for problem in found:
|
||||
print(f"::error::{problem}")
|
||||
return 1 if found else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,231 @@
|
||||
"""OCI validation record: generated list, record checks, credited authors and the report bot."""
|
||||
import datetime
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
spec = importlib.util.spec_from_file_location("oci_validation", ROOT / ".github/scripts/oci_validation.py")
|
||||
validation = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(validation)
|
||||
|
||||
TODAY = datetime.date(2026, 9, 28)
|
||||
REPORT = "https://github.com/MacRimi/ProxMenux/issues/400"
|
||||
INDEX = {"applications": [
|
||||
{"id": "glances", "title": "Glances", "category_label": "Monitoring", "automatic_install_candidate": True},
|
||||
{"id": "2fauth", "title": "2FAuth", "category_label": "Security", "automatic_install_candidate": True},
|
||||
{"id": "legacy", "title": "Legacy | App", "category_label": "Other", "automatic_install_candidate": False},
|
||||
{"id": "hidden", "title": "Hidden", "hidden": True},
|
||||
]}
|
||||
|
||||
|
||||
def form(**answers):
|
||||
fields = {"Application": "Glances", "Image version": "4.3", "Image digest": "2026-09-06 · B1F339CF",
|
||||
"Proxmox VE version": "9.1", "Install mode": "Default configuration",
|
||||
"Data storage": "Volume on Proxmox storage", "Network": "DHCP", "GPU": "No GPU",
|
||||
"Result": "Works", "Notes": "_No response_"}
|
||||
fields.update(answers)
|
||||
return "\n\n".join(f"### {label}\n\n{value}" for label, value in fields.items())
|
||||
|
||||
|
||||
def event(body=None, labels=("oci-validation", "validated"), user="Tester-1"):
|
||||
return {"issue": {"number": 400, "html_url": REPORT, "user": {"login": user}, "body": form() if body is None else body,
|
||||
"labels": [{"name": name} for name in labels]}}
|
||||
|
||||
|
||||
class Render(unittest.TestCase):
|
||||
def test_lists_community_and_installable_lab_entries(self):
|
||||
record = {"applications": {
|
||||
"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27", "report": REPORT}},
|
||||
"2fauth": {"status": "laboratory-validated"},
|
||||
"legacy": {"status": "laboratory-validated"}}}
|
||||
text = validation.render(record, INDEX)
|
||||
self.assertIn("2 of 3 applications", text)
|
||||
self.assertIn("| Glances | [@Vaso73](https://github.com/Vaso73) | 2026-09-27 | — | — | [Report](" + REPORT + ") |", text)
|
||||
self.assertIn("| 2FAuth | Security |", text)
|
||||
# An application that cannot be installed is not listed as verified.
|
||||
self.assertNotIn("Legacy", text)
|
||||
|
||||
def test_empty_record_and_escaped_titles(self):
|
||||
text = validation.render({"applications": {}}, INDEX)
|
||||
self.assertIn("No application has been tested by the community yet.", text)
|
||||
record = {"applications": {"legacy": {"community_tested": {"by": "a", "date": "2026-09-27"}}}}
|
||||
self.assertIn("| Legacy \\| App |", validation.render(record, INDEX))
|
||||
|
||||
def test_repository_record_is_valid_and_current(self):
|
||||
record = validation.load(validation.VERIFICATION)
|
||||
index = validation.load(validation.INDEX)
|
||||
self.assertEqual(validation.problems(record, index, validation.OUTPUT.read_text(encoding="utf-8")), [])
|
||||
|
||||
|
||||
class Problems(unittest.TestCase):
|
||||
def check(self, entry):
|
||||
return validation.problems({"applications": {"glances": entry}}, INDEX, today=TODAY)
|
||||
|
||||
def test_valid_entries(self):
|
||||
self.assertEqual(self.check({"status": "laboratory-validated"}), [])
|
||||
self.assertEqual(self.check({"community_tested": {"by": "Vaso73", "date": "2026-09-27", "report": REPORT}}), [])
|
||||
self.assertEqual(self.check({"community_tested": {"by": "Vaso73", "date": "2026-09-27",
|
||||
"report": "https://github.com/MacRimi/ProxMenux/discussions/392#discussioncomment-18623868"}}), [])
|
||||
|
||||
def test_rejected_entries(self):
|
||||
cases = [
|
||||
({"status": "works"}, "status can only be"),
|
||||
({"community_tested": {"by": "bad user", "date": "2026-09-27"}}, "GitHub user name"),
|
||||
({"community_tested": {"by": "a", "date": "27/09/2026"}}, "YYYY-MM-DD"),
|
||||
({"community_tested": {"by": "a", "date": "2027-01-01"}}, "in the future"),
|
||||
({"community_tested": {"by": "a", "date": "2026-09-27", "report": "https://example.com/x"}}, "must link"),
|
||||
({"community_tested": {"by": "a", "date": "2026-09-27", "extra": 1}}, "unknown community_tested keys"),
|
||||
({"verified": True}, "unknown keys"),
|
||||
({}, "non-empty object"),
|
||||
]
|
||||
for entry, message in cases:
|
||||
with self.subTest(entry=entry):
|
||||
self.assertTrue(any(message in problem for problem in self.check(entry)), self.check(entry))
|
||||
self.assertTrue(validation.problems({"applications": {"nope": {"status": "laboratory-validated"}}}, INDEX))
|
||||
|
||||
def test_stale_list_is_reported(self):
|
||||
record = {"applications": {"2fauth": {"status": "laboratory-validated"}}}
|
||||
self.assertTrue(any("out of date" in p for p in validation.problems(record, INDEX, "old", TODAY)))
|
||||
|
||||
|
||||
class Authors(unittest.TestCase):
|
||||
base = {"applications": {"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27"}},
|
||||
"2fauth": {"status": "laboratory-validated"}}}
|
||||
|
||||
def head(self, **changes):
|
||||
data = json.loads(json.dumps(self.base))
|
||||
data["applications"].update(changes)
|
||||
return data
|
||||
|
||||
def test_author_adds_own_test(self):
|
||||
head = self.head(pocketbase={"community_tested": {"by": "f3rs3n", "date": "2026-09-28"}})
|
||||
self.assertEqual(validation.author_problems(self.base, head, "F3rs3n"), [])
|
||||
|
||||
def test_crediting_someone_else_is_refused(self):
|
||||
head = self.head(pocketbase={"community_tested": {"by": "Vaso73", "date": "2026-09-28"}})
|
||||
self.assertEqual(validation.author_problems(self.base, head, "f3rs3n"),
|
||||
["pocketbase: credits @Vaso73, but the pull request is from @f3rs3n"])
|
||||
|
||||
def test_lab_status_and_removals_are_for_the_maintainer(self):
|
||||
head = self.head(glances={}, pocketbase={"status": "laboratory-validated"})
|
||||
found = validation.author_problems(self.base, head, "f3rs3n")
|
||||
self.assertIn("glances: a community test is removed by the maintainer only", found)
|
||||
self.assertIn("pocketbase: the ProxMenux lab status is changed by the maintainer only", found)
|
||||
self.assertEqual(validation.author_problems(self.base, head, "MacRimi"), [])
|
||||
|
||||
def test_missing_base_file(self):
|
||||
head = self.head(pocketbase={"community_tested": {"by": "f3rs3n", "date": "2026-09-28"}})
|
||||
self.assertEqual(validation.author_problems({}, {"applications": {"pocketbase": head["applications"]["pocketbase"]}},
|
||||
"f3rs3n"), [])
|
||||
|
||||
|
||||
class Record(unittest.TestCase):
|
||||
def test_parse_form_reads_github_issue_form_body(self):
|
||||
fields = validation.parse_form(form(Notes="Line one\r\nLine two"))
|
||||
self.assertEqual(fields["Application"], "Glances")
|
||||
self.assertEqual(fields["Notes"], "Line one\nLine two")
|
||||
self.assertEqual(validation.parse_form(form())["Notes"], "")
|
||||
|
||||
def test_records_the_report_author(self):
|
||||
data, app, message = validation.record(event(), {"applications": {"glances": {"status": "laboratory-validated"}}},
|
||||
INDEX, TODAY)
|
||||
self.assertEqual(app, "glances")
|
||||
self.assertEqual(data["applications"]["glances"], {"status": "laboratory-validated", "community_tested": {
|
||||
"by": "Tester-1", "date": "2026-09-28", "report": REPORT, "digest": "b1f339cf",
|
||||
"scenario": "Default configuration · Volume on Proxmox storage · DHCP · No GPU · Proxmox VE 9.1 · image 4.3"}})
|
||||
self.assertIn("@Tester-1", message)
|
||||
self.assertEqual(validation.problems(data, INDEX, today=TODAY), [])
|
||||
|
||||
def test_matches_id_or_title_and_replaces_older_test(self):
|
||||
record = {"applications": {"glances": {"community_tested": {"by": "Vaso73", "date": "2026-09-27"}}}}
|
||||
data, app, message = validation.record(event(form(Application=" glances ")), record, INDEX, TODAY)
|
||||
self.assertEqual(data["applications"]["glances"]["community_tested"]["by"], "Tester-1")
|
||||
self.assertIn("replaces the previous one from @Vaso73", message)
|
||||
self.assertEqual(record["applications"]["glances"]["community_tested"]["by"], "Vaso73")
|
||||
|
||||
def test_refusals(self):
|
||||
for case, message in [
|
||||
(event(form(Application="Unknown")), "does not match"),
|
||||
(event(form(Result="Does not work")), "does not work"),
|
||||
(event(labels=("validated",)), "not an OCI validation report"),
|
||||
(event(user="bad user"), "could not be read"),
|
||||
(event(form(**{"Image digest": "latest"})), "does not hold an image digest"),
|
||||
]:
|
||||
with self.subTest(message=message):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
validation.record(case, {"applications": {}}, INDEX, TODAY)
|
||||
|
||||
|
||||
class ReviewerComment(unittest.TestCase):
|
||||
allowed = frozenset({"macrimi", "vaso73", "f3rs3n"})
|
||||
|
||||
def comment(self, reviewer, author="Tester-1"):
|
||||
data = event(labels=("oci-validation",), user=author)
|
||||
data["comment"] = {"user": {"login": reviewer}, "body": "/validated"}
|
||||
return data
|
||||
|
||||
def test_listed_reviewer_validates_and_is_named(self):
|
||||
data, app, message = validation.record(self.comment("Vaso73"), {"applications": {}}, INDEX, TODAY, self.allowed)
|
||||
self.assertEqual(data["applications"]["glances"]["community_tested"]["by"], "Tester-1")
|
||||
self.assertIn("Validated by @Vaso73.", message)
|
||||
|
||||
def test_unlisted_reviewer_and_own_report_are_refused(self):
|
||||
with self.assertRaisesRegex(ValueError, "not in the list of OCI validation reviewers"):
|
||||
validation.record(self.comment("someone"), {"applications": {}}, INDEX, TODAY, self.allowed)
|
||||
with self.assertRaisesRegex(ValueError, "other than its author"):
|
||||
validation.record(self.comment("f3rs3n", author="f3rs3n"), {"applications": {}}, INDEX, TODAY, self.allowed)
|
||||
|
||||
def test_maintainer_may_validate_own_report_and_label_needs_no_list(self):
|
||||
validation.record(self.comment("MacRimi", author="MacRimi"), {"applications": {}}, INDEX, TODAY, self.allowed)
|
||||
validation.record(event(), {"applications": {}}, INDEX, TODAY)
|
||||
|
||||
def test_reviewer_list_file(self):
|
||||
self.assertEqual(validation.reviewers("# comment\nMacRimi\n Vaso73 # tester\nbad name\n"),
|
||||
{"macrimi", "vaso73"})
|
||||
listed = validation.reviewers(validation.REVIEWERS.read_text(encoding="utf-8"))
|
||||
self.assertEqual(listed, self.allowed)
|
||||
|
||||
|
||||
class ImageAndScenario(unittest.TestCase):
|
||||
def test_digest_as_the_monitor_shows_it_or_in_full(self):
|
||||
full = "sha256:" + "ab" * 32
|
||||
self.assertEqual(validation.image_digest("2026-09-06 · b1f339cf"), "b1f339cf")
|
||||
self.assertEqual(validation.image_digest(full.upper()), "ab" * 32)
|
||||
for value in ("", "b1f3", "latest", "sha256:xyz12345"):
|
||||
self.assertIsNone(validation.image_digest(value))
|
||||
|
||||
def test_scenario_names_problems_and_skips_empty_fields(self):
|
||||
fields = validation.parse_form(form(Result="Works with problems", GPU="_No response_"))
|
||||
self.assertEqual(validation.scenario(fields), "Works with problems: Default configuration · "
|
||||
"Volume on Proxmox storage · DHCP · Proxmox VE 9.1 · image 4.3")
|
||||
|
||||
def test_list_shows_image_and_scenario(self):
|
||||
record = {"applications": {"glances": {"community_tested": {
|
||||
"by": "Vaso73", "date": "2026-09-27", "digest": "b1f339cf08ae", "scenario": "Default | DHCP"}}}}
|
||||
self.assertIn("| `b1f339cf08ae` | Default \\| DHCP |", validation.render(record, INDEX))
|
||||
|
||||
def test_bad_digest_and_long_scenario_are_rejected(self):
|
||||
found = validation.problems({"applications": {"glances": {"community_tested": {
|
||||
"by": "a", "date": "2026-09-27", "digest": "XYZ", "scenario": "x" * 301}}}}, INDEX, today=TODAY)
|
||||
self.assertTrue(any("digest" in f for f in found) and any("scenario" in f for f in found), found)
|
||||
|
||||
|
||||
class IssueForm(unittest.TestCase):
|
||||
def test_form_labels_match_the_bot(self):
|
||||
text = (ROOT / ".github/ISSUE_TEMPLATE/oci-validation.yml").read_text(encoding="utf-8")
|
||||
labels = re.findall(r"^\s+label: (.+)$", text, re.M)
|
||||
self.assertIn("Application", labels)
|
||||
self.assertIn("Result", labels)
|
||||
self.assertIn("Image digest", labels)
|
||||
for name in validation.SCENARIO_FIELDS:
|
||||
self.assertIn(name, labels)
|
||||
self.assertIn("- Does not work", text)
|
||||
self.assertIn('labels: ["oci-validation"]', text)
|
||||
self.assertIn("template=oci-validation.yml", validation.FORM)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Label OCI validation pull requests
|
||||
|
||||
# A pull request that changes verification.json gets the "oci-validation"
|
||||
# label, which adds it to the ProxMenux Roadmap. The pull request's code is
|
||||
# never checked out: only its list of changed files is read.
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, reopened, synchronize]
|
||||
paths:
|
||||
- 'oci/catalog/verification.json'
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
label:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
steps:
|
||||
- env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: gh api "repos/$GITHUB_REPOSITORY/issues/$NUMBER/labels" -f "labels[]=oci-validation" --silent
|
||||
@@ -0,0 +1,86 @@
|
||||
name: Record OCI validation
|
||||
|
||||
# When an OCI validation report is validated, the report's author is recorded
|
||||
# in oci/catalog/verification.json on develop and oci/VALIDATION.md is
|
||||
# regenerated. A report is validated by the "validated" label, which only users
|
||||
# with write access can add, or by a /validated comment from a user listed in
|
||||
# .github/oci-validation-reviewers, checked by the script. The issue body and
|
||||
# the comment are read from the event file, never interpolated into a shell.
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [labeled]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
concurrency:
|
||||
group: oci-validation-record
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
record:
|
||||
if: >-
|
||||
contains(github.event.issue.labels.*.name, 'oci-validation') && (
|
||||
(github.event_name == 'issues' && github.event.label.name == 'validated') ||
|
||||
(github.event_name == 'issue_comment' && !github.event.issue.pull_request &&
|
||||
startsWith(github.event.comment.body, '/validated')))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: develop
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Record the report
|
||||
id: record
|
||||
run: |
|
||||
set +e
|
||||
python3 .github/scripts/oci_validation.py record --event "$GITHUB_EVENT_PATH" --message "$RUNNER_TEMP/reply.md"
|
||||
echo "code=$?" >> "$GITHUB_OUTPUT"
|
||||
- name: Commit + push
|
||||
if: steps.record.outputs.code == '0'
|
||||
env:
|
||||
APP: ${{ steps.record.outputs.app }}
|
||||
NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
git config user.name "ProxMenuxBot"
|
||||
git config user.email "bot@proxmenux.local"
|
||||
git add oci/catalog/verification.json oci/VALIDATION.md
|
||||
git commit -m "chore(oci): record the validation of $APP (#$NUMBER)"
|
||||
for attempt in 1 2 3 4 5; do
|
||||
git fetch origin develop
|
||||
if git rebase origin/develop && git push origin HEAD:develop; then
|
||||
exit 0
|
||||
fi
|
||||
git rebase --abort 2>/dev/null || true
|
||||
sleep $(( attempt * 3 ))
|
||||
done
|
||||
echo "push failed after 5 attempts"
|
||||
exit 1
|
||||
- name: Reply and close
|
||||
if: steps.record.outputs.code == '0'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/reply.md"
|
||||
gh issue close "$NUMBER" --repo "$GITHUB_REPOSITORY" --reason completed
|
||||
- name: Explain why it was not recorded
|
||||
if: steps.record.outputs.code == '2'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
NUMBER: ${{ github.event.issue.number }}
|
||||
run: |
|
||||
gh issue comment "$NUMBER" --repo "$GITHUB_REPOSITORY" --body-file "$RUNNER_TEMP/reply.md"
|
||||
if [ "$GITHUB_EVENT_NAME" = issues ]; then
|
||||
gh api -X DELETE "repos/$GITHUB_REPOSITORY/issues/$NUMBER/labels/validated" --silent
|
||||
fi
|
||||
- name: Fail on an unexpected error
|
||||
if: steps.record.outputs.code != '0' && steps.record.outputs.code != '2'
|
||||
run: exit 1
|
||||
@@ -0,0 +1,49 @@
|
||||
name: OCI validation record
|
||||
|
||||
# verification.json is the record of OCI applications tested for real, and
|
||||
# oci/VALIDATION.md is generated from it. A pull request only adds or changes
|
||||
# the community tests credited to its own author.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'oci/catalog/verification.json'
|
||||
- 'oci/catalog/index.json'
|
||||
- 'oci/VALIDATION.md'
|
||||
- '.github/scripts/oci_validation.py'
|
||||
- '.github/workflows/oci-validation.yml'
|
||||
push:
|
||||
branches: [main, develop]
|
||||
paths:
|
||||
- 'oci/catalog/verification.json'
|
||||
- 'oci/catalog/index.json'
|
||||
- 'oci/VALIDATION.md'
|
||||
- '.github/scripts/oci_validation.py'
|
||||
- '.github/workflows/oci-validation.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Check the record and oci/VALIDATION.md
|
||||
run: python3 .github/scripts/oci_validation.py check
|
||||
- name: Check who the pull request credits
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
BASE: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
git show "$BASE:oci/catalog/verification.json" > "$RUNNER_TEMP/base.json" 2>/dev/null || : > "$RUNNER_TEMP/base.json"
|
||||
python3 .github/scripts/oci_validation.py authors --base "$RUNNER_TEMP/base.json" --author "$AUTHOR"
|
||||
Reference in New Issue
Block a user