diff --git a/.github/scripts/generate_app_tracking_catalog.py b/.github/scripts/generate_app_tracking_catalog.py new file mode 100644 index 00000000..7dc4eaca --- /dev/null +++ b/.github/scripts/generate_app_tracking_catalog.py @@ -0,0 +1,1349 @@ +#!/usr/bin/env python3 +"""Build a verified ProxMenux LXC application-version tracking catalog. + +The source of truth is a pinned snapshot of community-scripts/ProxmoxVE, +downloaded through the GitHub REST API. Only ct/*.sh launchers are considered. + +An operational hint is emitted only when two independent pieces of the helper +scripts agree: + +* file: the LXC update script reads the cache written by the shared deploy + helper, and the matching install script deploys the same app/repository; or +* dpkg/apk: the package is installed by the install script and checked or + explicitly upgraded by the LXC update script. + +Everything else is retained in the audit report rather than guessed into the +runtime JSON. The script uses only Python's standard library and runs on macOS. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import re +import shutil +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + + +DEFAULT_REPOSITORY = "community-scripts/ProxmoxVE" +DEFAULT_REF = "main" +API_VERSION = "2022-11-28" +USER_AGENT = "ProxMenux-app-tracking-catalog/1.0" + +# Both the helper cache and common GitHub tags are handled. The first capture +# group is deliberately the normalized version consumed by lxc_apps.py. +DEFAULT_VERSION_REGEX = ( + r"(?i)(?:v|release[-_/]?)?" + r"(\d+(?:\.\d+){1,3}(?:[-+._][0-9A-Za-z.-]+)?)" +) +VERSION_FORMATS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r'(?i)^v?\d{6,14}$'), r"(?i)v?(\d{6,14})"), + ( + re.compile(r'(?i)^\d{6,14}-[0-9a-f]{6,40}$'), + r"(?i)(\d{6,14}(?:-[0-9a-f]{6,40})?)", + ), + (re.compile(r'(?i)^r\d{4,}$'), r"(?i)r?(\d{4,})"), + ( + re.compile(r'(?i)^SQUID_\d+(?:_\d+){1,3}$'), + r"(?i)(?:SQUID_)?(\d+(?:[._]\d+){1,3})", + ), + ( + re.compile(r'(?i)^release\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z(?:\.\d+)?$'), + r"(?i)(?:release\.)?(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z(?:\.\d+)?)", + ), +] + +APP_RE = re.compile(r'^\s*APP=["\']([^"\']+)["\']', re.MULTILINE) +CHECK_RE = re.compile( + r'\bcheck_for_gh_release\s+["\']([^"\']+)["\']\s+["\']' + r'([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)["\']' +) +FETCH_RE = re.compile( + r'\bfetch_and_deploy_gh_release\s+["\']([^"\']+)["\']\s+["\']' + r'([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)["\']' +) +HEADER_GITHUB_RE = re.compile( + r'Github:\s*https?://github\.com/([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)', + re.IGNORECASE, +) +HEADER_SOURCE_RE = re.compile(r'^\s*#\s*Source:\s*(https?://\S+)', re.IGNORECASE | re.MULTILINE) +EXPLICIT_VERSION_FILE_RE = re.compile( + r'(?:>|tee\s+)(?:["\']?)' + r'(?:~|\$HOME|\$\{HOME\})/(\.[A-Za-z0-9_.-]+)' +) +DOCKER_IMAGE_RE = re.compile( + r'(?:docker\s+(?:run|pull)\b[\s\S]{0,800}?)' + r'((?:ghcr\.io|docker\.io|quay\.io|lscr\.io)/[A-Za-z0-9_./-]+:[A-Za-z0-9_.-]+)', + re.IGNORECASE, +) +DOCKER_NAME_RE = re.compile(r'\bdocker\s+run\b[\s\S]{0,1200}?--name(?:=|\s+)([A-Za-z0-9_.-]+)', re.IGNORECASE) +EXECSTART_RE = re.compile(r'^\s*ExecStart=(/[A-Za-z0-9_./+@:-]+)', re.MULTILINE) +EXISTENCE_PATH_RE = re.compile(r'\[\[?[^\n]{0,80}?!?\s+-[fx]\s+(/[A-Za-z0-9_./+@:-]+)') +PACKAGE_TOKEN_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9.+:@_-]*$') + + +class CatalogError(RuntimeError): + pass + + +class GitHubClient: + def __init__(self, token: str | None = None) -> None: + self.token = token or os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + self.rate_remaining: str | None = None + + def request(self, url: str, *, accept: str = "application/vnd.github+json") -> bytes: + headers = { + "Accept": accept, + "X-GitHub-Api-Version": API_VERSION, + "User-Agent": USER_AGENT, + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=90) as response: + self.rate_remaining = response.headers.get("X-RateLimit-Remaining") + return response.read() + except urllib.error.HTTPError as exc: + remaining = exc.headers.get("X-RateLimit-Remaining") + if exc.code == 403 and remaining == "0": + raise CatalogError( + "GitHub API rate limit exhausted. Set GITHUB_TOKEN (or GH_TOKEN) " + "and run again." + ) from exc + raise CatalogError(f"GitHub API HTTP {exc.code} for {url}") from exc + except urllib.error.URLError as exc: + raise CatalogError(f"Cannot reach GitHub API: {exc}") from exc + + def json(self, path: str) -> Any: + url = path if path.startswith("https://") else f"https://api.github.com{path}" + return json.loads(self.request(url).decode("utf-8")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def safe_extract_tar(payload: bytes, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + root = destination.resolve() + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:gz") as archive: + members = archive.getmembers() + for member in members: + parts = Path(member.name).parts + if len(parts) < 2: + continue + relative = Path(*parts[1:]) + target = (destination / relative).resolve() + if target != root and root not in target.parents: + raise CatalogError(f"Unsafe path in GitHub tarball: {member.name}") + if member.issym() or member.islnk() or member.isdev(): + raise CatalogError(f"Unsupported link/device in GitHub tarball: {member.name}") + if not (member.isfile() or member.isdir()): + continue + member.name = str(relative) + if member.name != ".": + # Python 3.9 (the system Python on several macOS releases) + # predates tarfile's `filter=` argument. Paths and special + # members have already been validated above. + archive.extract(member, destination) + + +def obtain_snapshot( + client: GitHubClient, + repository: str, + ref: str, + cache_dir: Path, +) -> tuple[Path, str, str]: + commit = client.json(f"/repos/{repository}/commits/{ref}") + commit_sha = str(commit.get("sha") or "") + if not re.fullmatch(r"[0-9a-f]{40}", commit_sha): + raise CatalogError(f"Unexpected commit SHA for {repository}@{ref}") + + snapshot_dir = cache_dir / repository.replace("/", "--") / commit_sha + marker = snapshot_dir / ".snapshot-complete" + if marker.exists() and (snapshot_dir / "ct").is_dir(): + return snapshot_dir, commit_sha, marker.read_text(encoding="utf-8").strip() + + payload = client.request( + f"https://api.github.com/repos/{repository}/tarball/{commit_sha}", + accept="application/vnd.github+json", + ) + archive_sha = hashlib.sha256(payload).hexdigest() + temp_parent = snapshot_dir.parent + temp_parent.mkdir(parents=True, exist_ok=True) + temp_dir = Path(tempfile.mkdtemp(prefix=f"{commit_sha}.tmp-", dir=temp_parent)) + try: + safe_extract_tar(payload, temp_dir) + (temp_dir / ".snapshot-complete").write_text(archive_sha + "\n", encoding="utf-8") + if snapshot_dir.exists(): + shutil.rmtree(snapshot_dir) + temp_dir.rename(snapshot_dir) + except Exception: + shutil.rmtree(temp_dir, ignore_errors=True) + raise + return snapshot_dir, commit_sha, archive_sha + + +def normalize(value: str) -> str: + return re.sub(r"[^a-z0-9]", "", value.lower()) + + +def cache_key(app: str) -> str: + # Mirrors tools.func: lower-case then `tr -d ' '`. + return app.lower().replace(" ", "") + + +def version_regex_for_tag(tag: str) -> str: + if not tag or re.search(DEFAULT_VERSION_REGEX, tag): + return DEFAULT_VERSION_REGEX + for matcher, pattern in VERSION_FORMATS: + if matcher.fullmatch(tag): + return pattern + return DEFAULT_VERSION_REGEX + + +def read_text(path: Path | None) -> str: + if path is None or not path.is_file(): + return "" + return path.read_text(encoding="utf-8", errors="replace") + + +def marker_stores_url(text: str, marker: str) -> bool: + marker_pattern = re.escape(marker) + for line in text.splitlines(): + if not re.search(rf'(?:~|\$HOME|\$\{{HOME\}})/{marker_pattern}\b', line): + continue + variable = re.search(r'echo\s+["\']?\$\{?([A-Za-z_][A-Za-z0-9_]*)', line) + if not variable: + continue + assignment = re.search( + rf'^\s*{re.escape(variable.group(1))}=([^\n]*(?:\n(?![A-Za-z_][A-Za-z0-9_]*=)[^\n]*){{0,3}})', + text, + re.MULTILINE, + ) + if assignment and re.search(r"grep\s+-o[^\n]*https?://|DownloadLocation", assignment.group(0), re.IGNORECASE): + return True + return False + + +def unique_pairs(items: Iterable[tuple[str, str]]) -> list[tuple[str, str]]: + result: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for item in items: + key = (item[0], item[1].lower()) + if key not in seen: + seen.add(key) + result.append(item) + return result + + +def extract_command_blocks(text: str) -> list[str]: + lines = text.splitlines() + blocks: list[str] = [] + i = 0 + while i < len(lines): + line = lines[i].strip() + block = line + while block.rstrip().endswith("\\") and i + 1 < len(lines): + block = block.rstrip()[:-1] + " " + lines[i + 1].strip() + i += 1 + blocks.append(block) + i += 1 + return blocks + + +def packages_from_command(text: str, manager: str) -> set[str]: + packages: set[str] = set() + command_re = ( + re.compile(r'\b(?:apt|apt-get)\b[^\n]*?\b(?:install|upgrade)\b\s+(.+)$') + if manager == "dpkg" + else re.compile(r'\bapk\b[^\n]*?\b(?:add|upgrade)\b\s+(.+)$') + ) + for block in extract_command_blocks(text): + match = command_re.search(block) + if not match: + continue + for token in re.split(r"\s+", match.group(1)): + token = token.strip("'\"") + if ( + not token + or token.startswith("-") + or token.startswith("$") + or "/" in token + or "=" in token + or not PACKAGE_TOKEN_RE.fullmatch(token) + ): + continue + packages.add(token) + return packages + + +def checked_packages(text: str, manager: str) -> set[str]: + patterns = ( + [r'\bdpkg\s+-s\s+([A-Za-z0-9.+:@_-]+)', r'\bdpkg-query\b[^\n]*?\s([A-Za-z0-9.+:@_-]+)\s*(?:[>&]|$)'] + if manager == "dpkg" + else [r'\bapk\s+info\b[^\n]*?\s([A-Za-z0-9.+:@_-]+)\s*(?:[>&]|$)'] + ) + result: set[str] = set() + for pattern in patterns: + result.update(re.findall(pattern, text)) + return result + + +def load_helper_catalog(path: Path | None) -> dict[str, dict[str, Any]]: + if path is None or not path.is_file(): + return {} + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise CatalogError("helpers_cache.json must contain a list") + result: dict[str, dict[str, Any]] = {} + for item in raw: + if not isinstance(item, dict) or item.get("type") != "lxc": + continue + slug = str(item.get("slug") or "") + if slug and slug not in result: + result[slug] = item + return result + + +@dataclass +class Candidate: + app: str + repo: str + score: int + reasons: list[str] = field(default_factory=list) + install_fetch: bool = False + update_fetch: bool = False + + +def select_release_candidate( + slug: str, + app_name: str, + launcher: str, + installer: str, +) -> tuple[Candidate | None, list[dict[str, Any]], str | None]: + checks = unique_pairs(CHECK_RE.findall(launcher)) + update_fetches = {(a, r.lower()) for a, r in FETCH_RE.findall(launcher)} + install_fetches = {(a, r.lower()) for a, r in FETCH_RE.findall(installer)} + candidates: list[Candidate] = [] + + for app, repo in checks: + key = (app, repo.lower()) + score = 0 + identity_score = 0 + reasons: list[str] = [] + n_app, n_slug, n_name = normalize(app), normalize(slug), normalize(app_name) + n_repo = normalize(repo.split("/", 1)[1]) + if n_app == n_slug: + score += 100 + identity_score += 100 + reasons.append("check app name matches LXC slug") + elif n_app and (n_app in n_slug or n_slug in n_app): + score += 55 + identity_score += 55 + reasons.append("check app name closely matches LXC slug") + if n_app == n_name: + score += 45 + identity_score += 45 + reasons.append("check app name matches APP label") + if n_repo == n_slug or (n_repo and (n_repo in n_slug or n_slug in n_repo)): + score += 30 + identity_score += 30 + reasons.append("repository name matches LXC slug") + install_fetch = key in install_fetches + update_fetch = key in update_fetches + if install_fetch: + score += 80 + reasons.append("matching deploy call exists in install script") + if update_fetch: + score += 30 + reasons.append("matching deploy call exists in update script") + if identity_score == 0: + # Auxiliary components (Ollama inside Open WebUI, web vault + # assets inside Vaultwarden, etc.) must never become the primary + # application merely because their deploy helper is present. + score -= 1000 + reasons.append("does not identify the primary LXC application") + candidates.append(Candidate(app, repo, score, reasons, install_fetch, update_fetch)) + + candidates.sort(key=lambda candidate: candidate.score, reverse=True) + audit_candidates = [ + { + "app": c.app, + "repo": c.repo, + "cache_file": f"/root/.{cache_key(c.app)}", + "score": c.score, + "install_fetch": c.install_fetch, + "update_fetch": c.update_fetch, + "reasons": c.reasons, + } + for c in candidates + ] + if not candidates: + return None, audit_candidates, "no literal check_for_gh_release call" + winner = candidates[0] + if winner.score < 70: + return None, audit_candidates, "no release check identifies the primary LXC application" + if len(candidates) > 1 and winner.score == candidates[1].score: + return None, audit_candidates, "ambiguous primary GitHub application" + if not winner.install_fetch: + return None, audit_candidates, "version cache is not proven to exist immediately after installation" + return winner, audit_candidates, None + + +def select_install_only_release( + slug: str, + app_name: str, + installer: str, + header_repos: list[str], +) -> Candidate | None: + candidates: list[Candidate] = [] + for app, repo in unique_pairs(FETCH_RE.findall(installer)): + n_app, n_slug, n_name = normalize(app), normalize(slug), normalize(app_name) + n_repo = normalize(repo.split("/", 1)[1]) + score = 0 + reasons: list[str] = [] + if n_app == n_slug: + score += 100 + reasons.append("deploy app matches LXC slug") + elif n_app and (n_app in n_slug or n_slug in n_app): + score += 50 + reasons.append("deploy app closely matches LXC slug") + if n_app == n_name: + score += 45 + reasons.append("deploy app matches APP label") + if n_repo == n_slug or (n_repo and (n_repo in n_slug or n_slug in n_repo)): + score += 30 + reasons.append("repository name matches LXC slug") + if repo.lower() in {item.lower() for item in header_repos}: + score += 20 + reasons.append("repository matches script header") + candidates.append(Candidate(app, repo, score, reasons, install_fetch=True)) + candidates.sort(key=lambda candidate: candidate.score, reverse=True) + if not candidates or candidates[0].score < 70: + return None + if len(candidates) > 1 and candidates[0].score == candidates[1].score: + return None + return candidates[0] + + +def choose_package( + slug: str, + app_name: str, + launcher: str, + installer: str, + manager: str, +) -> tuple[str | None, dict[str, Any]]: + installed = packages_from_command(installer, manager) + updated = packages_from_command(launcher, manager) + checked = checked_packages(launcher, manager) + raw_proven = installed & (updated | checked) + # Some official repositories are updated with a plain `apt upgrade` or + # `apk upgrade`, so the package is not repeated in the update command. + # Accept only an exact app/slug match in that case; dependencies remain + # excluded. + has_generic_upgrade = bool( + re.search(r'\b(?:apt|apt-get)\b[^\n]*\bupgrade\b', launcher) + if manager == "dpkg" + else re.search(r'\bapk\b[^\n]*\bupgrade\b', launcher) + ) + if has_generic_upgrade: + expected = {normalize(slug), normalize(app_name)} + raw_proven.update(package for package in installed if normalize(package) in expected) + expected = {normalize(slug), normalize(app_name)} + + def is_app_package(package: str) -> bool: + normalized = normalize(package) + return any( + candidate and ( + normalized == candidate + or normalized in candidate + or candidate in normalized + ) + for candidate in expected + ) + + proven = {package for package in raw_proven if is_app_package(package)} + evidence = { + "installed_packages": sorted(installed), + "updated_packages": sorted(updated), + "checked_packages": sorted(checked), + "proven_packages": sorted(proven), + "rejected_unrelated_packages": sorted(raw_proven - proven), + "generic_upgrade": has_generic_upgrade, + } + if not proven: + return None, evidence + + def score(package: str) -> tuple[int, int, str]: + n_pkg = normalize(package) + n_slug = normalize(slug) + n_name = normalize(app_name) + value = 0 + if n_pkg == n_slug: + value += 100 + elif n_pkg in n_slug or n_slug in n_pkg: + value += 45 + if n_pkg == n_name: + value += 50 + if package in checked: + value += 30 + return value, -len(package), package + + ranked = sorted(proven, key=score, reverse=True) + if len(ranked) > 1 and score(ranked[0])[:2] == score(ranked[1])[:2]: + return None, evidence + return ranked[0], evidence + + +def repo_from_helper(item: dict[str, Any]) -> str: + repo = str(item.get("github_repo") or "").strip() + if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repo): + return repo + raw = str(item.get("github") or "").strip() + match = re.search(r'(?:github\.com/)?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)', raw) + return match.group(1) if match else "" + + +def build_catalog( + source: Path, + helpers: dict[str, dict[str, Any]], +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: + catalog: dict[str, Any] = {} + v2_apps: dict[str, Any] = {} + records: list[dict[str, Any]] = [] + launchers = sorted((source / "ct").glob("*.sh")) + tools_func = source / "misc" / "tools.func" + tools_text = read_text(tools_func) + shared_helper_verified = bool( + re.search(r'local version_file="\$HOME/\.\$\{app_lc\}"', tools_text) + and re.search(r'echo "\$version" >"\$version_file"', tools_text) + and re.search(r'local current_file="\$HOME/\.\$\{app_lc\}"', tools_text) + ) + if not shared_helper_verified: + raise CatalogError("Could not verify the shared GitHub release cache contract in misc/tools.func") + + for launcher_path in launchers: + slug = launcher_path.stem + launcher = read_text(launcher_path) + installer_path = source / "install" / f"{slug}-install.sh" + installer = read_text(installer_path) + app_match = APP_RE.search(launcher) + app_name = app_match.group(1) if app_match else slug + helper_item = helpers.get(slug, {}) + helper_repo = repo_from_helper(helper_item) + helper_version = str(helper_item.get("github_version") or "").strip() + version_regex = version_regex_for_tag(helper_version) + header_repos = HEADER_GITHUB_RE.findall(launcher + "\n" + installer) + official_sources = sorted(set(HEADER_SOURCE_RE.findall(launcher + "\n" + installer))) + docker_images = sorted(set(DOCKER_IMAGE_RE.findall(installer))) + docker_names = sorted(set(DOCKER_NAME_RE.findall(installer))) + launcher_version_files = set(EXPLICIT_VERSION_FILE_RE.findall(launcher)) + installer_version_files = set(EXPLICIT_VERSION_FILE_RE.findall(installer)) + explicit_files = sorted(launcher_version_files | installer_version_files) + relevant_markers = sorted( + marker + for marker in launcher_version_files & installer_version_files + if normalize(marker.lstrip(".")) in {normalize(slug), normalize(app_name)} + and not marker_stores_url(launcher, marker) + and not marker_stores_url(installer, marker) + ) + + package_evidence: dict[str, Any] = {} + package_detectors: list[dict[str, Any]] = [] + for manager in ("dpkg", "apk"): + package, evidence = choose_package(slug, app_name, launcher, installer, manager) + package_evidence[manager] = evidence + if package and helper_repo: + upstream_verified = bool( + helper_version and re.search(version_regex, helper_version) + ) + package_detectors.append( + { + "installed_via": manager, + "package": package, + "repo": helper_repo, + "github_source": "releases", + "tag_regex": version_regex, + "install_scope": ["community-script", "manual-if-same-package"], + "verification": ( + "verified-static" + if upstream_verified + else "candidate-needs-upstream-verification" + ), + "evidence": { + "install": f"install/{installer_path.name}: package installation", + "update": f"ct/{launcher_path.name}: package check or named upgrade", + }, + } + ) + + binary_paths = sorted(set(EXECSTART_RE.findall(installer)) & set(EXISTENCE_PATH_RE.findall(launcher))) + binary_detectors = [ + { + "installed_via": "binary", + "binary_path": path, + "binary_args": ["--version"], + "repo": helper_repo or None, + "github_source": "releases", + "tag_regex": version_regex, + "install_scope": ["community-script", "manual-if-same-path"], + "verification": "candidate-needs-version-probe", + "evidence": { + "install": f"install/{installer_path.name}: systemd ExecStart", + "update": f"ct/{launcher_path.name}: installation existence check", + }, + } + for path in binary_paths + ] + + docker_detectors: list[dict[str, Any]] = [] + if docker_names or docker_images: + # Keep all discovered data when a script has multiple containers; + # pairing by shell position is intentionally left to an override. + docker_detectors.append( + { + "installed_via": "docker", + "container_names": docker_names, + "images": docker_images, + "version_sources": [ + {"type": "oci_label", "name": "org.opencontainers.image.version"}, + {"type": "image_ref_tag"}, + ], + "repo": helper_repo or None, + "github_source": "releases", + "tag_regex": version_regex, + "install_scope": ["community-script", "manual-docker"], + "verification": "requires-detector-change", + "evidence": {"install": f"install/{installer_path.name}: docker run/pull"}, + } + ) + + record: dict[str, Any] = { + "slug": slug, + "name": app_name, + "launcher": f"ct/{launcher_path.name}", + "installer": f"install/{installer_path.name}" if installer_path.is_file() else None, + "status": "excluded", + "method": None, + "reason": None, + "helper_repo": helper_repo or None, + "helper_upstream_version": helper_version or None, + "header_repositories": header_repos, + "official_sources": official_sources, + "docker_images": docker_images, + "docker_names": docker_names, + "explicit_version_files": [f"/root/{item}" for item in explicit_files], + } + + release, release_candidates, release_error = select_release_candidate( + slug, app_name, launcher, installer + ) + install_only_release = select_install_only_release( + slug, app_name, installer, header_repos + ) if release is None else None + record["release_candidates"] = release_candidates + v2_detectors: list[dict[str, Any]] = [] + if release is not None: + hint = { + "installed_via": "file", + "file_path": f"/root/.{cache_key(release.app)}", + "file_regex": DEFAULT_VERSION_REGEX, + "repo": release.repo, + "github_source": "releases", + "tag_regex": version_regex, + } + hint["file_regex"] = version_regex + catalog[slug] = hint + v2_detectors.append( + { + **hint, + "install_scope": ["community-script"], + "verification": "verified-static", + "evidence": { + "install": f"install/{installer_path.name}: matching deploy helper", + "update": f"ct/{launcher_path.name}: matching release check", + "contract": "misc/tools.func: shared version-cache contract", + }, + } + ) + v2_detectors.extend(package_detectors) + v2_detectors.extend(binary_detectors) + v2_detectors.extend(docker_detectors) + record.update( + { + "status": "verified", + "method": "file", + "reason": "install deploy and update check share the same helper cache", + "selected": hint, + "evidence": { + "install": f"install/{installer_path.name}: fetch_and_deploy_gh_release({release.app}, {release.repo})", + "update": f"ct/{launcher_path.name}: check_for_gh_release({release.app}, {release.repo})", + "contract": "misc/tools.func writes and reads /root/.", + }, + } + ) + records.append(record) + v2_apps[slug] = { + "name": app_name, + "repo": release.repo, + "official_sources": official_sources, + "detectors": v2_detectors, + } + continue + + if len(relevant_markers) == 1 and helper_repo and helper_version: + marker = relevant_markers[0] + hint = { + "installed_via": "file", + "file_path": f"/root/{marker}", + "file_regex": version_regex, + "repo": helper_repo, + "github_source": "releases", + "tag_regex": version_regex, + } + catalog[slug] = hint + v2_detectors.append( + { + **hint, + "install_scope": ["community-script"], + "verification": "verified-static", + "evidence": { + "install": f"install/{installer_path.name}: writes {hint['file_path']}", + "update": f"ct/{launcher_path.name}: writes {hint['file_path']}", + }, + } + ) + v2_detectors.extend(package_detectors) + v2_detectors.extend(binary_detectors) + v2_detectors.extend(docker_detectors) + record.update( + { + "status": "verified", + "method": "file", + "reason": "install and update scripts write the same app-specific version marker", + "selected": hint, + "evidence": v2_detectors[0]["evidence"], + } + ) + records.append(record) + v2_apps[slug] = { + "name": app_name, + "repo": helper_repo, + "official_sources": official_sources, + "detectors": v2_detectors, + } + continue + + if install_only_release is not None: + v2_detectors.append( + { + "installed_via": "file", + "file_path": f"/root/.{cache_key(install_only_release.app)}", + "file_regex": version_regex, + "repo": install_only_release.repo, + "github_source": "releases", + "tag_regex": version_regex, + "install_scope": ["community-script"], + "verification": "candidate-install-cache-may-stale", + "evidence": { + "install": f"install/{installer_path.name}: deploy helper writes the version cache", + "limitation": "no matching update check proves that later updates refresh this cache", + }, + } + ) + + package_selected = False + for detector in package_detectors: + manager = detector["installed_via"] + if not package_selected and detector["verification"] == "verified-static": + hint = {key: value for key, value in detector.items() if key in { + "installed_via", "package", "repo", "github_source", "tag_regex" + }} + catalog[slug] = hint + record.update( + { + "status": "verified", + "method": manager, + "reason": "package is present in both install and update/check paths", + "selected": hint, + "evidence": package_evidence[manager], + } + ) + package_selected = True + record["package_evidence"] = package_evidence + if package_selected: + v2_detectors.extend(package_detectors) + v2_detectors.extend(binary_detectors) + v2_detectors.extend(docker_detectors) + records.append(record) + v2_apps[slug] = { + "name": app_name, + "repo": helper_repo or None, + "official_sources": official_sources, + "detectors": v2_detectors, + } + continue + + if docker_images or re.search(r'\bsetup_docker\b|\bdocker\s+(?:run|compose|pull)\b', installer): + record["reason"] = "Docker installation requires detector support not present in lxc_apps.py" + record["required_detector"] = "docker" + elif release_error: + record["reason"] = release_error + elif not installer_path.is_file(): + record["reason"] = "no matching install script" + elif helper_repo and any( + item["proven_packages"] for item in package_evidence.values() + ): + record["reason"] = "package detector needs upstream repository/release verification" + elif not helper_repo: + record["reason"] = "no verified upstream GitHub repository for package tracking" + else: + record["reason"] = "no supported detection method could be proven from both scripts" + records.append(record) + v2_detectors.extend(package_detectors) + v2_detectors.extend(binary_detectors) + v2_detectors.extend(docker_detectors) + v2_apps[slug] = { + "name": app_name, + "repo": helper_repo or (header_repos[0] if header_repos else None), + "official_sources": official_sources, + "detectors": v2_detectors, + } + + status_counts: dict[str, int] = {} + method_counts: dict[str, int] = {} + reason_counts: dict[str, int] = {} + tag_validation = {"matched": 0, "missing": 0, "mismatched": []} + for record in records: + status_counts[record["status"]] = status_counts.get(record["status"], 0) + 1 + method = record.get("method") or "none" + method_counts[method] = method_counts.get(method, 0) + 1 + reason = record.get("reason") or "none" + reason_counts[reason] = reason_counts.get(reason, 0) + 1 + if record.get("status") == "verified": + upstream = record.get("helper_upstream_version") + pattern = (record.get("selected") or {}).get("tag_regex") + if not upstream: + tag_validation["missing"] += 1 + elif pattern and re.search(pattern, upstream): + tag_validation["matched"] += 1 + else: + tag_validation["mismatched"].append( + {"slug": record["slug"], "version": upstream, "tag_regex": pattern} + ) + + audit = { + "summary": { + "lxc_launchers": len(launchers), + "operational_hints": len(catalog), + "coverage_percent": round((len(catalog) / len(launchers) * 100), 2) if launchers else 0, + "status_counts": status_counts, + "method_counts": method_counts, + "reason_counts": reason_counts, + "shared_release_cache_contract_verified": shared_helper_verified, + "helper_upstream_tag_validation": tag_validation, + }, + "records": records, + } + v2 = { + "schema_version": 2, + "detector_policy": { + "strategy": "try detectors in order and retain the first successful detector", + "operational_verification": ["verified-static", "verified-runtime"], + "non_operational_verification": [ + "candidate-needs-version-probe", + "requires-detector-change", + "candidate-needs-runtime-validation", + "candidate-install-cache-may-stale", + "candidate-needs-upstream-verification", + "candidate-helper-marker", + ], + }, + "apps": dict(sorted(v2_apps.items())), + } + return dict(sorted(catalog.items())), audit, v2 + + +def compare_existing(generated: dict[str, Any], existing_path: Path | None) -> dict[str, Any]: + if existing_path is None or not existing_path.is_file(): + return {"existing_file": None, "added": sorted(generated), "removed": [], "changed": []} + existing = json.loads(existing_path.read_text(encoding="utf-8")) + if not isinstance(existing, dict): + raise CatalogError("Existing catalog must be a JSON object") + return { + "existing_file": str(existing_path), + "added": sorted(set(generated) - set(existing)), + "removed": sorted(set(existing) - set(generated)), + "changed": sorted( + slug for slug in set(existing) & set(generated) if existing[slug] != generated[slug] + ), + "unchanged": sorted( + slug for slug in set(existing) & set(generated) if existing[slug] == generated[slug] + ), + } + + +def demote_generic_helper_markers( + catalog: dict[str, Any], + v2: dict[str, Any], + audit: dict[str, Any], +) -> list[str]: + """Remove generic /root/.app caches from the operational catalog. + + Even when install and update scripts both write the marker, it records + helper/update state rather than interrogating the installed application. + Runtime checks also found these files absent on legacy and manually + updated LXC. They remain useful candidates/fallbacks in v2, not verified + primary detectors. + """ + demoted: list[str] = [] + apps = v2.get("apps", {}) + for record in audit.get("records", []): + slug = record.get("slug") + hint = catalog.get(slug) + if not isinstance(hint, dict): + continue + path = hint.get("file_path") + if hint.get("installed_via") != "file" or not isinstance(path, str): + continue + if not re.fullmatch(r"/root/\.[A-Za-z0-9_.-]+", path): + continue + catalog.pop(slug, None) + record["status"] = "candidate" + record["reason"] = "generic helper marker is not guaranteed on legacy/manual installations" + for detector in (apps.get(slug) or {}).get("detectors", []): + if detector.get("installed_via") == "file" and detector.get("file_path") == path: + detector["verification"] = "candidate-helper-marker" + detector["limitation"] = ( + "Observed absent on legacy/manual LXC; use only as fallback or after runtime probe" + ) + demoted.append(slug) + return sorted(demoted) + + +def verify_upstream_releases( + client: GitHubClient, + catalog: dict[str, Any], + cache_dir: Path, +) -> dict[str, Any]: + """Verify repositories and current tags directly with the GitHub API.""" + repos = sorted({hint["repo"] for hint in catalog.values() if hint.get("repo")}) + if len(repos) > 40 and not client.token: + raise CatalogError( + f"--verify-upstream needs GITHUB_TOKEN or GH_TOKEN for {len(repos)} repositories " + "(the anonymous GitHub API limit is only 60 requests/hour)." + ) + cache_file = cache_dir / "upstream-releases.json" + cache_file.parent.mkdir(parents=True, exist_ok=True) + try: + cache = json.loads(cache_file.read_text(encoding="utf-8")) + if not isinstance(cache, dict): + cache = {} + except (OSError, json.JSONDecodeError): + cache = {} + + now = int(time.time()) + results: dict[str, Any] = {} + for index, repo in enumerate(repos, start=1): + cached = cache.get(repo, {}) + if isinstance(cached, dict) and now - int(cached.get("fetched_at", 0)) < 24 * 3600: + results[repo] = cached + continue + tag = "" + source = "releases" + error = "" + try: + payload = client.json(f"/repos/{repo}/releases/latest") + if isinstance(payload, dict): + tag = str(payload.get("tag_name") or payload.get("name") or "").strip() + except CatalogError as exc: + error = str(exc) + try: + tags = client.json(f"/repos/{repo}/tags?per_page=30") + if isinstance(tags, list) and tags and isinstance(tags[0], dict): + tag = str(tags[0].get("name") or "").strip() + source = "tags" + error = "" + except CatalogError as tag_exc: + error = f"release: {exc}; tags: {tag_exc}" + results[repo] = { + "tag": tag or None, + "source": source, + "error": error or None, + "fetched_at": now, + } + if index % 25 == 0: + print(f"Verified upstream repositories: {index}/{len(repos)}", file=sys.stderr) + cache_file.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + + matched: list[dict[str, str]] = [] + mismatched: list[dict[str, str]] = [] + unavailable: list[dict[str, str]] = [] + for slug, hint in catalog.items(): + result = results.get(hint.get("repo"), {}) + tag = result.get("tag") + if not tag: + unavailable.append({"slug": slug, "repo": hint.get("repo", ""), "error": result.get("error") or "no tag"}) + elif re.search(hint["tag_regex"], tag): + matched.append({"slug": slug, "repo": hint["repo"], "tag": tag}) + else: + mismatched.append( + {"slug": slug, "repo": hint["repo"], "tag": tag, "tag_regex": hint["tag_regex"]} + ) + return { + "repositories_queried": len(repos), + "matched": len(matched), + "mismatched": mismatched, + "unavailable": unavailable, + "results": results, + } + + +def merge_existing_as_runtime_candidates(v2: dict[str, Any], existing_path: Path | None) -> dict[str, Any]: + """Retain hand-curated/manual-install hints without declaring them proven. + + Existing entries are valuable for official/manual layouts, but static + analysis found that several no longer match current Community Scripts. + They therefore enter v2 as runtime-validation candidates and never enter + the compatible v1 output automatically. + """ + result = {"merged": [], "unmatched": [], "skipped_duplicates": []} + if existing_path is None or not existing_path.is_file(): + return result + raw = json.loads(existing_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + return result + apps = v2.get("apps", {}) + allowed = { + "installed_via", "package", "file_path", "file_regex", "binary_path", + "repo", "github_source", "tag_regex", + } + for slug, hint in raw.items(): + if slug not in apps or not isinstance(hint, dict): + result["unmatched"].append(slug) + continue + detector = {key: value for key, value in hint.items() if key in allowed} + if not detector.get("installed_via"): + continue + signature = json.dumps(detector, sort_keys=True) + existing_signatures = { + json.dumps({key: value for key, value in item.items() if key in allowed}, sort_keys=True) + for item in apps[slug]["detectors"] + } + if signature in existing_signatures: + result["skipped_duplicates"].append(slug) + continue + detector.update( + { + "install_scope": ["manual", "legacy-catalog"], + "verification": "candidate-needs-runtime-validation", + "evidence": {"catalog": str(existing_path)}, + } + ) + apps[slug]["detectors"].append(detector) + result["merged"].append(slug) + for key in result: + result[key].sort() + return result + + +def enrich_catalog_metadata( + catalog: dict[str, Any], + v2: dict[str, Any], + helpers: dict[str, dict[str, Any]], + existing_path: Path | None, +) -> dict[str, int]: + """Add presentation metadata without weakening detector verification. + + Community Scripts provides one primary port and a curated logo. Existing + manual `default_ports` take precedence because they may describe multi-port + applications. Detector fields and their evidence remain untouched. + """ + existing: dict[str, Any] = {} + if existing_path and existing_path.is_file(): + try: + payload = json.loads(existing_path.read_text(encoding="utf-8")) + if isinstance(payload, dict): + existing = payload + except (OSError, json.JSONDecodeError): + pass + + counts = {"apps_with_ports": 0, "apps_with_logos": 0, "selfhst_logos": 0} + apps = v2.get("apps", {}) + for slug, app in apps.items(): + helper = helpers.get(slug, {}) + prior = existing.get(slug, {}) if isinstance(existing.get(slug), dict) else {} + + ports: list[int] = [] + prior_ports = prior.get("default_ports") + if isinstance(prior_ports, list): + for value in prior_ports: + try: + port = int(value) + except (TypeError, ValueError): + continue + if 1 <= port <= 65535 and port not in ports: + ports.append(port) + if not ports: + raw_port = helper.get("port") + if isinstance(raw_port, int) and 1 <= raw_port <= 65535: + ports.append(raw_port) + + logo = str(prior.get("logo") or helper.get("logo") or "").strip() + if logo and not re.match(r"^https://[A-Za-z0-9.-]+/", logo): + logo = "" + website = str(helper.get("website") or "").strip() + + if ports: + app["default_ports"] = ports + counts["apps_with_ports"] += 1 + if logo: + app["logo"] = logo + app["logo_source"] = ( + "selfh.st/icons via jsDelivr" + if "cdn.jsdelivr.net/gh/selfhst/icons@" in logo + else "community-scripts catalog" + ) + counts["apps_with_logos"] += 1 + if app["logo_source"].startswith("selfh.st"): + counts["selfhst_logos"] += 1 + if website.startswith("https://"): + app["website"] = website + + # v1 only contains operationally verified apps. Extra metadata is + # ignored safely by validate_config but is available to suggestions/UI. + if slug in catalog: + if ports: + catalog[slug]["default_ports"] = ports + if logo: + catalog[slug]["logo"] = logo + if website.startswith("https://"): + catalog[slug]["website"] = website + return counts + + +def apply_runtime_overrides( + catalog: dict[str, Any], + v2: dict[str, Any], + overrides_path: Path | None, +) -> dict[str, Any]: + """Apply detectors proven against real containers. + + The generated/static catalog is intentionally conservative. This optional + overlay promotes only detectors carrying runtime evidence. Unsupported + future methods (for example ``python_dist`` or ``docker_label``) are kept + in v2 but are not written to the current-compatible v1 catalog. + """ + result: dict[str, Any] = { + "file": str(overrides_path) if overrides_path else None, + "promoted_to_v1": [], + "v2_only": [], + "invalid": [], + } + if overrides_path is None or not overrides_path.is_file(): + return result + raw = json.loads(overrides_path.read_text(encoding="utf-8")) + apps_raw = raw.get("apps") if isinstance(raw, dict) else None + if not isinstance(apps_raw, dict): + raise CatalogError("runtime overrides must contain an 'apps' object") + + supported_v1 = {"dpkg", "apk", "file", "binary"} + v2_apps = v2.get("apps", {}) + detector_keys = { + "installed_via", "package", "file_path", "file_regex", + "binary_path", "binary_args", "python_path", "distribution", + "container_name", "label", "repo", "github_source", "tag_regex", + "installed_regex", + } + passthrough_keys = { + "file_fallbacks", "alt_detectors", "default_ports", "logo", "website", + } + for slug, spec in apps_raw.items(): + if not isinstance(spec, dict) or not isinstance(spec.get("detector"), dict): + result["invalid"].append(slug) + continue + detector = {k: v for k, v in spec["detector"].items() if k in detector_keys} + method = detector.get("installed_via") + if not isinstance(method, str) or not method: + result["invalid"].append(slug) + continue + evidence = spec.get("evidence") if isinstance(spec.get("evidence"), list) else [] + v2_detector = { + **detector, + "install_scope": spec.get("install_scope") or ["runtime-observed"], + "verification": "verified-runtime", + "evidence": evidence, + } + app = v2_apps.get(slug) + if not isinstance(app, dict): + result["invalid"].append(slug) + continue + app.setdefault("detectors", []).insert(0, v2_detector) + app["runtime_evidence"] = evidence + + if bool(spec.get("remove_from_v1")): + catalog.pop(slug, None) + + operational = bool(spec.get("operational", True)) + if operational and method in supported_v1: + # Preserve presentation metadata already enriched from helpers. + presentation_source = dict(app) + presentation_source.update(catalog.get(slug, {})) + presentation = { + key: value + for key, value in presentation_source.items() + if key in {"default_ports", "logo", "website"} + } + hint = {k: v for k, v in detector.items() if k not in { + "binary_args", "python_path", "distribution", "container_name", + "label", "installed_regex", + }} + for key in passthrough_keys: + if key in spec: + hint[key] = spec[key] + hint.update(presentation) + catalog[slug] = hint + result["promoted_to_v1"].append(slug) + else: + result["v2_only"].append(slug) + + for key in ("promoted_to_v1", "v2_only", "invalid"): + result[key].sort() + return result + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", default=DEFAULT_REPOSITORY) + parser.add_argument("--ref", default=DEFAULT_REF) + parser.add_argument( + "--source-dir", + type=Path, + help="Analyze an existing checkout/snapshot instead of downloading through GitHub API", + ) + parser.add_argument( + "--cache-dir", + type=Path, + default=Path.home() / ".cache" / "proxmenux-app-tracking", + ) + parser.add_argument("--helpers-cache", type=Path) + parser.add_argument("--existing", type=Path) + parser.add_argument( + "--runtime-overrides", + type=Path, + help="JSON overlay with detectors verified against real LXC installations", + ) + parser.add_argument( + "--include-helper-markers", + action="store_true", + help="Keep generic /root/.app helper caches in v1 (not recommended for legacy/manual LXC)", + ) + parser.add_argument("--output", type=Path, default=Path("app_tracking_hints.generated.json")) + parser.add_argument("--audit-output", type=Path, default=Path("app_tracking_hints.audit.json")) + parser.add_argument("--v2-output", type=Path, default=Path("app_tracking_catalog.v2.json")) + parser.add_argument( + "--verify-upstream", + action="store_true", + help="Verify every repository's current release/tag directly through GitHub API (token recommended)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + started = time.time() + client = GitHubClient() + commit_sha = "local-source" + archive_sha = "not-applicable" + try: + if args.source_dir: + source = args.source_dir.resolve() + if re.fullmatch(r"[0-9a-fA-F]{40}", source.name): + commit_sha = source.name.lower() + else: + source, commit_sha, archive_sha = obtain_snapshot( + client, args.repository, args.ref, args.cache_dir.expanduser().resolve() + ) + if not (source / "ct").is_dir() or not (source / "misc" / "tools.func").is_file(): + raise CatalogError(f"Not a valid ProxmoxVE source tree: {source}") + helpers = load_helper_catalog(args.helpers_cache) + catalog, audit, v2 = build_catalog(source, helpers) + audit["demoted_helper_markers"] = ( + [] if args.include_helper_markers else demote_generic_helper_markers(catalog, v2, audit) + ) + audit["metadata"] = enrich_catalog_metadata(catalog, v2, helpers, args.existing) + audit["v2_existing_candidates"] = merge_existing_as_runtime_candidates(v2, args.existing) + audit["runtime_overrides"] = apply_runtime_overrides( + catalog, v2, args.runtime_overrides + ) + if args.verify_upstream: + audit["github_upstream_verification"] = verify_upstream_releases( + client, catalog, args.cache_dir.expanduser().resolve() + ) + audit["provenance"] = { + "repository": args.repository, + "ref": args.ref, + "commit_sha": commit_sha, + "archive_sha256": archive_sha, + "source_dir": str(source), + "generated_at_unix": int(time.time()), + "generator_sha256": sha256_file(Path(__file__).resolve()), + "github_api_rate_remaining": client.rate_remaining, + "helpers_cache": str(args.helpers_cache) if args.helpers_cache else None, + } + audit["existing_comparison"] = compare_existing(catalog, args.existing) + audit["summary"]["operational_hints"] = len(catalog) + audit["summary"]["coverage_percent"] = round( + len(catalog) / max(1, audit["summary"]["lxc_launchers"]) * 100, 2 + ) + method_counts: dict[str, int] = {} + for hint in catalog.values(): + method = str(hint.get("installed_via") or "none") + method_counts[method] = method_counts.get(method, 0) + 1 + method_counts["none"] = max( + 0, audit["summary"]["lxc_launchers"] - len(catalog) + ) + audit["summary"]["method_counts"] = method_counts + audit["summary"]["elapsed_seconds"] = round(time.time() - started, 3) + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.audit_output.parent.mkdir(parents=True, exist_ok=True) + args.v2_output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + args.audit_output.write_text(json.dumps(audit, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + v2["provenance"] = audit["provenance"] + args.v2_output.write_text(json.dumps(v2, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + except (CatalogError, OSError, json.JSONDecodeError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + summary = audit["summary"] + print(f"Pinned source: {args.repository}@{commit_sha}") + print(f"LXC launchers analyzed: {summary['lxc_launchers']}") + print(f"Verified operational hints: {summary['operational_hints']} ({summary['coverage_percent']}%)") + print(f"Methods: {summary['method_counts']}") + print(f"Catalog: {args.output.resolve()}") + print(f"Audit: {args.audit_output.resolve()}") + print(f"Multi-detector catalog v2: {args.v2_output.resolve()}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/update-app-tracking-hints.yml b/.github/workflows/update-app-tracking-hints.yml new file mode 100644 index 00000000..373918a3 --- /dev/null +++ b/.github/workflows/update-app-tracking-hints.yml @@ -0,0 +1,160 @@ +name: Update App Tracking Hints + +on: + # Manual trigger from the Actions UI + workflow_dispatch: + + # Re-merge whenever the generator, workflow, or the maintainer- + # curated runtime overrides change. `runtime_verified_overrides.json` + # is the file to edit when a real LXC reveals a canonical path the + # community-scripts helper doesn't ship (legacy /app/package.json, + # /opt/vaultwarden/bin/vaultwarden, etc.) β€” the generator folds it + # into the operational catalog every run. + push: + branches: [main] + paths: + - ".github/scripts/generate_app_tracking_catalog.py" + - ".github/workflows/update-app-tracking-hints.yml" + - "json/runtime_verified_overrides.json" + + # Regen every 6h β€” picks up new community-scripts LXC apps and + # detector-relevant script edits without needing a manual trigger. + schedule: + - cron: "0 */6 * * *" + +jobs: + update-hints: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: ⬇️ Checkout the repository + uses: actions/checkout@v6 + + - name: 🐍 Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: βš™οΈ Generate app_tracking_hints.generated.json (intermediate) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # The generator writes 4 files; only `.generated.json` is + # consumed downstream by the merge step. The v2 catalog and + # per-app audit are useful for local review but not kept in + # the repo β€” written under /tmp so they never appear as + # dirty files here. + # + # `--runtime-overrides` folds real-CT evidence into the + # operational hints (canonical paths, cross-method fallbacks + # per app) so the runtime doesn't get fed helper-marker + # false-positives. + run: | + python .github/scripts/generate_app_tracking_catalog.py \ + --helpers-cache json/helpers_cache.json \ + --existing json/app_tracking_hints.json \ + --runtime-overrides json/runtime_verified_overrides.json \ + --output json/app_tracking_hints.generated.json \ + --v2-output /tmp/app_tracking_catalog.v2.json \ + --audit-output /tmp/app_tracking_hints.audit.json + + - name: 🧬 Smart-merge generated into app_tracking_hints.json + # Single source of truth: `app_tracking_hints.json` is the ONE + # file. It contains 3 kinds of entries: + # 1. Auto-verified from community-scripts (the generator + # manages every "generator-owned" field on these). + # 2. User-edited additions to those entries β€” extra fields + # the generator doesn't touch (default_ports, + # file_fallbacks, custom logo overrides…). + # 3. User-only entries the generator can't verify (Docker, + # AdGuard, Pi-hole, WireGuard, …) β€” left alone. + # Merge rule: for slugs the generator produces, refresh only + # the whitelisted fields; preserve everything else. For slugs + # NOT in the generator's output, keep the existing entry + # untouched. + run: | + python - <<'PY' + import json + from pathlib import Path + + GEN = Path("json/app_tracking_hints.generated.json") + OUT = Path("json/app_tracking_hints.json") + + # Fields owned by the generator β€” refreshed on every run. + # These are all populated deterministically by the generator + # (the audit script folds `runtime_verified_overrides.json` + # in as it runs), so a local hand-edit for a generator-known + # slug would get overwritten on the next tick. To add a new + # canonical path or a cross-method fallback for a slug the + # generator already knows, edit `runtime_verified_overrides + # .json` β€” that file IS the maintainer-controlled input. + # + # For user-only slugs (Docker, WireGuard, Pi-hole and any + # other entry not in the generator's output) EVERY field is + # preserved verbatim by the merge below β€” the whitelist only + # governs generator-covered slugs. + GENERATOR_FIELDS = { + "installed_via", "package", "file_path", "file_regex", + "binary_path", "repo", "github_source", "tag_regex", + "installed_regex", + # Upstream source discriminator + per-type fields + # (http_json + docker_hub). Kept in the whitelist so a + # curated entry in runtime_verified_overrides.json can + # supply them and the smart merge won't drop them on the + # next regeneration. + "upstream_type", "upstream_url", "upstream_json_path", + "docker_image", + "logo", "website", + "default_ports", "file_fallbacks", "alt_detectors", + } + + generated = json.loads(GEN.read_text(encoding="utf-8")) + existing = {} + if OUT.is_file(): + try: + existing = json.loads(OUT.read_text(encoding="utf-8")) + if not isinstance(existing, dict): + existing = {} + except json.JSONDecodeError: + existing = {} + + merged = {} + for slug, gen_entry in generated.items(): + base = dict(existing.get(slug) or {}) + # Refresh generator-owned fields (add/update). + for k, v in gen_entry.items(): + if k in GENERATOR_FIELDS: + base[k] = v + # Drop generator-owned fields that the generator no + # longer emits for this slug (e.g. path renamed away). + for k in list(base): + if k in GENERATOR_FIELDS and k not in gen_entry: + del base[k] + merged[slug] = base + # Preserve user-only entries the generator can't verify. + for slug, entry in existing.items(): + if slug not in generated and isinstance(entry, dict): + merged[slug] = dict(entry) + + OUT.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + added = sorted(set(generated) - set(existing)) + removed = sorted(set(existing) - set(generated) - {s for s, e in existing.items() if not ( + set(e.keys()) - GENERATOR_FIELDS + )}) + print(f"merged: {len(merged)} entries " + f"(generated={len(generated)}, existing={len(existing)})") + if added: + print(f" new from generator: {len(added)}") + # Clean up the intermediate file so it doesn't get committed. + GEN.unlink() + PY + + - name: πŸ“€ Commit + push if changed + run: | + git config user.name "ProxMenuxBot" + git config user.email "bot@proxmenux.local" + git add json/app_tracking_hints.json + git diff --cached --quiet || git commit -m "Update app tracking hints" + git push diff --git a/AppImage/components/health-status-modal.tsx b/AppImage/components/health-status-modal.tsx index 014c9b85..e9a0a494 100644 --- a/AppImage/components/health-status-modal.tsx +++ b/AppImage/components/health-status-modal.tsx @@ -32,7 +32,7 @@ import { FileText, RefreshCw, Shield, - Download, + ArrowUpCircle, X, Clock, BellOff, @@ -777,7 +777,7 @@ export function HealthStatusModal({ open, onOpenChange, getApiUrl }: HealthStatu onClick={() => setShowUpdateTerminal(true)} className="bg-purple-600/15 hover:bg-purple-600/25 border border-purple-500/40 text-purple-300 hover:text-purple-200" > - + {t("healthStatus.updateNow")} diff --git a/AppImage/components/lxc-app-panel.tsx b/AppImage/components/lxc-app-panel.tsx new file mode 100644 index 00000000..fd369eb8 --- /dev/null +++ b/AppImage/components/lxc-app-panel.tsx @@ -0,0 +1,1895 @@ +"use client" + +/** + * LxcAppPanel β€” Body of the "App" tab in the LXC modal. + * + * Handles a LIST of apps per CT (one CT can host several services + * β€” e.g. Frigate on port 5000 + go2rtc on 1984, or Docker + two + * containerised apps). Each app has: + * β€’ an install method (dpkg / apk / file / binary / docker) + * β€’ an optional GitHub repo for upstream version tracking + * β€’ a list of ports, each with a description and web path + * + * Docker apps are "register-only": they exist to produce clickable + * links, ProxMenux does NOT try to track their version and NEVER + * emits warnings for them β€” updates for Docker apps are handled by + * Docker itself. + * + * For ProxMenux-managed OCI CTs (Secure Gateway) the panel is + * read-only β€” the actual update lifecycle lives in Security β†’ + * Secure Gateway. + */ + +import { useCallback, useEffect, useMemo, useState } from "react" +import { + Loader2, Save, RefreshCw, Trash2, Package, ExternalLink, + AlertTriangle, Info, PlusCircle, Pencil, ChevronDown, ChevronRight, EyeOff, + ArrowUpCircle, RotateCcw, Check, Settings2, ShieldCheck, CheckCircle2, +} from "lucide-react" +import { Card, CardContent } from "./ui/card" +import { Button } from "./ui/button" +import { Input } from "./ui/input" +import { Label } from "./ui/label" +import { Badge } from "./ui/badge" +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "./ui/select" +import { fetchApi } from "../lib/api-config" +import { useT } from "@/lib/i18n/provider" + +// installed_via is optional now β€” an empty value means "register only, +// no version tracking, no warnings, just a clickable link". Docker +// apps and casual "just want a link" registrations use this default. +type InstalledVia = "" | "dpkg" | "apk" | "file" | "binary" | + "python_dist" | "docker_label" | "docker_exec" | + "command" | "manual" +type GithubSource = "releases" | "tags" + +interface PortEntry { + port: number | "" + description?: string + scheme?: "http" | "https" + web_path?: string + logo_url?: string +} + +interface AppConfig { + name: string + installed_via?: InstalledVia + package?: string + file_path?: string + file_regex?: string + binary_path?: string + binary_args?: string[] + python_path?: string + distribution?: string + container_name?: string + label?: string + command_argv?: string[] + installed_version?: string + installed_regex?: string + // Upstream source discriminator + fields. When `upstream_type` is + // "github" (default when `repo` is set) the classic repo / + // github_source / tag_regex fields drive the check. "http_json" and + // "docker_hub" open two new source types validated separately on + // the backend. + upstream_type?: "github" | "http_json" | "docker_hub" | "" + repo?: string + github_source?: GithubSource + upstream_url?: string + upstream_json_path?: string + docker_image?: string + tag_regex?: string + ports: PortEntry[] + health_path?: string + logo_url?: string + helper_slug?: string +} + +interface DetectedApp { + slug: string + name: string + logo_url?: string | null + default_ports?: number[] + tracking_suggestion?: TrackingSuggestion | null +} + +interface AppState { + installed_version: string | null + latest_version: string | null + update_available: boolean | null + error: string | null + checked_at: string | null +} + +interface AppEntry extends AppConfig { + id: string + state?: AppState + created_at?: string +} + +interface SidecarResponse { + vmid: number + apps: AppEntry[] + dismissed_slugs?: string[] + created_at?: string + updated_at?: string +} + +interface TrackingSuggestion { + installed_via: "dpkg" | "apk" | "file" | "binary" + package?: string + file_path?: string + file_regex?: string + binary_path?: string + repo?: string + github_source?: "releases" | "tags" + tag_regex?: string +} + +interface Suggestions { + name_suggestion: string | null + helper_slug: string | null + port_suggestions: number[] + web_path_hint: string | null + tracking_suggestion?: TrackingSuggestion | null + default_ports?: number[] + logo_url?: string | null + extras?: DetectedApp[] +} + +// Compact catalog entry β€” one row for every registerable app the +// picker can offer. Fetched once from /api/apps/catalog on panel +// mount, filtered client-side while the user types. +interface CatalogEntry { + slug: string + name: string + logo: string + default_port: number + has_tracking: boolean +} + +// Full detail for a picked catalog entry β€” server merges catalog +// metadata + curated tracking_suggestion (when available) so the +// editor can pre-fill every field in one round-trip. +interface CatalogDetail { + slug: string + name: string + logo_url: string | null + website: string + default_ports: number[] + tracking_suggestion?: TrackingSuggestion | null +} + +interface ManagedAppInfo { + managed_oci_app_id: string + name: string + installed_version?: string | null + latest_version?: string | null + update_available?: boolean | null + checked_at?: string | null + error?: string | null +} + +interface Props { + vmid: number + ctIp?: string | null + onChange?: () => void + managed?: ManagedAppInfo | null +} + +const EMPTY_APP: AppConfig = { + name: "", + installed_via: "", // no tracking by default β€” just a link + package: "", + upstream_type: "", + repo: "", + github_source: "releases", + upstream_url: "", + upstream_json_path: "", + docker_image: "", + tag_regex: "v?(\\d+\\.\\d+\\.\\d+)", + ports: [], + logo_url: "", +} + +// Default scheme heuristic for freshly-added ports β€” only used to +// pre-select the dropdown. The user always has the final say via +// the http/https selector next to the port input. +const HTTPS_HINT_PORTS = new Set([443, 4443, 8443, 9443]) +const defaultSchemeFor = (port: number | ""): "http" | "https" => + HTTPS_HINT_PORTS.has(Number(port)) ? "https" : "http" + +function buildWebUrl(ip: string | undefined | null, port: number | "", scheme?: "http" | "https") { + if (!ip || ip === "DHCP" || !port) return null + return `${scheme || defaultSchemeFor(port)}://${ip}:${port}` +} + +// Suggest a dpkg/apk package name from a friendly app name β€” lowercase, +// spaces and slashes to hyphens, drop punctuation. Only used as a +// placeholder / auto-fill; user can always override. +function suggestPackageName(name: string) { + return name + .trim() + .toLowerCase() + .replace(/[\s/]+/g, "-") + .replace(/[^a-z0-9._+@:-]/g, "") + .replace(/-{2,}/g, "-") + .replace(/^-+|-+$/g, "") +} + +export function LxcAppPanel({ vmid, ctIp, onChange, managed }: Props) { + const t = useT() + const [loading, setLoading] = useState(true) + const [sidecar, setSidecar] = useState(null) + const [suggestions, setSuggestions] = useState(null) + const [error, setError] = useState(null) + // Editor state + const [editing, setEditing] = useState<{ appId: string | null; draft: AppConfig } | null>(null) + const [saving, setSaving] = useState(false) + const [busyAppId, setBusyAppId] = useState(null) + // Advanced section (version tracking) is collapsed by default so the + // basic Name + Ports flow stays approachable. Auto-expanded when + // editing an app that already has installed_via set, or when the + // user clicked Register on an auto-detected chip whose hint carries + // tracking metadata β€” the user sees the fields we auto-filled and + // can tweak or opt out before saving. + const [showAdvanced, setShowAdvanced] = useState(false) + + // Catalog picker: 700+ apps fetched once from /api/apps/catalog and + // filtered client-side while the user types in the Name input. The + // dropdown shows top 20 matches. Selecting one calls the detail + // endpoter to seed name / logo / ports / tracking_suggestion at once. + const [catalog, setCatalog] = useState([]) + const [pickerOpen, setPickerOpen] = useState(false) + + // "Register a different app" browse panel: when the user has hidden + // some detections we surface them here with a Restore button before + // falling through to the blank-form path. If there's nothing to + // restore, this panel is skipped entirely and the button opens the + // editor directly (fast path for the common case). + const [browseOpen, setBrowseOpen] = useState(false) + + // Global "manage apps" mode. When ON, every app card grows a footer + // with Remove / Check / Edit fields actions. When OFF the cards are + // pure info β€” the only surfaced action is a hover-reveal Check icon + // on the LATEST UPSTREAM panel. Toggled from a single button next to + // "Add another application". + const [editMode, setEditMode] = useState(false) + + const load = useCallback(async () => { + if (managed) { setLoading(false); return } + setLoading(true) + setError(null) + try { + const r: SidecarResponse = await fetchApi(`/api/vms/${vmid}/apps`) + setSidecar(r) + // Always fetch suggestions β€” used both by the empty-state form + // seed AND by the "Also detected on this container" chip strip + // that surfaces unregistered detections even after the CT already + // has β‰₯1 registered app. Previously gated on `!r.apps?.length`, + // which meant a CT with 1 registered + 1 detected-but-not-yet- + // registered app never showed the second app until the user + // opened + cancelled the editor (which triggered a re-load path + // that happened to fetch it). + try { + const s: Suggestions = await fetchApi(`/api/vms/${vmid}/apps/suggestions`) + setSuggestions(s) + } catch { /* non-fatal */ } + } catch (e: any) { + setError(e?.message || "Could not load app configuration") + } finally { + setLoading(false) + } + }, [vmid, managed]) + + useEffect(() => { load() }, [load]) + + // Fetch the picker catalog once per panel mount. Best-effort β€” if + // the API is unreachable, the picker just stays empty and users + // type the app name manually (same as before this feature). + useEffect(() => { + let cancelled = false + fetchApi("/api/apps/catalog") + .then((data: CatalogEntry[]) => { + if (!cancelled && Array.isArray(data)) setCatalog(data) + }) + .catch(() => { /* non-fatal */ }) + return () => { cancelled = true } + }, []) + + // Derived state β€” computed here BEFORE any conditional early + // return so React sees the same hook order on every render. + // Rules of Hooks: `useMemo` after an `if (loading) return …` + // trips React error #310 the moment `loading` flips false. + const apps = sidecar?.apps || [] + + // Unified detection list β€” primary community-scripts install + + // every other app whose install signature was found on the CT + // (`extras[]` from the backend). Both use the same DetectedApp + // shape so the empty state renders them uniformly. + const detectedList: DetectedApp[] = useMemo(() => { + if (!suggestions) return [] + const out: DetectedApp[] = [] + if (suggestions.helper_slug && suggestions.name_suggestion) { + out.push({ + slug: suggestions.helper_slug, + name: suggestions.name_suggestion, + logo_url: suggestions.logo_url, + default_ports: suggestions.default_ports, + tracking_suggestion: suggestions.tracking_suggestion, + }) + } + const seen = new Set(out.map((d) => d.slug)) + for (const e of suggestions.extras || []) { + if (!seen.has(e.slug)) { + out.push(e) + seen.add(e.slug) + } + } + return out + }, [suggestions]) + + // Registered slugs β€” used to filter the detection list down to + // what the user hasn't already registered on this CT. + const registeredSlugs = useMemo( + () => new Set(apps.map((a) => a.helper_slug).filter(Boolean) as string[]), + [apps], + ) + // Dismissed slugs β€” persisted in the sidecar. Chips the user + // explicitly hid via the βœ• button stay hidden across reloads until + // they register the app (which also un-dismisses implicitly). + const dismissedSlugs = useMemo( + () => new Set(sidecar?.dismissed_slugs || []), + [sidecar], + ) + const visibleDetected = detectedList.filter( + (d) => !registeredSlugs.has(d.slug) && !dismissedSlugs.has(d.slug), + ) + // Alias for pre-existing consumers (post-registration chip strip). + const unregisteredDetected = visibleDetected + // Detections the user hid and could restore from the Register-a- + // different-app panel. Not affected by registration state. + const hiddenDetections = detectedList.filter((d) => dismissedSlugs.has(d.slug)) + + // "Register a different app" behavior: if there are hidden slugs, + // surface them first (with Restore) so the user can bring one back + // instead of typing everything by hand. If nothing to restore, go + // straight to the blank editor. + const openBrowseOrEditor = () => { + if (hiddenDetections.length > 0) setBrowseOpen(true) + else openEditor() + } + + const openEditor = useCallback(async ( + existing?: AppEntry, + opts?: { withTracking?: boolean, preset?: DetectedApp }, + ) => { + let seed: AppConfig + if (existing) { + seed = { + name: existing.name, + installed_via: (existing.installed_via as InstalledVia) || "", + package: existing.package || "", + file_path: existing.file_path || "", + file_regex: existing.file_regex || "", + binary_path: existing.binary_path || "", + binary_args: existing.binary_args ? [...existing.binary_args] : [], + python_path: existing.python_path || "", + distribution: existing.distribution || "", + container_name: existing.container_name || "", + label: existing.label || "", + command_argv: existing.command_argv ? [...existing.command_argv] : [], + installed_version: existing.installed_version || "", + installed_regex: existing.installed_regex || "", + upstream_type: existing.upstream_type || (existing.repo ? "github" : ""), + repo: existing.repo || "", + github_source: existing.github_source || "releases", + upstream_url: existing.upstream_url || "", + upstream_json_path: existing.upstream_json_path || "", + docker_image: existing.docker_image || "", + tag_regex: existing.tag_regex || "v?(\\d+\\.\\d+\\.\\d+)", + ports: existing.ports?.length ? existing.ports.map((p) => ({ ...p })) : [], + health_path: existing.health_path || "", + logo_url: existing.logo_url || "", + helper_slug: existing.helper_slug || "", + } + // Editing an existing app: expand Advanced when tracking is on + setShowAdvanced(!!seed.installed_via) + } else { + seed = { ...EMPTY_APP, ports: [] } + let s = suggestions + if (!s) { + try { + s = await fetchApi(`/api/vms/${vmid}/apps/suggestions`) + setSuggestions(s) + } catch { /* non-fatal */ } + } + // Preset path: a chip in the empty state (primary OR extra) was + // clicked. Seed EVERYTHING from the preset so this works + // regardless of whether it's the first or Nth app on the CT. + // Primary detection is `{...suggestions}`-shaped, an extra is + // `DetectedApp`-shaped β€” both carry name/logo/ports/tracking. + if (opts?.preset) { + const p = opts.preset + seed.name = p.name + seed.logo_url = p.logo_url || "" + seed.helper_slug = p.slug + if (p.default_ports?.length) { + seed.ports = p.default_ports.map((port) => ({ + port, + scheme: defaultSchemeFor(port), + web_path: s?.web_path_hint || "", + })) + } + if (opts.withTracking && p.tracking_suggestion) { + const t = p.tracking_suggestion + seed = { + ...seed, + installed_via: t.installed_via, + package: t.package || "", + file_path: t.file_path || "", + file_regex: t.file_regex || "", + binary_path: t.binary_path || "", + upstream_type: (t as any).upstream_type || (t.repo ? "github" : ""), + repo: t.repo || "", + github_source: t.github_source || "releases", + upstream_url: (t as any).upstream_url || "", + upstream_json_path: (t as any).upstream_json_path || "", + docker_image: (t as any).docker_image || "", + tag_regex: t.tag_regex || "v?(\\d+\\.\\d+\\.\\d+)", + } + setShowAdvanced(true) + } else { + setShowAdvanced(false) + } + } else { + // No preset (bare "+ Register application"): start empty so + // the user types name/ports/logo for a custom app the auto- + // detector doesn't know about. + setShowAdvanced(false) + } + } + setEditing({ appId: existing?.id || null, draft: seed }) + setError(null) + }, [suggestions, vmid, sidecar]) + + const closeEditor = () => { + setEditing(null) + setError(null) + } + + const save = async () => { + if (!editing) return + setSaving(true) + setError(null) + try { + const url = editing.appId + ? `/api/vms/${vmid}/apps/${editing.appId}` + : `/api/vms/${vmid}/apps` + const method = editing.appId ? "PUT" : "POST" + const r: SidecarResponse & { error?: string } = await fetchApi(url, { + method, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(editing.draft), + }) + if ((r as any).error) throw new Error((r as any).error) + setSidecar(r) + setEditing(null) + onChange?.() + } catch (e: any) { + setError(e?.message || t("vmLxc.appEditor.saveFailed")) + } finally { + setSaving(false) + } + } + + const checkOne = async (appId: string) => { + setBusyAppId(appId) + setError(null) + try { + const r: SidecarResponse = await fetchApi(`/api/vms/${vmid}/apps/${appId}/check`, { + method: "POST", + }) + setSidecar(r) + onChange?.() + } catch (e: any) { + setError(e?.message || "Check failed") + } finally { + setBusyAppId(null) + } + } + + const removeOne = async (appId: string) => { + if (!confirm("Remove this application from the CT's App tab?")) return + setBusyAppId(appId) + setError(null) + try { + await fetchApi(`/api/vms/${vmid}/apps/${appId}`, { method: "DELETE" }) + // Reload from server so the empty state re-fetches suggestions + await load() + onChange?.() + } catch (e: any) { + setError(e?.message || "Delete failed") + } finally { + setBusyAppId(null) + } + } + + // Hide an auto-detected chip. Optimistic UI: update the local + // sidecar state immediately so the chip disappears without + // waiting for the round-trip, then persist to the server. If the + // POST fails, reload from server to resync. + const dismissDetection = async (slug: string, name: string) => { + if (!confirm(t("vmLxc.appEditor.confirmHide", { name }))) return + setSidecar((prev) => prev + ? { ...prev, dismissed_slugs: [...(prev.dismissed_slugs || []), slug] } + : prev, + ) + try { + const r: SidecarResponse = await fetchApi(`/api/vms/${vmid}/apps/dismiss`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ slug, dismissed: true }), + }) + setSidecar(r) + } catch (e: any) { + setError(e?.message || "Could not dismiss detection") + await load() // resync on failure + } + } + + // Un-hide a previously dismissed slug. Used from the "Register a + // different app" panel. Optimistically drops the slug from local + // dismissed_slugs so the chip re-appears in the main list, then + // persists. + const restoreDetection = async (slug: string) => { + setSidecar((prev) => prev + ? { ...prev, dismissed_slugs: (prev.dismissed_slugs || []).filter((s) => s !== slug) } + : prev, + ) + try { + const r: SidecarResponse = await fetchApi(`/api/vms/${vmid}/apps/dismiss`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ slug, dismissed: false }), + }) + setSidecar(r) + } catch (e: any) { + setError(e?.message || "Could not restore detection") + await load() + } + } + + // ── Managed CT (Secure Gateway etc.) ────────────────────────── + // Mirrors the visual identity of a regular app card so managed OCI + // apps sit next to user-registered apps without a jarring style + // shift. Version data comes from managed_installs.update_check + // (already tracked by oci_manager β€” same source as the Security β†’ + // Secure Gateway page). No footer: no Edit, no Check, no Remove β€” + // the whole lifecycle lives in Security β†’ Secure Gateway. + if (managed) { + // Currently the only OCI managed app is Secure Gateway (Tailscale + // in an Alpine CT). When we add more OCI apps we'll swap this to + // a lookup keyed on managed_oci_app_id β†’ catalog metadata. + const isSecureGateway = managed.managed_oci_app_id === "secure-gateway" + const displayName = isSecureGateway ? "Secure Gateway" : (managed.name || "Managed app") + const displaySubtitle = isSecureGateway ? "Tailscale VPN Gateway" : "" + // Two variants β€” the selfh.st mark (dark logo on light bg) reads + // better in light mode; the homarr-labs "-light" variant (light + // logo on dark bg) reads better in dark mode. Both are rendered + // and Tailwind's dark: class picks which one is visible. + const upstreamLogoLightUrl = isSecureGateway + ? "https://cdn.jsdelivr.net/gh/selfhst/icons@main/webp/tailscale.webp" + : "" + const upstreamLogoDarkUrl = isSecureGateway + ? "https://cdn.jsdelivr.net/gh/homarr-labs/dashboard-icons/webp/tailscale-light.webp" + : "" + const upstreamName = isSecureGateway ? "Tailscale" : "" + const repo = isSecureGateway ? "tailscale/tailscale" : "" + const methodLine = isSecureGateway ? "apk Β· tailscale Β· managed" : "managed" + const hasUpdate = managed.update_available === true + const upToDate = managed.update_available === false && !!managed.installed_version + const showVersions = !!(managed.installed_version || managed.latest_version || repo) + + return ( +
+ + + {/* Block 1 β€” Secure Gateway identity (the ProxMenux + product). Big shield, title, catalog subtitle. */} +
+
+ +
+
+

{displayName}

+ {displaySubtitle && ( +
{displaySubtitle}
+ )} +
+
+ + {/* Block 2 β€” Underlying engine (Tailscale). Same visual + pattern as a regular app card so it's clear this is + what version tracking is anchored to. Repo link goes + here (top-right on desktop / stacked on mobile) because + the repo is the engine's, not Secure Gateway's. */} + {upstreamName && ( +
+
+
+ {upstreamLogoLightUrl || upstreamLogoDarkUrl ? ( + <> + {upstreamLogoLightUrl && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} + {upstreamLogoDarkUrl && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} + + ) : ( +
+ +
+ )} +
+
{upstreamName}
+
{methodLine}
+ {managed.checked_at && ( +
+ Checked {new Date(managed.checked_at).toLocaleString([], { dateStyle: "short", timeStyle: "short" })} +
+ )} + {repo && ( + + + {repo} + + )} +
+
+ {repo && ( + + {repo} + + + )} +
+
+ )} + + {showVersions && ( +
+ {managed.installed_version && ( +
+
{t("vmLxc.appEditor.installedStatus")}
+
+ {managed.installed_version} + {upToDate && ( + + )} +
+
+ )} + {repo && ( +
+
+ Latest upstream +
+
+ {managed.latest_version || {t("vmLxc.appEditor.checkingStatus")}} + {hasUpdate && managed.latest_version && ( + + )} +
+
+ )} +
+ )} + + {managed.error && ( +
+ +
{managed.error}
+
+ )} + + {/* Managed banner β€” green translucent badge signalling this + CT's lifecycle is owned by ProxMenux (not user CRUD). */} +
+ + {t("vmLxc.appEditor.installedManaged")} +
+ + +
+ ) + } + + if (loading) { + return ( +
+ + Loading applications… +
+ ) + } + + // ── Editor ───────────────────────────────────────────────────── + if (editing) { + const draft = editing.draft + const method = draft.installed_via || "" + const isPackaged = method === "dpkg" || method === "apk" + const setField = (patch: Partial) => + setEditing({ ...editing, draft: { ...draft, ...patch } }) + // Editing the Name auto-fills the Package field on packaged + // methods when it's still empty. Rationale: 90% of the time the + // dpkg/apk package name mirrors the friendly app name (jellyfin, + // adguardhome, portainer-ce). The user can still override. + const setName = (name: string) => { + const patch: Partial = { name } + if (isPackaged && !draft.package?.trim()) { + patch.package = suggestPackageName(name) + } + setField(patch) + } + const setPort = (i: number, patch: Partial) => { + const ports = draft.ports.map((p, idx) => (idx === i ? { ...p, ...patch } : p)) + setField({ ports }) + } + // "Add port" adds an EMPTY row for manual entry. Detected chips + // ("+5000", "+1984"…) add the port directly with the port + // pre-filled β€” no need for the user to open a row first. + const addEmptyPort = () => + setField({ ports: [...draft.ports, { port: "", description: "", scheme: "http" }] }) + const addDetectedPort = (port: number) => { + const scheme = defaultSchemeFor(port) + // If the last row is still empty, fill it instead of appending + // a duplicate. Avoids the two-lines-appear bug. + const last = draft.ports[draft.ports.length - 1] + if (last && last.port === "" && !last.description) { + const ports = [...draft.ports] + ports[ports.length - 1] = { port, description: "", scheme } + setField({ ports }) + } else { + setField({ ports: [...draft.ports, { port, description: "", scheme }] }) + } + } + const removePort = (i: number) => + setField({ ports: draft.ports.filter((_, idx) => idx !== i) }) + const usedPorts = new Set(draft.ports.map((p) => p.port)) + const suggestable = (suggestions?.port_suggestions || []).filter((p) => !usedPorts.has(p)) + + return ( +
+ {/* Editor card β€” opaque `bg-card` (not the `/50` used by + view-mode cards) so the darker `bg-background` inputs sit + visibly recessed against it. Matches the sunken-input + pattern used in the PBS setup wizard: card = surface, + fields = wells inside the surface. Reverts to `bg-card/50` + automatically because this render branch only fires when + `editing !== null`. */} + + +
+ + { setName(e.target.value); setPickerOpen(true) }} + onFocus={() => setPickerOpen(true)} + onBlur={() => setTimeout(() => setPickerOpen(false), 150)} + placeholder={suggestions?.name_suggestion || "Type to search 700+ apps, or type your own"} + maxLength={64} + autoComplete="off" + /> + {/* Catalog picker dropdown β€” filters the 700+ helpers_cache + entries by name substring while the user types. Top 20 + matches shown. Click one to auto-fill name / logo / + ports / tracking hint in one shot. Empty query with + the input focused shows a "start typing" hint. */} + {pickerOpen && catalog.length > 0 && (() => { + const q = (draft.name || "").trim().toLowerCase() + if (!q) return null + const matches = catalog + .filter((c) => c.name.toLowerCase().includes(q) || c.slug.toLowerCase().includes(q)) + .slice(0, 20) + if (!matches.length) return null + return ( +
+
+ {matches.length === 20 ? t("vmLxc.appEditor.top20Matches") : `${matches.length} match${matches.length === 1 ? "" : "es"}`} +
+ {matches.map((c) => ( + + ))} +
+ ) + })()} +
+ + {/* App-level logo URL β€” optional. Auto-filled from the + catalog for helper-scripts installs, blank otherwise. + For a custom app the user can paste a URL (typically + from https://selfh.st/icons); empty β†’ no logo in the + app card header. */} +
+ + setField({ logo_url: e.target.value })} + placeholder={t("vmLxc.appEditor.portLogoPlaceholder")} + maxLength={512} + className="text-sm mt-2 font-mono" + /> +
+ + {/* Web links β€” each port becomes a clickable link (built as + http[s]://:). Detected chips add the port + directly (no need to first open an empty row). */} +
+
+ + +
+ + {/* Detected chips FIRST β€” one-click add. Only shown when + there are chips left to suggest, so empty states stay + clean. Click on a chip: fills the current empty row + or adds a new one; never duplicates. */} + {suggestable.length > 0 && ( +
+ + Ports detected in the container β€” click to add: + + {suggestable.map((p) => ( + + ))} +
+ )} + + {draft.ports.length === 0 && suggestable.length === 0 && ( +
+ No web ports listening. Use "Add port" if you'd still + like to register one manually. +
+ )} + +
+ {draft.ports.map((entry, i) => ( +
+ + setPort(i, { port: e.target.value ? Number(e.target.value) : "" })} + placeholder={t("vmLxc.appEditor.portPortPlaceholder")} + min={1} + max={65535} + className="text-sm" + /> + setPort(i, { description: e.target.value })} + placeholder={t("vmLxc.appEditor.portDescriptionPlaceholder")} + maxLength={64} + className="text-sm" + /> + + {/* Per-link logo URL β€” spans cols 1-3 so its + right edge lines up with the description input + above (never covers the trash column). */} + setPort(i, { logo_url: e.target.value })} + placeholder={t("vmLxc.appEditor.portLogoLabel")} + maxLength={512} + className="col-start-1 col-end-4 text-xs font-mono h-8 opacity-70 focus:opacity-100" + type="url" + /> +
+ ))} +
+
+ + {/* ── Advanced (Version tracking, optional) ────────── + Collapsed by default so casual users never see the + technical fields. Auto-expanded when editing an app + that already has tracking configured, or when the + user registered from an auto-detected chip whose + hint carried tracking metadata. */} +
+ + + {showAdvanced && ( +
+

+ Optional. When enabled, ProxMenux checks the installed + version inside the container and compares it against + a GitHub repo β€” you'll see "update available" and get + an optional notification. Skip this if you only want + a clickable link. +

+ +
+
+ + +
+ + {isPackaged && ( +
+ + setField({ package: e.target.value })} + placeholder={method === "dpkg" ? "e.g. jellyfin-server" : "e.g. tailscale"} + /> +
+ Auto-filled from Name. Verify with{" "} + + {method === "dpkg" ? "dpkg -l | grep " : "apk info | grep "} + {" "} + inside the CT. +
+
+ )} + + {method === "binary" && ( +
+ + setField({ binary_path: e.target.value })} + placeholder={t("vmLxc.appEditor.binaryPathPlaceholder")} + className="font-mono text-xs" + /> +
+ Absolute path. Find it with{" "} + which <app> or{" "} + systemctl show <service> -p ExecStart. +
+
+ )} +
+ + {method === "file" && ( +
+
+ + setField({ file_path: e.target.value })} + placeholder="/opt/app/VERSION" + className="font-mono text-xs" + /> +
+
+ + setField({ file_regex: e.target.value })} + placeholder={t("vmLxc.appEditor.regexPlaceholderVersion")} + className="font-mono text-xs" + /> +
+
+ )} + + {method === "python_dist" && ( +
+
+ + setField({ python_path: e.target.value })} + placeholder={t("vmLxc.appEditor.pythonInterpreterPlaceholder")} + className="font-mono text-xs" + /> +
+
+ + setField({ distribution: e.target.value })} + placeholder={t("vmLxc.appEditor.pipDistPlaceholder")} + className="font-mono text-xs" + /> +
+
+ )} + + {method === "docker_label" && ( +
+
+ + setField({ container_name: e.target.value })} + placeholder={t("vmLxc.appEditor.containerNamePlaceholder")} + className="font-mono text-xs" + /> +
+
+ + setField({ label: e.target.value })} + placeholder={t("vmLxc.appEditor.ociLabelPlaceholder")} + className="font-mono text-xs" + /> +
+
+ )} + + {method === "docker_exec" && ( +
+
+ + setField({ container_name: e.target.value })} + placeholder={t("vmLxc.appEditor.containerNamePlaceholder")} + className="font-mono text-xs" + /> +
+
+ + setField({ binary_path: e.target.value })} + placeholder={t("vmLxc.appEditor.binaryPathBarePlaceholder")} + className="font-mono text-xs" + /> +
+
+ + setField({ + binary_args: e.target.value.split(",").map(s => s.trim()).filter(Boolean), + })} + placeholder={t("vmLxc.appEditor.binaryArgsPlaceholder")} + className="font-mono text-xs" + /> +
+ {t("vmLxc.appEditor.binaryArgsHintPrefix")} --version. {t("vmLxc.appEditor.binaryArgsHintGrafana")} server, -v. +
+
+
+ )} + + {method === "command" && ( +
+ + setField({ + command_argv: e.target.value.split(",").map(s => s.trim()).filter(Boolean), + })} + placeholder={t("vmLxc.appEditor.commandPlaceholder")} + className="font-mono text-xs" + /> +
+ Runs argv-style inside the CT as root β€” never through a shell. You are responsible for what the command does. Use installed_regex below to extract the version from the output. +
+
+ + setField({ installed_regex: e.target.value })} + placeholder={t("vmLxc.appEditor.tagRegexBare")} + className="font-mono text-xs mt-1" + /> +
+
+ )} + + {method === "manual" && ( +
+ + setField({ installed_version: e.target.value })} + placeholder="1.2.3" + maxLength={64} + className="font-mono text-xs" + /> +
+ Type the version you have installed. If repo is set below, ProxMenux still checks upstream and notifies on updates β€” after upgrading the app, come back here and update this string. +
+
+ )} + + {method && (() => { + // Upstream source selector β€” 3 methods (github, + // http_json, docker_hub). Legacy sidecars with a + // `repo` set but no `upstream_type` default to + // github so the classic behaviour keeps working + // until the user re-saves. + const upstreamType = draft.upstream_type + || (draft.repo ? "github" : "") + const setUpstream = (t: "" | "github" | "http_json" | "docker_hub") => { + // Clear other-type fields when switching so the + // backend doesn't receive stale data. + const patch: Partial = { upstream_type: t } + if (t !== "github") { + patch.repo = "" + patch.github_source = "releases" + } + if (t !== "http_json") { + patch.upstream_url = "" + patch.upstream_json_path = "" + } + if (t !== "docker_hub") { + patch.docker_image = "" + } + if (!t) patch.tag_regex = "" + setField(patch) + } + return ( + <> +
+ + +
+ How ProxMenux finds the latest version to compare against your installed one. +
+
+ + {upstreamType === "github" && ( +
+
+ + setField({ repo: e.target.value })} + placeholder={t("vmLxc.appEditor.githubRepoPlaceholder")} + /> +
+ Public GitHub repository where releases or tags are published. +
+
+
+ + +
+
+ )} + + {upstreamType === "http_json" && ( +
+
+ + setField({ upstream_url: e.target.value })} + placeholder={t("vmLxc.appEditor.endpointUrlPlaceholder")} + className="font-mono text-xs" + maxLength={512} + /> +
+ Public JSON endpoint that returns a version somewhere in the payload. +
+
+
+ + setField({ upstream_json_path: e.target.value })} + placeholder={t("vmLxc.appEditor.jsonPathPlaceholder")} + className="font-mono text-xs" + maxLength={128} + /> +
+ Dotted path with optional [N] array indices. +
+
+
+ )} + + {upstreamType === "docker_hub" && ( +
+ + setField({ docker_image: e.target.value })} + placeholder={t("vmLxc.appEditor.dockerImagePlaceholder")} + className="font-mono text-xs" + maxLength={255} + /> +
+ owner/name β€” or bare + name for official images. ProxMenux picks the highest semver tag + matching the filter below. +
+
+ )} + + {upstreamType && ( +
+ + setField({ tag_regex: e.target.value })} + placeholder={t("vmLxc.appEditor.tagRegexPlaceholder")} + className="font-mono text-xs" + /> +
+ {upstreamType === "github" && "Extracts the version from the release tag name."} + {upstreamType === "http_json" && "Optional β€” extract a substring from the endpoint's value."} + {upstreamType === "docker_hub" && "Filter which tags qualify (e.g. only semver). Applied before picking the highest."} +
+
+ )} + + ) + })()} +
+ )} +
+ + {error && ( +
+ + {error} +
+ )} + +
+ + +
+ + +
+ ) + } + + // Restore chip β€” same visual shell as detection chip but the + // action switches from Register/Hide to Restore. Used inside the + // Register-a-different-app panel when hidden slugs exist. + const renderRestoreChip = (d: DetectedApp) => ( +
+
+
+ {d.logo_url && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} +
+
{d.name}
+
{t("vmLxc.appEditor.hiddenBadge")}
+
+
+ +
+
+ ) + + // Uniform detection chip used in every context β€” empty state, + // post-registration "also detected" strip, and the Register-a- + // different-app panel. Actions layout responsive: + // β€’ Desktop (sm+): Register + Hide side-by-side, both labeled + // β€’ Mobile: same row, 3/4 Register (label+icon) + 1/4 Hide + // (eye icon only inside a red-translucent button) + // + // Single "Register" button covers both paths β€” with or without + // tracking hint β€” the editor opens pre-filled with whatever data + // we have, and the user can adjust in Advanced. + const renderDetectionChip = (d: DetectedApp) => ( +
+
+
+ {d.logo_url && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} +
+
{d.name}
+
{t("vmLxc.appEditor.detectedInContainer")}
+
+
+
+ + +
+
+
+ ) + + return ( +
+ {/* Browse panel β€” surfaces hidden detections with Restore + before falling through to the blank-form path. Rendered + before app cards / empty state so it takes precedence when + open. Closes automatically once all hidden slugs are + restored (nothing left to show β†’ back to normal flow). */} + {browseOpen && ( + + +
+
+

{t("vmLxc.appEditor.registerDifferent")}

+

+ {t(hiddenDetections.length === 1 ? "vmLxc.appEditor.hiddenDetectionsHelpSingular" : "vmLxc.appEditor.hiddenDetectionsHelpPlural", { count: hiddenDetections.length })} +

+
+ +
+ {hiddenDetections.length > 0 && ( +
+ {hiddenDetections.map(renderRestoreChip)} +
+ )} +
+ +
+ + + )} + + {/* Empty state β€” always uniform chip list regardless of how + many detections there are (0, 1, or many). Below the chips, + a single "Register a different app" button lets the user + add something the auto-detector doesn't know about. */} + {apps.length === 0 && ( + + +
+ +
+

+ {t("vmLxc.appEditor.noAppsTitle")} +

+

+ {t("vmLxc.appEditor.noAppsBody")} +

+ {visibleDetected.length > 0 && ( +
+ {visibleDetected.map(renderDetectionChip)} +
+ )} +
+ +
+ + + )} + + {/* App cards */} + {apps.map((app) => { + const st = app.state + // Version tracking is on when installed_via is set. Without a + // method the app is register-only β€” no cards, no warnings. + const tracking = !!app.installed_via + const hasUpdate = tracking && st?.update_available === true + const upToDate = tracking && st?.update_available === false && !!st?.installed_version + return ( + + +
+
+ {app.logo_url && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} +
+

{app.name}

+ {tracking && ( +
+ {app.installed_via === "dpkg" && app.package && <>dpkg Β· {app.package}} + {app.installed_via === "apk" && app.package && <>apk Β· {app.package}} + {app.installed_via === "file" && app.file_path && <>file Β· {app.file_path}} + {app.installed_via === "binary" && app.binary_path && <>binary Β· {app.binary_path}} +
+ )} + {tracking && st?.checked_at && ( +
+ Checked {new Date(st.checked_at).toLocaleString([], { dateStyle: "short", timeStyle: "short" })} +
+ )} + {/* Mobile-only repo link: falls into the metadata + stack below Checked, full-width so long repo + names wrap cleanly instead of competing with + the top-right on narrow screens. */} + {app.repo && tracking && ( + + + {app.repo} + + )} +
+
+ {/* Desktop-only repo link: same row as the title on md+, + hidden on mobile where the stacked variant above + handles it. */} + {app.repo && tracking && ( + + {app.repo} + + + )} +
+ + {/* Version panels β€” always same-line (grid-cols-2), on + every viewport when ANY upstream source is configured + (github repo, http_json endpoint, or docker_hub + image). The signal that an update exists is the + LATEST UPSTREAM number turning purple + a small + ArrowUpCircle next to it, matching the project-wide + update button convention. Absence of purple = up to + date; no colored banners, no green/orange noise. + Labels stay muted so numbers keep visual priority. + When no upstream is configured, the LATEST panel is + omitted entirely and INSTALLED takes the full width. */} + {(() => { + const hasUpstream = !!(app.repo || app.upstream_type) + if (!tracking || !(st?.installed_version || hasUpstream)) return null + return ( +
+ {st?.installed_version && ( +
+
{t("vmLxc.appEditor.installedStatus")}
+
+ {st.installed_version} + {upToDate && ( + + )} +
+
+ )} + {hasUpstream && ( +
+
+ Latest upstream +
+
+ {st?.latest_version || {t("vmLxc.appEditor.checkingStatus")}} + {hasUpdate && st?.latest_version && ( + + )} +
+
+ )} +
+ ) + })()} + + {tracking && st?.error && ( +
+ +
{st.error}
+
+ )} + + {/* Web links β€” one row per port. Each row: + [logo 56px] Description or app name + β†— http://IP:PORT + Logo is optional (per-port `logo_url`); when absent + the row indents naturally to align with the text. + If we can't resolve an IP for the CT we hide the row. */} + {app.ports && app.ports.length > 0 && ( +
+ {app.ports.map((p) => { + const url = buildWebUrl(ctIp, p.port, p.scheme) + if (!url) return null + const label = p.description || app.name + return ( +
+ {p.logo_url && ( + { (e.currentTarget as HTMLImageElement).style.display = "none" }} + /> + )} +
+ {label} + + + {url} + +
+
+ ) + })} +
+ )} + + {/* Footer with per-card actions β€” only rendered in the + global edit mode (toggled from the "Edit" button next + to Add another application). View mode keeps cards + chrome-free; Check is still reachable via the + hover-reveal icon on the LATEST panel. Buttons match + the Settings-page section style (h-8, outline, + small icon + label) for visual consistency across + the app. */} + {editMode && ( +
+ +
+ {tracking && ( + + )} + +
+
+ )} + + + ) + })} + + {/* Post-registration "also detected" strip β€” every hint slug + whose install signature is present on the CT AND that + hasn't been registered yet is shown as a chip with a + one-click Register button. Filtered against the sidecar's + `helper_slug` field so a registered app never re-appears. */} + {apps.length > 0 && unregisteredDetected.length > 0 && ( +
+
+ Also detected on this container +
+ {unregisteredDetected.map(renderDetectionChip)} +
+ )} + + {/* Add-more + Edit toggle. Edit is a global toggle that reveals + the per-card action footer (Remove / Check / Edit fields). + Add-more is disabled while editing so the two flows don't + overlap. Routes through the browse panel if there are hidden + detections, so the user gets one-click Restore before hand- + typing a custom app. */} + {apps.length > 0 && ( +
+ + +
+ )} + + {error && ( +
+ + {error} +
+ )} +
+ ) +} diff --git a/AppImage/components/ui/input.tsx b/AppImage/components/ui/input.tsx index 0bc8c5de..6a991a66 100644 --- a/AppImage/components/ui/input.tsx +++ b/AppImage/components/ui/input.tsx @@ -16,7 +16,7 @@ const Input = React.forwardRef(({ className, type, // 1px blue ring + matching border so a focused input now sits at the // same visual weight as the colored card selectors used elsewhere // (Backend picker, etc.). - "flex h-10 w-full rounded-lg border border-input bg-background px-4 py-2 text-sm shadow-sm transition-all file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-blue-500 focus-visible:border-blue-500 disabled:cursor-not-allowed disabled:opacity-50 hover:border-ring/50", + "flex h-10 w-full rounded-lg border border-input bg-background px-4 py-2 text-sm shadow-sm transition-all file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground placeholder:opacity-40 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-blue-500 focus-visible:border-blue-500 disabled:cursor-not-allowed disabled:opacity-50 hover:border-ring/50", className, )} ref={ref} diff --git a/AppImage/components/ui/textarea.tsx b/AppImage/components/ui/textarea.tsx index 1260ba68..4415b582 100644 --- a/AppImage/components/ui/textarea.tsx +++ b/AppImage/components/ui/textarea.tsx @@ -10,7 +10,7 @@ const Textarea = React.forwardRef( return (