mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-05 04:12:10 +00:00
Refresh OCI catalog automatically with local cache and app counters
This commit is contained in:
@@ -54,6 +54,7 @@ Thumbs.db
|
||||
|
||||
# Cache
|
||||
.cache
|
||||
/oci/.catalog-cache/
|
||||
/web/.cache
|
||||
|
||||
# IDE/editor files
|
||||
|
||||
+7
-1
@@ -8967,5 +8967,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Festplattendaten werden NICHT gelöscht.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ Die Festplatte wird ausgehängt und aus /etc/fstab entfernt.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ Der /etc/fstab-Eintrag wird entfernt.",
|
||||
"⚠ The disk will be unmounted.": "⚠ Die Festplatte wird ausgehängt."
|
||||
"⚠ The disk will be unmounted.": "⚠ Die Festplatte wird ausgehängt.",
|
||||
"Available applications: {count}": "Verfügbare Anwendungen: {count}",
|
||||
"New applications since your last visit: {count}": "Neue Anwendungen seit Ihrem letzten Besuch: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Aktualisierte Rezepte seit Ihrem letzten Besuch: {count}",
|
||||
"Refreshing the application catalog...": "Anwendungskatalog wird aktualisiert...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Der Katalog konnte nicht aktualisiert werden; der lokale Katalog wird verwendet.",
|
||||
"Refresh catalog": "Katalog aktualisieren"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8937,5 +8937,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Los datos del disco NO se borrarán.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ El disco se desmontará y se eliminará de /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ Se eliminará la entrada /etc/fstab.",
|
||||
"⚠ The disk will be unmounted.": "⚠ El disco se desmontará."
|
||||
"⚠ The disk will be unmounted.": "⚠ El disco se desmontará.",
|
||||
"Available applications: {count}": "Aplicaciones disponibles: {count}",
|
||||
"New applications since your last visit: {count}": "Aplicaciones nuevas desde tu última visita: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Recetas actualizadas desde tu última visita: {count}",
|
||||
"Refreshing the application catalog...": "Actualizando el catálogo de aplicaciones...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "No se pudo actualizar el catálogo; se está usando la copia local.",
|
||||
"Refresh catalog": "Actualizar catálogo"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8966,5 +8966,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Les données du disque ne seront PAS effacées.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ Le disque sera démonté et supprimé de /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ L'entrée /etc/fstab sera supprimée.",
|
||||
"⚠ The disk will be unmounted.": "⚠ Le disque sera démonté."
|
||||
"⚠ The disk will be unmounted.": "⚠ Le disque sera démonté.",
|
||||
"Available applications: {count}": "Applications disponibles : {count}",
|
||||
"New applications since your last visit: {count}": "Nouvelles applications depuis votre dernière visite : {count}",
|
||||
"Updated recipes since your last visit: {count}": "Recettes mises à jour depuis votre dernière visite : {count}",
|
||||
"Refreshing the application catalog...": "Mise à jour du catalogue des applications...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Impossible de mettre à jour le catalogue ; le catalogue local est utilisé.",
|
||||
"Refresh catalog": "Actualiser le catalogue"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8967,5 +8967,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ I dati del disco NON verranno cancellati.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ Il disco verrà smontato e rimosso da /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ La voce /etc/fstab verrà rimossa.",
|
||||
"⚠ The disk will be unmounted.": "⚠ Il disco verrà smontato."
|
||||
"⚠ The disk will be unmounted.": "⚠ Il disco verrà smontato.",
|
||||
"Available applications: {count}": "Applicazioni disponibili: {count}",
|
||||
"New applications since your last visit: {count}": "Nuove applicazioni dalla tua ultima visita: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Ricette aggiornate dalla tua ultima visita: {count}",
|
||||
"Refreshing the application catalog...": "Aggiornamento del catalogo delle applicazioni...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Impossibile aggiornare il catalogo; viene usata la copia locale.",
|
||||
"Refresh catalog": "Aggiorna catalogo"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8967,5 +8967,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Os dados do disco NÃO serão apagados.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ O disco será desmontado e removido de /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ A entrada /etc/fstab será removida.",
|
||||
"⚠ The disk will be unmounted.": "⚠ O disco será desmontado."
|
||||
"⚠ The disk will be unmounted.": "⚠ O disco será desmontado.",
|
||||
"Available applications: {count}": "Aplicações disponíveis: {count}",
|
||||
"New applications since your last visit: {count}": "Novas aplicações desde a sua última visita: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Receitas atualizadas desde a sua última visita: {count}",
|
||||
"Refreshing the application catalog...": "A atualizar o catálogo de aplicações...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Não foi possível atualizar o catálogo; está a ser usada a cópia local.",
|
||||
"Refresh catalog": "Atualizar catálogo"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8967,5 +8967,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Dáta na disku sa NEVYMAŽÚ.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ Disk sa odpojí a odstráni z /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ Záznam v /etc/fstab bude odstránený.",
|
||||
"⚠ The disk will be unmounted.": "⚠ Disk bude odpojený."
|
||||
"⚠ The disk will be unmounted.": "⚠ Disk bude odpojený.",
|
||||
"Available applications: {count}": "Dostupné aplikácie: {count}",
|
||||
"New applications since your last visit: {count}": "Nové aplikácie od vašej poslednej návštevy: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Aktualizované recepty od vašej poslednej návštevy: {count}",
|
||||
"Refreshing the application catalog...": "Aktualizuje sa katalóg aplikácií...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Katalóg sa nepodarilo aktualizovať; používa sa miestna kópia.",
|
||||
"Refresh catalog": "Aktualizovať katalóg"
|
||||
}
|
||||
|
||||
+7
-1
@@ -8967,5 +8967,11 @@
|
||||
"⚠ Disk data will NOT be erased.": "⚠ Diskdata kommer INTE att raderas.",
|
||||
"⚠ Disk will be unmounted and removed from /etc/fstab.": "⚠ Disken kommer att avmonteras och tas bort från /etc/fstab.",
|
||||
"⚠ The /etc/fstab entry will be removed.": "⚠ /etc/fstab-posten kommer att tas bort.",
|
||||
"⚠ The disk will be unmounted.": "⚠ Disken kommer att avmonteras."
|
||||
"⚠ The disk will be unmounted.": "⚠ Disken kommer att avmonteras.",
|
||||
"Available applications: {count}": "Tillgängliga appar: {count}",
|
||||
"New applications since your last visit: {count}": "Nya appar sedan ditt senaste besök: {count}",
|
||||
"Updated recipes since your last visit: {count}": "Uppdaterade recept sedan ditt senaste besök: {count}",
|
||||
"Refreshing the application catalog...": "Uppdaterar appkatalogen...",
|
||||
"Catalog refresh unavailable; using the local catalog.": "Katalogen kunde inte uppdateras; den lokala katalogen används.",
|
||||
"Refresh catalog": "Uppdatera katalogen"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# Automatic catalog refresh
|
||||
|
||||
Opening OCI Manager displays its local catalog immediately and starts a separate
|
||||
worker. The menu shows the available total, additions since the last displayed
|
||||
catalog, and changed recipes. Returning from a search or installation does not
|
||||
start another download. A running wizard keeps the catalog it started with.
|
||||
The menu also offers **Refresh catalog** to check again manually or retry after
|
||||
a connection failure. Selecting it during a refresh does not start a second worker.
|
||||
|
||||
The worker checks the latest commit affecting `oci/catalog` in the official
|
||||
MacRimi/ProxMenux repository. GitHub's ETag avoids repeated transfers. When the
|
||||
revision changes, its immutable Git tree supplies the file manifest. Only changed
|
||||
JSON files are downloaded from that commit; Git blob hashes and sizes are checked.
|
||||
The complete snapshot, including overlays, categories, volume policy and
|
||||
verification records, is validated before publishing its pointer atomically.
|
||||
Runtime Python or shell files are never downloaded by catalog refresh.
|
||||
|
||||
State and snapshots live in `/usr/local/share/proxmenux/oci/catalog-cache`, outside
|
||||
the engine replaced by the normal ProxMenux updater. The bundled catalog remains
|
||||
the fallback. A network failure, incomplete manifest, invalid recipe, different
|
||||
template schema or unsupported catalog engine API leaves the previous catalog
|
||||
active. The private `CHANNEL-result.json` records the diagnostic.
|
||||
|
||||
The channel follows `install_branch`, an active beta program or `beta_version.txt`;
|
||||
otherwise it is `main`. A host can explicitly select `main` or `develop` with
|
||||
`/usr/local/share/proxmenux/oci/catalog-source.json`:
|
||||
|
||||
```json
|
||||
{"channel": "develop", "local_apps": ["my-local-preview"]}
|
||||
```
|
||||
|
||||
`local_apps` optionally keeps explicitly named bundled preview recipes available
|
||||
when they are absent upstream. These are local overrides, not automatic retention
|
||||
of withdrawn upstream applications. An upstream recipe with the same ID wins.
|
||||
|
||||
The first visit uses the bundled/current catalog as its baseline. Additions are
|
||||
compared by application ID, not net growth, and only visible installable entries
|
||||
are counted, using the same rule as the existing menu. A downloaded catalog is
|
||||
acknowledged only after the menu displays it. Exiting during a download therefore
|
||||
preserves its news for the next visit. The new-app count remains visible throughout
|
||||
the current visit and becomes zero on the next visit after acknowledgement.
|
||||
|
||||
Catalog maintainers must increase `catalog/compatibility.json`'s `engine_api` when
|
||||
recipes require incompatible runtime capabilities. The consumer currently
|
||||
supports API 1. Changes to the template schema also require a ProxMenux engine
|
||||
update. Catalog delivery alone neither changes installed CTs nor applies a new
|
||||
recipe to an existing deployment; the established management actions remain in
|
||||
charge of those changes.
|
||||
@@ -1117,3 +1117,9 @@ have not been tested. No custom supervisor or image entrypoint is introduced.
|
||||
DeepSeek OCR remains deferred at the user's request: registry inspection on
|
||||
2026-09-14 found only `v2.2.0` for both IceWhaleTech images and no `latest` tag.
|
||||
No fixed-version exception or deployment has been introduced.
|
||||
|
||||
## Automatic catalog refresh
|
||||
|
||||
OCI Manager opens with its local catalog and checks for new applications and recipes
|
||||
in the background. **Refresh catalog** repeats the check manually. See
|
||||
[Automatic catalog refresh](CATALOG-REFRESH.md) for caching, channels and compatibility.
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"engine_api": 1,
|
||||
"description": "Increment engine_api when catalog recipes require runtime capabilities unavailable in earlier catalog-refresh clients."
|
||||
}
|
||||
@@ -56,9 +56,13 @@ def is_multi_container(template: dict[str, Any]) -> bool:
|
||||
|
||||
|
||||
class Catalog:
|
||||
def __init__(self, root: Path, source: GitHubSource | None = None) -> None:
|
||||
def __init__(self, root: Path, source: GitHubSource | None = None, *,
|
||||
catalog_dir: Path | None = None) -> None:
|
||||
self.root = root
|
||||
self.catalog_dir = root / "catalog"
|
||||
from .catalog_refresh import current_catalog
|
||||
self.catalog_dir = catalog_dir if catalog_dir is not None else current_catalog(root)
|
||||
self._include_local = catalog_dir is None and self.catalog_dir != root / "catalog"
|
||||
self.local_app_ids: set[str] = set()
|
||||
self.apps_dir = self.catalog_dir / "apps"
|
||||
self.curated_dir = self.catalog_dir / "curated"
|
||||
self.overlays_dir = self.catalog_dir / "overlays"
|
||||
@@ -345,6 +349,16 @@ class Catalog:
|
||||
payload = json.loads(self.index_path.read_text(encoding="utf-8"))
|
||||
self._enrich_index_from_templates(payload)
|
||||
self._apply_verification_to_index(payload)
|
||||
if self._include_local:
|
||||
from .catalog_refresh import source_settings
|
||||
requested = source_settings(self.root).get("local_apps", [])
|
||||
if isinstance(requested, list) and requested:
|
||||
bundled = Catalog(self.root, catalog_dir=self.root / "catalog").load_index()
|
||||
present = {item["id"] for item in payload["applications"]}
|
||||
local = [item for item in bundled["applications"]
|
||||
if item["id"] in requested and item["id"] not in present]
|
||||
self.local_app_ids = {item["id"] for item in local}
|
||||
payload["applications"].extend(local)
|
||||
return payload
|
||||
|
||||
def categories(self) -> dict[str, Any]:
|
||||
@@ -588,7 +602,7 @@ class Catalog:
|
||||
item["id"],
|
||||
)
|
||||
elif item.get("template_family") == "curated-profile":
|
||||
template = json.loads((self.root / item["curated_path"]).read_text(encoding="utf-8"))
|
||||
template = json.loads((self._curated_path(item)).read_text(encoding="utf-8"))
|
||||
else:
|
||||
raise ConversionError(f"Proveedor no soportado: {provider}")
|
||||
catalog_id = item["id"]
|
||||
@@ -641,7 +655,7 @@ class Catalog:
|
||||
)
|
||||
elif item.get("template_family") == "curated-profile":
|
||||
template = json.loads(
|
||||
(self.root / item["curated_path"]).read_text(encoding="utf-8")
|
||||
(self._curated_path(item)).read_text(encoding="utf-8")
|
||||
)
|
||||
else:
|
||||
raise ConversionError(f"Proveedor no soportado: {item_provider}")
|
||||
@@ -785,13 +799,27 @@ class Catalog:
|
||||
if errors:
|
||||
raise ConversionError(f"La plantilla generada no cumple el esquema: {'; '.join(errors)}")
|
||||
|
||||
def _curated_path(self, item: dict[str, Any]) -> Path:
|
||||
path = Path(item["curated_path"])
|
||||
if path.parts and path.parts[0] == "catalog":
|
||||
return self.catalog_dir / path.relative_to("catalog")
|
||||
# Older local profiles may use an engine-relative path. Downloaded
|
||||
# catalogs are checked separately and must use catalog/curated paths.
|
||||
return self.root / path
|
||||
|
||||
def compose(self, app_id: str) -> dict[str, Any]:
|
||||
"""The installable template, built only from the files shipped with
|
||||
ProxMenux: the curated profile or the pre-generated template, with its
|
||||
overlay applied. Nothing is downloaded and nothing is written."""
|
||||
item = self.find_item(app_id)
|
||||
return self._compose_item(self.find_item(app_id))
|
||||
|
||||
def _compose_item(self, item: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Compose an already loaded item when validating a complete snapshot."""
|
||||
app_id = item["id"]
|
||||
if app_id in self.local_app_ids:
|
||||
return Catalog(self.root, catalog_dir=self.root / "catalog").compose(app_id)
|
||||
if item.get("template_family") == "curated-profile":
|
||||
path = self.root / item["curated_path"]
|
||||
path = self._curated_path(item)
|
||||
else:
|
||||
path = self.apps_dir / f"{item['id']}.json"
|
||||
if not path.exists():
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Refresh the reviewed ProxMenux catalog without blocking the OCI menu.
|
||||
|
||||
GitHub's catalog commit and tree form the manifest. Files are fetched from
|
||||
that immutable commit and checked against their Git blob IDs. Runtime code is
|
||||
never downloaded. An installed engine always retains its bundled fallback.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .i18n import translate
|
||||
|
||||
REPOSITORY = "MacRimi/ProxMenux"
|
||||
CATALOG_API = 1
|
||||
MAX_FILE = 4 * 1024 * 1024
|
||||
MAX_TOTAL = 40 * 1024 * 1024
|
||||
SHA = re.compile(r"[0-9a-f]{40}")
|
||||
ROOT_FILES = {"index.json", "categories.json", "volume-policy.json",
|
||||
"exclusions.json", "verification.json", "compatibility.json"}
|
||||
|
||||
|
||||
def source_settings(root):
|
||||
root = Path(root)
|
||||
base = (root.parent.parent if root.name == "engine" else
|
||||
root.parent if root.name == "oci" else root)
|
||||
return read_json(base / "oci/catalog-source.json")
|
||||
|
||||
|
||||
def channel_for(root):
|
||||
"""Keep stable and beta catalogs separate; allow a local channel choice."""
|
||||
root = Path(root)
|
||||
base = (root.parent.parent if root.name == "engine" else
|
||||
root.parent if root.name == "oci" else root)
|
||||
for name in ("oci/catalog-source.json", "config.json"):
|
||||
try:
|
||||
value = json.loads((base / name).read_text())
|
||||
channel = value.get("channel") or value.get("install_branch")
|
||||
if channel in ("main", "develop"):
|
||||
return channel
|
||||
if (value.get("beta_program") or {}).get("status") == "active":
|
||||
return "develop"
|
||||
except (OSError, ValueError, TypeError):
|
||||
pass
|
||||
return "develop" if (base / "beta_version.txt").exists() else "main"
|
||||
|
||||
|
||||
def cache_root(root):
|
||||
root = Path(root)
|
||||
return root.parent / "catalog-cache" if root.name == "engine" else root / ".catalog-cache"
|
||||
|
||||
|
||||
def read_json(path, default=None):
|
||||
try:
|
||||
value = json.loads(Path(path).read_text())
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return {} if default is None else default
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, name = tempfile.mkstemp(prefix=".write-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump(value, stream, ensure_ascii=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(name, path)
|
||||
finally:
|
||||
Path(name).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def schema_hash(root):
|
||||
return hashlib.sha256((Path(root) / "schemas/oci-template.schema.json").read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def current_catalog(root, channel=None):
|
||||
root = Path(root)
|
||||
channel = channel or channel_for(root)
|
||||
value = read_json(cache_root(root) / f"{channel}-current.json")
|
||||
revision = value.get("revision", "")
|
||||
if (isinstance(revision, str) and SHA.fullmatch(revision)
|
||||
and value.get("schema_hash") == schema_hash(root)
|
||||
and value.get("engine_api") == CATALOG_API):
|
||||
path = cache_root(root) / "snapshots" / revision
|
||||
index = read_json(path / "index.json")
|
||||
if (index.get("kind") == "proxmenux.oci-catalog-index"
|
||||
and isinstance(index.get("applications"), list) and index["applications"]):
|
||||
return path
|
||||
return root / "catalog"
|
||||
|
||||
|
||||
def git_blob(data):
|
||||
return hashlib.sha1(b"blob " + str(len(data)).encode() + b"\0" + data).hexdigest()
|
||||
|
||||
|
||||
def request(url, etag=None, limit=MAX_FILE):
|
||||
headers = {"User-Agent": "ProxMenux-OCI-Catalog", "Accept": "application/vnd.github+json"}
|
||||
if etag:
|
||||
headers["If-None-Match"] = etag
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=8) as response:
|
||||
body = response.read(limit + 1)
|
||||
if len(body) > limit:
|
||||
raise ValueError("Catalog response exceeds the size limit")
|
||||
return body, response.headers.get("ETag")
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 304:
|
||||
return None, etag
|
||||
raise
|
||||
|
||||
|
||||
def catalog_path(name):
|
||||
path = PurePosixPath(name)
|
||||
if (str(path) != name or path.is_absolute() or ".." in path.parts
|
||||
or path.suffix != ".json"):
|
||||
raise ValueError("Invalid catalog file path")
|
||||
if name not in ROOT_FILES and not (len(path.parts) == 2 and path.parts[0] in
|
||||
("apps", "curated", "overlays")):
|
||||
raise ValueError("Unsupported catalog file path")
|
||||
return path
|
||||
|
||||
|
||||
def validate_snapshot(root, directory):
|
||||
from .catalog import Catalog
|
||||
from jsonschema import Draft202012Validator
|
||||
compatibility = (json.loads((directory / "compatibility.json").read_text())
|
||||
if (directory / "compatibility.json").exists() else {"engine_api": 1})
|
||||
if compatibility.get("engine_api") != CATALOG_API:
|
||||
raise ValueError("This catalog requires a newer OCI engine")
|
||||
validator = Draft202012Validator(json.loads((Path(root) / "schemas/oci-template.schema.json").read_text()))
|
||||
for folder in ("apps", "curated"):
|
||||
for file in (directory / folder).glob("*.json"):
|
||||
validator.validate(json.loads(file.read_text()))
|
||||
index = json.loads((directory / "index.json").read_text())
|
||||
if index.get("kind") != "proxmenux.oci-catalog-index":
|
||||
raise ValueError("Invalid catalog index")
|
||||
ids = set()
|
||||
for item in index.get("applications", []):
|
||||
app_id = item.get("id", "")
|
||||
if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]*", app_id) or app_id in ids:
|
||||
raise ValueError("Invalid or duplicate catalog application")
|
||||
ids.add(app_id)
|
||||
if item.get("template_family") == "curated-profile":
|
||||
curated = item.get("curated_path", "")
|
||||
if not curated.startswith("catalog/"):
|
||||
raise ValueError("Invalid curated recipe path")
|
||||
catalog_path(curated.removeprefix("catalog/"))
|
||||
if not ids:
|
||||
raise ValueError("The remote catalog is empty")
|
||||
catalog = Catalog(Path(root), catalog_dir=directory)
|
||||
for item in catalog.load_index()["applications"]:
|
||||
if not item.get("hidden") and item.get("automatic_install_candidate"):
|
||||
validator.validate(catalog._compose_item(item))
|
||||
|
||||
|
||||
def synchronize(root, channel, fetch=request):
|
||||
"""Download changed JSON files into a private snapshot, then publish once."""
|
||||
root = Path(root)
|
||||
cache = cache_root(root)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
metadata_path = cache / f"{channel}-remote.json"
|
||||
previous = read_json(metadata_path)
|
||||
active = current_catalog(root, channel)
|
||||
etag = previous.get("etag") if active != root / "catalog" else None
|
||||
url = f"https://api.github.com/repos/{REPOSITORY}/commits?sha={channel}&path=oci/catalog&per_page=1"
|
||||
body, etag = fetch(url, etag)
|
||||
if body is None:
|
||||
return False
|
||||
commits = json.loads(body)
|
||||
revision = commits[0]["sha"]
|
||||
if not isinstance(revision, str) or not SHA.fullmatch(revision):
|
||||
raise ValueError("Invalid catalog revision")
|
||||
if active.name == revision:
|
||||
write_json(metadata_path, {"etag": etag})
|
||||
return False
|
||||
body, _ = fetch(f"https://api.github.com/repos/{REPOSITORY}/git/trees/{revision}?recursive=1")
|
||||
tree = json.loads(body)
|
||||
if tree.get("truncated"):
|
||||
raise ValueError("GitHub returned an incomplete catalog manifest")
|
||||
entries = {}
|
||||
schema = None
|
||||
for entry in tree["tree"]:
|
||||
name = entry["path"]
|
||||
if name == "oci/schemas/oci-template.schema.json":
|
||||
schema = entry["sha"]
|
||||
if not name.startswith("oci/catalog/") or entry.get("type") != "blob":
|
||||
continue
|
||||
relative = name.removeprefix("oci/catalog/")
|
||||
if relative.startswith("generation-report"):
|
||||
continue
|
||||
catalog_path(relative)
|
||||
if entry.get("mode") != "100644" or not SHA.fullmatch(entry.get("sha", "")):
|
||||
raise ValueError("Invalid catalog manifest entry")
|
||||
if type(entry.get("size")) is not int or not 0 <= entry["size"] <= MAX_FILE:
|
||||
raise ValueError("Invalid catalog file size")
|
||||
entries[relative] = entry
|
||||
if schema != git_blob((root / "schemas/oci-template.schema.json").read_bytes()):
|
||||
raise ValueError("Catalog schema changed; update ProxMenux first")
|
||||
if not entries or len(entries) > 3000 or sum(e["size"] for e in entries.values()) > MAX_TOTAL:
|
||||
raise ValueError("Catalog exceeds the size limit")
|
||||
if not {"index.json", "categories.json", "volume-policy.json", "verification.json"} <= entries.keys():
|
||||
raise ValueError("Catalog manifest is incomplete")
|
||||
snapshots = cache / "snapshots"
|
||||
snapshots.mkdir(exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=".download-", dir=snapshots) as name:
|
||||
staging = Path(name)
|
||||
|
||||
def download(pair):
|
||||
relative, entry = pair
|
||||
old = active / relative
|
||||
data = old.read_bytes() if old.is_file() else b""
|
||||
if git_blob(data) != entry["sha"]:
|
||||
data, _ = fetch(f"https://raw.githubusercontent.com/{REPOSITORY}/{revision}/oci/catalog/{relative}")
|
||||
if data is None or len(data) != entry["size"] or git_blob(data) != entry["sha"]:
|
||||
raise ValueError("Catalog file failed its integrity check")
|
||||
json.loads(data)
|
||||
target = staging / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(data)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
list(pool.map(download, entries.items()))
|
||||
validate_snapshot(root, staging)
|
||||
destination = snapshots / revision
|
||||
if destination.exists():
|
||||
# Revalidate an existing immutable snapshot after an interrupted publication.
|
||||
validate_snapshot(root, destination)
|
||||
else:
|
||||
os.replace(staging, destination)
|
||||
write_json(cache / f"{channel}-current.json", {"revision": revision,
|
||||
"schema_hash": schema_hash(root), "engine_api": CATALOG_API})
|
||||
write_json(metadata_path, {"etag": etag})
|
||||
return True
|
||||
|
||||
|
||||
def worker(root, channel):
|
||||
cache = cache_root(root)
|
||||
cache.mkdir(parents=True, exist_ok=True)
|
||||
with (cache / "refresh.lock").open("a") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
try:
|
||||
changed = synchronize(root, channel)
|
||||
write_json(cache / f"{channel}-result.json", {"ok": True, "changed": changed})
|
||||
except Exception as error:
|
||||
write_json(cache / f"{channel}-result.json", {"ok": False, "error": str(error)[:400]})
|
||||
|
||||
|
||||
def visible_recipes(catalog):
|
||||
"""Same availability rule as the menu; count new IDs, not net growth."""
|
||||
result = {}
|
||||
policies = {}
|
||||
for item in catalog.load_index()["applications"]:
|
||||
if item.get("hidden") or not item.get("automatic_install_candidate"):
|
||||
continue
|
||||
directory = catalog.root / "catalog" if item["id"] in catalog.local_app_ids else catalog.catalog_dir
|
||||
if item.get("template_family") == "curated-profile":
|
||||
path = directory / item["curated_path"].removeprefix("catalog/")
|
||||
else:
|
||||
path = directory / "apps" / f"{item['id']}.json"
|
||||
overlay = directory / "overlays" / f"{item['id']}.json"
|
||||
recipe = json.loads(path.read_text()) if path.exists() else {}
|
||||
# Formatting and regenerated source timestamps are not recipe changes.
|
||||
recipe.pop("source", None)
|
||||
recipe.pop("validation", None)
|
||||
correction = json.loads(overlay.read_text()) if overlay.exists() else {}
|
||||
if directory not in policies:
|
||||
policies[directory] = read_json(directory / "volume-policy.json")
|
||||
policy = policies[directory]
|
||||
paths = {volume.get("container_path") for volume in recipe.get("container_contract", {}).get("volumes", [])}
|
||||
relevant = {"shared_paths": sorted(paths & set(policy.get("shared_paths", []))),
|
||||
"application": policy.get("applications", {}).get(item["id"], []),
|
||||
"sizes": policy.get("sizes", {}).get(item["id"], {})}
|
||||
data = json.dumps([recipe, correction, relevant, item.get("category"), item.get("category_label")],
|
||||
sort_keys=True, separators=(",", ":")).encode()
|
||||
result[item["id"]] = hashlib.sha256(data).hexdigest()
|
||||
return result
|
||||
|
||||
|
||||
class CatalogRefresh:
|
||||
def __init__(self, root, catalog):
|
||||
self.root = Path(root)
|
||||
self.channel = channel_for(root)
|
||||
self.seen_path = cache_root(root) / f"{self.channel}-seen.json"
|
||||
self.recipes = visible_recipes(catalog)
|
||||
baseline = read_json(self.seen_path).get("recipes")
|
||||
self.baseline = baseline if isinstance(baseline, dict) else self.recipes.copy()
|
||||
self.process = None
|
||||
self.failed = False
|
||||
self._finished = False
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def start(self):
|
||||
if self.active:
|
||||
return False
|
||||
self._finished = False
|
||||
self.failed = False
|
||||
environment = dict(os.environ, PYTHONPATH=str(self.root / "src"))
|
||||
try:
|
||||
self.process = subprocess.Popen([sys.executable, "-m", __name__, str(self.root), self.channel],
|
||||
env=environment, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True)
|
||||
except OSError:
|
||||
self.failed = True
|
||||
self._finished = True
|
||||
return False
|
||||
return True
|
||||
|
||||
def __exit__(self, *args):
|
||||
# The worker can finish downloading after the user leaves. Its result
|
||||
# is not marked seen until a later menu actually displays that catalog.
|
||||
if self.process and self.process.poll() is not None:
|
||||
self.process.wait()
|
||||
elif self.process:
|
||||
threading.Thread(target=self.process.wait, daemon=True).start()
|
||||
|
||||
@property
|
||||
def active(self):
|
||||
return self.process is not None and not self._finished
|
||||
|
||||
def changed(self):
|
||||
return self.active and self.process.poll() is not None
|
||||
|
||||
def consume(self):
|
||||
if not self.changed():
|
||||
return False
|
||||
self._finished = True
|
||||
result = read_json(cache_root(self.root) / f"{self.channel}-result.json")
|
||||
self.failed = self.process.returncode != 0 or not result.get("ok")
|
||||
return not self.failed and bool(result.get("changed"))
|
||||
|
||||
def describe(self):
|
||||
text = translate("Available applications: {count}").format(count=len(self.recipes))
|
||||
added = len(self.recipes.keys() - self.baseline.keys())
|
||||
updated = sum(self.recipes[key] != self.baseline[key] for key in self.recipes.keys() & self.baseline.keys())
|
||||
text += "\n" + translate("New applications since your last visit: {count}").format(count=added)
|
||||
if updated:
|
||||
text += "\n" + translate("Updated recipes since your last visit: {count}").format(count=updated)
|
||||
if self.active:
|
||||
text += "\n" + translate("Refreshing the application catalog...")
|
||||
elif self.failed:
|
||||
text += "\n" + translate("Catalog refresh unavailable; using the local catalog.")
|
||||
return text
|
||||
|
||||
def mark_seen(self):
|
||||
try:
|
||||
write_json(self.seen_path, {"recipes": self.recipes})
|
||||
except OSError:
|
||||
self.failed = True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("root", type=Path)
|
||||
parser.add_argument("channel", choices=("main", "develop"))
|
||||
args = parser.parse_args()
|
||||
worker(args.root, args.channel)
|
||||
@@ -557,13 +557,7 @@ def _search(catalog: Catalog, ui, applications: list[dict[str, Any]]) -> None:
|
||||
keep_order=True)
|
||||
|
||||
|
||||
def interactive(catalog: Catalog) -> int:
|
||||
from .ui import MenuRefresh
|
||||
from .updates import UpdateCheck, select_updates
|
||||
ui = interactive_ui()
|
||||
if not catalog.index_path.exists():
|
||||
ui.message(translate("The OCI catalog is not installed. Update ProxMenux and try again."))
|
||||
return 1
|
||||
def _catalog_options(catalog):
|
||||
applications = _installable(catalog.load_index()["applications"])
|
||||
categories: dict[str, list[dict[str, Any]]] = {}
|
||||
labels: dict[str, str] = {}
|
||||
@@ -577,17 +571,34 @@ def interactive(catalog: Catalog) -> int:
|
||||
("search", translate("Search applications")),
|
||||
("all", f"{translate('All applications'):<35} ({len(applications):>3})"),
|
||||
("manage", translate("Manage installed OCI applications")),
|
||||
("catalog-refresh", translate("Refresh catalog")),
|
||||
("registries", translate("Registry accounts")),
|
||||
("custom", translate("Install an image that is not in the catalog")),
|
||||
("", ""),
|
||||
] + [(number, f"{translate(labels[key]):<35} ({len(categories[key]):>3})")
|
||||
for number, key in category_by_index.items()]
|
||||
return applications, categories, labels, category_by_index, options
|
||||
|
||||
|
||||
def interactive(catalog: Catalog) -> int:
|
||||
from .ui import MenuRefresh
|
||||
from .catalog_refresh import CatalogRefresh, visible_recipes
|
||||
from .updates import UpdateCheck, select_updates
|
||||
ui = interactive_ui()
|
||||
if not catalog.index_path.exists():
|
||||
ui.message(translate("The OCI catalog is not installed. Update ProxMenux and try again."))
|
||||
return 1
|
||||
applications, categories, labels, category_by_index, options = _catalog_options(catalog)
|
||||
selection = "search"
|
||||
with UpdateCheck(PROJECT_ROOT) as updates:
|
||||
with CatalogRefresh(PROJECT_ROOT, catalog) as catalog_refresh, UpdateCheck(PROJECT_ROOT) as updates:
|
||||
while True:
|
||||
if catalog_refresh.consume():
|
||||
catalog = Catalog(PROJECT_ROOT)
|
||||
applications, categories, labels, category_by_index, options = _catalog_options(catalog)
|
||||
catalog_refresh.recipes = visible_recipes(catalog)
|
||||
state = updates.status()
|
||||
entries = ReadableTags(options[:3] + [('updates', updates.label(state))] + options[3:])
|
||||
text = translate("Select a category or search for applications:")
|
||||
text = catalog_refresh.describe() + "\n\n" + translate("Select a category or search for applications:")
|
||||
if notice := updates.notice(state):
|
||||
text += '\n\n' + notice
|
||||
if updates.active and not getattr(ui, 'live_refresh', False):
|
||||
@@ -595,10 +606,13 @@ def interactive(catalog: Catalog) -> int:
|
||||
updates.wait()
|
||||
continue
|
||||
try:
|
||||
kwargs = {'refresh': updates.changed} if updates.active else {}
|
||||
kwargs = ({'refresh': lambda: updates.changed() or catalog_refresh.changed()}
|
||||
if updates.active or catalog_refresh.active else {})
|
||||
selected = ui.choose(text, entries, selection, size=MENU_SIZE, **kwargs)
|
||||
except MenuRefresh:
|
||||
continue
|
||||
finally:
|
||||
catalog_refresh.mark_seen()
|
||||
if selected is None:
|
||||
return 0
|
||||
selection = selected
|
||||
@@ -611,6 +625,8 @@ def interactive(catalog: Catalog) -> int:
|
||||
elif selection == "manage":
|
||||
from .management import interactive_management
|
||||
interactive_management(PROJECT_ROOT, ui)
|
||||
elif selection == "catalog-refresh":
|
||||
catalog_refresh.start()
|
||||
elif selection == "updates":
|
||||
if not updates.active:
|
||||
if state['available']:
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
"""Catalog delivery is atomic, offline-safe, and never acknowledges unseen apps."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
from proxmenux_oci.catalog import Catalog
|
||||
from proxmenux_oci import catalog_refresh as refresh
|
||||
|
||||
|
||||
class CatalogRefreshTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name) / "oci" / "engine"
|
||||
self.root.mkdir(parents=True)
|
||||
shutil.copytree(ROOT / "schemas", self.root / "schemas")
|
||||
index = json.loads((ROOT / "catalog/index.json").read_text())
|
||||
self.item = next(x for x in index["applications"]
|
||||
if x.get("template_family") == "curated-profile"
|
||||
and x.get("automatic_install_candidate") and not x.get("hidden"))
|
||||
self.files = {}
|
||||
for name in ("categories.json", "volume-policy.json", "exclusions.json", "verification.json"):
|
||||
self.files[name] = (ROOT / "catalog" / name).read_bytes()
|
||||
for name in (f'apps/{self.item["id"]}.json', self.item['curated_path'].removeprefix("catalog/")):
|
||||
self.files[name] = (ROOT / "catalog" / name).read_bytes()
|
||||
self.files["index.json"] = json.dumps({"kind": "proxmenux.oci-catalog-index",
|
||||
"applications": [self.item]}).encode()
|
||||
for name, data in self.files.items():
|
||||
file = self.root / "catalog" / name
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_bytes(data)
|
||||
self.revision = "a" * 40
|
||||
self.requests = []
|
||||
|
||||
def fetch(self, url, etag=None):
|
||||
self.requests.append((url, etag))
|
||||
if "/commits?" in url:
|
||||
return json.dumps([{"sha": self.revision}]).encode(), '"catalog-etag"'
|
||||
if "/git/trees/" in url:
|
||||
entries = [{"path": "oci/catalog/" + name, "type": "blob", "mode": "100644",
|
||||
"sha": refresh.git_blob(data), "size": len(data)}
|
||||
for name, data in self.files.items()]
|
||||
schema = (self.root / "schemas/oci-template.schema.json").read_bytes()
|
||||
entries.append({"path": "oci/schemas/oci-template.schema.json", "type": "blob",
|
||||
"sha": refresh.git_blob(schema)})
|
||||
return json.dumps({"tree": entries, "truncated": False}).encode(), None
|
||||
name = url.split("/oci/catalog/", 1)[1]
|
||||
return self.files[name], None
|
||||
|
||||
def sync(self):
|
||||
return refresh.synchronize(self.root, "main", self.fetch)
|
||||
|
||||
def add_app(self):
|
||||
item = dict(self.item, id="sample-added", template_id="image-sample-added",
|
||||
curated_path="catalog/curated/sample-added.json", template="apps/sample-added.json")
|
||||
recipe = json.loads(self.files[self.item["curated_path"].removeprefix("catalog/")])
|
||||
recipe["id"] = item["template_id"]
|
||||
for name in ("apps/sample-added.json", "curated/sample-added.json"):
|
||||
self.files[name] = json.dumps(recipe).encode()
|
||||
index = json.loads(self.files["index.json"])
|
||||
index["applications"].append(item)
|
||||
self.files["index.json"] = json.dumps(index).encode()
|
||||
|
||||
def test_complete_catalog_is_selected_atomically_without_redownloading_unchanged_files(self):
|
||||
self.assertTrue(self.sync())
|
||||
directory = refresh.current_catalog(self.root, "main")
|
||||
self.assertEqual(directory.name, self.revision)
|
||||
self.assertEqual(Catalog(self.root).compose(self.item["id"])["id"], self.item["template_id"])
|
||||
self.assertFalse(any("raw.githubusercontent" in url for url, _ in self.requests))
|
||||
self.assertTrue((self.root / "catalog/index.json").exists())
|
||||
|
||||
def test_changed_file_uses_immutable_commit_and_updates_recipe(self):
|
||||
name = self.item["curated_path"].removeprefix("catalog/")
|
||||
recipe = json.loads(self.files[name]); recipe["catalog_ui"]["tips"].append("New setup tip")
|
||||
self.files[name] = json.dumps(recipe).encode()
|
||||
self.sync()
|
||||
self.assertTrue(any(f"/{self.revision}/oci/catalog/{name}" in u for u, _ in self.requests))
|
||||
self.assertIn("New setup tip", Catalog(self.root).compose(self.item["id"])["catalog_ui"]["tips"])
|
||||
|
||||
def test_unchanged_revision_does_not_fetch_tree_or_recipes(self):
|
||||
self.sync(); self.requests.clear()
|
||||
self.assertFalse(self.sync())
|
||||
self.assertEqual(len(self.requests), 1)
|
||||
self.assertEqual(self.requests[0][1], '"catalog-etag"')
|
||||
|
||||
def test_not_modified_uses_cached_snapshot(self):
|
||||
self.sync()
|
||||
fetch = Mock(return_value=(None, '"catalog-etag"'))
|
||||
self.assertFalse(refresh.synchronize(self.root, "main", fetch))
|
||||
self.assertEqual(fetch.call_count, 1)
|
||||
|
||||
def test_formatting_and_unrelated_categories_do_not_count_as_recipe_updates(self):
|
||||
before = refresh.visible_recipes(Catalog(self.root))
|
||||
for name in self.files:
|
||||
path = self.root / 'catalog' / name
|
||||
value = json.loads(path.read_text())
|
||||
if name == 'categories.json':
|
||||
value['applications']['unrelated-app'] = 'misc'
|
||||
path.write_text(json.dumps(value, indent=4) + '\n')
|
||||
self.assertEqual(refresh.visible_recipes(Catalog(self.root)), before)
|
||||
|
||||
def test_failure_keeps_previous_catalog_and_does_not_commit_remote_etag(self):
|
||||
self.sync()
|
||||
old = refresh.current_catalog(self.root)
|
||||
self.revision = "b" * 40
|
||||
self.files[self.item["curated_path"].removeprefix("catalog/")] = b'{}'
|
||||
with self.assertRaises(Exception):
|
||||
self.sync()
|
||||
self.assertEqual(refresh.current_catalog(self.root), old)
|
||||
self.assertFalse((refresh.cache_root(self.root) / "snapshots" / self.revision).exists())
|
||||
|
||||
def test_missing_recipe_cannot_publish_index_alone(self):
|
||||
self.files.pop(self.item["curated_path"].removeprefix("catalog/"))
|
||||
with self.assertRaises(Exception):
|
||||
self.sync()
|
||||
self.assertEqual(refresh.current_catalog(self.root), self.root / "catalog")
|
||||
|
||||
def test_hash_mismatch_is_rejected(self):
|
||||
real = self.fetch
|
||||
def corrupt(url, etag=None):
|
||||
body, tag = real(url, etag)
|
||||
if "/git/trees/" in url:
|
||||
tree = json.loads(body)
|
||||
tree["tree"][0]["sha"] = "f" * 40
|
||||
body = json.dumps(tree).encode()
|
||||
return body, tag
|
||||
with self.assertRaisesRegex(ValueError, "integrity"):
|
||||
refresh.synchronize(self.root, "main", corrupt)
|
||||
self.assertEqual(refresh.current_catalog(self.root), self.root / "catalog")
|
||||
|
||||
def test_schema_mismatch_requires_engine_update(self):
|
||||
real = self.fetch
|
||||
def incompatible(url, etag=None):
|
||||
body, tag = real(url, etag)
|
||||
if "/git/trees/" in url:
|
||||
tree = json.loads(body); tree["tree"][-1]["sha"] = "f" * 40
|
||||
body = json.dumps(tree).encode()
|
||||
return body, tag
|
||||
with self.assertRaisesRegex(ValueError, "schema changed"):
|
||||
refresh.synchronize(self.root, "main", incompatible)
|
||||
|
||||
def test_new_engine_api_requires_engine_update(self):
|
||||
self.files["compatibility.json"] = b'{"engine_api": 999}'
|
||||
with self.assertRaisesRegex(ValueError, "newer OCI engine"):
|
||||
self.sync()
|
||||
|
||||
def test_paths_and_symlink_entries_are_rejected(self):
|
||||
for name in ("../index.json", "/index.json", "apps/../../secret.json", "remote/install.sh"):
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
refresh.catalog_path(name)
|
||||
|
||||
def test_offline_worker_keeps_local_catalog(self):
|
||||
with patch.object(refresh, "synchronize", side_effect=OSError("Network unavailable")):
|
||||
refresh.worker(self.root, "main")
|
||||
self.assertFalse(refresh.read_json(refresh.cache_root(self.root) / "main-result.json")["ok"])
|
||||
self.assertEqual(refresh.current_catalog(self.root), self.root / "catalog")
|
||||
|
||||
def test_first_visit_has_zero_new_apps_and_second_visit_clears_acknowledged_news(self):
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
self.assertIn("New applications since your last visit: 0", status.describe())
|
||||
status.recipes["new-app"] = "new-recipe-hash"
|
||||
self.assertIn("New applications since your last visit: 1", status.describe())
|
||||
status.mark_seen()
|
||||
same = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
same.recipes = status.recipes.copy()
|
||||
self.assertIn("New applications since your last visit: 0", same.describe())
|
||||
|
||||
def test_new_app_count_is_not_net_growth_and_modified_recipes_are_separate(self):
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
original = self.item["id"]
|
||||
status.baseline = {original: "old-recipe", "removed-app": "hash"}
|
||||
status.recipes["new-app"] = "hash"
|
||||
self.assertIn("Available applications: 2", status.describe())
|
||||
self.assertIn("New applications since your last visit: 1", status.describe())
|
||||
self.assertIn("Updated recipes since your last visit: 1", status.describe())
|
||||
|
||||
def test_exit_before_download_finishes_does_not_acknowledge_remote_apps(self):
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
status.mark_seen()
|
||||
status.process = Mock(); status.process.poll.return_value = None
|
||||
with patch.object(refresh.threading, "Thread"):
|
||||
status.__exit__(None, None, None)
|
||||
status.process.terminate.assert_not_called()
|
||||
self.add_app()
|
||||
self.sync()
|
||||
next_visit = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
self.assertIn("New applications since your last visit: 1", next_visit.describe())
|
||||
|
||||
def test_main_menu_redraws_totals_and_news_after_background_refresh(self):
|
||||
from proxmenux_oci import cli, updates
|
||||
from proxmenux_oci.ui import MenuRefresh
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
status.process = Mock(returncode=0)
|
||||
status.process.poll.return_value = None
|
||||
screens = []
|
||||
owner = self
|
||||
class UI:
|
||||
live_refresh = True
|
||||
def choose(self, text, entries, default, **kwargs):
|
||||
screens.append((text, dict(entries)))
|
||||
if len(screens) == 1:
|
||||
owner.add_app()
|
||||
owner.sync()
|
||||
refresh.write_json(refresh.cache_root(owner.root) / 'main-result.json',
|
||||
{"ok": True, "changed": True})
|
||||
status.process.poll.return_value = 0
|
||||
self_refreshed = kwargs['refresh']()
|
||||
owner.assertTrue(self_refreshed)
|
||||
raise MenuRefresh()
|
||||
return None
|
||||
checker = Mock(active=False)
|
||||
checker.status.return_value = {"available": []}
|
||||
checker.notice.return_value = ""
|
||||
checker.label.return_value = "Check for updates"
|
||||
checker.changed.return_value = False
|
||||
checker_context = Mock()
|
||||
checker_context.__enter__ = Mock(return_value=checker)
|
||||
checker_context.__exit__ = Mock(return_value=False)
|
||||
original_class = refresh.CatalogRefresh
|
||||
with patch.object(original_class, '__enter__', return_value=status), \
|
||||
patch.object(refresh, 'CatalogRefresh', return_value=status), \
|
||||
patch.object(cli, 'PROJECT_ROOT', self.root), \
|
||||
patch.object(cli, 'interactive_ui', return_value=UI()), \
|
||||
patch.object(updates, 'UpdateCheck', return_value=checker_context):
|
||||
self.assertEqual(cli.interactive(Catalog(self.root)), 0)
|
||||
self.assertIn('Available applications: 1', screens[0][0])
|
||||
self.assertIn('Available applications: 2', screens[1][0])
|
||||
self.assertIn('New applications since your last visit: 1', screens[1][0])
|
||||
self.assertIn('( 2)', screens[1][1]['all'])
|
||||
next_visit = original_class(self.root, Catalog(self.root))
|
||||
self.assertIn('New applications since your last visit: 0', next_visit.describe())
|
||||
|
||||
def test_explicit_local_apps_survive_refresh_without_preventing_upstream_removal(self):
|
||||
# This app is a local preview, not a member of the upstream manifest.
|
||||
self.add_app()
|
||||
for name, data in self.files.items():
|
||||
file = self.root / 'catalog' / name
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_bytes(data)
|
||||
index = json.loads(self.files['index.json'])
|
||||
index['applications'] = [self.item]
|
||||
self.files['index.json'] = json.dumps(index).encode()
|
||||
for name in ('apps/sample-added.json', 'curated/sample-added.json'):
|
||||
self.files.pop(name)
|
||||
with patch.object(refresh, 'source_settings', return_value={'local_apps': ['sample-added']}):
|
||||
self.sync()
|
||||
catalog = Catalog(self.root)
|
||||
self.assertIn('sample-added', refresh.visible_recipes(catalog))
|
||||
self.assertEqual(catalog.compose('sample-added')['id'], 'image-sample-added')
|
||||
self.assertNotIn('sample-added', refresh.visible_recipes(Catalog(self.root)))
|
||||
|
||||
def test_channel_switch_does_not_use_other_channel_snapshot(self):
|
||||
self.sync()
|
||||
self.assertEqual(refresh.current_catalog(self.root, "develop"), self.root / "catalog")
|
||||
|
||||
def test_engine_schema_change_invalidates_cached_catalog(self):
|
||||
self.sync()
|
||||
schema = self.root / "schemas/oci-template.schema.json"
|
||||
schema.write_text(schema.read_text() + "\n")
|
||||
self.assertEqual(refresh.current_catalog(self.root), self.root / "catalog")
|
||||
|
||||
def test_corrupted_cached_index_falls_back_to_bundled_catalog(self):
|
||||
self.sync()
|
||||
snapshot = refresh.current_catalog(self.root)
|
||||
(snapshot / 'index.json').write_text('{broken json')
|
||||
self.assertEqual(refresh.current_catalog(self.root), self.root / 'catalog')
|
||||
|
||||
def test_refresh_process_is_started_without_waiting(self):
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
with patch.object(refresh.subprocess, "Popen") as spawn:
|
||||
spawn.return_value.poll.return_value = None
|
||||
with patch.object(refresh.threading, "Thread") as thread:
|
||||
with status:
|
||||
self.assertTrue(status.active)
|
||||
self.assertIn("Refreshing", status.describe())
|
||||
thread.return_value.start.assert_called_once()
|
||||
spawn.return_value.wait.assert_not_called()
|
||||
|
||||
def test_manual_refresh_does_not_duplicate_worker_and_can_retry_failure(self):
|
||||
status = refresh.CatalogRefresh(self.root, Catalog(self.root))
|
||||
with patch.object(refresh.subprocess, 'Popen') as spawn:
|
||||
spawn.return_value.poll.return_value = None
|
||||
self.assertTrue(status.start())
|
||||
self.assertFalse(status.start())
|
||||
self.assertEqual(spawn.call_count, 1)
|
||||
spawn.return_value.poll.return_value = 0
|
||||
spawn.return_value.returncode = 0
|
||||
refresh.write_json(refresh.cache_root(self.root) / 'main-result.json', {'ok': False})
|
||||
self.assertFalse(status.consume())
|
||||
self.assertTrue(status.failed)
|
||||
self.assertTrue(status.start())
|
||||
self.assertFalse(status.failed)
|
||||
self.assertEqual(spawn.call_count, 2)
|
||||
|
||||
def test_manual_refresh_is_offered_in_the_main_menu(self):
|
||||
from proxmenux_oci.cli import _catalog_options
|
||||
options = dict(_catalog_options(Catalog(self.root))[-1])
|
||||
self.assertEqual(options['catalog-refresh'], 'Refresh catalog')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -126,8 +126,13 @@ class RegistryMenuTests(unittest.TestCase):
|
||||
catalog = SimpleNamespace(index_path=SimpleNamespace(exists=lambda: True),
|
||||
load_index=lambda: {'applications': []})
|
||||
ui = UI(['registries', None])
|
||||
with patch.object(cli, 'interactive_ui', return_value=ui), \
|
||||
with patch('proxmenux_oci.catalog_refresh.CatalogRefresh') as refresh_type, \
|
||||
patch.object(cli, 'interactive_ui', return_value=ui), \
|
||||
patch.object(accounts, 'open_registry_accounts') as open_accounts:
|
||||
refresher = refresh_type.return_value.__enter__.return_value
|
||||
refresher.active = False
|
||||
refresher.consume.return_value = False
|
||||
refresher.describe.return_value = ''
|
||||
self.assertEqual(cli.interactive(catalog), 0)
|
||||
open_accounts.assert_called_once_with(ui)
|
||||
|
||||
|
||||
@@ -280,7 +280,14 @@ class MainMenuTests(unittest.TestCase):
|
||||
checker.running = False; checker.done = True
|
||||
raise MenuRefresh()
|
||||
return None
|
||||
with patch.object(cli, 'interactive_ui', return_value=UI()), \
|
||||
catalog_refresh = Mock(active=False)
|
||||
catalog_refresh.consume.return_value = False
|
||||
catalog_refresh.describe.return_value = 'Catalog: 0 applications'
|
||||
catalog_context = Mock()
|
||||
catalog_context.__enter__ = Mock(return_value=catalog_refresh)
|
||||
catalog_context.__exit__ = Mock(return_value=False)
|
||||
with patch('proxmenux_oci.catalog_refresh.CatalogRefresh', return_value=catalog_context), \
|
||||
patch.object(cli, 'interactive_ui', return_value=UI()), \
|
||||
patch.object(updates, 'UpdateCheck', return_value=checker):
|
||||
self.assertEqual(cli.interactive(cli.Catalog(ROOT)), 0)
|
||||
self.assertEqual(seen, ['Check for updates', 'Checking for updates (0/2)',
|
||||
|
||||
@@ -33,6 +33,7 @@ initialize_cache
|
||||
ensure_oci_dependencies() {
|
||||
local -a missing=()
|
||||
python3 -c 'import yaml' >/dev/null 2>&1 || missing+=("python3-yaml")
|
||||
python3 -c 'import jsonschema' >/dev/null 2>&1 || missing+=("python3-jsonschema")
|
||||
command -v skopeo >/dev/null 2>&1 || missing+=("skopeo")
|
||||
[[ ${#missing[@]} -eq 0 ]] && return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user