Generalize OCI device setup and remove unused catalog hashes

This commit is contained in:
MacRimi
2026-09-26 01:22:37 +02:00
parent 88acceb8ef
commit f7266e7b44
26 changed files with 689 additions and 487 deletions
+2
View File
@@ -115,6 +115,8 @@ def build_suite(template, ui, mode='advanced'):
if mode != DEFAULT_MODE:
from .custom_mounts import ask_stack_custom_mounts
ask_stack_custom_mounts(ui, services, storage)
from .extra_devices import ask_stack_extra_devices
ask_stack_extra_devices(ui, services)
return {'deployment_kind':'generic-multi-lxc-stack','suite_arr':True,'lifecycle_mode':'independent','stack_name':name,
'base_vmid':int(base) if base else None,'services':services,'shared_media':shared,'media_player':player,
'completion_notes':[
+24 -9
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import json
import hashlib
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
@@ -176,7 +175,6 @@ class Catalog:
),
"template": f"apps/{repo.app_id}.json" if repo.app_id in existing else None,
"template_status": existing.get(repo.app_id),
"content_hash": self._template_hash(repo.app_id) if repo.app_id in existing else None,
}
for repo, summary in discovered
]
@@ -291,7 +289,6 @@ class Catalog:
"category_label": metadata.get("category_label"),
"template": f"apps/{catalog_id}.json" if catalog_id in existing else None,
"template_status": existing.get(catalog_id),
"content_hash": self._template_hash(catalog_id) if catalog_id in existing else None,
}
)
@@ -515,6 +512,24 @@ class Catalog:
except (OSError, json.JSONDecodeError, KeyError, TypeError):
return
def _preserve_optional_environment(self, app_id: str, template: dict[str, Any]) -> None:
existing_path = self.apps_dir / f"{app_id}.json"
if not existing_path.is_file():
return
try:
existing = json.loads(existing_path.read_text(encoding="utf-8"))
if (existing["container_contract"]["image"]["repository"] !=
template["container_contract"]["image"]["repository"]):
return
optional = {item["name"]: item for item in existing["container_contract"]["environment"]
if item.get("required") is False}
for item in template["container_contract"]["environment"]:
previous = optional.get(item["name"])
if previous and previous.get("example") == item.get("example"):
item["required"] = False
except (OSError, json.JSONDecodeError, KeyError, TypeError):
return
def generate(self, app_id: str) -> tuple[Path, dict[str, Any]]:
item = self.find_item(app_id)
provider = item.get("provider", "linuxserver.io")
@@ -541,6 +556,7 @@ class Catalog:
raise ConversionError(f"Proveedor no soportado: {provider}")
catalog_id = item["id"]
self._apply_overlay(catalog_id, template)
self._preserve_optional_environment(catalog_id, template)
self._preserve_registry_state(catalog_id, template)
self.validate(template)
self.apps_dir.mkdir(parents=True, exist_ok=True)
@@ -593,6 +609,7 @@ class Catalog:
else:
raise ConversionError(f"Proveedor no soportado: {item_provider}")
self._apply_overlay(item["id"], template)
self._preserve_optional_environment(item["id"], template)
self._preserve_registry_state(item["id"], template)
self.validate(template)
return item["id"], template
@@ -661,7 +678,6 @@ class Catalog:
item["architectures"] = supported_architectures(
template["catalog_ui"]["architectures"]
)
item["content_hash"] = self._template_hash(app_id)
self._write_json(self.index_path, index)
generated.sort()
failed.sort(key=lambda item: item["id"])
@@ -759,7 +775,6 @@ class Catalog:
"curated_path": str(path.relative_to(self.root)),
"template": f"apps/{app_id}.json" if app_id in existing else None,
"template_status": existing.get(app_id),
"content_hash": self._template_hash(app_id) if app_id in existing else None,
}
)
return result
@@ -768,7 +783,11 @@ class Catalog:
path = self.overlays_dir / f"{app_id}.json"
if path.exists():
overlay = json.loads(path.read_text(encoding="utf-8"))
environment_overrides = overlay.pop("environment_overrides", {})
self._deep_merge(template, overlay)
for item in template.get("container_contract", {}).get("environment", []):
if item.get("name") in environment_overrides:
item.update(environment_overrides[item["name"]])
from .stack import apply_stack_support
apply_stack_support(template)
from .gpu import apply_gpu_contract
@@ -809,7 +828,6 @@ class Catalog:
"untranslated_blockers": template["compatibility"][
"untranslated_blockers"
],
"content_hash": self._template_hash(app_id),
}
)
if item.get("template_family") == "curated-profile":
@@ -826,6 +844,3 @@ class Catalog:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
temporary.replace(path)
def _template_hash(self, app_id: str) -> str:
return hashlib.sha256((self.apps_dir / f"{app_id}.json").read_bytes()).hexdigest()
+17 -3
View File
@@ -366,10 +366,24 @@ def _install(catalog: Catalog, ui, item: dict[str, Any], mode: str) -> None:
def install_template(ui, template: dict[str, Any], identifier: str, mode: str) -> dict[str, Any] | None:
"""Configures and installs one template, from the catalog or written from a
definition the user gave."""
deployment = build_deployment(template, ui, mode)
if not ui.review(_deployment_summary_text(template, deployment), translate("Installation summary"),
question=translate("Install with this configuration?")):
from .ui import BacktrackUI, RestartWizard
wizard = BacktrackUI(ui)
try:
while True:
candidate = copy.deepcopy(template)
try:
deployment = build_deployment(candidate, wizard, mode)
approved = wizard.review(_deployment_summary_text(candidate, deployment),
translate("Installation summary"),
question=translate("Install with this configuration?"))
break
except RestartWizard:
wizard.restart()
finally:
wizard.close()
if not approved:
return None
template = candidate
console.show_logo()
console.msg_title(f"{source_text(template['catalog_ui']['title']) or identifier} · {APP_TITLE}")
try:
+104
View File
@@ -0,0 +1,104 @@
"""Explicit native LXC devices beyond an application's curated profile."""
import copy
import re
from pathlib import Path
from .i18n import translate
from .ui import UserCancelled
GPU_NODE = re.compile(r'/dev/dri/(?:renderD|card)[0-9]+|/dev/kfd')
USB_NODE = re.compile(r'/dev/(?:ttyUSB[0-9]+|ttyACM[0-9]+|bus/usb/[0-9]{3}/[0-9]{3})')
CORAL_NODE = re.compile(r'/dev/apex_[0-9]+')
def ask_extra_devices(ui, devices, unprivileged, allow_coral=False):
"""Keep manual attachments separate from image-owned GPU profiles."""
result = list(devices)
while ui.confirm(translate('Add another GPU or USB device manually?'), False):
options = [
('gpu', translate('Intel/AMD DRM node (device only)')),
('nvidia', translate('NVIDIA runtime (device and host driver libraries)')),
('usb', translate('USB or serial device node')),
]
if allow_coral:
options.append(('coral', translate('Coral PCIe/M.2 device node')))
kind = ui.choose(translate('Device to attach'), options)
if kind is None:
raise UserCancelled(translate('Device configuration cancelled'))
if kind == 'nvidia':
if any(item.get('kind') == 'nvidia-runtime' for item in result):
raise ValueError(translate('NVIDIA is already attached'))
if not ui.confirm(translate('Passing NVIDIA does not enable acceleration inside the application. '
'The host needs NVIDIA Container Toolkit and a compatible image. Continue?'), False):
continue
result.append({'id': 'manual-nvidia', 'kind': 'nvidia-runtime',
'device_selection': 'all-requested-by-compose',
'runtime_mode': 'dynamic' if unprivileged else 'static'})
continue
if kind == 'gpu':
candidates = sorted(str(path) for path in Path('/dev/dri').glob('renderD*'))
default = candidates[0] if candidates else '/dev/dri/renderD128'
path = ui.ask(translate('Host DRM node (e.g. /dev/dri/renderD128)'), default)
valid = GPU_NODE.fullmatch(path)
elif kind == 'usb':
path = ui.ask(translate('Host USB node (e.g. /dev/ttyACM0 or /dev/bus/usb/003/004)'),
'/dev/ttyACM0')
valid = USB_NODE.fullmatch(path)
else:
path = ui.ask(translate('Host Coral node (e.g. /dev/apex_0)'), '/dev/apex_0')
valid = CORAL_NODE.fullmatch(path)
if not valid:
raise ValueError(translate('Choose a specific supported GPU or USB node'))
if any(item.get('host_path') == path for item in result):
raise ValueError(translate('This device is already attached'))
if kind == 'usb' and '/bus/usb/' in path:
ui.info(translate('USB bus numbers can change after reconnecting or rebooting.'))
result.append({'id': 'manual-' + path.removeprefix('/dev/').replace('/', '-'),
'kind': 'character-device', 'host_path': path, 'container_path': path,
'mode': '0660', 'deny_write': False,
'gid_strategy': 'host-device-gid'})
return result
def device_permissions(image, devices, existing=None):
if existing:
return existing
repository = image.split('@', 1)[0].rsplit(':', 1)[0]
if repository.startswith(('lscr.io/linuxserver/', 'linuxserver/', 'docker.io/linuxserver/')) and any(
item.get('kind') == 'character-device' for item in devices):
return {'strategy': 'linuxserver-native-init', 'service_user': 'abc',
'environment': 'ATTACHED_DEVICES_PERMS',
'paths': 'all-resolved-selected-character-devices'}
return None
def ask_stack_extra_devices(ui, services):
"""Ask once per device, then select the stack members that need it."""
devices = ask_extra_devices(ui, [], True)
if not devices:
return
options = [(service['name'], service['name']) for service in services]
for device in devices:
selected = ui.checklist(
f"{translate('Containers that will receive this device')}: "
f"{device.get('host_path', 'NVIDIA')}", options,
[service['name'] for service in services if service.get('main')] or [options[-1][0]])
if not selected or set(selected) - {name for name, _ in options}:
raise ValueError(translate('Select at least one stack container'))
for service in services:
if service['name'] not in selected:
continue
plan = service['deployment']
existing = plan.setdefault('devices', [])
if any(item.get('host_path') == device.get('host_path') if device.get('host_path')
else item.get('kind') == 'nvidia-runtime' for item in existing):
raise ValueError(translate('This device is already attached'))
member_device = copy.deepcopy(device)
if member_device['kind'] == 'nvidia-runtime':
member_device['runtime_mode'] = (
'dynamic' if plan.get('security', {}).get('unprivileged', True) else 'static')
existing.append(member_device)
image = service['template']['container_contract']['image']['reference']
plan['device_permissions'] = device_permissions(
image, existing, plan.get('device_permissions'))
+13 -1
View File
@@ -20,6 +20,7 @@ from . import network as access
from .i18n import translate
from .ui import DialogUI, TerminalUI, UserCancelled
from .custom_mounts import ask_custom_mounts
from .extra_devices import device_permissions
class InstallError(RuntimeError):
@@ -379,6 +380,15 @@ def build_deployment(
devices, selected_hardware_profile, post_start_configurations, environment = configure_acceleration(
installer_profile, environment, unprivileged, ui, mode)
if advanced:
from .extra_devices import ask_extra_devices
reference = template['container_contract']['image']['reference']
repository = reference.split('@', 1)[0].rsplit(':', 1)[0]
devices = ask_extra_devices(
ui, devices, unprivileged,
allow_coral=repository in ('ghcr.io/blakeblackshear/frigate',
'codeproject/ai-server', 'docker.io/codeproject/ai-server'))
from .gpu import apply_profile_image
apply_profile_image(template, selected_hardware_profile)
@@ -455,7 +465,9 @@ def build_deployment(
"tmpfs_mounts": tmpfs_mounts,
"devices": devices,
"hardware_profile": selected_hardware_profile,
"device_permissions": installer_profile.get("device_permissions") if devices else None,
"device_permissions": (device_permissions(template['container_contract']['image']['reference'],
devices, installer_profile.get('device_permissions'))
if devices else None),
"post_start_configurations": post_start_configurations,
"extra_hosts": installer_profile.get("extra_hosts", []),
}
+15 -4
View File
@@ -216,10 +216,21 @@ def manage_instance(project, ui, row, action=None, lifecycle_args=()):
if action == 'recreate':
from .recreation import edit_recreation
from .cli import _deployment_summary_text
proposal = edit_recreation(record, ui)
if not ui.review(_deployment_summary_text(proposal['candidate']['template'],
proposal['candidate']['deployment']), translate('Recreate OCI'),
question=translate('Recreate with these options?'), default=True):
from .ui import BacktrackUI, RestartWizard
wizard = BacktrackUI(ui)
try:
while True:
try:
proposal = edit_recreation(record, wizard)
approved = wizard.review(_deployment_summary_text(proposal['candidate']['template'],
proposal['candidate']['deployment']), translate('Recreate OCI'),
question=translate('Recreate with these options?'), default=True)
break
except RestartWizard:
wizard.restart()
finally:
wizard.close()
if not approved:
return False
elif not ui.review(translate('The current image of the saved channel will be checked and downloaded. Resources, paths and GPU are kept. The CT is stopped during the replacement and a native backup is created first.'),
translate('Update OCI'), question=translate('Update now?'), default=True):
+20 -51
View File
@@ -104,31 +104,6 @@ def edit_environment(deployment, ui):
environment.append(item)
def edit_peripherals(deployment, ui, allow_coral=False):
devices = deployment.setdefault('devices', [])
label = 'Coral/USB' if allow_coral else 'USB'
example = '/dev/apex_0, ' if allow_coral else ''
while ui.confirm(f"{translate('Add or change a device')} ({label})?", False):
path = ui.ask(f"{translate('Host device node')} ({translate('e.g.')} {example}/dev/ttyACM0, /dev/bus/usb/003/004)")
if not re.fullmatch(r'/dev/(apex_[0-9]+|ttyUSB[0-9]+|ttyACM[0-9]+|bus/usb/[0-9]{3}/[0-9]{3})', path):
raise ValueError(translate('Select a specific Coral or USB node, not the whole /dev'))
if path.startswith('/dev/apex_') and not allow_coral:
raise ValueError(translate('Coral is only offered for Frigate and CodeProject.AI'))
if '/bus/usb/' in path:
ui.info(translate('The USB number can change after reconnecting or rebooting. This profile does not remap it automatically or handle Coral USB re-enumeration. Do not share a dongle already used by another service.'))
old = next((d for d in devices if d.get('host_path') == path), None)
mode = ui.ask(translate('Node octal permissions (e.g. 0660)'), (old or {}).get('mode', '0660'))
if not re.fullmatch(r'0?[0-7]{3}', mode):
raise ValueError(translate('Invalid octal permissions'))
item = dict(old or {}, id=(old or {}).get('id', 'peripheral-' + path.removeprefix('/dev/').replace('/', '-')),
kind='character-device', host_path=path, container_path=path,
mode=mode, gid_strategy='host-device-gid', deny_write=False)
if old:
devices[devices.index(old)] = item
else:
devices.append(item)
def edit_recreation(record, ui):
candidate = copy.deepcopy(record)
refresh_template(candidate, ui)
@@ -136,32 +111,21 @@ def edit_recreation(record, ui):
resources = deployment['resources']
resources['cores'] = positive_integer(ui, translate('Cores'), resources['cores'])
resources['memory_mb'] = positive_integer(ui, translate('RAM in MiB'), resources['memory_mb'], 128)
while ui.confirm(translate('Add a custom data path?'), False):
target = absolute_path(ui.ask(translate('Path inside the container'), '/data/custom'))
existing = [m['container_path'].rstrip('/') for m in deployment['mounts']]
if any(target == p or target.startswith(p + '/') or p.startswith(target + '/') for p in existing):
raise ValueError(translate('The path overlaps an existing mount'))
mode = ui.choose(translate('Persistence for the new path'),
[('managed-volume', translate('Container volume (included in backups)')),
('host-bind', translate('Host directory (not included in Proxmox backups)'))],
'managed-volume')
if mode is None:
raise UserCancelled(translate('Custom path cancelled'))
mount = {'type': mode, 'container_path': target, 'read_only': False}
if mode == 'managed-volume':
mount.update(source=ui.ask(translate('Proxmox storage for the volume'), deployment['rootfs']['storage']),
size_gb=positive_integer(ui, translate('Volume size in GB'), 4), backup=True)
else:
mount.update(source=absolute_path(ui.ask(translate('Host directory'),
'/mnt/oci-shared/custom')), size_gb=None, backup=False,
create_if_missing=True)
deployment['mounts'].append(mount)
from .custom_mounts import ask_custom_mounts
deployment['mounts'] = ask_custom_mounts(
ui, deployment['mounts'], deployment['rootfs']['storage'])
if ui.confirm(translate('Change the access network?'), False):
edit_network(deployment, ui)
edit_acceleration(candidate, ui)
from .extra_devices import ask_extra_devices
reference = candidate.get('template', {}).get('container_contract', {}).get('image', {}).get('reference', '')
repository = reference.split('@')[0].rsplit(':', 1)[0]
edit_peripherals(deployment, ui, repository in ('ghcr.io/blakeblackshear/frigate', 'codeproject/ai-server', 'docker.io/codeproject/ai-server'))
deployment['devices'] = ask_extra_devices(
ui, deployment.get('devices', []), deployment.get('security', {}).get('unprivileged', True),
allow_coral=repository in ('ghcr.io/blakeblackshear/frigate', 'codeproject/ai-server', 'docker.io/codeproject/ai-server'))
from .extra_devices import device_permissions
deployment['device_permissions'] = device_permissions(
reference, deployment['devices'], deployment.get('device_permissions'))
edit_environment(deployment, ui)
proposal = {'operation': 'recreate', 'candidate': candidate}
if record.get('observed', {}).get('config_sha256'):
@@ -172,14 +136,19 @@ def edit_recreation(record, ui):
def refresh_template(candidate, ui):
from .catalog import Catalog
old = candidate.get('template', {})
name = old.get('id', '').removeprefix('image-')
if not re.fullmatch(r'[a-z0-9][a-z0-9-]+', name):
template_id = old.get('id', '')
if not re.fullmatch(r'[a-z0-9][a-z0-9-]+', template_id):
return
root = Path(__file__).resolve().parents[2]
path = root / 'catalog' / 'apps' / (name + '.json')
if not path.is_file() or not old.get('container_contract', {}).get('image'):
if not old.get('container_contract', {}).get('image'):
return
latest = Catalog(root).load_template(name, generate_if_missing=False)
catalog = Catalog(root)
matching = [item for item in catalog.load_index()['applications']
if item.get('template_id') == template_id]
if len(matching) != 1:
return
name = matching[0]['id']
latest = catalog.compose(name)
if latest == old:
return
if not ui.confirm(translate('Apply the options from the current catalog template? Your data and configuration are kept.'), True):
+2
View File
@@ -355,6 +355,8 @@ def build_stack(template, ui, mode='advanced'):
if mode != DEFAULT_MODE:
from .custom_mounts import ask_stack_custom_mounts
ask_stack_custom_mounts(ui, plans, volumes)
from .extra_devices import ask_stack_extra_devices
ask_stack_extra_devices(ui, plans)
return {'deployment_kind':'generic-multi-lxc-stack','stack_name':name,'base_vmid':int(vmid) if vmid else None,
'completion_notes':template.get('proxmox',{}).get('stack_completion_notes',[]),
'rootfs_storage':root,'template_storage':cache,'onboot':onboot,'start_after_create':True,
+96 -3
View File
@@ -16,17 +16,96 @@ class UserCancelled(RuntimeError):
pass
class BackRequested(RuntimeError):
pass
class RestartWizard(RuntimeError):
pass
class BacktrackUI:
"""Replay prior answers when returning to a previous wizard question."""
def __init__(self, base):
self.base = base
self.answers = []
self.cursor = 0
self.previous_back_enabled = getattr(base, 'back_enabled', False)
base.back_enabled = True
def __getattr__(self, name):
return getattr(self.base, name)
def close(self):
self.base.back_enabled = self.previous_back_enabled
def restart(self):
self.cursor = 0
def _call(self, name, *args, **kwargs):
if self.cursor < len(self.answers):
saved_name, value = self.answers[self.cursor]
if saved_name != name:
self.answers = self.answers[:self.cursor]
else:
self.cursor += 1
return value
try:
value = getattr(self.base, name)(*args, **kwargs)
except BackRequested:
if self.cursor:
self.answers = self.answers[:self.cursor - 1]
self.cursor = 0
raise RestartWizard()
raise UserCancelled(translate('Wizard cancelled'))
self.answers.append((name, value))
self.cursor += 1
return value
def ask(self, *args, **kwargs):
return self._call('ask', *args, **kwargs)
def password(self, *args, **kwargs):
return self._call('password', *args, **kwargs)
def confirm(self, *args, **kwargs):
return self._call('confirm', *args, **kwargs)
def choose(self, *args, **kwargs):
return self._call('choose', *args, **kwargs)
def checklist(self, *args, **kwargs):
return self._call('checklist', *args, **kwargs)
def detail_menu(self, *args, **kwargs):
return self._call('detail_menu', *args, **kwargs)
def review(self, *args, **kwargs):
try:
return self.base.review(*args, **kwargs)
except BackRequested:
if self.cursor:
self.answers = self.answers[:self.cursor - 1]
self.cursor = 0
raise RestartWizard()
raise UserCancelled(translate('Wizard cancelled'))
APP_TITLE = "OCI manager Apps (beta)"
@dataclass
class TerminalUI:
title: str = APP_TITLE
back_enabled: bool = False
def ask(self, text: str, default: str | None = None, required: bool = True) -> str:
suffix = f" [{default}]" if default not in (None, "") else ""
while True:
value = input(f"{text}{suffix}: ").strip()
if self.back_enabled and value == ':back':
raise BackRequested()
if value:
return value
if default is not None:
@@ -38,18 +117,25 @@ class TerminalUI:
def password(self, text: str, required: bool = True) -> str:
while True:
value = getpass.getpass(f"{text}: ")
if self.back_enabled and value == ':back':
raise BackRequested()
if not value:
if not required:
return ""
print(translate("This value is required."))
continue
if value == getpass.getpass(f"{translate('Repeat to confirm')}: "):
repeated = getpass.getpass(f"{translate('Repeat to confirm')}: ")
if self.back_enabled and repeated == ':back':
raise BackRequested()
if value == repeated:
return value
print(translate("The values do not match. Enter them again."))
def confirm(self, text: str, default: bool = False) -> bool:
suffix = " [Y/n]" if default else " [y/N]"
value = input(f"{text}{suffix}: ").strip().casefold()
if self.back_enabled and value == ':back':
raise BackRequested()
if not value:
return default
return value in {"y", "yes", "s", "si"}
@@ -93,6 +179,7 @@ class DialogUI:
title: str = APP_TITLE
backtitle: str = "ProxMenux"
back_enabled: bool = False
@staticmethod
def available() -> bool:
@@ -103,10 +190,16 @@ class DialogUI:
if environment.get("TERM", "").casefold() in {"", "dumb", "unknown"}:
environment["TERM"] = "xterm-256color"
# dialog draws on the terminal and writes the selection to stderr.
return subprocess.run(
["dialog", "--no-collapse", "--backtitle", self.backtitle, "--title", title or self.title, *widget],
back_widget = ['--extra-button', '--extra-label', 'Volver'] if self.back_enabled and any(
flag in widget for flag in ('--inputbox', '--passwordbox', '--yesno', '--menu', '--checklist')) else []
result = subprocess.run(
["dialog", "--no-collapse", "--backtitle", self.backtitle, "--title", title or self.title,
*back_widget, *widget],
stdout=None, stderr=subprocess.PIPE, text=True, check=False, env=environment,
)
if result.returncode == 3 and back_widget:
raise BackRequested()
return result
@staticmethod
def _size(text: str, min_height: int, width: int, extra: int = 6) -> tuple[str, str]: