mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-01 10:22:08 +00:00
feat(oci): update and modify every application, and keep its address
- The backup taken before an update or a modification is made on another storage when it does not fit on the system disk; an operation without room says so before anything is stopped. - Privileged containers, relaxed AppArmor or seccomp, process limits, the host monitors Glances and Netdata and single-file host mounts are rebuilt and recovered like any other application. - HAOS One installs the docker-haos image: port 8123, host time zone, unprivileged, watchdog proposed by default; it is updated and modified with Home Assistant's own check on the rebuilt container. - A container that reboots itself gets its DHCP client back. - The installer takes the address of the container's own interface, and the notes of a suite or stack container show its LAN address. - Seerr installs: its volume belongs to the image user and no-new-privileges goes through the include of the container. - Removing an application deletes its seccomp profile.
This commit is contained in:
MacRimi
@@ -27,6 +27,9 @@ def note(text):
|
||||
"""Detail for the run log; without one, for stderr."""
|
||||
try:
|
||||
if OCI_LOG:
|
||||
if not sys.stderr.isatty():
|
||||
# The progress line goes to a file, which may be this log: end it first.
|
||||
print(file=sys.stderr, flush=True)
|
||||
log(OCI_LOG, text)
|
||||
return
|
||||
except OSError:
|
||||
@@ -110,7 +113,7 @@ def probe(vmid, ip, remaining):
|
||||
raise Pending(translate('Supervisor does not confirm healthy and supported yet'))
|
||||
if not http_ready(f'http://{ip}:4357/', max(0.1, min(5, remaining()))):
|
||||
raise Pending(translate('Observer is not responding on port 4357'))
|
||||
for port in (80, 8123):
|
||||
for port in (8123, 80):
|
||||
if http_ready(f'http://{ip}:{port}/', max(0.1, min(5, remaining()))):
|
||||
return [{'label': 'Home Assistant', 'url': f'http://{ip}:{port}/'},
|
||||
{'label': 'Home Assistant Observer', 'url': f'http://{ip}:4357/'}]
|
||||
|
||||
@@ -103,6 +103,24 @@ def persist_instances(deployment, services, primary_id=None):
|
||||
oci_instances.publish_stack(oci_instances.ROOT, None, template, deployment, services)
|
||||
|
||||
|
||||
def refresh_notes(services):
|
||||
"""The notes of each container were written when it was created, before
|
||||
it had its leg on the local network: write them again with the address
|
||||
the application is reached at."""
|
||||
from oci_description import render
|
||||
from oci_native_stack import access_address as reachable_address
|
||||
for service in services:
|
||||
vmid = int(service['vmid'])
|
||||
try:
|
||||
record = oci_instances.read(oci_instances.ROOT, vmid)
|
||||
running = 'running' in run('pct', 'status', vmid)
|
||||
description = render(record['template'], '', record['installation_id'],
|
||||
reachable_address(vmid, wait=20 if running else 0))
|
||||
run('pct', 'set', vmid, '--description', description)
|
||||
except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error:
|
||||
log(LOG, f'Notes of CT {vmid} not refreshed: {error}')
|
||||
|
||||
|
||||
def run(*args, capture=True, timeout=180):
|
||||
argv = list(map(str,args))
|
||||
if capture:
|
||||
@@ -811,6 +829,7 @@ def main():
|
||||
fcntl.flock(lock,fcntl.LOCK_UN)
|
||||
result = finish_independent_suite(services,subnet)
|
||||
persist_instances(deployment, services)
|
||||
refresh_notes(services)
|
||||
result.update(completion_notes=list(deployment.get('completion_notes',[])), log=LOG)
|
||||
print('PROXMENUX_RESULT='+base64.b64encode(json.dumps(result).encode()).decode(),flush=True)
|
||||
return
|
||||
@@ -852,6 +871,7 @@ def main():
|
||||
msg_ok(translate('LAN address applied to the application URLs'))
|
||||
result=results[primary['name']]
|
||||
persist_instances(deployment, services, primary_id)
|
||||
refresh_notes(services)
|
||||
# The credentials of the main service come from its own installation.
|
||||
result.update(vmid=primary_id,ip=lan,stack_vmids={s['name']:s['vmid'] for s in services},
|
||||
urls=[{'label':'Web UI','url':public_url}],
|
||||
|
||||
+28
-12
@@ -745,6 +745,21 @@ apply_post_start_configuration() {
|
||||
esac
|
||||
}
|
||||
|
||||
# The first address of the container on an interface Proxmox gave it. A
|
||||
# container that runs Docker inside also lists the ones of its inner bridges.
|
||||
# Fails when the interfaces cannot be read, so the caller takes what is listed.
|
||||
own_ipv4() {
|
||||
local name listed found own="" looked=0
|
||||
listed=$(lxc-info -n "$VMID" -iH 2>/dev/null | grep -E '^[0-9]+\.') || return 1
|
||||
for name in $(pct config "$VMID" 2>/dev/null | sed -n 's/^net[0-9]*: .*name=\([A-Za-z0-9_.-]*\).*/\1/p'); do
|
||||
found=$(lxc-attach -n "$VMID" -s NETWORK -- ip -4 -o addr show dev "$name" scope global 2>/dev/null) || continue
|
||||
looked=1
|
||||
own+=$(sed -n 's/.* inet \([0-9.]*\)\/.*/\1/p' <<<"$found")$'\n'
|
||||
done
|
||||
(( looked == 1 )) || return 1
|
||||
grep -m1 -Fx -f <(sed '/^$/d' <<<"$own") <<<"$listed" || true
|
||||
}
|
||||
|
||||
# $1: optional result label shown before the address.
|
||||
detect_container_ipv4() {
|
||||
local attempt addresses label=${1:-}
|
||||
@@ -755,10 +770,12 @@ detect_container_ipv4() {
|
||||
fi
|
||||
IP=""
|
||||
for attempt in $(seq 1 15); do
|
||||
IP=$(lxc-info -n "$VMID" -iH 2>/dev/null | grep -m1 -E '^[0-9]+\.' || true)
|
||||
if [[ -z $IP ]]; then
|
||||
addresses=$(pct exec "$VMID" -- hostname -I 2>/dev/null || true)
|
||||
IP=$(tr ' ' '\n' <<<"$addresses" | grep -m1 -E '^[0-9]+\.' || true)
|
||||
if ! IP=$(own_ipv4); then
|
||||
IP=$(lxc-info -n "$VMID" -iH 2>/dev/null | grep -m1 -E '^[0-9]+\.' || true)
|
||||
if [[ -z $IP ]]; then
|
||||
addresses=$(pct exec "$VMID" -- hostname -I 2>/dev/null || true)
|
||||
IP=$(tr ' ' '\n' <<<"$addresses" | grep -m1 -E '^[0-9]+\.' || true)
|
||||
fi
|
||||
fi
|
||||
[[ -n $IP ]] && break
|
||||
msg_progress "$(translate "Waiting for the network address...") $((attempt * 2))/30 s"
|
||||
@@ -1705,7 +1722,8 @@ else
|
||||
fi
|
||||
|
||||
SYSCTL_COUNT=$(jq '.security.sysctls? // [] | length' "$DEPLOYMENT_FILE")
|
||||
if (( SYSCTL_COUNT > 0 )); then
|
||||
NO_NEW_PRIVILEGES=$(jq -r '.security.options.no_new_privileges? // false' "$DEPLOYMENT_FILE")
|
||||
if (( SYSCTL_COUNT > 0 )) || [[ $NO_NEW_PRIVILEGES == true ]]; then
|
||||
SYSCTL_INCLUDE="/etc/pve/proxmenux/${VMID}.sysctls"
|
||||
mkdir -p /etc/pve/proxmenux || die "$(translate "Could not create the ProxMenux folder in /etc/pve")"
|
||||
SYSCTL_TEMP=$(mktemp)
|
||||
@@ -1718,12 +1736,14 @@ if (( SYSCTL_COUNT > 0 )); then
|
||||
printf 'lxc.sysctl.%s = %s\n' "$SYSCTL_NAME" "$SYSCTL_VALUE" >>"$SYSCTL_TEMP"
|
||||
oci_log "Network sysctl prepared: ${SYSCTL_NAME}=${SYSCTL_VALUE}"
|
||||
done < <(jq -r '.security.sysctls[]? | [.name,.value] | @tsv' "$DEPLOYMENT_FILE")
|
||||
# Proxmox does not take this setting in the configuration of a container.
|
||||
[[ $NO_NEW_PRIVILEGES != true ]] || printf 'lxc.no_new_privs = 1\n' >>"$SYSCTL_TEMP"
|
||||
[[ ! -L $SYSCTL_INCLUDE ]] || die "$(translate "The sysctl include is a link:") $SYSCTL_INCLUDE"
|
||||
# pmxcfs assigns its own permissions and rejects chmod.
|
||||
cat "$SYSCTL_TEMP" >"$SYSCTL_INCLUDE"
|
||||
rm -f "$SYSCTL_TEMP"
|
||||
set_lxc_directive "lxc.include" "$SYSCTL_INCLUDE"
|
||||
msg_ok "$(translate "Network sysctls prepared:") $SYSCTL_COUNT"
|
||||
(( SYSCTL_COUNT == 0 )) || msg_ok "$(translate "Network sysctls prepared:") $SYSCTL_COUNT"
|
||||
fi
|
||||
|
||||
REQUIRED_CAPABILITIES=$(jq -r '.security.required_capabilities? // [] | join(",")' "$DEPLOYMENT_FILE")
|
||||
@@ -1731,13 +1751,9 @@ if [[ -n $REQUIRED_CAPABILITIES ]]; then
|
||||
msg_ok "$(translate "Compose capabilities validated in the LXC user namespace:") $REQUIRED_CAPABILITIES"
|
||||
fi
|
||||
|
||||
NO_NEW_PRIVILEGES=$(jq -r '.security.options.no_new_privileges? // false' "$DEPLOYMENT_FILE")
|
||||
APPARMOR_PROFILE=$(jq -r '.security.options.apparmor_profile? // empty' "$DEPLOYMENT_FILE")
|
||||
SECCOMP_PROFILE=$(jq -r '.security.options.seccomp_profile? // empty' "$DEPLOYMENT_FILE")
|
||||
SELINUX_LABEL_DISABLED=$(jq -r '.security.options.selinux_label_disabled? // false' "$DEPLOYMENT_FILE")
|
||||
if [[ $NO_NEW_PRIVILEGES == true ]]; then
|
||||
set_lxc_directive "lxc.no_new_privs" "1"
|
||||
fi
|
||||
if [[ $(jq -r '.security.options.drop_all_capabilities? // false' "$DEPLOYMENT_FILE") == true ]]; then
|
||||
[[ -z $REQUIRED_CAPABILITIES ]] || die "$(translate "Capabilities cannot be kept and all dropped at the same time")"
|
||||
set_lxc_directive "lxc.cap.drop" ""
|
||||
@@ -2057,9 +2073,9 @@ if [[ $START_AFTER == 1 && $HAS_STARTUP_HEALTHCHECK == 1 ]]; then
|
||||
HC_PORT=$(jq -er '.port' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_PATH=$(jq -er '.path' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_TIMEOUT=$(jq -er '.timeout_seconds' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_REQUEST_TIMEOUT=$(jq -er '.request_timeout_seconds' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_REQUEST_TIMEOUT=$(jq -r '.request_timeout_seconds // 5' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_STABILITY=$(jq -r '.stability_seconds // 0' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_VERIFY_TLS=$(jq -r '.verify_tls' <<<"$STARTUP_HEALTHCHECK")
|
||||
HC_VERIFY_TLS=$(jq -r '.verify_tls // false' <<<"$STARTUP_HEALTHCHECK")
|
||||
[[ $HC_SCHEME == http || $HC_SCHEME == https ]] || die "$(translate "Invalid healthcheck scheme")"
|
||||
[[ $HC_PORT =~ ^[0-9]+$ && $HC_PORT -ge 1 && $HC_PORT -le 65535 ]] \
|
||||
|| die "$(translate "Invalid healthcheck port")"
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Marks each start of an OCI container in its console log, so the log can be
|
||||
# read from the last start. Run by the container's lxc.hook.pre-start.
|
||||
# Run by the lxc.hook.pre-start of an OCI container. Marks each start in its
|
||||
# console log, so the log can be read from the last start, and queues the
|
||||
# check of its DHCP address for when it is up.
|
||||
case "$1" in
|
||||
''|*[!0-9]*) exit 0 ;;
|
||||
esac
|
||||
printf '\n=== ProxMenux: container started %s ===\n' "$(date '+%Y-%m-%d %H:%M:%S')" \
|
||||
>> "/var/log/proxmenux/oci/$1.console.log" 2>/dev/null
|
||||
lease="${0%/*}/oci_dhcp_lease.py"
|
||||
if [ -r "$lease" ] && command -v systemd-run >/dev/null 2>&1; then
|
||||
systemd-run --quiet --collect --on-active=20 /usr/bin/python3 "$lease" "$1" >/dev/null 2>&1
|
||||
fi
|
||||
exit 0
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Give its DHCP address back to an OCI container that restarted itself.
|
||||
|
||||
Proxmox runs the DHCP client of a host-managed interface from the host and
|
||||
starts it only when it starts the container. A container that reboots from
|
||||
inside, as Home Assistant OS does from its settings, is started again by
|
||||
Proxmox without that client and comes back with no address. The start hook of
|
||||
every OCI container queues this check: once the container is up, the client
|
||||
that is missing is started the way Proxmox starts it.
|
||||
|
||||
The client lives where the container was started from: one started from the
|
||||
Monitor ends when the Monitor is restarted, and the address would be lost
|
||||
when its lease runs out. The Monitor asks for every container when it starts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
CONFIGS = Path('/etc/pve/lxc')
|
||||
HOOKS = Path('/var/lib/lxc')
|
||||
PROC = Path('/proc')
|
||||
LOCKS = Path('/run/proxmenux')
|
||||
# Proxmox starts its own client right after the container; one that just
|
||||
# started is left to it, and to the check its start hook queued.
|
||||
SETTLE = 30
|
||||
NAME_RE = re.compile(r'[A-Za-z0-9_.-]{1,15}')
|
||||
START = 'PVE::LXC::manage_dhclient("start", $ARGV[0], $ARGV[1], $ARGV[2], "/proc/$ARGV[3]/root")'
|
||||
|
||||
|
||||
def managed(config):
|
||||
"""The host-managed interfaces of a container configuration that take
|
||||
their address by DHCP: [(name, ip version)]."""
|
||||
found = []
|
||||
for line in config.split('\n[', 1)[0].splitlines():
|
||||
match = re.match(r'net\d+:\s*(.*)', line)
|
||||
if not match:
|
||||
continue
|
||||
options = dict(part.split('=', 1) for part in match.group(1).split(',') if '=' in part)
|
||||
name = options.get('name', '')
|
||||
if options.get('host-managed') != '1' or options.get('link_down') == '1' or not NAME_RE.fullmatch(name):
|
||||
continue
|
||||
found += [(name, version) for key, version in (('ip', 4), ('ip6', 6)) if options.get(key) == 'dhcp']
|
||||
return found
|
||||
|
||||
|
||||
def client_running(vmid, name, version):
|
||||
"""Whether a DHCP client of the interface runs, with or without a lease yet."""
|
||||
mark = f'{HOOKS}/{vmid}/hook/dhclient{version}-{name}.pid'.encode()
|
||||
for entry in PROC.glob('[0-9]*/cmdline'):
|
||||
try:
|
||||
if mark in entry.read_bytes().split(b'\0'):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def init_pid(vmid):
|
||||
result = subprocess.run(['lxc-info', '-n', str(vmid), '-pH'], capture_output=True, text=True, check=False, timeout=30)
|
||||
pid = result.stdout.strip()
|
||||
return int(pid) if result.returncode == 0 and pid.isdigit() and int(pid) > 0 else None
|
||||
|
||||
|
||||
def running_for(pid):
|
||||
"""Seconds since the init of the container started."""
|
||||
try:
|
||||
return time.time() - (PROC / str(pid)).stat().st_ctime
|
||||
except OSError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def containers():
|
||||
"""The containers of this node with a host-managed interface on DHCP."""
|
||||
found = []
|
||||
for path in sorted(CONFIGS.glob('*.conf')):
|
||||
try:
|
||||
if path.stem.isdigit() and managed(path.read_text(errors='replace')):
|
||||
found.append(int(path.stem))
|
||||
except OSError:
|
||||
continue
|
||||
return found
|
||||
|
||||
|
||||
def start_client(vmid, name, version, pid):
|
||||
# In a scope of its own: the client stays when this check ends.
|
||||
result = subprocess.run(['systemd-run', '--scope', '--quiet', '--collect', 'perl', '-MPVE::LXC', '-e', START,
|
||||
str(vmid), str(version), name, str(pid)],
|
||||
capture_output=True, text=True, check=False, timeout=180)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def repair(vmid, settle=0):
|
||||
"""Start the DHCP clients a running container is missing. Returns the
|
||||
interfaces that got one: [(name, ip version)]."""
|
||||
try:
|
||||
config = (CONFIGS / f'{vmid}.conf').read_text(errors='replace')
|
||||
except OSError:
|
||||
return []
|
||||
wanted = managed(config)
|
||||
if not wanted:
|
||||
return []
|
||||
pid = init_pid(vmid)
|
||||
if pid is None or running_for(pid) < settle:
|
||||
return []
|
||||
started = []
|
||||
for name, version in wanted:
|
||||
if not client_running(vmid, name, version) and start_client(vmid, name, version, pid):
|
||||
started.append((name, version))
|
||||
return started
|
||||
|
||||
|
||||
def check(vmid, settle=0):
|
||||
with (LOCKS / f'oci-dhcp-lease-{vmid}.lock').open('w') as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
return
|
||||
for name, version in repair(vmid, settle):
|
||||
print(f'CT {vmid}: DHCP client of {name} (IPv{version}) started again', flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2 or not (sys.argv[1].isdigit() or sys.argv[1] == '--all'):
|
||||
print('usage: oci_dhcp_lease.py <vmid> | --all', file=sys.stderr)
|
||||
return 2
|
||||
LOCKS.mkdir(parents=True, exist_ok=True)
|
||||
if sys.argv[1] == '--all':
|
||||
for vmid in containers():
|
||||
check(vmid, SETTLE)
|
||||
else:
|
||||
check(int(sys.argv[1]))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -34,9 +34,13 @@ def snapshot(source, allow_missing=False):
|
||||
'mode': stat.S_IMODE(info.st_mode)}
|
||||
|
||||
|
||||
def validate_source(source, allow_missing=False):
|
||||
def validate_source(source, allow_missing=False, declared=False):
|
||||
"""`declared` is a source the recipe of the application names itself, as a
|
||||
host monitor does with /proc and /sys: it is not one the user typed."""
|
||||
valid_path(source)
|
||||
value = snapshot(source, allow_missing)
|
||||
if declared:
|
||||
return dict(value, declared=True)
|
||||
protected = ('/etc', '/usr', '/bin', '/sbin', '/lib', '/lib64', '/dev', '/proc', '/sys', '/run')
|
||||
protected += tuple(str(Path(p).resolve()) for p in protected)
|
||||
resolved = value['resolved_path']
|
||||
@@ -52,7 +56,8 @@ def same_source(a, b):
|
||||
|
||||
def verify_sources(expected):
|
||||
for source, previous in expected.items():
|
||||
current = validate_source(source, allow_missing=not previous['exists'])
|
||||
current = validate_source(source, allow_missing=not previous['exists'],
|
||||
declared=previous.get('declared', False))
|
||||
if not same_source(previous, current):
|
||||
raise ValueError(f"{translate('The operation was stopped because a shared directory changed its identity:')} {source}")
|
||||
|
||||
|
||||
@@ -195,6 +195,7 @@ def resolve_candidate(reference, architecture):
|
||||
# version inside it moved.
|
||||
return {'manifest_digest': digest, 'registry_digest': registry_digest,
|
||||
'layers': [layer['digest'] for layer in json.loads(raw).get('layers', [])],
|
||||
'size': sum(int(layer.get('size') or 0) for layer in json.loads(raw).get('layers', [])),
|
||||
'defaults': defaults, 'version': version, 'created': config.get('created')}
|
||||
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ import oci_host_mounts as host_mounts
|
||||
import oci_accelerators as gpu_devices
|
||||
import oci_runtime_settings as runtime_settings
|
||||
import oci_image_cache as image_cache
|
||||
import oci_work_backup as work_backup
|
||||
import oci_ui
|
||||
from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error, msg_info2
|
||||
|
||||
@@ -48,7 +49,9 @@ BASIC = {'arch', 'cmode', 'console', 'tty', 'cores', 'cpulimit', 'cpuunits', 'de
|
||||
# The hook that marks each start in that log, set the same way.
|
||||
'lxc.hook.pre-start'}
|
||||
# Their output is data (and may hold saved secrets); it is never logged.
|
||||
DATA_COMMANDS = {('pct', 'config'), ('pvesh', 'get')}
|
||||
DATA_COMMANDS = {('pct', 'config'), ('pvesh', 'get'), ('pct', 'df')}
|
||||
# Kept across a modification: the watchdog and where the backups are made.
|
||||
HOST_PREFERENCES = ('watchdog', 'work_storage')
|
||||
LOG_DIR = Path(os.environ.get('OCI_LOG_DIR', '/var/log/proxmenux/oci'))
|
||||
ANSI = re.compile(r'\x1b\[[0-9;?]*[A-Za-z]')
|
||||
SPINNER_FRAME = re.compile('^ ?[' + ''.join(oci_ui.FRAMES) + ']')
|
||||
@@ -316,13 +319,49 @@ def mounts(config):
|
||||
return result
|
||||
|
||||
|
||||
def is_file_bind(mount, files=()):
|
||||
return mount['type'] == 'host-bind' and (mount['source'] in files or Path(mount['source']).is_file())
|
||||
|
||||
|
||||
def declared_sources(contract):
|
||||
"""The host paths the recipe of a host monitor mounts on its own."""
|
||||
if not contract['deployment'].get('host_monitor'):
|
||||
return set()
|
||||
profile = contract['template'].get('proxmox', {}).get('installer_profile', {})
|
||||
return {item['source'] for item in profile.get('host_monitor_mounts', [])}
|
||||
|
||||
|
||||
def profile_keys(deployment):
|
||||
"""The raw LXC settings the installer writes for what the application
|
||||
declares, and writes again on the rebuilt container."""
|
||||
keys = set()
|
||||
security = deployment.get('security', {})
|
||||
options = security.get('options') or {}
|
||||
if options.get('apparmor_profile'):
|
||||
keys.add('lxc.apparmor.profile')
|
||||
if options.get('seccomp_profile'):
|
||||
keys.add('lxc.seccomp.profile')
|
||||
if options.get('no_new_privileges'):
|
||||
keys.add('lxc.include')
|
||||
if options.get('drop_all_capabilities'):
|
||||
keys.update(('lxc.cap.drop', 'lxc.cap.keep'))
|
||||
keys.update(f"lxc.prlimit.{limit['name']}" for limit in deployment.get('resources', {}).get('rlimits') or [])
|
||||
if deployment.get('host_monitor'):
|
||||
keys.update(('lxc.include', 'lxc.net.0.type', 'lxc.hook.mount'))
|
||||
return keys
|
||||
|
||||
|
||||
def effective_healthcheck(template):
|
||||
"""The template's own health check or, when it has none, one derived from
|
||||
its web address: any HTTP answer below 500 means the application is up."""
|
||||
profile = template.get('proxmox', {}).get('installer_profile', {})
|
||||
check = profile.get('startup_healthcheck')
|
||||
if check or profile.get('haos_healthcheck') or profile.get('host_monitor'):
|
||||
if check:
|
||||
return check
|
||||
if profile.get('haos_healthcheck'):
|
||||
# Home Assistant OS has a check of its own: Supervisor, Core and the
|
||||
# services it runs inside. The installer runs it on the new container.
|
||||
return {'type': 'haos', 'timeout_seconds': int(profile['haos_healthcheck'].get('timeout_seconds', 1200))}
|
||||
for endpoint in template.get('first_run', {}).get('endpoints', []):
|
||||
path = str(endpoint.get('path') or '/')
|
||||
if (endpoint.get('scheme') in ('http', 'https') and str(endpoint.get('port', '')).isdigit()
|
||||
@@ -336,7 +375,7 @@ def effective_healthcheck(template):
|
||||
|
||||
def with_default_healthcheck(contract):
|
||||
check = effective_healthcheck(contract['template'])
|
||||
if check:
|
||||
if check and check.get('type') != 'haos':
|
||||
contract['template'].setdefault('proxmox', {}).setdefault('installer_profile', {})['startup_healthcheck'] = check
|
||||
return contract
|
||||
|
||||
@@ -365,6 +404,13 @@ def candidate_contract(record, operation, proposal=None):
|
||||
result = copy.deepcopy(record)
|
||||
result.update(template=copy.deepcopy(candidate['template']),
|
||||
deployment=copy.deepcopy(candidate['deployment']))
|
||||
# What ProxMenux was told about how to treat the application is not part
|
||||
# of what is being edited: it stays as the record has it now.
|
||||
for key in HOST_PREFERENCES:
|
||||
if key in record['deployment']:
|
||||
result['deployment'][key] = record['deployment'][key]
|
||||
else:
|
||||
result['deployment'].pop(key, None)
|
||||
return with_default_healthcheck(result)
|
||||
|
||||
|
||||
@@ -435,23 +481,32 @@ def preflight(record, candidate, config, coordinated=None):
|
||||
runtime_keys.add('lxc.mount.entry')
|
||||
if deployment.get('security', {}).get('sysctls'):
|
||||
runtime_keys.add('lxc.include')
|
||||
runtime_keys |= profile_keys(deployment)
|
||||
files = runtime_settings.file_binds(config)
|
||||
if files:
|
||||
runtime_keys.add('lxc.mount.entry')
|
||||
runtime_settings.check(config, deployment, record['vmid'])
|
||||
coordinated_keys = {'hookscript', *KEPT_AS_IS} if coordinated else set(KEPT_AS_IS)
|
||||
if '[' in config.decode() or keys - BASIC - runtime_keys - coordinated_keys - {k for k in keys if re.fullmatch(r'(mp|dev)[0-9]+', k)}:
|
||||
raise ValueError(translate('The container has advanced Proxmox settings outside the supported profile'))
|
||||
for plan in (deployment, desired):
|
||||
security = plan.get('security', {})
|
||||
if (security.get('unprivileged') is not True
|
||||
or security.get('options') or plan.get('host_monitor')
|
||||
or plan.get('resources', {}).get('rlimits')):
|
||||
raise ValueError(translate('Updates are not available yet in this beta for applications that use '
|
||||
'a privileged container or advanced LXC settings'))
|
||||
unprivileged = deployment.get('security', {}).get('unprivileged', True) is not False
|
||||
if (desired.get('security', {}).get('unprivileged', True) is not False) != unprivileged \
|
||||
or desired.get('host_monitor') != deployment.get('host_monitor'):
|
||||
# The owners of the data belong to one mode, and a monitor of the
|
||||
# host has no network of its own to keep.
|
||||
raise ValueError(translate('Changing between a privileged and an unprivileged container, or the host monitor mode, requires installing the application again'))
|
||||
for template, plan in ((record['template'], deployment), (candidate['template'], desired)):
|
||||
runtime_settings.sysctl_content(plan)
|
||||
runtime_settings.tmpfs_lines(plan)
|
||||
gpu_devices.planned(plan)
|
||||
paths = []
|
||||
# What the recipe of a host monitor mounts of the host, such as /sys
|
||||
# and a folder inside it, is read only and laid out by the recipe.
|
||||
declared = declared_sources({'template': template, 'deployment': plan})
|
||||
for mount in plan.get('mounts', []):
|
||||
target = host_mounts.valid_path(mount['container_path'])
|
||||
if mount['type'] == 'host-bind' and mount['source'] in declared and mount.get('read_only'):
|
||||
continue
|
||||
if (any(target == p or target.startswith(p.rstrip('/') + '/')
|
||||
or p.startswith(target.rstrip('/') + '/') for p in paths)):
|
||||
raise ValueError(translate('Mount paths must not overlap'))
|
||||
@@ -465,15 +520,18 @@ def preflight(record, candidate, config, coordinated=None):
|
||||
else:
|
||||
raise ValueError(translate('Unsupported mount type'))
|
||||
paths.append(target)
|
||||
if cfg.get('unprivileged') != '1' or ':' not in cfg.get('rootfs', ''):
|
||||
raise ValueError(translate('A managed rootfs and an unprivileged container are required'))
|
||||
if cfg.get('unprivileged', '0') != ('1' if unprivileged else '0') or ':' not in cfg.get('rootfs', ''):
|
||||
raise ValueError(translate('The container disks do not match the saved record'))
|
||||
if desired['rootfs'] != deployment['rootfs']:
|
||||
raise ValueError(translate('Changing the rootfs or its storage requires a separate migration'))
|
||||
actual = mounts(config)
|
||||
gpu_devices.check(config, deployment)
|
||||
gpu_devices.verify_baseline(record['observed'].get('gpu_devices', {}), deployment)
|
||||
old = {m['container_path']: m for m in deployment.get('mounts', [])}
|
||||
new = {m['container_path']: m for m in desired.get('mounts', [])}
|
||||
recorded_files = {m['source']: bool(m.get('read_only')) for m in deployment.get('mounts', []) if is_file_bind(m, files)}
|
||||
if files != recorded_files:
|
||||
raise ValueError(translate('A container mount has a source, backup or permission different from the saved record'))
|
||||
old = {m['container_path']: m for m in deployment.get('mounts', []) if not is_file_bind(m, files)}
|
||||
new = {m['container_path']: m for m in desired.get('mounts', []) if not is_file_bind(m, files)}
|
||||
if set(actual) != set(old):
|
||||
raise ValueError(translate('The container disks do not match the saved record'))
|
||||
for target, mount in actual.items():
|
||||
@@ -500,24 +558,31 @@ def preflight(record, candidate, config, coordinated=None):
|
||||
check = effective_healthcheck(candidate['template'])
|
||||
if not check and not coordinated:
|
||||
raise ValueError(translate('Updates are not available yet for this application in this beta'))
|
||||
mac = next((item[7:] for item in cfg['net0'].split(',') if item.startswith('hwaddr=')), None)
|
||||
if deployment.get('host_monitor') and 'net0' not in cfg:
|
||||
return cfg, actual, None
|
||||
mac = next((item[7:] for item in cfg.get('net0', '').split(',') if item.startswith('hwaddr=')), None)
|
||||
if not mac:
|
||||
raise ValueError(translate('The MAC address of the container cannot be kept'))
|
||||
return cfg, actual, mac
|
||||
|
||||
|
||||
def freeze_host_sources(record, candidate, acknowledge_external_data):
|
||||
old = {m['source'] for m in record['deployment'].get('mounts', []) if m['type'] == 'host-bind'}
|
||||
wanted = {m['source']: m for m in candidate['deployment'].get('mounts', []) if m['type'] == 'host-bind'}
|
||||
if (old or wanted) and not acknowledge_external_data:
|
||||
def freeze_host_sources(record, candidate, acknowledge_external_data, files=()):
|
||||
# A single file mounted from the host holds no data of the application:
|
||||
# the installer mounts it again, and nothing of it is pinned.
|
||||
old = {m['source'] for m in record['deployment'].get('mounts', [])
|
||||
if m['type'] == 'host-bind' and not is_file_bind(m, files)}
|
||||
wanted = {m['source']: m for m in candidate['deployment'].get('mounts', [])
|
||||
if m['type'] == 'host-bind' and not is_file_bind(m, files)}
|
||||
declared = declared_sources(record) | declared_sources(candidate)
|
||||
if (old | set(wanted)) - declared and not acknowledge_external_data:
|
||||
raise ValueError(translate('Host data is not restored by the backup; confirm it with --acknowledge-external-data'))
|
||||
baseline = record['observed'].get('host_bind_sources', {})
|
||||
original = {p: host_mounts.validate_source(p) for p in old}
|
||||
original = {p: host_mounts.validate_source(p, declared=p in declared) for p in old}
|
||||
for source, previous in baseline.items():
|
||||
if source in original and not host_mounts.same_source(previous, original[source]):
|
||||
raise ValueError(translate('A shared source does not match its recorded identity'))
|
||||
desired = {p: original[p] if p in original else
|
||||
host_mounts.validate_source(p, allow_missing=m.get('create_if_missing') is True)
|
||||
host_mounts.validate_source(p, allow_missing=m.get('create_if_missing') is True, declared=p in declared)
|
||||
for p, m in wanted.items()}
|
||||
if old or wanted:
|
||||
log('host directories: not included in the backup and not reverted by a recovery')
|
||||
@@ -529,7 +594,10 @@ def freeze_host_sources(record, candidate, acknowledge_external_data):
|
||||
def check_runtime_mounts(config, deployment):
|
||||
runtime_settings.check(config, deployment, deployment['vmid'])
|
||||
actual = mounts(config)
|
||||
declared = {m['container_path']: m for m in deployment.get('mounts', [])}
|
||||
files = runtime_settings.file_binds(config)
|
||||
if files != {m['source']: bool(m.get('read_only')) for m in deployment.get('mounts', []) if is_file_bind(m, files)}:
|
||||
raise ValueError(translate('The mounts of the new container do not match the proposal'))
|
||||
declared = {m['container_path']: m for m in deployment.get('mounts', []) if not is_file_bind(m, files)}
|
||||
if actual.keys() != declared.keys():
|
||||
raise ValueError(translate('The mounts of the new container do not match the proposal'))
|
||||
for target, value in actual.items():
|
||||
@@ -548,7 +616,7 @@ def pin_host_source(root, vmid, journal, source):
|
||||
or record['installation_id'] != state['record']['installation_id'] or record['status'] != 'updating'):
|
||||
raise ValueError(translate('Mount not authorized by the operation'))
|
||||
expected = state['desired_host_sources'][source]
|
||||
current = host_mounts.validate_source(source)
|
||||
current = host_mounts.validate_source(source, declared=expected.get('declared', False))
|
||||
if ((expected['exists'] and not host_mounts.same_source(expected, current))
|
||||
or current['resolved_path'] != expected['resolved_path']):
|
||||
raise ValueError(translate('The host directory changed before it was mounted'))
|
||||
@@ -582,10 +650,48 @@ def stop(vmid):
|
||||
raise ValueError(translate('The container did not stop; its disks are not touched'))
|
||||
|
||||
|
||||
def own_addresses(vmid):
|
||||
"""The IPv4 addresses of the interfaces Proxmox gave the container. One
|
||||
that runs Docker inside also has the addresses of its inner bridges."""
|
||||
names = re.findall(r'^net[0-9]+: (?:.*,)?name=([A-Za-z0-9_.-]+)', run('pct', 'config', str(vmid)).decode(), re.M)
|
||||
found = []
|
||||
for name in names:
|
||||
try:
|
||||
listed = run('lxc-attach', '-n', str(vmid), '-s', 'NETWORK', '--',
|
||||
'ip', '-4', '-o', 'addr', 'show', 'dev', name, 'scope', 'global').decode()
|
||||
except RuntimeError:
|
||||
continue
|
||||
found += re.findall(r'\binet ([0-9.]+)/', listed)
|
||||
return found
|
||||
|
||||
|
||||
def haos_healthcheck(vmid, timeout):
|
||||
import haos_healthcheck as haos
|
||||
haos.note = log
|
||||
started, ip = time.monotonic(), None
|
||||
while ip is None and time.monotonic() - started < 60:
|
||||
if run('pct', 'status', str(vmid)).strip() != b'status: running':
|
||||
raise ValueError(translate('The restored service stopped; the recovery is not confirmed'))
|
||||
ip = next(iter(own_addresses(vmid)), None)
|
||||
if ip is None:
|
||||
time.sleep(2)
|
||||
try:
|
||||
if ip is None:
|
||||
raise RuntimeError('no address')
|
||||
url = haos.wait_ready(vmid, ip, timeout)[0]['url']
|
||||
except RuntimeError as error:
|
||||
log(f'restored Home Assistant OS not confirmed: {error}')
|
||||
raise ValueError(translate('The restored service did not pass its health check'))
|
||||
log(f'restored service responding: {url}')
|
||||
return url
|
||||
|
||||
|
||||
def healthcheck(vmid, template):
|
||||
check = effective_healthcheck(template)
|
||||
if not check:
|
||||
return None
|
||||
if check.get('type') == 'haos':
|
||||
return haos_healthcheck(vmid, int(check['timeout_seconds']))
|
||||
if check.get('type') == 'running':
|
||||
stability = int(check.get('stability_seconds', 20))
|
||||
started = time.monotonic()
|
||||
@@ -603,10 +709,12 @@ def healthcheck(vmid, template):
|
||||
or not 0 <= stability < timeout <= 3600):
|
||||
raise ValueError(translate('Invalid health check'))
|
||||
started, stable_since = time.monotonic(), None
|
||||
# A monitor of the host answers on the network of the host itself.
|
||||
shared = parse_config(run('pct', 'config', str(vmid))).get('lxc.net.0.type') == 'none'
|
||||
while time.monotonic() - started < timeout:
|
||||
if run('pct', 'status', str(vmid)).strip() != b'status: running':
|
||||
raise ValueError(translate('The restored service stopped; the recovery is not confirmed'))
|
||||
addresses = run('lxc-info', '-n', str(vmid), '-iH').decode().splitlines()
|
||||
addresses = ['127.0.0.1'] if shared else run('lxc-info', '-n', str(vmid), '-iH').decode().splitlines()
|
||||
ip = next((a for a in addresses if re.fullmatch(r'[0-9]+(?:\.[0-9]+){3}', a)), None)
|
||||
ok = False
|
||||
if ip:
|
||||
@@ -820,16 +928,6 @@ def backup_size(vmid):
|
||||
return total
|
||||
|
||||
|
||||
def require_backup_space(directory, vmids):
|
||||
"""The data in use on the backed-up volumes, plus a margin, must fit in
|
||||
`directory`; data that is already compressed does not shrink."""
|
||||
needed = int(sum(backup_size(vmid) for vmid in vmids) * 1.1) + 1024**3
|
||||
free = shutil.disk_usage(directory).free
|
||||
if free < needed:
|
||||
raise ValueError(f"{translate('Not enough free space for the backup')} "
|
||||
f"({translate('needed')}: {gib(needed)}, {translate('free')}: {gib(free)}, {directory})")
|
||||
|
||||
|
||||
def prune_backups(root, vmid):
|
||||
"""The backups of closed operations are removed once the container works
|
||||
with its new image; their journal and log stay."""
|
||||
@@ -843,9 +941,7 @@ def prune_backups(root, vmid):
|
||||
continue
|
||||
if phase not in ('committed', 'rolled-back'):
|
||||
continue
|
||||
for backup in (directory / 'backup').glob('vzdump-lxc-*'):
|
||||
if backup.is_file() and not backup.is_symlink():
|
||||
backup.unlink()
|
||||
work_backup.clear(directory)
|
||||
|
||||
|
||||
def check_archive(archive):
|
||||
@@ -892,7 +988,8 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
|
||||
before = run('pct', 'config', str(vmid))
|
||||
cfg, actual, mac = preflight(record, candidate, before, coordinated)
|
||||
description = original_description({'record': record, 'before_config': before.decode()})
|
||||
original_sources, desired_sources = freeze_host_sources(record, candidate, acknowledge_external_data)
|
||||
original_sources, desired_sources = freeze_host_sources(record, candidate, acknowledge_external_data,
|
||||
runtime_settings.file_binds(before))
|
||||
original_gpu = gpu_devices.planned(record['deployment'])
|
||||
desired_gpu = gpu_devices.planned(candidate['deployment'])
|
||||
resources = json.loads(run('pvesh', 'get', '/cluster/ha/resources', '--output-format', 'json'))
|
||||
@@ -912,10 +1009,17 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
|
||||
for p in set(image['defaults'].get('Volumes') or {}) - non_persistent_image_volumes(candidate['template'])):
|
||||
raise ValueError(translate('The new image requires additional persistent paths; use Modify'))
|
||||
directory = instances.location(root, vmid).parent / 'transactions' / uuid.uuid4().hex
|
||||
# Where the backup goes is settled, and checked for room, before anything
|
||||
# is stopped.
|
||||
work = directory if coordinated else work_backup.locate(directory, [vmid], record, vmid)
|
||||
private_directory(directory)
|
||||
open_log(directory)
|
||||
if not coordinated:
|
||||
require_backup_space(directory, [vmid])
|
||||
if work != directory:
|
||||
work_backup.prepare(work)
|
||||
work_backup.remember(directory, work)
|
||||
if show:
|
||||
msg_info2(f"{translate('This backup does not fit on the system disk of the host; it is made on the storage:')} "
|
||||
f"{record['deployment'].get(work_backup.KEY)}")
|
||||
journal = directory / 'transaction.json'
|
||||
runtime_deployment = copy.deepcopy(candidate['deployment'])
|
||||
runtime_deployment['network']['mac_address'] = mac
|
||||
@@ -970,7 +1074,7 @@ def apply(root, vmid, archive, operation, proposal=None, registry_digest=None, i
|
||||
run('zstd', '-t', backup['archive'])
|
||||
state.update(backup=backup['archive'], backup_sha256=backup['sha256'])
|
||||
else:
|
||||
backup_dir = directory / 'backup'
|
||||
backup_dir = work / 'backup'
|
||||
private_directory(backup_dir)
|
||||
backup = verified_backup(vmid, backup_dir, backup_compression,
|
||||
translate('The backup could not be identified; the image is not replaced'), show)
|
||||
@@ -1195,6 +1299,8 @@ def recover(root, journal):
|
||||
if show:
|
||||
msg_ok(translate('Recovery completed. The container had not been modified yet.'))
|
||||
return
|
||||
if not Path(backup).is_file():
|
||||
raise ValueError(f"{translate('The backup of this operation cannot be read. If it is on another storage, make it available and recover the operation again:')} {backup}")
|
||||
if filehash(backup) != state['backup_sha256']:
|
||||
raise ValueError(translate('The backup was altered; recovery blocked'))
|
||||
run('gzip' if state.get('backup_compression') == 'gzip' else 'zstd', '-t', backup)
|
||||
|
||||
@@ -17,6 +17,7 @@ import subprocess
|
||||
import sys
|
||||
|
||||
import oci_image_cache as image_cache
|
||||
import oci_work_backup as work_backup
|
||||
import oci_instances as instances
|
||||
from oci_installation_state import parse_config
|
||||
import oci_console
|
||||
@@ -119,9 +120,11 @@ def _unlink(path):
|
||||
|
||||
def remove_host_state(vmid):
|
||||
"""What the installation kept on the host for a container that is gone:
|
||||
its sysctl include, the Rclone mount hookscript and the views it
|
||||
published, and its registration in the App tab of ProxMenux Monitor."""
|
||||
for include in (runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid)):
|
||||
its sysctl include, its seccomp profile, the Rclone mount hookscript and
|
||||
the views it published, and its registration in the App tab of ProxMenux
|
||||
Monitor."""
|
||||
for include in (runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid),
|
||||
runtime_settings.seccomp_path(vmid)):
|
||||
_unlink(include)
|
||||
hook = SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh'
|
||||
if hook.is_file() and not hook.is_symlink():
|
||||
@@ -179,7 +182,7 @@ def release_shared_host_files(hookscripts):
|
||||
def _leftovers(vmid):
|
||||
"""Whether anything of the container is still on the host."""
|
||||
paths = [runtime_settings.include_path(vmid), runtime_settings.legacy_include_path(vmid),
|
||||
SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh', MONITOR_APPS / f'{int(vmid)}.json',
|
||||
runtime_settings.seccomp_path(vmid), SNIPPETS / f'proxmenux-rclone-{int(vmid)}-fuse-hook.sh', MONITOR_APPS / f'{int(vmid)}.json',
|
||||
CLUSTER_RECORDS / f'{int(vmid)}.json',
|
||||
*oci_console.LOG_DIR.glob(f'{int(vmid)}.console.log*')]
|
||||
return any(path.exists() for path in paths)
|
||||
@@ -193,6 +196,7 @@ def sweep_orphans(root):
|
||||
found = {int(d.name) for d in root.iterdir() if d.name.isdecimal()} if root.is_dir() else set()
|
||||
for directory, pattern in ((runtime_settings.include_path(0).parent, r'([0-9]+)\.sysctls'),
|
||||
(runtime_settings.legacy_include_path(0).parent, r'([0-9]+)\.proxmenux-sysctls'),
|
||||
(runtime_settings.seccomp_path(0).parent, r'([0-9]+)\.proxmenux-seccomp'),
|
||||
(oci_console.LOG_DIR, r'([0-9]+)\.console\.log.*'),
|
||||
(SNIPPETS, r'proxmenux-rclone-([0-9]+)-fuse-hook\.sh'),
|
||||
(CLUSTER_RECORDS, r'([0-9]+)\.json')):
|
||||
@@ -347,6 +351,7 @@ def remove(root, vmid):
|
||||
lifecycle.unlink()
|
||||
for member in members:
|
||||
directory = instances.location(root, member).parent
|
||||
work_backup.forget(root, member)
|
||||
if directory.is_dir() and not directory.is_symlink():
|
||||
shutil.rmtree(directory)
|
||||
msg_ok(translate('Saved record removed'))
|
||||
|
||||
@@ -18,6 +18,9 @@ def sysctl_content(deployment):
|
||||
raise ValueError(translate('Invalid or duplicated network sysctl'))
|
||||
seen.add(name)
|
||||
result.append(f'lxc.sysctl.{name} = {value}\n')
|
||||
# Proxmox does not take this setting in the configuration of a container.
|
||||
if (deployment.get('security', {}).get('options') or {}).get('no_new_privileges'):
|
||||
result.append('lxc.no_new_privs = 1\n')
|
||||
return ''.join(result)
|
||||
|
||||
|
||||
@@ -58,6 +61,28 @@ def legacy_include_path(vmid):
|
||||
return Path(f'/etc/pve/lxc/{int(vmid)}.proxmenux-sysctls')
|
||||
|
||||
|
||||
# A host monitor shares the process and network namespaces of the host. Proxmox
|
||||
# takes those two settings from an include, the same for every monitor.
|
||||
HOST_MONITOR_INCLUDES = (CLUSTER_DIR / 'host-monitor', Path('/etc/pve/lxc/proxmenux-host-monitor'))
|
||||
HOST_MONITOR_CONTENT = 'lxc.namespace.share.pid = 1\nlxc.namespace.share.net = 1\n'
|
||||
|
||||
|
||||
def host_monitor_include(includes):
|
||||
"""Whether the includes are the one of a host monitor, unchanged."""
|
||||
if len(includes) != 1 or Path(includes[0]) not in HOST_MONITOR_INCLUDES:
|
||||
return False
|
||||
try:
|
||||
return Path(includes[0]).read_text() == HOST_MONITOR_CONTENT
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def seccomp_path(vmid):
|
||||
"""The seccomp profile the installer writes for a container that asks for
|
||||
it relaxed."""
|
||||
return Path(f'/etc/pve/lxc/{int(vmid)}.proxmenux-seccomp')
|
||||
|
||||
|
||||
def check(config, deployment, vmid):
|
||||
expected = tmpfs_lines(deployment)
|
||||
lines = config.decode().splitlines()
|
||||
@@ -65,6 +90,10 @@ def check(config, deployment, vmid):
|
||||
if sorted(actual) != sorted(expected):
|
||||
raise ValueError(translate('The tmpfs mounts of the container differ from the saved record'))
|
||||
includes = [line.split(': ',1)[1] for line in lines if line.startswith('lxc.include: ')]
|
||||
if deployment.get('host_monitor'):
|
||||
if not host_monitor_include(includes):
|
||||
raise ValueError(translate('The sysctl include is unknown or differs from the saved record'))
|
||||
return
|
||||
content = sysctl_content(deployment)
|
||||
allowed = [[str(include_path(vmid))], [str(legacy_include_path(vmid))]] if content else [[]]
|
||||
if includes not in allowed:
|
||||
@@ -78,12 +107,34 @@ def check(config, deployment, vmid):
|
||||
raise ValueError(translate('The sysctl content was modified outside the saved record'))
|
||||
|
||||
|
||||
FILE_BIND = re.compile(r'(/\S+) \S+ none bind,create=file(,ro)? 0 0')
|
||||
|
||||
|
||||
def file_binds(config):
|
||||
"""The host files mounted in the container: {source: read only}. Proxmox
|
||||
takes a directory as a mount point and a single file as a raw LXC entry,
|
||||
so these are not among the mount points."""
|
||||
found = {}
|
||||
for line in config.decode().splitlines():
|
||||
match = FILE_BIND.fullmatch(line.split(': ', 1)[1]) if line.startswith('lxc.mount.entry: ') else None
|
||||
if match:
|
||||
found[match.group(1)] = bool(match.group(2))
|
||||
return found
|
||||
|
||||
|
||||
def filter_config(config, deployment):
|
||||
"""The configuration without the entries of the mounts the deployment
|
||||
declares itself: what is left belongs to the acceleration profile."""
|
||||
declared = set(tmpfs_lines(deployment))
|
||||
files = {m['source'] for m in deployment.get('mounts', []) if m.get('type') == 'host-bind'}
|
||||
|
||||
def own(line):
|
||||
entry = line.decode().strip().split(': ', 1)[1]
|
||||
match = FILE_BIND.fullmatch(entry)
|
||||
return entry in declared or bool(match and match.group(1) in files)
|
||||
return b''.join(line for line in config.splitlines(keepends=True)
|
||||
if not line.startswith(b'lxc.include: ') and not (
|
||||
line.startswith(b'lxc.mount.entry: ')
|
||||
and line.decode().strip().split(': ',1)[1] in declared))
|
||||
line.startswith(b'lxc.mount.entry: ') and own(line)))
|
||||
|
||||
|
||||
def check_recovery(config, state):
|
||||
@@ -94,6 +145,8 @@ def check_recovery(config, state):
|
||||
raise ValueError(translate('A tmpfs mount is not part of the journal; recovery blocked'))
|
||||
if line.startswith('lxc.include: '):
|
||||
path = Path(line.split(': ', 1)[1])
|
||||
if any(plan.get('host_monitor') for plan in plans) and host_monitor_include([str(path)]):
|
||||
continue
|
||||
if path not in (include_path(state['vmid']), legacy_include_path(state['vmid'])):
|
||||
raise ValueError(translate('An include is not part of the journal; recovery blocked'))
|
||||
contents = {sysctl_content(plan) for plan in plans} - {''}
|
||||
|
||||
@@ -17,6 +17,7 @@ import oci_instance_transaction as member_tx
|
||||
import oci_stack_plan
|
||||
import oci_stack_transaction as stack_tx
|
||||
import oci_stack_replay as replay
|
||||
import oci_work_backup as work_backup
|
||||
from oci_installation_state import image_from_archive, parse_config, private_directory, sha
|
||||
from oci_update_current import resolve_archive
|
||||
from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error
|
||||
@@ -251,7 +252,17 @@ class NativeAdapter:
|
||||
for s in self.services.values() if s['vmid'] != primary_id and not s.get('deferred_setup')]
|
||||
if spec.get('schema') != 1 or spec.get('dependencies') != expected:
|
||||
raise ValueError(translate('The dependency hook and the stack recipe differ'))
|
||||
member_tx.require_backup_space(self.journal.parent, list(self.records))
|
||||
self.work_base()
|
||||
|
||||
def work_base(self):
|
||||
"""Where the backups of this operation go. It is decided, and checked
|
||||
for room, until the first backup exists; afterwards it is read back,
|
||||
so the space the backups themselves take is not asked for again."""
|
||||
directory = self.journal.parent
|
||||
if (directory / work_backup.MARK).exists() or any(directory.glob('backup-*')):
|
||||
return work_backup.recall(directory)
|
||||
primary = self.plan['primary_vmid']
|
||||
return work_backup.locate(directory, list(self.records), self.records[primary], primary)
|
||||
|
||||
def is_running(self, vmid):
|
||||
return instances.command('pct', 'status', str(vmid)).strip() == b'status: running'
|
||||
@@ -331,7 +342,13 @@ class NativeAdapter:
|
||||
|
||||
def backup(self, vmid, identity):
|
||||
self.validate(self.plan)
|
||||
directory = self.journal.parent / ('backup-%s' % vmid)
|
||||
base = self.work_base()
|
||||
if base != self.journal.parent and not (self.journal.parent / work_backup.MARK).exists():
|
||||
work_backup.prepare(base)
|
||||
work_backup.remember(self.journal.parent, base)
|
||||
msg_warn(f"{translate('This backup does not fit on the system disk of the host; it is made on the storage:')} "
|
||||
f"{self.records[self.plan['primary_vmid']]['deployment'].get(work_backup.KEY)}")
|
||||
directory = base / ('backup-%s' % vmid)
|
||||
private_directory(directory)
|
||||
archive = member_tx.verified_backup(vmid, directory, 'zstd',
|
||||
translate('The backup of a member could not be identified'))
|
||||
@@ -339,6 +356,8 @@ class NativeAdapter:
|
||||
|
||||
def verify_backups(self, backups):
|
||||
for backup in backups.values():
|
||||
if not Path(backup['archive']).is_file():
|
||||
raise ValueError(f"{translate('The backup of this operation cannot be read. If it is on another storage, make it available and recover the operation again:')} {backup['archive']}")
|
||||
if member_tx.filehash(backup['archive']) != backup['sha256']:
|
||||
raise ValueError(translate('A backup was modified'))
|
||||
member_tx.run('zstd', '-t', backup['archive'])
|
||||
@@ -579,7 +598,7 @@ class NativeAdapter:
|
||||
self.release_stages(recovered=state['phase'] == 'rolled-back')
|
||||
if self.keep_backup and state['phase'] == 'committed':
|
||||
import oci_keep_backup
|
||||
for backup in sorted(self.journal.parent.glob('backup-*/vzdump-lxc-*.tar.zst')):
|
||||
for backup in sorted(work_backup.recall(self.journal.parent).glob('backup-*/vzdump-lxc-*.tar.zst')):
|
||||
kept = oci_keep_backup.keep(backup, self.keep_backup)
|
||||
if kept:
|
||||
msg_ok(f"{translate('Backup kept in')} {self.keep_backup}: {Path(kept).name}")
|
||||
@@ -621,9 +640,7 @@ class NativeAdapter:
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if phase in stack_tx.TERMINAL:
|
||||
for backup in directory.glob('backup-*/vzdump-lxc-*'):
|
||||
if backup.is_file() and not backup.is_symlink():
|
||||
backup.unlink()
|
||||
work_backup.clear(directory)
|
||||
|
||||
|
||||
# The stack journal of this run, for the summary after a failure.
|
||||
|
||||
@@ -12,6 +12,7 @@ import tempfile
|
||||
|
||||
import oci_instances as instances
|
||||
import oci_instance_transaction as transaction
|
||||
import oci_work_backup as work_backup
|
||||
from oci_installation_state import image_from_archive, same_image
|
||||
from oci_ui import translate, msg_info, msg_ok, msg_warn, msg_error, msg_info2
|
||||
|
||||
@@ -102,6 +103,12 @@ def resolve_archive(desired, config, current=None, check=None, required_digest=N
|
||||
return archive, digest
|
||||
msg_warn(translate('The cached image is damaged; it will be downloaded again.'))
|
||||
archive.unlink()
|
||||
# The compressed layers are what the download writes; nothing was changed yet.
|
||||
size, free = int(candidate.get('size') or 0), work_backup.available(archive.parent)
|
||||
if size and free < int(size * 1.1) + 256 * 1024**2:
|
||||
raise ValueError(f"{translate('The new image does not fit on the storage where images are downloaded:')} {storage}. "
|
||||
f"{translate('Needed:')} {work_backup.gib(int(size * 1.1) + 256 * 1024**2)}. "
|
||||
f"{translate('Free:')} {work_backup.gib(free)}. {translate('The application was not modified.')}")
|
||||
# A download can come back complete yet damaged when the connection drops
|
||||
# and the transfer resumes; the integrity check catches it, and a second
|
||||
# download is what repairs it.
|
||||
@@ -170,8 +177,8 @@ def update(vmid, acknowledge_external_data=False, proposal=None, keep_backup=Non
|
||||
transaction.preflight(record, desired, config)
|
||||
msg_ok(translate('Container checked'))
|
||||
current = record['observed']['image'] if operation == 'update' else None
|
||||
archive, digest = resolve_archive(desired, config, current, lambda: transaction.require_backup_space(
|
||||
instances.location(instances.ROOT, vmid).parent, [vmid]))
|
||||
archive, digest = resolve_archive(desired, config, current, lambda: work_backup.locate(
|
||||
instances.location(instances.ROOT, vmid).parent / 'transactions' / 'check', [vmid], record, vmid))
|
||||
if archive is None:
|
||||
msg_ok(translate('The image is already up to date; nothing was changed.'))
|
||||
return
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Where the backup taken before changing an OCI application is written.
|
||||
|
||||
An update, a modification or a recreation stops the application and takes a
|
||||
verified backup first, to go back to if the new container does not work; it
|
||||
is deleted when the operation ends. That backup goes next to the record of
|
||||
the application, on the system disk of the host. An application with a lot
|
||||
of data may not fit there: the user can then choose a file storage of the
|
||||
host that accepts backups, and the choice is kept in the record so later
|
||||
operations, the scheduled ones included, use it without asking.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
import oci_instances as instances
|
||||
from oci_ui import translate
|
||||
|
||||
FOLDER = 'proxmenux-oci-work'
|
||||
KEY = 'work_storage'
|
||||
MARK = 'backup-location'
|
||||
STORAGE_RE = re.compile(r'[A-Za-z0-9._-]{1,64}')
|
||||
|
||||
|
||||
def gib(size):
|
||||
return f'{size / 1024**3:.1f} GB'
|
||||
|
||||
|
||||
def required(vmids):
|
||||
"""The data in use on the backed-up volumes plus a margin: data that is
|
||||
already compressed does not shrink."""
|
||||
import oci_instance_transaction as transaction
|
||||
return int(sum(transaction.backup_size(vmid) for vmid in vmids) * 1.1) + 1024**3
|
||||
|
||||
|
||||
def available(path):
|
||||
"""Free bytes where `path` is or will be created."""
|
||||
path = Path(path)
|
||||
while not path.exists() and path != path.parent:
|
||||
path = path.parent
|
||||
return shutil.disk_usage(path).free
|
||||
|
||||
|
||||
def file_storages():
|
||||
"""The storages of this node that accept backups and are a directory of
|
||||
the host, with their free space: [{'storage', 'path', 'free'}]."""
|
||||
node = socket.gethostname().split('.', 1)[0]
|
||||
listed = subprocess.run(['pvesh', 'get', f'/nodes/{node}/storage', '--content', 'backup', '--enabled', '1',
|
||||
'--output-format', 'json'], capture_output=True, text=True, timeout=60)
|
||||
if listed.returncode != 0:
|
||||
return []
|
||||
found = []
|
||||
for entry in json.loads(listed.stdout):
|
||||
name = str(entry.get('storage') or '')
|
||||
if not entry.get('active') or not STORAGE_RE.fullmatch(name):
|
||||
continue
|
||||
path = storage_path(name)
|
||||
if path is not None and path.is_dir():
|
||||
found.append({'storage': name, 'path': path, 'free': available(path)})
|
||||
return found
|
||||
|
||||
|
||||
def storage_path(storage):
|
||||
"""The directory of a file storage that accepts backups, or None."""
|
||||
if not STORAGE_RE.fullmatch(storage or ''):
|
||||
return None
|
||||
result = subprocess.run(['pvesh', 'get', f'/storage/{storage}', '--output-format', 'json'],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
info = json.loads(result.stdout)
|
||||
if 'backup' not in str(info.get('content', '')).split(',') or not info.get('path'):
|
||||
return None
|
||||
return Path(info['path'])
|
||||
|
||||
|
||||
def explain(needed, free, storage=None, storage_free=None):
|
||||
text = (f"{translate('The backup made before changing the application does not fit on the system disk of the host.')} "
|
||||
f"{translate('Needed:')} {gib(needed)}. {translate('Free:')} {gib(free)}.")
|
||||
if storage:
|
||||
text += (f" {translate('The storage chosen for this backup has no room either:')} {storage}"
|
||||
+ (f" ({gib(storage_free)})." if storage_free is not None else '.'))
|
||||
return (f"{text} {translate('The application was not modified.')} "
|
||||
f"{translate('Free space on the host, or choose another storage for this backup from Manage installed OCI applications.')}")
|
||||
|
||||
|
||||
def locate(default, vmids, record, owner):
|
||||
"""The directory the backups of one operation are written under: `default`
|
||||
when they fit there, otherwise a folder of its own on the storage chosen
|
||||
for the application. Nothing is created. Raises ValueError with what is
|
||||
needed when there is no room."""
|
||||
default = Path(default)
|
||||
needed = required(vmids)
|
||||
free = available(default)
|
||||
if free >= needed:
|
||||
return default
|
||||
storage = (record.get('deployment') or {}).get(KEY)
|
||||
path = storage_path(storage) if storage else None
|
||||
if path is None or not path.is_dir():
|
||||
raise ValueError(explain(needed, free))
|
||||
storage_free = available(path)
|
||||
if storage_free < needed:
|
||||
raise ValueError(explain(needed, free, storage, storage_free))
|
||||
return path / FOLDER / str(int(owner)) / default.name
|
||||
|
||||
|
||||
def prepare(base):
|
||||
"""Create the folder of an operation on another storage and make sure it
|
||||
can be written, before anything is stopped."""
|
||||
base = Path(base)
|
||||
try:
|
||||
base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
probe = base / '.write-test'
|
||||
probe.write_text('')
|
||||
probe.unlink()
|
||||
except OSError as error:
|
||||
raise ValueError(f"{translate('The storage chosen for this backup cannot be written:')} {base} ({error.strerror or error})")
|
||||
|
||||
|
||||
def remember(directory, base):
|
||||
"""Leave in the folder of the operation where its backups are, when they
|
||||
are not in that folder: a recovery or a cleanup reads it from there."""
|
||||
directory, base = Path(directory), Path(base)
|
||||
if base != directory:
|
||||
(directory / MARK).write_text(str(base) + '\n')
|
||||
|
||||
|
||||
def recall(directory):
|
||||
"""Where the backups of the operation kept in `directory` are."""
|
||||
directory = Path(directory)
|
||||
try:
|
||||
text = (directory / MARK).read_text().strip()
|
||||
except OSError:
|
||||
return directory
|
||||
base = Path(text)
|
||||
return base if base.is_absolute() and FOLDER in base.parts else directory
|
||||
|
||||
|
||||
def clear(directory):
|
||||
"""Delete the backups of a closed operation, and its folder on another
|
||||
storage once it is empty. The journal and the log of the operation stay."""
|
||||
directory = Path(directory)
|
||||
base = recall(directory)
|
||||
if base != directory and not base.exists():
|
||||
return
|
||||
for backup in list(base.glob('backup/vzdump-lxc-*')) + list(base.glob('backup-*/vzdump-lxc-*')):
|
||||
if backup.is_file() and not backup.is_symlink():
|
||||
backup.unlink()
|
||||
if base == directory:
|
||||
return
|
||||
for folder in sorted(base.glob('backup*'), reverse=True):
|
||||
_remove_empty(folder)
|
||||
# The folder of the operation and the one of the application, never the
|
||||
# one shared by every application nor the storage itself.
|
||||
for folder in (base, base.parent):
|
||||
if folder.parent.name == FOLDER or folder.parent.parent.name == FOLDER:
|
||||
_remove_empty(folder)
|
||||
|
||||
|
||||
def _remove_empty(folder):
|
||||
try:
|
||||
if folder.is_dir() and not folder.is_symlink():
|
||||
folder.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def forget(root, vmid):
|
||||
"""Before the record of an application is deleted: remove what its
|
||||
operations left on another storage."""
|
||||
transactions = instances.location(root, vmid).parent / 'transactions'
|
||||
if not transactions.is_dir():
|
||||
return
|
||||
for directory in transactions.iterdir():
|
||||
base = recall(directory)
|
||||
if base != directory and base.exists() and FOLDER in base.parts:
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
_remove_empty(base.parent)
|
||||
|
||||
|
||||
def application(root, vmid):
|
||||
"""The containers of the application `vmid` belongs to."""
|
||||
record = instances.read(root, vmid)
|
||||
primary_id = int((record.get('stack_member') or {}).get('primary_vmid') or vmid)
|
||||
primary = record if primary_id == vmid else instances.read(root, primary_id)
|
||||
members = [int(member['vmid']) for member in (primary.get('stack') or {}).get('members') or []]
|
||||
return primary_id, members or [vmid]
|
||||
|
||||
|
||||
def status(root, vmid):
|
||||
"""What the menu needs to know before an operation: whether the backup
|
||||
fits on the system disk, the storage chosen earlier and the ones that
|
||||
have room."""
|
||||
primary_id, vmids = application(root, vmid)
|
||||
record = instances.read(root, primary_id)
|
||||
default = instances.location(root, primary_id).parent
|
||||
needed, free = required(vmids), available(default)
|
||||
saved = (record.get('deployment') or {}).get(KEY)
|
||||
path = storage_path(saved) if saved else None
|
||||
saved_free = available(path) if path is not None and path.is_dir() else None
|
||||
return {'needed': needed, 'free': free, 'fits': free >= needed, 'saved': saved,
|
||||
'saved_fits': saved_free is not None and saved_free >= needed,
|
||||
'candidates': [{'storage': item['storage'], 'free': item['free']}
|
||||
for item in file_storages() if item['free'] >= needed]}
|
||||
|
||||
|
||||
def choose(root, vmid, storage):
|
||||
"""Keep the storage chosen for the backups of an application in the
|
||||
record of each of its containers. None forgets it."""
|
||||
import oci_carried_record
|
||||
if storage is not None and storage_path(storage) is None:
|
||||
raise ValueError(f"{translate('The storage does not accept backups:')} {storage}")
|
||||
with instances.locked(root):
|
||||
_, vmids = application(root, vmid)
|
||||
for member in vmids:
|
||||
record = instances.read(root, member)
|
||||
deployment = record.setdefault('deployment', {})
|
||||
if storage is None:
|
||||
deployment.pop(KEY, None)
|
||||
else:
|
||||
deployment[KEY] = storage
|
||||
instances.write(instances.location(root, member), record)
|
||||
oci_carried_record.carry(root, member, mount_stopped=False)
|
||||
return vmids
|
||||
Reference in New Issue
Block a user