mirror of
https://github.com/MacRimi/ProxMenux.git
synced 2026-11-02 10:52:22 +00:00
feat: discover native LXC applications on demand with evidence
This commit is contained in:
VAIO73
@@ -149,6 +149,7 @@ cp "$SCRIPT_DIR/managed_installs.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "
|
||||
cp "$SCRIPT_DIR/lxc_update_observation.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_update_finalization.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_check_job.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/native_app_discovery.py" "$APP_DIR/usr/bin/"
|
||||
cp "$SCRIPT_DIR/lxc_apps.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ lxc_apps.py not found"
|
||||
cp "$SCRIPT_DIR/custom_links.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ custom_links.py not found"
|
||||
cp "$SCRIPT_DIR/recreate_docker_container.py" "$APP_DIR/usr/bin/" 2>/dev/null || echo "⚠️ recreate_docker_container.py not found"
|
||||
|
||||
@@ -2276,6 +2276,8 @@ def _handle_guest_lifecycle(vmid: str, vm_type: str, action: str) -> None:
|
||||
# or stops; drop the cached IP so the next /api/vms poll re-reads
|
||||
# it via `lxc-info`. QEMU guests do not touch this cache.
|
||||
if guest_type == 'lxc':
|
||||
import native_app_discovery
|
||||
native_app_discovery.mark_stale(guest_id)
|
||||
_invalidate_lxc_ip(guest_id)
|
||||
if action in ('start', 'reboot'):
|
||||
if guest_type == 'lxc':
|
||||
@@ -14090,6 +14092,20 @@ def api_vm_apps_suggestions(vmid):
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/api/vms/<int:vmid>/apps/native-discovery', methods=['GET', 'POST'])
|
||||
@require_auth
|
||||
def api_vm_apps_native_discovery(vmid):
|
||||
"""Only the Find applications button requests POST. GET never probes."""
|
||||
import native_app_discovery
|
||||
import lxc_apps
|
||||
result = (native_app_discovery.discover(vmid) if request.method == 'POST'
|
||||
else native_app_discovery.cached(vmid))
|
||||
suggestions = _vm_cache_get(_vm_app_suggestions_cache, vmid, _VM_APP_SUGGESTIONS_TTL) or {}
|
||||
sidecar = lxc_apps.load_sidecar(vmid) or {}
|
||||
return jsonify(native_app_discovery.filter_candidates(
|
||||
result, suggestions, sidecar.get('apps') or []))
|
||||
|
||||
|
||||
@app.route('/api/vms/<int:vmid>/apps/adguard-setup', methods=['GET'])
|
||||
@require_auth
|
||||
def api_vm_apps_adguard_setup(vmid):
|
||||
|
||||
@@ -1179,6 +1179,13 @@ def validate_config(payload: dict) -> tuple[bool, Any]:
|
||||
return _err("logo_url must be an http(s) URL (max 512 chars)")
|
||||
conf["logo_url"] = logo
|
||||
|
||||
# Native discovery identity is separate from a curated helper/updater.
|
||||
discovery_id = payload.get("discovery_id")
|
||||
if discovery_id is not None:
|
||||
if not isinstance(discovery_id, str) or not re.fullmatch(r"native-[0-9a-f]{24}", discovery_id):
|
||||
return _err("invalid discovery_id")
|
||||
conf["discovery_id"] = discovery_id
|
||||
|
||||
# Optional helper_slug — set by the frontend when the user picks
|
||||
# an auto-detected app (primary or extra). Persisted so we can
|
||||
# filter the "also detected" chip list against apps already
|
||||
|
||||
@@ -0,0 +1,487 @@
|
||||
#!/usr/bin/env python3
|
||||
# ==========================================================
|
||||
# ProxMenux - A menu-driven script for Proxmox VE management
|
||||
# Author : VAIO73
|
||||
# GitHub : github.com/Vaso73
|
||||
# Maintainer : MacRimi
|
||||
# Copyright : (c) 2026 MacRimi & contributors
|
||||
# License : (GPL-3.0) (https://github.com/MacRimi/ProxMenux/blob/main/LICENSE)
|
||||
# Version : 1.0
|
||||
# Last Updated: 14/10/2026
|
||||
# ==========================================================
|
||||
# Description:
|
||||
# On-demand native LXC discovery from service and installation metadata.
|
||||
# Reads bounded data only; never executes discovered programs, starts guests,
|
||||
# registers applications or infers their update mechanism.
|
||||
# ==========================================================
|
||||
"""Separate from catalog detection: callers must explicitly request a scan."""
|
||||
|
||||
import copy
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import selectors
|
||||
import signal
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
TOTAL_SECONDS = 20
|
||||
PROBE_SECONDS = 3
|
||||
OUTPUT_BYTES = 64 * 1024
|
||||
MAX_RECORDS = 256
|
||||
MAX_CANDIDATES = 50
|
||||
CACHE_SECONDS = 300
|
||||
LOCAL_CONFIG = Path('/etc/pve/local/lxc')
|
||||
OCI_ROOT = Path('/usr/local/share/proxmenux/oci/instances')
|
||||
UPDATE_LOCK_ROOT = Path('/run/lock')
|
||||
_cache = {}
|
||||
_locks = {}
|
||||
_mutex = threading.Lock()
|
||||
_PATH = re.compile(r'/[A-Za-z0-9_.+/@-]{1,510}\Z')
|
||||
_NAME = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.+@-]{0,63}\Z')
|
||||
_VERSION = re.compile(r'v?\d+(?:\.\d+){1,4}(?:[-+._][A-Za-z0-9.-]+)?\Z')
|
||||
_SYSTEM = {
|
||||
'ssh', 'sshd', 'cron', 'crond', 'postfix', 'rsyslog', 'syslog', 'syslog-ng',
|
||||
'dbus', 'dbus-broker', 'systemd', 'udev', 'networking', 'network', 'networkmanager',
|
||||
'dhcpcd', 'dhclient', 'chrony', 'chronyd', 'ntp', 'ntpd', 'openntpd', 'agetty',
|
||||
'getty', 'login', 'polkit', 'polkitd', 'accounts-daemon', 'acpid', 'anacron',
|
||||
'docker', 'containerd', 'containerd-shim', 'podman', 'qemu-guest-agent', 'rngd',
|
||||
'auditd', 'logrotate', 'local', 'bootmisc', 'hostname', 'hwclock', 'modules', 'master',
|
||||
'proxmox-boot-sync', 'pve-container', 'pve-firewall', 'pvefw-logger',
|
||||
'pvedaemon', 'pvestatd', 'pveproxy', 'pvescheduler', 'spiceproxy', 'lxc',
|
||||
'mount-ro', 'mdev', 'sysctl', 'sysfs', 'devfs', 'cgroups', 'rc', 'rcs', 'swap',
|
||||
'sleep', 'true', 'false', 'echo', 'cat', 'head', 'tail', 'tee', 'logger',
|
||||
'apt', 'apt-get', 'dpkg', 'dpkg-query', 'apk', 'unattended-upgrades',
|
||||
'fstrim', 'fsck', 'fsckd', 'e2scrub', 'e2scrub-all', 'e2scrub-reap',
|
||||
'irqbalance', 'smartd', 'smartmontools', 'rpcbind', 'nfs-common', 'nfs-server',
|
||||
'rpc-statd', 'rpc-statd-notify', 'rpc-gssd', 'rpc-svcgssd', 'nftables', 'iptables',
|
||||
'ip6tables', 'firewalld', 'ufw', 'resolvconf', 'wpa-supplicant', 'modemmanager',
|
||||
'console-setup', 'keyboard-setup', 'plymouth', 'halt', 'reboot', 'shutdown',
|
||||
'rc-local', 'ntpsec', 'networkd-dispatcher', 'busybox', 'atd', 'uuidd',
|
||||
}
|
||||
_RUNTIMES = re.compile(r'(?:python[0-9.]*|node(?:js)?|sh|bash|dash|env|java|ruby|perl)\Z')
|
||||
|
||||
|
||||
class ProbeError(Exception):
|
||||
"""Safe error code only; guest output must never become an API error."""
|
||||
|
||||
|
||||
def bounded_run(argv, deadline):
|
||||
"""Cap combined stdout/stderr while reading, and kill/reap the process group."""
|
||||
remaining = min(PROBE_SECONDS, deadline - time.monotonic())
|
||||
if remaining <= 0:
|
||||
raise ProbeError('deadline')
|
||||
end = time.monotonic() + remaining
|
||||
try:
|
||||
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
start_new_session=True)
|
||||
except OSError:
|
||||
raise ProbeError('unavailable') from None
|
||||
outputs = [bytearray(), bytearray()]
|
||||
try:
|
||||
with selectors.DefaultSelector() as poll:
|
||||
for index, stream in enumerate((proc.stdout, proc.stderr)):
|
||||
os.set_blocking(stream.fileno(), False)
|
||||
poll.register(stream, selectors.EVENT_READ, index)
|
||||
while poll.get_map():
|
||||
wait = end - time.monotonic()
|
||||
if wait <= 0:
|
||||
raise ProbeError('timeout')
|
||||
for key, _ in poll.select(wait):
|
||||
block = os.read(key.fileobj.fileno(), 4096)
|
||||
if not block:
|
||||
poll.unregister(key.fileobj)
|
||||
continue
|
||||
if sum(map(len, outputs)) + len(block) > OUTPUT_BYTES:
|
||||
raise ProbeError('output_limit')
|
||||
outputs[key.data].extend(block)
|
||||
try:
|
||||
proc.wait(timeout=max(0.001, end - time.monotonic()))
|
||||
except subprocess.TimeoutExpired:
|
||||
raise ProbeError('timeout') from None
|
||||
if proc.returncode:
|
||||
raise ProbeError('read_failed')
|
||||
return outputs[0].decode('utf-8', errors='strict')
|
||||
except UnicodeError:
|
||||
raise ProbeError('invalid_data') from None
|
||||
finally:
|
||||
# Also clean up descendants that outlive a successfully exited wrapper.
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
proc.wait()
|
||||
proc.stdout.close()
|
||||
proc.stderr.close()
|
||||
|
||||
|
||||
# These are fixed read-only scripts, not guest-supplied shell or init scripts.
|
||||
# awk emits only executable/script paths and WorkingDirectory, never arguments
|
||||
# or environment values. OpenRC files are parsed as data, never sourced.
|
||||
SERVICE_SCRIPT = r'''
|
||||
command -v awk >/dev/null && command -v head >/dev/null || exit 1
|
||||
n=0
|
||||
for dir in /etc/systemd/system /usr/lib/systemd/system /lib/systemd/system /etc/init.d; do
|
||||
[ ! -d "$dir" ] || [ -r "$dir" ] || exit 1
|
||||
for f in "$dir"/*; do
|
||||
[ -f "$f" ] || continue
|
||||
case "$dir:$f" in /etc/init.d:*) kind=openrc;; *:*.service) kind=systemd;; *) continue;; esac
|
||||
n=$((n+1)); [ "$n" -le 256 ] || { printf 'LIMIT\n'; exit 0; }
|
||||
[ -r "$f" ] || exit 1
|
||||
content=$(timeout -s KILL 1 head -c 16385 "$f") || exit 1
|
||||
[ "${#content}" -le 16384 ] || { printf 'LIMIT\n'; exit 0; }
|
||||
printf '%s\n' "$content" | awk -v kind="$kind" -v file="$f" '
|
||||
function clean(v) {gsub(/^[ \t]+|[ \t]+$/, "", v); gsub(/^["\047]|["\047]$/, "", v); return v}
|
||||
/^ExecStart=/ {v=substr($0,11); sub(/^[-+!:@]+/, "", v); split(v,a,/[ \t]+/); exe=clean(a[1]); script=clean(a[2]); if(script !~ /^\//) script=""}
|
||||
/^WorkingDirectory=/ {dir=clean(substr($0,18))}
|
||||
/^command=/ {exe=clean(substr($0,9))}
|
||||
/^directory=/ {dir=clean(substr($0,11))}
|
||||
/^command_args=/ {v=clean(substr($0,14)); split(v,a,/[ \t]+/); script=clean(a[1]); if(script !~ /^\//) script=""}
|
||||
END {printf "S\t%s\t%s\t%s\t%s\t%s\n", file, kind, exe, script, dir}'
|
||||
done
|
||||
done
|
||||
printf 'END\n'
|
||||
'''
|
||||
PROCESS_SCRIPT = r'''
|
||||
command -v readlink >/dev/null && command -v head >/dev/null || exit 1
|
||||
n=0
|
||||
for dir in /proc/[0-9]*; do
|
||||
[ -d "$dir" ] || continue
|
||||
n=$((n+1)); [ "$n" -le "$1" ] || { printf 'LIMIT\n'; exit 0; }
|
||||
# Never inspect Docker/OCI child workloads or read cmdline/environ.
|
||||
[ -d "$dir" ] || continue
|
||||
[ -r "$dir/cgroup" ] || exit 1
|
||||
group=$(timeout -s KILL 1 head -c 8193 "$dir/cgroup") || exit 1
|
||||
[ "${#group}" -le 8192 ] || { printf 'LIMIT\n'; exit 0; }
|
||||
case "$group" in *docker*|*libpod*|*kubepods*|*containerd*) continue;; esac
|
||||
# Kernel threads and vanished processes have no application executable.
|
||||
[ -L "$dir/exe" ] || continue
|
||||
exe=$(timeout -s KILL 1 readlink "$dir/exe") || continue
|
||||
cwd=$(timeout -s KILL 1 readlink "$dir/cwd") || continue
|
||||
printf 'P\t%s\t%s\t%s\n' "${dir##*/}" "$exe" "$cwd"
|
||||
done
|
||||
printf 'END\n'
|
||||
'''
|
||||
PORT_SCRIPT = r'''
|
||||
command -v awk >/dev/null || exit 1
|
||||
if command -v ss >/dev/null; then
|
||||
(timeout -s KILL 1 ss -H -lntp || printf 'FAILED\n') | awk -v cap="$1" '
|
||||
/^FAILED$/ {failed=1; exit}
|
||||
NR > cap {limited=1; exit}
|
||||
{port=$4; sub(/^.*:/,"",port); rest=$0
|
||||
# More than one owner is ambiguous; never choose the first arbitrarily.
|
||||
if (gsub(/pid=/,"pid=",rest)!=1) next
|
||||
if (match(rest,/pid=[0-9]+/)) {pid=substr(rest,RSTART+4,RLENGTH-4); printf "L\t%s\t%s\n",pid,port}}
|
||||
END {print failed ? "FAILED" : (limited ? "LIMIT" : "END")}'
|
||||
elif command -v netstat >/dev/null; then
|
||||
(timeout -s KILL 1 netstat -lntp || printf 'FAILED\n') | awk -v cap="$1" '
|
||||
/^FAILED$/ {failed=1; exit}
|
||||
/^tcp/ {n++; if(n>cap) {limited=1; exit}; port=$4; sub(/^.*:/,"",port)
|
||||
pid=$7; sub(/\/.*$/,"",pid); if(pid ~ /^[0-9]+$/) printf "L\t%s\t%s\n",pid,port}
|
||||
END {print failed ? "FAILED" : (limited ? "LIMIT" : "END")}'
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
'''
|
||||
FOOTPRINT_SCRIPT = r'''
|
||||
[ -f "$1" ] && [ -r "$1" ] || exit 1
|
||||
printf 'FILE\n'
|
||||
if [ -f "$2/package.json" ]; then
|
||||
[ -r "$2/package.json" ] || exit 1
|
||||
timeout -s KILL 1 head -c 8192 "$2/package.json"
|
||||
fi
|
||||
'''
|
||||
|
||||
|
||||
def _path(value):
|
||||
return bool(isinstance(value, str) and _PATH.fullmatch(value)
|
||||
and '..' not in value.split('/'))
|
||||
|
||||
|
||||
def _system(name):
|
||||
raw = name.lower().removesuffix('.service')
|
||||
stem = raw.split('@')[0]
|
||||
return raw.startswith('user@') or stem in _SYSTEM or stem.startswith((
|
||||
'systemd-', 'dbus-', 'getty', 'containerd-', 'apt-', 'dpkg-', 'e2scrub',
|
||||
'unattended-', 'ifup', 'ifdown', 'user@', 'btrfs-', 'lvm2-', 'lxc-'))
|
||||
|
||||
|
||||
def _custom(path):
|
||||
return _path(path) and path.startswith(('/opt/', '/srv/', '/usr/local/'))
|
||||
|
||||
|
||||
def parse_records(text, kind):
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[-1] not in ('END', 'LIMIT'):
|
||||
raise ProbeError('incomplete')
|
||||
if len(lines) - 1 > MAX_RECORDS:
|
||||
raise ProbeError('metadata_limit')
|
||||
records = []
|
||||
seen_services = set()
|
||||
for line in lines[:-1]:
|
||||
parts = line.split('\t')
|
||||
if kind == 'S' and len(parts) == 6 and parts[0] == 'S':
|
||||
_, source, manager, exe, script, directory = parts
|
||||
if not _path(source) or manager not in ('systemd', 'openrc'):
|
||||
raise ProbeError('invalid_data')
|
||||
service = Path(source).name.removesuffix('.service')
|
||||
if service in seen_services:
|
||||
continue # /etc overrides vendor units; no duplicate installations.
|
||||
seen_services.add(service)
|
||||
if not _NAME.fullmatch(service) or _system(service) or not _path(exe):
|
||||
continue
|
||||
runtime = bool(_RUNTIMES.fullmatch(Path(exe).name))
|
||||
install = script if runtime else exe
|
||||
if (not _path(install) or _system(Path(install).name)
|
||||
or install.startswith(('/usr/lib/systemd/', '/lib/systemd/',
|
||||
'/usr/lib/udev/', '/lib/udev/', '/usr/lib/apt/'))):
|
||||
continue
|
||||
root = directory if _custom(directory) else str(Path(install).parent)
|
||||
# Generic runtime without an absolute application script is ambiguous.
|
||||
if runtime and not _custom(install):
|
||||
continue
|
||||
records.append({'name': service, 'path': install, 'root': root,
|
||||
'service': source, 'manager': manager, 'executable': exe})
|
||||
elif kind == 'P' and len(parts) == 4 and parts[0] == 'P':
|
||||
_, pid, exe, directory = parts
|
||||
if not pid.isdigit() or not _path(exe) or not _path(directory):
|
||||
continue
|
||||
records.append({'pid': pid, 'executable': exe, 'root': directory})
|
||||
elif kind == 'L' and len(parts) == 3 and parts[0] == 'L':
|
||||
_, pid, port = parts
|
||||
if pid.isdigit() and port.isdigit() and 0 < int(port) < 65536:
|
||||
records.append({'pid': pid, 'port': int(port)})
|
||||
else:
|
||||
raise ProbeError('invalid_data')
|
||||
else:
|
||||
raise ProbeError('invalid_data')
|
||||
return records, lines[-1] == 'LIMIT', len(lines) - 1
|
||||
|
||||
|
||||
def candidate(vmid, record, footprint):
|
||||
if not footprint.startswith('FILE\n'):
|
||||
raise ProbeError('invalid_data')
|
||||
version, metadata_issue = None, None
|
||||
metadata = footprint[5:].strip()
|
||||
if metadata:
|
||||
try:
|
||||
manifest = json.loads(metadata)
|
||||
except (ValueError, TypeError):
|
||||
manifest = None
|
||||
# Only the whitelisted version field leaves this function. Never retain
|
||||
# manifest scripts, repository guesses, config or arbitrary extra fields.
|
||||
if not isinstance(manifest, dict):
|
||||
metadata_issue = 'invalid_metadata'
|
||||
manifest = {}
|
||||
value = manifest.get('version')
|
||||
if (isinstance(value, str) and len(value) <= 64 and _VERSION.fullmatch(value)
|
||||
and record['path'].startswith(record['root'].rstrip('/') + '/')):
|
||||
version = value
|
||||
identity = 'native-' + hashlib.sha256(
|
||||
f'{int(vmid)}:{record["path"]}:{record["root"]}'.encode()).hexdigest()[:24]
|
||||
evidence = [{'kind': 'installation_file', 'value': record['path']}]
|
||||
if record.get('service'):
|
||||
evidence.append({'kind': record['manager'], 'value': record['service']})
|
||||
else:
|
||||
evidence.extend([{'kind': 'process_executable', 'value': record['process']},
|
||||
{'kind': 'working_directory', 'value': record['root']}])
|
||||
ports = sorted(set(record.get('listening_ports') or []))
|
||||
evidence.extend({'kind': 'tcp_listener', 'value': str(port)} for port in ports)
|
||||
return {'slug': identity, 'discovery_id': identity, 'source': 'native_metadata',
|
||||
'name': record['name'], 'installed_version': version, 'evidence': evidence,
|
||||
'installation_path': record['path'], 'default_ports': [],
|
||||
'listening_ports': ports,
|
||||
'metadata_issue': metadata_issue,
|
||||
'tracking_suggestion': ({'installed_via': 'manual', 'installed_version': version,
|
||||
'detected_version': version, 'detector_verified': True}
|
||||
if version else None)}
|
||||
|
||||
|
||||
def _guard(vmid, deadline, run):
|
||||
# The local node path rejects QEMU, missing and remote CTs before pct exec.
|
||||
path = LOCAL_CONFIG / f'{vmid}.conf'
|
||||
if not path.is_file():
|
||||
raise ProbeError('not_local_lxc')
|
||||
# Presence, including a damaged/in-progress record, excludes generic probes.
|
||||
try:
|
||||
os.lstat(OCI_ROOT / str(vmid))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
raise ProbeError('oci_unknown') from None
|
||||
else:
|
||||
raise ProbeError('managed_oci')
|
||||
try:
|
||||
with path.open('rb') as handle:
|
||||
raw = handle.read(OUTPUT_BYTES + 1)
|
||||
if len(raw) > OUTPUT_BYTES:
|
||||
raise ProbeError('config_limit')
|
||||
config = raw.decode('utf-8')
|
||||
except (OSError, UnicodeError):
|
||||
raise ProbeError('config_unavailable') from None
|
||||
if re.search(r'^lock:', config, re.M):
|
||||
raise ProbeError('guest_busy')
|
||||
update_lock = UPDATE_LOCK_ROOT / f'proxmenux-lxc-update-{vmid}.lock'
|
||||
try:
|
||||
with update_lock.open('rb') as handle:
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise ProbeError('guest_busy') from None
|
||||
finally:
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
raise ProbeError('lock_unavailable') from None
|
||||
if re.search(r'^ostype:\s*unmanaged\s*$', config, re.M):
|
||||
raise ProbeError('unsupported_guest')
|
||||
if not re.search(r'^ostype:\s*(?:debian|ubuntu|alpine|archlinux|centos|fedora|opensuse|gentoo)\s*$', config, re.M):
|
||||
raise ProbeError('unsupported_guest')
|
||||
status = run(['pct', 'status', str(vmid)], deadline).strip()
|
||||
if status != 'status: running':
|
||||
raise ProbeError('guest_stopped' if status == 'status: stopped' else 'status_unknown')
|
||||
|
||||
|
||||
def scan(vmid, run=bounded_run):
|
||||
deadline = time.monotonic() + TOTAL_SECONDS
|
||||
_guard(vmid, deadline, run)
|
||||
records, issues, consumed, processes, listeners = [], [], 0, [], []
|
||||
for kind, script in (('S', SERVICE_SCRIPT), ('P', PROCESS_SCRIPT), ('L', PORT_SCRIPT)):
|
||||
try:
|
||||
if consumed >= MAX_RECORDS:
|
||||
issues.append('metadata_limit')
|
||||
break
|
||||
parsed, limited, count = parse_records(run(
|
||||
['pct', 'exec', str(vmid), '--', 'timeout', '-s', 'KILL', '2',
|
||||
'sh', '-c', script, 'discovery', str(MAX_RECORDS - consumed)], deadline), kind)
|
||||
if consumed + count > MAX_RECORDS:
|
||||
issues.append('metadata_limit')
|
||||
parsed = parsed[:MAX_RECORDS - consumed]
|
||||
consumed = min(MAX_RECORDS, consumed + count)
|
||||
# The entire collector, not each metadata source, has 256 records.
|
||||
capacity = MAX_RECORDS - len(records)
|
||||
if kind == 'S':
|
||||
records.extend(parsed[:capacity])
|
||||
elif kind == 'P':
|
||||
processes = parsed[:capacity]
|
||||
else:
|
||||
listeners = parsed[:capacity]
|
||||
if limited or len(parsed) > capacity:
|
||||
issues.append('metadata_limit')
|
||||
except ProbeError as exc:
|
||||
issues.append(str(exc))
|
||||
for process in processes:
|
||||
exe, directory = process['executable'], process['root']
|
||||
related = [record for record in records if record['executable'] == exe and (
|
||||
not _RUNTIMES.fullmatch(Path(exe).name) or record['root'] == directory)]
|
||||
ports = [entry['port'] for entry in listeners if entry['pid'] == process['pid']]
|
||||
if len(related) == 1:
|
||||
related[0].setdefault('listening_ports', []).extend(ports)
|
||||
elif not related and (_custom(exe) and _custom(directory)
|
||||
and not _system(Path(exe).name)
|
||||
and not _RUNTIMES.fullmatch(Path(exe).name)
|
||||
and exe.startswith(directory.rstrip('/') + '/')
|
||||
and _NAME.fullmatch(Path(exe).name)):
|
||||
records.append({'name': Path(exe).name, 'path': exe, 'root': directory,
|
||||
'process': exe, 'executable': exe, 'listening_ports': ports})
|
||||
items, seen = [], set()
|
||||
for record in records:
|
||||
key = (record['path'], record['root'])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if len(items) >= MAX_CANDIDATES:
|
||||
issues.append('candidate_limit')
|
||||
break
|
||||
try:
|
||||
footprint = run(['pct', 'exec', str(vmid), '--', 'timeout', '-s', 'KILL', '2',
|
||||
'sh', '-c', FOOTPRINT_SCRIPT,
|
||||
'discovery', record['path'], record['root']], deadline)
|
||||
item = candidate(vmid, record, footprint)
|
||||
items.append(item)
|
||||
if item['metadata_issue']:
|
||||
issues.append(item['metadata_issue'])
|
||||
except ProbeError as exc:
|
||||
issues.append(str(exc))
|
||||
if str(exc) == 'deadline':
|
||||
break
|
||||
return {'vmid': vmid, 'status': 'partial' if issues else 'complete',
|
||||
'candidates': items, 'issues': sorted(set(issues)), 'stale': False,
|
||||
'scanned_at': time.time()}
|
||||
|
||||
|
||||
def cached(vmid):
|
||||
with _mutex:
|
||||
value = copy.deepcopy(_cache.get(int(vmid)))
|
||||
if value is None:
|
||||
return {'vmid': int(vmid), 'status': 'not_scanned', 'candidates': [], 'stale': False}
|
||||
if time.time() - value['scanned_at'] >= CACHE_SECONDS:
|
||||
value['stale'] = True
|
||||
return value
|
||||
|
||||
|
||||
def mark_stale(vmid):
|
||||
"""Lifecycle events retire freshness, never collect new guest metadata."""
|
||||
with _mutex:
|
||||
if int(vmid) in _cache:
|
||||
_cache[int(vmid)]['stale'] = True
|
||||
|
||||
|
||||
def discover(vmid):
|
||||
vmid = int(vmid)
|
||||
with _mutex:
|
||||
lock = _locks.setdefault(vmid, threading.Lock())
|
||||
if not lock.acquire(blocking=False):
|
||||
return {**cached(vmid), 'status': 'busy', 'stale': True}
|
||||
try:
|
||||
try:
|
||||
result = scan(vmid)
|
||||
except ProbeError as exc:
|
||||
result = {**cached(vmid), 'status': 'error', 'issues': [str(exc)], 'stale': True}
|
||||
result.setdefault('scanned_at', time.time())
|
||||
with _mutex:
|
||||
_cache[vmid] = copy.deepcopy(result)
|
||||
return result
|
||||
previous = cached(vmid)
|
||||
if result['status'] != 'complete' and previous['candidates']:
|
||||
# Partial output cannot prove that previously seen apps disappeared.
|
||||
ids = {item['discovery_id'] for item in result['candidates']}
|
||||
result['candidates'].extend(item for item in previous['candidates']
|
||||
if item['discovery_id'] not in ids)
|
||||
result['candidates'] = result['candidates'][:MAX_CANDIDATES]
|
||||
result['stale'] = True
|
||||
with _mutex:
|
||||
_cache[vmid] = copy.deepcopy(result)
|
||||
return result
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
|
||||
def filter_candidates(result, suggestions, apps):
|
||||
"""Pure composition: never run catalog probes, load or write registries."""
|
||||
known = [*apps, *(suggestions.get('extras') or []),
|
||||
*(suggestions.get('docker_workloads') or [])]
|
||||
if suggestions.get('helper_slug'):
|
||||
known.append({'name': suggestions.get('name_suggestion'),
|
||||
'helper_slug': suggestions['helper_slug'],
|
||||
'tracking_suggestion': suggestions.get('tracking_suggestion')})
|
||||
identities = {a.get('discovery_id') for a in known if a.get('discovery_id')}
|
||||
names, paths = set(), set()
|
||||
for app in known:
|
||||
for field in ('name', 'slug', 'helper_slug', 'package'):
|
||||
if app.get(field):
|
||||
names.add(str(app[field]).casefold())
|
||||
detector = app.get('tracking_suggestion') or app
|
||||
for field in ('binary_path', 'file_path'):
|
||||
if detector.get(field):
|
||||
paths.add(detector[field])
|
||||
items = [item for item in result['candidates']
|
||||
if item['discovery_id'] not in identities
|
||||
and item['name'].casefold() not in names
|
||||
and item['installation_path'] not in paths]
|
||||
return {**result, 'candidates': items}
|
||||
@@ -0,0 +1,441 @@
|
||||
"""Host-independent discovery fixtures; no real pct, guest, auth or registry."""
|
||||
import ast
|
||||
import copy
|
||||
import fcntl
|
||||
from functools import wraps
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import native_app_discovery as native
|
||||
import lxc_apps
|
||||
|
||||
|
||||
def service(name='handmade', exe='/opt/handmade/server', directory='/opt/handmade', manager='systemd', script=''):
|
||||
path = ('/etc/systemd/system/' + name + '.service' if manager == 'systemd'
|
||||
else '/etc/init.d/' + name)
|
||||
return '\t'.join(['S', path, manager, exe, script, directory]) + '\nEND\n'
|
||||
|
||||
|
||||
class DiscoveryFixtures(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.config = self.root / 'lxc'
|
||||
self.config.mkdir()
|
||||
(self.config / '101.conf').write_text('ostype: debian\n')
|
||||
self.oci = self.root / 'oci'
|
||||
self.oci.mkdir()
|
||||
for context in (patch.object(native, 'LOCAL_CONFIG', self.config),
|
||||
patch.object(native, 'OCI_ROOT', self.oci),
|
||||
patch.object(native, 'UPDATE_LOCK_ROOT', self.root)):
|
||||
context.start()
|
||||
self.addCleanup(context.stop)
|
||||
native._cache.clear()
|
||||
native._locks.clear()
|
||||
self.calls = []
|
||||
|
||||
def runner(self, services=None, processes='END\n', footprint='FILE\n', status='status: running', ports='END\n'):
|
||||
def run(argv, deadline):
|
||||
self.calls.append(argv)
|
||||
self.assertLessEqual(deadline - time.monotonic(), native.TOTAL_SECONDS)
|
||||
if argv[1] == 'status':
|
||||
return status
|
||||
self.assertEqual(argv[:6], ['pct', 'exec', '101', '--', 'timeout', '-s'])
|
||||
self.assertEqual(argv[6:9], ['KILL', '2', 'sh'])
|
||||
script = argv[10]
|
||||
if script == native.SERVICE_SCRIPT:
|
||||
return services if services is not None else service()
|
||||
if script == native.PROCESS_SCRIPT:
|
||||
return processes
|
||||
if script == native.PORT_SCRIPT:
|
||||
return ports
|
||||
if script == native.FOOTPRINT_SCRIPT:
|
||||
return footprint
|
||||
self.fail('Unexpected probe')
|
||||
return run
|
||||
|
||||
def test_unknown_version_is_present_with_evidence_and_no_capabilities(self):
|
||||
result = native.scan(101, self.runner())
|
||||
self.assertEqual(result['status'], 'complete')
|
||||
item, = result['candidates']
|
||||
self.assertIsNone(item['installed_version'])
|
||||
self.assertIsNone(item['tracking_suggestion'])
|
||||
self.assertEqual(item['default_ports'], [])
|
||||
self.assertEqual({e['kind'] for e in item['evidence']}, {'installation_file', 'systemd'})
|
||||
for field in ('helper_slug', 'repo', 'update_command', 'binary_path', 'upstream_type'):
|
||||
self.assertNotIn(field, item)
|
||||
|
||||
def test_openrc_node_script_known_version_is_manual_and_secret_free(self):
|
||||
result = native.scan(101, self.runner(
|
||||
service(exe='/usr/bin/node', script='/opt/handmade/server.js', manager='openrc'),
|
||||
footprint='FILE\n' + json.dumps({'version': '1.2.3', 'scripts': {'start': 'SECRET'}, 'env': 'PASSWORD'})))
|
||||
item, = result['candidates']
|
||||
self.assertEqual(item['installed_version'], '1.2.3')
|
||||
self.assertEqual(item['tracking_suggestion']['installed_via'], 'manual')
|
||||
self.assertNotIn('SECRET', json.dumps(result))
|
||||
self.assertNotIn('PASSWORD', json.dumps(result))
|
||||
self.assertFalse(any('systemctl' in arg for call in self.calls for arg in call))
|
||||
|
||||
def test_inactive_service_and_packaged_application_are_not_hidden(self):
|
||||
result = native.scan(101, self.runner(service(name='legit-app', exe='/usr/bin/legit-app', directory='')))
|
||||
self.assertEqual(result['candidates'][0]['name'], 'legit-app')
|
||||
|
||||
def test_system_services_and_ambiguous_runtimes_have_zero_footprint_probes(self):
|
||||
for name, exe in [('ssh', '/usr/sbin/sshd'), ('cron', '/usr/sbin/cron'),
|
||||
('postfix', '/usr/lib/postfix/master'), ('rsyslog', '/usr/sbin/rsyslogd'),
|
||||
('systemd-networkd', '/usr/lib/systemd/systemd-networkd'),
|
||||
('ambiguous', '/usr/bin/python3')]:
|
||||
self.calls.clear()
|
||||
result = native.scan(101, self.runner(service(name=name, exe=exe)))
|
||||
self.assertEqual(result['candidates'], [])
|
||||
self.assertFalse(any(native.FOOTPRINT_SCRIPT in call for call in self.calls))
|
||||
|
||||
def test_missing_remote_qemu_stopped_busy_oci_guards_precede_exec(self):
|
||||
scenarios = ['missing', 'stopped', 'busy', 'oci', 'invalid-oci', 'unmanaged']
|
||||
for scenario in scenarios:
|
||||
with self.subTest(scenario=scenario):
|
||||
(self.config / '101.conf').write_text('ostype: debian\n')
|
||||
if scenario == 'missing':
|
||||
(self.config / '101.conf').unlink()
|
||||
if scenario == 'busy':
|
||||
(self.config / '101.conf').write_text('lock: backup\n')
|
||||
if scenario == 'unmanaged':
|
||||
(self.config / '101.conf').write_text('ostype: unmanaged\n')
|
||||
if 'oci' in scenario:
|
||||
(self.oci / '101').mkdir(exist_ok=True)
|
||||
(self.oci / '101' / 'oci-compose.json').write_text('broken')
|
||||
self.calls.clear()
|
||||
with self.assertRaises(native.ProbeError):
|
||||
native.scan(101, self.runner(status='status: stopped' if scenario == 'stopped' else 'status: running'))
|
||||
self.assertFalse(any(call[1] == 'exec' for call in self.calls))
|
||||
if (self.oci / '101').exists():
|
||||
(self.oci / '101' / 'oci-compose.json').unlink()
|
||||
(self.oci / '101').rmdir()
|
||||
|
||||
def test_failure_and_partial_output_are_not_successful_empty_results(self):
|
||||
for text in ('', 'malformed\nEND\n', 'S\tpartial', 'LIMIT\n'):
|
||||
result = native.scan(101, self.runner(services=text))
|
||||
self.assertEqual(result['status'], 'partial')
|
||||
run = self.runner()
|
||||
def failed(argv, deadline):
|
||||
if argv[1] == 'exec':
|
||||
raise native.ProbeError('read_failed')
|
||||
return run(argv, deadline)
|
||||
self.assertEqual(native.scan(101, failed)['status'], 'partial')
|
||||
|
||||
def test_optional_invalid_version_metadata_keeps_proven_candidate(self):
|
||||
result = native.scan(101, self.runner(footprint='FILE\n{broken'))
|
||||
self.assertEqual(result['status'], 'partial')
|
||||
item, = result['candidates']
|
||||
self.assertIsNone(item['installed_version'])
|
||||
self.assertIsNone(item['tracking_suggestion'])
|
||||
|
||||
def test_working_directory_manifest_does_not_version_an_unrelated_executable(self):
|
||||
result = native.scan(101, self.runner(service(exe='/opt/other/server'),
|
||||
footprint='FILE\n{"version":"1.2.3"}'))
|
||||
self.assertIsNone(result['candidates'][0]['installed_version'])
|
||||
|
||||
def test_path_injection_and_port_only_data_do_not_become_candidates(self):
|
||||
for text in (service(exe='/opt/app/$(id)'), service(exe='/opt/../server'),
|
||||
service(name='app;id'), service(exe='/bin/sh')):
|
||||
self.assertEqual(native.scan(101, self.runner(text))['candidates'], [])
|
||||
with self.assertRaises(native.ProbeError):
|
||||
native.parse_records('PORT\t8080\nEND\n', 'P')
|
||||
|
||||
def test_same_process_deduplicates_but_two_apps_using_node_do_not_merge(self):
|
||||
services = service(exe='/usr/bin/node', script='/opt/handmade/server.js').removesuffix('END\n')
|
||||
services += service(name='second', exe='/usr/bin/node', script='/opt/second/server.js', directory='/opt/second')
|
||||
result = native.scan(101, self.runner(services))
|
||||
self.assertEqual(len(result['candidates']), 2)
|
||||
result = native.scan(101, self.runner(processes='P\t123\t/opt/handmade/server\t/opt/handmade\nEND\n'))
|
||||
self.assertEqual(len(result['candidates']), 1)
|
||||
other = native.candidate(102, native.parse_records(service(), 'S')[0][0], 'FILE\n')
|
||||
self.assertNotEqual(other['discovery_id'], result['candidates'][0]['discovery_id'])
|
||||
|
||||
def test_record_and_candidate_caps(self):
|
||||
too_many = 'P\t123\t/opt/a/app\t/opt/a\n' * 257 + 'END\n'
|
||||
with self.assertRaises(native.ProbeError):
|
||||
native.parse_records(too_many, 'P')
|
||||
services = ''.join(service(name=f'app{i}', exe=f'/opt/app{i}/server', directory=f'/opt/app{i}').removesuffix('END\n') for i in range(51)) + 'END\n'
|
||||
result = native.scan(101, self.runner(services))
|
||||
self.assertEqual(len(result['candidates']), 50)
|
||||
self.assertIn('candidate_limit', result['issues'])
|
||||
|
||||
def test_cached_ttl_never_runs_scan_and_error_preserves_last_known(self):
|
||||
complete = native.scan(101, self.runner())
|
||||
with patch.object(native, 'scan', return_value=complete) as scan:
|
||||
native.discover(101)
|
||||
for age, stale in ((299.9, False), (300, True), (300.1, True)):
|
||||
with patch.object(native.time, 'time', return_value=complete['scanned_at'] + age):
|
||||
self.assertEqual(native.cached(101)['stale'], stale)
|
||||
self.assertEqual(scan.call_count, 1)
|
||||
with patch.object(native, 'scan', side_effect=native.ProbeError('timeout')):
|
||||
result = native.discover(101)
|
||||
self.assertEqual(result['candidates'], complete['candidates'])
|
||||
self.assertEqual(native.cached(101)['status'], 'error')
|
||||
self.assertTrue(native.cached(101)['stale'])
|
||||
partial = {**complete, 'status': 'partial', 'candidates': [], 'issues': ['read_failed']}
|
||||
with patch.object(native, 'scan', return_value=partial):
|
||||
self.assertEqual(native.discover(101)['candidates'], complete['candidates'])
|
||||
with patch.object(native, 'scan', return_value={**complete, 'candidates': []}):
|
||||
self.assertEqual(native.discover(101)['candidates'], [])
|
||||
|
||||
def test_same_ct_busy_and_different_ct_cache_isolation(self):
|
||||
lock = threading.Lock()
|
||||
lock.acquire()
|
||||
native._locks[101] = lock
|
||||
with patch.object(native, 'scan') as scanner:
|
||||
self.assertEqual(native.discover(101)['status'], 'busy')
|
||||
scanner.assert_not_called()
|
||||
self.assertEqual(native.cached(102)['status'], 'not_scanned')
|
||||
|
||||
def test_filter_registered_curated_and_docker_children_without_mutation(self):
|
||||
result = native.scan(101, self.runner())
|
||||
original = copy.deepcopy(result)
|
||||
for suggestions, apps in (({}, [{'discovery_id': result['candidates'][0]['discovery_id']}]),
|
||||
({'extras': [{'name': 'Handmade'}]}, []),
|
||||
({'docker_workloads': [{'name': 'Handmade'}]}, []),
|
||||
({'helper_slug': 'handmade', 'name_suggestion': 'Handmade'}, [])):
|
||||
self.assertEqual(native.filter_candidates(result, suggestions, apps)['candidates'], [])
|
||||
self.assertEqual(result, original)
|
||||
|
||||
def test_actual_save_first_check_and_recheck_never_probe_or_configure_updater(self):
|
||||
with patch.object(lxc_apps, '_APPS_DIR', str(self.root / 'apps')), \
|
||||
patch.object(lxc_apps, '_pct_exec', side_effect=AssertionError('forbidden guest execution')) as execute, \
|
||||
patch.object(lxc_apps, 'fetch_latest_upstream_details', side_effect=AssertionError('forbidden upstream')) as upstream:
|
||||
for version in (None, '1.2.3'):
|
||||
item = native.candidate(101, native.parse_records(service(), 'S')[0][0],
|
||||
'FILE\n' + (json.dumps({'version': version}) if version else ''))
|
||||
payload = {'name': item['name'], 'discovery_id': item['discovery_id'],
|
||||
'update_method': 'none', **(item['tracking_suggestion'] or {})}
|
||||
ok, sidecar = lxc_apps.add_app(101, payload)
|
||||
self.assertTrue(ok, sidecar)
|
||||
app = sidecar['apps'][-1]
|
||||
self.assertEqual(app['state']['installed_version'], version)
|
||||
self.assertIsNone(app['state']['update_available'])
|
||||
self.assertEqual(app['update_method'], 'none')
|
||||
for field in ('helper_slug', 'update_command', 'repo', 'upstream_type'):
|
||||
self.assertNotIn(field, app)
|
||||
lxc_apps.check_app(101, app['id'], force=True)
|
||||
self.assertIsNone(lxc_apps.get_schedule(101))
|
||||
self.assertIsNone(lxc_apps.get_bulk_update(101))
|
||||
execute.assert_not_called()
|
||||
upstream.assert_not_called()
|
||||
|
||||
def test_failed_save_has_no_first_check(self):
|
||||
with patch.object(lxc_apps, '_write_sidecar', return_value=False), \
|
||||
patch.object(lxc_apps, '_read_sidecar', return_value=None), \
|
||||
patch.object(lxc_apps, 'check_app') as check:
|
||||
self.assertFalse(lxc_apps.add_app(101, {'name': 'handmade'})[0])
|
||||
check.assert_not_called()
|
||||
|
||||
def test_active_update_lock_blocks_all_guest_probes_without_writing(self):
|
||||
path = self.root / 'proxmenux-lxc-update-101.lock'
|
||||
path.write_bytes(b'unchanged')
|
||||
with path.open('rb') as handle:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||
with self.assertRaisesRegex(native.ProbeError, 'guest_busy'):
|
||||
native.scan(101, self.runner())
|
||||
self.assertEqual(path.read_bytes(), b'unchanged')
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
def test_actual_service_parser_reads_data_never_executes_or_sources_it(self):
|
||||
for manager in ('systemd', 'openrc'):
|
||||
directory = self.root / manager
|
||||
directory.mkdir()
|
||||
marker = self.root / ('forbidden-' + manager)
|
||||
unit = directory / ('handmade.service' if manager == 'systemd' else 'handmade')
|
||||
content = ('[Service]\nExecStart=/usr/bin/node /opt/handmade/server.js --password SECRET\nWorkingDirectory=/opt/handmade\n'
|
||||
if manager == 'systemd' else 'command="/usr/bin/node"\ncommand_args="/opt/handmade/server.js --password SECRET"\ndirectory="/opt/handmade"\n')
|
||||
unit.write_text(content + '\ntouch ' + str(marker) + '\n')
|
||||
script = native.SERVICE_SCRIPT.replace('/etc/systemd/system /usr/lib/systemd/system /lib/systemd/system /etc/init.d', str(directory))
|
||||
# Substitute only fixture paths, not the collector/parser behavior.
|
||||
if manager == 'openrc':
|
||||
script = script.replace('/etc/init.d:*)', str(directory) + ':*)')
|
||||
output = native.bounded_run(['sh', '-c', script], time.monotonic() + 3)
|
||||
record, = native.parse_records(output, 'S')[0]
|
||||
self.assertEqual(record['path'], '/opt/handmade/server.js')
|
||||
self.assertNotIn('SECRET', output)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def test_actual_process_parser_skips_docker_and_never_reads_environ_cmdline(self):
|
||||
proc = self.root / 'proc'
|
||||
proc.mkdir()
|
||||
for pid, group in ((1, '0::/\n'), (2, '0::/docker/child\n')):
|
||||
directory = proc / str(pid)
|
||||
directory.mkdir()
|
||||
(directory / 'cgroup').write_text(group)
|
||||
(directory / 'exe').symlink_to('/opt/handmade/server')
|
||||
(directory / 'cwd').symlink_to('/opt/handmade')
|
||||
(directory / 'cmdline').write_text('SECRET')
|
||||
(directory / 'environ').write_text('PASSWORD')
|
||||
script = native.PROCESS_SCRIPT.replace('/proc/[0-9]*', str(proc) + '/[0-9]*')
|
||||
output = native.bounded_run(['sh', '-c', script, 'fixture', '256'], time.monotonic() + 3)
|
||||
self.assertEqual(output, 'P\t1\t/opt/handmade/server\t/opt/handmade\nEND\n')
|
||||
self.assertNotIn('"$dir/cmdline"', native.PROCESS_SCRIPT)
|
||||
self.assertNotIn('"$dir/environ"', native.PROCESS_SCRIPT)
|
||||
self.assertNotIn('SECRET', output)
|
||||
|
||||
def test_ports_require_exact_process_ownership_and_do_not_infer_http(self):
|
||||
process = 'P\t123\t/opt/handmade/server\t/opt/handmade\nEND\n'
|
||||
result = native.scan(101, self.runner(processes=process, ports='L\t123\t8080\nL\t999\t9000\nEND\n'))
|
||||
item, = result['candidates']
|
||||
self.assertEqual(item['listening_ports'], [8080])
|
||||
self.assertEqual(item['default_ports'], [], 'TCP listener does not prove HTTP')
|
||||
result = native.scan(101, self.runner(ports='L\t123\t8080\nEND\n'))
|
||||
self.assertEqual(result['candidates'][0]['listening_ports'], [])
|
||||
|
||||
def test_shared_runtime_ports_are_assigned_only_by_working_directory(self):
|
||||
services = service(exe='/usr/bin/node', script='/opt/handmade/server.js').removesuffix('END\n')
|
||||
services += service(name='second', exe='/usr/bin/node', script='/opt/second/server.js', directory='/opt/second')
|
||||
process = 'P\t123\t/usr/bin/node\t/opt/second\nEND\n'
|
||||
result = native.scan(101, self.runner(services, processes=process, ports='L\t123\t8080\nEND\n'))
|
||||
self.assertEqual(result['candidates'][0]['listening_ports'], [])
|
||||
self.assertEqual(result['candidates'][1]['listening_ports'], [8080])
|
||||
|
||||
def test_ss_and_alpine_netstat_parser_are_bounded_and_secret_free(self):
|
||||
for tool, line in (('ss', 'LISTEN 0 128 0.0.0.0:8080 0.0.0.0:* users:(("SECRET",pid=123,fd=7))'),
|
||||
('netstat', 'tcp 0 0 0.0.0.0:8080 0.0.0.0:* LISTEN 123/SECRET')):
|
||||
bin_dir = self.root / tool
|
||||
bin_dir.mkdir()
|
||||
executable = bin_dir / tool
|
||||
executable.write_text('#!/bin/sh\nprintf \'%s\\n\' \'' + line + '\'\n')
|
||||
executable.chmod(0o755)
|
||||
(bin_dir / 'awk').symlink_to('/usr/bin/awk')
|
||||
(bin_dir / 'timeout').symlink_to('/usr/bin/timeout')
|
||||
script = 'PATH=' + str(bin_dir) + '\n' + native.PORT_SCRIPT
|
||||
output = native.bounded_run(['sh', '-c', script, 'fixture', '256'], time.monotonic() + 3)
|
||||
self.assertEqual(output, 'L\t123\t8080\nEND\n')
|
||||
self.assertNotIn('SECRET', output)
|
||||
executable.write_text('#!/bin/sh\nexit 1\n')
|
||||
output = native.bounded_run(['sh', '-c', script, 'fixture', '256'], time.monotonic() + 3)
|
||||
with self.assertRaises(native.ProbeError):
|
||||
native.parse_records(output, 'L')
|
||||
|
||||
|
||||
class BoundedRunner(unittest.TestCase):
|
||||
def test_output_limit_is_enforced_during_read(self):
|
||||
with self.assertRaisesRegex(native.ProbeError, 'output_limit'):
|
||||
native.bounded_run([sys.executable, '-c', 'import os; os.write(2,b"x"*1000000)'], time.monotonic() + 3)
|
||||
|
||||
def test_pipe_holding_child_does_not_bypass_timeout(self):
|
||||
start = time.monotonic()
|
||||
with self.assertRaisesRegex(native.ProbeError, 'timeout'):
|
||||
native.bounded_run([sys.executable, '-c', 'import os,time; os.fork(); time.sleep(10)'], start + .15)
|
||||
self.assertLess(time.monotonic() - start, .8)
|
||||
|
||||
def test_failures_never_reveal_stderr_and_deadline_blocks_spawn(self):
|
||||
with self.assertRaisesRegex(native.ProbeError, '^read_failed$'):
|
||||
native.bounded_run([sys.executable, '-c', 'import sys; sys.stderr.write("SECRET"); sys.exit(1)'], time.monotonic() + 3)
|
||||
with patch.object(native.subprocess, 'Popen') as launch:
|
||||
with self.assertRaisesRegex(native.ProbeError, 'deadline'):
|
||||
native.bounded_run(['forbidden'], time.monotonic() - 1)
|
||||
launch.assert_not_called()
|
||||
|
||||
|
||||
class Routes(unittest.TestCase):
|
||||
"""Execute actual Flask route bodies; fixture auth, no server startup jobs."""
|
||||
def setUp(self):
|
||||
from flask import Flask, request, jsonify, g
|
||||
self.app = Flask('native-fixture')
|
||||
self.authorized = True
|
||||
def require_auth(fn):
|
||||
@wraps(fn)
|
||||
def wrapped(*args, **kwargs):
|
||||
if not self.authorized:
|
||||
return jsonify({'error': 'auth_required'}), 401
|
||||
return fn(*args, **kwargs)
|
||||
return wrapped
|
||||
self.globals = {'app': self.app, 'request': request, 'jsonify': jsonify,
|
||||
'require_auth': require_auth, '_vm_app_suggestions_cache': {},
|
||||
'_VM_APP_SUGGESTIONS_TTL': 300,
|
||||
'_vm_cache_get': lambda cache, vmid, ttl: cache.get(vmid),
|
||||
'_vm_cache_put': lambda cache, vmid, value: cache.update({vmid: value}),
|
||||
'_pvesh_cache': {}, '_vm_disk_cache': {}, '_vm_cache_invalidate': Mock(),
|
||||
'_invalidate_lxc_ip': Mock(), '_schedule_started_guest_refresh': Mock(),
|
||||
'_publish_guest_modal_cache_revision': Mock()}
|
||||
tree = ast.parse((SCRIPTS / 'flask_server.py').read_text())
|
||||
routes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in (
|
||||
'api_vm_apps_suggestions', 'api_vm_apps_native_discovery', 'api_vm_apps_add',
|
||||
'_handle_guest_lifecycle')]
|
||||
self.assertEqual(len(routes), 4)
|
||||
exec(compile(ast.Module(body=routes, type_ignores=[]), str(SCRIPTS / 'flask_server.py'), 'exec'), self.globals)
|
||||
self.client = self.app.test_client()
|
||||
|
||||
def test_explicit_post_only_get_and_existing_post_never_native_scan(self):
|
||||
result = {'vmid': 101, 'status': 'complete', 'candidates': [], 'stale': False}
|
||||
with patch.object(native, 'discover', return_value=result) as scan, \
|
||||
patch.object(native, 'cached', return_value=result), \
|
||||
patch.object(lxc_apps, 'get_suggestions', return_value={'extras': []}) as suggestions, \
|
||||
patch.object(lxc_apps, 'load_sidecar', return_value={'apps': []}), \
|
||||
patch.object(lxc_apps, '_write_sidecar') as write:
|
||||
self.assertEqual(self.client.get('/api/vms/101/apps/native-discovery').status_code, 200)
|
||||
self.client.get('/api/vms/101/apps/suggestions')
|
||||
# OCI refresh and Find's existing catalog path retain their purpose.
|
||||
self.client.post('/api/vms/101/apps/suggestions')
|
||||
with self.app.test_request_context('/api/vms/101/apps/suggestions'):
|
||||
from flask import g
|
||||
g._internal_call = True
|
||||
self.globals['_vm_app_suggestions_cache'].clear()
|
||||
self.globals['api_vm_apps_suggestions'](101)
|
||||
scan.assert_not_called()
|
||||
self.client.post('/api/vms/101/apps/native-discovery')
|
||||
scan.assert_called_once_with(101)
|
||||
self.assertEqual(suggestions.call_count, 2)
|
||||
write.assert_not_called()
|
||||
|
||||
def test_unauthorized_and_wrong_method_have_zero_scans(self):
|
||||
with patch.object(native, 'discover') as scan, patch.object(lxc_apps, 'load_sidecar') as read:
|
||||
self.authorized = False
|
||||
self.assertEqual(self.client.post('/api/vms/101/apps/native-discovery').status_code, 401)
|
||||
self.assertEqual(self.client.put('/api/vms/101/apps/native-discovery').status_code, 405)
|
||||
scan.assert_not_called()
|
||||
read.assert_not_called()
|
||||
|
||||
def test_actual_lifecycle_marks_last_known_stale_without_scanning(self):
|
||||
native._cache[101] = {'vmid': 101, 'status': 'complete', 'stale': False,
|
||||
'candidates': [], 'scanned_at': time.time()}
|
||||
with patch.object(native, 'discover') as scan, \
|
||||
patch.object(lxc_apps, 'clear_schedule_reboot_required') as clear:
|
||||
for action in ('start', 'reboot', 'stop'):
|
||||
self.globals['_handle_guest_lifecycle']('101', 'lxc', action)
|
||||
self.assertTrue(native.cached(101)['stale'])
|
||||
scan.assert_not_called()
|
||||
self.assertEqual(clear.call_count, 2)
|
||||
|
||||
def test_actual_repeated_prewarmer_has_zero_native_scans(self):
|
||||
tree = ast.parse((SCRIPTS / 'flask_server.py').read_text())
|
||||
warmer = next(node for node in tree.body if isinstance(node, ast.FunctionDef)
|
||||
and node.name == '_vm_modal_prewarmer_pass')
|
||||
environment = dict(self.globals)
|
||||
for name in ('_vm_details_cache', '_vm_backups_cache', '_vm_schedule_cache', '_vm_mounts_cache'):
|
||||
environment[name] = {}
|
||||
for name in ('_VM_DETAILS_TTL', '_VM_BACKUPS_TTL', '_VM_SCHEDULE_TTL', '_VM_MOUNTS_TTL'):
|
||||
environment[name] = 300
|
||||
for name in ('get_vm_config', 'api_vm_backups', 'api_vm_apps_schedule', 'api_lxc_mount_points'):
|
||||
environment[name] = lambda vmid: None
|
||||
environment.update({'get_cached_pvesh_cluster_resources_vm': lambda: [
|
||||
{'vmid': 101, 'type': 'lxc', 'status': 'running'}], 'time': Mock(), 'sys': sys})
|
||||
exec(compile(ast.Module(body=[warmer], type_ignores=[]), 'flask_server.py', 'exec'), environment)
|
||||
with patch.object(native, 'discover') as scan, \
|
||||
patch.object(lxc_apps, 'ensure_oci_registration'), \
|
||||
patch.object(lxc_apps, 'load_sidecar', return_value={'apps': []}), \
|
||||
patch.object(lxc_apps, 'get_suggestions', return_value={'extras': []}):
|
||||
for _ in range(2):
|
||||
environment['_vm_app_suggestions_cache'].clear()
|
||||
self.assertEqual(environment['_vm_modal_prewarmer_pass'](), 1)
|
||||
scan.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user