New version 1.2.6

Restores AI Assistant support for OpenAI-compatible endpoints on private IPs, loopback and Docker networks (LiteLLM, LM Studio, LocalAI, vLLM, OmniRoute, self-hosted proxies) and surfaces the server's error under the Load button (#325). Aligns the Secure Gateway wizard's Alpine template selection and pct create with the host's real architecture on x86_64 and arm64 (#324). Consolidates changes landing on develop: atomic notification delivery, custom SSH port for Borg remote targets (#236), optional GitHub API token for app version tracking (#306), and richer replication failure notifications.
2026-09-15 11:16:50 +00:00 · 2026-09-02 22:18:52 +02:00
parent eecd93bf1a
commit ea8c29ecbd
24 changed files with 884 additions and 118 deletions
@@ -471,7 +471,14 @@ def get_provider_models():
            if not ok:
                return jsonify({'success': False, 'models': [], 'message': f'Invalid ollama_url: {err}'}), 400
        if provider == 'openai' and openai_base_url:
-            ok, err = validate_external_url(openai_base_url, allow_loopback=False)
+            # OpenAI-compatible endpoints (LiteLLM, LM Studio, Ollama-proxy,
+            # LocalAI, vLLM, OmniRoute, opencode.ai, …) are LOCAL by design:
+            # documented deployments run on localhost, on the same LAN or
+            # inside a Docker network — all of which use loopback or
+            # RFC1918 addresses. Blocking them made the "Custom Base URL"
+            # feature unusable in practice (issue #325). The AWS metadata
+            # host stays blocked via _SSRF_BLOCKED_HOSTS regardless.
+            ok, err = validate_external_url(openai_base_url, allow_loopback=True)
            if not ok:
                return jsonify({'success': False, 'models': [], 'message': f'Invalid openai_base_url: {err}'}), 400
        
@@ -665,7 +672,14 @@ def test_ai_connection():
            if not ok:
                return jsonify({'success': False, 'message': f'Invalid ollama_url: {err}', 'model': ''}), 400
        if provider == 'openai' and openai_base_url:
-            ok, err = validate_external_url(openai_base_url, allow_loopback=False)
+            # OpenAI-compatible endpoints (LiteLLM, LM Studio, Ollama-proxy,
+            # LocalAI, vLLM, OmniRoute, opencode.ai, …) are LOCAL by design:
+            # documented deployments run on localhost, on the same LAN or
+            # inside a Docker network — all of which use loopback or
+            # RFC1918 addresses. Blocking them made the "Custom Base URL"
+            # feature unusable in practice (issue #325). The AWS metadata
+            # host stays blocked via _SSRF_BLOCKED_HOSTS regardless.
+            ok, err = validate_external_url(openai_base_url, allow_loopback=True)
            if not ok:
                return jsonify({'success': False, 'message': f'Invalid openai_base_url: {err}', 'model': ''}), 400

@@ -2020,6 +2020,20 @@ def _handle_guest_lifecycle(vmid: str, vm_type: str, action: str) -> None:
    if guest_type == 'lxc':
        _invalidate_lxc_ip(guest_id)
    if action in ('start', 'reboot'):
+        if guest_type == 'lxc':
+            # TaskWatcher already owns the authoritative lifecycle event.
+            # Reuse it to retire a reboot-required warning left by the last
+            # scheduled update instead of adding another polling loop.
+            try:
+                import lxc_apps
+                lxc_apps.clear_schedule_reboot_required(guest_id)
+                _vm_cache_invalidate(guest_id, _vm_schedule_cache)
+            except Exception as exc:
+                print(
+                    f'[ProxMenux] could not clear scheduled-update reboot state '
+                    f'for CT {guest_id}: {exc}',
+                    flush=True,
+                )
        _schedule_started_guest_refresh(guest_id, guest_type)
        return
    if action == 'stop':
@@ -13626,6 +13640,42 @@ def api_vm_apps_schedule(vmid):
    return jsonify(result)


+@app.route('/api/vms/<int:vmid>/schedule/log', methods=['GET'])
+@require_auth
+def api_vm_apps_schedule_log(vmid):
+    """Return the bounded tail of the latest scheduled-update log."""
+    try:
+        import lxc_apps
+        schedule = lxc_apps.get_schedule(vmid) or {}
+    except Exception as exc:
+        return jsonify({'error': f'lxc_apps unavailable: {exc}'}), 500
+    name = os.path.basename(str(schedule.get('last_run_log') or ''))
+    if not _LXC_UPDATE_LOG_RE.fullmatch(name) or not name.startswith(f'{vmid}-'):
+        return jsonify({'error': 'no scheduled update log is available'}), 404
+    path = os.path.join(_LXC_UPDATE_LOG_DIR, name)
+    if not os.path.isfile(path):
+        return jsonify({'error': 'scheduled update log was not found'}), 404
+    try:
+        size = os.path.getsize(path)
+        offset = max(0, size - _LXC_UPDATE_LOG_READ_LIMIT)
+        with open(path, 'rb') as stream:
+            stream.seek(offset)
+            content = stream.read(_LXC_UPDATE_LOG_READ_LIMIT).decode('utf-8', errors='replace')
+        if offset:
+            newline = content.find('\n')
+            if newline >= 0:
+                content = content[newline + 1:]
+        return jsonify({
+            'content': content,
+            'size': size,
+            'truncated': bool(offset),
+            'run_at': schedule.get('last_run_at'),
+            'status': schedule.get('last_run_status'),
+        })
+    except OSError as exc:
+        return jsonify({'error': str(exc)}), 500
+
+
@app.route('/api/vms/<int:vmid>/bulk-update', methods=['GET', 'PUT', 'DELETE'])
@require_auth
 def api_vm_bulk_update(vmid):
@@ -14098,6 +14148,8 @@ def _lxc_update_details(
    after: dict,
    verification_pending: bool,
    verification_errors: list[str],
+    reboot_required: bool | None,
+    reboot_packages: list[str],
 ) -> str:
    lines = [
        f"Source: {'Scheduled' if source == 'scheduled' else 'Manual'}",
@@ -14170,6 +14222,12 @@ def _lxc_update_details(
        lines.append('Deferred targets: ' + ', '.join(deferred_targets))
    if reason:
        lines.append(f'Reason: {reason}')
+    if reboot_required is True:
+        lines.append('Restart required: yes')
+        if reboot_packages:
+            lines.append('Restart-triggering packages: ' + ', '.join(reboot_packages[:12]))
+    elif reboot_required is False:
+        lines.append('Restart required: no')
    if verification_pending:
        lines.append('Verification pending until the container is running')
    for error in verification_errors[:4]:
@@ -14194,6 +14252,8 @@ def _finalize_lxc_update(
    reason: str | None = None,
    refresh_docker_inventory: bool = False,
    before_snapshot: dict | None = None,
+    reboot_required: bool | None = None,
+    reboot_packages=None,
 ) -> dict:
    safe_run_id = _normalise_lxc_update_run_id(run_id)
    key = (int(vmid), safe_run_id)
@@ -14273,6 +14333,8 @@ def _finalize_lxc_update(
        after=after,
        verification_pending=verification_pending,
        verification_errors=verification_errors,
+        reboot_required=reboot_required,
+        reboot_packages=list(reboot_packages or []),
    )
    try:
        notification_manager.emit_event(
@@ -14303,6 +14365,8 @@ def _finalize_lxc_update(
        'verification_pending': verification_pending,
        'verification_errors': verification_errors,
        'docker_inventory': docker_inventory,
+        'reboot_required': reboot_required,
+        'reboot_packages': list(reboot_packages or []),
    }
    with _lxc_update_finalization_lock:
        _lxc_update_finalizations[key] = {
@@ -21578,6 +21642,97 @@ _DOCKER_ENGINE_INTEGRATED_COMMAND = (
    'update_docker_engine.py --vmid "$VMID"'
 )
 _scheduled_fired_this_minute: set = set()
+_LXC_UPDATE_LOG_DIR = "/usr/local/share/proxmenux/logs/lxc-updates"
+_LXC_UPDATE_LOG_RE = re.compile(r'^[1-9][0-9]*-scheduled-[a-f0-9]{32}\.log$')
+_LXC_UPDATE_LOG_KEEP_PER_CT = 10
+_LXC_UPDATE_LOG_READ_LIMIT = 2 * 1024 * 1024
+
+
+def _create_lxc_update_log(vmid: int, run_id: str) -> tuple[str | None, str | None]:
+    """Create a private, persistent log for one scheduled LXC run."""
+    name = f'{int(vmid)}-{run_id}.log'
+    if not _LXC_UPDATE_LOG_RE.fullmatch(name):
+        return None, None
+    try:
+        os.makedirs(_LXC_UPDATE_LOG_DIR, mode=0o700, exist_ok=True)
+        path = os.path.join(_LXC_UPDATE_LOG_DIR, name)
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        with os.fdopen(fd, 'w', encoding='utf-8', errors='replace') as stream:
+            stream.write(f'=== ProxMenux scheduled LXC update — CT {vmid} ===\n')
+            stream.write(f'Run ID: {run_id}\n')
+            stream.write(f'Started: {datetime.now().astimezone().isoformat()}\n\n')
+        return name, path
+    except OSError as exc:
+        print(f'[ProxMenux] scheduler: could not create update log for CT {vmid}: {exc}',
+              flush=True)
+        return None, None
+
+
+def _append_lxc_update_log(path: str | None, text: str) -> None:
+    if not path:
+        return
+    try:
+        with open(path, 'a', encoding='utf-8', errors='replace') as stream:
+            stream.write(text)
+    except OSError as exc:
+        print(f'[ProxMenux] scheduler: could not append update log: {exc}', flush=True)
+
+
+def _prune_lxc_update_logs(vmid: int) -> None:
+    try:
+        candidates = sorted(
+            glob.glob(os.path.join(_LXC_UPDATE_LOG_DIR, f'{int(vmid)}-scheduled-*.log')),
+            key=os.path.getmtime,
+            reverse=True,
+        )
+        for path in candidates[_LXC_UPDATE_LOG_KEEP_PER_CT:]:
+            if _LXC_UPDATE_LOG_RE.fullmatch(os.path.basename(path)):
+                os.unlink(path)
+    except OSError as exc:
+        print(f'[ProxMenux] scheduler: update-log retention failed for CT {vmid}: {exc}',
+              flush=True)
+
+
+def _inspect_lxc_reboot_requirement(
+    vmid: int,
+    *,
+    update_succeeded: bool,
+    restart_requested: bool,
+    originally_running: bool,
+) -> tuple[bool | None, list[str], str | None]:
+    """Read Debian's reboot marker without installing extra guest tools."""
+    if update_succeeded and (restart_requested or not originally_running):
+        return False, [], None
+    if _fast_guest_status(vmid, 'lxc') != 'running':
+        return None, [], 'container is not running; reboot marker could not be checked'
+    try:
+        marker = subprocess.run(
+            ['/usr/sbin/pct', 'exec', str(vmid), '--',
+             'test', '-f', '/var/run/reboot-required'],
+            capture_output=True, text=True, timeout=8,
+        )
+    except (FileNotFoundError, OSError, subprocess.TimeoutExpired) as exc:
+        return None, [], str(exc)
+    if marker.returncode == 1:
+        return False, [], None
+    if marker.returncode != 0:
+        return None, [], (marker.stderr or marker.stdout or 'reboot marker check failed').strip()[:300]
+    packages: list[str] = []
+    try:
+        result = subprocess.run(
+            ['/usr/sbin/pct', 'exec', str(vmid), '--',
+             'cat', '/var/run/reboot-required.pkgs'],
+            capture_output=True, text=True, timeout=8,
+        )
+        if result.returncode == 0:
+            packages = list(dict.fromkeys(
+                line.strip()[:160]
+                for line in result.stdout.splitlines()
+                if line.strip()
+            ))[:32]
+    except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
+        pass
+    return True, packages, None


 def _normalise_schedule_targets(sched: dict) -> list[str]:
@@ -21908,16 +22063,40 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
    """Run one scheduled update and finalize it through the shared path."""
    started_at = time.monotonic()
    run_id = f'scheduled-{uuid.uuid4().hex}'
+    log_name, log_path = _create_lxc_update_log(vmid, run_id)
+    originally_running = _fast_guest_status(vmid, 'lxc') == 'running'
    requested_targets = _normalise_schedule_targets(sched)
    targets = list(requested_targets)
    before = _lxc_update_snapshot(vmid)
    deferred_targets: list[str] = []
    reasons: list[str] = []
+    reboot_required: bool | None = None
+    reboot_packages: list[str] = []
+    reboot_check_error: str | None = None

    def finish(status: str, actual_target: str, executed: list[str]) -> dict:
        reason = '; '.join(dict.fromkeys(value for value in reasons if value)) or None
        duration_seconds = max(0, int(time.monotonic() - started_at))
        labels = _lxc_update_target_labels(requested_targets, [], before)
+        footer = [
+            '',
+            '=== ProxMenux result ===',
+            f'Finished: {datetime.now().astimezone().isoformat()}',
+            f'Status: {status}',
+            f'Duration: {duration_seconds}s',
+        ]
+        if reason:
+            footer.append(f'Reason: {reason}')
+        if reboot_required is True:
+            footer.append('Restart required: yes')
+            if reboot_packages:
+                footer.append('Restart-triggering packages: ' + ', '.join(reboot_packages))
+        elif reboot_required is False:
+            footer.append('Restart required: no')
+        elif reboot_check_error:
+            footer.append(f'Restart check unavailable: {reboot_check_error}')
+        _append_lxc_update_log(log_path, '\n'.join(footer) + '\n')
+        _prune_lxc_update_logs(vmid)
        finalization = _finalize_lxc_update(
            vmid,
            run_id=run_id,
@@ -21935,6 +22114,8 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
                value.startswith('docker-') for value in requested_targets
            ),
            before_snapshot=before,
+            reboot_required=reboot_required,
+            reboot_packages=reboot_packages,
        )
        return {
            'status': status,
@@ -21945,6 +22126,9 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
            'executed_targets': list(executed),
            'deferred_targets': list(deferred_targets),
            'duration_seconds': duration_seconds,
+            'log_name': log_name,
+            'reboot_required': reboot_required,
+            'reboot_packages': list(reboot_packages),
            'finalization': finalization,
        }

@@ -22063,13 +22247,32 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
        "1" if any(value.startswith('docker-') for value in requested_targets) else "0"
    )
    try:
-        r = subprocess.run(
-            ["bash", _APPLY_UPDATES_SCRIPT],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60 * 60,  # 1h hard cap so a stuck run doesn't
-                              # block the queue forever
+        if log_path:
+            with open(log_path, 'a', encoding='utf-8', errors='replace') as log_stream:
+                r = subprocess.run(
+                    ["bash", _APPLY_UPDATES_SCRIPT],
+                    env=env,
+                    stdout=log_stream,
+                    stderr=subprocess.STDOUT,
+                    text=True,
+                    timeout=60 * 60,  # 1h hard cap so a stuck run doesn't
+                                      # block the queue forever
+                )
+        else:
+            r = subprocess.run(
+                ["bash", _APPLY_UPDATES_SCRIPT],
+                env=env,
+                capture_output=True,
+                text=True,
+                timeout=60 * 60,
+            )
+        reboot_required, reboot_packages, reboot_check_error = (
+            _inspect_lxc_reboot_requirement(
+                vmid,
+                update_succeeded=r.returncode == 0,
+                restart_requested=bool(sched.get("restart")),
+                originally_running=originally_running,
+            )
        )
        if r.returncode != 0:
            reasons.append(f'update runner exited with code {r.returncode}')
@@ -22078,6 +22281,14 @@ def _run_scheduled_update(vmid: int, sched: dict) -> dict:
            return finish('partial', target, targets)
        return finish('success', target, targets)
    except subprocess.TimeoutExpired:
+        reboot_required, reboot_packages, reboot_check_error = (
+            _inspect_lxc_reboot_requirement(
+                vmid,
+                update_succeeded=False,
+                restart_requested=False,
+                originally_running=originally_running,
+            )
+        )
        reasons.append('scheduled update timed out')
        return finish('failure', target, targets)
    except Exception as exc:
@@ -22131,7 +22342,17 @@ def _scheduler_loop():
                    actual_target = outcome.get('target') or 'both'
                    reason = outcome.get('reason')
                    try:
-                        lxc_apps.record_schedule_run(_vmid, status, actual_target, reason)
+                        lxc_apps.record_schedule_run(
+                            _vmid,
+                            status,
+                            actual_target,
+                            reason,
+                            log_name=outcome.get('log_name'),
+                            reboot_required=outcome.get('reboot_required'),
+                            reboot_packages=outcome.get('reboot_packages'),
+                        )
+                        _vm_cache_invalidate(_vmid, _vm_schedule_cache)
+                        _publish_guest_modal_cache_revision(_vmid)
                    except Exception as e:
                        print(f"[ProxMenux] scheduler: could not record run for {_vmid}: {e}")
                    print(f"[ProxMenux] scheduler: CT {_vmid} finished with status={status}"
@@ -3018,7 +3018,16 @@ def scheduled_app_release_gate(
    return {"allowed": True, "status": "ready", "reason": None}


-def record_schedule_run(vmid, status: str, target: str, reason: Optional[str] = None) -> bool:
+def record_schedule_run(
+    vmid,
+    status: str,
+    target: str,
+    reason: Optional[str] = None,
+    *,
+    log_name: Optional[str] = None,
+    reboot_required: Optional[bool] = None,
+    reboot_packages: Optional[list[str]] = None,
+) -> bool:
    """Called by the scheduler after a fired run completes. Updates
    the schedule with last_run_at + last_run_status so the UI can show
    the outcome. `status` is one of "success" | "failure" |
@@ -3034,6 +3043,38 @@ def record_schedule_run(vmid, status: str, target: str, reason: Optional[str] =
            sidecar["schedule"]["last_run_reason"] = str(reason)[:300]
        else:
            sidecar["schedule"].pop("last_run_reason", None)
+        if log_name:
+            sidecar["schedule"]["last_run_log"] = os.path.basename(str(log_name))[:220]
+        else:
+            sidecar["schedule"].pop("last_run_log", None)
+        if reboot_required is None:
+            sidecar["schedule"].pop("last_run_reboot_required", None)
+        else:
+            sidecar["schedule"]["last_run_reboot_required"] = bool(reboot_required)
+        packages = [
+            str(package).strip()[:160]
+            for package in (reboot_packages or [])[:32]
+            if str(package).strip()
+        ]
+        if reboot_required and packages:
+            sidecar["schedule"]["last_run_reboot_packages"] = packages
+        else:
+            sidecar["schedule"].pop("last_run_reboot_packages", None)
+        sidecar["updated_at"] = _now_iso()
+        return _write_sidecar(vmid, sidecar)
+
+
+def clear_schedule_reboot_required(vmid) -> bool:
+    """Clear a persisted reboot warning after the CT starts or reboots."""
+    with _cache_lock:
+        sidecar = _read_sidecar(vmid)
+        schedule = (sidecar or {}).get("schedule")
+        if not isinstance(schedule, dict):
+            return False
+        if schedule.get("last_run_reboot_required") is not True:
+            return True
+        schedule["last_run_reboot_required"] = False
+        schedule.pop("last_run_reboot_packages", None)
        sidecar["updated_at"] = _now_iso()
        return _write_sidecar(vmid, sidecar)

@@ -361,32 +361,73 @@ def get_available_storages() -> List[Dict[str, Any]]:
    return storages


+def _host_arch() -> str:
+    """Return the host's dpkg architecture (`amd64`, `arm64`, ...).
+
+    Falls back to mapping `uname -m` when `dpkg --print-architecture` is
+    unavailable — the Alpine LXC template filenames use the dpkg style
+    (`amd64`, `arm64`), so `uname -m`'s `x86_64` / `aarch64` gets
+    translated to that form.
+    """
+    try:
+        rc, out, _ = _run_pve_cmd(["dpkg", "--print-architecture"], timeout=5)
+        arch = out.strip()
+        if rc == 0 and arch:
+            return arch
+    except Exception:
+        pass
+    try:
+        machine = os.uname().machine.lower()
+    except Exception:
+        machine = ''
+    return {
+        'x86_64': 'amd64', 'amd64': 'amd64',
+        'aarch64': 'arm64', 'arm64': 'arm64',
+        'armv7l': 'armhf', 'armhf': 'armhf',
+        'i686': 'i386', 'i386': 'i386',
+    }.get(machine, 'amd64')
+
+
 def _download_alpine_template(storage: str = DEFAULT_STORAGE) -> bool:
-    """Download the latest Alpine LXC template using pveam."""
+    """Download the latest Alpine LXC template using pveam.
+
+    Filters by the host's architecture so an x86_64 host does not end up
+    with an arm64 template — the previous naive `for line in out` loop
+    kept the last match and could pick any arch that `pveam available`
+    happened to list (issue #324).
+    """
    print("[*] Downloading Alpine Linux template...")
    logger.info("Downloading Alpine template via pveam")
-    
+
+    host_arch = _host_arch()
+    logger.info(f"Host architecture detected: {host_arch}")
+
    # Update template list first
    rc, out, err = _run_pve_cmd(["pveam", "update"], timeout=60)
    if rc != 0:
        logger.warning(f"Failed to update template list: {err}")
-    
+
    # Get available Alpine templates
    rc, out, err = _run_pve_cmd(["pveam", "available", "--section", "system"], timeout=30)
    if rc != 0:
        logger.error(f"Failed to list available templates: {err}")
        return False
-    
-    # Find latest Alpine template
+
+    # Find latest Alpine template FOR THIS HOST'S ARCH. Template names
+    # follow `alpine-<ver>-default_<date>_<arch>.tar.xz`; we require the
+    # arch token to match the host before considering the candidate.
    alpine_template = None
+    arch_token = f"_{host_arch}."
    for line in out.strip().split('\n'):
-        if 'alpine-' in line.lower():
-            parts = line.split()
-            if len(parts) >= 2:
-                alpine_template = parts[1]  # Template name is usually second column
-    
+        low = line.lower()
+        if 'alpine-' not in low or arch_token not in low:
+            continue
+        parts = line.split()
+        if len(parts) >= 2:
+            alpine_template = parts[1]  # Template name is usually second column
+
    if not alpine_template:
-        logger.error("No Alpine template found in available templates")
+        logger.error(f"No Alpine template found for architecture {host_arch}")
        return False
    
    # Download the template
@@ -410,11 +451,22 @@ def _find_alpine_template(storage: str = DEFAULT_STORAGE, auto_download: bool =
    if rc == 0 and out.strip():
        template_dir = os.path.dirname(out.strip())
    
-    # Look for Alpine templates
+    # Look for Alpine templates matching the host architecture. Without
+    # this filter, a host that happened to have an arm64 Alpine template
+    # sitting in its cache from a previous experiment would be handed
+    # that template — resulting in `arch: arm64` on the container and
+    # the `Exec format error` from issue #324.
+    host_arch = _host_arch()
+    arch_token = f"_{host_arch}."
    try:
        templates = os.listdir(template_dir)
-        alpine_templates = [t for t in templates if t.startswith("alpine-") and t.endswith((".tar.xz", ".tar.gz", ".tar"))]
-        
+        alpine_templates = [
+            t for t in templates
+            if t.startswith("alpine-")
+            and t.endswith((".tar.xz", ".tar.gz", ".tar"))
+            and arch_token in t.lower()
+        ]
+
        if alpine_templates:
            # Sort to get latest version
            alpine_templates.sort(reverse=True)
@@ -882,6 +934,7 @@ def deploy_app(app_id: str, config: Dict[str, Any], installed_by: str = "web") -
    
    pct_cmd = [
        "pct", "create", str(vmid), template,
+        "--arch", _host_arch(),
        "--hostname", hostname,
        "--memory", str(container_def.get("memory", 512)),
        "--cores", str(container_def.get("cores", 1)),