diff --git a/.github/scripts/tests/test_oci_storage_sk.py b/.github/scripts/tests/test_oci_storage_sk.py index e6c72dec..99a5bb76 100644 --- a/.github/scripts/tests/test_oci_storage_sk.py +++ b/.github/scripts/tests/test_oci_storage_sk.py @@ -209,10 +209,9 @@ class InputAndCancellationTests(StorageFixture): # The real background worker has its own catalog lifecycle tests. catalog_check=catalog_checker.return_value.__enter__.return_value catalog_check.active=False;catalog_check.consume.return_value=False - catalog_check.describe.return_value='' - catalog_check.label.return_value=i18n.translate('Refresh catalog') + catalog_check.failed=False;catalog_check.label.return_value='Refresh catalog' check=checker.return_value.__enter__.return_value check.active=False;check.notice.return_value='';check.status.return_value={'available':[]} - check.label.return_value=i18n.translate('Check for updates') + check.label.return_value='Check for updates' self.assertEqual(cli.interactive(self.catalog),0) ui.message.assert_called_once_with('Operáciu sa nepodarilo dokončiť:\n\nfiction-detail') diff --git a/.github/workflows/test-features.yml b/.github/workflows/test-features.yml index 27349ac0..b4031997 100644 --- a/.github/workflows/test-features.yml +++ b/.github/workflows/test-features.yml @@ -22,6 +22,9 @@ on: - 'tests/test_vm_backup_jobs_api.py' - 'tests/test_backup_storage_retention.cjs' - 'tests/test_figurine_installation.py' + - 'scripts/gpu_tpu/nvidia_installer.sh' + - 'tests/test_nvidia*.py' + - 'AppImage/scripts/managed_installs.py' - 'scripts/post_install/customizable_post_install.sh' - 'AppImage/scripts/post_install_versions.py' - 'AppImage/scripts/flask_proxmenux_routes.py' @@ -101,6 +104,9 @@ on: - 'AppImage/scripts/tests/test_proxmox_node_name.py' - 'AppImage/scripts/tests/test_pve_notifications_cfg_writes.py' - 'tests/test_lvm_disk_read_policy.py' + - 'scripts/gpu_tpu/nvidia_installer.sh' + - 'tests/test_nvidia*.py' + - 'AppImage/scripts/managed_installs.py' - 'oci/**' - 'AppImage/scripts/oci_menu_bridge.py' - 'AppImage/scripts/lxc_apps.py' @@ -169,6 +175,21 @@ jobs: NODE_PATH: ${{ github.workspace }}/.github/ci/offline-node/node_modules run: node tests/test_backup_storage_retention.cjs + nvidia-installer: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Check NVIDIA parser, cache and Secure Boot regressions + run: | + bash -n scripts/gpu_tpu/nvidia_installer.sh + python3 -B -m unittest discover -s tests -p 'test_nvidia*.py' -v + installer-and-release: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/AppImage/scripts/managed_installs.py b/AppImage/scripts/managed_installs.py index b05ab307..fe3dc74f 100644 --- a/AppImage/scripts/managed_installs.py +++ b/AppImage/scripts/managed_installs.py @@ -1188,21 +1188,15 @@ def _check_oci_app(entry: dict) -> dict: # ── NVIDIA driver checker ── # -# Source of truth for what's available upstream: -# `https://download.nvidia.com/XFree86/Linux-x86_64/latest.txt` -# returns the single newest version, e.g. "580.105.08" -# `https://download.nvidia.com/XFree86/Linux-x86_64/` -# HTML directory listing — we scrape it for per-branch latest -# (so a user on 570.x gets 570.x's latest, without an automatic -# cross-branch upgrade). -# -# Cache TTL is 7 days because NVIDIA's release cadence on each branch -# is roughly monthly. The cache is in-memory only; AppImage restarts -# refresh it for free. +# The driver update is the version the NVIDIA installer itself would +# recommend on this host: it filters by the running kernel, NVIDIA's +# maintained branches, the PCI IDs of the host's GPUs and the NVENC patch, +# and keeps the installed branch when it can. Asking the installer means the +# Monitor never announces a version the installer then refuses. The installer +# keeps its own caches of NVIDIA's pages for 24 hours. -_NVIDIA_BASE = "https://download.nvidia.com/XFree86/Linux-x86_64" -_NVIDIA_CACHE_TTL = 7 * 86400 -_nvidia_cache: dict[str, Any] = {"versions": [], "fetched_at": 0} +_NVIDIA_INSTALLER = "/usr/local/share/proxmenux/scripts/gpu_tpu/nvidia_installer.sh" +_NVIDIA_CHECK_TIMEOUT = 300 def _version_tuple(v: str) -> tuple: @@ -1219,64 +1213,46 @@ def _version_tuple(v: str) -> tuple: return tuple(out[:3]) -def _fetch_nvidia_versions(force: bool = False) -> list[str]: - """Return the cached list of all upstream versions, or fetch fresh.""" - now = time.time() - if not force and _nvidia_cache["versions"] and \ - now - _nvidia_cache["fetched_at"] < _NVIDIA_CACHE_TTL: - return _nvidia_cache["versions"] +def _nvidia_recommended_version(current: str) -> tuple[Optional[str], Optional[str]]: + """The version the installer recommends for this host, or an error.""" try: - req = urllib.request.Request( - _NVIDIA_BASE + "/", - headers={"User-Agent": "ProxMenux-Monitor/1.0"}, + # An older installer would ignore the option and open its menu. + with open(_NVIDIA_INSTALLER, encoding="utf-8", errors="replace") as handle: + if "--recommended-version" not in handle.read(): + return None, "the NVIDIA installer on this host is older than ProxMenux Monitor" + done = subprocess.run( + ["bash", _NVIDIA_INSTALLER, "--recommended-version", current], + capture_output=True, text=True, timeout=_NVIDIA_CHECK_TIMEOUT, + # The host's curl and Python, not the AppImage's libraries. + env={k: v for k, v in os.environ.items() if k not in ("PYTHONPATH", "LD_LIBRARY_PATH")}, ) - with urllib.request.urlopen(req, timeout=15) as resp: - html = resp.read().decode("utf-8", errors="replace") - except Exception as e: - print(f"[ProxMenux] NVIDIA version fetch failed: {e}") - return _nvidia_cache.get("versions", []) - versions = sorted( - {m.group(1) for m in re.finditer( - r"""href=['"](\d+\.\d+(?:\.\d+)?)/?['"]""", html)}, - key=_version_tuple, - reverse=True, - ) + except (OSError, subprocess.TimeoutExpired) as e: + return None, f"the NVIDIA installer could not be run: {e}" + versions = [line.strip() for line in done.stdout.splitlines() + if re.fullmatch(r"\d+\.\d+(\.\d+)?", line.strip())] if versions: - _nvidia_cache["versions"] = versions - _nvidia_cache["fetched_at"] = now - return versions + return versions[-1], None + if done.returncode == 0: + return None, None + return None, "the NVIDIA installer could not read the version list" def _check_nvidia_xfree86(entry: dict) -> dict: - """Report same-branch bugfix upgrades for the installed driver.""" + """Report the update the NVIDIA installer would recommend.""" current = entry.get("current_version") if not current or not re.match(r"^\d+\.\d+(\.\d+)?$", current): return {"available": False, "latest": None, "last_check": _now_iso(), "error": "no installed version"} - versions = _fetch_nvidia_versions() - if not versions: - return {"available": False, "latest": None, - "last_check": _now_iso(), - "error": "could not parse upstream version listing"} - - current_branch = current.split(".")[0] - same_branch = [v for v in versions if v.split(".")[0] == current_branch] - same_branch_latest = same_branch[0] if same_branch else None - - available = False - latest: Optional[str] = None - if same_branch_latest and \ - _version_tuple(same_branch_latest) > _version_tuple(current): - latest = same_branch_latest - available = True - + recommended, error = _nvidia_recommended_version(current) + available = bool(recommended) and _version_tuple(recommended) > _version_tuple(current) + same_branch = bool(recommended) and recommended.split(".")[0] == current.split(".")[0] return { "available": available, - "latest": latest, + "latest": recommended if available else None, "last_check": _now_iso(), - "error": None, - "_upgrade_kind": "patch" if available else None, + "error": error, + "_upgrade_kind": ("patch" if same_branch else "branch") if available else None, } @@ -1842,13 +1818,9 @@ def check_for_updates(force: bool = False) -> list[dict]: UI reads ``get_active_items()`` to render the inline "update available" line. - ``force`` invalidates the per-source caches (currently only the - NVIDIA versions list — OCI keeps its own internal cache). + ``force`` is kept for callers; the NVIDIA installer and OCI keep + their own caches. """ - if force: - _nvidia_cache["versions"] = [] - _nvidia_cache["fetched_at"] = 0 - updates_available: list[dict] = [] successful_lxc_checks: list[tuple[int, int]] = [] with _lock: diff --git a/scripts/gpu_tpu/nvidia_installer.sh b/scripts/gpu_tpu/nvidia_installer.sh index a296c786..53e2fcd9 100644 --- a/scripts/gpu_tpu/nvidia_installer.sh +++ b/scripts/gpu_tpu/nvidia_installer.sh @@ -866,11 +866,9 @@ KEYLASE_PATCH_URL="https://raw.githubusercontent.com/keylase/nvidia-patch/master NVIDIA_BRANCHES_CACHE="/var/cache/proxmenux/nvidia_stable_branches.txt" NVIDIA_PRODUCTION_HEAD_CACHE="/var/cache/proxmenux/nvidia_production_head.txt" NVIDIA_BRANCH_HEADS_CACHE="/var/cache/proxmenux/nvidia_branch_heads.txt" -# v2: the first generation of these files scraped the whole page and so -# included the legacy table (see _nvidia_supported_ids_from_page). Those -# files say a branch supports GPUs it has dropped, so they are left -# behind rather than reused. -NVIDIA_GPU_SUPPORT_CACHE_PREFIX="/var/cache/proxmenux/nvidia_gpu_support_v2_" +# v3: discard older caches, which either included legacy GPUs (v1) or +# omitted devices listed only with subsystem qualifiers (v2). +NVIDIA_GPU_SUPPORT_CACHE_PREFIX="/var/cache/proxmenux/nvidia_gpu_support_v3_" NVIDIA_BRANCHES_TTL_SECONDS=$((24 * 3600)) NVIDIA_BRANCHES_URL="https://www.nvidia.com/en-us/drivers/unix/" @@ -975,13 +973,17 @@ get_nvidia_branch_head() { # card NVIDIA ever shipped — which is how a Quadro P1000 came to be # offered the 595 branch that refuses it at probe time, after compiling # and installing perfectly. Everything from that heading on is cut away. +# NVIDIA identifies each GPU row as devidXXXX or devidXXXX_VENDOR_SUBDEVICE. +# Read the primary device ID from that identifier, not an exact four-digit +# table cell: T400/T1000 variants have three IDs in the cell. Subsystem IDs +# and GPU names such as A100 must not become independent supported devices. _nvidia_supported_ids_from_page() { awk ' /legacy GPUs that are no longer supported/ { exit } { print } ' \ - | grep -aoiE '[0-9A-F]{4}' \ - | grep -aoiE '[0-9A-F]{4}' \ + | grep -aoiE '/dev/null) versions_list=$(list_available_versions 2>/dev/null) if [[ -z "$latest" ]] && [[ -z "$versions_list" ]]; then - stop_spinner - hybrid_msgbox "$(translate 'Error')" \ - "$(translate 'Could not retrieve versions list from NVIDIA. Please check your internet connection.')\n\nURL: ${NVIDIA_BASE_URL}" 10 80 - DRIVER_VERSION="cancel" return 1 fi @@ -2115,6 +2116,47 @@ show_version_menu() { latest=$(printf '%s\n' "$current_list" | head -n1 | tr -d '[:space:]') fi + NVIDIA_OFFERED_VERSIONS="$current_list" + NVIDIA_RECOMMENDED_VERSION="$latest" + NVIDIA_PATCH_FILTERED="$patch_filtered" + NVIDIA_PATCH_FILTER_NOTE="$patch_filter_note" +} + +# For ProxMenux Monitor: the version the menu would recommend on this host for +# the driver in use (passed in, or read from nvidia-smi), with no interface and +# no change to the host. Prints nothing when none is recommended. +print_recommended_version() { + detect_nvidia_gpus + $NVIDIA_GPU_PRESENT || return 1 + CURRENT_DRIVER_VERSION="${1:-}" + if [[ -z "$CURRENT_DRIVER_VERSION" ]] && command -v nvidia-smi >/dev/null 2>&1; then + CURRENT_DRIVER_VERSION=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -n1) + fi + compute_nvidia_offer || return 2 + [[ -z "$NVIDIA_RECOMMENDED_VERSION" ]] || printf '%s\n' "$NVIDIA_RECOMMENDED_VERSION" +} + +show_version_menu() { + local latest versions_list + local kernel_version + kernel_version=$(uname -r) + + show_proxmenux_logo + msg_title "$(translate 'NVIDIA GPU Driver Installation')" + msg_info "$(translate 'Fetching NVIDIA driver versions supported by your GPU...')" + + if ! compute_nvidia_offer; then + stop_spinner + hybrid_msgbox "$(translate 'Error')" \ + "$(translate 'Could not retrieve versions list from NVIDIA. Please check your internet connection.')\n\nURL: ${NVIDIA_BASE_URL}" 10 80 + DRIVER_VERSION="cancel" + return 1 + fi + local current_list="$NVIDIA_OFFERED_VERSIONS" + local patch_filtered="$NVIDIA_PATCH_FILTERED" + local patch_filter_note="$NVIDIA_PATCH_FILTER_NOTE" + latest="$NVIDIA_RECOMMENDED_VERSION" + local menu_text="$(translate 'Select the NVIDIA driver version to install:')\n\n" menu_text+="$(translate 'Versions shown belong to maintained NVIDIA branches that list your GPU PCI ID and are new enough to build against the running kernel. DKMS compilation is the final validation. The recommended version keeps the current branch, or uses the NVIDIA Production Branch on a fresh install.')" menu_text+="\n\n$(translate 'Running kernel:') ${kernel_version} — $(translate 'oldest driver offered:') $(nvidia_kernel_driver_floor)" @@ -2546,5 +2588,9 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then auto_reinstall_from_state exit $? fi + if [[ "${1:-}" == "--recommended-version" ]]; then + print_recommended_version "${2:-}" + exit $? + fi main fi diff --git a/tests/test_nvidia_monitor_uses_installer.py b/tests/test_nvidia_monitor_uses_installer.py new file mode 100644 index 00000000..202f46b5 --- /dev/null +++ b/tests/test_nvidia_monitor_uses_installer.py @@ -0,0 +1,91 @@ +"""ProxMenux Monitor announces the NVIDIA driver the installer would recommend, +worked out by the same installer function the version menu uses.""" +from pathlib import Path +import re +import subprocess +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +INSTALLER = (ROOT / "scripts/gpu_tpu/nvidia_installer.sh").read_text() +sys.path.insert(0, str(ROOT / "AppImage/scripts")) + +import managed_installs # noqa: E402 + + +def function(name): + return re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", INSTALLER, re.M | re.S).group(0) + + +def answer(stdout, returncode=0): + return subprocess.CompletedProcess([], returncode, stdout, "") + + +class MonitorAsksTheInstallerTests(unittest.TestCase): + def setUp(self): + installer = patch.object(managed_installs, "_NVIDIA_INSTALLER", str(ROOT / "scripts/gpu_tpu/nvidia_installer.sh")) + installer.start() + self.addCleanup(installer.stop) + + def check(self, current, stdout, returncode=0): + with patch.object(managed_installs.subprocess, "run", return_value=answer(stdout, returncode)) as run: + result = managed_installs._check_nvidia_xfree86({"current_version": current}) + return result, run + + def test_an_older_installer_is_not_run(self): + import tempfile + with tempfile.NamedTemporaryFile("w", suffix=".sh") as old: + old.write("main\n"); old.flush() + with patch.object(managed_installs, "_NVIDIA_INSTALLER", old.name), \ + patch.object(managed_installs.subprocess, "run") as run: + result = managed_installs._check_nvidia_xfree86({"current_version": "595.80"}) + run.assert_not_called() + self.assertFalse(result["available"]) + self.assertIn("older", result["error"]) + + def test_the_recommended_version_is_announced_and_the_host_tools_are_used(self): + with patch.dict(managed_installs.os.environ, {"LD_LIBRARY_PATH": "/appimage/lib"}): + result, run = self.check("595.80", "595.112.07\n") + self.assertEqual((result["available"], result["latest"], result["_upgrade_kind"]), + (True, "595.112.07", "patch")) + argv, kwargs = run.call_args.args[0], run.call_args.kwargs + self.assertEqual(argv[-2:], ["--recommended-version", "595.80"]) + self.assertNotIn("LD_LIBRARY_PATH", kwargs["env"]) + + def test_nothing_is_announced_when_the_installer_recommends_the_installed_driver(self): + result, _ = self.check("580.196.07", "580.196.07\n") + self.assertEqual((result["available"], result["latest"], result["error"]), (False, None, None)) + + def test_nothing_is_announced_when_the_installer_offers_nothing(self): + # A GPU no maintained branch supports: the installer recommends nothing. + result, _ = self.check("470.256.02", "") + self.assertEqual((result["available"], result["error"]), (False, None)) + + def test_a_failed_version_list_is_an_error_not_an_update(self): + result, _ = self.check("595.80", "", returncode=2) + self.assertFalse(result["available"]) + self.assertIn("could not read", result["error"]) + + def test_a_branch_change_is_marked(self): + result, _ = self.check("570.211.01", "595.112.07\n") + self.assertEqual(result["_upgrade_kind"], "branch") + + +class OneCalculationTests(unittest.TestCase): + def test_the_menu_and_the_monitor_mode_share_the_calculation(self): + self.assertIn("compute_nvidia_offer", function("show_version_menu")) + self.assertIn("compute_nvidia_offer", function("print_recommended_version")) + self.assertNotIn("download.nvidia.com", (ROOT / "AppImage/scripts/managed_installs.py").read_text()) + + def test_the_monitor_mode_prints_only_the_recommendation(self): + stubs = ('detect_nvidia_gpus() { NVIDIA_GPU_PRESENT=true; }\n' + 'compute_nvidia_offer() { NVIDIA_RECOMMENDED_VERSION="595.112.07"; }\n') + done = subprocess.run(["bash", "-c", stubs + function("print_recommended_version") + + "print_recommended_version 595.80"], + capture_output=True, text=True) + self.assertEqual((done.returncode, done.stdout), (0, "595.112.07\n")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_nvidia_supported_gpu_ids.py b/tests/test_nvidia_supported_gpu_ids.py new file mode 100644 index 00000000..a50d7b43 --- /dev/null +++ b/tests/test_nvidia_supported_gpu_ids.py @@ -0,0 +1,72 @@ +"""Parse NVIDIA device rows without confusing subsystem or legacy IDs.""" +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +SOURCE = (Path(__file__).resolve().parents[1] / "scripts/gpu_tpu/nvidia_installer.sh").read_text() + + +def function(name): + return re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", SOURCE, re.M | re.S).group(0) + + +class SupportedGpuIdsTests(unittest.TestCase): + def parse(self, html): + result = subprocess.run( + ["bash", "-c", function("_nvidia_supported_ids_from_page") + + "_nvidia_supported_ids_from_page"], + input=html, text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.splitlines() + + def test_subsystem_qualified_t400_and_t1000_are_supported(self): + # NVIDIA 595.112.07 lists these devices only in qualified rows. + self.assertEqual(self.parse(''' +NVIDIA T1000 8GB +1FF0 1028 1612 +NVIDIA T400 4GB +1FF2 10DE 18FF +'''), ["1ff0", "1ff2"]) + + def test_unqualified_and_duplicate_rows_are_sorted_and_deduplicated(self): + self.assertEqual(self.parse(''' +Quadro RTX 60001E30 +Quadro RTX 8000 +A10020B0 +'''), ["1e30", "20b0"]) + + def test_legacy_rows_and_unrelated_hex_cells_are_excluded(self): + self.assertEqual(self.parse(''' +A10020B0 +1234 +

Below are the legacy GPUs that are no longer supported in the +current driver.

+Quadro P10001CB1 +'''), ["20b0"]) + + def test_cache_refresh_does_not_reuse_fresh_v2_device_list(self): + prefix = re.search(r'^NVIDIA_GPU_SUPPORT_CACHE_PREFIX="([^"]+)"', SOURCE, re.M).group(1) + self.assertTrue(prefix.endswith("_v3_")) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "nvidia_gpu_support_v2_595.txt").write_text("1e30\n") + script = '\n'.join([ + 'NVIDIA_GPU_SUPPORT_CACHE_PREFIX="' + str(root / Path(prefix).name) + '"', + 'NVIDIA_BRANCHES_TTL_SECONDS=86400', + 'get_nvidia_branch_head() { echo 595.112.07; }', + 'curl() { echo \'1FF2 10DE 18FF\'; }', + function("_nvidia_supported_ids_from_page"), + function("refresh_nvidia_gpu_support_for_major"), + 'refresh_nvidia_gpu_support_for_major 595', + ]) + result = subprocess.run(["bash", "-c", script], text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / "nvidia_gpu_support_v3_595.txt").read_text(), "1ff2\n") + + +if __name__ == "__main__": + unittest.main()