Merge pull request #593 from jcastro/fix/nvidia-subsystem-device-detection

fix: recognize subsystem-qualified NVIDIA GPUs

.github/workflows/test-features.yml

@@ -22,6 +22,8 @@ on:
       - 'tests/test_vm_backup_jobs_api.py'
       - 'tests/test_backup_storage_retention.cjs'
       - 'tests/test_figurine_installation.py'
+      - 'scripts/gpu_tpu/nvidia_installer.sh'
+      - 'tests/test_nvidia*.py'
       - 'scripts/post_install/customizable_post_install.sh'
       - 'AppImage/scripts/post_install_versions.py'
       - 'AppImage/scripts/flask_proxmenux_routes.py'
@@ -169,6 +171,21 @@ jobs:
           NODE_PATH: ${{ github.workspace }}/.github/ci/offline-node/node_modules
         run: node tests/test_backup_storage_retention.cjs
 
+  nvidia-installer:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Check NVIDIA parser, cache and Secure Boot regressions
+        run: |
+          bash -n scripts/gpu_tpu/nvidia_installer.sh
+          python3 -B -m unittest discover -s tests -p 'test_nvidia*.py' -v
+
   installer-and-release:
     runs-on: ubuntu-latest
     timeout-minutes: 10

scripts/gpu_tpu/nvidia_installer.sh

@@ -866,11 +866,9 @@ KEYLASE_PATCH_URL="https://raw.githubusercontent.com/keylase/nvidia-patch/master
 NVIDIA_BRANCHES_CACHE="/var/cache/proxmenux/nvidia_stable_branches.txt"
 NVIDIA_PRODUCTION_HEAD_CACHE="/var/cache/proxmenux/nvidia_production_head.txt"
 NVIDIA_BRANCH_HEADS_CACHE="/var/cache/proxmenux/nvidia_branch_heads.txt"
-# v2: the first generation of these files scraped the whole page and so
+# v3: discard older caches, which either included legacy GPUs (v1) or
-# included the legacy table (see _nvidia_supported_ids_from_page). Those
+# omitted devices listed only with subsystem qualifiers (v2).
-# files say a branch supports GPUs it has dropped, so they are left
+NVIDIA_GPU_SUPPORT_CACHE_PREFIX="/var/cache/proxmenux/nvidia_gpu_support_v3_"
-# behind rather than reused.
-NVIDIA_GPU_SUPPORT_CACHE_PREFIX="/var/cache/proxmenux/nvidia_gpu_support_v2_"
 NVIDIA_BRANCHES_TTL_SECONDS=$((24 * 3600))
 NVIDIA_BRANCHES_URL="https://www.nvidia.com/en-us/drivers/unix/"
 
@@ -975,13 +973,17 @@ get_nvidia_branch_head() {
 # card NVIDIA ever shipped — which is how a Quadro P1000 came to be
 # offered the 595 branch that refuses it at probe time, after compiling
 # and installing perfectly. Everything from that heading on is cut away.
+# NVIDIA identifies each GPU row as devidXXXX or devidXXXX_VENDOR_SUBDEVICE.
+# Read the primary device ID from that identifier, not an exact four-digit
+# table cell: T400/T1000 variants have three IDs in the cell. Subsystem IDs
+# and GPU names such as A100 must not become independent supported devices.
 _nvidia_supported_ids_from_page() {
   awk '
     /legacy GPUs that are no longer supported/ { exit }
     { print }
   ' \
-    | grep -aoiE '<td>[0-9A-F]{4}</td>' \
+    | grep -aoiE '<tr[[:space:]]+id="devid[0-9A-F]{4}([_][0-9A-F]{4})*"' \
-    | grep -aoiE '[0-9A-F]{4}' \
+    | sed -E 's/.*devid([0-9A-Fa-f]{4}).*/\1/' \
     | tr 'A-F' 'a-f' \
     | sort -u
 }

tests/test_nvidia_supported_gpu_ids.py

@@ -0,0 +1,72 @@
+"""Parse NVIDIA device rows without confusing subsystem or legacy IDs."""
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import unittest
+
+
+SOURCE = (Path(__file__).resolve().parents[1] / "scripts/gpu_tpu/nvidia_installer.sh").read_text()
+
+
+def function(name):
+    return re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", SOURCE, re.M | re.S).group(0)
+
+
+class SupportedGpuIdsTests(unittest.TestCase):
+    def parse(self, html):
+        result = subprocess.run(
+            ["bash", "-c", function("_nvidia_supported_ids_from_page")
+             + "_nvidia_supported_ids_from_page"],
+            input=html, text=True, capture_output=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+        return result.stdout.splitlines()
+
+    def test_subsystem_qualified_t400_and_t1000_are_supported(self):
+        # NVIDIA 595.112.07 lists these devices only in qualified rows.
+        self.assertEqual(self.parse('''
+<tr id="devid1FF0_1028_1612"><td>NVIDIA T1000 8GB</td>
+<td>1FF0 1028 1612</td></tr>
+<tr id="devid1FF2_10DE_18FF"><td>NVIDIA T400 4GB</td>
+<td>1FF2 10DE 18FF</td></tr>
+'''), ["1ff0", "1ff2"])
+
+    def test_unqualified_and_duplicate_rows_are_sorted_and_deduplicated(self):
+        self.assertEqual(self.parse('''
+<tr id="devid1E30"><td>Quadro RTX 6000</td><td>1E30</td></tr>
+<tr id="devid1E30_1028_129E"><td>Quadro RTX 8000</td></tr>
+<tr id="devid20B0"><td>A100</td><td>20B0</td></tr>
+'''), ["1e30", "20b0"])
+
+    def test_legacy_rows_and_unrelated_hex_cells_are_excluded(self):
+        self.assertEqual(self.parse('''
+<tr id="devid20B0"><td>A100</td><td>20B0</td></tr>
+<td>1234</td>
+<p>Below are the legacy GPUs that are no longer supported in the
+current driver.</p>
+<tr id="devid1CB1"><td>Quadro P1000</td><td>1CB1</td></tr>
+'''), ["20b0"])
+
+    def test_cache_refresh_does_not_reuse_fresh_v2_device_list(self):
+        prefix = re.search(r'^NVIDIA_GPU_SUPPORT_CACHE_PREFIX="([^"]+)"', SOURCE, re.M).group(1)
+        self.assertTrue(prefix.endswith("_v3_"))
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "nvidia_gpu_support_v2_595.txt").write_text("1e30\n")
+            script = '\n'.join([
+                'NVIDIA_GPU_SUPPORT_CACHE_PREFIX="' + str(root / Path(prefix).name) + '"',
+                'NVIDIA_BRANCHES_TTL_SECONDS=86400',
+                'get_nvidia_branch_head() { echo 595.112.07; }',
+                'curl() { echo \'<tr id="devid1FF2_10DE_18FF"><td>1FF2 10DE 18FF</td></tr>\'; }',
+                function("_nvidia_supported_ids_from_page"),
+                function("refresh_nvidia_gpu_support_for_major"),
+                'refresh_nvidia_gpu_support_for_major 595',
+            ])
+            result = subprocess.run(["bash", "-c", script], text=True, capture_output=True)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual((root / "nvidia_gpu_support_v3_595.txt").read_text(), "1ff2\n")
+
+
+if __name__ == "__main__":
+    unittest.main()