ProxMenux/web/messages/en/guides/nvidia-manual.json

{
  "meta": {
    "title": "Installing NVIDIA Drivers on Proxmox VE 9 (manual procedure) | ProxMenux Guides",
    "description": "Manual installation and configuration of NVIDIA drivers on a Proxmox VE 9 host, plus the LXC wiring needed to expose the GPU to one or more containers. Covers driver install, persistence service, optional NVENC patch and the per-container setup.",
    "ogTitle": "Installing NVIDIA Drivers on Proxmox VE 9 (manual procedure)",
    "ogDescription": "Manual NVIDIA driver install on PVE 9 — host driver, persistence service, optional NVENC patch and per-container LXC wiring."
  },
  "header": {
    "title": "Installing NVIDIA Graphics Card Drivers on Proxmox VE 9 (manual procedure)",
    "description": "Manual installation and configuration of NVIDIA drivers on a Proxmox VE 9 host, plus the LXC wiring needed to expose the GPU to one or more containers. Covers driver install, persistence service, optional NVENC patch and the per-container setup.",
    "section": "Guides"
  },
  "intro": {
    "calloutTitle": "Note",
    "calloutBody": "This is the manual procedure preserved as a reference. For most users the recommended path is the automated ProxMenux flow: <strong>Install NVIDIA Drivers (Host)</strong> in the <em>GPUs and Coral-TPU</em> menu, and <strong>Add GPU to LXC</strong> (or <strong>Add GPU to VM</strong>) for the per-guest wiring. The automated flow handles the same steps documented here (with extra safety checks: kernel-headers compatibility, IOMMU validation, VFIO conflict resolution). This page is for users who want to understand each command, or who need to deviate from the standard flow.",
    "targetNote": "Targeted at <strong>Proxmox VE 9</strong> (Debian Trixie). PVE 7 (Bullseye) and PVE 8 (Bookworm) are no longer covered.",
    "stepsTitle": "What you'll do",
    "steps": [
      "Prepare the Proxmox VE 9 host (blacklist nouveau, repos, prerequisites).",
      "Install the NVIDIA driver on the host.",
      "Install the NVIDIA persistence service.",
      "(Optional) Apply the keylase patch to lift the NVENC concurrent-session limit on consumer GPUs.",
      "Wire the GPU into one or more LXC containers and install the matching driver inside each."
    ]
  },
  "prepareHost": {
    "heading": "1. Prepare the host (PVE 9)",
    "blacklistHeading": "1.1 Blacklist nouveau",
    "blacklistBody": "Check whether the open-source <code>nouveau</code> driver is already blacklisted:",
    "blacklistCheckCode": "cat /etc/modprobe.d/blacklist.conf",
    "blacklistAdd": "If <code>blacklist nouveau</code> does not appear, add it and reboot:",
    "blacklistAddCode": "echo \"blacklist nouveau\" >> /etc/modprobe.d/blacklist.conf\nupdate-initramfs -u -k all\nreboot",
    "blacklistImageAlt": "Blacklist check",
    "reposHeading": "1.2 Verify repositories (PVE 9 / Trixie)",
    "reposBody": "Post-install tools may already have configured the repositories. Inspect the existing sources first and skip changes that are already in place; do not create duplicate entries.",
    "reposOtherwise": "Otherwise, on a vanilla PVE 9 install with no enterprise subscription:",
    "reposEditCode": "nano /etc/apt/sources.list.d/proxmox.sources",
    "reposPveBody": "Make sure it contains the no-subscription source for Trixie:",
    "reposPveCode": "Types: deb\nURIs: http://download.proxmox.com/debian/pve\nSuites: trixie\nComponents: pve-no-subscription\nSigned-By: /usr/share/keyrings/proxmox-archive-keyring.gpg",
    "reposDebianBody": "And the Debian sources at <code>/etc/apt/sources.list.d/debian.sources</code>:",
    "reposDebianCode": "Types: deb\nURIs: http://deb.debian.org/debian/\nSuites: trixie trixie-updates\nComponents: main contrib non-free non-free-firmware\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg\n\nTypes: deb\nURIs: http://security.debian.org/debian-security/\nSuites: trixie-security\nComponents: main contrib non-free non-free-firmware\nSigned-By: /usr/share/keyrings/debian-archive-keyring.gpg",
    "updateHeading": "1.3 Update the system and install prerequisites",
    "updateCode": "apt update && apt dist-upgrade -y",
    "buildToolsBody": "Install the build tools and kernel headers needed to compile the NVIDIA kernel module:",
    "buildToolsCode": "apt-get install -y git\napt-get install -qqy pve-headers-$(uname -r) gcc make"
  },
  "installDriver": {
    "heading": "2. Install the NVIDIA driver on the host",
    "pickHeading": "2.1 Pick a driver version",
    "pickBody": "Choose a driver branch that supports your GPU and running kernel. The newest branch may not support older GPU models; check NVIDIA’s supported-products list for the chosen release:",
    "pickUrlCode": "https://download.nvidia.com/XFree86/Linux-x86_64/latest.txt",
    "nvencCallout": "If you plan to apply the NVENC patch (step 4), verify the patch supports your chosen driver version first: <patchLink>github.com/keylase/nvidia-patch</patchLink>",
    "nvencCalloutTitle": "Heads up",
    "pickReplace": "Replace <code>latest.txt</code> in the URL with the version number to find the installer file ending in <code>.run</code>. The full driver list is at:",
    "pickListCode": "https://download.nvidia.com/XFree86/Linux-x86_64/",
    "pickImageAlt": "NVIDIA driver download",
    "pickVersionNote": "Throughout the rest of this guide, replace <code>'<'VERSION'>'</code> with the actual version (for example <code>580.95.05</code>).",
    "downloadHeading": "2.2 Download and run the installer",
    "downloadCode": "mkdir -p /opt/nvidia\ncd /opt/nvidia\nwget https://download.nvidia.com/XFree86/Linux-x86_64/VERSION/NVIDIA-Linux-x86_64-VERSION.run\nchmod +x NVIDIA-Linux-x86_64-VERSION.run",
    "firstPassBody": "Run the installer. The option below can disable <code>nouveau</code> if necessary; installation may need a reboot before it can finish:",
    "firstPassCode": "./NVIDIA-Linux-x86_64-VERSION.run --no-questions --ui=none --disable-nouveau\nreboot",
    "secondPassBody": "After the host comes back, run the installer again to compile and install the kernel module:",
    "secondPassCode": "/opt/nvidia/NVIDIA-Linux-x86_64-VERSION.run --no-questions --ui=none",
    "modulesHeading": "2.3 Load NVIDIA modules at boot",
    "modulesBody": "Edit the modules-load configuration:",
    "modulesEditCode": "nano /etc/modules-load.d/modules.conf",
    "modulesAddBody": "Add the NVIDIA modules needed for host/LXC use. VFIO modules are for VM passthrough and are not required for this LXC setup:",
    "modulesAddCode": "nvidia\nnvidia_uvm",
    "modulesSaveBody": "Save (<code>Ctrl+X</code>) and rebuild initramfs:",
    "modulesSaveCode": "update-initramfs -u -k all",
    "udevHeading": "2.4 Create udev rules",
    "udevBody": "So that <code>/dev/nvidia*</code> device nodes are created when the modules load:",
    "udevEditCode": "nano /etc/udev/rules.d/70-nvidia.rules",
    "udevRulesCode": "# /etc/udev/rules.d/70-nvidia.rules\nKERNEL==\"nvidia\", RUN+=\"/bin/bash -c '/usr/bin/nvidia-smi -L'\"\nKERNEL==\"nvidia_uvm\", RUN+=\"/bin/bash -c '/usr/bin/nvidia-modprobe -c0 -u'\"",
    "udevSaveBody": "Save (<code>Ctrl+X</code>)."
  },
  "persistence": {
    "heading": "3. NVIDIA driver persistence service",
    "body": "The persistence daemon keeps the GPU initialized when no other application is using it. This reduces repeated initialization latency and preserves GPU-initialization state between clients:",
    "installCode": "cd /opt/nvidia\ngit clone https://github.com/NVIDIA/nvidia-persistenced.git\ncd nvidia-persistenced/init\n./install.sh\nreboot",
    "verifyBody": "Verify the driver is loaded and the service is running after reboot:",
    "verifySmiCode": "nvidia-smi",
    "smiImageAlt": "NVIDIA SMI output",
    "verifyServiceCode": "systemctl status nvidia-persistenced",
    "serviceImageAlt": "NVIDIA persistence service status"
  },
  "nvenc": {
    "heading": "4. (Optional) Lift the NVENC concurrent-session limit",
    "body": "NVENC session limits depend on the GPU model and driver version; consult NVIDIA’s support matrix rather than assuming a fixed limit by GPU generation. The keylase patch removes the software limit for supported driver versions. It can help encoding workloads such as Plex or Jellyfin transcoding.",
    "code": "cd /opt/nvidia\ngit clone https://github.com/keylase/nvidia-patch.git\ncd nvidia-patch\n./patch.sh",
    "imageAlt": "NVIDIA patch application",
    "after": "The patch must be re-applied after every driver update. The keylase repo also includes <code>patch-fbc.sh</code> for the FBC (frame buffer capture) limit if you need it."
  },
  "lxcSetup": {
    "heading": "5. Configure an LXC container to use the GPU",
    "identifyHeading": "5.1 Identify the device numbers",
    "identifyBody": "On the host:",
    "identifyCode": "ls -l /dev/nv*",
    "identifyImageAlt": "NVIDIA device list",
    "identifyNote": "Note the major numbers — they vary between systems. Typical values:",
    "tableHeaders": {
      "device": "Device",
      "major": "Typical major"
    },
    "tableRows": [
      {
        "device": "<code>/dev/nvidia0</code>, <code>/dev/nvidiactl</code>",
        "major": "195"
      },
      {
        "device": "<code>/dev/nvidia-uvm</code>, <code>/dev/nvidia-uvm-tools</code>",
        "major": "509 (varies)"
      },
      {
        "device": "<code>/dev/dri/*</code>",
        "major": "226"
      },
      {
        "device": "<code>/dev/nvidia-modeset</code>",
        "major": "195 (shares with nvidia)"
      }
    ],
    "editHeading": "5.2 Edit the LXC config",
    "editBody": "Stop the container first if it's running. Open its config file (replace <code>'<'CTID'>'</code> with the container ID):",
    "editCode": "nano /etc/pve/lxc/CTID.conf",
    "editConfigBody": "Comment out any pre-existing <code>lxc.cgroup2.devices.allow</code> or <code>/dev/dri</code> lines that conflict, then append the NVIDIA wiring (adjust the major numbers to match what <code>ls -l /dev/nv*</code> showed on <strong>your</strong> host):",
    "editConfigCode": "lxc.cgroup2.devices.allow: c 195:* rwm\nlxc.cgroup2.devices.allow: c 509:* rwm\nlxc.mount.entry: /dev/nvidia0 dev/nvidia0 none bind,optional,create=file\nlxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file\nlxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file\nlxc.mount.entry: /dev/nvidia-modeset dev/nvidia-modeset none bind,optional,create=file\nlxc.mount.entry: /dev/nvidia-uvm-tools dev/nvidia-uvm-tools none bind,optional,create=file",
    "editConfigImageAlt": "LXC configuration",
    "editSave": "Save (<code>Ctrl+X</code>) and start the container.",
    "installCtHeading": "5.3 Install the driver inside the container",
    "installCtCalloutTitle": "Important",
    "installCtCalloutBody": "This part runs <strong>inside</strong> the container, not on the host.",
    "installCtBody": "The kernel module is already loaded by the host — the container only needs the userland libraries that match the same driver version:",
    "installCtCode": "mkdir -p /opt/nvidia\ncd /opt/nvidia\nwget https://download.nvidia.com/XFree86/Linux-x86_64/VERSION/NVIDIA-Linux-x86_64-VERSION.run\nchmod +x NVIDIA-Linux-x86_64-VERSION.run\n./NVIDIA-Linux-x86_64-VERSION.run --no-kernel-module",
    "installCtAfter": "Review the prompts and keep this installation limited to user-space libraries. Do not build or install a kernel module inside the LXC.",
    "installCtImageAlt": "NVIDIA driver installation",
    "verifyCtHeading": "5.4 Verify inside the container",
    "verifyCtSmiCode": "nvidia-smi",
    "verifyCtSmiImageAlt": "NVIDIA SMI in LXC",
    "verifyCtLsCode": "ls -l /dev/nv*",
    "verifyCtLsImageAlt": "NVIDIA devices in LXC",
    "verifyCtAfter": "You should see the GPU listed and the device nodes mounted into the container's filesystem.",
    "workloadHeading": "5.5 Confirm a real workload picks up the GPU",
    "workloadBody": "Enable NVIDIA hardware acceleration in Plex or Jellyfin, start a video transcode and check its dashboard or logs for actual GPU use. Installing the driver alone does not enable acceleration in the application.",
    "workloadImage1Alt": "Plex using NVIDIA GPU",
    "workloadImage2Alt": "Plex using NVIDIA GPU - active session",
    "repeatNote": "Repeat <strong>section 5</strong> for each additional container that needs GPU access. Keep the user-space driver libraries in every container aligned when the host driver is updated."
  },
  "docker": {
    "heading": "6. (Optional) NVIDIA Docker inside an LXC",
    "body": "If Docker already runs inside the LXC, install <code>nvidia-container-toolkit</code> there to expose the GPU to its Docker containers. GPU access from the LXC must work first. The following commands are for a rootful Docker installation:",
    "code": "apt-get update\napt-get install -y ca-certificates curl gnupg\ncurl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg\ncurl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' > /etc/apt/sources.list.d/nvidia-container-toolkit.list\napt-get update\napt-get install -y nvidia-container-toolkit\nnvidia-ctk runtime configure --runtime=docker\nsystemctl restart docker",
    "after": "These commands configure NVIDIA’s repository, install the toolkit, configure the Docker runtime and restart Docker. LXC device permissions and cgroup restrictions still apply; verify GPU use with a test workload."
  },
  "troubleshoot": {
    "heading": "Troubleshooting",
    "items": [
      "<strong><code>nvidia-smi</code> works on the host but reports \"No devices found\" in the container:</strong> check device mounts, permissions and matching driver libraries. If versions differ, install the host’s <code>'<'VERSION'>'</code> in the container with <code>--no-kernel-module</code>.",
      "<strong>Driver compilation fails:</strong> inspect the build log. \"No precompiled kernel interface was found\" can simply mean the installer must compile locally. Install headers matching <code>uname -r</code> using <code>apt-get install pve-headers-$(uname -r)</code>, or the equivalent proxmox-headers package for that kernel.",
      "<strong>NVENC sessions remain limited:</strong> confirm the patch supports the active driver. A driver update can overwrite it. For a supported version, reapply <code>./patch.sh</code> from <code>/opt/nvidia/nvidia-patch</code> and review its output.",
      "<strong>GPU stops responding after idle:</strong> check <code>systemctl status nvidia-persistenced</code> and enable the service if intended. Also inspect kernel and driver logs; a stopped persistence daemon is not the only possible cause.",
      "<strong>Container starts but <code>ls /dev/nv*</code> shows no NVIDIA devices:</strong> check the bind-mount entries and host device paths. Use <code>ls -l /dev/nv*</code> on the host to verify NVIDIA major numbers, then check the <code>lxc.cgroup2.devices.allow</code> rules and permissions."
    ]
  }
}