{ "meta": { "title": "Switch GPU Mode (VM ↔ LXC) | ProxMenux Documentation", "description": "Move an already-assigned GPU between VM mode (vfio-pci) and LXC mode (native driver) on a Proxmox host. ProxMenux detects the current binding, prompts per-workload conflict policies, handles orphan audio cleanup, and rebuilds initramfs only when needed." }, "header": { "title": "Switch GPU Mode (VM ↔ LXC)", "description": "Move an assigned GPU between VM passthrough and LXC sharing. ProxMenux manages the host driver binding, VFIO configuration, module settings and initramfs, and asks how to handle affected guests. Review those choices and verify the resulting configuration after reboot.", "section": "Hardware: GPUs and Coral-TPU" }, "intro": { "title": "What this does", "body": "A GPU on a Proxmox host lives in one of two modes: bound to vfio-pci (reserved for a VM) or bound to its native driver (i915 / amdgpu / nvidia) so the host + LXCs can share it. Switch GPU Mode flips between those two without you having to hand-edit vfio.conf, manage blacklists, or remember which VM / LXC lines point to the card. It also warns you cleanly if a workload still references the GPU so you don't end up with a broken VM at boot." }, "graphics": { "lxcTitle": "Ready for LXC containers", "lxcDesc": "Native driver active", "vmTitle": "Ready for VM passthrough", "vmDesc": "VFIO-PCI driver active" }, "when": { "heading": "When should I use this?", "intro": "Use this script when a GPU is already assigned and you want to move it:", "headerSituation": "Situation", "headerUse": "Use this page?", "rows": [ { "situation": "GPU is free — never assigned. Want to give it to a VM.", "useRich": "No — use Add GPU to VM" }, { "situation": "GPU is free — never assigned. Want to give it to an LXC.", "useRich": "No — use Add GPU to LXC" }, { "situationRich": "GPU is in a VM via vfio-pci, I want to use it from an LXC instead.", "useRich": "Yes — this page." }, { "situation": "GPU is shared with an LXC, I want to dedicate it to a VM.", "useRich": "Yes — this page." }, { "situation": "I just want to completely unbind a GPU from everything.", "use": "LXC mode restores host-driver ownership; the VM conflict policy can remove hostpci entries, but LXC device entries must be removed separately. This does not leave the GPU unbound from every driver." } ] }, "prereqs": { "title": "Before you start", "assigned": "A GPU already assigned — either in a VM via VFIO or attached to at least one LXC. If you haven't assigned it yet, start from Add GPU to VM / LXC instead.", "iommu": "IOMMU must be active on the host when switching to VM mode. Firmware settings and kernel support both matter. If intel_iommu=on or amd_iommu=on is missing from the running kernel, the script adds it with iommu=pt to /etc/kernel/cmdline (ZFS-root marker) or /etc/default/grub, then runs proxmox-boot-tool refresh or update-grub. Check any warnings and reboot when prompted.", "iommuCheck": "dmesg | grep -i 'IOMMU enabled' | head -1", "reboot": "Plan for a reboot. Persistent driver and boot changes may require rebuilding initramfs and rebooting. The script presents a reboot prompt at the end.", "knowList": "Know which VMs / LXCs are using the GPU. The script will find them and ask what to do with each, but it's faster if you already know the list." }, "blocklist": { "title": "Not all GPUs are safe to pass to a VM", "body": "The script blocks VM mode for device IDs 8086:5a84 and 8086:5a85 under its compatibility policy. If a selected GPU matches, it stops the VM-mode switch and returns to GPU selection, or exits when it is the only GPU. This blocklist does not prevent switching to LXC mode; other checks can still apply." }, "running": { "heading": "Running the script", "body": "Open ProxMenux on the host, go to Hardware: GPUs and Coral-TPU → Switch GPU Mode (VM ↔ LXC).", "imageAlt": "Menu entry for 'Switch GPU Mode (VM ↔ LXC)' inside Hardware: GPUs and Coral-TPU" }, "howRuns": { "heading": "How the script runs", "body": "GPU selection and direction are reviewed before the main switch. A separate stale-VFIO cleanup can ask for confirmation earlier, and affected guests have their own conflict prompts. Read each confirmation: there is not one final confirmation covering every possible change." }, "walkthrough": { "heading": "Walking through the flow", "detect": { "title": "Detect GPUs and their current binding", "body": "The script scans every VGA / 3D / Display controller on the host and inspects /sys/bus/pci/devices/*/driver to find the current kernel driver. You'll see each GPU labelled with its name, PCI slot and current driver binding — so you can tell at a glance which mode it's in.", "imageAlt": "GPU checklist showing each detected GPU with its current driver (vfio-pci / nvidia / amdgpu / i915) and PCI slot" }, "pickGpu": { "title": "Pick the GPU(s) to switch", "body": "Single GPU → auto-selected. Multiple GPUs → checklist. You can tick several, but they must all be in the same current mode — otherwise the script can't pick a target mode for the batch and you get a \"mixed mode\" warning asking you to narrow the selection.", "tipTitle": "Batching switches", "tipBody": "Useful when you're rebuilding a host: \"All three NVIDIAs go to VM mode, then the iGPU goes back to LXC.\" Two runs, each with uniform target, much less friction than one-at-a-time." }, "direction": { "title": "Review the proposed direction", "intro": "Based on the current mode, the script proposes the opposite as target:", "items": [ "VM → LXC: unbind from vfio-pci, let the native driver (nvidia, amdgpu, i915) reclaim the card so LXCs can share it. On NVIDIA, the per-BDF entry is removed from /etc/udev/rules.d/10-proxmenux-vfio-bind.rules so the nvidia module reclaims the GPU after reboot.", "LXC → VM: prepares vfio-pci ownership. AMD / Intel use native-driver blacklisting and options vfio-pci ids=…. For NVIDIA, nvidia is globally blacklisted only when every host NVIDIA GPU is in VFIO; otherwise per-BDF rules set driver_override=vfio-pci for the affected IOMMU-group devices, and NVIDIA GPUs outside those groups keep nvidia. Review every group member before switching." ], "outro": "Confirm the direction or cancel." }, "conflict": { "title": "Conflict policy per affected workload", "body": "The script checks /etc/pve/lxc/*.conf for GPU-device references by driver family, and /etc/pve/qemu-server/*.conf for PCI assignments. LXC detection is not a precise per-card BDF match. Review the detected guests and choose how to handle each:", "headerPolicy": "Policy", "headerEffect": "Effect", "headerWhen": "When to pick", "keepPolicy": "Keep config, disable onboot", "keepEffect": "pct set CTID -onboot 0 (or qm set VMID -onboot 0). GPU entries stay in the configuration.", "keepWhen": "You plan to come back to this VM/LXC once the GPU is back in its original mode. Safe default.", "removePolicy": "Remove GPU from config", "removeEffect": "Removes identified hostpci / dev entries and supported legacy device references.", "removeWhen": "Choose this if the guest should operate without the GPU. Check its application settings first: CPU fallback is application-dependent, and removing the device does not automatically reconfigure the app.", "imageAlt": "Dialog asking per-VM / per-LXC conflict policy when switching a GPU that's currently assigned" }, "audio": { "title": "Orphan audio cleanup (only when leaving VM mode)", "body1": "Many dedicated NVIDIA / AMD GPUs expose HDMI audio at function .1 of the same slot; additional audio devices may also have been assigned. Removing the GPU can leave obsolete hostpci audio entries in the VM. Review which devices should remain assigned.", "body2": "The script discovers them (precise BDF match, no substring false-positives) and shows a checklist so you can remove them cleanly. It also cleans their vendor:device IDs from /etc/modprobe.d/vfio.conf — but only if no other VM still uses those audio IDs." }, "apply": { "title": "Apply host + workload changes", "body": "Once you confirm, the script writes the host-side changes — vfio.conf, blacklist, modules, and (for NVIDIA) the per-BDF udev rule at /etc/udev/rules.d/10-proxmenux-vfio-bind.rules plus the BDF state at /etc/proxmenux/vfio-bind.bdfs. It also applies the chosen conflict policy to each affected VM/LXC. If the host config actually changed, it runs update-initramfs -u -k all followed by proxmox-boot-tool refresh; otherwise both are skipped." }, "reboot": { "title": "Reboot", "body": "The new GPU binding only takes effect after a reboot. The script prompts you; you can reboot now or later, but don't start the target VM/LXC until the host has rebooted — otherwise the GPU is still held by the previous driver.", "imageAlt": "Summary dialog listing what changed, followed by the reboot prompt" } }, "manual": { "heading": "Manual inspection", "intro": "Before a VM → LXC switch, inspect the driver and existing per-BDF policy. This NVIDIA example uses vendor:device 10de:2204; adapt it to your card. These commands inspect state; the script also manages affected guests and IOMMU-group members.", "lxcToVm": "Before LXC → VM, inspect the selected PCI address and its IOMMU group. Replace the example address with yours and review all group members before assigning devices.", "oneVmTitle": "Only one VM can use a given vfio-pci GPU at a time", "oneVmBody": "Putting multiple hostpci entries with the same PCI slot in two VMs is valid config but only one of the VMs can start with the GPU — the second one will fail. The ProxMenux conflict policy step is exactly about avoiding this trap." }, "verification": { "heading": "Verification after reboot" }, "troubleshoot": { "heading": "Troubleshooting", "stillVfioTitle": "GPU still shows vfio-pci after switching to LXC mode", "stillVfioBody": "A loaded VFIO module does not prove which driver owns this GPU. Check the card binding and whether update-initramfs and the reboot completed. lsmod | grep vfio only lists modules. For AMD / Intel inspect vfio.conf; for NVIDIA inspect /etc/proxmenux/vfio-bind.bdfs and /etc/udev/rules.d/10-proxmenux-vfio-bind.rules. Correct stale policy before running update-initramfs -u -k all and rebooting.", "vmFailTitle": "A VM won't start after switching a GPU to LXC mode", "vmFailBody": "Check whether the VM still has obsolete hostpci references. The Remove GPU from config policy can remove them during the intended switch; inspect the proposed direction before rerunning the script so you do not reverse the mode unintentionally. Alternatively, remove the obsolete assignment in Proxmox with the VM stopped.", "smiFailTitle": "nvidia-smi fails with 'Driver/library version mismatch' after going back to LXC", "smiFailBody": "Compare the loaded host NVIDIA module with the installed userspace libraries, including those in LXCs. Align versions first; a reboot loads the installed module but cannot repair mismatched library packages. Reloading with modprobe -r nvidia and modprobe nvidia requires stopped GPU clients and may fail while dependent modules are in use.", "logTitle": "Install log", "logBody": "Every run writes to /tmp/proxmenux_gpu_switch_mode.log on the host. Attach it when asking for help on GitHub." }, "related": { "heading": "Related", "items": [ { "label": "Add GPU to VM (Passthrough)", "href": "/docs/hardware/gpu-vm-passthrough", "tail": " — first-time VM mode setup." }, { "label": "Add GPU to LXC", "href": "/docs/hardware/igpu-acceleration-lxc", "tail": " — first-time LXC mode setup." }, { "label": "Install NVIDIA Drivers (Host)", "href": "/docs/hardware/nvidia-host", "tail": " — required for LXC mode on NVIDIA GPUs." }, { "label": "ProxMenux Monitor — Hardware tab", "href": "/docs/monitor/dashboard/hardware", "tail": " — the Graphics Cards section where each GPU shows its current mode and exposes the inline switch control." }, { "label": "GPU Passthrough commands", "href": "/docs/help-info/gpu-commands", "tail": " — quick command reference." } ] } }