From a0b64f928aa6ab5512dee4690935bfe291e33eff Mon Sep 17 00:00:00 2001 From: shaq918 Date: Tue, 14 Jul 2026 13:50:36 -0700 Subject: [PATCH] Handle gdrdrv kernel module in _unload_driver() The _unload_driver() function accounts for nvidia_modeset, nvidia_uvm, and nvidia_peermem when computing nvidia_deps, but does not account for gdrdrv (loaded by the nvidia-gdrcopy-ctr sidecar container). When RDMA is enabled in the GPU Operator ClusterPolicy, the driver pod runs three containers: nvidia-driver-ctr, nvidia-peermem-ctr, and nvidia-gdrcopy-ctr. The gdrcopy container loads the gdrdrv kernel module, which holds a reference on the nvidia module. If the driver container restarts (e.g. due to upgrade, pod eviction, or node reboot), it calls _unload_driver() while gdrdrv is still loaded. The nvidia module refcount (4) exceeds the expected nvidia_deps count (3), causing the unload to fail with "Could not unload NVIDIA driver kernel modules, driver is in use". The driver container then enters CrashLoopBackOff. This adds gdrdrv to the set of known nvidia-dependent modules, matching the existing pattern used for nvidia_modeset, nvidia_uvm, and nvidia_peermem. When gdrdrv is present, it is added to rmmod_args and nvidia_deps is incremented so the refcount check passes correctly. Signed-off-by: Shashank Mohankumar --- rhel10/nvidia-driver | 4 ++++ rhel10/precompiled/nvidia-driver | 4 ++++ rhel8/nvidia-driver | 4 ++++ rhel8/precompiled/nvidia-driver | 4 ++++ rhel9/nvidia-driver | 4 ++++ rhel9/precompiled/nvidia-driver | 4 ++++ ubuntu22.04/nvidia-driver | 4 ++++ ubuntu22.04/precompiled/nvidia-driver | 4 ++++ ubuntu24.04/nvidia-driver | 4 ++++ ubuntu24.04/precompiled/nvidia-driver | 4 ++++ ubuntu26.04/precompiled/nvidia-driver | 4 ++++ 11 files changed, 44 insertions(+) diff --git a/rhel10/nvidia-driver b/rhel10/nvidia-driver index 13994cd3b..8a3b53413 100755 --- a/rhel10/nvidia-driver +++ b/rhel10/nvidia-driver @@ -531,6 +531,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/rhel10/precompiled/nvidia-driver b/rhel10/precompiled/nvidia-driver index 1c82533d2..0d1fc5b09 100755 --- a/rhel10/precompiled/nvidia-driver +++ b/rhel10/precompiled/nvidia-driver @@ -344,6 +344,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/rhel8/nvidia-driver b/rhel8/nvidia-driver index 58753091b..033376426 100755 --- a/rhel8/nvidia-driver +++ b/rhel8/nvidia-driver @@ -508,6 +508,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/rhel8/precompiled/nvidia-driver b/rhel8/precompiled/nvidia-driver index de85549bc..c94c7819a 100755 --- a/rhel8/precompiled/nvidia-driver +++ b/rhel8/precompiled/nvidia-driver @@ -281,6 +281,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/rhel9/nvidia-driver b/rhel9/nvidia-driver index faaa2181a..e6de1258c 100755 --- a/rhel9/nvidia-driver +++ b/rhel9/nvidia-driver @@ -527,6 +527,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/rhel9/precompiled/nvidia-driver b/rhel9/precompiled/nvidia-driver index 4e18e22e0..90dc84a41 100755 --- a/rhel9/precompiled/nvidia-driver +++ b/rhel9/precompiled/nvidia-driver @@ -370,6 +370,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ ${nvidia_refs} -gt ${nvidia_deps} ] || [ ${nvidia_uvm_refs} -gt 0 ] || [ ${nvidia_modeset_refs} -gt 0 ] || [ ${nvidia_peermem_refs} -gt 0 ]; then echo "Could not unload NVIDIA driver kernel modules, driver is in use" >&2 return 1 diff --git a/ubuntu22.04/nvidia-driver b/ubuntu22.04/nvidia-driver index 283dbbb68..025001af6 100755 --- a/ubuntu22.04/nvidia-driver +++ b/ubuntu22.04/nvidia-driver @@ -482,6 +482,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ -f /sys/module/nvidia/refcnt ]; then nvidia_refs=$(< /sys/module/nvidia/refcnt) rmmod_args+=("nvidia") diff --git a/ubuntu22.04/precompiled/nvidia-driver b/ubuntu22.04/precompiled/nvidia-driver index bb63653a3..80b481587 100755 --- a/ubuntu22.04/precompiled/nvidia-driver +++ b/ubuntu22.04/precompiled/nvidia-driver @@ -360,6 +360,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ -f /sys/module/nvidia/refcnt ]; then nvidia_refs=$(< /sys/module/nvidia/refcnt) rmmod_args+=("nvidia") diff --git a/ubuntu24.04/nvidia-driver b/ubuntu24.04/nvidia-driver index f3e0fdb66..a76d25890 100755 --- a/ubuntu24.04/nvidia-driver +++ b/ubuntu24.04/nvidia-driver @@ -423,6 +423,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ -f /sys/module/nvidia/refcnt ]; then nvidia_refs=$(< /sys/module/nvidia/refcnt) rmmod_args+=("nvidia") diff --git a/ubuntu24.04/precompiled/nvidia-driver b/ubuntu24.04/precompiled/nvidia-driver index 6a5cb5804..898b415ef 100755 --- a/ubuntu24.04/precompiled/nvidia-driver +++ b/ubuntu24.04/precompiled/nvidia-driver @@ -361,6 +361,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ -f /sys/module/nvidia/refcnt ]; then nvidia_refs=$(< /sys/module/nvidia/refcnt) rmmod_args+=("nvidia") diff --git a/ubuntu26.04/precompiled/nvidia-driver b/ubuntu26.04/precompiled/nvidia-driver index 54a926aed..6f458b900 100755 --- a/ubuntu26.04/precompiled/nvidia-driver +++ b/ubuntu26.04/precompiled/nvidia-driver @@ -331,6 +331,10 @@ _unload_driver() { rmmod_args+=("nvidia-peermem") ((++nvidia_deps)) fi + if [ -f /sys/module/gdrdrv/refcnt ]; then + rmmod_args+=("gdrdrv") + ((++nvidia_deps)) + fi if [ -f /sys/module/nvidia/refcnt ]; then nvidia_refs=$(< /sys/module/nvidia/refcnt) rmmod_args+=("nvidia")