diff --git a/pkgs/os-specific/linux/nvidia-x11/generic.nix b/pkgs/os-specific/linux/nvidia-x11/generic.nix
index 3b1b6f9863f8..bde8ad361dcd 100644
--- a/pkgs/os-specific/linux/nvidia-x11/generic.nix
+++ b/pkgs/os-specific/linux/nvidia-x11/generic.nix
@@ -62,7 +62,8 @@ let
 
     libPath = makeLibraryPath [ xorg.libXext xorg.libX11 xorg.libXv xorg.libXrandr zlib stdenv.cc.cc ];
 
-    nativeBuildInputs = [ perl nukeReferences ] ++ kernel.moduleBuildDependencies;
+    nativeBuildInputs = [ perl nukeReferences ]
+      ++ optionals (!libsOnly) kernel.moduleBuildDependencies;
 
     disallowedReferences = optional (!libsOnly) [ kernel.dev ];