diff --git a/pkgs/os-specific/linux/kernel/common-config.nix b/pkgs/os-specific/linux/kernel/common-config.nix
index 0553f0eef934..a48d2d466e22 100644
--- a/pkgs/os-specific/linux/kernel/common-config.nix
+++ b/pkgs/os-specific/linux/kernel/common-config.nix
@@ -392,8 +392,8 @@ let
       FRAMEBUFFER_CONSOLE_ROTATION = yes;
       FRAMEBUFFER_CONSOLE_DETECT_PRIMARY = yes;
       FB_GEODE            = mkIf (stdenv.hostPlatform.system == "i686-linux") yes;
-      # On 5.14 this conflicts with FB_SIMPLE.
-      DRM_SIMPLEDRM = whenAtLeast "5.14" no;
+      # Use simplefb on older kernels where we don't have simpledrm (enabled below)
+      FB_SIMPLE           = whenOlder "5.15" yes;
       DRM_FBDEV_EMULATION = yes;
     };
 
@@ -409,10 +409,19 @@ let
     video = let
       whenHasDevicePrivate = mkIf (!stdenv.isx86_32 && versionAtLeast version "5.1");
     in {
+      # compile in DRM so simpledrm can load before initrd if necessary
+      AGP = yes;
+      DRM = yes;
+
       DRM_LEGACY = whenOlder "6.8" no;
 
       NOUVEAU_LEGACY_CTX_SUPPORT = whenBetween "5.2" "6.3" no;
 
+      # Enable simpledrm and use it for generic framebuffer
+      # Technically added in 5.14, but adding more complex configuration is not worth it
+      DRM_SIMPLEDRM = whenAtLeast "5.15" yes;
+      SYSFB_SIMPLEFB = whenAtLeast "5.15" yes;
+
       # Allow specifying custom EDID on the kernel command line
       DRM_LOAD_EDID_FIRMWARE = yes;
       VGA_SWITCHEROO         = yes; # Hybrid graphics support
@@ -1178,11 +1187,6 @@ let
       CMA_DEBUGFS = yes;
       CMA_SYSFS = yes;
 
-      # Many ARM SBCs hand off a pre-configured framebuffer.
-      # This always can can be replaced by the actual native driver.
-      # Keeping it a built-in ensures it will be used if possible.
-      FB_SIMPLE = yes;
-
       # https://docs.kernel.org/arch/arm/mem_alignment.html
       # tldr:
       #  when buggy userspace code emits illegal misaligned LDM, STM,