diff --git a/.gitignore b/.gitignore
index b2be92b..726d2d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 result
+.direnv
diff --git a/dev/checks.nix b/dev/checks.nix
index a4c35ff..6baf41d 100644
--- a/dev/checks.nix
+++ b/dev/checks.nix
@@ -1,12 +1,7 @@
-{ prefix, srvos, nixpkgs, system }:
+{ prefix, srvos, pkgs, lib, system }:
 let
-  pkgs = nixpkgs.legacyPackages.${system};
-
-  inherit (nixpkgs) lib;
-
   nixosTest = import "${pkgs.path}/nixos/lib/testing-python.nix" {
-    inherit pkgs;
-    system = pkgs.system;
+    inherit pkgs system;
   };
 
   moduleTests = {
@@ -25,7 +20,7 @@ let
   };
 
   configurations = import ./test-configurations.nix {
-    inherit srvos nixpkgs system;
+    inherit srvos pkgs lib system;
   };
 
   # Add all the nixos configurations to the checks
diff --git a/dev/test-configurations.nix b/dev/test-configurations.nix
index cdf7a34..6e9bbf2 100644
--- a/dev/test-configurations.nix
+++ b/dev/test-configurations.nix
@@ -1,9 +1,8 @@
 # We use the nixosConfigurations to test all the modules below.
 #
 # This is not optimal, but it gets the job done
-{ srvos, nixpkgs, system }:
+{ srvos, pkgs, lib, system }:
 let
-  inherit (nixpkgs) lib;
   inherit (lib) nixosSystem;
 
   # some example configuration to make it eval
@@ -13,6 +12,10 @@ let
     users.users.root.initialPassword = "fnord23";
     boot.loader.grub.devices = lib.mkForce [ "/dev/sda" ];
     fileSystems."/".device = lib.mkDefault "/dev/sda";
+
+    # Don't reinstantiate nixpkgs for every nixos eval.
+    # Also important to have nixpkgs config which allows for some required insecure packages
+    nixpkgs = { inherit pkgs; };
   };
 in
 {
diff --git a/flake.nix b/flake.nix
index 3621fc0..477d535 100644
--- a/flake.nix
+++ b/flake.nix
@@ -9,10 +9,20 @@
     let
       srvos = self;
       inherit (nixpkgs) lib;
+
+      permittedInsecurePackages = [
+        "nodejs-16.20.0"
+        "nodejs-16.20.1"
+        "nodejs-16.20.2"
+      ];
+
       eachSystem = f:
         lib.genAttrs
           srvos.lib.supportedSystems
-          (system: f nixpkgs.legacyPackages.${system});
+          (system: f (import nixpkgs {
+            inherit system;
+            config = { inherit permittedInsecurePackages; };
+          }));
     in
     {
       lib.supportedSystems = [
@@ -23,13 +33,17 @@
       ];
 
       checks = eachSystem (pkgs:
-        { } // (lib.optionalAttrs (pkgs.system == "x86_64-linux") (import ./dev/checks.nix {
-          inherit srvos nixpkgs;
+        (lib.optionalAttrs (pkgs.system == "x86_64-linux") (import ./dev/checks.nix {
+          inherit srvos pkgs lib;
           prefix = "nixos";
           system = pkgs.system;
         })) // (lib.optionalAttrs (pkgs.system == "x86_64-linux") (import ./dev/checks.nix {
           inherit srvos;
-          nixpkgs = nixos-23_05;
+          pkgs = import nixos-23_05 {
+            inherit (pkgs) system;
+            config = { inherit permittedInsecurePackages; };
+          };
+          inherit (nixos-23_05) lib;
           prefix = "nixos-23_05";
           system = pkgs.system;
         })));
diff --git a/nixos/modules/github-runners/default.nix b/nixos/modules/github-runners/default.nix
index aed23db..b4b37dd 100644
--- a/nixos/modules/github-runners/default.nix
+++ b/nixos/modules/github-runners/default.nix
@@ -57,12 +57,6 @@ in
           })
         cfg);
 
-    nixpkgs.config.permittedInsecurePackages = [
-      "nodejs-16.20.0"
-      "nodejs-16.20.1"
-      "nodejs-16.20.2"
-    ];
-
     systemd.services = flip mapAttrs' cfg (n: v:
       let
         svcName = "github-runner-${n}";