feat: module system improvements

- add documentation to the subsystem-loading interface
- expose evaled modules publicly via `framework`
- add option config.modules to allow users to include their own modules
- update examples for extending dream2nix

docs/src/extending-dream2nix.md

@@ -3,6 +3,11 @@
 `dream2nix` can be extended while you are `init`ializing it. This can be done in a few ways.
 For extending, you need to utilize the `config.extra` option of the dream2nix config.
 
+## Module System Transition
+The dream2nix architecture is currently in the process of being refactored into nixos modules. The actual APIs can deviate from what is described in this document.
+For up-to-date examples see `/examples/_d2n-extended` and `/examples/_d2n-extended-new-subsystem`
+Once the transition is completed we will update this document and remove this notice.
+
 ## Declare `extra`s from a nix file
 
 ```nix

examples/_d2n-extended-new-subsystem/extra.nix

@@ -11,28 +11,6 @@
         })
       ];
     };
-    translators.dummy = {...}: {
-      type = "pure";
-      translate = {hello, ...}: {...}: {
-        result = {
-          _generic = {
-            subsystem = "hello";
-            defaultPackage = "hello";
-            location = "";
-            sourcesAggregatedHash = null;
-            packages = {${hello.pname} = hello.version;};
-          };
-          _subsystem = {};
-          cyclicDependencies = {};
-          dependencies.${hello.pname}.${hello.version} = [];
-          sources.${hello.pname}.${hello.version} = {
-            type = "http";
-            url = hello.src.url;
-            hash = hello.src.outputHash;
-          };
-        };
-      };
-    };
     builders.dummy = {...}: {
       type = "pure";
       build = {hello, ...}: {...}: {

examples/_d2n-extended-new-subsystem/flake.nix

@@ -11,6 +11,9 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       config.extra = ./extra.nix;
+      config.modules = [
+        ./translators.nix
+      ];
       source = ./.;
       settings = [
         {

examples/_d2n-extended-new-subsystem/translators.nix

@@ -0,0 +1,26 @@
+{
+  translators.dummy = {...}: {
+    type = "pure";
+    name = "dummy";
+    subsystem = "hello";
+    translate = {hello, ...}: {...}: {
+      result = {
+        _generic = {
+          subsystem = "hello";
+          defaultPackage = "hello";
+          location = "";
+          sourcesAggregatedHash = null;
+          packages = {${hello.pname} = hello.version;};
+        };
+        _subsystem = {};
+        cyclicDependencies = {};
+        dependencies.${hello.pname}.${hello.version} = [];
+        sources.${hello.pname}.${hello.version} = {
+          type = "http";
+          url = hello.src.url;
+          hash = hello.src.outputHash;
+        };
+      };
+    };
+  };
+}

examples/_d2n-extended/flake.nix

@@ -13,10 +13,20 @@
     (dream2nix.lib.makeFlakeOutputs {
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
+      config.modules = [
+        (builtins.toFile "cargo-toml-new.nix" ''
+          {
+            translators.cargo-toml-new = {
+              imports = ["${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml"];
+              name = "cargo-toml-new";
+              subsystem = "rust";
+            };
+          }
+        '')
+      ];
       config.extra = {
         subsystems.rust = {
           builders.brp-new = "${inp.dream2nix}/src/subsystems/rust/builders/build-rust-package";
-          translators.cargo-toml-new = "${inp.dream2nix}/src/subsystems/rust/translators/cargo-toml";
           discoverers.default = "${inp.dream2nix}/src/subsystems/rust/discoverers/default";
         };
         fetchers.crates-io = "${inp.dream2nix}/src/fetchers/crates-io";

src/default.nix

@@ -49,17 +49,22 @@ in let
 
   configFile = pkgs.writeText "dream2nix-config.json" (b.toJSON config);
 
+  # pass spacialArgs itself in specialArgs in order to forward it to submodules.
+  specialModuleArgs = l.fix (self: {
+    specialArgs = self;
+    inherit
+      callPackageDream
+      dlib
+      ;
+  });
+
   evaledModules = lib.evalModules {
-    modules = [
+    modules =
-      ./modules/top-level.nix
+      [./modules/top-level.nix]
-    ];
+      ++ (config.modules or []);
+
     # TODO: remove specialArgs once all functionality is moved to /src/modules
-    specialArgs = {
+    specialArgs = specialModuleArgs;
-      inherit
-        callPackageDream
-        dlib
-        ;
-    };
   };
 
   framework = evaledModules.config;
@@ -709,6 +714,7 @@ in {
     callPackageDream
     dream2nixWithExternals
     fetchers
+    framework
     indexers
     fetchSources
     realizeProjects

src/modules/functions.subsystem-loading/implementation.nix

@@ -17,10 +17,6 @@ modules:
   subsystemsDir = lib.toString ../../subsystems;
   subsystems = dlib.dirNames subsystemsDir;
 
-  /*
-  Discover modules in:
-    /src/subsystems/{subsystem}/{module-type}/{module-name}
-  */
   collect = moduleDirName:
     lib.concatMap
     (subsystem: let
@@ -43,10 +39,6 @@ modules:
         names)
     subsystems;
 
-  /*
-  Imports discovered module files.
-  Adds name and subsystem attributes to each module derived from the path.
-  */
   import_ = collectedModules:
     lib.mapAttrs
     (name: description:
@@ -54,12 +46,6 @@ modules:
       // {inherit (description) name subsystem;})
     (lib.listToAttrs collectedModules);
 
-  /*
-  To keep module implementations simpler, additional generic logic is added
-  by a loader.
-
-  The loader is subsytem specific and needs to be passed as an argument.
-  */
   instantiate = importedModules: loader:
     lib.mapAttrs
     (name: module:
@@ -67,10 +53,6 @@ modules:
       // {inherit (module) name subsystem;})
     importedModules;
 
-  /*
-  re-structures the instantiated instances into a deeper attrset like:
-    {subsytem}.{module-name} = ...
-  */
   structureBySubsystem = instances:
     lib.foldl
     lib.recursiveUpdate

src/modules/functions.subsystem-loading/interface.nix

@@ -3,9 +3,33 @@
   t = lib.types;
 in {
   options.functions.subsystem-loading = {
-    collect = lib.mkOption {type = t.functionTo t.anything;};
+    collect = lib.mkOption {
-    import_ = lib.mkOption {type = t.functionTo t.anything;};
+      type = t.functionTo t.anything;
-    instantiate = lib.mkOption {type = t.functionTo t.anything;};
+      description = ''
-    structureBySubsystem = lib.mkOption {type = t.functionTo t.anything;};
+        Discover modules in /src/subsystems/{subsystem}/{module-type}/{module-name}
+      '';
+    };
+    import_ = lib.mkOption {
+      type = t.functionTo t.anything;
+      description = ''
+        Imports discovered module files.
+        Adds name and subsystem attributes to each module derived from the path.
+      '';
+    };
+    instantiate = lib.mkOption {
+      type = t.functionTo t.anything;
+      description = ''
+        To keep module implementations simpler, additional generic logic is added
+        by a loader.
+        The loader is subsytem specific and needs to be passed as an argument.
+      '';
+    };
+    structureBySubsystem = lib.mkOption {
+      type = t.functionTo t.anything;
+      description = ''
+        re-structures the instantiated instances into a deeper attrset like:
+        {subsytem}.{module-name} = ...
+      '';
+    };
   };
 }

src/modules/translators/interface-translator.nix

@@ -44,6 +44,7 @@ in {
       default = null;
     };
     extraArgs = lib.mkOption {
+      default = {};
       type = t.attrsOf (t.submodule {
         options = {
           description = lib.mkOption {

src/modules/translators/interface.nix

@@ -1,13 +1,17 @@
 {
   config,
   lib,
+  specialArgs,
   ...
 }: let
   t = lib.types;
 in {
   options = {
     translators = lib.mkOption {
-      type = t.attrsOf (t.submodule ./interface-translator.nix);
+      type = t.attrsOf (t.submoduleWith {
+        modules = [./interface-translator.nix];
+        inherit specialArgs;
+      });
       description = ''
         Translator module definitions
       '';

tests/examples/default.nix

@@ -29,7 +29,7 @@
       cp -r ${examples}/$dir/* .
       chmod -R +w .
       nix flake lock --override-input dream2nix ${../../.}
-      nix run .#resolveImpure
+      nix run .#resolveImpure --show-trace
       # disable --read-only check for these because they do IFD so they will
       # write to store at eval time
       evalBlockList=("haskell_cabal-plan" "haskell_stack-lock")