Merge pull request #415 from nix-community/no-auto

Disable automatic project discovery in makeFlakeOutputs

README.md

@@ -72,6 +72,7 @@ Minimal Example `flake.nix`:
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = ./.;
+      projects = ./projects.toml;
     };
 }
 ```
@@ -87,6 +88,10 @@ Extensive Example `flake.nix`:
 
       source = ./.;
 
+      # `projects` can alternatively be an attrset.
+      # `projects` can be omitted if `autoProjects = true` is defined.
+      projects = ./projects.toml;
+
       # Configure the behavior of dream2nix when translating projects.
       # A setting applies to all discovered projects if `filter` is unset,
       # or just to a subset or projects if `filter` is used.

docs/src/define-targets.md

@@ -32,6 +32,7 @@ Alternatively, we can define the targets in the `flake.nix` like so:
       systems = ["x86_64-linux"];         # <- This line.
       config.projectRoot = ./.;
       source = ./.;
+      projects = ./projects.toml;
     };
 }
 ```

docs/src/guides/getting-started-nodejs.md

@@ -60,6 +60,7 @@ to create a `flake.nix`:
       systemsFromFile = ./nix_systems;
       config.projectRoot = ./.;
       source = ./.;
+      projects = ./projects.toml;
     };
 }
 ```
@@ -87,12 +88,11 @@ git+file:///tmp/my_project
 │   └───x86_64-linux
 │       ├───cowsay: development environment 'nix-shell'
 │       └───default: development environment 'nix-shell'
-├───packages
-│   └───x86_64-linux
-│       ├───cowsay: package 'cowsay-1.5.0'
-│       ├───default: package 'cowsay-1.5.0'
-│       └───resolveImpure: package 'resolve'
-└───projectsJson: unknown
+└───packages
+    └───x86_64-linux
+        ├───cowsay: package 'cowsay-1.5.0'
+        ├───default: package 'cowsay-1.5.0'
+        └───resolveImpure: package 'resolve'
 ```
 
 We can see that:
@@ -207,12 +207,11 @@ git+file:///tmp/my_project
 │   └───x86_64-linux
 │       ├───cowsay: development environment 'nix-shell'
 │       └───default: development environment 'nix-shell'
-├───packages
-│   └───x86_64-linux
-│       ├───cowsay: package 'cowsay-1.5.0'
-│       ├───default: package 'cowsay-1.5.0'
-│       └───resolveImpure: package 'resolve'
-└───projectsJson: unknown
+└───packages
+    └───x86_64-linux
+        ├───cowsay: package 'cowsay-1.5.0'
+        ├───default: package 'cowsay-1.5.0'
+        └───resolveImpure: package 'resolve'
 ```
 When we enter the `cowsay` development shell, we will get `node` in our
 `PATH`, together with all the binaries from our dependencies packages.

docs/src/guides/getting-started-python.md

@@ -42,11 +42,10 @@ warning: Git tree '/tmp/my_project' is dirty
 warning: creating lock file '/tmp/my_project/flake.lock'
 warning: Git tree '/tmp/my_project' is dirty
 git+file:///tmp/my_project
-├───packages
-│   └───x86_64-linux
-│       ├───main: package 'main'
-│       └───resolveImpure: package 'resolve'
-└───projectsJson: unknown
+└───packages
+    └───x86_64-linux
+        ├───main: package 'main'
+        └───resolveImpure: package 'resolve'
 ```
 
 What we can observe here:
@@ -57,11 +56,10 @@ Our flake.nix imported external libraries. The versions of these libraries have
 1.
     ```
       git+file:///tmp/my_project
-      ├───packages
-      │   └───x86_64-linux
-      │       ├───main: package 'main'
-      │       └───resolveImpure: package 'resolve'
-      └───projectsJson: unknown
+      └───packages
+          └───x86_64-linux
+              ├───main: package 'main'
+              └───resolveImpure: package 'resolve'
     ```
     Similar like a .json file defines a structure of data, our flake.nix defines a structure of `nix attributes` which are things that we can build or run with nix.
     We can see that it contains packages for my current platform `x86_64-linux`.

examples/_d2n-auto-project-detection/flake.nix

@@ -0,0 +1,22 @@
+{
+  inputs = {
+    dream2nix.url = "github:nix-community/dream2nix";
+    src.url = "github:yusdacra/linemd/v0.4.0";
+    src.flake = false;
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+    src,
+  } @ inp:
+    (dream2nix.lib.makeFlakeOutputs {
+      systems = ["x86_64-linux"];
+      config.projectRoot = ./.;
+      source = src;
+      autoProjects = true;
+    })
+    // {
+      # checks.x86_64-linux.linemd = self.packages.x86_64-linux.linemd;
+    };
+}

examples/_d2n-extend-devShell/flake.nix

@@ -24,6 +24,7 @@
       inherit systems;
       config.projectRoot = ./.;
       source = src;
+      projects = ./projects.toml;
     };
   in
     dream2nix.lib.dlib.mergeFlakes [

examples/_d2n-extend-devShell/projects.toml

@@ -0,0 +1,9 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[prettier]
+name = "prettier"
+relPath = ""
+subsystem = "nodejs"
+translator = "yarn-lock"

examples/_d2n-extended-new-subsystem/flake.nix

@@ -16,12 +16,8 @@
         ./builders.nix
       ];
       source = ./.;
-      settings = [
-        {
-          builder = "dummy";
-          translator = "dummy";
-        }
-      ];
+      # The dummy discoverer will discover a project `hello` automatically.
+      autoProjects = true;
     })
     // {
       checks.x86_64-linux.hello = self.packages.x86_64-linux.hello;

examples/_d2n-extended/flake.nix

@@ -50,12 +50,7 @@
         '')
       ];
       source = src;
-      settings = [
-        {
-          builder = "brp-new";
-          translator = "cargo-toml-new";
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks.x86_64-linux.linemd = self.packages.x86_64-linux.linemd;

examples/_d2n-extended/projects.toml

@@ -0,0 +1,16 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[linemd]
+name = "linemd"
+relPath = ""
+subsystem = "rust"
+translator = "cargo-lock"
+translators = [ "cargo-lock", "cargo-toml",]
+
+[[linemd.subsystemInfo.crates]]
+name = "linemd"
+relPath = ""
+version = "0.4.0"
+

examples/_d2n-init-pkgs/flake.nix

@@ -18,11 +18,6 @@
       pkgs = allPkgs;
       config.projectRoot = ./.;
       source = inp.src;
-      settings = [
-        {
-          builder = "build-rust-package";
-          translator = "cargo-lock";
-        }
-      ];
+      projects = ./projects.toml;
     };
 }

examples/_d2n-init-pkgs/projects.toml

@@ -0,0 +1,16 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[linemd]
+name = "linemd"
+relPath = ""
+subsystem = "rust"
+translator = "cargo-lock"
+builder = "build-rust-package"
+
+[[linemd.subsystemInfo.crates]]
+name = "linemd"
+relPath = ""
+version = "0.4.0"
+

examples/_d2n-projects-as-attrs/flake.nix

@@ -0,0 +1,28 @@
+{
+  inputs = {
+    dream2nix.url = "github:nix-community/dream2nix";
+    src.url = "github:yusdacra/linemd/v0.4.0";
+    src.flake = false;
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+    src,
+  } @ inp:
+    (dream2nix.lib.makeFlakeOutputs {
+      systems = ["x86_64-linux"];
+      config.projectRoot = ./.;
+      source = src;
+      projects = {
+        linemd = {
+          name = "linemd";
+          subsystem = "rust";
+          translator = "cargo-lock";
+        };
+      };
+    })
+    // {
+      # checks.x86_64-linux.linemd = self.packages.x86_64-linux.linemd;
+    };
+}

examples/haskell_cabal-freeze/flake.nix

@@ -13,10 +13,6 @@
     systems = ["x86_64-linux"];
     config.projectRoot = ./.;
     source = src;
-    settings = [
-      {
-        translator = "cabal-freeze";
-      }
-    ];
+    projects = ./projects.toml;
   });
 }

examples/haskell_cabal-freeze/projects.toml

@@ -0,0 +1,9 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[main]
+name = "main"
+relPath = ""
+subsystem = "haskell"
+translator = "cabal-freeze"

examples/haskell_cabal-plan/flake.nix

@@ -14,11 +14,7 @@
       pkgs = dream2nix.inputs.nixpkgs.legacyPackages.x86_64-linux;
       source = src;
       config.projectRoot = ./.;
-      settings = [
-        {
-          translator = "cabal-plan";
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks.x86_64-linux.cabal2json = self.packages.x86_64-linux.cabal2json.overrideAttrs (old: {

examples/haskell_cabal-plan/projects.toml

@@ -0,0 +1,9 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[main]
+name = "main"
+relPath = ""
+subsystem = "haskell"
+translator = "cabal-plan"

examples/haskell_stack-lock/flake.nix

@@ -14,15 +14,7 @@
       pkgs = dream2nix.inputs.nixpkgs.legacyPackages.x86_64-linux;
       source = src;
       config.projectRoot = ./.;
-      settings = [
-        {
-          # Optionally, override the compiler version
-          subsystemInfo.compiler = {
-            name = "ghc";
-            version = "8.10.7";
-          };
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks = self.packages;

examples/haskell_stack-lock/projects.toml

@@ -0,0 +1,14 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[main]
+name = "main"
+relPath = ""
+subsystem = "haskell"
+translator = "stack-lock"
+translators = [ "stack-lock",]
+
+[[main.subsystemInfo.compiler]]
+name = "ghc"
+version = "8.10.7"

examples/nodejs_eslint-aggreagted/flake.nix

@@ -14,12 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          subsystemInfo.noDev = true;
-          aggregate = true;
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       checks = self.packages;

examples/nodejs_eslint-aggreagted/projects.toml

@@ -0,0 +1,15 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[eslint]
+name = "eslint"
+relPath = ""
+subsystem = "nodejs"
+translator = "package-json"
+translators = [ "package-json",]
+
+[eslint.subsystemInfo]
+noDev = true
+noedjs = 18
+aggregated = true

examples/nodejs_eslint/flake.nix

@@ -14,12 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          subsystemInfo.noDev = true;
-          subsystemInfo.nodejs = 18;
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks = self.packages;

examples/nodejs_eslint/projects.toml

@@ -0,0 +1,14 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[eslint]
+name = "eslint"
+relPath = ""
+subsystem = "nodejs"
+translator = "package-json"
+translators = [ "package-json",]
+
+[eslint.subsystemInfo]
+noDev = true
+noedjs = 18

examples/nodejs_prettier/flake.nix

@@ -14,6 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
+      projects = ./projects.toml;
     })
     // {
       checks.x86_64-linux.prettier = self.packages.x86_64-linux.prettier;

examples/nodejs_prettier/projects.toml

@@ -0,0 +1,9 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[prettier]
+name = "prettier"
+relPath = ""
+subsystem = "nodejs"
+translator = "yarn-lock"

examples/nodejs_workspace/flake.nix

@@ -11,8 +11,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = ./.;
-      settings = [
-      ];
+      autoProjects = true;
     })
     // {
       checks = self.packages;

examples/php_composer/flake.nix

@@ -14,7 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [];
+      projects = ./projects.toml;
     })
     // {
       # checks = self.packages;

examples/php_composer/projects.toml

@@ -0,0 +1,12 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+["gipetto/cowsay"]
+name = "gipetto/cowsay"
+relPath = ""
+subsystem = "php"
+translator = "composer-lock"
+translators = [ "composer-lock", "composer-json",]
+
+["gipetto/cowsay".subsystemInfo]

examples/python_pip/flake.nix

@@ -14,14 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          # optionally define python version
-          subsystemInfo.pythonVersion = "3.8";
-          # optionally define extra setup requirements;
-          subsystemInfo.extraSetupDeps = ["cython > 0.29"];
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       checks.x86_64-linux.aiohttp = self.packages.x86_64-linux.main;

examples/python_pip/projects.toml

@@ -0,0 +1,21 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[llhttp]
+name = "llhttp"
+relPath = "vendor/llhttp"
+subsystem = "nodejs"
+translator = "package-lock"
+
+[main]
+name = "main"
+relPath = ""
+subsystem = "python"
+translator = "pip"
+
+[llhttp.subsystemInfo]
+
+[main.subsystemInfo]
+pythonAttr = "python3"
+pythonVersion = "3.8"

examples/racket/flake.nix

@@ -14,5 +14,6 @@
     systems = ["x86_64-linux"];
     config.projectRoot = ./.;
     source = goblins;
+    projects = ./projects.toml;
   });
 }

examples/racket/projects.toml

@@ -0,0 +1,11 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[goblins]
+name = "goblins"
+relPath = "goblins"
+subsystem = "racket"
+translator = "racket-impure"
+
+[goblins.subsystemInfo]

examples/rust_no-cargo-lock/flake.nix

@@ -14,12 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          builder = "crane";
-          translator = "cargo-toml";
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks.x86_64-linux.linemd = self.packages.x86_64-linux.linemd;

examples/rust_no-cargo-lock/projects.toml

@@ -0,0 +1,17 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[linemd]
+name = "linemd"
+relPath = ""
+subsystem = "rust"
+translator = "cargo-toml"
+builder = "crane"
+
+
+[[linemd.subsystemInfo.crates]]
+name = "linemd"
+relPath = ""
+version = "0.4.0"
+

examples/rust_ripgrep/flake.nix

@@ -14,12 +14,7 @@
       systems = ["x86_64-linux"];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          builder = "crane";
-          translator = "cargo-lock";
-        }
-      ];
+      projects = ./projects.toml;
     })
     // {
       # checks.x86_64-linux.ripgrep = self.packages.x86_64-linux.ripgrep;

examples/rust_ripgrep/projects.toml

@@ -0,0 +1,64 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[ripgrep]
+name = "ripgrep"
+relPath = ""
+subsystem = "rust"
+translator = "cargo-lock"
+translators = [ "cargo-lock", "cargo-toml",]
+builder = "crane"
+
+[ripgrep.subsystemInfo]
+workspaceMembers = [ "crates/globset", "crates/grep", "crates/cli", "crates/matcher", "crates/pcre2", "crates/printer", "crates/regex", "crates/searcher", "crates/ignore",]
+[[ripgrep.subsystemInfo.crates]]
+name = "ripgrep"
+relPath = ""
+version = "13.0.0"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-cli"
+relPath = "crates/cli"
+version = "0.1.6"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "globset"
+relPath = "crates/globset"
+version = "0.4.7"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep"
+relPath = "crates/grep"
+version = "0.2.8"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "ignore"
+relPath = "crates/ignore"
+version = "0.4.18"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-matcher"
+relPath = "crates/matcher"
+version = "0.1.5"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-pcre2"
+relPath = "crates/pcre2"
+version = "0.1.5"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-printer"
+relPath = "crates/printer"
+version = "0.1.6"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-regex"
+relPath = "crates/regex"
+version = "0.1.9"
+
+[[ripgrep.subsystemInfo.crates]]
+name = "grep-searcher"
+relPath = "crates/searcher"
+version = "0.1.8"
+

examples/rust_set-rust-toolchain/flake.nix

@@ -23,12 +23,7 @@
       systems = [system];
       config.projectRoot = ./.;
       source = src;
-      settings = [
-        {
-          builder = "crane";
-          translator = "cargo-lock";
-        }
-      ];
+      projects = ./projects.toml;
       packageOverrides = {
         # override all packages and set a toolchain
         "^.*" = {

examples/rust_set-rust-toolchain/projects.toml

@@ -0,0 +1,15 @@
+# To re-generate this file, run:
+#   nix run github:nix-community/dream2nix#detect-projects $source
+# ... where `$source` points to the source of your project.
+
+[linemd]
+name = "linemd"
+relPath = ""
+subsystem = "rust"
+translator = "cargo-lock"
+
+[[linemd.subsystemInfo.crates]]
+name = "linemd"
+relPath = ""
+version = "0.4.0"
+

src/apps/detect-projects/default.nix

@@ -0,0 +1,23 @@
+{
+  pkgs,
+  utils,
+  apps,
+  ...
+}:
+utils.writePureShellScriptBin
+"detect-projects"
+(with pkgs; [
+  coreutils
+  yq
+])
+''
+  set -e
+
+  # Use $1 as source if set. Otherwise default to ./.
+  export source=$(realpath ''${1:-.})
+
+  ${apps.callNixWithD2N} eval --json \
+    "dream2nix.functions.discoverers.discoverProjects2 {source = builtins.getEnv \"source\";}" \
+    | yq --toml-output \
+    | (cat ${./template.toml} && echo "" && cat)
+''

src/apps/detect-projects/template.toml

@@ -0,0 +1,9 @@
+# projects.toml file describing inputs for dream2nix
+#
+# To re-generate this file, run:
+#   nix run .#detect-projects $source
+# ... where `$source` points to the source of your project.
+#
+# If the local flake is unavailable, alternatively execute the app from the
+# upstream dream2nix flake:
+#   nix run github:nix-community/dream2nix#detect-projects $source

src/lib.nix

@@ -71,6 +71,24 @@
   }:
     initDream2nix (loadConfig config) pkgs;
 
+  missingProjectsError = source: ''
+    Please pass `projects` to makeFlakeOutputs.
+    `projects` can be:
+      - an attrset
+      - a path to a .toml file (not empty & added to git)
+      - a path to a .json file (not empty & added to git)
+
+    To generate a projects.toml file automatically:
+      1. execute:
+        nix run github:nix-community/dream2nix#detect-projects ${source} > projects.toml
+      2. review the ./projects.toml and edit it if necessary.
+      3. pass `projects = ./projects.toml` to makeFlakeOutputs.
+
+    Alternatively pass `autoProjects = true` to makeFlakeOutputs.
+    This is not recommended as it doesn't allow you to review or filter the list
+      of detected projects.
+  '';
+
   makeFlakeOutputs = {
     source,
     pkgs ? null,
@@ -81,6 +99,7 @@
     pname ? throw "Please pass `pname` to makeFlakeOutputs",
     packageOverrides ? {},
     projects ? {},
+    autoProjects ? false,
     settings ? [],
     sourceOverrides ? oldSources: {},
   } @ args: let
@@ -101,6 +120,21 @@
           file = systemsFromFile;
         };
 
+    # if projects provided via `.json` or `.toml` file, parse to attrset
+    projects = let
+      givenProjects = args.projects or {};
+    in
+      if autoProjects && args ? projects
+      then throw "Don't pass `projects` to makeFlakeOutputs when `autoProjects = true`"
+      else if l.isPath givenProjects
+      then
+        if ! l.pathExists givenProjects
+        then throw (missingProjectsError source)
+        else if l.hasSuffix ".toml" (l.toString givenProjects)
+        then l.fromTOML (l.readFile givenProjects)
+        else l.fromJSON (l.readFile givenProjects)
+      else givenProjects;
+
     allPkgs = makeNixpkgs pkgs systems;
 
     initD2N = initDream2nix config;
@@ -128,7 +162,9 @@
               proj
               (l.head projectsList);
           })
-      else discoveredProjects;
+      else if autoProjects == true
+      then discoveredProjects
+      else throw (missingProjectsError source);
 
     allBuilderOutputs =
       l.mapAttrs
@@ -146,7 +182,12 @@
             ;
         };
       in
-        allOutputs)
+        framework.dlib.recursiveUpdateUntilDrv
+        allOutputs
+        {
+          apps.detect-projects =
+            dream2nixFor.${system}.flakeApps.detect-projects;
+        })
       allPkgs;
 
     flakifiedOutputsList =
@@ -159,12 +200,8 @@
       (allOutputs: output: lib.recursiveUpdate allOutputs output)
       {}
       flakifiedOutputsList;
-
-    flakeOutputs =
-      {projectsJson = l.toJSON finalProjects;}
-      // flakeOutputsBuilders;
   in
-    flakeOutputs;
+    flakeOutputsBuilders;
 
   makeFlakeOutputsForIndexes = {
     systems ? [],

src/modules/apps/implementation.nix

@@ -19,10 +19,11 @@ in {
       {
         inherit
           (config.apps)
-          install
-          translate
-          runNixCmdInSrc
+          detect-projects
           index
+          install
+          runNixCmdInSrc
+          translate
           translate-index
           ;
       };

src/modules/functions.discoverers/implementation.nix

@@ -2,6 +2,68 @@
   l = config.lib // builtins;
   dlib = config.dlib;
 
+  /*
+  Stripped down discoverProjects without settings merging and dreamLockPath
+    represented as an attrset:
+    {
+      ${project-name} = {
+        relPath = ...;
+        subsystem = ...;
+        subsystemInfo = {
+          ...
+        };
+        translator = "some-translator";
+
+        # all compatible translators
+        translators = [
+          "some-translator"
+          "some-alternative-translator"
+        ];
+      }
+    }
+  */
+  discoverProjects2 = {
+    source ? throw "Pass either `source` or `tree` to discoverProjects",
+    tree ? dlib.prepareSourceTree {inherit source;},
+  }: let
+    discoveredProjects =
+      l.flatten
+      (
+        l.map
+        (discoverer: discoverer.discover {inherit tree;})
+        (l.attrValues config.discoverers)
+      );
+
+    discoveredProjectsSorted = let
+      sorted =
+        l.sort
+        (p1: p2: l.hasPrefix p1.relPath or "" p2.relPath or "")
+        discoveredProjects;
+    in
+      sorted;
+
+    allProjects = discoveredProjectsSorted;
+
+    projectsExtended =
+      l.forEach allProjects
+      (proj:
+        proj
+        // {
+          relPath = proj.relPath or "";
+          translator = proj.translator or (l.head proj.translators);
+        });
+  in
+    l.listToAttrs (
+      l.map
+      (proj: l.nameValuePair proj.name proj)
+      projectsExtended
+    );
+
+  /*
+  legacy function doing some custom merging on `settings`.
+  We should remove that function once there is no more usage within the
+    frameworks code.
+  */
   discoverProjects = {
     source ? throw "Pass either `source` or `tree` to discoverProjects",
     tree ? dlib.prepareSourceTree {inherit source;},
@@ -69,6 +131,7 @@ in {
   functions.discoverers = {
     inherit
       discoverProjects
+      discoverProjects2
       getDreamLockPath
       applyProjectSettings
       ;

src/modules/functions.discoverers/interface.nix

@@ -3,6 +3,9 @@
   t = l.types;
 in {
   options.functions.discoverers = {
+    discoverProjects2 = l.mkOption {
+      type = t.uniq (t.functionTo (t.listOf t.attrs));
+    };
     discoverProjects = l.mkOption {
       type = t.uniq (t.functionTo (t.listOf t.attrs));
     };

src/subsystems/nodejs/discoverers/discoverer-nodejs/default.nix

@@ -10,29 +10,7 @@
       inherit tree;
     };
   in
-    filterProjects projects;
-
-  # One translator call can process a whole workspace containing all
-  # sub-packages of that workspace.
-  # Therefore we can filter out projects which are children of a workspace.
-  filterProjects = projects: let
-    workspaceRoots =
-      l.filter
-      (proj: proj.subsystemInfo.workspaces or [] != [])
-      projects;
-
-    allWorkspaceChildren =
-      l.flatten
-      (l.map
-        (root: root.subsystemInfo.workspaces)
-        workspaceRoots);
-
-    childrenRemoved =
-      l.filter
-      (proj: (! l.elem proj.relPath allWorkspaceChildren))
-      projects;
-  in
-    childrenRemoved;
+    projects;
 
   getTranslatorNames = tree: let
     packageJson = tree.files."package.json".jsonContent;
@@ -51,49 +29,48 @@
   in
     translators;
 
-  # returns the parsed package.json of a given directory
-  getPackageJson = dirPath:
-    l.fromJSON (l.readFile "${dirPath}/package.json");
-
   # returns all relative paths to workspaces defined by a glob
-  getWorkspacePaths = glob: tree:
-    if l.hasSuffix "*" glob
-    then let
-      prefix = l.removeSuffix "*" glob;
-      path = "${tree.fullPath}/${prefix}";
+  getWorkspacePaths = glob: tree: let
+    paths =
+      if l.hasSuffix "*" glob
+      then let
+        prefix = l.removeSuffix "*" glob;
+        path = "${tree.fullPath}/${prefix}";
 
-      dirNames =
-        if l.pathExists path
-        then dlib.listDirs path
-        else
-          l.trace
-          "WARNING: Detected workspace ${glob} does not exist."
-          [];
-
-      existingWsPaths =
-        l.filter
-        (wsPath:
-          if l.pathExists "${path}/${wsPath}/package.json"
-          then true
-          else let
-            notExistingPath =
-              dlib.sanitizeRelativePath "${prefix}/${wsPath}";
-          in
+        dirNames =
+          if l.pathExists path
+          then dlib.listDirs path
+          else
             l.trace
-            "WARNING: Detected workspace ${notExistingPath} does not exist."
-            false)
-        dirNames;
-    in
-      l.map (dname: "${prefix}/${dname}") existingWsPaths
-    else if l.pathExists "${tree.fullPath}/${glob}/package.json"
-    then [glob]
-    else
-      l.trace
-      "WARNING: Detected workspace ${glob} does not exist."
-      [];
+            "WARNING: Detected workspace ${glob} does not exist."
+            [];
+
+        existingWsPaths =
+          l.filter
+          (wsPath:
+            if l.pathExists "${path}/${wsPath}/package.json"
+            then true
+            else let
+              notExistingPath =
+                dlib.sanitizeRelativePath "${prefix}/${wsPath}";
+            in
+              l.trace
+              "WARNING: Detected workspace ${notExistingPath} does not exist."
+              false)
+          dirNames;
+      in
+        l.map (dname: "${prefix}/${dname}") existingWsPaths
+      else if l.pathExists "${tree.fullPath}/${glob}/package.json"
+      then [glob]
+      else
+        l.trace
+        "WARNING: Detected workspace ${glob} does not exist."
+        [];
+  in
+    map dlib.sanitizeRelativePath paths;
 
   # collect project info for workspaces defined by current package.json
-  getWorkspaces = tree: parentInfo: let
+  getWorkspaceRelPaths = tree: parentInfo: let
     packageJson = tree.files."package.json".jsonContent;
     workspacesRaw = packageJson.workspaces or [];
 
@@ -110,29 +87,7 @@
   in
     l.flatten
     (l.forEach workspacesFlattened
-      (glob: let
-        workspacePaths = getWorkspacePaths glob tree;
-      in
-        l.forEach workspacePaths
-        (wPath: makeWorkspaceProjectInfo tree wPath parentInfo)));
-
-  makeWorkspaceProjectInfo = tree: wsRelPath: parentInfo:
-    dlib.construct.discoveredProject {
-      name =
-        (getPackageJson "${tree.fullPath}/${wsRelPath}").name
-        or "${parentInfo.name}/${wsRelPath}";
-      subsystem = "nodejs";
-      relPath = dlib.sanitizeRelativePath "${tree.relPath}/${wsRelPath}";
-      translators =
-        l.unique
-        (
-          (lib.filter (trans: l.elem trans ["package-lock" "yarn-lock"]) parentInfo.translators)
-          ++ (getTranslatorNames "${tree.fullPath}/${wsRelPath}")
-        );
-      subsystemInfo = {
-        workspaceParent = tree.relPath;
-      };
-    };
+      (glob: getWorkspacePaths glob tree));
 
   discoverInternal = {
     tree,
@@ -165,35 +120,41 @@
         name = tree.files."package.json".jsonContent.name or tree.relPath;
         subsystem = "nodejs";
         translators = getTranslatorNames tree;
-        subsystemInfo = l.optionalAttrs (workspaces != []) {
+        subsystemInfo = l.optionalAttrs (workspaceRelPaths != []) {
           workspaces =
             l.map
-            (w: l.removePrefix "${tree.relPath}/" w.relPath)
-            workspaces;
+            (w: l.removePrefix "${tree.relPath}/" w)
+            workspaceRelPaths;
         };
       };
 
-      workspaces = getWorkspaces tree currentProjectInfo;
+      workspaceRelPaths = getWorkspaceRelPaths tree currentProjectInfo;
 
-      # list of all projects infos found by the current iteration
-      foundProjects =
-        # current directories project info
-        [currentProjectInfo]
-        # workspaces defined by the current directory
-        ++ workspaces;
+      excludePaths =
+        l.map
+        (childRelPath:
+          dlib.sanitizeRelativePath
+          (currentProjectInfo.relPath + "/" + childRelPath))
+        workspaceRelPaths;
+
+      # contains all workspace children
+      excludes =
+        l.genAttrs
+        excludePaths
+        (_: null);
 
       # index of already found projects
       # This is needed, because sub-projects also contain a `package.json`,
       # and would otherwise be discovered again as an independent project.
       alreadyDiscovered' =
         alreadyDiscovered
-        // (l.genAttrs
-          (l.map (p: p.relPath) foundProjects)
-          (relPath: null));
+        // excludes
+        // {
+          ${currentProjectInfo.relPath} = null;
+        };
     in
-      # l.trace tree.directories
       # the current directory
-      foundProjects
+      [currentProjectInfo]
       # sub-directories
       # Thanks to `alreadyDiscovered`, workspace projects won't be discovered
       # a second time.

src/subsystems/rust/discoverers/cargo/default.nix

@@ -5,25 +5,7 @@
 }: let
   l = lib // builtins;
 
-  discoverCrates = {tree}: let
-    cargoToml = tree.files."Cargo.toml".tomlContent or {};
-
-    subdirCrates =
-      l.flatten
-      (l.mapAttrsToList
-        (dirName: dir: discoverCrates {tree = dir;})
-        (tree.directories or {}));
-  in
-    if cargoToml ? package.name
-    then
-      [
-        {
-          inherit (cargoToml.package) name version;
-          inherit (tree) relPath;
-        }
-      ]
-      ++ subdirCrates
-    else subdirCrates;
+  discoverCrates = import ../../findAllCrates.nix {inherit lib dlib;};
 
   discoverProjects = {
     tree,

src/subsystems/rust/findAllCrates.nix

@@ -0,0 +1,26 @@
+{
+  lib,
+  dlib,
+  ...
+}: let
+  discoverCrates = {tree}: let
+    cargoToml = tree.files."Cargo.toml".tomlContent or {};
+
+    subdirCrates =
+      lib.flatten
+      (lib.mapAttrsToList
+        (dirName: dir: discoverCrates {tree = dir;})
+        (tree.directories or {}));
+  in
+    if cargoToml ? package.name
+    then
+      [
+        {
+          inherit (cargoToml.package) name version;
+          inherit (tree) relPath;
+        }
+      ]
+      ++ subdirCrates
+    else subdirCrates;
+in
+  discoverCrates

src/subsystems/rust/translators/cargo-lock/default.nix

@@ -29,7 +29,16 @@ in {
     projectTree = rootTree.getNodeFromPath project.relPath;
     rootSource = rootTree.fullPath;
     projectSource = dlib.sanitizePath "${rootSource}/${project.relPath}";
-    subsystemInfo = project.subsystemInfo;
+    subsystemInfo = project.subsystemInfo or {};
+    # pull all crates from subsystemInfo, if not find all of them
+    # this is mainly helpful when `projects` is defined manually in which case
+    # crates won't be available, so we will reduce burden on the user here.
+    allCrates =
+      subsystemInfo.crates
+      or (
+        (import ../../findAllCrates.nix {inherit lib dlib;})
+        {tree = rootTree;}
+      );
 
     # Get the root toml
     rootToml = {
@@ -330,7 +339,7 @@ in {
                 )
                 cargoPackages;
               workspaceCrate = findCrate workspaceCrates;
-              nonWorkspaceCrate = findCrate (subsystemInfo.crates or []);
+              nonWorkspaceCrate = findCrate allCrates;
             in
               if
                 (package.name == dependencyObject.name)

templates/simple/flake.nix

@@ -5,5 +5,6 @@
       systemsFromFile = ./nix_systems;
       config.projectRoot = ./.;
       source = ./.;
+      projects = ./projects.toml;
     };
 }

tests/integration-d2n-flakes/tests/nodejs_built-binary/flake.nix

@@ -23,5 +23,6 @@
     systems = ["x86_64-linux"];
     config.projectRoot = ./.;
     source = ./.;
+    autoProjects = true;
   });
 }

tests/integration-d2n-flakes/tests/nodejs_dependency-versions-mismatch/flake.nix

@@ -24,5 +24,6 @@
       config.projectRoot = ./.;
       packageOverrides = {};
       source = ./.;
+      autoProjects = true;
     };
 }

tests/integration-d2n-flakes/tests/nodejs_direct-priority-over-transitive/flake.nix

@@ -24,6 +24,7 @@
     systems = ["x86_64-linux"];
     config.projectRoot = ./.;
     source = ./.;
+    autoProjects = true;
     packageOverrides = {
       test = {
         "check-linked-bin-version" = {