feat: use modules system for indexers

src/apps/index/default.nix

@@ -30,7 +30,7 @@ utils.writePureShellScriptBin
   resultBin="$TMPDIR/result"
 
   ${callNixWithD2N} build -L --keep-failed --show-trace --out-link $resultBin \
-    "dream2nix.indexers.indexers.$name.indexBin"
+    "dream2nix.framework.indexers.$name.indexBin"
 
   $resultBin $inputFile
 ''

src/default.nix

@@ -94,7 +94,6 @@ in let
       inherit externalSources;
       inherit inputs;
       inherit framework;
-      inherit indexers;
       inherit dream2nixWithExternals;
       inherit utils;
       inherit nix;
@@ -118,9 +117,6 @@ in let
   # apps for CLI and installation
   apps = callPackageDream ./apps {};
 
-  # indexer implementations
-  indexers = callPackageDream ./indexers {};
-
   # updater modules to find newest package versions
   updaters = callPackageDream ./updaters {};
 
@@ -732,7 +728,6 @@ in {
     callPackageDream
     dream2nixWithExternals
     framework
-    indexers
     fetchSources
     realizeProjects
     translateProjects

src/indexers/crates-io-simple/default.nix

@@ -1,14 +1,11 @@
-{...}: {
-  indexBin = {
-    utils,
-    coreutils,
-    curl,
-    jq,
-    python3,
-    ...
-  }:
+{
+  pkgs,
+  utils,
+  ...
+}: {
+  indexBin =
     utils.writePureShellScript
-    [coreutils curl jq python3]
+    (with pkgs; [coreutils curl jq python3])
     ''
       input=''${1:?"please provide an input as a JSON file"}
 

src/indexers/crates-io/default.nix

@@ -1,15 +1,16 @@
-{...}: {
-  indexBin = {
-    rustPlatform,
-    runCommandLocal,
-    openssl,
-    pkg-config,
-    zlib,
-    curl,
-    libssh2,
-    libgit2,
-    ...
-  }: let
+{pkgs, ...}: {
+  indexBin = let
+    inherit
+      (pkgs)
+      rustPlatform
+      runCommandLocal
+      openssl
+      pkg-config
+      zlib
+      curl
+      libssh2
+      libgit2
+      ;
     package = rustPlatform.buildRustPackage {
       name = "indexer";
 

src/indexers/default.nix

@@ -1,13 +0,0 @@
-{
-  lib,
-  # dream2nix
-  callPackageDream,
-  dlib,
-  ...
-}: let
-  b = builtins;
-  callIndexer = module:
-    module // {indexBin = callPackageDream module.indexBin {};};
-in rec {
-  indexers = dlib.indexers.mapIndexers callIndexer;
-}

src/indexers/libraries-io/default.nix

@@ -14,16 +14,13 @@ libraries.io also supports other interesting popularity metrics:
   - forks
   - rank (source rank, see https://libraries.io/api#project-sourcerank)
 */
-{...}: {
-  indexBin = {
-    utils,
-    coreutils,
-    curl,
-    jq,
-    lib,
-    python3,
-    ...
-  }: let
+{
+  pkgs,
+  utils,
+  lib,
+  ...
+}: {
+  indexBin = let
     l = lib // builtins;
     platformMap = {
       npm = "npm";
@@ -33,7 +30,7 @@ libraries.io also supports other interesting popularity metrics:
     };
   in
     utils.writePureShellScript
-    [coreutils curl jq python3]
+    (with pkgs; [coreutils curl jq python3])
     ''
       input=''${1:?"please provide an input as a JSON file"}
 

src/indexers/npm/default.nix

@@ -1,14 +1,11 @@
-{...}: {
-  indexBin = {
-    utils,
-    coreutils,
-    curl,
-    jq,
-    python3,
-    ...
-  }:
+{
+  pkgs,
+  utils,
+  ...
+}: {
+  indexBin =
     utils.writePureShellScript
-    [coreutils curl jq]
+    (with pkgs; [coreutils curl jq python3])
     ''
       input=''${1:?"please provide an input as a JSON file"}
 
@@ -19,6 +16,6 @@
 
       url="https://registry.npmjs.org/-/v1/search?text=$text&popularity=1.0&quality=0.0&maintenance=0.0&size=$size"
 
-      curl -k "$url" | ${python3}/bin/python ${./process-result.py} > $(realpath $outFile)
+      curl -k "$url" | python3 ${./process-result.py} > $(realpath $outFile)
     '';
 }

src/lib/default.nix

@@ -13,7 +13,6 @@
       construct
       containsMatchingFile
       dirNames
-      indexers
       latestVersion
       listDirs
       listFiles
@@ -27,7 +26,6 @@
       subsystems
       systemsFromFile
       traceJ
-      modules
       warnIfIfd
       parseSpdxId
       ;
@@ -43,9 +41,6 @@
 
   # other libs
   construct = import ./construct.nix {inherit lib;};
-  indexers = import ./indexers.nix {inherit dlib lib;};
-
-  modules = import ./modules.nix {inherit config dlib lib;};
 
   simpleTranslate2 =
     import ./simpleTranslate2.nix {inherit dlib lib;};

src/lib/indexers.nix

@@ -1,40 +0,0 @@
-{
-  dlib,
-  lib,
-}: let
-  l = lib // builtins;
-
-  # TODO:
-  validator = module: true;
-
-  callIndexer = file: extraArgs:
-    dlib.modules.importModule {inherit file extraArgs validator;};
-
-  # get information for builtin indexers
-  indexersDir = ../indexers;
-  indexerNames = dlib.dirNames indexersDir;
-
-  # get the builtin indexers
-  builtinIndexers =
-    l.map
-    (name: {
-      file = "${indexersDir}/${name}";
-      extraArgs = {inherit name;};
-    })
-    indexerNames;
-  # get extra indexers
-  extraIndexers = dlib.modules.extra.indexers or [];
-
-  # import indexers
-  importedIndexers =
-    l.map
-    (
-      module:
-        (callIndexer module.file module.extraArgs)
-        // {inherit (module.extraArgs) name;}
-    )
-    (builtinIndexers ++ extraIndexers);
-  # create the attrset
-  indexers = l.listToAttrs (l.map (f: l.nameValuePair f.name f) importedIndexers);
-  mapIndexers = f: l.mapAttrs (_: indexer: f indexer) indexers;
-in {inherit indexers callIndexer mapIndexers;}

src/lib/modules.nix

@@ -1,197 +0,0 @@
-{
-  config,
-  dlib,
-  lib,
-}: let
-  l = lib // builtins;
-
-  configFuncMsg = ''
-    consider passing a path to a file instead of a function:
-    functions can't be encoded to JSON, and as such most features of
-    dream2nix won't work because of this since they require passing around
-    the config as JSON.
-  '';
-
-  # imports a module.
-  #
-  # - 'file' can be a function or a path to a function.
-  # - 'dlib', 'lib', 'config' and attributes in
-  # 'extraArgs' are passed to the function.
-  importModule = {
-    file,
-    validator ? _: true,
-    extraArgs ? {},
-  }: let
-    _module =
-      if l.isFunction file
-      then file
-      else import file;
-    module =
-      if l.isFunction _module
-      then _module ({inherit config dlib lib;} // extraArgs)
-      else throw "module file (${file}) must return a function that takes an attrset";
-  in
-    l.seq (validator module) module;
-
-  # collects extra modules from a list
-  # ex: [{translators = [module1];} {translators = [module2];}] -> {translators = [module1 module2];}
-  collectExtraModules =
-    l.foldl'
-    (acc: el:
-      acc
-      // (
-        l.mapAttrs
-        (category: modules: modules ++ (acc.${category} or []))
-        el
-      ))
-    {};
-  # processes one extra (config.extra)
-  # returns extra modules like {fetchers = [...]; translators = [...];}
-  processOneExtra = _extra: let
-    isAttrs = val: l.isAttrs val && (! val ? drvPath);
-    # imports a modules declaration
-    importDecl = decl:
-      if l.isFunction decl
-      then l.throw configFuncMsg
-      else if isAttrs decl
-      then decl
-      else import decl {inherit config dlib lib;};
-    # extra attrset itself
-    # config.extra is imported here if it's a path
-    extra = importDecl _extra;
-    # throw if declaring a module as a function
-    throwIfModuleNotPath = module:
-      l.throwIf ((isAttrs _extra) && (l.isFunction module)) configFuncMsg module;
-    # collect subsystem modules (translators, discoverers, builders)
-    _extraSubsystemModules =
-      l.mapAttrsToList
-      (subsystem: categories:
-        l.mapAttrs
-        (category: modules:
-          l.mapAttrsToList
-          (name: module: {
-            file = throwIfModuleNotPath module;
-            extraArgs = {inherit subsystem name;};
-          })
-          (importDecl modules))
-        categories)
-      (importDecl (extra.subsystems or {}));
-    extraSubsystemModules =
-      collectExtraModules (l.flatten _extraSubsystemModules);
-    # collect fetcher modules
-    extraFetcherModules =
-      l.mapAttrsToList
-      (name: fetcher: {
-        file = throwIfModuleNotPath fetcher;
-        extraArgs = {inherit name;};
-      })
-      (importDecl (extra.fetchers or {}));
-  in
-    extraSubsystemModules
-    // {
-      fetchers = extraFetcherModules;
-    };
-  # collect all extras
-  extra =
-    collectExtraModules
-    (l.map processOneExtra (l.flatten [(config.extra or [])]));
-
-  mapSubsystemModules = f: modules:
-    l.mapAttrs
-    (
-      subsystem: names:
-        l.mapAttrs
-        (name: module: f module)
-        names
-    )
-    modules;
-
-  # collect subsystem modules into a list
-  # ex: {rust.cargo-lock = cargo-lock; go.gomod2nix = gomod2nix;} -> [cargo-lock gomod2nix]
-  collectSubsystemModules = modules: let
-    allModules = l.flatten (l.map l.attrValues (l.attrValues modules));
-    hasModule = module: modules:
-      l.any
-      (
-        omodule:
-          (omodule.name == module.name)
-          && (omodule.subsystem == module.subsystem)
-      )
-      modules;
-  in
-    l.foldl'
-    (acc: el:
-      if hasModule el acc
-      then acc
-      else acc ++ [el])
-    []
-    allModules;
-
-  # create subsystem modules
-  # returns ex: {rust = {moduleName = module;}; go = {moduleName = module;};}
-  makeSubsystemModules = {
-    modulesCategory,
-    validator,
-    extraModules ? extra.${modulesCategory} or [],
-    defaults ? {},
-  }: let
-    callModule = file: extraArgs:
-      importModule {inherit file validator extraArgs;};
-
-    # extract builtin modules from subsystems directory
-    modulesBuiltin = l.flatten (
-      l.map
-      (
-        subsystem: let
-          dir = "${toString ../subsystems}/${subsystem}/${modulesCategory}";
-          moduleNames =
-            if l.pathExists dir
-            then dlib.dirNames dir
-            else [];
-          makeModule = name: {
-            file = "${dir}/${name}";
-            extraArgs = {inherit subsystem name;};
-          };
-        in
-          l.map makeModule moduleNames
-      )
-      dlib.subsystems
-    );
-
-    # import the modules
-    importedModules = l.filter (t: t.disabled or false == false) (
-      l.map
-      (
-        module:
-          (callModule module.file module.extraArgs)
-          // {inherit (module.extraArgs) subsystem name;}
-      )
-      (modulesBuiltin ++ extraModules)
-    );
-    # create the modules attrset
-    _modules =
-      l.foldl'
-      (acc: el: l.recursiveUpdate acc {"${el.subsystem}"."${el.name}" = el;})
-      {}
-      importedModules;
-    # add default module attribute to a subsystem if declared in `defaults`
-    modules =
-      l.mapAttrs
-      (
-        subsystem: modules:
-          if l.hasAttr subsystem defaults
-          then modules // {default = modules.${defaults.${subsystem}};}
-          else modules
-      )
-      _modules;
-    mapModules = f: mapSubsystemModules f modules;
-  in {inherit callModule mapModules modules;};
-in {
-  inherit
-    importModule
-    makeSubsystemModules
-    collectSubsystemModules
-    mapSubsystemModules
-    extra
-    ;
-}

src/modules/indexers/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./interface.nix
+    ./implementation.nix
+  ];
+}

src/modules/indexers/implementation.nix

@@ -0,0 +1,12 @@
+{config, ...}: let
+  l = config.lib;
+  indexerDir = ../../indexers;
+  indexerModules = l.readDir indexerDir;
+in {
+  config = {
+    indexers =
+      l.mapAttrs
+      (name: _: import "${indexerDir}/${name}" config)
+      indexerModules;
+  };
+}

src/modules/indexers/interface.nix

@@ -0,0 +1,19 @@
+{config, ...}: let
+  l = config.lib;
+  t = l.types;
+in {
+  options = {
+    indexers = l.mkOption {
+      type = t.attrsOf (
+        t.submoduleWith {
+          modules = [../interfaces.indexer];
+          specialArgs = {framework = config;};
+        }
+      );
+      default = {};
+      description = ''
+        The indexers.
+      '';
+    };
+  };
+}

src/modules/interfaces.indexer/default.nix

@@ -0,0 +1,3 @@
+{
+  imports = [./interface.nix];
+}

src/modules/interfaces.indexer/interface.nix

@@ -0,0 +1,13 @@
+{framework, ...}: let
+  l = framework.lib;
+  t = l.types;
+in {
+  options = {
+    indexBin = l.mkOption {
+      type = t.uniq (t.either t.package t.path);
+      description = ''
+        The program to run to index using the passed indexer arguments.
+      '';
+    };
+  };
+}

src/modules/top-level.nix

@@ -24,6 +24,7 @@ in {
     ./discoverers.default-discoverer
     ./fetchers
     ./translators
+    ./indexers
   ];
   options = {
     apps = lib.mkOption {