feat: make fetchers use modules

examples/_d2n-extended/flake.nix

@@ -41,10 +41,14 @@
             };
           }
         '')
+        (builtins.toFile "crates-io-new.nix" ''
+          {
+            fetchers.crates-io = {
+              imports = ["${inp.dream2nix}/src/fetchers/crates-io"];
+            };
+          }
+        '')
       ];
-      config.extra = {
-        fetchers.crates-io = "${inp.dream2nix}/src/fetchers/crates-io";
-      };
       source = src;
       settings = [
         {

src/default.nix

@@ -51,7 +51,7 @@ in let
   };
 
   framework = import ./modules/framework.nix {
-    inherit lib dlib callPackageDream;
+    inherit lib dlib callPackageDream pkgs utils;
     dream2nixConfig = config;
   };
 
@@ -238,8 +238,8 @@ in let
   # detect if granular or combined fetching must be used
   findFetcher = dreamLock:
     if null != dreamLock._generic.sourcesAggregatedHash
-    then fetchers.combinedFetcher
-    else fetchers.defaultFetcher;
+    then framework.functions.fetchers.combinedFetcher
+    else framework.functions.fetchers.defaultFetcher;
 
   # fetch only sources and do not build
   fetchSources = {

src/lib.nix

@@ -94,6 +94,8 @@
       dream2nixConfig = config;
       callPackageDream =
         throw "callPackageDream is not available before nixpkgs is imported";
+      pkgs = throw "pkgs is not available before nixpkgs is imported";
+      utils = throw "utils is not available before nixpkgs is imported";
     };
 
     initD2N = initDream2nix config;

src/modules/fetchers/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./interface.nix
+    ./implementation.nix
+  ];
+}

src/modules/fetchers/implementation.nix

@@ -0,0 +1,26 @@
+{
+  dlib,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+  fetchersDir = ../../fetchers;
+  fetcherNames = l.attrNames (
+    l.filterAttrs
+    (_: type: type == "directory")
+    (l.readDir fetchersDir)
+  );
+  fetcherModules =
+    l.genAttrs
+    fetcherNames
+    (
+      name:
+        import "${fetchersDir}/${name}" {
+          inherit dlib lib;
+        }
+    );
+in {
+  config = {
+    fetchers = fetcherModules;
+  };
+}

src/modules/fetchers/interface.nix

@@ -0,0 +1,19 @@
+{
+  lib,
+  specialArgs,
+  ...
+}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options = {
+    fetchers = l.mkOption {
+      type = t.attrsOf (
+        t.submoduleWith {
+          modules = [../interfaces.fetcher];
+          inherit specialArgs;
+        }
+      );
+    };
+  };
+}

src/modules/framework.nix

@@ -1,8 +1,10 @@
 {
   dream2nixConfig,
   callPackageDream,
+  pkgs,
   dlib,
   lib,
+  utils,
 }: let
   evaledModules = lib.evalModules {
     modules = [./top-level.nix] ++ (dream2nixConfig.modules or []);
@@ -12,7 +14,9 @@
       inherit
         dream2nixConfig
         callPackageDream
+        pkgs
         dlib
+        utils
         ;
     };
   };

src/modules/functions.combined-fetcher/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./interface.nix
+    ./implementation.nix
+  ];
+}

src/modules/functions.combined-fetcher/implementation.nix

@@ -0,0 +1,235 @@
+{
+  lib,
+  dlib,
+  utils,
+  config,
+  pkgs,
+  ...
+}: let
+  defaultFetcher = config.functions.fetchers.defaultFetcher;
+  inherit
+    (pkgs)
+    nix
+    stdenv
+    writeScript
+    async
+    bash
+    coreutils
+    ;
+  func = {
+    # sources attrset from dream lock
+    sources,
+    sourcesAggregatedHash,
+    sourceOverrides,
+    ...
+  } @ args: let
+    b = builtins;
+    l = lib // builtins;
+
+    # resolve to individual fetcher calls
+    defaultFetched = (defaultFetcher args).fetchedSources;
+
+    isFOD = drv:
+      lib.all
+      (attr: drv ? "${attr}")
+      ["outputHash" "outputHashAlgo" "outputHashMode"];
+
+    drvArgs = drv:
+      (drv.overrideAttrs (args: {
+        passthru.originalArgs = args;
+      }))
+      .originalArgs;
+
+    # extract the arguments from the individual fetcher calls
+    FODArgsAll = let
+      FODArgsAll' =
+        lib.mapAttrs
+        (
+          name: versions:
+            lib.mapAttrs
+            (version: fetched:
+              # handle FOD sources
+                if isFOD fetched
+                then
+                  (drvArgs fetched)
+                  // {
+                    isOriginal = false;
+                    outPath = let
+                      sanitizedName = l.strings.sanitizeDerivationName name;
+                    in "${sanitizedName}/${version}/${fetched.name}";
+                  }
+                # handle already extracted sources
+                else if fetched ? original && isFOD fetched.original
+                then
+                  (drvArgs fetched.original)
+                  // {
+                    isOriginal = true;
+                    outPath = let
+                      sanitizedName = l.strings.sanitizeDerivationName name;
+                    in "${sanitizedName}/${version}/${fetched.original.name}";
+                  }
+                # handle path sources
+                else if lib.isString fetched
+                then "ignore"
+                # handle store path sources
+                else if lib.isStorePath fetched
+                then "ignore"
+                # handle unknown sources
+                else if fetched == "unknown"
+                then "ignore"
+                # error out on unknown source types
+                else
+                  throw ''
+                    Error while generating FOD fetcher for combined sources.
+                    Cannot classify source of ${name}#${version}.
+                  '')
+            versions
+        )
+        defaultFetched;
+    in
+      lib.filterAttrs
+      (name: versions: versions != {})
+      (lib.mapAttrs
+        (name: versions:
+          lib.filterAttrs
+          (version: fetcherArgs: fetcherArgs != "ignore")
+          versions)
+        FODArgsAll');
+
+    FODArgsAllList =
+      lib.flatten
+      (lib.mapAttrsToList
+        (name: versions:
+          b.attrValues versions)
+        FODArgsAll);
+
+    # convert arbitrary types to string, like nix does with derivation arguments
+    toString' = x:
+      if lib.isBool x
+      then
+        if x
+        then "1"
+        else ""
+      else if lib.isList x
+      then ''"${lib.concatStringsSep " " (lib.forEach x (y: toString' y))}"''
+      else if x == null
+      then ""
+      else b.toJSON x;
+
+    # set up nix build env for signle item
+    itemScript = fetcherArgs: ''
+
+      # export arguments for builder
+      ${lib.concatStringsSep "\n" (lib.mapAttrsToList (argName: argVal: ''
+          export ${argName}=${
+            lib.replaceStrings ["$" ''\n''] [''\$'' "\n"] (toString' argVal)
+          }
+        '')
+        fetcherArgs)}
+
+      # run builder
+      bash ${fetcherArgs.builder}
+    '';
+
+    mkScriptForItem = fetcherArgs: ''
+      # configure $out
+      OUT_ORIG=$out
+      export out=$OUT_ORIG/${fetcherArgs.outPath}
+      mkdir -p $(dirname $out)
+
+      # set up TMP and TMPDIR
+      workdir=$(mktemp -d)
+      TMP=$workdir/TMP
+      TMPDIR=$TMP
+      mkdir -p $TMP
+
+      # do the work
+      pushd $workdir
+      ${itemScript fetcherArgs}
+      popd
+      rm -r $workdir
+    '';
+
+    # builder which wraps several other FOD builders
+    # and executes these after each other inside a single build
+    builder = writeScript "multi-source-fetcher" ''
+      #!${bash}/bin/bash
+      export PATH=${coreutils}/bin:${bash}/bin
+
+      mkdir $out
+
+      S="/$TMP/async_socket"
+      async=${async}/bin/async
+      $async -s="$S" server --start -j40
+
+      # remove if resolved: https://github.com/ctbur/async/issues/6
+      sleep 1
+
+      ${lib.concatStringsSep "\n"
+        (b.map
+          (fetcherArgs: ''
+            $async -s="$S" cmd -- bash -c '${mkScriptForItem fetcherArgs}'
+          '')
+          FODArgsAllList)}
+
+      $async -s="$S" wait
+
+      ${lib.concatStringsSep "\n"
+        (b.map
+          (fetcherArgs: ''
+            if [ ! -e "$out/${fetcherArgs.outPath}" ]; then
+              echo "builder for ${fetcherArgs.name} terminated without creating out path: ${fetcherArgs.outPath}"
+              exit 1
+            fi
+          '')
+          FODArgsAllList)}
+
+      echo "FOD_HASH=$(${nix}/bin/nix --extra-experimental-features "nix-command flakes" hash path $out)"
+    '';
+
+    FODAllSources = let
+      nativeBuildInputs' =
+        lib.unique
+        (lib.foldl (a: b: a ++ b) []
+          (b.map
+            (fetcherArgs: (fetcherArgs.nativeBuildInputs or []))
+            FODArgsAllList));
+    in
+      stdenv.mkDerivation rec {
+        name = "sources-combined";
+        inherit builder;
+        nativeBuildInputs =
+          nativeBuildInputs'
+          ++ [
+            coreutils
+          ];
+        outputHashAlgo = "sha256";
+        outputHashMode = "recursive";
+        outputHash = sourcesAggregatedHash;
+      };
+  in {
+    FOD = FODAllSources;
+
+    fetchedSources =
+      lib.mapAttrs
+      (name: versions:
+        lib.mapAttrs
+        (version: source:
+          if FODArgsAll ? "${name}"."${version}".outPath
+          then
+            if FODArgsAll ? "${name}"."${version}".isOriginal
+            then
+              utils.extractSource {
+                source = "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}";
+                name = l.strings.sanitizeDerivationName name;
+              }
+            else "${FODAllSources}/${FODArgsAll."${name}"."${version}".outPath}"
+          else defaultFetched."${name}"."${version}")
+        versions)
+      defaultFetched;
+  };
+in {
+  config = {
+    functions.fetchers.combinedFetcher = func;
+  };
+}

src/modules/functions.combined-fetcher/interface.nix

@@ -0,0 +1,10 @@
+{lib, ...}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options = {
+    functions.fetchers.combinedFetcher = l.mkOption {
+      type = t.functionTo t.attrs;
+    };
+  };
+}

src/modules/functions.default-fetcher/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./interface.nix
+    ./implementation.nix
+  ];
+}

src/modules/functions.default-fetcher/implementation.nix

@@ -0,0 +1,76 @@
+{
+  lib,
+  dlib,
+  config,
+  ...
+}: let
+  fetchers = config.fetchers;
+  fetchSource = config.functions.fetchers.fetchSource;
+  func = {
+    # sources attrset from dream lock
+    defaultPackage,
+    defaultPackageVersion,
+    sourceOverrides,
+    sources,
+    sourceRoot ? null,
+    ...
+  }: let
+    l = lib // builtins;
+
+    fetchedSources =
+      l.mapAttrs
+      (name: versions:
+        l.mapAttrs
+        (version: source:
+          if source.type == "unknown"
+          then "unknown"
+          else if source.type == "path"
+          then let
+            path =
+              if l.isStorePath (l.concatStringsSep "/" (l.take 4 (l.splitString "/" source.path)))
+              then source.path
+              else if name == source.rootName && version == source.rootVersion
+              then throw "source for ${name}@${version} is referencing itself"
+              else if source.rootName != null && source.rootVersion != null
+              then "${overriddenSources."${source.rootName}"."${source.rootVersion}"}/${source.path}"
+              else if sourceRoot != null
+              then "${sourceRoot}/${source.path}"
+              else throw "${name}-${version}: cannot determine path source";
+          in
+            l.path {
+              inherit path;
+              name = l.strings.sanitizeDerivationName "${name}-${version}-source";
+            }
+          else if fetchers.fetchers ? "${source.type}"
+          then
+            fetchSource {
+              source =
+                source
+                // {
+                  pname = source.pname or name;
+                  version = source.version or version;
+                };
+            }
+          else throw "unsupported source type '${source.type}'")
+        versions)
+      sources;
+
+    overriddenSources =
+      l.zipAttrsWith
+      (name: versionsSets:
+        l.zipAttrsWith
+        (version: sources: l.last sources)
+        versionsSets)
+      [
+        fetchedSources
+        (sourceOverrides fetchedSources)
+      ];
+  in {
+    # attrset: pname -> path of downloaded source
+    fetchedSources = overriddenSources;
+  };
+in {
+  config = {
+    functions.fetchers.defaultFetcher = func;
+  };
+}

src/modules/functions.default-fetcher/interface.nix

@@ -0,0 +1,10 @@
+{lib, ...}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options = {
+    functions.fetchers.defaultFetcher = l.mkOption {
+      type = t.functionTo t.attrs;
+    };
+  };
+}

src/modules/functions.fetchers/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./interface.nix
+    ./implementation.nix
+  ];
+}

src/modules/functions.fetchers/implementation.nix

@@ -0,0 +1,233 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  b = builtins;
+  fetchers = config.fetchers;
+
+  renderUrlArgs = kwargs: let
+    asStr =
+      lib.concatStringsSep
+      "&"
+      (lib.mapAttrsToList
+        (name: val: "${name}=${val}")
+        kwargs);
+  in
+    if asStr == ""
+    then ""
+    else "?" + asStr;
+in {
+  config = rec {
+    constructSource = {
+      type,
+      reComputeHash ? false,
+      ...
+    } @ args: let
+      fetcher = fetchers."${type}";
+      argsKeep = b.removeAttrs args ["reComputeHash"];
+      fetcherOutputs =
+        fetcher.outputs
+        (b.removeAttrs argsKeep ["dir" "hash" "type"]);
+    in
+      argsKeep
+      # if the hash was not provided, calculate hash on the fly (impure)
+      // (lib.optionalAttrs reComputeHash {
+        hash = fetcherOutputs.calcHash "sha256";
+      });
+
+    # update source spec to different version
+    updateSource = {
+      source,
+      newVersion,
+      ...
+    }:
+      constructSource (source
+        // {
+          reComputeHash = true;
+        }
+        // {
+          "${fetchers."${source.type}".versionField}" = newVersion;
+        });
+
+    # fetch a source defined via a dream lock source spec
+    fetchSource = {
+      source,
+      extract ? false,
+    }: let
+      fetcher = fetchers."${source.type}";
+      fetcherArgs = b.removeAttrs source ["dir" "hash" "type"];
+      fetcherOutputs = fetcher.outputs fetcherArgs;
+      maybeArchive = fetcherOutputs.fetched (source.hash or null);
+    in
+      if source ? dir
+      then "${maybeArchive}/${source.dir}"
+      else maybeArchive;
+
+    # fetch a source defined by a shortcut
+    fetchShortcut = {
+      shortcut,
+      extract ? false,
+    }:
+      fetchSource {
+        source = translateShortcut {inherit shortcut;};
+        inherit extract;
+      };
+
+    parseShortcut = shortcut: let
+      # in: "git+https://foo.com/bar?kwarg1=lol&kwarg2=hello"
+      # out: [ "git+" "git" "https" "//" "foo.com/bar" "?kwarg1=lol&kwarg2=hello" "kwarg1=lol&kwarg2=hello" ]
+      split =
+        b.match
+        ''(([[:alnum:]]+)\+)?([[:alnum:]-]+):(//)?([^\?]*)(\?(.*))?''
+        shortcut;
+
+      parsed = {
+        proto1 = b.elemAt split 1;
+        proto2 = b.elemAt split 2;
+        path = b.elemAt split 4;
+        allArgs = b.elemAt split 6;
+        kwargs = b.removeAttrs kwargs_ ["dir"];
+        dir = kwargs_.dir or null;
+      };
+
+      kwargs_ =
+        if parsed.allArgs == null
+        then {}
+        else
+          lib.listToAttrs
+          (map
+            (kwarg: let
+              split = lib.splitString "=" kwarg;
+            in
+              lib.nameValuePair
+              (b.elemAt split 0)
+              (b.elemAt split 1))
+            (lib.splitString "&" parsed.allArgs));
+    in
+      if split == null
+      then throw "Unable to parse shortcut: ${shortcut}"
+      else parsed;
+
+    # translate shortcut to dream lock source spec
+    translateShortcut = {
+      shortcut ? throw "pass shortcut",
+      parsed ? parseShortcut shortcut,
+      computeHash ? (! parsed.kwargs ? hash),
+    }: let
+      parsed = parseShortcut shortcut;
+
+      checkArgs = fetcherName: args: let
+        fetcher = fetchers."${fetcherName}";
+        unknownArgNames = lib.filter (argName: ! lib.elem argName fetcher.inputs) (lib.attrNames args);
+        missingArgNames = lib.filter (inputName: ! args ? "${inputName}") fetcher.inputs;
+      in
+        if lib.length unknownArgNames > 0
+        then throw "Received unknown arguments for fetcher '${fetcherName}': ${b.toString unknownArgNames}"
+        else if lib.length missingArgNames > 0
+        then throw "Missing arguments for fetcher '${fetcherName}': ${b.toString missingArgNames}"
+        else args;
+
+      translateHttpUrl = let
+        fetcher = fetchers.http;
+
+        urlArgsFinal = renderUrlArgs parsed.kwargs;
+
+        url = with parsed; "${proto2}://${path}${urlArgsFinal}";
+
+        fetcherOutputs = fetchers.http.outputs {
+          inherit url;
+        };
+      in
+        constructSource
+        {
+          inherit url;
+          type = "http";
+        }
+        // (lib.optionalAttrs (parsed.dir != null) {
+          dir = parsed.dir;
+        })
+        // (lib.optionalAttrs computeHash {
+          hash = fetcherOutputs.calcHash "sha256";
+        });
+
+      translateProtoShortcut = let
+        kwargsUrl = b.removeAttrs parsed.kwargs fetcher.inputs;
+
+        urlArgs = renderUrlArgs kwargsUrl;
+
+        url = with parsed; "${proto2}://${path}${urlArgs}";
+
+        fetcherName = parsed.proto1;
+
+        fetcher = fetchers."${fetcherName}";
+
+        args = parsed.kwargs // {inherit url;};
+
+        fetcherOutputs = fetcher.outputs (checkArgs fetcherName args);
+      in
+        constructSource
+        (parsed.kwargs
+          // {
+            type = fetcherName;
+            inherit url;
+          }
+          // (lib.optionalAttrs (parsed.dir != null) {
+            dir = parsed.dir;
+          })
+          // (lib.optionalAttrs computeHash {
+            hash = fetcherOutputs.calcHash "sha256";
+          }));
+
+      translateRegularShortcut = let
+        fetcherName = parsed.proto2;
+
+        path = lib.removeSuffix "/" parsed.path;
+
+        params = lib.splitString "/" path;
+
+        fetcher = fetchers."${fetcherName}";
+
+        args =
+          if fetcher ? parseParams
+          then fetcher.parseParams params
+          else if b.length params != b.length fetcher.inputs
+          then
+            throw ''
+              Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
+              Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
+            ''
+          else
+            lib.listToAttrs
+            (lib.forEach
+              (lib.range 0 ((lib.length fetcher.inputs) - 1))
+              (
+                idx:
+                  lib.nameValuePair
+                  (lib.elemAt fetcher.inputs idx)
+                  (lib.elemAt params idx)
+              ));
+
+        fetcherOutputs = fetcher.outputs (args // parsed.kwargs);
+      in
+        constructSource (args
+          // parsed.kwargs
+          // {
+            type = fetcherName;
+          }
+          // (lib.optionalAttrs (parsed.dir != null) {
+            dir = parsed.dir;
+          })
+          // (lib.optionalAttrs computeHash {
+            hash = fetcherOutputs.calcHash "sha256";
+          }));
+    in
+      if parsed.proto1 != null
+      then translateProtoShortcut
+      else if
+        lib.hasPrefix "http://" shortcut
+        || lib.hasPrefix "https://" shortcut
+      then translateHttpUrl
+      else translateRegularShortcut;
+  };
+}

src/modules/functions.fetchers/interface.nix

@@ -0,0 +1,39 @@
+{lib, ...}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options = {
+    functions.fetchers = {
+      constructSource = l.mkOption {
+        type = t.functionTo t.attrs;
+      };
+      updateSource = l.mkOption {
+        type = t.functionTo t.attrs;
+        description = ''
+          update source spec to different version
+        '';
+      };
+      fetchSource = l.mkOption {
+        type = t.functionTo t.package;
+        description = ''
+          fetch a source defined via a dream lock source spec
+        '';
+      };
+      fetchShortcut = l.mkOption {
+        type = t.functionTo t.package;
+        description = ''
+          fetch a source defined by a shortcut
+        '';
+      };
+      parseShortcut = l.mkOption {
+        type = t.functionTo t.attrs;
+      };
+      translateShortcut = l.mkOption {
+        type = t.functionTo t.attrs;
+        description = ''
+          translate shortcut to dream lock source spec
+        '';
+      };
+    };
+  };
+}

src/modules/interfaces.fetcher/default.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ./interface.nix
+  ];
+}

src/modules/interfaces.fetcher/interface.nix

@@ -0,0 +1,45 @@
+{lib, ...}: let
+  l = lib // builtins;
+  t = l.types;
+
+  outputsOptions = {
+    options = {
+      calcHash = l.mkOption {
+        type = t.functionTo t.str;
+        description = ''
+          Function that will calculate the hash of the source.
+          It should output the hash in the specified algo.
+        '';
+      };
+      fetched = l.mkOption {
+        type = t.functionTo t.package;
+        description = ''
+          Function that will output the source itself, given a hash.
+        '';
+      };
+    };
+  };
+in {
+  options = {
+    inputs = l.mkOption {
+      type = t.listOf t.str;
+      description = ''
+        The inputs that are passed to this fetcher.
+      '';
+    };
+    versionField = l.mkOption {
+      type = t.str;
+      description = ''
+        The name of an input that corresponds to the "version" of this source.
+        This can be semantic versioning, a git revision, etc.
+      '';
+    };
+    outputs = l.mkOption {
+      type = t.functionTo (t.functionTo (
+        t.submoduleWith {
+          modules = [outputsOptions];
+        }
+      ));
+    };
+  };
+}

src/modules/top-level.nix

@@ -6,13 +6,17 @@
   t = lib.types;
 in {
   imports = [
-    ./functions.subsystem-loading
-    ./functions.translators
     ./functions.discoverers
-    ./translators
+    ./functions.fetchers
+    ./functions.default-fetcher
+    ./functions.combined-fetcher
+    ./functions.translators
+    ./functions.subsystem-loading
     ./builders
     ./discoverers
     ./discoverers.default-discoverer
+    ./fetchers
+    ./translators
   ];
   options = {
     lib = lib.mkOption {