introduce fetcher modules

flake.nix

@@ -14,34 +14,34 @@
 
       supportedSystems = [ "x86_64-linux" ];
 
-      forAllSystems = f: lib.genAttrs supportedSystems (system: f system);
+      forAllSystems = f: lib.genAttrs supportedSystems (system:
+        f system (import nixpkgs { inherit system; overlays = [ self.overlay ]; })
+      );
 
-      nixpkgsFor = forAllSystems (system: import nixpkgs {
-        inherit system;
-        overlays = [ self.overlay ];
-      });
-
-      externalSourcesFor = forAllSystems (system: nixpkgsFor."${system}".runCommand "dream2nix-vendored" {} ''
+      externalSourcesFor = forAllSystems (system: pkgs: pkgs.runCommand "dream2nix-vendored" {} ''
         mkdir -p $out/{npmlock2nix,node2nix}
         cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/
         cp ${node2nix}/{nix/node-env.nix,LICENSE} $out/node2nix/
       '');
 
-      dream2nixFor = forAllSystems (system: import ./src rec {
-        pkgs = nixpkgsFor."${system}";
+      dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
         externalSources = externalSourcesFor."${system}";
         inherit lib;
       });
 
     in
       {
-        overlay = curr: prev: {};
+        overlay = new: old: {
+          nix = old.writeScriptBin "nix" ''
+            ${new.nixUnstable}/bin/nix --option experimental-features "nix-command flakes" "$@"
+          '';
+        };
 
         lib.dream2nix = dream2nixFor;
 
-        defaultApp = forAllSystems (system: self.apps."${system}".cli);
+        defaultApp = forAllSystems (system: pkgs: self.apps."${system}".cli);
 
-        apps = forAllSystems (system:
+        apps = forAllSystems (system: pkgs:
           lib.mapAttrs (appName: app:
             {
               type = "app";
@@ -50,8 +50,8 @@
           ) dream2nixFor."${system}".apps
         );
 
-        devShell = forAllSystems (system: nixpkgsFor."${system}".mkShell {
-          buildInputs = with nixpkgsFor."${system}"; [
+        devShell = forAllSystems (system: pkgs: pkgs.mkShell {
+          buildInputs = with pkgs; [
             cntr
             nixUnstable
           ];

src/apps/default.nix

@@ -1,7 +1,7 @@
 {
   pkgs,
 
-  callPackage,
+  callPackageDream,
   externalSources,
   location,
   translators,
@@ -10,14 +10,14 @@
 {
 
   # the unified translator cli
-  cli = callPackage (import ./cli) {};
-  cli2 = callPackage (import ./cli2) {};
+  cli = callPackageDream (import ./cli) {};
+  cli2 = callPackageDream (import ./cli2) {};
 
   # the contribute cli
-  contribute = callPackage (import ./contribute) {};
+  contribute = callPackageDream (import ./contribute) {};
 
   # install the framework to a specified location by copying the code
-  install = callPackage ({ writeScript, }:
+  install = callPackageDream ({ writeScript, }:
     writeScript
       "install"
       ''

src/builders/default.nix

@@ -1,5 +1,5 @@
 {
-  callPackage,
+  callPackageDream,
   ...
 }:
 {
@@ -7,14 +7,14 @@
 
     default = simpleBuilder;
 
-    simpleBuilder = callPackage ./python/simple-builder {};
+    simpleBuilder = callPackageDream ./python/simple-builder {};
   };
 
   nodejs =  rec {
 
     default = node2nix;
 
-    node2nix = callPackage ./nodejs/node2nix {};
+    node2nix = callPackageDream ./nodejs/node2nix {};
   };
   
 }

src/default.nix

@@ -12,10 +12,10 @@
 
 let
 
-  utils = callPackage ./utils.nix {};
+  utils = pkgs.callPackage ./utils.nix {};
 
-  callPackage = f: args: pkgs.callPackage f (args // {
-    inherit callPackage;
+  callPackageDream = f: args: pkgs.callPackage f (args // {
+    inherit callPackageDream;
     inherit externals;
     inherit externalSources;
     inherit location;
@@ -31,18 +31,18 @@ let
   config = builtins.fromJSON (builtins.readFile ./config.json);
 
   # apps for CLI and installation
-  apps = callPackage ./apps {};
+  apps = callPackageDream ./apps {};
 
   # builder implementaitons for all subsystems
-  builders = callPackage ./builders {};
+  builders = callPackageDream ./builders {};
 
   # fetcher implementations
-  fetchers = callPackage ./fetchers {
+  fetchers = callPackageDream ./fetchers {
     inherit (config) allowBuiltinFetchers;
   };
 
   # the translator modules and utils for all subsystems
-  translators = callPackage ./translators {};
+  translators = callPackageDream ./translators {};
 
 
   # the location of the dream2nix framework for self references (update scripts, etc.)

src/fetchers/default-fetcher.nix

@@ -1,11 +1,9 @@
 {
-  # fetchers
-  fetchFromGitHub,
-  fetchFromGitLab,
-  fetchgit,
-  fetchurl,
-
   lib,
+
+  # dream2nix attributes
+  fetchSource,
+  fetchers,
   ...
 }:
 {
@@ -15,73 +13,13 @@
   ...
 }:
 
-let
-  githubMissingHashErrorText = pname: ''
-    Error: Cannot verify the integrity of the source of '${pname}'
-    It is a github reference with no hash providedand.
-    Solve this problem via any of the wollowing ways:
-
-    - (alternative 1): allow the use of builtin fetchers (which can verify using git rev).
-      ```
-        dream2nix.buildPackage {
-          ...
-          allowBuiltinFetchers = true;
-          ...
-        }
-      ```
-
-    - (alternative 2): add a hash to the source via override
-      ```
-        dream2nix.buildPackage {
-          ...
-          sourceOverrides = oldSources: {
-            "${pname}" = oldSources."${pname}".overrideAttrs (_:{
-              hash = "";
-            })
-          }
-          ...
-        }
-      ```
-
-  '';
-in
-
 {
   # attrset: pname -> path of downloaded source
   fetchedSources = lib.mapAttrs (pname: source:
-    if source.type == "github" then
-      # handle when no hash is provided
-      if ! source ? hash then
-        if allowBuiltinFetchers then
-          builtins.fetchGit {
-            inherit (source) rev;
-            allRefs = true;
-            url = "https://github.com/${source.owner}/${source.repo}";
-          }
-        else
-          throw githubMissingHashErrorText pname
-      else
-        fetchFromGitHub {
-          inherit (source) url owner repo rev;
-          hash = source.hash or null;
-        }
-    else if source.type == "gitlab" then 
-      fetchFromGitLab {
-        inherit (source) url owner repo rev;
-        hash = source.hash or null;
-      }
-    else if source.type == "git" then
-      fetchgit {
-        inherit (source) url rev;
-        hash = source.hash or null;
-      }
-    else if source.type == "fetchurl" then
-      fetchurl {
-        inherit (source) url;
-        hash = source.hash or null;
-      }
-    else if source.type == "unknown" then
+    if source.type == "unknown" then
       "unknown"
+    else if fetchers ? "${source.type}" then
+      fetchSource source
     else throw "unsupported source type '${source.type}'"
   ) sources;
 }

src/fetchers/default.nix

@@ -1,12 +1,105 @@
 {
-  callPackage,
+  callPackageDream,
+  lib,
 
+  utils,
   # config
   allowBuiltinFetchers,
   ...
 }:
+
+let
+  b = builtins;
+  callFetcher = file: args: callPackageDream file ({
+    inherit allowBuiltinFetchers;
+  } // args);
+in
+
 rec {
-  defaultFetcher = callPackage ./default-fetcher.nix {};
+
+  fetchers = lib.genAttrs (utils.dirNames ./.) (name:
+    callFetcher (./. + "/${name}") {}
+  );
+
+  defaultFetcher = callPackageDream ./default-fetcher.nix { inherit fetchers fetchSource; };
   
-  combinedFetcher = callPackage ./combined-fetcher.nix { inherit defaultFetcher; };
+  combinedFetcher = callPackageDream ./combined-fetcher.nix { inherit defaultFetcher; };
+
+  fetchSource = source:
+    let
+      fetcher = fetchers."${source.type}";
+      fetcherOutputs = fetcher.outputs source;
+    in
+      fetcherOutputs.fetched (source.hash or null);
+
+  fetchViaShortcut = shortcut:
+    let
+
+      fetchViaHttpUrl = 
+        let
+          fetcher = fetchers.fetchurl;
+          fetcherOutputs = fetchers.http.outputs { url = shortcut; };
+        in
+          rec {
+            hash = fetcherOutputs.calcHash "sha256";
+            fetched = fetcherOutputs.fetched hash;
+          };
+
+      fetchViaGitShortcut =
+        let
+          urlAndParams = lib.elemAt (lib.splitString "+" shortcut) 1;
+          splitUrlParams = lib.splitString "?" urlAndParams;
+          url = lib.head splitUrlParams;
+          params = lib.listToAttrs (lib.forEach (lib.tail splitUrlParams) (keyVal:
+            let
+              split = lib.splitString "=" keyVal;
+              name = lib.elemAt split 0;
+              value = lib.elemAt split 1;
+            in
+              lib.nameValuePair name value
+          ));
+          fetcher = fetchers.git;
+          args = params // { inherit url; };
+          fetcherOutputs = fetcher.outputs args;
+        in
+          rec {
+            hash = fetcherOutputs.calcHash "sha256";
+            fetched = fetcherOutputs.fetched hash;
+          };
+        
+      fetchViaRegularShortcut =
+        let
+          splitNameParams = lib.splitString ":" (lib.removeSuffix "/" shortcut);
+          fetcherName = lib.elemAt splitNameParams 0;
+          paramsStr = lib.elemAt splitNameParams 1;
+          params = lib.splitString "/" paramsStr;
+          
+          fetcher = fetchers."${fetcherName}";
+          args = lib.listToAttrs 
+            (lib.forEach
+              (lib.range 0 ((lib.length fetcher.inputs) - 1))
+              (idx:
+                lib.nameValuePair
+                  (lib.elemAt fetcher.inputs idx)
+                  (lib.elemAt params idx)
+              ));
+          fetcherOutputs = fetcher.outputs args;
+        in
+          if b.length params != b.length fetcher.inputs then
+            throw ''
+              Wrong number of arguments provided in shortcut for fetcher '${fetcherName}'
+              Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
+            ''
+          else
+            rec {
+              hash = fetcherOutputs.calcHash "sha256";
+              fetched = fetcherOutputs.fetched hash;
+            };
+    in
+      if lib.hasPrefix "git+" (lib.head (lib.splitString ":" shortcut)) then
+        fetchViaGitShortcut
+      else if lib.hasPrefix "http://" shortcut || lib.hasPrefix  "https://" shortcut then
+        fetchViaHttpUrl
+      else
+        fetchViaRegularShortcut;
 }

src/fetchers/fetchurl/default.nix

@@ -0,0 +1,31 @@
+{
+  fetchurl,
+
+  utils,
+  # config
+  allowBuiltinFetchers,
+  ...
+}:
+{
+
+  inputs = [
+    "url"
+  ];
+
+  outputs = { url, ... }@inp: 
+    let
+      b = builtins;
+    in
+    {
+
+      calcHash = algo: utils.hashPath algo (b.fetchurl {
+        inherit url;
+      });
+
+      fetched = hash:
+        fetchurl {
+          inherit url hash;
+        };
+
+    };
+}

src/fetchers/git/default.nix

@@ -0,0 +1,60 @@
+{
+  fetchgit,
+  lib,
+
+  utils,
+  # config
+  allowBuiltinFetchers,
+  ...
+}:
+let
+  b = builtins;
+in
+{
+
+  inputs = [
+    "url"
+    "rev"
+  ];
+
+  outputs = { url, ref ? null, rev ? null, ... }@inp: 
+    if ref == null && rev == null then
+      throw "At least 'rev' or 'ref' must be specified for fetcher 'git'"
+    else if rev != null && ! (builtins.match "[a-f0-9]*" rev) then
+      throw "Argument 'rev' for fetcher git must be a sha1 hash. Try using 'ref' instead"
+    else if ref != null && b.match "refs/(heads|tags)/.*" ref == null then
+      throw ''ref must be of format "refs/heads/branch-name" or "refs/tags/tag-name"''
+    else
+    let
+      b = builtins;
+      refAndRev =
+        (lib.optionalAttrs (ref != null) {
+          inherit ref;
+        })
+        //
+        (lib.optionalAttrs (rev != null) {
+          inherit rev;
+        });
+    in
+    {
+
+      calcHash = algo: utils.hashPath algo (b.fetchGit
+        ({ inherit url; } // refAndRev)
+      );
+
+      fetched = hash:
+        if hash == null then
+          if rev == null then
+            throw "Cannot fetch git repo without integrity. Specify at least 'rev' or 'sha256'"
+          else
+            b.fetchGit (
+              { inherit url;} // refAndRev
+            )
+        else
+          fetchgit {
+            inherit url;
+            rev = if rev != null then rev else ref;
+            sha256 = hash;
+          };
+    };
+}

src/fetchers/github/default.nix

@@ -0,0 +1,75 @@
+{
+  fetchFromGitHub,
+  lib,
+  nix,
+  runCommand,
+
+  utils,
+  # config
+  allowBuiltinFetchers,
+  ...
+}:
+{
+
+  inputs = [
+    "owner"
+    "repo"
+    "rev"
+  ];
+
+  outputs = { owner, repo, rev, ... }@inp: 
+    let
+      b = builtins;
+
+      githubMissingHashErrorText = pname: ''
+        Error: Cannot verify the integrity of the source of '${pname}'
+        It is a github reference with no hash providedand.
+        Solve this problem via any of the wollowing ways:
+
+        - (alternative 1): allow the use of builtin fetchers (which can verify using git rev).
+          ```
+            dream2nix.buildPackage {
+              ...
+              allowBuiltinFetchers = true;
+              ...
+            }
+          ```
+
+        - (alternative 2): add a hash to the source via override
+          ```
+            dream2nix.buildPackage {
+              ...
+              sourceOverrides = oldSources: {
+                "${pname}" = oldSources."${pname}".overrideAttrs (_:{
+                  hash = "";
+                })
+              }
+              ...
+            }
+          ```
+
+      '';
+    in
+    {
+
+      calcHash = algo: utils.hashPath algo (b.fetchTarball {
+        url = "https://github.com/${owner}/${repo}/tarball/${rev}";
+      });
+
+      fetched = hash:
+        if hash == null then
+          if allowBuiltinFetchers then
+            builtins.fetchGit {
+              inherit rev;
+              allRefs = true;
+              url = "https://github.com/${owner}/${repo}";
+            }
+          else
+            throw githubMissingHashErrorText (inp.pname or repo)
+        else
+          fetchFromGitHub {
+            inherit owner repo rev hash;
+          };
+
+    };
+}

src/fetchers/gitlab/default.nix

@@ -0,0 +1,33 @@
+{
+  fetchFromGitLab,
+
+  utils,
+  # config
+  allowBuiltinFetchers,
+  ...
+}:
+{
+
+  inputs = [
+    "owner"
+    "repo"
+    "rev"
+  ];
+
+  outputs = { owner, repo, rev, ... }@inp: 
+    let
+      b = builtins;
+    in
+    {
+
+      calcHash = algo: utils.hashPath algo (b.fetchTarball {
+        url = "https://gitlab.com/${owner}/${repo}/-/archive/${rev}/${repo}-${rev}.tar.gz";
+      });
+
+      fetched = hash:
+        fetchFromGitLab {
+          inherit owner repo rev hash;
+        };
+
+    };
+}

src/translators/default.nix

@@ -1,6 +1,6 @@
 {
   lib,
-  callPackage,
+  callPackageDream,
   pkgs,
 
   externalSources,
@@ -15,7 +15,7 @@ let
 
   callTranslator = subsystem: type: name: file: args: 
     let
-      translator = callPackage file (args // {
+      translator = callPackageDream file (args // {
         inherit externals;
         translatorName = name;
       });
@@ -35,16 +35,13 @@ let
       };
       
 
-  buildSystems = dirNames ./.;
+  buildSystems = utils.dirNames ./.;
 
   translatorTypes = [ "impure" "ifd" "pure" ];
 
   # every translator must provide 'bin/translate'
   translatorExec = translatorPkg: "${translatorPkg}/bin/translate";
 
-  # directory names of a given directory
-  dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
-
   # adds a translateBin to a pure translator
   wrapPureTranslator = translatorAttrPath:
     let
@@ -71,7 +68,7 @@ let
 
   # attrset of: subsystem -> translator-type -> (function subsystem translator-type)
   mkTranslatorsSet = function:
-    lib.genAttrs (dirNames ./.) (subsystem:
+    lib.genAttrs (utils.dirNames ./.) (subsystem:
       lib.genAttrs
         (lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) translatorTypes)
         (transType: function subsystem transType)
@@ -79,7 +76,7 @@ let
 
   # attrset of: subsystem -> translator-type -> translator
   translators = mkTranslatorsSet (subsystem: type:
-    lib.genAttrs (dirNames (./. + "/${subsystem}/${type}")) (translatorName:
+    lib.genAttrs (utils.dirNames (./. + "/${subsystem}/${type}")) (translatorName:
       callTranslator subsystem type translatorName (./. + "/${subsystem}/${type}/${translatorName}") {}
     )
   );

src/utils.nix

@@ -1,7 +1,13 @@
 {
   lib,
+  nix,
+  runCommand,
   ...
 }:
+let
+  b = builtins;
+in
+
 rec {
   basename = path: lib.last (lib.splitString "/" path);
 
@@ -13,6 +19,9 @@ rec {
 
   listFiles = path: lib.filterAttrs (n: v: v == "regular") (builtins.listDir path);
 
+  # directory names of a given directory
+  dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
+
   matchTopLevelFiles = pattern: path:
     # is dir
     if isDir path then
@@ -42,4 +51,13 @@ rec {
     else
       func args;
 
+  
+  hashPath = algo: path:
+    let
+      hashFile = runCommand "hash-${algo}" {} ''
+        ${nix}/bin/nix hash-path ${path} | tr --delete '\n' > $out
+      '';
+    in
+      b.readFile hashFile;
+
 }