Merge pull request #45 from DavHau/dev

Improve: CLI and falkes interfaces + document override system
2024-09-11 14:35:30 +03:00 · 2021-11-07 15:22:19 +07:00 · 2021-11-07 15:22:19 +07:00 · 71decc4946
commit 71decc4946
parent f1ab3e1b84 d56edddc3e
23 changed files with 649 additions and 322 deletions
--- a/README.md
+++ b/README.md
@ -30,4 +30,5 @@ The intention is to integrate many existing 2nix converters into this framework,

 - [Summary of the core concepts and benefits](/docs/concepts-and-benefits.md)
 - [How would this improve the packaging situation in nixpkgs](/docs/nixpkgs-improvements.md)
+- [Override System](/docs/override-system.md)
 - [Contributors Guide](/docs/contributors-guide.md)
--- a/docs/override-system.md
+++ b/docs/override-system.md
@ -0,0 +1,60 @@
+The override system plays an important role when packaging software with drem2nix. Overrides are the only way to express package specific logic in dream2nix. This serves the purpose of strictly separating:
+```
+  - generic logic     (builders)
+  - specific logic    (overrides)
+  - data              (dream-lock.json)
+```
+
+To optimize for scalalable workflows, the structure of dream2nix overrides differs from the ones seen in other projects.
+dream2nix overrides have the following properties:
+  - **referenceable**: each override is assigned to a key through which it can be referenced. This allows for better inspection, selective debugging, replacing, etc.
+  - **conditional**: each override can declare a condition, so that the override only applies when the condiiton evaluates positively.
+  - **attribute-oriented**: The relevant parameters are attributes, not override functions. dream2nix will automatically figure out which underlying function (eg. override, overrideAttrs, ...) needs to be called to update each given attribute. The user is not confronted with this by default.
+
+Each subsytem in dream2nix like `nodejs` or `python` manages its overrides in a separate directory to avoid package name collisions.
+
+dream2nix supports packaging different versions of the same package within one repository. Therefore conditions are used to make overrides apply only to certain package versions.
+
+Currently a collection of overrides is maintained at https://github.com/DavHau/dreampkgs
+
+Example for nodejs overrides:
+```nix
+{
+  # The name of a package.
+  # Contains all overrides which can apply to the package `enhanced-resolve`
+  enhanced-resolve = {
+
+    # first override for enhanced-resolve named `preserve-symlinks-v4`
+    preserve-symlinks-v4 = {
+
+      # override will apply for packages with major version 4
+      _condition = satisfiesSemver "^4.0.0";
+
+      # this statement replaces exisiting patches
+      # (for appending see next example)
+      patches = [
+        ./enhanced-resolve/npm-preserve-symlinks-v4.patch
+      ];
+
+    };
+
+    # second override for enhanced-resolve
+    preserve-symlinks-v5 = {
+
+      # override will apply for packages with major version 5
+      _condition = satisfiesSemver "^5.0.0";
+
+      # this statement adds a patch to the exsiting list of patches
+      patches = old: old ++ [
+        ./enhanced-resolve/npm-preserve-symlinks-v5.patch
+      ];
+    };
+
+  };
+
+  # another package name
+  webpack = {
+    # overrides for webpack
+  };
+}
+```
--- a/flake.lock
+++ b/flake.lock
@ -18,11 +18,11 @@
    "nix-parsec": {
      "flake": false,
      "locked": {
-        "lastModified": 1614150306,
-        "narHash": "sha256-h5fZgIhwvekg5WTNDdrV5EJ4zxXvlSqITmPVuz/OM+w=",
+        "lastModified": 1635533376,
+        "narHash": "sha256-/HrG0UPGnI5VdkhrNrpDiM2+nhdL6lD/bqyGtYv0QDE=",
        "owner": "nprindle",
        "repo": "nix-parsec",
-        "rev": "a28e73fce98227cc46edfdbb8518697c0982e034",
+        "rev": "1bf25dd9c5de1257a1c67de3c81c96d05e8beb5e",
        "type": "github"
      },
      "original": {
@ -62,12 +62,30 @@
        "type": "github"
      }
    },
+    "poetry2nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1632969109,
+        "narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=",
+        "owner": "nix-community",
+        "repo": "poetry2nix",
+        "rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "1.21.0",
+        "repo": "poetry2nix",
+        "type": "github"
+      }
+    },
    "root": {
      "inputs": {
        "mach-nix": "mach-nix",
        "nix-parsec": "nix-parsec",
        "nixpkgs": "nixpkgs",
-        "node2nix": "node2nix"
+        "node2nix": "node2nix",
+        "poetry2nix": "poetry2nix"
      }
    }
  },
--- a/flake.nix
+++ b/flake.nix
@ -12,9 +12,19 @@

    # required for builder nodejs/node2nix
    node2nix = { url = "github:svanderburg/node2nix"; flake = false; };
+
+    # required for utils.satisfiesSemver
+    poetry2nix = { url = "github:nix-community/poetry2nix/1.21.0"; flake = false; };
  };

-  outputs = { self, mach-nix, nix-parsec, nixpkgs, node2nix, }@inp:
+  outputs = {
+    self,
+    mach-nix,
+    nix-parsec,
+    nixpkgs,
+    node2nix,
+    poetry2nix,
+  }@inp:
    let

      b = builtins;
@ -46,24 +56,20 @@
        nix-parsec = [
          "parsec.nix"
          "lexer.nix"
+          "LICENSE"
+        ];
+        poetry2nix = [
+          "semver.nix"
+          "LICENSE"
        ];
      };

      # create a directory containing the files listed in externalPaths
-      makeExternalDir = pkgs: pkgs.runCommand "dream2nix-external" {}
-        (lib.concatStringsSep "\n"
-          (lib.mapAttrsToList
-            (inputName: paths:
-              lib.concatStringsSep "\n"
-                (lib.forEach
-                  paths
-                  (path: ''
-                    mkdir -p $out/${inputName}/$(dirname ${path})
-                    cp ${inp."${inputName}"}/${path} $out/${inputName}/${path}
-                  '')))
-            externalPaths));
+      makeExternalDir = import ./src/utils/external-dir.nix;

-      externalDirFor = forAllSystems (system: makeExternalDir);
+      externalDirFor = forAllSystems (system: pkgs: makeExternalDir {
+        inherit externalPaths externalSources pkgs;
+      });

      # An interface to access files of external projects.
      # This implementation aceeses the flake inputs directly,
@ -75,12 +81,15 @@
          (lib.attrNames externalPaths)
          (inputName: inp."${inputName}");

-      overridesDir = "${./overrides}";
+      overridesDirs =  [ "${./overrides}" ];

      # system specific dream2nix api
      dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
        externalDir = externalDirFor."${system}";
-        inherit externalSources lib overridesDir pkgs;
+        inherit externalSources lib pkgs;
+        config = {
+          inherit overridesDirs;
+        };
      });

    in
@ -97,7 +106,7 @@
        # Similar to drem2nixFor but will require 'system(s)' or 'pkgs' as an argument.
        # Produces flake-like output schema.
        lib = (import ./src/lib.nix {
-          inherit externalSources overridesDir lib;
+          inherit externalPaths externalSources lib;
          nixpkgsSrc = "${nixpkgs}";
        })
        # system specific dream2nix library
@ -106,23 +115,19 @@
            inherit
              externalSources
              lib
-              overridesDir
+              overridesDirs
              pkgs
            ;
          }
        ));

        # the dream2nix cli to be used with 'nix run dream2nix'
-        defaultApp = forAllSystems (system: pkgs: self.apps."${system}".cli);
+        defaultApp =
+          forAllSystems (system: pkgs: self.apps."${system}".dream2nix);

        # all apps including cli, install, etc.
        apps = forAllSystems (system: pkgs:
-          lib.mapAttrs (appName: app:
-            {
-              type = "app";
-              program = b.toString app.program;
-            }
-          ) dream2nixFor."${system}".apps.apps
+          dream2nixFor."${system}".apps.flakeApps
        );

        # a dev shell for working on dream2nix
@ -140,7 +145,7 @@
            export NIX_PATH=nixpkgs=${nixpkgs}
            export d2nExternalDir=${externalDirFor."${system}"}
            export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
-            export d2nOverridesDir=${./overrides}
+            export d2nOverridesDirs=${./overrides}

            echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
          '';
--- a/overrides/nodejs/default.nix
+++ b/overrides/nodejs/default.nix
@ -1,6 +1,10 @@
 {
  lib,
  pkgs,
+
+  # dream2nix
+  satisfiesSemver,
+  ...
 }:

 let
@ -8,106 +12,7 @@ let
 in

 {
-  degit = {
-    run-build = {
-      installScript = ''
-        npm run build
-        cp help.md ./dist
-      '';
-    };
-  };
-
-  esbuild = {
-    "add-binary-0.12.17" = {
-      _condition = pkg: pkg.version == "0.12.17";
-      ESBUILD_BINARY_PATH =
-        let
-          esbuild = pkgs.buildGoModule rec {
-            pname = "esbuild";
-            version = "0.12.17";
-
-            src = pkgs.fetchFromGitHub {
-              owner = "evanw";
-              repo = "esbuild";
-              rev = "v${version}";
-              sha256 = "sha256-wZOBjNOgGmwIQNCrhzwGPmI/fW/yZiDqq8l4oSDTvZs=";
-            };
-
-            vendorSha256 = "sha256-2ABWPqhK2Cf4ipQH7XvRrd+ZscJhYPc3SV2cGT0apdg=";
-          };
-        in
-          "${esbuild}/bin/esbuild";
-    };
-  };
-
-  geckodriver = {
-    add-binary = {
-      GECKODRIVER_FILEPATH = "${pkgs.geckodriver}/bin/geckodriver";
-    };
-  };
-
-  gifsicle = {
-    add-binary = {
-      installScript = ''
-        ln -s ${pkgs.gifsicle}/bin/gifsicle ./vendor/gifsicle
-        npm run postinstall
-      '';
-    };
-  };
-
-  mozjpeg = {
-    add-binary = {
-      installScript = ''
-        ln -s ${pkgs.mozjpeg}/bin/cjpeg ./vendor/cjpeg
-        npm run postinstall
-      '';
-    };
-  };
-
-  optipng-bin = {
-    add-binary = {
-      installScript = ''
-        ln -s ${pkgs.optipng}/bin/optipng ./vendor/optipng
-        npm run postinstall
-      '';
-    };
-  };
-
-  pngquant-bin = {
-    add-binary = {
-      installScript = ''
-        ln -s ${pkgs.pngquant}/bin/pngquant ./vendor/pngquant
-        npm run postinstall
-      '';
-    };
-  };
-
-  webpack = {
-    remove-webpack-cli-check = {
-      _condition = pkg: pkg.version == "5.41.1";
-      ignoreScripts = false;
-      patches = [
-        ./webpack/remove-webpack-cli-check.patch
-      ];
-    };
-  };
-
-  webpack-cli = {
-    remove-webpack-check = {
-      _condition = pkg: pkg.version == "4.7.2";
-      ignoreScripts = false;
-      patches = [
-        ./webpack-cli/remove-webpack-check.patch
-      ];
-    };
-  };
-
-  "@mattermost/webapp" = {
-    run-webpack = {
-      installScript = ''
-        NODE_ENV=production node --max-old-space-size=8192 ./node_modules/webpack/bin/webpack.js
-      '';
-    };
-  };
+  # this serves as a template
+  # overrides are currently maintained at https://github.com/DavHau/dreampkgs

 }
--- a/overrides/nodejs/webpack-cli/remove-webpack-check.patch
+++ b/overrides/nodejs/webpack-cli/remove-webpack-check.patch
@ -1,12 +0,0 @@
-diff --git a/packages/webpack-cli/bin/cli.js b/packages/webpack-cli/bin/cli.js
--- a/bin/cli.js
-+++ b/bin/cli.js
-@@ -21,7 +21,7 @@ if (!process.env.WEBPACK_CLI_SKIP_IMPORT_LOCAL) {
- 
- process.title = "webpack";
- 
-if (utils.packageExists("webpack")) {
-+if (true) {
-     runCLI(process.argv, originalModuleCompile);
- } else {
-     const { promptInstallation, logger, colors } = utils;
--- a/overrides/nodejs/webpack/remove-webpack-cli-check.patch
+++ b/overrides/nodejs/webpack/remove-webpack-cli-check.patch
@ -1,12 +0,0 @@
-diff --git a/bin/webpack.js b/bin/webpack.js
--- a/bin/webpack.js
-+++ b/bin/webpack.js
-@@ -87,7 +87,7 @@ const cli = {
- 	url: "https://github.com/webpack/webpack-cli"
- };
- 
-if (!cli.installed) {
-+if (false) {
- 	const path = require("path");
- 	const fs = require("graceful-fs");
- 	const readLine = require("readline");
--- a/src/apps/cli/cli.py
+++ b/src/apps/cli/cli.py
@ -1,10 +1,10 @@
 from cleo import Application

-from commands.package import PackageCommand
+from commands.add import AddCommand
 from commands.update import UpdateCommand

-application = Application("package")
-application.add(PackageCommand())
+application = Application("dream2nix")
+application.add(AddCommand())
 application.add(UpdateCommand())

 if __name__ == '__main__':
--- a/src/apps/cli/commands/package.py
+++ b/src/apps/cli/commands/package.py
@ -6,30 +6,35 @@ import sys
 import tempfile

 import networkx as nx
-from cleo import Command, option
+from cleo import Command, argument, option

-from utils import dream2nix_src, checkLockJSON, callNixFunction, buildNixFunction, buildNixAttribute, \
+from utils import config, dream2nix_src, checkLockJSON, callNixFunction, buildNixFunction, buildNixAttribute, \
  list_translators_for_source, strip_hashes_from_lock


-class PackageCommand(Command):
+class AddCommand(Command):

  description = (
-    "Package a software project using nix"
+    f"Add a package to {config['repoName']}"
  )

-  name = "package"
+  name = "add"
+
+  arguments = [
+    argument(
+      "source",
+      "source of the package, can be a path, tarball URL, or flake-style spec")
+  ]

  options = [
-    option(
-      "source",
-      None,
-      "source of the package, can be a path or flake-like spec",
-      flag=False,
-      multiple=False
-    ),
    option("translator", None, "which translator to use", flag=False),
-    option("output", None, "output file/directory for the dream-lock.json", flag=False),
+    option("target", None, "target file/directory for the dream-lock.json", flag=False),
+    option(
+      "--packages-root",
+      None,
+      "Put package under a new directory inside packages-root",
+      flag=False
+    ),
    option(
      "combined",
      None,
@ -45,7 +50,7 @@ class PackageCommand(Command):
      multiple=True
    ),
    option("force", None, "override existing files", flag=True),
-    option("default-nix", None, "create default.nix", flag=True),
+    option("no-default-nix", None, "create default.nix", flag=True),
  ]

  def handle(self):
@ -60,40 +65,32 @@ class PackageCommand(Command):
      )
    }

-    # ensure output directory
-    output = self.option("output")
-    if not output:
-      output = './.'
-    if not os.path.isdir(output):
-      os.mkdir(output)
-    filesToCreate = ['dream-lock.json']
-    if self.option('default-nix'):
-      filesToCreate.append('default.nix')
-    if self.option('force'):
-      for f in filesToCreate:
-        if os.path.isfile(f):
-          os.remove(f)
+    # ensure packages-root
+    if self.option("packages-root"):
+      packages_root = self.option("packages-root")
+    elif config['packagesDir']:
+      packages_root = config['packagesDir']
    else:
-      existingFiles = set(os.listdir(output))
-      if any(f in existingFiles for f in filesToCreate):
-        print(
-          f"output directory {output} already contains a 'default.nix' "
-          "or 'dream-lock.json'. Delete first, or user '--force'.",
-          file=sys.stderr,
-        )
-        exit(1)
-    output = os.path.realpath(output)
-    outputDreamLock = f"{output}/dream-lock.json"
-    outputDefaultNix = f"{output}/default.nix"
+      packages_root = './.'
+    if not os.path.isdir(packages_root):
+      print(
+        f"Packages direcotry {packages_root} does not exist. Please create.",
+        file = sys.stderr,
+      )

    # verify source
-    source = self.option("source")
-    if not source:
+    source = self.argument("source")
+    if not source and not config['packagesDir']:
      source = os.path.realpath('./.')
      print(
        f"Source not specified. Defaulting to current directory: {source}",
        file=sys.stderr,
      )
+    # else:
+    #   print(
+    #     f"Source not specified. Defaulting to current directory: {source}",
+    #     file=sys.stderr,
+    #   )
    # check if source is valid fetcher spec
    sourceSpec = {}
    # handle source shortcuts
@ -228,38 +225,102 @@ class PackageCommand(Command):
              if specified_extra_args[arg_name]:
                break

-    # arguments for calling the translator nix module
-    translator_input = dict(
-      inputFiles=[],
-      inputDirectories=[source],
-      outputFile=outputDreamLock,
-    )
-    translator_input.update(specified_extra_args)
-
    # build the translator bin
    t = translator
    translator_path = buildNixAttribute(
      f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
    )

-    # dump translator arguments to json file and execute translator
-    print("\nTranslating upstream metadata")
-    with tempfile.NamedTemporaryFile("w") as input_json_file:
-      json.dump(translator_input, input_json_file, indent=2)
-      input_json_file.seek(0) # flushes write cache
+    # direct outputs of translator to temporary file
+    with tempfile.NamedTemporaryFile("r") as output_temp_file:

-      # execute translator
-      sp.run(
-        [f"{translator_path}/bin/run", input_json_file.name]
+      # arguments for calling the translator nix module
+      translator_input = dict(
+        inputFiles=[],
+        inputDirectories=[source],
+        outputFile=output_temp_file.name,
      )
+      translator_input.update(specified_extra_args)

-    # raise error if output wasn't produced
-    if not os.path.isfile(outputDreamLock):
-      raise Exception(f"Translator failed to create dream-lock.json")
+      # dump translator arguments to json file and execute translator
+      print("\nTranslating project metadata")
+      with tempfile.NamedTemporaryFile("w") as input_json_file:
+        json.dump(translator_input, input_json_file, indent=2)
+        input_json_file.seek(0) # flushes write cache

-    # read produced lock file
-    with open(outputDreamLock) as f:
-      lock = json.load(f)
+        # execute translator
+        sp.run(
+          [f"{translator_path}/bin/run", input_json_file.name]
+        )
+
+      # raise error if output wasn't produced
+      if not output_temp_file.read():
+        raise Exception(f"Translator failed to create dream-lock.json")
+
+      # read produced lock file
+      with open(output_temp_file.name) as f:
+        lock = json.load(f)
+
+
+    # get package name and version from lock
+    mainPackageName = lock['_generic']['mainPackageName']
+    mainPackageVersion = lock['_generic']['mainPackageVersion']
+
+    # calculate output directory
+    mainPackageDirName = mainPackageName.strip('@').replace('/', '-')
+
+    # verify / change main package dir name
+    def update_name(mainPackageDirName):
+      print(f"Current package attribute name is: {mainPackageDirName}")
+      new_name = self.ask(
+        "Specify new attribute name or leave empty to keep current:"
+      )
+      if new_name:
+        return new_name
+      return mainPackageDirName
+
+    mainPackageDirName = update_name(mainPackageDirName)
+
+    if self.option('target'):
+      if self.option('target').startswith('/'):
+        output = self.option('target')
+      else:
+        output = f"{packages_root}/{self.option('target')}"
+    else:
+      output = f"{packages_root}/{mainPackageDirName}"
+
+    # collect files to create
+    filesToCreate = ['dream-lock.json']
+    if not os.path.isdir(output):
+      os.mkdir(output)
+    existingFiles = set(os.listdir(output))
+    if not self.option('no-default-nix')\
+        and not 'default.nix' in existingFiles\
+        and not config['packagesDir']:
+      if self.confirm(
+          'Create a default.nix for debugging purposes',
+          default=True):
+        filesToCreate.append('default.nix')
+
+    # overwrite existing files only if --force is set
+    if self.option('force'):
+      for f in filesToCreate:
+        if os.path.isfile(f):
+          os.remove(f)
+    # raise error if any file exists already
+    else:
+      if any(f in existingFiles for f in filesToCreate):
+        print(
+          f"output directory {output} already contains a 'default.nix' "
+          "or 'dream-lock.json'. Resolve via one of these:\n"
+          "  - use --force to overwrite files\n"
+          "  - use --target to specify another target dir",
+          file=sys.stderr,
+        )
+        exit(1)
+    output = os.path.realpath(output)
+    outputDreamLock = f"{output}/dream-lock.json"
+    outputDefaultNix = f"{output}/default.nix"

    # write translator information to lock file
    combined = self.option('combined')
@ -274,8 +335,6 @@ class PackageCommand(Command):
    ])

    # add main package source
-    mainPackageName = lock['_generic']['mainPackageName']
-    mainPackageVersion = lock['_generic']['mainPackageVersion']
    mainSource = sourceSpec.copy()
    if not mainSource:
      mainSource = dict(
@ -384,7 +443,7 @@ class PackageCommand(Command):
      sourcePathRelative = os.path.relpath(source, os.path.dirname(outputDefaultNix))
    )
    # with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
-    if self.option('default-nix'):
+    if 'default.nix' in filesToCreate:
      with open(outputDefaultNix, 'w') as defaultNix:
        defaultNix.write(template)
        print(f"Created {output}/default.nix")
--- a/src/apps/cli/commands/update.py
+++ b/src/apps/cli/commands/update.py
@ -4,31 +4,42 @@ import subprocess as sp
 import sys
 import tempfile

-from cleo import Command, option
+from cleo import Command, argument, option

-from utils import buildNixFunction, callNixFunction
+from utils import config, buildNixFunction, callNixFunction


 class UpdateCommand(Command):
  description = (
-    "Update an existing dream2nix based package"
+    f"Update an existing package in {config['repoName']}"
  )

  name = "update"

+  arguments = [
+    argument(
+      "name",
+      "name of the package or path containing a dream-lock.json",
+    ),
+  ]
+
  options = [
-    option("dream-lock", None, "dream-lock.json file or its parent directory", flag=False, value_required=True),
    option("updater", None, "name of the updater module to use", flag=False),
-    option("new-version", None, "new package version", flag=False),
+    option("to-version", None, "target package version", flag=False),
  ]

  def handle(self):
    if self.io.is_interactive():
      self.line(f"\n{self.description}\n")

-    dreamLockFile = os.path.abspath(self.option("dream-lock"))
-    if not dreamLockFile.endswith('dream-lock.json'):
-      dreamLockFile = os.path.abspath(dreamLockFile + "/dream-lock.json")
+    if config['packagesDir'] and '/' not in self.argument("name"):
+      dreamLockFile =\
+        os.path.abspath(
+          f"{config['packagesDir']}/{self.argument('name')}/dream-lock.json")
+    else:
+      dreamLockFile = os.path.abspath(self.argument("name"))
+      if not dreamLockFile.endswith('dream-lock.json'):
+        dreamLockFile = os.path.abspath(dreamLockFile + "/dream-lock.json")

    # parse dream lock
    with open(dreamLockFile) as f:
@ -44,9 +55,11 @@ class UpdateCommand(Command):
          file=sys.stderr,
        )
        exit(1)
+    print(f"updater module is: {updater}")

    # find new version
-    version = self.option('new-version')
+    old_version = lock['_generic']['mainPackageVersion']
+    version = self.option('to-version')
    if not version:
      update_script = buildNixFunction(
        "updaters.makeUpdateScript",
@ -55,7 +68,7 @@ class UpdateCommand(Command):
      )
      update_proc = sp.run([f"{update_script}/bin/run"], capture_output=True)
      version = update_proc.stdout.decode().strip()
-    print(f"Updating to version {version}")
+    print(f"Updating from version {old_version} to {version}")

    cli_py = os.path.abspath(f"{__file__}/../../cli.py")
    # delete the hash
@ -73,8 +86,8 @@ class UpdateCommand(Command):
      tmpDreamLock.seek(0)  # flushes write cache
      sp.run(
        [
-          sys.executable, f"{cli_py}", "package", "--force", "--source", tmpDreamLock.name,
-          "--output", os.path.abspath(os.path.dirname(dreamLockFile))
+          sys.executable, f"{cli_py}", "add", tmpDreamLock.name, "--force",
+          "--target", os.path.abspath(os.path.dirname(dreamLockFile))
        ]
        + lock['_generic']['translatorParams'].split()
      )
--- a/src/apps/cli/default.nix
+++ b/src/apps/cli/default.nix
@ -1,13 +1,16 @@
 {
  # from dream2nix
+  configFile,
  dream2nixWithExternals,
  fetchers,
+  nix,
  translators,
+  utils,

  # from nixpkgs
+  gitMinimal,
  lib,
  python3,
-  writeScript,
  ...
 }:

@ -19,11 +22,25 @@ let

 in
 {
-  program = writeScript "cli" ''
-    dream2nixSrc=${dream2nixWithExternals} \
-    fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
-      ${cliPython}/bin/python ${./.}/cli.py "$@"
-  '';
+  program = 
+    let
+      script = utils.writePureShellScript
+        [
+          gitMinimal
+          nix
+        ]
+        ''
+          # escape the temp dir created by writePureShellScript
+          cd - > /dev/null
+
+          # run the cli
+          dream2nixConfig=${configFile} \
+          dream2nixSrc=${dream2nixWithExternals} \
+          fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
+            ${cliPython}/bin/python ${./.}/cli.py "$@"
+        '';
+    in
+      "${script}/bin/run";

  templateDefaultNix =
    {
--- a/src/apps/cli/utils.py
+++ b/src/apps/cli/utils.py
@ -8,6 +8,25 @@ from jsonschema import validate

 dream2nix_src = os.environ.get("dream2nixSrc")

+def find_repo_root():
+  proc = sp.run(
+    ['git', 'rev-parse', '--show-toplevel'],
+    capture_output=True,
+  )
+  if proc.returncode:
+    print(proc.stderr.decode(), file=sys.stderr)
+    print(
+      f"\nProbably not inside git repo {config['repoName']}\n"
+      f"Please clone the repo first.",
+      file=sys.stderr
+    )
+    exit(1)
+  return proc.stdout.decode().strip()
+
+with open(os.environ.get("dream2nixConfig")) as f:
+  config = json.load(f)
+  if config['repoName'] and config ['packagesDir']:
+    config['packagesDir'] = f"{find_repo_root()}/{config['packagesDir']}"

 def checkLockJSON(lock):
  lock_schema_raw=open(dream2nix_src+"/specifications/dream-lock-schema.json").read()
--- a/src/apps/default.nix
+++ b/src/apps/default.nix
@ -1,19 +1,37 @@
 {
+  lib,
  pkgs,

+  # dream2nix
  callPackageDream,
  translators,
  ...
 }:
-rec {
-  apps = { inherit cli contribute install; };

-  # the unified translator cli
+let
+  b = builtins;
+in
+
+rec {
+  apps = {
+    inherit cli contribute install;
+    dream2nix = cli;
+  };
+
+  flakeApps =
+    lib.mapAttrs (appName: app:
+      {
+        type = "app";
+        program = b.toString app.program;
+      }
+    ) apps;
+
+  # the dream2nix cli
  cli = callPackageDream (import ./cli) {};

  # the contribute cli
  contribute = callPackageDream (import ./contribute) {};

-  # install the framework to a specified location by copying the code
+  # instrall the framework to a specified location by copying the code
  install = callPackageDream (import ./install) {};
 }
--- a/src/builders/nodejs/granular/default.nix
+++ b/src/builders/nodejs/granular/default.nix
@ -123,17 +123,11 @@ let
          # prevents running into ulimits
          passAsFile = [ "dependenciesJson" "nodeDeps" ];

-          preBuildPhases = [ "d2nPatchPhase" "d2nInstallDependenciesPhase" ];
-
-          preFixupPhases = [ "d2nPostInstallPhase" ];
-
-          # not used by default but can be enabled if needed
-          dontConfigure = true;
-          dontBuild = true;
+          preConfigurePhases = [ "d2nPatchPhase" ];

          # can be overridden to define alternative install command
          # (defaults to 'npm run postinstall')
-          installScript = null;
+          buildScript = null;

          # python script to modify some metadata to support installation
          # (see comments below on d2nPatchPhase)
@ -192,9 +186,8 @@ let
          # The python script wich is executed in this phase:
          #   - ensures that the package is compatible to the current system
          #   - ensures the main version in package.json matches the expected
-          #   - deletes "devDependencies" and "peerDependencies" from package.json
-          #     (might block npm install in case npm install is used)
          #   - pins dependency versions in package.json
+          #     (some npm commands might otherwise trigger networking)
          #   - creates symlinks for executables declared in package.json
          # Apart from that:
          #   - Any usage of 'link:' in package.json is replaced with 'file:'
@ -209,7 +202,7 @@ let
            cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
            rm $nodeModules/$packageName/package.json.old

-            # run python script (see commend above):
+            # run python script (see comment above):
            cp package.json package.json.bak
            python $fixPackage \
            || \
@ -225,11 +218,14 @@ let
          '';

          # - links all direct node dependencies into the node_modules directory
-          # - adds executables of direct node dependencies to PATH
+          # - adds executables of direct node module dependencies to PATH
          # - adds the current node module to NODE_PATH
          # - sets HOME=$TMPDIR, as this is required by some npm scripts
          # TODO: don't install dev dependencies. Load into NODE_PATH instead
-          d2nInstallDependenciesPhase = ''
+          # TODO: move all linking to python script, as `ln` calls perform badly
+          configurePhase = ''
+            runHook preConfigure
+
            # symlink dependency packages into node_modules
            for dep in $(cat $nodeDepsPath); do
              # add bin to PATH
@ -254,34 +250,41 @@ let
              fi
            done

+            # symlink sub dependencies as well as this imitates npm better
+            python ${./symlink-deps.py}
+
+            # add dependencies to NODE_PATH
            export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"

            export HOME=$TMPDIR
+
+            runHook postConfigure
          '';

-          # Run the install command which defaults to 'npm run postinstall'.
-          # Allows using custom install command by overriding 'installScript'.
-          installPhase = ''
-            runHook preInstall
+          # Runs the install command which defaults to 'npm run postinstall'.
+          # Allows using custom install command by overriding 'buildScript'.
+          buildPhase = ''
+            runHook preBuild

            # execute install command
-            if [ -n "$installScript" ]; then
-              if [ -f "$installScript" ]; then
-                exec $installScript
+            if [ -n "$buildScript" ]; then
+              if [ -f "$buildScript" ]; then
+                exec $buildScript
              else
-                echo "$installScript" | bash
+                echo "$buildScript" | bash
              fi
+            # by default, only for top level packages, `npm run build` is executed
            elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
              npm run build
            elif [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
              npm --production --offline --nodedir=$nodeSources run postinstall
            fi

-            runHook postInstall
+            runHook postBuild
          '';

          # Symlinks executables and manual pages to correct directories
-          d2nPostInstallPhase = ''
+          installPhase = ''
            
            echo "Symlinking exectuables to /bin"
            if [ -d "$nodeModules/.bin" ]
--- a/src/builders/nodejs/granular/symlink-deps.py
+++ b/src/builders/nodejs/granular/symlink-deps.py
@ -0,0 +1,69 @@
+import json
+import os
+import pathlib
+import sys
+
+
+out = os.environ.get('out')
+pname = os.environ.get('packageName')
+version = os.environ.get('version')
+root = f"{out}/lib/node_modules/{pname}/node_modules"
+
+if not os.path.isdir(root):
+  exit()
+
+
+with open(os.environ.get("nodeDepsPath")) as f:
+  nodeDeps = f.read().split()
+
+def getDependencies(root, depth):
+  if not os.path.isdir(root):
+    return []
+  dirs = os.listdir(root)
+
+  currentDeps = []
+  for d in dirs:
+    if d.rpartition('/')[-1].startswith('@'):
+      subdirs = os.listdir(f"{root}/{d}")
+      for sd in subdirs:
+        cur_dir = f"{root}/{d}/{sd}"
+        currentDeps.append(f"{cur_dir}")
+    else:
+      cur_dir = f"{root}/{d}"
+      currentDeps.append(cur_dir)
+  
+  if depth == 0:
+    return currentDeps
+  else:
+    depsOfDeps =\
+      map(lambda dep: getDependencies(f"{dep}/node_modules", depth - 1), currentDeps)
+    result = []
+    for deps in depsOfDeps:
+      result += deps
+    return result
+
+deps = getDependencies(root, 1)
+
+# symlink deps non-colliding deps
+for dep in deps:
+
+  # compute module path
+  d1, d2 = dep.split('/')[-2:]
+  if d1.startswith('@'):
+    path = f"{root}/{d1}/{d2}"
+  else:
+    path = f"{root}/{d2}"
+
+  # check for collision
+  if os.path.isdir(path):
+    continue
+
+  # create parent dir
+  pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
+
+  # symlink dependency
+  os.symlink(dep, path)
+
+  
+
+  
--- a/src/config.json
+++ b/src/config.json
@ -1,3 +0,0 @@
-{
-
-}
--- a/src/default.nix
+++ b/src/default.nix
@ -9,42 +9,58 @@

  # the dream2nix cli depends on some nix 2.4 features
  nix ? pkgs.writeScriptBin "nix" ''
+    #!${pkgs.bash}/bin/bash
    ${pkgs.nixUnstable}/bin/nix --option experimental-features "nix-command flakes" "$@"
  '',

+  # default to empty dream2nix config
+  config ?
+    # if called via CLI, load cnfig via env
+    if builtins ? getEnv && builtins.getEnv "d2nConfigFile" != "" then
+      builtins.toPath (builtins.getEnv "d2nConfigFile")
+    # load from default directory
+    else
+      {},
+
  # dependencies of dream2nix
  externalSources ?
    lib.genAttrs
      (lib.attrNames (builtins.readDir externalDir))
      (inputName: "${externalDir}/${inputName}"),

+  # will be defined if called via flake
+  externalPaths ? null,
+
  # required for non-flake mode
  externalDir ?
+    # if flake is used, construct external dir from flake inputs
+    if externalPaths != null then
+      (import ./utils/external-dir.nix {
+        inherit externalPaths externalSources pkgs;
+      })
    # if called via CLI, load externals via env
-    if builtins ? getEnv && builtins.getEnv "d2nExternalDir" != "" then
+    else if builtins ? getEnv && builtins.getEnv "d2nExternalDir" != "" then
      builtins.getEnv "d2nExternalDir"
    # load from default directory
    else
      ./external,

-  # dream2nix overrides
-  overridesDir ?
-     # if called via CLI, load externals via env
-    if builtins ? getEnv && builtins.getEnv "d2nOverridesDir" != "" then
-      builtins.getEnv "d2nOverridesDir"
-    # load from default directory
-    else
-      ./overrides,
-}:
+}@args:

 let

  b = builtins;

+  config = (import ./utils/config.nix).loadConfig args.config or {};
+
+  configFile = pkgs.writeText "dream2nix-config.json" (b.toJSON config);
+
  # like pkgs.callPackage, but includes all the dream2nix modules
  callPackageDream = f: args: pkgs.callPackage f (args // {
    inherit builders;
    inherit callPackageDream;
+    inherit config;
+    inherit configFile;
    inherit externals;
    inherit externalSources;
    inherit fetchers;
@ -57,8 +73,6 @@ let

  utils = callPackageDream ./utils {};

-  config = builtins.fromJSON (builtins.readFile ./config.json);
-
  # apps for CLI and installation
  apps = callPackageDream ./apps {};

@ -75,18 +89,20 @@ let
  translators = callPackageDream ./translators {};

  externals = {
-    node2nix = nodejs: pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" { inherit nodejs; };
+    node2nix = nodejs:
+      pkgs.callPackage "${externalSources.node2nix}/nix/node-env.nix" {
+        inherit nodejs;
+      };
    nix-parsec = rec {
-      lexer = import "${externalSources.nix-parsec}/lexer.nix" { inherit parsec; };
+      lexer = import "${externalSources.nix-parsec}/lexer.nix" {
+        inherit parsec;
+      };
      parsec = import "${externalSources.nix-parsec}/parsec.nix";
    };
  };

-  dreamOverrides = lib.genAttrs (utils.dirNames overridesDir) (name:
-    import (overridesDir + "/${name}") {
-      inherit lib pkgs;
-    }
-  );
+  dreamOverrides =
+    utils.loadOverridesDirs config.overridesDirs pkgs;

  # the location of the dream2nix framework for self references (update scripts, etc.)
  dream2nixWithExternals =
--- a/src/lib.nix
+++ b/src/lib.nix
@ -6,15 +6,21 @@
  nixpkgsSrc ? <nixpkgs>,
  lib ? (import nixpkgsSrc {}).lib,

-  # (if called impurely ./default.nix will handle externals and overrides)
-  externalSources ? null,
-  overridesDir ? null,
-}:
+  externalSources,
+
+  externalPaths,
+
+}@args:

 let

  b = builtins;

+  dream2nixForSystem = config: system: pkgs:
+    import ./default.nix
+      { inherit config externalPaths externalSources pkgs; };
+
+
  # TODO: design output schema for cross compiled packages
  makePkgsKey = pkgs:
    let
@ -62,9 +68,57 @@ let
        devShell."${system}" = outputs.devShell;
      });

-in
+  init =
+    {
+      pkgs ? null,
+      systems ? [],
+      config ? {},
+    }@argsInit:
+    let
+
+      config = (import ./utils/config.nix).loadConfig argsInit.config or {};
+
+      overridesDirs' = config.overridesDirs;
+
+      allPkgs = makeNixpkgs pkgs systems;
+
+      forAllSystems = f:
+        lib.mapAttrs f allPkgs;
+
+      dream2nixFor = forAllSystems (dream2nixForSystem config);
+    in
+      {
+        riseAndShine = riseAndShineArgs:
+          let
+            allBuilderOutputs =
+              lib.mapAttrs
+                (system: pkgs:
+                  dream2nixFor."${system}".riseAndShine riseAndShineArgs)
+                allPkgs;
+
+            flakifiedOutputs =
+              lib.mapAttrsToList
+                (system: outputs: flakifyBuilderOutputs system outputs)
+                allBuilderOutputs;
+
+          in
+            b.foldl'
+              (allOutputs: output: lib.recursiveUpdate allOutputs output)
+              {}
+              flakifiedOutputs;
+
+        apps =
+          forAllSystems
+            (system: pkgs:
+              dream2nixFor."${system}".apps.flakeApps);
+              
+        defaultApp =
+          forAllSystems
+            (system: pkgs:
+              dream2nixFor."${system}".apps.flakeApps.dream2nix);
+        
+      };

-{
  riseAndShine = 
    {
      pkgs ? null,
@ -78,17 +132,7 @@ in
      allPkgs = makeNixpkgs pkgs systems;

      dream2nixFor =
-        lib.mapAttrs
-          (system: pkgs:
-            import ./default.nix
-              ({ inherit pkgs; }
-              // (lib.optionalAttrs (externalSources != null) {
-                inherit externalSources;
-              })
-              // (lib.optionalAttrs (overridesDir != null) {
-                inherit overridesDir;
-              })))
-          allPkgs;
+        lib.mapAttrs (dream2nixForSystem {}) allPkgs;

      allBuilderOutputs =
        lib.mapAttrs
@ -106,4 +150,8 @@ in
        (allOutputs: output: lib.recursiveUpdate allOutputs output)
        {}
        flakifiedOutputs;
+
+in
+{
+  inherit init riseAndShine;
 }
--- a/src/utils/config.nix
+++ b/src/utils/config.nix
@ -0,0 +1,34 @@
+let
+
+  b = builtins;
+
+  # loads attrs either from s:
+  #   - json file
+  #   - json string
+  #   - attrset (no changes)
+  loadAttrs = input:
+    if b.isPath input then
+      b.fromJSON (b.readFile input)
+    else if b.isString input then
+      b.fromJSON input
+    else if b.isAttrs input then
+      input
+    else
+      throw "input for loadAttrs must be json file or string or attrs";
+  
+  # load dream2nix config extending with defaults
+  loadConfig = configInput:
+    let
+      config = loadAttrs configInput;
+      defaults =  {
+        overridesDirs = [];
+        packagesDir = null;
+        repoName = "this repo";
+      };
+    in
+      defaults // config;
+
+in
+{
+  inherit loadConfig;
+}
--- a/src/utils/default.nix
+++ b/src/utils/default.nix
@ -9,6 +9,7 @@

  # dream2nix inputs
  callPackageDream,
+  externalSources,
  ...
 }:
 let
@ -23,6 +24,16 @@ let

  translatorUtils = callPackageDream ./translator.nix {};

+  poetry2nixSemver = import "${externalSources.poetry2nix}/semver.nix" {
+    inherit lib;
+    # copied from poetry2nix
+    ireplace = idx: value: list: (
+      lib.genList
+        (i: if i == idx then value else (b.elemAt list i))
+        (b.length list)
+    );
+  };
+
 in

 parseUtils
@ -152,4 +163,10 @@ rec {
        (v1: v2: versionGreater v1 v2)
        versions);

+  satisfiesSemver = poetry2nixSemver.satisfiesSemver;
+
+  # like nixpkgs recursiveUpdateUntil, but the depth of the 
+  recursiveUpdateUntilDepth = depth: lhs: rhs:
+    lib.recursiveUpdateUntil (path: l: r: (b.length path) > depth) lhs rhs;
+
 }
--- a/src/utils/dream-lock.nix
+++ b/src/utils/dream-lock.nix
@ -120,7 +120,7 @@ let
          lib.zipAttrsWith
            (name: versions:
              lib.zipAttrsWith
-                (version: deps: lib.flatten deps)
+                (version: deps: lib.unique (lib.flatten deps))
                versions)
            [
              oldDependencyGraph
--- a/src/utils/external-dir.nix
+++ b/src/utils/external-dir.nix
@ -0,0 +1,19 @@
+{
+  lib ? pkgs.lib,
+  pkgs,
+  externalSources,
+  externalPaths,
+}:
+
+pkgs.runCommand "dream2nix-external-dir" {}
+  (lib.concatStringsSep "\n"
+    (lib.mapAttrsToList
+      (inputName: paths:
+        lib.concatStringsSep "\n"
+          (lib.forEach
+            paths
+            (path: ''
+              mkdir -p $out/${inputName}/$(dirname ${path})
+              cp ${externalSources."${inputName}"}/${path} $out/${inputName}/${path}
+            '')))
+      externalPaths))
--- a/src/utils/override.nix
+++ b/src/utils/override.nix
@ -1,11 +1,30 @@
 {
  lib,
+
+  # dream2nix
+  utils,
  ...
 }:
 let

  b = builtins;

+  loadOverridesDirs = overridesDirs: pkgs:
+    let
+      loadOverrides = dir:
+        lib.genAttrs (utils.dirNames dir) (name:
+          import (dir + "/${name}") {
+            inherit lib pkgs;
+            satisfiesSemver = constraint: pkg:
+              utils.satisfiesSemver pkg.version constraint;
+          });
+    in
+      b.foldl'
+        (loaded: nextDir:
+          utils.recursiveUpdateUntilDepth 3 loaded (loadOverrides nextDir))
+        {}
+        overridesDirs;
+
  throwErrorUnclearAttributeOverride = pname: overrideName: attrName:
    throw ''
      Error while applying override for ${pname}: '${overrideName}'
@ -44,21 +63,22 @@ let

        # if condition is unset, it will be assumed true
        evalCondition = condOverride: pkg:
-          if condOverride ? _condition then
-            condOverride._condition pkg
-          else
-            true;
+            if condOverride ? _condition then
+              condOverride._condition pkg
+            else
+              true;

        # filter the overrides by the package name and conditions
        overridesToApply =
          let
-            regexOverrides =
-              lib.filterAttrs
-                (name: data:
-                  lib.hasPrefix "^" name
-                  &&
-                  b.match name pname != null)
-                conditionalOverrides;
+            # TODO: figure out if regex names will be useful
+            regexOverrides = {};
+              # lib.filterAttrs
+              #   (name: data:
+              #     lib.hasPrefix "^" name
+              #     &&
+              #     b.match name pname != null)
+              #   conditionalOverrides;

            overridesForPackage =
              b.foldl'
@ -86,6 +106,19 @@ let
        # the condition is not evaluated anymore here
        applyOneOverride = pkg: condOverride:
          let
+            base_derivation =
+              if condOverride ? _replace then
+                if lib.isFunction condOverride._replace then
+                  condOverride._replace pkg
+                else if lib.isDerivation condOverride._replace then
+                  condOverride._replace
+                else
+                  throw
+                    ("override attr ${pname}.${condOverride._name}._replace"
+                    + " must either be a derivation or a function")
+              else
+                pkg;
+
            overrideFuncs =
              lib.mapAttrsToList
                (funcName: func: { inherit funcName func; })
@ -98,7 +131,7 @@ let
                    (funcName: func: lib.attrNames (lib.functionArgs func))
                    (lib.filterAttrs
                      (funcName: func: lib.hasPrefix "override" funcName)
-                      pkg);
+                      base_derivation);

                getOverrideFuncNameForAttrName = attrName:
                  let
@ -140,7 +173,7 @@ let
                      (attrName: functionOrValue:
                        applySingleAttributeOverride old."${attrName}" functionOrValue)
                      updateAttrsFuncs))
-              pkg
+              base_derivation
              (overrideFuncs ++ singleArgOverrideFuncs);
      in
        # apply the overrides to the given pkg
@ -151,5 +184,5 @@ let

 in
 {
-  inherit applyOverridesToPackage;
+  inherit applyOverridesToPackage loadOverridesDirs;
 }