From d3a279875e661caf25881b84883406531a79d7db Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 2 Dec 2021 14:15:40 +0700 Subject: [PATCH] feature: support multiple packages per source --- flake.nix | 18 +++++- src/apps/cli/commands/add.py | 42 +++++++++----- src/apps/cli/default.nix | 2 +- src/builders/nodejs/granular/default.nix | 24 ++++++-- src/default.nix | 56 ++++++++++++++----- .../nodejs/pure/package-lock/default.nix | 2 - .../nodejs/pure/yarn-lock/default.nix | 6 +- src/utils/config.nix | 4 +- src/utils/dream-lock.nix | 41 ++++++++++++-- src/utils/override.nix | 12 +++- 10 files changed, 153 insertions(+), 54 deletions(-) diff --git a/flake.nix b/flake.nix index 5d9ab8be..45e0e9b6 100644 --- a/flake.nix +++ b/flake.nix @@ -158,9 +158,23 @@ export NIX_PATH=nixpkgs=${nixpkgs} export d2nExternalDir=${externalDirFor."${system}"} export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals} - export d2nOverridesDirs=${./overrides} - echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'" + if [ -e ./overrides ]; then + export d2nOverridesDir=$(realpath ./overrides) + else + export d2nOverridesDir=${./overrides} + echo -e "\nManually execute 'export d2nOverridesDir={path to your dream2nix overrides dir}'" + fi + + if [ -e ../dream2nix ]; then + export dream2nixWithExternals=$(realpath ./src) + else + export dream2nixWithExternals=${./src} + echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'" + fi + + + ''; }); diff --git a/src/apps/cli/commands/add.py b/src/apps/cli/commands/add.py index 6b72dcb4..5f9eef6f 100644 --- a/src/apps/cli/commands/add.py +++ b/src/apps/cli/commands/add.py @@ -24,7 +24,7 @@ class AddCommand(Command): argument( "source", "Sources of the packages. Can be a paths, tarball URLs, or flake-style specs", - # multiple=True + multiple=True ) ] @@ -57,29 +57,40 @@ class AddCommand(Command): ] def handle(self): + sources = self.argument("source") + + # ensure packages-root + package_root = self.find_package_root() + + main_package_dir_name = self.handle_one(package_root, sources[0]) + + sub_package_root = f"{package_root}/{main_package_dir_name}" + + for source in sources[1:]: + self.handle_one(sub_package_root, source) + + + + def handle_one(self, package_root, source): + if self.io.is_interactive(): self.line(f"\n{self.description}\n") # parse extra args specified_extra_args = self.parse_extra_args() - # ensure packages-root - packages_root = self.find_packages_root() - - - source = self.argument("source") lock, sourceSpec, specified_extra_args, translator =\ - self.translate_from_source(specified_extra_args, self.argument("source")) + self.translate_from_source(specified_extra_args, source) # get package name and version from lock mainPackageName = lock['_generic']['mainPackageName'] mainPackageVersion = lock['_generic']['mainPackageVersion'] # calculate output directory and attribute name - mainPackageDirName = self.define_attribute_name(mainPackageName) + main_package_dir_name = self.define_attribute_name(mainPackageName) # calculate output files - filesToCreate, output = self.calc_outputs(mainPackageDirName, packages_root) + filesToCreate, output = self.calc_outputs(main_package_dir_name, package_root) outputDreamLock = f"{output}/dream-lock.json" outputDefaultNix = f"{output}/default.nix" @@ -116,6 +127,8 @@ class AddCommand(Command): if config['isRepo']: sp.run(["git", "add", "-N", output]) + return main_package_dir_name + def translate_from_source(self, specified_extra_args, source): # get source path and spec source, sourceSpec = self.parse_source(source) @@ -147,7 +160,7 @@ class AddCommand(Command): defaultNix.write(template) print(f"Created {output}/default.nix") - def find_packages_root(self): + def find_package_root(self): if self.option("packages-root"): packages_root = self.option("packages-root") elif config['packagesDir']: @@ -248,8 +261,9 @@ class AddCommand(Command): type="unknown", ) else: - del mainSource['pname'] - del mainSource['version'] + for key in ['pname', 'version']: + if key in mainSource: + del mainSource[key] if mainPackageName not in lock['sources']: lock['sources'][mainPackageName] = { mainPackageVersion: mainSource @@ -270,14 +284,14 @@ class AddCommand(Command): f"--arg {n}={v}" for n, v in specified_extra_args.items() ]) - def calc_outputs(self, mainPackageDirName, packages_root): + def calc_outputs(self, main_package_dir_name, packages_root): if self.option('target'): if self.option('target').startswith('/'): output = self.option('target') else: output = f"{packages_root}/{self.option('target')}" else: - output = f"{packages_root}/{mainPackageDirName}" + output = f"{packages_root}/{main_package_dir_name}" # collect files to create filesToCreate = ['dream-lock.json'] if not os.path.isdir(output): diff --git a/src/apps/cli/default.nix b/src/apps/cli/default.nix index c5451ad0..db2f63be 100644 --- a/src/apps/cli/default.nix +++ b/src/apps/cli/default.nix @@ -62,7 +62,7 @@ in }: dream2nix.riseAndShine { - dreamLock = ./dream-lock.json; + source = ./dream-lock.json; ${lib.optionalString (dreamLock.sources."${mainPackageName}"."${mainPackageVersion}".type == "unknown") '' sourceOverrides = oldSources: { "${mainPackageName}"."${mainPackageVersion}" = ./${sourcePathRelative}; diff --git a/src/builders/nodejs/granular/default.nix b/src/builders/nodejs/granular/default.nix index e3fbdf51..d7394194 100644 --- a/src/builders/nodejs/granular/default.nix +++ b/src/builders/nodejs/granular/default.nix @@ -254,21 +254,28 @@ let python ${./symlink-deps.py} # resolve symlinks to copies - if [ "$installMethod" == "copy" ]; then - echo "transforming symlinked dependencies to copies..." - chmod +wx . - for f in $(find . -type l); do + symlinksToCopies() { + local dir="$1" + + echo "transforming symlinks to copies..." + for f in $(find -L "$dir" -xtype l); do if [ -f $f ]; then continue fi + echo "copying $f" chmod +wx $(dirname "$f") mv "$f" "$f.bak" mkdir "$f" if [ -n "$(ls -A "$f.bak/")" ]; then cp -r "$f.bak"/* "$f/" + chmod -R +w $f fi rm "$f.bak" done + } + + if [ "$installMethod" == "copy" ]; then + symlinksToCopies . fi # add dependencies to NODE_PATH @@ -294,8 +301,13 @@ let # by default, only for top level packages, `npm run build` is executed elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then npm run build - elif [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then - npm --production --offline --nodedir=$nodeSources run postinstall + else + if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run install + fi + if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then + npm --production --offline --nodedir=$nodeSources run postinstall + fi fi runHook postBuild diff --git a/src/default.nix b/src/default.nix index fda4a398..d5a04701 100644 --- a/src/default.nix +++ b/src/default.nix @@ -103,7 +103,16 @@ let }; dreamOverrides = - utils.loadOverridesDirs config.overridesDirs pkgs; + let + overridesDirs = + config.overridesDirs + ++ + (lib.optionals (b ? getEnv && b.getEnv "d2nOverridesDir" != "") [ + (b.getEnv "d2nOverridesDir") + ]); + + in + utils.loadOverridesDirs overridesDirs pkgs; # the location of the dream2nix framework for self references (update scripts, etc.) dream2nixWithExternals = @@ -231,6 +240,7 @@ let inject, sourceOverrides, packageOverrides, + allOutputs, }@args: let @@ -247,7 +257,7 @@ let produceDerivation = name: pkg: utils.applyOverridesToPackage { inherit pkg; - packages = formattedOutputs.packages; + outputs = allOutputs; pname = name; conditionalOverrides = packageOverrides; }; @@ -279,6 +289,8 @@ let dreamLock = subDreamLockLoaded.lock; + + outputs = allOutputs; }; outputs = builder ( builderArgs // { @@ -333,37 +345,38 @@ let # produce outputs for a dream-lock or a source riseAndShine = { - dreamLock ? null, + source, # source tree or dream-lock builder ? null, fetcher ? null, inject ? {}, - source ? null, sourceOverrides ? oldSources: {}, packageOverrides ? {}, builderArgs ? {}, translatorArgs ? {}, }@args: - # ensure either dreamLock or source is used - if source != null && dreamLock != null then - throw "Define either 'dreamLock' or 'source', not both" - else if source == null && dreamLock == null then - throw "Define either 'dreamLock' or 'source'" - else - let dreamLock' = - if source != null then - makeDreamLockForSource { inherit source translatorArgs; } + if lib.isAttrs args.source + || lib.hasSuffix "dream-lock.json" source then + args.source else - args.dreamLock; + makeDreamLockForSource { inherit source translatorArgs; }; # parse dreamLock dreamLockLoaded = utils.readDreamLock { dreamLock = dreamLock'; }; dreamLock = dreamLockLoaded.lock; dreamLockInterface = dreamLockLoaded.interface; + # rise and shine sub packages + builderOutputsSub = + b.mapAttrs + (dirName: dreamLock: + riseAndShine + (args // {source = dreamLock.lock; })) + dreamLockInterface.subDreamLocks; + builder' = if builder == null then findBuilder dreamLock @@ -386,6 +399,7 @@ let inherit dreamLock fetchedSources + allOutputs sourceOverrides ; @@ -402,8 +416,20 @@ let utils.dreamLock.decompressDependencyGraph args.inject or {}; }; + allOutputs = + { subPackages = builderOutputsSub; } + // + # merge with sub package outputs + b.foldl' + (old: new: old // { + packages = new.packages or {} // old.packages; + }) + builderOutputs + (b.attrValues builderOutputsSub); + in - builderOutputs; + allOutputs; + in { diff --git a/src/translators/nodejs/pure/package-lock/default.nix b/src/translators/nodejs/pure/package-lock/default.nix index eb0271a1..402d3b06 100644 --- a/src/translators/nodejs/pure/package-lock/default.nix +++ b/src/translators/nodejs/pure/package-lock/default.nix @@ -185,14 +185,12 @@ } else rec { - version = getVersion dependencyObject; url = dependencyObject.resolved; hash = dependencyObject.integrity; }; path = dependencyObject: rec { - version = getVersion dependencyObject; path = getPath dependencyObject; }; }; diff --git a/src/translators/nodejs/pure/yarn-lock/default.nix b/src/translators/nodejs/pure/yarn-lock/default.nix index 8e5ded56..bad13e2b 100644 --- a/src/translators/nodejs/pure/yarn-lock/default.nix +++ b/src/translators/nodejs/pure/yarn-lock/default.nix @@ -191,11 +191,10 @@ if b.length githubUrlInfos == 7 then let rev = lib.elemAt githubUrlInfos 6; - version = dependencyObject.version; in { url = "https://github.com/${owner}/${repo}"; - inherit rev version; + inherit rev; } else if b.length githubUrlInfos == 5 then let @@ -215,13 +214,11 @@ path = dependencyObject: if lib.hasInfix "@link:" dependencyObject.yarnName then { - version = dependencyObject.version; path = lib.last (lib.splitString "@link:" dependencyObject.yarnName); } else if lib.hasInfix "@file:" dependencyObject.yarnName then { - version = dependencyObject.version; path = lib.last (lib.splitString "@file:" dependencyObject.yarnName); } @@ -231,7 +228,6 @@ http = dependencyObject: { type = "http"; - version = dependencyObject.version; hash = if dependencyObject ? integrity then dependencyObject.integrity diff --git a/src/utils/config.nix b/src/utils/config.nix index c8f2354e..eb5d50d0 100644 --- a/src/utils/config.nix +++ b/src/utils/config.nix @@ -15,7 +15,7 @@ let input else throw "input for loadAttrs must be json file or string or attrs"; - + # load dream2nix config extending with defaults loadConfig = configInput: let @@ -31,4 +31,4 @@ let in { inherit loadConfig; -} \ No newline at end of file +} diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix index 9812a640..b5edaeeb 100644 --- a/src/utils/dream-lock.nix +++ b/src/utils/dream-lock.nix @@ -9,16 +9,34 @@ let b = builtins; + subDreamLockNames = dreamLockFile: + let + dir = b.dirOf dreamLockFile; + + directories = utils.listDirs dir; + + dreamLockDirs = + lib.filter + (d: b.pathExists ("${dir}/${d}/dream-lock.json")) + directories; + + in + dreamLockDirs; + + readDreamLock = { dreamLock, }@args: let - lockMaybeCompressed = - if b.isPath dreamLock + isFile = + b.isPath dreamLock || b.isString dreamLock - || lib.isDerivation dreamLock then + || lib.isDerivation dreamLock; + + lockMaybeCompressed = + if isFile then b.fromJSON (b.readFile dreamLock) else dreamLock; @@ -29,6 +47,20 @@ let else decompressDreamLock lockMaybeCompressed; + subDreamLocks = + if ! isFile then + {} + else + let + dir = b.dirOf dreamLock; + in + lib.genAttrs + (subDreamLockNames dreamLock) + (d: + readDreamLock + { dreamLock = "${dir}/${d}/dream-lock.json"; }); + + mainPackageName = lock._generic.mainPackageName; mainPackageVersion = lock._generic.mainPackageVersion; @@ -56,7 +88,7 @@ let in { inherit lock; - interface = rec { + interface = { inherit mainPackageName @@ -65,6 +97,7 @@ let getCyclicDependencies getDependencies packageVersions + subDreamLocks ; }; }; diff --git a/src/utils/override.nix b/src/utils/override.nix index e1153390..5109f9a2 100644 --- a/src/utils/override.nix +++ b/src/utils/override.nix @@ -70,7 +70,7 @@ let conditionalOverrides, pkg, pname, - packages, + outputs, }: let @@ -113,7 +113,13 @@ let # apply single attribute override applySingleAttributeOverride = oldVal: functionOrValue: if b.isFunction functionOrValue then - functionOrValue oldVal + if lib.functionArgs functionOrValue == {} then + functionOrValue oldVal + else + functionOrValue { + old = oldVal; + inherit outputs; + } else functionOrValue; @@ -167,7 +173,7 @@ let lib.filterAttrs (n: v: ! lib.hasPrefix "override" n && ! lib.hasPrefix "_" n) condOverride; - + in lib.mapAttrsToList (attrName: funcOrValue: {