feat: implement makeOutputs that outputs systemless structure (#161)

* feat: return underlying d2n instance if pkgs was passed to init, document init pkgs

* feat: implement makeOutputs

* tests: dont fail if resolveImpure fails

* feat: make init return a dream2nix instance, rework makeFlakeOutputs code

* feat: default to source if config.projectRoot is not specified, update examples and readme

* fix: update the simple template

* docs: clarify init in readme

* docs: change readme numbering to back to 1

* refactor: dont default projectRoot to source

* docs: make extensive example use makeFlakeOutputs, link per-pkgs d2n example in readme

* fix: call loadConfig correctly
This commit is contained in:
Yusuf Bera Ertan 2022-06-01 23:53:46 +03:00 committed by GitHub
parent ca7f4d0a7f
commit 9b6638a1a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 449 additions and 199 deletions

View File

@ -42,14 +42,10 @@ Minimal Example `flake.nix`:
```nix
{
inputs.dream2nix.url = "github:nix-community/dream2nix";
outputs = { self, dream2nix }@inputs:
let
dream2nix = inputs.dream2nix.lib.init {
# modify according to your supported systems
systems = [ "x86_64-linux" ];
config.projectRoot = ./. ;
};
in dream2nix.makeFlakeOutputs {
outputs = { self, dream2nix }:
dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
source = ./.;
};
}
@ -59,31 +55,21 @@ Extensive Example `flake.nix`:
```nix
{
inputs.dream2nix.url = "github:nix-community/dream2nix";
outputs = { self, dream2nix }@inputs:
let
system = "x86_64-linux";
outputs = { self, dream2nix }:
dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
pkgs = inputs.dream2nix.inputs.nixpkgs.legacyPackages.${system};
dream2nix = inputs.dream2nix.lib.init {
# modify according to your supported systems
systems = [ system ];
config.projectRoot = ./. ;
};
in dream2nix.makeFlakeOutputs {
source = ./.;
# Configure the behavior of dream2nix when translating projects.
# A setting applies to all discovered projects if `filter` is unset,
# or just to a subset or projects if `filter` is used.
settings = [
# prefer aggregated source fetching (large FODs)
{
aggregate = true;
}
# for all impure nodejs projects with just a `package.json`,
# add arguments for the `package-json` translator
{
@ -102,7 +88,7 @@ Extensive Example `flake.nix`:
# override attributes
preBuild = "...";
# update attributes
buildInputs = old: old ++ [ pkgs.hello ];
buildInputs = old: old ++ [pkgs.hello];
};
};
};
@ -132,6 +118,9 @@ Extensive Example `flake.nix`:
}
```
An example for instancing dream2nix per pkgs and using it to create outputs
can be found at [`examples/d2n-init-pkgs`](./examples/d2n-init-pkgs/flake.nix).
### Watch the presentation
(The code examples of the presentation are outdated)
[![dream2nix - A generic framework for 2nix tools](https://gist.githubusercontent.com/DavHau/755fed3774e89c0b9b8953a0a25309fa/raw/3c8b2c56f5fca3bf5c343ffc179136eef39d4d6a/dream2nix-youtube-talk.png)](https://www.youtube.com/watch?v=jqCfHMvCsfQ)

View File

@ -6,14 +6,11 @@
outputs = {
self,
dream2nix,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
config.extra = ./extra.nix;
};
in
(dream2nix.makeFlakeOutputs {
source = ./.;
settings = [
{

View File

@ -9,8 +9,8 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
config.extra = {
@ -21,9 +21,6 @@
};
fetchers.crates-io = "${inp.dream2nix}/src/fetchers/crates-io";
};
};
in
(dream2nix.makeFlakeOutputs {
source = src;
settings = [
{

View File

@ -0,0 +1,229 @@
{
"nodes": {
"alejandra": {
"inputs": {
"flakeCompat": "flakeCompat",
"nixpkgs": [
"dream2nix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1652972885,
"narHash": "sha256-OKTV5Mi0WyDGsF6GcTwWkgJPNRkskD5yqCZZmghZYHI=",
"owner": "kamadorueda",
"repo": "alejandra",
"rev": "69d2075e432c562099965829d8bc4da701b10d20",
"type": "github"
},
"original": {
"owner": "kamadorueda",
"repo": "alejandra",
"type": "github"
}
},
"crane": {
"flake": false,
"locked": {
"lastModified": 1644785799,
"narHash": "sha256-VpAJO1L0XeBvtCuNGK4IDKp6ENHIpTrlaZT7yfBCvwo=",
"owner": "ipetkov",
"repo": "crane",
"rev": "fc7a94f841347c88f2cb44217b2a3faa93e2a0b2",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"dream2nix": {
"inputs": {
"alejandra": "alejandra",
"crane": "crane",
"flake-utils-pre-commit": "flake-utils-pre-commit",
"gomod2nix": "gomod2nix",
"mach-nix": "mach-nix",
"nixpkgs": [
"nixpkgs"
],
"node2nix": "node2nix",
"poetry2nix": "poetry2nix",
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 0,
"narHash": "sha256-cMxGUD65rheZKUfWX0DDgFUqkKXvSS1syweUtmn3peM=",
"path": "../..",
"type": "path"
},
"original": {
"path": "../..",
"type": "path"
}
},
"flake-utils-pre-commit": {
"locked": {
"lastModified": 1644229661,
"narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flakeCompat": {
"flake": false,
"locked": {
"lastModified": 1648199409,
"narHash": "sha256-JwPKdC2PoVBkG6E+eWw3j6BMR6sL3COpYWfif7RVb8Y=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "64a525ee38886ab9028e6f61790de0832aa3ef03",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"gomod2nix": {
"flake": false,
"locked": {
"lastModified": 1627572165,
"narHash": "sha256-MFpwnkvQpauj799b4QTBJQFEddbD02+Ln5k92QyHOSk=",
"owner": "tweag",
"repo": "gomod2nix",
"rev": "67f22dd738d092c6ba88e420350ada0ed4992ae8",
"type": "github"
},
"original": {
"owner": "tweag",
"repo": "gomod2nix",
"type": "github"
}
},
"mach-nix": {
"flake": false,
"locked": {
"lastModified": 1634711045,
"narHash": "sha256-m5A2Ty88NChLyFhXucECj6+AuiMZPHXNbw+9Kcs7F6Y=",
"owner": "DavHau",
"repo": "mach-nix",
"rev": "4433f74a97b94b596fa6cd9b9c0402104aceef5d",
"type": "github"
},
"original": {
"id": "mach-nix",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1653581809,
"narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "83658b28fe638a170a19b8933aa008b30640fbd1",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"node2nix": {
"flake": false,
"locked": {
"lastModified": 1634916276,
"narHash": "sha256-lov2b/8ydYjq+MhKQugmWV2lFnq35AU5RTRBTfLq7B4=",
"owner": "svanderburg",
"repo": "node2nix",
"rev": "644e90c0304038a446ed53efc97e9eb1e2831e71",
"type": "github"
},
"original": {
"owner": "svanderburg",
"repo": "node2nix",
"type": "github"
}
},
"poetry2nix": {
"flake": false,
"locked": {
"lastModified": 1632969109,
"narHash": "sha256-jPDclkkiAy5m2gGLBlKgH+lQtbF7tL4XxBrbSzw+Ioc=",
"owner": "nix-community",
"repo": "poetry2nix",
"rev": "aee8f04296c39d88155e05d25cfc59dfdd41cc77",
"type": "github"
},
"original": {
"owner": "nix-community",
"ref": "1.21.0",
"repo": "poetry2nix",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-utils": [
"dream2nix",
"flake-utils-pre-commit"
],
"nixpkgs": [
"dream2nix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1646153636,
"narHash": "sha256-AlWHMzK+xJ1mG267FdT8dCq/HvLCA6jwmx2ZUy5O8tY=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "b6bc0b21e1617e2b07d8205e7fae7224036dfa4b",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"dream2nix": "dream2nix",
"nixpkgs": "nixpkgs",
"src": "src"
}
},
"src": {
"flake": false,
"locked": {
"lastModified": 1623499944,
"narHash": "sha256-udEh+Re2PeO3DnX4fQThsaT1Y3MBHFfrX5Q5EN2XrF0=",
"owner": "BurntSushi",
"repo": "ripgrep",
"rev": "af6b6c543b224d348a8876f0c06245d9ea7929c5",
"type": "github"
},
"original": {
"owner": "BurntSushi",
"ref": "13.0.0",
"repo": "ripgrep",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

View File

@ -0,0 +1,45 @@
{
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
dream2nix.url = "path:../..";
dream2nix.inputs.nixpkgs.follows = "nixpkgs";
src.url = "github:BurntSushi/ripgrep/13.0.0";
src.flake = false;
};
outputs = inp: let
l = inp.nixpkgs.lib // builtins;
allPkgs =
l.map
(system: inp.nixpkgs.legacyPackages.${system})
["x86_64-linux" "aarch64-linux"];
initD2N = pkgs:
inp.dream2nix.lib.init {
inherit pkgs;
config.projectRoot = ./.;
};
makeOutputs = pkgs: let
outputs = (initD2N pkgs).makeOutputs {
source = inp.src;
settings = [
{
builder = "build-rust-package";
translator = "cargo-lock";
}
];
};
in rec {
packages.${pkgs.system} = outputs.packages;
checks.${pkgs.system} = {
inherit (outputs.packages) ripgrep;
};
};
allOutputs = l.map makeOutputs allPkgs;
outputs = l.foldl' l.recursiveUpdate {} allOutputs;
in
outputs;
}

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
settings = [
{

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
settings = [
{

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
})
// {

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
settings = [
{

View File

@ -6,13 +6,10 @@
outputs = {
self,
dream2nix,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = ./.;
settings = [
];

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
})
// {

View File

@ -9,13 +9,10 @@
self,
dream2nix,
src,
} @ inp: let
dream2nix = inp.dream2nix.lib2.init {
} @ inp:
(dream2nix.lib.makeFlakeOutputs {
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
(dream2nix.makeFlakeOutputs {
source = src;
settings = [
{

View File

@ -563,6 +563,100 @@
l.foldl' recursiveUpdateUntilDrv {} projectOutputs;
in
mergedOutputs;
generateImpureResolveScript = {
source,
impureDiscoveredProjects,
}: let
impureResolveScriptsList =
l.listToAttrs
(
l.map
(
project:
l.nameValuePair
"Name: ${project.name}; Subsystem: ${project.subsystem}; relPath: ${project.relPath}"
(utils.makeTranslateScript {inherit project source;})
)
impureDiscoveredProjects
);
resolveImpureScript =
utils.writePureShellScript
[]
''
cd $WORKDIR
${l.concatStringsSep "\n"
(l.mapAttrsToList
(title: script: ''
echo "Resolving:: ${title}"
${script}/bin/resolve
'')
impureResolveScriptsList)}
'';
in
resolveImpureScript;
makeOutputs = {
source ? throw "pass a 'source' to 'makeOutputs'",
discoveredProjects ? dlib.discoverers.discoverProjects {inherit settings source;},
pname ? null,
settings ? [],
packageOverrides ? {},
sourceOverrides ? old: {},
inject ? {},
}: let
impureDiscoveredProjects =
l.filter
(proj:
subsystems
."${proj.subsystem}"
.translators
."${proj.translator}"
.type
== "impure")
discoveredProjects;
resolveImpureScript = generateImpureResolveScript {
inherit impureDiscoveredProjects source;
};
translatedProjects = translateProjects {
inherit
discoveredProjects
pname
settings
source
;
};
realizedProjects = realizeProjects {
inherit
inject
packageOverrides
sourceOverrides
translatedProjects
source
;
};
in
realizedProjects
// l.optionalAttrs (l.length impureDiscoveredProjects > 0) {
apps =
l.warnIf
(realizeProjects.apps.resolveImpure or null != null)
''
a builder outputted an app named 'resolveImpure'
this will be overrided by dream2nix!
''
((realizeProjects.apps or {})
// {
resolveImpure = {
type = "app";
program = l.toString resolveImpureScript;
};
});
};
in {
inherit
apps
@ -576,6 +670,7 @@ in {
updaters
utils
makeOutputsForDreamLock
makeOutputs
subsystems
;
}

View File

@ -8,13 +8,22 @@
externalSources,
externalPaths,
} @ args: let
b = builtins;
l = lib // builtins;
dream2nixForSystem = config: system: pkgs:
initDream2nix = config: pkgs:
import ./default.nix
{inherit config externalPaths externalSources pkgs;};
{inherit config pkgs externalPaths externalSources;};
loadConfig = config'': let
config' = (import ./utils/config.nix).loadConfig config'';
config =
config'
// {
overridesDirs = args.overridesDirs ++ config'.overridesDirs;
};
in
config;
# TODO: design output schema for cross compiled packages
makePkgsKey = pkgs: let
@ -35,87 +44,48 @@
# only pkgs is specified
else if pkgsList != null
then
if b.isList pkgsList
if l.isList pkgsList
then
lib.listToAttrs
(pkgs: lib.nameValuePair (makePkgsKey pkgs) pkgs)
l.listToAttrs
(pkgs: l.nameValuePair (makePkgsKey pkgs) pkgs)
pkgsList
else {"${makePkgsKey pkgsList}" = pkgsList;}
# only systems is specified
else
lib.genAttrs systems
l.genAttrs systems
(system: import nixpkgsSrc {inherit system;});
flakifyBuilderOutputs = system: outputs:
l.mapAttrs
(ouputType: outputValue: {"${system}" = outputValue;})
(_: outputValue: {"${system}" = outputValue;})
outputs;
init = {
pkgs ? throw "please pass 'pkgs' (a nixpkgs instance) to 'init'",
config ? {},
}:
initDream2nix (loadConfig config) pkgs;
makeFlakeOutputs = {
source,
pkgs ? null,
systems ? [],
config ? {},
} @ argsInit: let
config' = (import ./utils/config.nix).loadConfig argsInit.config or {};
config =
config'
// {
overridesDirs = args.overridesDirs ++ config'.overridesDirs;
};
allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: lib.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
in {
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutputs = mArgs:
makeFlakeOutputsFunc
(
{inherit config pkgs systems;}
// mArgs
);
apps =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".apps.flakeApps);
defaultApp =
forAllSystems
(system: pkgs:
dream2nixFor."${system}".apps.flakeApps.dream2nix);
};
makeFlakeOutputsFunc = {
config ? {},
inject ? {},
pname ? throw "Please pass `pname` to makeFlakeOutputs",
pkgs ? null,
packageOverrides ? {},
settings ? [],
source,
sourceOverrides ? oldSources: {},
systems ? [],
translator ? null,
translatorArgs ? {},
} @ args: let
config = args.config or ((import ./utils/config.nix).loadConfig {});
allPkgs = makeNixpkgs pkgs systems;
forAllSystems = f: b.mapAttrs f allPkgs;
dream2nixFor = forAllSystems (dream2nixForSystem config);
config = loadConfig (args.config or {});
dlib = import ./lib {inherit lib config;};
getInvalidationHash = project:
dlib.calcInvalidationHash {
inherit project source;
# TODO: add translatorArgs
translatorArgs = {};
translator = project.translator;
};
initD2N = initDream2nix config;
dream2nixFor = l.mapAttrs (_: pkgs: initD2N pkgs) allPkgs;
discoveredProjects = dlib.discoverers.discoverProjects {
inherit settings;
@ -123,80 +93,31 @@
};
allBuilderOutputs =
lib.mapAttrs
l.mapAttrs
(system: pkgs: let
dream2nix = dream2nixFor."${system}";
impureDiscoveredProjects =
l.filter
(proj:
dream2nix
.subsystems
."${proj.subsystem}"
.translators
."${proj.translator}"
.type
== "impure")
discoveredProjects;
impureResolveScriptsList =
l.listToAttrs
(l.forEach impureDiscoveredProjects
(project:
l.nameValuePair
"Name: ${project.name}; Subsystem: ${project.subsystem}; relPath: ${project.relPath}"
(dream2nix.utils.makeTranslateScript {
inherit project source;
invalidationHash = getInvalidationHash project;
})));
resolveImpureScript =
dream2nix.utils.writePureShellScript
[]
''
cd $WORKDIR
${l.concatStringsSep "\n"
(l.mapAttrsToList
(title: script: ''
echo "Resolving:: ${title}"
${script}/bin/resolve
'')
impureResolveScriptsList)}
'';
translatedProjects = dream2nix.translateProjects {
inherit pname settings source;
};
realizedProjects = dream2nix.realizeProjects {
allOutputs = dream2nix.makeOutputs {
inherit
inject
packageOverrides
sourceOverrides
translatedProjects
source
pname
discoveredProjects
settings
sourceOverrides
packageOverrides
inject
;
};
allOutputs =
realizedProjects
// {
apps.resolveImpure = {
type = "app";
program = l.toString resolveImpureScript;
};
};
in
allOutputs)
allPkgs;
flakifiedOutputsList =
lib.mapAttrsToList
l.mapAttrsToList
(system: outputs: flakifyBuilderOutputs system outputs)
allBuilderOutputs;
flakeOutputsBuilders =
b.foldl'
l.foldl'
(allOutputs: output: lib.recursiveUpdate allOutputs output)
{}
flakifiedOutputsList;
@ -207,8 +128,7 @@
in
flakeOutputs;
in {
inherit init;
inherit init makeFlakeOutputs;
dlib = import ./lib {inherit lib;};
riseAndShine = throw "Use makeFlakeOutputs instead of riseAndShine.";
makeFlakeOutputs = makeFlakeOutputsFunc;
}

View File

@ -168,9 +168,15 @@ in
satisfiesSemver = poetry2nixSemver.satisfiesSemver;
makeTranslateScript = {
invalidationHash,
source,
project,
invalidationHash ?
dlib.calcInvalidationHash {
inherit project source;
# TODO: translatorArgs
translatorArgs = {};
translator = project.translator;
},
} @ args: let
aggregate = project.aggregate or false;

View File

@ -1,16 +1,9 @@
{
inputs.dream2nix.url = "github:nix-community/dream2nix";
outputs = {
self,
dream2nix,
} @ inputs: let
dream2nix = inputs.dream2nix.lib.init {
outputs = inp:
inp.dream2nix.lib.makeFlakeOutputs {
# modify according to your supported systems
systems = ["x86_64-linux"];
config.projectRoot = ./.;
};
in
dream2nix.makeFlakeOutputs {
source = ./.;
};
}

View File

@ -29,7 +29,7 @@
cp -r ${examples}/$dir/* .
chmod -R +w .
nix flake lock --override-input dream2nix ${../../.}
nix run .#resolveImpure
nix run .#resolveImpure || echo "no resolveImpure probably?"
nix flake check "$@"
'';
in