nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2024-11-30 10:07:33 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #44 from DavHau/dev

Browse Source 
Refactor/Improve dream-lock structure
This commit is contained in:
DavHau 2021-11-04 15:49:09 +07:00 committed by GitHub
commit f1ab3e1b84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 441 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/concepts-and-benefits.md
View File

@ -135,26 +135,26 @@ Potery uses `pyproject.toml` and `poetry.lock` to lock dependencies
},
// generic metadata (not specific to python)
"generic": {
"_generic": {
// this indicates which builder must be used
"buildSystem": "python",
"subsystem": "python",
// translator which generated this file
// (not relevant for building)
"producedBy": "translator-poetry-1",
// dependency graph of the packages
"dependencyGraph": {
"dependencies": {
"requests": [
"certifi"
]
}
},
// all fields inside 'buildSystem' are specific to
// the selected buildSystem (python)
"buildSystem": {
// all fields inside 'subsystem' are specific to
// the selected subsystem (python)
"_subsystem": {
// tell the python builder how the inputs must be handled
"sourceFormats": {
@ -168,8 +168,8 @@ Potery uses `pyproject.toml` and `poetry.lock` to lock dependencies
- be dumped to a .json file and committed to a repo
- passed directly to the fetching/building layer
- the fetcher will only read the sources section and translate it to standard fetcher calls.
- the building layer will read the "buildSystem" attribute and select the python builder for building.
- the python builder will read all information from "buildSystem" and translate the data to a final derivation.
- the building layer will read the "subsystem" attribute and select the python builder for building.
- the python builder will read all information from "subsystem" and translate the data to a final derivation.
Notes on IFD, FOD and code generation:
- No matter which type of tanslator is used, it is always possible to export the generic lock to a file, which can later be evaluated without using IFD or FOD, similar to current nix code generators, just with a standardized format.

4
docs/nixpkgs-improvements.md
View File

@ -76,7 +76,7 @@
#### Solution
- dream2nix provides good interfaces for customizability which are unified as much as possible independently from the underlying buildsystems.
- dream2nix provides good interfaces for customizability which are unified as much as possible independently from the underlying subsystems.
### Inefficient/Slow Innovation
@ -88,5 +88,5 @@
#### Solution
- Since dream2nix centrally handles many core elements of packaging like different strategies for fetching and building, it is much easier to fix problems at large scale and apply new innovations to all underlysing buildsystems at once.
- Since dream2nix centrally handles many core elements of packaging like different strategies for fetching and building, it is much easier to fix problems at large scale and apply new innovations to all underlysing subsystems at once.
- Experimenting with and adding support for new nix features will be easier as the framework offers better abstractions than existing 2nix converters and allows adding/modifying strategies more easily.

2
flake.nix
View File

@ -142,7 +142,7 @@
export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
export d2nOverridesDir=${./overrides}
echo "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
echo -e "\nManually execute 'export dream2nixWithExternals={path to your dream2nix checkout}'"
'';
});

120
src/apps/cli/commands/package.py
View File

@ -9,7 +9,7 @@ import networkx as nx
from cleo import Command, option
from utils import dream2nix_src, checkLockJSON, callNixFunction, buildNixFunction, buildNixAttribute, \
list_translators_for_source, sort_dict, strip_hashes_from_lock
list_translators_for_source, strip_hashes_from_lock
class PackageCommand(Command):
@ -29,7 +29,7 @@ class PackageCommand(Command):
multiple=False
),
option("translator", None, "which translator to use", flag=False),
option("output", None, "output file/directory for the dream.lock", flag=False),
option("output", None, "output file/directory for the dream-lock.json", flag=False),
option(
"combined",
None,
@ -66,7 +66,7 @@ class PackageCommand(Command):
output = './.'
if not os.path.isdir(output):
os.mkdir(output)
filesToCreate = ['dream.lock']
filesToCreate = ['dream-lock.json']
if self.option('default-nix'):
filesToCreate.append('default.nix')
if self.option('force'):
@ -78,12 +78,12 @@ class PackageCommand(Command):
if any(f in existingFiles for f in filesToCreate):
print(
f"output directory {output} already contains a 'default.nix' "
"or 'dream.lock'. Delete first, or user '--force'.",
"or 'dream-lock.json'. Delete first, or user '--force'.",
file=sys.stderr,
)
exit(1)
output = os.path.realpath(output)
outputDreamLock = f"{output}/dream.lock"
outputDreamLock = f"{output}/dream-lock.json"
outputDefaultNix = f"{output}/default.nix"
# verify source
@ -111,13 +111,13 @@ class PackageCommand(Command):
print(f"Input source '{source}' does not exist", file=sys.stdout)
exit(1)
source = os.path.realpath(source)
# handle source from dream.lock
if source.endswith('dream.lock'):
print(f"fetching source defined via existing dream.lock")
# handle source from dream-lock.json
if source.endswith('dream-lock.json'):
print(f"fetching source defined via existing dream-lock.json")
with open(source) as f:
sourceDreamLock = json.load(f)
sourceMainPackageName = sourceDreamLock['generic']['mainPackageName']
sourceMainPackageVersion = sourceDreamLock['generic']['mainPackageVersion']
sourceMainPackageName = sourceDreamLock['_generic']['mainPackageName']
sourceMainPackageVersion = sourceDreamLock['_generic']['mainPackageVersion']
sourceSpec =\
sourceDreamLock['sources'][sourceMainPackageName][sourceMainPackageVersion]
source = \
@ -210,20 +210,23 @@ class PackageCommand(Command):
file=sys.stderr
)
exit(1)
# interactively retrieve unswers for unspecified extra arguments
# interactively retrieve answers for unspecified extra arguments
else:
print(f"\nThe translator '{translator['name']}' requires additional options")
for arg_name, arg in unspecified_extra_args.items():
print('')
if arg['type'] == 'flag':
print(f"Please specify '{arg_name}': {arg['description']}")
print(f"Please specify '{arg_name}'")
specified_extra_args[arg_name] = self.confirm(f"{arg['description']}:", False)
else:
print(f"Please specify '{arg_name}': {arg['description']}")
print(f"Example values: " + ', '.join(arg['examples']))
if 'default' in arg:
print(f"\nLeave empty for default ({arg['default']})")
specified_extra_args[arg_name] = self.ask(f"{arg_name}:", arg.get('default'))
print(f"Leave empty for default ({arg['default']})")
while True:
specified_extra_args[arg_name] = self.ask(f"{arg_name}:", arg.get('default'))
if specified_extra_args[arg_name]:
break
# arguments for calling the translator nix module
translator_input = dict(
@ -252,7 +255,7 @@ class PackageCommand(Command):
# raise error if output wasn't produced
if not os.path.isfile(outputDreamLock):
raise Exception(f"Translator failed to create dream.lock")
raise Exception(f"Translator failed to create dream-lock.json")
# read produced lock file
with open(outputDreamLock) as f:
@ -260,8 +263,8 @@ class PackageCommand(Command):
# write translator information to lock file
combined = self.option('combined')
lock['generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
lock['generic']['translatorParams'] = " ".join([
lock['_generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
lock['_generic']['translatorParams'] = " ".join([
'--translator',
f"{translator['subsystem']}.{translator['type']}.{translator['name']}",
] + (
@ -271,8 +274,8 @@ class PackageCommand(Command):
])
# add main package source
mainPackageName = lock['generic']['mainPackageName']
mainPackageVersion = lock['generic']['mainPackageVersion']
mainPackageName = lock['_generic']['mainPackageName']
mainPackageVersion = lock['_generic']['mainPackageVersion']
mainSource = sourceSpec.copy()
if not mainSource:
mainSource = dict(
@ -287,45 +290,49 @@ class PackageCommand(Command):
# clean up dependency graph
# remove empty entries
if 'dependencyGraph' in lock['generic']:
depGraph = lock['generic']['dependencyGraph']
if 'dependencyGraph' in lock['generic']:
if 'dependencies' in lock['_generic']:
depGraph = lock['_generic']['dependencies']
if 'dependencies' in lock['_generic']:
for pname, deps in depGraph.copy().items():
if not deps:
del depGraph[pname]
# remove cyclic dependencies
edges = set()
for pname, deps in depGraph.items():
for dep in deps:
edges.add((pname, dep))
for pname, versions in depGraph.items():
for version, deps in versions.items():
for dep in deps:
edges.add(((pname, version), tuple(dep)))
G = nx.DiGraph(sorted(list(edges)))
cycle_count = 0
removed_edges = []
for pname in list(depGraph.keys()):
try:
while True:
cycle = nx.find_cycle(G, pname)
cycle_count += 1
# remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
node_from, node_to = cycle[-1][0], cycle[-1][1]
G.remove_edge(node_from, node_to)
removed_edges.append((node_from, node_to))
except nx.NetworkXNoCycle:
continue
lock['generic']['cyclicDependencies'] = {}
for pname, versions in depGraph.items():
for version in versions.keys():
key = (pname, version)
try:
while True:
cycle = nx.find_cycle(G, key)
cycle_count += 1
# remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
node_from, node_to = cycle[-1][0], cycle[-1][1]
G.remove_edge(node_from, node_to)
removed_edges.append((node_from, node_to))
except nx.NetworkXNoCycle:
continue
lock['cyclicDependencies'] = {}
if removed_edges:
lock['generic']['cyclicDependencies'] = {}
removed_cycles_text = 'Removed Cyclic dependencies:'
for node, removed_node in removed_edges:
removed_cycles_text += f"\n {node} -> {removed_node}"
n_name, n_ver = node.split('#')
if n_name not in lock['generic']['cyclicDependencies']:
lock['generic']['cyclicDependencies'][n_name] = {}
if n_ver not in lock['generic']['cyclicDependencies'][n_name]:
lock['generic']['cyclicDependencies'][n_name][n_ver] = []
lock['generic']['cyclicDependencies'][n_name][n_ver].append(removed_node)
print(removed_cycles_text)
cycles_text = 'Detected Cyclic dependencies:'
for node, removed in removed_edges:
n_name, n_ver = node[0], node[1]
r_name, r_ver = removed[0], removed[1]
cycles_text +=\
f"\n {n_name}#{n_ver} -> {r_name}#{r_ver}"
if n_name not in lock['cyclicDependencies']:
lock['cyclicDependencies'][n_name] = {}
if n_ver not in lock['cyclicDependencies'][n_name]:
lock['cyclicDependencies'][n_name][n_ver] = []
lock['cyclicDependencies'][n_name][n_ver].append(removed)
print(cycles_text)
# calculate combined hash if --combined was specified
if combined:
@ -334,7 +341,7 @@ class PackageCommand(Command):
# remove hashes from lock file and init sourcesCombinedHash with empty string
strip_hashes_from_lock(lock)
lock['generic']['sourcesCombinedHash'] = ""
lock['_generic']['sourcesCombinedHash'] = ""
with open(outputDreamLock, 'w') as f:
json.dump(lock, f, indent=2)
@ -357,12 +364,17 @@ class PackageCommand(Command):
print(f"Computed FOD hash: {hash}")
# store the hash in the lock
lock['generic']['sourcesCombinedHash'] = hash
lock['_generic']['sourcesCombinedHash'] = hash
# re-write dream.lock
checkLockJSON(sort_dict(lock))
# re-write dream-lock.json
checkLockJSON(lock)
lockStr = json.dumps(lock, indent=2, sort_keys = True)
lockStr = lockStr\
.replace("[\n ", "[ ")\
.replace("\"\n ]", "\" ]")\
.replace(",\n ", ", ")
with open(outputDreamLock, 'w') as f:
json.dump(sort_dict(lock), f, indent=2)
f.write(lockStr)
# create default.nix
template = callNixFunction(
@ -377,4 +389,4 @@ class PackageCommand(Command):
defaultNix.write(template)
print(f"Created {output}/default.nix")
print(f"Created {output}/dream.lock")
print(f"Created {output}/dream-lock.json")

14
src/apps/cli/commands/update.py
View File

@ -17,7 +17,7 @@ class UpdateCommand(Command):
name = "update"
options = [
option("dream-lock", None, "dream.lock file or its parent directory", flag=False, value_required=True),
option("dream-lock", None, "dream-lock.json file or its parent directory", flag=False, value_required=True),
option("updater", None, "name of the updater module to use", flag=False),
option("new-version", None, "new package version", flag=False),
]
@ -27,8 +27,8 @@ class UpdateCommand(Command):
self.line(f"\n{self.description}\n")
dreamLockFile = os.path.abspath(self.option("dream-lock"))
if not dreamLockFile.endswith('dream.lock'):
dreamLockFile = os.path.abspath(dreamLockFile + "/dream.lock")
if not dreamLockFile.endswith('dream-lock.json'):
dreamLockFile = os.path.abspath(dreamLockFile + "/dream-lock.json")
# parse dream lock
with open(dreamLockFile) as f:
@ -59,8 +59,8 @@ class UpdateCommand(Command):
cli_py = os.path.abspath(f"{__file__}/../../cli.py")
# delete the hash
mainPackageName = lock['generic']['mainPackageName']
mainPackageVersion = lock['generic']['mainPackageVersion']
mainPackageName = lock['_generic']['mainPackageName']
mainPackageVersion = lock['_generic']['mainPackageVersion']
mainPackageSource = lock['sources'][mainPackageName][mainPackageVersion]
updatedSourceSpec = callNixFunction(
"fetchers.updateSource",
@ -68,7 +68,7 @@ class UpdateCommand(Command):
newVersion=version,
)
lock['sources'][mainPackageName][mainPackageVersion] = updatedSourceSpec
with tempfile.NamedTemporaryFile("w", suffix="dream.lock") as tmpDreamLock:
with tempfile.NamedTemporaryFile("w", suffix="dream-lock.json") as tmpDreamLock:
json.dump(lock, tmpDreamLock, indent=2)
tmpDreamLock.seek(0) # flushes write cache
sp.run(
@ -76,5 +76,5 @@ class UpdateCommand(Command):
sys.executable, f"{cli_py}", "package", "--force", "--source", tmpDreamLock.name,
"--output", os.path.abspath(os.path.dirname(dreamLockFile))
]
+ lock['generic']['translatorParams'].split()
+ lock['_generic']['translatorParams'].split()
)

6
src/apps/cli/default.nix
View File

@ -32,8 +32,8 @@ in
sourcePathRelative,
}:
let
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageVersion = dreamLock.generic.mainPackageVersion;
mainPackageName = dreamLock._generic.mainPackageName;
mainPackageVersion = dreamLock._generic.mainPackageVersion;
in
''
{
@ -48,7 +48,7 @@ in
}:
dream2nix.riseAndShine {
dreamLock = ./dream.lock;
dreamLock = ./dream-lock.json;
${lib.optionalString (dreamLock.sources."${mainPackageName}"."${mainPackageVersion}".type == "unknown") ''
sourceOverrides = oldSources: {
"${mainPackageName}#${mainPackageVersion}" = ./${sourcePathRelative};

4
src/apps/cli/utils.py
View File

@ -106,10 +106,6 @@ def list_translators_for_source(sourcePath):
return list(sorted(translatorsList, key=lambda t: t['compatible']))
def sort_dict(d):
return {k: sort_dict(v) if isinstance(v, dict) else v
for k, v in sorted(d.items())}
def strip_hashes_from_lock(lock):
for name, versions in lock['sources'].items():
for source in versions.values():

30
src/apps/contribute/contribute.py
View File

@ -17,7 +17,7 @@ class ContributeCommand(Command):
options = [
option("module", None, "Which kind of module to contribute", flag=False),
option("buildsystem", None, "which kind of buildsystem", flag=False),
option("subsystem", None, "which kind of subsystem", flag=False),
option("type", None, "pure or impure translator", flag=False),
option("name", None, "name of the new module", flag=False),
option(
@ -49,22 +49,22 @@ class ContributeCommand(Command):
module = f"{module}s"
module_dir = dream2nix_src + f"/{module}/"
buildsystem = self.option('buildsystem')
known_buildsystems = list(dir for dir in os.listdir(module_dir) if os.path.isdir(module_dir + dir))
if not buildsystem:
buildsystem = self.choice(
'Select buildsystem',
known_buildsystems
subsystem = self.option('subsystem')
known_subsystems = list(dir for dir in os.listdir(module_dir) if os.path.isdir(module_dir + dir))
if not subsystem:
subsystem = self.choice(
'Select subsystem',
known_subsystems
+
[
" -> add new"
],
0
)
if buildsystem == " -> add new":
buildsystem = self.ask('Please enter the name of a new buildsystem:')
if buildsystem in known_buildsystems:
raise Exception(f"buildsystem {buildsystem} already exists")
if subsystem == " -> add new":
subsystem = self.ask('Please enter the name of a new subsystem:')
if subsystem in known_subsystems:
raise Exception(f"subsystem {subsystem} already exists")
if module == 'translators':
@ -81,12 +81,12 @@ class ContributeCommand(Command):
name = self.ask('Specify name of new module:')
for path in (
module_dir + f"{buildsystem}",
module_dir + f"{buildsystem}/{type}",
module_dir + f"{buildsystem}/{type}/{name}"):
module_dir + f"{subsystem}",
module_dir + f"{subsystem}/{type}",
module_dir + f"{subsystem}/{type}/{name}"):
if not os.path.isdir(path):
os.mkdir(path)
target_file = module_dir + f"{buildsystem}/{type}/{name}/default.nix"
target_file = module_dir + f"{subsystem}/{type}/{name}/default.nix"
with open(dream2nix_src + f"/templates/{module}/{type}.nix") as template:
with open(target_file, 'w') as new_file:
new_file.write(template.read())

14
src/builders/nodejs/granular/default.nix
View File

@ -21,8 +21,8 @@
buildPackageWithOtherBuilder,
# attributes
buildSystemAttrs,
cyclicDependencies,
subsystemAttrs,
getCyclicDependencies,
mainPackageName,
mainPackageVersion,
packageVersions,
@ -43,12 +43,12 @@ let
# tells if a dependency introduces a cycle
# -> needs to be built in a combined derivation
isCyclic = name: version:
cyclicDependencies ? "${name}"."${version}";
(getCyclicDependencies name version) != [];
mainPackageKey =
"${mainPackageName}#${mainPackageVersion}";
nodejsVersion = buildSystemAttrs.nodejsVersion;
nodejsVersion = subsystemAttrs.nodejsVersion;
nodejs =
pkgs."nodejs-${builtins.toString nodejsVersion}_x"
@ -80,11 +80,7 @@ let
buildPackageWithOtherBuilder {
inherit name version;
builder = builders.nodejs.node2nix;
inject =
lib.optionalAttrs (cyclicDependencies ? "${name}"."${version}") {
"${name}"."${version}" =
cyclicDependencies."${name}"."${version}";
};
inject = {};
};
in
built.defaultPackage;

65
src/builders/nodejs/node2nix/default.nix
View File

@ -12,8 +12,7 @@
}:
{
buildSystemAttrs,
cyclicDependencies,
subsystemAttrs,
mainPackageName,
mainPackageVersion,
getCyclicDependencies,
@ -28,14 +27,15 @@
let
b = builtins;
getDependencies = name: version:
(args.getDependencies name version) ++ (args.getCyclicDependencies name version);
getAllDependencies = name: version:
(args.getDependencies name version)
++ (args.getCyclicDependencies name version);
mainPackageKey = "${mainPackageName}#${mainPackageVersion}";
mainPackageDependencies = getDependencies mainPackageName mainPackageVersion;
mainPackageDependencies = getAllDependencies mainPackageName mainPackageVersion;
nodejsVersion = buildSystemAttrs.nodejsVersion;
nodejsVersion = subsystemAttrs.nodejsVersion;
nodejs =
pkgs."nodejs-${builtins.toString nodejsVersion}_x"
@ -43,45 +43,24 @@ let
node2nixEnv = node2nix nodejs;
# allSources =
# lib.mapAttrs
# (packageName: versions:
# lib.genAttrs versions
# (version: {
# inherit packageName version;
# name = utils.sanitizeDerivationName packageName;
# src = getSource packageName version;
# dependencies =
# # b.trace "current package: ${packageName}#${version}"
# lib.forEach
# (lib.filter
# (dep: (! builtins.elem dep mainPackageDependencies))
# (getDependencies packageName version))
# (dep:
# # b.trace "accessing allSources.${dep.name}.${dep.version}"
# b.trace "${dep.name}#${dep.version}"
# allSources."${dep.name}"."${dep.version}"
# );
# }))
# packageVersions;
makeSource = packageName: version: prevDeps:
let
depsFiltered =
(lib.filter
(dep:
! b.elem dep prevDeps)
(getAllDependencies packageName version));
parentDeps =
prevDeps ++ depsFiltered;
in
rec {
inherit packageName version;
name = utils.sanitizeDerivationName packageName;
src = getSource packageName version;
dependencies =
let
parentDeps = prevDeps ++ depsFiltered;
depsFiltered =
(lib.filter
(dep:
! b.elem dep prevDeps)
(getDependencies packageName version));
in
lib.forEach
depsFiltered
(dep: makeSource dep.name dep.version parentDeps);
lib.forEach
depsFiltered
(dep: makeSource dep.name dep.version parentDeps);
};
node2nixDependencies =
@ -96,14 +75,6 @@ let
packageName = mainPackageName;
version = mainPackageVersion;
dependencies = node2nixDependencies;
# buildInputs ? []
# npmFlags ? ""
# dontNpmInstall ? false
# preRebuild ? ""
# dontStrip ? true
# unpackPhase ? "true"
# buildPhase ? "true"
# meta ? {}
production = true;
bypassCache = true;
reconstructLock = true;

8
src/builders/python/simple-builder/default.nix
View File

@ -12,19 +12,19 @@
}:
let
python = pkgs."${dreamLock.buildSystem.pythonAttr}";
python = pkgs."${dreamLock._subsystem.pythonAttr}";
buildFunc =
if dreamLock.buildSystem.application then
if dreamLock._subsystem.application then
python.pkgs.buildPythonApplication
else
python.pkgs.buildPythonPackage;
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageName = dreamLock._generic.mainPackageName;
packageName =
if mainPackageName == null then
if dreamLock.buildSystem.application then
if dreamLock._subsystem.application then
"application"
else
"environment"

36
src/default.nix
View File

@ -101,20 +101,20 @@ let
cp -r ${externalDir}/* $out/external/
'';
# automatically find a suitable builder for a given generic lock
# automatically find a suitable builder for a given dream lock
findBuilder = dreamLock:
let
buildSystem = dreamLock.generic.buildSystem;
subsystem = dreamLock._generic.subsystem;
in
if ! builders ? "${buildSystem}" then
throw "Could not find any builder for subsystem '${buildSystem}'"
if ! builders ? "${subsystem}" then
throw "Could not find any builder for subsystem '${subsystem}'"
else
builders."${buildSystem}".default;
builders."${subsystem}".default;
# detect if granular or combined fetching must be used
findFetcher = dreamLock:
if null != dreamLock.generic.sourcesCombinedHash then
if null != dreamLock._generic.sourcesCombinedHash then
fetchers.combinedFetcher
else
fetchers.defaultFetcher;
@ -127,7 +127,7 @@ let
extract ? false,
}@args:
let
# if generic lock is a file, read and parse it
# if dream lock is a file, read and parse it
dreamLock' = (utils.readDreamLock { inherit dreamLock; }).lock;
fetcher =
@ -137,10 +137,10 @@ let
args.fetcher;
fetched = fetcher {
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageVersion = dreamLock.generic.mainPackageVersion;
mainPackageName = dreamLock._generic.mainPackageName;
mainPackageVersion = dreamLock._generic.mainPackageVersion;
sources = dreamLock'.sources;
sourcesCombinedHash = dreamLock'.generic.sourcesCombinedHash;
sourcesCombinedHash = dreamLock'._generic.sourcesCombinedHash;
};
fetchedSources = fetched.fetchedSources;
@ -194,7 +194,7 @@ let
});
dreamLock = lib.recursiveUpdate dreamLock' {
sources."${dreamLock'.generic.mainPackageName}"."${dreamLock'.generic.mainPackageVersion}" = {
sources."${dreamLock'._generic.mainPackageName}"."${dreamLock'._generic.mainPackageVersion}" = {
type = "path";
path = "${source}";
};
@ -231,8 +231,8 @@ let
builder,
name,
version,
inject,
}@args2:
inject ? {},
}:
let
subDreamLockLoaded =
utils.readDreamLock {
@ -255,8 +255,7 @@ let
;
inherit (dreamLockInterface)
buildSystemAttrs
cyclicDependencies
subsystemAttrs
getDependencies
getCyclicDependencies
mainPackageName
@ -272,7 +271,7 @@ let
outputs;
# produce outputs for a dream.lock or a source
# produce outputs for a dream-lock or a source
riseAndShine =
{
dreamLock ? null,
@ -327,16 +326,17 @@ let
inherit
dreamLock
fetchedSources
inject
sourceOverrides
;
builder = builder';
builderArgs = (args.builderArgs or {}) // {
packageOverrides =
lib.recursiveUpdate
(dreamOverrides."${dreamLock.generic.buildSystem}" or {})
(dreamOverrides."${dreamLock._generic.subsystem}" or {})
(args.packageOverrides or {});
};
inject =
utils.dreamLock.decompressDependencyGraph args.inject or {};
};
# Makes the packages tree compatible with flakes schema.

2
src/fetchers/combined-fetcher.nix
View File

@ -13,7 +13,7 @@
...
}:
{
# sources attrset from generic lock
# sources attrset from dream lock
sources,
sourcesCombinedHash,
}:

2
src/fetchers/default-fetcher.nix
View File

@ -7,7 +7,7 @@
...
}:
{
# sources attrset from generic lock
# sources attrset from dream lock
mainPackageName,
mainPackageVersion,
sources,

26
src/specifications/dream-lock-example.json
View File

@ -1,25 +1,29 @@
{
"generic": {
"buildSystem": "python",
"_generic": {
"_subsystem": "python",
"translatedBy": "python.impure.pip",
"translatorArgs": "",
"mainPackageName": "requests",
"mainPackageVersion": "1.2.3",
"removedDependencies": true,
"dependencyGraph": {
"requests": [
"certifi"
]
}
"mainPackageVersion": "1.2.3"
},
"buildSystem": {
"_subsystem": {
"pythonAttr": "python38",
"sourceFormats": {
"requests": "sdist",
"certifi": "wheel"
}
},
"cyclicDependencies": {
},
"dependencies": {
"requests": [
"certifi"
]
},
"sources": {
"requests": {

30
src/specifications/dream-lock-schema.json
View File

@ -4,7 +4,6 @@
"type": "object",
"properties": {
"sources": {
"type": "object",
"patternProperties": {
@ -134,28 +133,27 @@
}
},
"generic": {
"_generic": {
"type": "object",
"properties": {
"buildSystem": { "type": "string" },
"producedBy": { "type": "string" },
"dependencyGraph": {
"type": "object",
"properties": {
"^.*$": {
"type": "array",
"items": { "type": "string" }
}
}
}
"_subsystem": { "type": "string" },
"producedBy": { "type": "string" }
}
},
"buildSystem": {
"_subsystem": {
"description": "build system specifics",
"type": "object"
},
"dependencies": {
"type": "object",
"properties": {
"^.*$": {
"type": "array",
"items": { "type": "string" }
}
}
}
}
}

2
src/specifications/subsystems/nodejs/dream.lock.example.json
View File

@ -1,5 +1,5 @@
{
"buildSystem": {
"_subsystem": {
"nodejsVersion": 14
}
}

2
src/specifications/translator-call-example.json
View File

@ -9,6 +9,6 @@
],
"outputFile": [
"./a/b/c/dream.lock"
"./a/b/c/dream-lock.json"
]
}

14
src/templates/translators/pure.nix
View File

@ -21,23 +21,27 @@
in
# TODO: produce dream lock like in /specifications/dream-lock-example.json
rec {
sources = ;
generic = {
buildSystem = "nodejs";
_generic = {
subsystem = "nodejs";
producedBy = translatorName;
mainPackageName = "some_name";
mainPackageVersion = "some_version";
dependencyGraph = ;
sourcesCombinedHash = null;
};
# build system specific attributes
buildSystem = {
_subsystem = {
# example
nodejsVersion = 14;
};
dependencies = {};
cyclicDependencies = {};
sources = ;
};

16
src/translators/default.nix
View File

@ -39,7 +39,7 @@ let
};
buildSystems = utils.dirNames ./.;
subsystems = utils.dirNames ./.;
translatorTypes = [ "impure" "ifd" "pure" ];
@ -57,19 +57,19 @@ let
outputFile=$(jq '.outputFile' -c -r $jsonInputFile)
nix eval --show-trace --impure --raw --expr "
builtins.toJSON (
let
dream2nix = import ${dream2nixWithExternals} {};
dreamLock =
(import ${dream2nixWithExternals} {}).translators.translators.${
dream2nix.translators.translators.${
lib.concatStringsSep "." translatorAttrPath
}.translate
(builtins.fromJSON (builtins.readFile '''$1'''));
in
# don't use nix to detect cycles, this will be more efficient in python
dreamLock // {
generic = builtins.removeAttrs dreamLock.generic [ \"cyclicDependencies\" ];
}
)
dream2nix.utils.dreamLock.toJSON
# don't use nix to detect cycles, this will be more efficient in python
(dreamLock // {
_generic = builtins.removeAttrs dreamLock._generic [ \"cyclicDependencies\" ];
})
" | jq > $outputFile
'';
in

4
src/translators/nodejs/pure/package-lock/default.nix
View File

@ -82,8 +82,8 @@
(lib.filterAttrs
(pname: pdata: ! (pdata.dev or false) || dev)
parsedDependencies);
buildSystemName = "nodejs";
buildSystemAttrs = { nodejsVersion = args.nodejs; };
subsystemName = "nodejs";
subsystemAttrs = { nodejsVersion = args.nodejs; };
# functions
serializePackages = inputData:

23
src/translators/nodejs/pure/yarn-lock/default.nix
View File

@ -16,7 +16,6 @@
# extraArgs
name,
noDev,
noOptional,
peer,
...
}:
@ -24,7 +23,6 @@
let
b = builtins;
dev = ! noDev;
optional = ! noOptional;
sourceDir = lib.elemAt inputDirectories 0;
yarnLock = utils.readTextFile "${sourceDir}/yarn.lock";
@ -52,7 +50,7 @@
mainPackageName =
packageJSON.name or
(if name != null then name else
(if name != "{automatic}" then name else
throw (
"Could not identify package name. "
+ "Please specify extra argument 'name'"
@ -60,9 +58,9 @@
mainPackageVersion = packageJSON.version or "unknown";
buildSystemName = "nodejs";
subsystemName = "nodejs";
buildSystemAttrs = {
subsystemAttrs = {
nodejsVersion = 14;
};
@ -114,7 +112,7 @@
let
dependencies =
dependencyObject.dependencies or []
++ (lib.optionals optional (dependencyObject.optionalDependencies or []));
++ dependencyObject.optionalDependencies or [];
in
lib.forEach
dependencies
@ -213,9 +211,10 @@
hash =
lib.last (lib.splitString "#" dependencyObject.resolved);
in
if lib.stringLength hash == 40 then hash
else
throw "Missing integrity for ${dependencyObject.yarnName}";
if lib.stringLength hash == 40 then
hash
else
throw "Missing integrity for ${dependencyObject.yarnName}";
url = lib.head (lib.splitString "#" dependencyObject.resolved);
};
};
@ -260,6 +259,7 @@
"react"
"@babel/code-frame"
];
default = "{automatic}";
type = "argument";
};
@ -268,11 +268,6 @@
type = "flag";
};
noOptional = {
description = "Exclude optional dependencies";
type = "flag";
};
peer = {
description = "Include peer dependencies";
type = "flag";

2
src/translators/python/impure/pip/default.nix
View File

@ -93,7 +93,7 @@ in
-r $tmpBuild/computed_requirements
# -r ''${inputFiles/$'\n'/$' -r '}
# generate the generic lock from the downloaded list of files
# generate the dream lock from the downloaded list of files
NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \
$tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput

12
src/translators/python/impure/pip/generate-dream-lock.py
View File

@ -51,20 +51,20 @@ def main():
format=format
)
# create generic lock
# create dream lock
# This translator is not aware of the exact dependency graph.
# This restricts us to use a single derivation builder later,
# which will install all packages at once
dream_lock = dict(
sources={},
generic={
"buildSystem": "python",
_generic={
"subsystem": "python",
"mainPackageName": os.environ.get('NAME'),
"mainPackageVersion": os.environ.get('VERSION'),
"sourcesCombinedHash": None,
},
buildSystem={
_subsystem={
"application": jsonInput['application'],
"pythonAttr": f"python{sys.version_info.major}{sys.version_info.minor}",
"sourceFormats":
@ -72,7 +72,7 @@ def main():
}
)
# populate sources of generic lock
# populate sources of dream lock
for pname, data in packages.items():
if pname not in dream_lock['sources']:
dream_lock['sources'][pname] = {}
@ -82,7 +82,7 @@ def main():
type='http',
)
# dump generic lock to stdout (json)
# dump dream lock to $ouputFile
print(jsonInput['outputFile'])
with open(jsonInput['outputFile'], 'w') as lock:
json.dump(dream_lock, lock, indent=2)

14
src/utils/default.nix
View File

@ -39,6 +39,8 @@ rec {
readTextFile = file: lib.replaceStrings [ "\r\n" ] [ "\n" ] (b.readFile file);
traceJ = toTrace: eval: b.trace (b.toJSON toTrace) eval;
isFile = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "regular";
isDir = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "directory";
@ -109,16 +111,8 @@ rec {
sanitizeDerivationName = name:
lib.replaceStrings [ "@" "/" ] [ "__at__" "__slash__" ] name;
keyToNameVersion = key:
let
split = lib.splitString "#" key;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
{ inherit name version; };
nameVersionToKey = nameVersion:
"${nameVersion.name}#${nameVersion.version}";
nameVersionPair = name: version:
{ inherit name version; };
# determines if version v1 is greater than version v2
versionGreater = v1: v2:

194
src/utils/dream-lock.nix
View File

@ -15,7 +15,7 @@ let
}@args:
let
lock =
lockMaybeCompressed =
if b.isPath dreamLock
|| b.isString dreamLock
|| lib.isDerivation dreamLock then
@ -23,69 +23,32 @@ let
else
dreamLock;
mainPackageName = lock.generic.mainPackageName;
mainPackageVersion = lock.generic.mainPackageVersion;
lock =
if lockMaybeCompressed.decompressed or false then
lockMaybeCompressed
else
decompressDreamLock lockMaybeCompressed;
buildSystemAttrs = lock.buildSystem;
mainPackageName = lock._generic.mainPackageName;
mainPackageVersion = lock._generic.mainPackageVersion;
subsystemAttrs = lock._subsystem;
sources = lock.sources;
dependencyGraph = lock.generic.dependencyGraph;
dependencyGraph = lock.dependencies;
packageVersions =
let
allDependencyKeys =
lib.attrNames
(lib.genAttrs
(lib.flatten
((lib.attrValues dependencyGraph)
++ (lib.attrNames dependencyGraph)
++ [ "${mainPackageName}#${mainPackageVersion}" ]))
(x: null));
in
lib.foldl'
(packageVersions: dep:
packageVersions // {
"${dep.name}" = (packageVersions."${dep.name}" or []) ++ [
dep.version
];
})
{}
(b.map (utils.keyToNameVersion) allDependencyKeys);
cyclicDependencies =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: removedKeys:
lib.forEach removedKeys
(rKey: utils.keyToNameVersion rKey))
versions)
lock.generic.cyclicDependencies or {};
# Format:
# {
# "{name}#{version}": [
# { name=...; version=...; }
# { name=...; version=...; }
# ]
# }
dependenciesAttrs =
lib.mapAttrs
(key: deps:
lib.forEach deps
(dep: utils.keyToNameVersion dep))
(name: versions: lib.attrNames versions)
dependencyGraph;
cyclicDependencies = lock.cyclicDependencies;
getDependencies = pname: version:
if dependenciesAttrs ? "${pname}#${version}" then
# filter out cyclicDependencies
lib.filter
(dep: ! b.elem dep (cyclicDependencies."${pname}"."${version}" or []))
dependenciesAttrs."${pname}#${version}"
# assume no deps if package not found in dependencyGraph
else
[];
b.filter
(dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
dependencyGraph."${pname}"."${version}" or [];
getCyclicDependencies = pname: version:
cyclicDependencies."${pname}"."${version}" or [];
@ -98,8 +61,7 @@ let
inherit
mainPackageName
mainPackageVersion
buildSystemAttrs
cyclicDependencies
subsystemAttrs
getCyclicDependencies
getDependencies
packageVersions
@ -109,8 +71,8 @@ let
getMainPackageSource = dreamLock:
dreamLock.sources
."${dreamLock.generic.mainPackageName}"
."${dreamLock.generic.mainPackageVersion}";
."${dreamLock._generic.mainPackageName}"
."${dreamLock._generic.mainPackageVersion}";
getSource = fetchedSources: pname: version:
let
@ -141,7 +103,7 @@ let
in
lock // {
generic = lock.generic // {
_generic = lock._generic // {
mainPackageName = name;
mainPackageVersion = version;
};
@ -152,31 +114,123 @@ let
let
lock = (readDreamLock { inherit dreamLock; }).lock;
oldDependencyGraph = lock.generic.dependencyGraph;
oldDependencyGraph = lock.dependencies;
newDependencyGraph =
lib.mapAttrs
(key: deps:
let
oldDeps = oldDependencyGraph."${key}" or [];
in
(oldDeps
++
lib.filter (dep: ! b.elem dep oldDeps) deps))
(oldDependencyGraph // inject);
lib.zipAttrsWith
(name: versions:
lib.zipAttrsWith
(version: deps: lib.flatten deps)
versions)
[
oldDependencyGraph
inject
];
in
lib.recursiveUpdate lock {
generic.dependencyGraph = newDependencyGraph;
dependencies = newDependencyGraph;
};
decompressDependencyGraph = compGraph:
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: deps:
map
(dep: {
name = b.elemAt dep 0;
version = b.elemAt dep 1;
})
deps)
versions)
compGraph;
compressDependencyGraph = decompGraph:
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: deps: map ( dep: [ dep.name dep.version ]) deps)
versions)
decompGraph;
decompressDreamLock = comp:
let
dependencyGraphDecomp =
decompressDependencyGraph (comp.dependencies or {});
cyclicDependencies =
decompressDependencyGraph (comp.cyclicDependencies or {});
emptyDependencyGraph =
lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: source: [])
versions)
comp.sources;
dependencyGraph =
lib.recursiveUpdate
emptyDependencyGraph
dependencyGraphDecomp;
in
comp // {
decompressed = true;
cyclicDependencies = cyclicDependencies;
dependencies = dependencyGraph;
};
compressDreamLock = uncomp:
let
dependencyGraphComp =
compressDependencyGraph
uncomp.dependencies;
cyclicDependencies =
compressDependencyGraph
uncomp.cyclicDependencies;
dependencyGraph =
lib.filterAttrs
(name: versions: versions != {})
(lib.mapAttrs
(name: versions:
lib.filterAttrs
(version: deps: deps != [])
versions)
dependencyGraphComp);
in
(b.removeAttrs uncomp [ "decompressed" ]) // {
inherit cyclicDependencies;
dependencies = dependencyGraph;
};
toJSON = dreamLock:
let
lock =
if dreamLock.decompressed or false then
compressDreamLock dreamLock
else
dreamLock;
json = b.toJSON lock;
in
json;
in
{
inherit
compressDreamLock
decompressDreamLock
decompressDependencyGraph
getMainPackageSource
getSource
getSubDreamLock
readDreamLock
injectDependencies
toJSON
;
}

173
src/utils/translator.nix
View File

@ -17,8 +17,8 @@ let
mainPackageName,
mainPackageVersion,
mainPackageDependencies,
buildSystemName,
buildSystemAttrs,
subsystemName,
subsystemAttrs,
# functions
serializePackages,
@ -66,45 +66,46 @@ let
})
{}
serializedPackagesList;
dependencyGraph =
{
"${mainPackageName}#${mainPackageVersion}" =
lib.forEach mainPackageDependencies
(dep: "${dep.name}#${dep.version}");
}
//
lib.listToAttrs
(lib.forEach
serializedPackagesList
(pkgData: lib.nameValuePair
"${getName pkgData}#${getVersion pkgData}"
(b.map
(depNameVer: "${depNameVer.name}#${depNameVer.version}")
(getDependencies pkgData getDepByNameVer dependenciesByOriginalID))));
let
depGraph =
(lib.mapAttrs
(name: versions:
lib.mapAttrs
(version: pkgData:
getDependencies
pkgData
getDepByNameVer
dependenciesByOriginalID)
versions)
allDependencies);
in
depGraph // {
"${mainPackageName}" = depGraph."${mainPackageName}" or {} // {
"${mainPackageVersion}" = mainPackageDependencies;
};
};
allDependencyKeys =
lib.attrNames
(lib.genAttrs
(b.foldl'
(a: b: a ++ b)
[]
(lib.attrValues dependencyGraph))
(x: null));
let
depsWithDuplicates =
lib.flatten
(lib.flatten
(lib.mapAttrsToList
(name: versions: lib.attrValues versions)
dependencyGraph));
in
lib.unique depsWithDuplicates;
missingDependencies =
lib.flatten
(lib.forEach allDependencyKeys
(depKey:
let
split = lib.splitString "#" depKey;
name = b.elemAt split 0;
version = b.elemAt split 1;
in
if sources ? "${name}" && sources."${name}" ? "${version}" then
[]
else
{ inherit name version; }));
(dep:
if sources ? "${dep.name}"."${dep.version}" then
[]
else
dep));
generatedSources =
if missingDependencies == [] then
@ -127,17 +128,33 @@ let
allDependencies."${name}"."${version}";
cyclicDependencies =
# TODO: inefficient! Implement some kind of early cutoff
let
findCycles = node: prevNodes: cycles:
let
children = dependencyGraph."${node}";
cyclicChildren = lib.filter (child: prevNodes ? "${child}") children;
nonCyclicChildren = lib.filter (child: ! prevNodes ? "${child}") children;
children = dependencyGraph."${node.name}"."${node.version}";
cyclicChildren =
lib.filter
(child: prevNodes ? "${child.name}#${child.version}")
children;
nonCyclicChildren =
lib.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++
(b.map (child: { from = node; to = child; }) cyclicChildren);
prevNodes' = prevNodes // { "${node}" = null; };
# use set for efficient lookups
prevNodes' =
prevNodes
// { "${node.name}#${node.version}" = null; };
in
if nonCyclicChildren == [] then
cycles'
@ -147,59 +164,65 @@ let
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList = findCycles "${mainPackageName}#${mainPackageVersion}" {} [];
cyclesList =
findCycles
(utils.nameVersionPair mainPackageName mainPackageVersion)
{}
[];
in
b.foldl'
(cycles: cycle:
(
let
fromNameVersion = utils.keyToNameVersion cycle.from;
fromName = fromNameVersion.name;
fromVersion = fromNameVersion.version;
toNameVersion = utils.keyToNameVersion cycle.to;
toName = toNameVersion.name;
toVersion = toNameVersion.version;
reverse = (cycles."${toName}"."${toVersion}" or []);
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if reverse edge already in cycles, do nothing
if b.elem cycle.from reverse then
# if edge or reverse edge already in cycles, do nothing
if b.elem cycle.from reverse
|| b.elem cycle.to existing then
cycles
else
lib.recursiveUpdate
cycles
{
"${fromName}"."${fromVersion}" =
let
existing = cycles."${fromName}"."${fromVersion}" or [];
in
if b.elem cycle.to existing then
existing
else
existing ++ [ cycle.to ];
})
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [ cycle.to ];
}))
{}
cyclesList;
in
{
sources = allSources;
decompressed = true;
generic =
{
inherit
cyclicDependencies
mainPackageName
mainPackageVersion
;
buildSystem = buildSystemName;
sourcesCombinedHash = null;
translator = translatorName;
}
//
(lib.optionalAttrs (getDependencies != null) { inherit dependencyGraph; });
_generic =
{
inherit
mainPackageName
mainPackageVersion
;
subsystem = subsystemName;
sourcesCombinedHash = null;
translator = translatorName;
};
# build system specific attributes
_subsystem = subsystemAttrs;
# build system specific attributes
buildSystem = buildSystemAttrs;
};
inherit cyclicDependencies;
sources = allSources;
}
//
(lib.optionalAttrs
(getDependencies != null)
{ dependencies = dependencyGraph; });
in
{