nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2025-01-02 19:34:08 +03:00
Code Issues Projects Releases Wiki Activity

refactor dream.lock sources structure:

Browse Source 
- now accessed like sources.{name}.{version}
This commit is contained in:
DavHau 2021-10-22 14:29:38 +07:00
parent 8a96fb0f07
commit 6a4a1da0f1
16 changed files with 227 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
flake.nix
View File

@ -74,6 +74,7 @@
shellHook = ''
export NIX_PATH=nixpkgs=${nixpkgs}
export d2nExternalSources=${externalSourcesFor."${system}"}
export dream2nixWithExternals=${dream2nixFor."${system}".dream2nixWithExternals}
'';
});
};

24
src/apps/cli/commands/package.py
View File

@ -115,8 +115,10 @@ class PackageCommand(Command):
print(f"fetching source defined via existing dream.lock")
with open(source) as f:
sourceDreamLock = json.load(f)
sourceMainPackageName = sourceDreamLock['generic']['mainPackageName']
sourceMainPackageVersion = sourceDreamLock['generic']['mainPackageVersion']
sourceSpec =\
sourceDreamLock['sources'][sourceDreamLock['generic']['mainPackage']]
sourceDreamLock['sources'][sourceMainPackageName][sourceMainPackageVersion]
source = \
buildNixFunction("fetchers.fetchSource", source=sourceSpec, extract=True)
@ -268,15 +270,17 @@ class PackageCommand(Command):
])
# add main package source
mainPackage = lock['generic']['mainPackage']
if mainPackage:
mainSource = sourceSpec.copy()
if not mainSource:
mainSource = dict(
type="unknown",
version="unknown",
)
lock['sources'][mainPackage] = mainSource
mainPackageName = lock['generic']['mainPackageName']
mainPackageVersion = lock['generic']['mainPackageVersion']
mainSource = sourceSpec.copy()
if not mainSource:
mainSource = dict(
type="unknown",
version="unknown",
)
lock['sources'][mainPackageName] = {
mainPackageVersion: mainSource
}
# clean up dependency graph
# remove empty entries

6
src/apps/cli/commands/update.py
View File

@ -59,13 +59,15 @@ class UpdateCommand(Command):
cli_py = os.path.abspath(f"{__file__}/../../cli.py")
# delete the hash
mainPackageSource = lock['sources'][lock['generic']['mainPackage']]
mainpakcageName = lock['generic']['mainPackage']
mainpakcageVersion = lock['generic']['mainpackageVersion']
mainPackageSource = lock['sources'][mainPackageName][mainPackageVersion]
updatedSourceSpec = callNixFunction(
"fetchers.updateSource",
source=mainPackageSource,
newVersion=version,
)
lock['sources'][lock['generic']['mainPackage']] = updatedSourceSpec
lock['sources'][mainpakcageName][mainPackageVersion] = updatedSourceSpec
with tempfile.NamedTemporaryFile("w", suffix="dream.lock") as tmpDreamLock:
json.dump(lock, tmpDreamLock, indent=2)
tmpDreamLock.seek(0) # flushes write cache

16
src/apps/cli/default.nix
View File

@ -33,18 +33,26 @@ in
sourcePathRelative,
}:
let
mainPackage = dreamLock.generic.mainPackage;
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageVersion = dreamLock.generic.mainPackageVersion;
in
''
{
dream2nix ? import ${dream2nixLocationRelative} {},
dream2nix ? import (
let
dream2nixWithExternals = (builtins.getEnv "dream2nixWithExternals");
in
if dream2nixWithExternals != "" then dream2nixWithExternals else
throw '''
This default.nix is for debugging purposes and can only be evaluated within the dream2nix devShell env.
''') {},
}:
(dream2nix.riseAndShine {
dreamLock = ./dream.lock;
${lib.optionalString (dreamLock.sources."${mainPackage}".type == "unknown") ''
${lib.optionalString (dreamLock.sources."${mainPackageName}"."${mainPackageVersion}".type == "unknown") ''
sourceOverrides = oldSources: {
"${mainPackage}" = ./${sourcePathRelative};
"${mainPackageName}#${mainPackageVersion}" = ./${sourcePathRelative};
};
''}
}).package.overrideAttrs (old: {

9
src/apps/cli/templateDefault.nix
View File

@ -1,9 +0,0 @@
{
dream2nix,
}:
(dream2nix.riseAndShine {
dreamLock = ./dream.lock;
}).package.overrideAttrs (old: {
})

229
src/builders/nodejs/granular/default.nix
View File

@ -42,20 +42,28 @@ let
lib.genAttrs standaloneKeys
(key:
(builders.nodejs.node2nix (args // {
dreamLock = lib.recursiveUpdate dreamLock
{
generic.mainPackage = key;
dreamLock =
let
nameVer = lib.splitString "#" key;
name = b.elemAt nameVer 0;
version = b.elemAt nameVer 1;
in
lib.recursiveUpdate dreamLock
{
generic.mainPackagenName = name;
generic.mainPackagenVersion = Version;
# re-introduce removed dependencies
generic.dependencyGraph."${key}" =
# b.trace "re-introduce removed ${b.toString dreamLock.generic.dependenciesRemoved."${key}" or []}"
dreamLock.generic.dependencyGraph."${key}"
++ dreamLock.generic.dependenciesRemoved."${key}" or [];
};
# re-introduce removed dependencies
generic.dependencyGraph."${key}" =
# b.trace "re-introduce removed ${b.toString dreamLock.generic.dependenciesRemoved."${key}" or []}"
dreamLock.generic.dependencyGraph."${key}"
++ dreamLock.generic.dependenciesRemoved."${key}" or [];
};
})).package
);
mainPackageKey = dreamLock.generic.mainPackage;
mainPackageKey =
"${dreamLock.generic.mainPackageName}#${dreamLock.generic.mainPackageVersion}";
nodejsVersion = dreamLock.buildSystem.nodejsVersion;
@ -70,125 +78,126 @@ let
allPackages = lib.genAttrs (lib.attrNames fetchedSources) makePackage;
makePackage = pkgKey:
standAlonePackages."${pkgKey}"
or
(stdenv.mkDerivation (applyOverrides rec {
makePackage = name: version:
let
pkgKey = "${name}#${version}";
in
standAlonePackages."${name}#${version}"
or
(stdenv.mkDerivation (applyOverrides rec {
packageName = (lib.head (lib.splitString "#" pkgKey));
pname =
lib.replaceStrings [ "@" "/" ] [ "_at_" "__" ]
packageName;
packageName = name;
pname = lib.straings.sanitizeDerivationName name;
version = dreamLock.sources."${pkgKey}".version;
inherit version;
src = fetchedSources."${pkgKey}";
src = fetchedSources."${pkgKey}";
buildInputs = [ nodejs ];
buildInputs = [ nodejs ];
nodeDeps =
lib.forEach
(dependencyGraph."${pkgKey}" or [])
(depKey:
allPackages."${depKey}"
)
++
lib.forEach (dreamLock.generic.dependenciesRemoved."${pkgKey}" or [])
(removedDep: standAlonePackages."${pkgKey}");
nodeDeps =
lib.forEach
(dependencyGraph."${pkgKey}" or [])
(depKey:
allPackages."${depKey}"
)
++
lib.forEach (dreamLock.generic.dependenciesRemoved."${pkgKey}" or [])
(removedDep: standAlonePackages."${pkgKey}");
dontUnpack = true;
dontUnpack = true;
installPhase = ''
runHook preInstall
installPhase = ''
runHook preInstall
nodeModules=$out/lib/node_modules
nodeModules=$out/lib/node_modules
mkdir -p $nodeModules
mkdir -p $nodeModules
cd $TMPDIR
cd $TMPDIR
unpackFile ${src}
unpackFile ${src}
# Make the base dir in which the target dependency resides first
mkdir -p "$(dirname "$nodeModules/${packageName}")"
# Make the base dir in which the target dependency resides first
mkdir -p "$(dirname "$nodeModules/${packageName}")"
# install source
if [ -f "${src}" ]
then
# Figure out what directory has been unpacked
packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w "$packageDir"
# Move the extracted tarball into the output folder
mv "$packageDir" "$nodeModules/${packageName}"
elif [ -d "${src}" ]
then
strippedName="$(stripHash ${src})"
# Restore write permissions
chmod -R u+w "$strippedName"
# Move the extracted directory into the output folder
mv "$strippedName" "$nodeModules/${packageName}"
fi
# repair 'link:' -> 'file:'
mv $nodeModules/${packageName}/package.json $nodeModules/${packageName}/package.json.old
cat $nodeModules/${packageName}/package.json.old | sed 's!link:!file\:!g' > $nodeModules/${packageName}/package.json
rm $nodeModules/${packageName}/package.json.old
# symlink dependency packages into node_modules
for dep in $nodeDeps; do
if [ -e $dep/lib/node_modules ]; then
for module in $(ls $dep/lib/node_modules); do
if [[ $module == @* ]]; then
for submodule in $(ls $dep/lib/node_modules/$module); do
mkdir -p $nodeModules/${packageName}/node_modules/$module
echo "ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule"
ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule
done
else
mkdir -p $nodeModules/${packageName}/node_modules/
echo "ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module"
ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module
fi
done
fi
done
cd "$nodeModules/${packageName}"
export HOME=$TMPDIR
npm --offline --production --nodedir=${nodeSources} install
# Create symlink to the deployed executable folder, if applicable
if [ -d "$nodeModules/.bin" ]
# install source
if [ -f "${src}" ]
then
ln -s $nodeModules/.bin $out/bin
# Figure out what directory has been unpacked
packageDir="$(find . -maxdepth 1 -type d | tail -1)"
# Restore write permissions
find "$packageDir" -type d -exec chmod u+x {} \;
chmod -R u+w "$packageDir"
# Move the extracted tarball into the output folder
mv "$packageDir" "$nodeModules/${packageName}"
elif [ -d "${src}" ]
then
strippedName="$(stripHash ${src})"
# Restore write permissions
chmod -R u+w "$strippedName"
# Move the extracted directory into the output folder
mv "$strippedName" "$nodeModules/${packageName}"
fi
# Create symlinks to the deployed manual page folders, if applicable
if [ -d "$nodeModules/${packageName}/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/${packageName}/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
# repair 'link:' -> 'file:'
mv $nodeModules/${packageName}/package.json $nodeModules/${packageName}/package.json.old
cat $nodeModules/${packageName}/package.json.old | sed 's!link:!file\:!g' > $nodeModules/${packageName}/package.json
rm $nodeModules/${packageName}/package.json.old
# symlink dependency packages into node_modules
for dep in $nodeDeps; do
if [ -e $dep/lib/node_modules ]; then
for module in $(ls $dep/lib/node_modules); do
if [[ $module == @* ]]; then
for submodule in $(ls $dep/lib/node_modules/$module); do
mkdir -p $nodeModules/${packageName}/node_modules/$module
echo "ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule"
ln -s $dep/lib/node_modules/$module/$submodule $nodeModules/${packageName}/node_modules/$module/$submodule
done
else
mkdir -p $nodeModules/${packageName}/node_modules/
echo "ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module"
ln -s $dep/lib/node_modules/$module $nodeModules/${packageName}/node_modules/$module
fi
done
fi
fi
done
runHook postInstall
'';
}));
cd "$nodeModules/${packageName}"
export HOME=$TMPDIR
npm --offline --production --nodedir=${nodeSources} install
# Create symlink to the deployed executable folder, if applicable
if [ -d "$nodeModules/.bin" ]
then
ln -s $nodeModules/.bin $out/bin
fi
# Create symlinks to the deployed manual page folders, if applicable
if [ -d "$nodeModules/${packageName}/man" ]
then
mkdir -p $out/share
for dir in "$nodeModules/${packageName}/man/"*
do
mkdir -p $out/share/man/$(basename "$dir")
for page in "$dir"/*
do
ln -s $page $out/share/man/$(basename "$dir")
done
done
fi
runHook postInstall
'';
}));
package = makePackage mainPackageKey;

23
src/builders/nodejs/node2nix/default.nix
View File

@ -17,9 +17,13 @@
...
}@args:
let
b = builtins;
dreamLock = utils.readDreamLock { inherit (args) dreamLock; };
mainPackageName = dreamLock.generic.mainPackage;
mainPackageName = dreamLock.generic.mainPackageName;
mainPackageVersion = dreamLock.generic.mainPackageVersion;
mainPackageKey = "${mainPackageName}#${mainPackageVersion}";
nodejsVersion = dreamLock.buildSystem.nodejsVersion;
@ -33,17 +37,18 @@ let
let
makeSource = name:
let
packageName = lib.head (lib.splitString "#" name);
nameVer = lib.splitString "#" name;
packageName = lib.elemAt nameVer 0;
version = lib.elemAt nameVer 1;
in
{
inherit packageName;
inherit packageName version;
name = lib.strings.sanitizeDerivationName packageName;
version = dreamLock.sources."${name}".version;
src = fetchedSources."${name}";
dependencies =
lib.forEach
(lib.filter
(depName: ! builtins.elem depName dreamLock.generic.dependencyGraph."${mainPackageName}")
(depName: ! builtins.elem depName dreamLock.generic.dependencyGraph."${mainPackageKey}")
(dreamLock.generic.dependencyGraph."${name}" or []))
(dependency:
makeSource dependency
@ -51,14 +56,14 @@ let
};
in
lib.forEach
dreamLock.generic.dependencyGraph."${mainPackageName}"
dreamLock.generic.dependencyGraph."${mainPackageKey}"
(dependency: makeSource dependency);
callNode2Nix = funcName: args:
node2nixEnv."${funcName}" rec {
name = lib.strings.sanitizeDerivationName packageName;
packageName = lib.head (lib.splitString "#" mainPackageName);
version = dreamLock.sources."${mainPackageName}".version;
packageName = mainPackageName;
version = mainPackageVersion;
dependencies = node2nixDependencies;
# buildInputs ? []
# npmFlags ? ""
@ -71,7 +76,7 @@ let
production = true;
bypassCache = true;
reconstructLock = true;
src = fetchedSources."${dreamLock.generic.mainPackage}";
src = fetchedSources."${mainPackageKey}";
}
// args;

2
src/builders/python/simple-builder/default.nix
View File

@ -20,7 +20,7 @@ let
else
python.pkgs.buildPythonPackage;
mainPackageName = dreamLock.generic.mainPackage;
mainPackageName = dreamLock.generic.mainPackageName;
packageName =
if mainPackageName == null then

2
src/default.nix
View File

@ -163,7 +163,7 @@ rec {
};
dreamLock = lib.recursiveUpdate dreamLock' {
sources."${dreamLock'.generic.mainPackage}" = {
sources."${dreamLock'.generic.mainPackageName}"."${dreamLock'.generic.mainPackageVersion}" = {
type = "path";
path = source;
version = "unknown";

27
src/fetchers/default-fetcher.nix
View File

@ -15,11 +15,24 @@
{
# attrset: pname -> path of downloaded source
fetchedSources = lib.mapAttrs (pname: source:
if source.type == "unknown" then
"unknown"
else if fetchers.fetchers ? "${source.type}" then
fetchSource { inherit source; }
else throw "unsupported source type '${source.type}'"
) sources;
fetchedSources =
lib.listToAttrs
(lib.flatten
(lib.mapAttrsToList
(pname: versions:
# list of name value pairs
lib.mapAttrsToList
(version: source:
lib.nameValuePair
"${pname}#${version}"
(if source.type == "unknown" then
"unknown"
else if fetchers.fetchers ? "${source.type}" then
fetchSource { inherit source; }
else throw "unsupported source type '${source.type}'")
)
versions
)
sources))
;
}

27
src/specifications/dream-lock-example.json
View File

@ -3,7 +3,8 @@
"buildSystem": "python",
"translatedBy": "python.impure.pip",
"translatorArgs": "",
"mainPackage": "requests",
"mainPackageName": "requests",
"mainPackageVersion": "1.2.3",
"removedDependencies": true,
"dependencyGraph": {
"requests": [
@ -21,17 +22,21 @@
},
"sources": {
"requests#1.2.3": {
"url": "https://download.pypi.org/requests/2.28.0",
"hash": "000000000000000000000000000000000000000",
"version": "1.2.3",
"type": "fetchurl"
"requests": {
"1.2.3": {
"url": "https://download.pypi.org/requests/2.28.0",
"hash": "000000000000000000000000000000000000000",
"version": "1.2.3",
"type": "fetchurl"
}
},
"certifi#2.3.4": {
"url": "https://download.pypi.org/certifi/2.0",
"hash": "000000000000000000000000000000000000000",
"version": "2.3.4",
"type": "fetchurl"
"certifi": {
"2.3.4": {
"url": "https://download.pypi.org/certifi/2.0",
"hash": "000000000000000000000000000000000000000",
"version": "2.3.4",
"type": "fetchurl"
}
}
}
}

3
src/templates/translators/pure.nix
View File

@ -26,7 +26,8 @@
generic = {
buildSystem = "nodejs";
producedBy = translatorName;
mainPackage = parsed.name;
mainPackageName = "some_name";
mainPackageVersion = "some_version";
dependencyGraph = ;
sourcesCombinedHash = null;
};

3
src/translators/python/impure/pip/default.nix
View File

@ -89,7 +89,8 @@ in
# -r ''${inputFiles/$'\n'/$' -r '}
# generate the generic lock from the downloaded list of files
MAIN=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
NAME=$(${jq}/bin/jq '.name' -c -r $tmpBuild/python.json) \
VERSION=$(${jq}/bin/jq '.version' -c -r $tmpBuild/python.json) \
$tmpBuild/python/bin/python ${./generate-dream-lock.py} $tmp $jsonInput
rm -rf $tmp $tmpBuild

10
src/translators/python/impure/pip/generate-dream-lock.py
View File

@ -36,7 +36,7 @@ def main():
# example: requests-2.26.0.tar.gz
else:
format = 'sdist'
pname, _, _ = file.rpartition('-')
pname, version, _ = file.rpartition('-')
pyver = 'source'
url = f"https://files.pythonhosted.org/packages/{pyver}/{pname[0]}/{pname}/{file}"
@ -45,6 +45,7 @@ def main():
sha256 = f"sha256-{base64.b64encode(hashlib.sha256(f.read()).digest()).decode()}"
packages[pname] = dict(
version=version,
url=url,
sha256=sha256,
format=format
@ -58,7 +59,8 @@ def main():
sources={},
generic={
"buildSystem": "python",
"mainPackage": os.environ.get('MAIN'),
"mainPackageName": os.environ.get('NAME'),
"mainPackageVersion": os.environ.get('VERSION'),
"sourcesCombinedHash": None,
},
@ -72,7 +74,9 @@ def main():
# populate sources of generic lock
for pname, data in packages.items():
dream_lock['sources'][pname] = dict(
if pname not in dream_lock['sources']:
dream_lock['sources'][pname] = {}
dream_lock['sources'][pname][data['version']] = dict(
url=data['url'],
hash=data['sha256'],
type='fetchurl',

4
src/utils/dream-lock.nix
View File

@ -22,7 +22,9 @@ let
lock;
getMainPackageSource = dreamLock:
dreamLock.sources."${dreamLock.generic.mainPackage}";
dreamLock.sources
."${dreamLock.generic.mainPackageName}"
."${dreamLock.generic.mainPackageVersion}";
in

20
src/utils/translator.nix
View File

@ -51,14 +51,16 @@ let
sources = b.foldl'
(result: pkgData: lib.recursiveUpdate result {
"${getName pkgData}#${getVersion pkgData}" =
let
type = getSourceType pkgData;
constructedArgs =
(sourceConstructors."${type}" pkgData)
// { inherit type; };
in
fetchers.constructSource constructedArgs;
"${getName pkgData}" = {
"${getVersion pkgData}" =
let
type = getSourceType pkgData;
constructedArgs =
(sourceConstructors."${type}" pkgData)
// { inherit type; };
in
fetchers.constructSource constructedArgs;
};
})
{}
serializedPackagesList;
@ -91,8 +93,8 @@ let
generic =
{
inherit mainPackageName mainPackageVersion;
buildSystem = buildSystemName;
mainPackage = "${mainPackageName}#${mainPackageVersion}";
sourcesCombinedHash = null;
translator = translatorName;
}