nix-community/dream2nix
1
1
Fork 0
mirror of https://github.com/nix-community/dream2nix.git synced 2024-09-21 11:32:15 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #17 from DavHau/dev

Browse Source 
translator for yarn.lock
This commit is contained in:
DavHau 2021-10-07 12:09:32 +07:00 committed by GitHub
commit 914a125f76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 361 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
flake.lock
View File

@ -1,5 +1,21 @@
{
"nodes": {
"nix-parsec": {
"flake": false,
"locked": {
"lastModified": 1614150306,
"narHash": "sha256-h5fZgIhwvekg5WTNDdrV5EJ4zxXvlSqITmPVuz/OM+w=",
"owner": "nprindle",
"repo": "nix-parsec",
"rev": "a28e73fce98227cc46edfdbb8518697c0982e034",
"type": "github"
},
"original": {
"owner": "nprindle",
"repo": "nix-parsec",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1631015389,
@ -49,6 +65,7 @@
},
"root": {
"inputs": {
"nix-parsec": "nix-parsec",
"nixpkgs": "nixpkgs",
"node2nix": "node2nix",
"npmlock2nix": "npmlock2nix"

22
flake.nix
View File

@ -3,11 +3,18 @@
inputs = {
nixpkgs.url = "nixpkgs/nixos-unstable";
# required for translator nodejs/pure/package-lock
nix-parsec = { url = "github:nprindle/nix-parsec"; flake = false; };
# required for builder nodejs/node2nix
node2nix = { url = "github:svanderburg/node2nix"; flake = false; };
# required for translator nodejs/pure/npmlock2nix
npmlock2nix = { url = "github:nix-community/npmlock2nix"; flake = false; };
};
outputs = { self, nixpkgs, node2nix, npmlock2nix }:
outputs = { self, nix-parsec, nixpkgs, node2nix, npmlock2nix }:
let
lib = nixpkgs.lib;
@ -19,9 +26,10 @@
);
externalSourcesFor = forAllSystems (system: pkgs: pkgs.runCommand "dream2nix-vendored" {} ''
mkdir -p $out/{npmlock2nix,node2nix}
mkdir -p $out/{npmlock2nix,node2nix,nix-parsec}
cp ${npmlock2nix}/{internal.nix,LICENSE} $out/npmlock2nix/
cp ${node2nix}/{nix/node-env.nix,LICENSE} $out/node2nix/
cp ${nix-parsec}/{parsec,lexer}.nix $out/nix-parsec/
'');
dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
@ -52,13 +60,15 @@
);
devShell = forAllSystems (system: pkgs: pkgs.mkShell {
buildInputs = with pkgs; [
cntr
buildInputs = with pkgs;
(with pkgs; [
nixUnstable
];
])
++ lib.optionals stdenv.isLinux [ cntr ];
shellHook = ''
export NIX_PATH=nixpkgs=${nixpkgs}
export d2nExternalSources=${externalSourcesFor."${system}"}
'';
});
};

35
src/apps/cli/cli.py
View File

@ -45,6 +45,7 @@ class PackageCommand(Command):
flag=False,
multiple=True
),
option("force", None, "override existing files", flag=True),
]
def handle(self):
@ -65,11 +66,17 @@ class PackageCommand(Command):
output = './.'
if not os.path.isdir(output):
os.mkdir(output)
filesToCreate = ('default.nix', 'dream.lock')
if self.option('force'):
for f in filesToCreate:
if os.path.isfile(f):
os.remove(f)
else:
existingFiles = set(os.listdir(output))
if any(f in existingFiles for f in ('default.nix', 'dream.lock')):
if any(f in existingFiles for f in filesToCreate):
print(
f"output directory {output} already contains a default.nix "
"or dream.lock. Delete first!",
f"output directory {output} already contains a 'default.nix' "
"or 'dream.lock'. Delete first, or user '--force'.",
file=sys.stderr,
)
exit(1)
@ -96,8 +103,9 @@ class PackageCommand(Command):
else:
# check if source path exists
if not os.path.exists(source):
raise print(f"Input path '{path}' does not exist", file=sys.stdout)
print(f"Input source '{source}' does not exist", file=sys.stdout)
exit(1)
source = os.path.realpath(source)
# select translator
translatorsSorted = sorted(
@ -122,6 +130,18 @@ class PackageCommand(Command):
lambda t: [t['subsystem'], t['type'], t['name']] == translator.split(' (')[0].split('.'),
translatorsSorted,
))[0]
else:
translator = translator.split('.')
if len(translator) == 3:
translator = list(filter(
lambda t: [t['subsystem'], t['type'], t['name']] == translator,
translatorsSorted,
))[0]
elif len(translator) == 1:
translator = list(filter(
lambda t: [t['name']] == translator,
translatorsSorted,
))[0]
# raise error if any specified extra arg is unknown
unknown_extra_args = set(specified_extra_args.keys()) - set(translator['specialArgs'].keys())
@ -211,6 +231,7 @@ class PackageCommand(Command):
type="unknown",
version="unknown",
)
else:
for field in ('versionField',):
if field in mainSource:
del mainSource[field]
@ -290,8 +311,10 @@ class PackageCommand(Command):
# create default.nix
template = callNixFunction(
'apps.apps.cli2.templateDefaultNix',
dream2nixLocationRelative=os.path.relpath(dream2nix_src, output)
'apps.apps.cli.templateDefaultNix',
dream2nixLocationRelative=os.path.relpath(dream2nix_src, output),
dreamLock = lock,
sourcePathRelative = os.path.relpath(source, os.path.dirname(outputDefaultNix))
)
# with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
with open(outputDefaultNix, 'w') as defaultNix:

10
src/apps/cli/default.nix
View File

@ -29,7 +29,12 @@ in
templateDefaultNix =
{
dream2nixLocationRelative,
dreamLock,
sourcePathRelative,
}:
let
mainPackage = dreamLock.generic.mainPackage;
in
''
{
dream2nix ? import ${dream2nixLocationRelative} {},
@ -37,6 +42,11 @@ in
(dream2nix.riseAndShine {
dreamLock = ./dream.lock;
${lib.optionalString (dreamLock.sources."${mainPackage}".type == "unknown") ''
sourceOverrides = oldSources: {
"${mainPackage}" = ./${sourcePathRelative};
};
''}
}).package.overrideAttrs (old: {
})

11
src/builders/nodejs/node2nix/default.nix
View File

@ -1,18 +1,23 @@
# builder imported from node2nix
{
externals,
node2nix ? externals.node2nix,
lib,
pkgs,
# dream2nix inputs
externals,
node2nix ? externals.node2nix,
utils,
...
}:
{
fetchedSources,
dreamLock,
}:
}@args:
let
dreamLock = utils.readDreamLock { inherit (args) dreamLock; };
mainPackageName = dreamLock.generic.mainPackage;
nodejsVersion = dreamLock.buildSystem.nodejsVersion;

6
src/default.nix
View File

@ -14,7 +14,7 @@ let
b = builtins;
utils = pkgs.callPackage ./utils.nix {};
utils = callPackageDream ./utils {};
callPackageDream = f: args: pkgs.callPackage f (args // {
inherit callPackageDream;
@ -29,6 +29,10 @@ let
externals = {
npmlock2nix = pkgs.callPackage "${externalSources}/npmlock2nix/internal.nix" {};
node2nix = nodejs: pkgs.callPackage "${externalSources}/node2nix/node-env.nix" { inherit nodejs; };
nix-parsec = rec {
lexer = import "${externalSources}/nix-parsec/lexer.nix" { inherit parsec; };
parsec = import "${externalSources}/nix-parsec/parsec.nix";
};
};
config = builtins.fromJSON (builtins.readFile ./config.json);

27
src/templates/translators/impure.nix
View File

@ -42,12 +42,15 @@
inputFiles,
}@args:
{
inputDirectories = [];
# TODO: insert regex here that matches valid input file names
# examples:
# - ".*(requirements).*\\.txt"
# - ".*(package-lock\\.json)"
inputFiles = lib.filter (f: builtins.match "# TODO: your regex" f != null) args.inputFiles;
# - ''.*requirements.*\.txt''
# - ''.*package-lock\.json''
inputDirectories = lib.filter
(utils.containsMatchingFile [ ''TODO: regex1'' ''TODO: regex2'' ])
args.inputDirectories;
inputFiles = [];
};
@ -58,8 +61,14 @@
# String arguments contain a default value and examples. Flags do not.
specialArgs = {
# Example: boolean option
# Flags always default to 'false' if not specified by the user
dev-dependenices = {
description = "Include dev dependencies";
type = "flag";
};
# Example: string option
# This will be exposed by the translate CLI command as --arg_the-answer
the-answer = {
default = "42";
description = "The Answer to the Ultimate Question of Life";
@ -70,13 +79,5 @@
type = "argument";
};
# Example: boolean option
# This will be exposed by the translate CLI command as --flag_example-flag
# The default value of boolean flags is always false
flat-earth = {
description = "Is the earth flat";
type = "flag";
};
};
}

27
src/templates/translators/pure.nix
View File

@ -49,12 +49,15 @@
inputFiles,
}@args:
{
inputDirectories = [];
# TODO: insert regex here that matches valid input file names
# examples:
# - ".*(requirements).*\\.txt"
# - ".*(package-lock\\.json)"
inputFiles = lib.filter (f: builtins.match "# TODO: your regex" f != null) args.inputFiles;
# - ''.*requirements.*\.txt''
# - ''.*package-lock\.json''
inputDirectories = lib.filter
(utils.containsMatchingFile [ ''TODO: regex1'' ''TODO: regex2'' ])
args.inputDirectories;
inputFiles = [];
};
@ -65,8 +68,14 @@
# String arguments contain a default value and examples. Flags do not.
specialArgs = {
# Example: boolean option
# Flags always default to 'false' if not specified by the user
dev-dependenices = {
description = "Include dev dependencies";
type = "flag";
};
# Example: string option
# This will be exposed by the translate CLI command as --arg_the-answer
the-answer = {
default = "42";
description = "The Answer to the Ultimate Question of Life";
@ -77,13 +86,5 @@
type = "argument";
};
# Example: boolean option
# This will be exposed by the translate CLI command as --flag_example-flag
# The default value of boolean flags is always false
flat-earth = {
description = "Is the earth flat";
type = "flag";
};
};
}

2
src/translators/default.nix
View File

@ -51,7 +51,7 @@ let
jsonInputFile=$(realpath $1)
outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile)
nix eval --impure --raw --expr "
nix eval --show-trace --impure --raw --expr "
builtins.toJSON (
(import ${dream2nixWithExternals} {}).translators.translators.${
lib.concatStringsSep "." translatorAttrPath

14
src/translators/nodejs/pure/npmlock2nix/default.nix
View File

@ -65,7 +65,7 @@
# handle github dependency
if pdata ? from && pdata ? version then
let
githubData = parseGithubDepedency pdata;
githubData = parseGithubDependency pdata;
in
[ rec {
name = "${pname}#${version}";
@ -125,15 +125,6 @@
producedBy = translatorName;
mainPackage = parsed.name;
dependencyGraph =
{
"${parsed.name}" =
lib.mapAttrsToList
(pname: pdata: "${pname}#${getVersion pdata}")
(lib.filterAttrs
(pname: pdata: ! (pdata.dev or false) || dev)
parsed.dependencies);
}
//
lib.listToAttrs
(map
(dep: lib.nameValuePair dep.name dep.depsExact)
@ -157,8 +148,7 @@
(utils.containsMatchingFile [ ''.*package-lock\.json'' ''.*package.json'' ])
args.inputDirectories;
inputFiles =
lib.filter (f: builtins.match ''.*package-lock\.json'' f != null) args.inputFiles;
inputFiles = [];
};
specialArgs = {

121
src/translators/nodejs/pure/yarn-lock/default.nix Normal file
View File

@ -0,0 +1,121 @@
{
lib,
externals,
translatorName,
utils,
...
}:
{
translate =
{
inputDirectories,
inputFiles,
...
}:
let
b = builtins;
yarnLock = "${lib.elemAt inputDirectories 0}/yarn.lock";
packageJSON = b.fromJSON (b.readFile "${lib.elemAt inputDirectories 0}/package.json");
parser = import ./parser.nix { inherit lib; inherit (externals) nix-parsec;};
parsedLock = lib.foldAttrs (n: a: n // a) {} (parser.parseLock yarnLock).value;
nameFromLockName = lockName:
let
version = lib.last (lib.splitString "@" lockName);
in
lib.removeSuffix "@${version}" lockName;
sources = lib.mapAttrs' (dependencyName: dependencyAttrs:
let
name = nameFromLockName dependencyName;
in
lib.nameValuePair ("${name}#${dependencyAttrs.version}") (
if ! lib.hasInfix "@github:" dependencyName then
{
version = dependencyAttrs.version;
hash = dependencyAttrs.integrity;
url = lib.head (lib.splitString "#" dependencyAttrs.resolved);
type = "fetchurl";
}
else
let
gitUrlInfos = lib.splitString "/" dependencyAttrs.resolved;
in
{
type = "github";
rev = lib.elemAt gitUrlInfos 6;
owner = lib.elemAt gitUrlInfos 3;
repo = lib.elemAt gitUrlInfos 4;
}
)) parsedLock;
dependencyGraph = lib.mapAttrs' (dependencyName: dependencyAttrs:
let
name = nameFromLockName dependencyName;
dependencies = dependencyAttrs.dependencies or [] ++ dependencyAttrs.optionalDependencies or [];
graph = lib.forEach dependencies (dependency:
builtins.head (
lib.mapAttrsToList (name: value:
let
yarnName = "${name}@${value}";
version = parsedLock."${yarnName}".version;
in
"${name}#${version}"
) dependency
)
);
in
lib.nameValuePair ("${name}#${dependencyAttrs.version}") graph) parsedLock;
in
# TODO: produce dream lock like in /specifications/dream-lock-example.json
rec {
inherit sources;
generic = {
buildSystem = "nodejs";
producedBy = translatorName;
mainPackage = packageJSON.name;
inherit dependencyGraph;
sourcesCombinedHash = null;
};
# build system specific attributes
buildSystem = {
# example
nodejsVersion = 14;
};
};
# From a given list of paths, this function returns all paths which can be processed by this translator.
# This allows the framework to detect if the translator is compatible with the given inputs
# to automatically select the right translator.
compatiblePaths =
{
inputDirectories,
inputFiles,
}@args:
{
inputDirectories = lib.filter
(utils.containsMatchingFile [ ''.*yarn\.lock'' ''.*package.json'' ])
args.inputDirectories;
inputFiles = [];
};
# If the translator requires additional arguments, specify them here.
# There are only two types of arguments:
# - string argument (type = "argument")
# - boolean flag (type = "flag")
# String arguments contain a default value and examples. Flags do not.
specialArgs = {
# optionalDependencies = {
# description = "Whether to include optional dependencies";
# type = "flag";
# };
};
}

76
src/translators/nodejs/pure/yarn-lock/parser.nix Normal file
View File

@ -0,0 +1,76 @@
# Example: parse a yarn.lock file
#
# Load in nix repl and test, e.g.:
#
# nix-repl> parseConfigFile ./yarn.lock
# { type = "success"; value = ...; }
{ lib, nix-parsec }:
with nix-parsec.parsec;
rec {
inherit (nix-parsec) parsec;
# Skip spaces and line comments and newlines
spaceComments = nix-parsec.lexer.space
(skipWhile1 (c: c == " " || c == "\t" || c == "\n"))
(nix-parsec.lexer.skipLineComment "#")
fail;
charInsideQuotes = c: c != "\"";
charOutsideQuotes = c: c != "\"" && c != "\n" && c != " " && c != "," && c != ":";
unquotedString = takeWhile1 charOutsideQuotes;
newLine = string "\n";
# use the nix-parsec quotedString
quotedString = between (string "\"") (string "\"") (takeWhile1 charInsideQuotes);
# TODO add the relevant fmap to add the attributes
version = skipThen (string " version ") (thenSkip quotedString newLine);
# TODO instead of nextLine use an exact count of space and newline ?:w
resolved = skipThen (string " resolved ") (thenSkip quotedString newLine);
integrity = skipThen (string " integrity ") (thenSkip unquotedString newLine);
dependencyList = thenSkip (sepBy singleDependency newLine) newLine;
dependencies = skipThen (string " dependencies:\n") dependencyList;
optionalDependencies = skipThen (string " optionalDependencies:\n") dependencyList;
singleDependency = bind (skipThen (string " ") (alt quotedString unquotedString)) (parsed_dependency:
skipThen (string " ") (
bind quotedString (parsed_version:
pure { "${parsed_dependency}" = parsed_version; }
)
)
);
dependencyNames = thenSkip (sepBy (alt quotedString unquotedString) (string ", ")) (string ":\n");
dependencyAttrs = bind version (parsedVersion:
bind resolved (parsedResolved:
bind (optional integrity) (parsedIntegrity:
bind (optional dependencies) (parsedDependencies:
bind (optional optionalDependencies) (parsedOptionalDependencies:
pure (
{ version = parsedVersion; resolved = parsedResolved; }
//
(if parsedIntegrity == [ ] then { } else { integrity = builtins.head parsedIntegrity; })
//
(if parsedDependencies == [ ] then { } else { dependencies = builtins.head parsedDependencies; })
//
(if parsedOptionalDependencies == [ ] then { } else { optionalDependencies = builtins.head parsedOptionalDependencies; })
)
)))));
namesToAttrsList = namesList: dependencyAttrs: map (dependencyName: lib.nameValuePair dependencyName dependencyAttrs) namesList;
group =
bind dependencyNames (namesList:
fmap (parsedAttrs: builtins.listToAttrs (namesToAttrsList namesList parsedAttrs))
dependencyAttrs);
configFile =
(skipThen spaceComments
(thenSkip (sepBy group newLine) eof));
parseLock = path: nix-parsec.parsec.runParser configFile (builtins.readFile path);
}

5
src/utils.nix → src/utils/default.nix
View File

@ -4,6 +4,9 @@
nix,
runCommand,
writeScriptBin,
# dream2nix inputs
callPackageDream,
...
}:
let
@ -12,6 +15,8 @@ in
rec {
readDreamLock = callPackageDream ./readDreamLock.nix {};
isFile = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "regular";
isDir = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "directory";

28
src/utils/readDreamLock.nix Normal file
View File

@ -0,0 +1,28 @@
{
lib,
...
}:
let
b = builtins;
in
{
dreamLock,
}@args:
let
lock =
if b.isPath dreamLock then
b.fromJSON (b.readFile dreamLock)
else
dreamLock;
mainPackage = lock.generic.mainPackage;
dependencyGraph = lock.generic.dependencyGraph;
in
lib.recursiveUpdate
lock
{
generic.dependencyGraph."${mainPackage}" = dependencyGraph."${mainPackage}" or lib.attrNames dependencyGraph;
}