finalize new CLI

This commit is contained in:
DavHau 2021-10-05 12:25:39 +07:00
parent ca22d7b59b
commit 7f19e0662f
17 changed files with 273 additions and 482 deletions

View File

@ -83,8 +83,8 @@
#### Problem
- Design issues (FOD-impurity, Maintainability, etc.) cannot be fixed easily and lead to long term suffering of maintainers.
- Innovation often happens on individual tools and are not adapted ecosystem wide
- New nix features will not be easily adapted as this will require updating many individual tools.
- Innovation often happens on individual tools and are not adopted ecosystem wide
- New nix features will not be easily adopted as this will require updating many individual tools.
#### Solution

View File

@ -26,6 +26,7 @@
dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
externalSources = externalSourcesFor."${system}";
inherit pkgs;
inherit lib;
});
@ -45,9 +46,9 @@
lib.mapAttrs (appName: app:
{
type = "app";
program = builtins.toString app;
program = builtins.toString app.program;
}
) dream2nixFor."${system}".apps
) dream2nixFor."${system}".apps.apps
);
devShell = forAllSystems (system: pkgs: pkgs.mkShell {

View File

@ -1,323 +0,0 @@
import argparse
import json
import os
import re
import subprocess as sp
import sys
import tempfile
import networkx as nx
with open (os.environ.get("translatorsJsonFile")) as f:
translators = json.load(f)
def strip_hashes_from_lock(lock):
for source in lock['sources'].values():
if 'hash' in source:
del source['hash']
def order_dict(d):
return {k: order_dict(v) if isinstance(v, dict) else v
for k, v in sorted(d.items())}
def list_translators(args):
out = "Available translators per build system"
for subsystem, trans_types in translators.items():
displayed = []
for trans_type, translators_ in trans_types.items():
for trans_name, translator in translators_.items():
lines = tuple(
f"{trans_type}.{trans_name}",
)
if translator:
lines += (
f"\n special args:",
)
for argName, argData in translator.items():
if argData['type'] == 'argument':
lines += (
f"\n --arg_{argName} {{value}}",
f"\n description: {argData['description']}",
f"\n default: {argData['default']}",
f"\n examples: {', '.join(argData['examples'])}",
)
elif argData['type'] == 'flag':
lines += (
f"\n --flag_{argName}",
f"\n description: {argData['description']}",
)
else:
raise Exception(f"Unknown type '{argData['type']}' of argument '{arg_Name}'")
displayed.append(''.join(lines))
nl = '\n'
out += f"\n\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}"
print(out)
def translate(args):
dream2nix_src = os.environ.get("dream2nixSrc")
inputPaths = args.input
# collect special args
specialArgs = {}
for argName, argVal in vars(args).items():
if argName.startswith("arg_"):
specialArgs[argName[4:]] = argVal
elif argName.startswith("flag_"):
specialArgs[argName[5:]] = True
# check if all inputs exist
for path in inputPaths:
if not os.path.exists(path):
raise Exception(f"Input path '{path}' does not exist")
inputFiles = list(filter(lambda p: os.path.isfile(p), inputPaths))
inputFiles = list(map(lambda p:os.path.realpath(p), inputFiles))
inputDirectories = list(filter(lambda p: os.path.isdir(p), inputPaths))
inputDirectories = list(map(lambda p:os.path.realpath(p), inputDirectories))
# determine output directory
if os.path.isdir(args.output):
output = f"{args.output}/dream.lock"
else:
output = args.output
output = os.path.realpath(output)
# translator arguments
translatorInput = dict(
inputFiles=inputFiles,
inputDirectories=inputDirectories,
outputFile=output,
selector=args.translator or "",
)
translatorInput.update(specialArgs)
# remove output file if exists
if os.path.exists(output):
os.remove(output)
# dump translator arguments to json file and execute translator
with tempfile.NamedTemporaryFile("w") as inputJsonFile:
json.dump(translatorInput, inputJsonFile, indent=2)
inputJsonFile.seek(0) # flushes write cache
env = os.environ.copy()
env.update(dict(
FUNC_ARGS=inputJsonFile.name
))
procEval = sp.run(
[
"nix", "eval", "--impure", "--raw", "--expr",
f"((import {dream2nix_src} {{}}).translators.selectTranslatorJSON {{}})",
],
capture_output=True,
env=env
)
if procEval.returncode:
print("Selecting translator failed", file=sys.stdout)
print(procEval.stderr.decode(), file=sys.stderr)
exit(1)
# parse data for auto selected translator
resultEval = json.loads(procEval.stdout)
subsystem = resultEval['subsystem']
trans_type = resultEval['type']
trans_name = resultEval['name']
# include default values into input data
translatorInputWithDefaults = resultEval['SpecialArgsDefaults']
translatorInputWithDefaults.update(translatorInput)
json.dump(translatorInputWithDefaults, inputJsonFile, indent=2)
inputJsonFile.seek(0)
# build the translator bin
procBuild = sp.run(
[
"nix", "build", "--impure", "-o", "translator", "--expr",
f"(import {dream2nix_src} {{}}).translators.translators.{subsystem}.{trans_type}.{trans_name}.translateBin",
],
capture_output=True,
)
if procBuild.returncode:
print("Building translator failed", file=sys.stdout)
print(procBuild.stderr.decode(), file=sys.stderr)
exit(1)
# execute translator
translatorPath = os.path.realpath("translator")
os.remove("translator")
sp.run(
[f"{translatorPath}/bin/translate", inputJsonFile.name] + sys.argv[1:]
)
# raise error if output wasn't produced
if not os.path.isfile(output):
raise Exception(f"Translator failed to create dream.lock")
# read produced lock file
with open(output) as f:
lock = json.load(f)
# write translator information to lock file
lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}"
lock['generic']['translatorParams'] = " ".join(sys.argv[2:])
# clean up dependency graph
# remove empty entries
depGraph = lock['generic']['dependencyGraph']
if 'dependencyGraph' in lock['generic']:
for pname, deps in depGraph.copy().items():
if not deps:
del depGraph[pname]
# remove cyclic dependencies
edges = set()
for pname, deps in depGraph.items():
for dep in deps:
edges.add((pname, dep))
G = nx.DiGraph(sorted(list(edges)))
cycle_count = 0
removed_edges = []
for pname in list(depGraph.keys()):
try:
while True:
cycle = nx.find_cycle(G, pname)
cycle_count += 1
# remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
node_from, node_to = cycle[-1][0], cycle[-1][1]
G.remove_edge(node_from, node_to)
depGraph[node_from].remove(node_to)
removed_edges.append((node_from, node_to))
except nx.NetworkXNoCycle:
continue
if removed_edges:
removed_cycles_text = 'Removed Cyclic dependencies:'
for node, removed_node in removed_edges:
removed_cycles_text += f"\n {node} -> {removed_node}"
print(removed_cycles_text)
lock['generic']['dependencyCyclesRemoved'] = True
# calculate combined hash if --combined was specified
if args.combined:
print("Building FOD of combined sources to retrieve output hash")
# remove hashes from lock file and init sourcesCombinedHash with emtpy string
strip_hashes_from_lock(lock)
lock['generic']['sourcesCombinedHash'] = ""
with open(output, 'w') as f:
json.dump(lock, f, indent=2)
# compute FOD hash of combined sources
proc = sp.run(
[
"nix", "build", "--impure", "-L", "--expr",
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {output}; }}"
],
capture_output=True,
)
# read the output hash from the failed build log
match = re.search(r"FOD_PATH=(.*=)", proc.stderr.decode())
if not match:
print(proc.stderr.decode())
print(proc.stdout.decode())
raise Exception("Could not find FOD hash in FOD log")
hash = match.groups()[0]
print(f"Computed FOD hash: {hash}")
# store the hash in the lock
lock['generic']['sourcesCombinedHash'] = hash
# re-write dream.lock
with open(output, 'w') as f:
json.dump(order_dict(lock), f, indent=2)
print(f"Created {output}")
def parse_args():
parser = argparse.ArgumentParser(
prog="nix run dream2nix --",
# description="translate projects to nix",
)
sub = parser.add_subparsers(
title='actions',
description='valid actions',
help='which action to execute'
)
list_parser = sub.add_parser(
"list",
description="list available translators"
)
list_parser.set_defaults(func=list_translators)
# PARSER FOR TRANSLATOR
translate_parser = sub.add_parser(
"translate",
prog="translate",
description="translate projects to nix",
)
translate_parser.set_defaults(func=translate)
translate_parser.add_argument(
"-t", "--translator",
help="select translator (list via: 'dream2nix list')",
default=""
)
translate_parser.add_argument(
"-o", "--output",
help="output file/directory for the dream.lock",
default="./dream.lock"
)
translate_parser.add_argument(
"-c", "--combined",
help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
action="store_true"
)
translate_parser.add_argument(
"input",
help="input files or directories containing sources and metadata",
nargs="+"
)
# parse special args
# (custom parameters required by individual translators)
parsed, unknown = translate_parser.parse_known_args()
for arg in unknown:
if arg.startswith("--arg_"):
translate_parser.add_argument(arg.split('=')[0])
if arg.startswith("--flag_"):
translate_parser.add_argument(arg.split('=')[0], action='store_true')
args = parser.parse_args()
if not hasattr(args, "func"):
parser.print_help()
exit(1)
args.func(args)
def main():
args = parse_args()
if __name__ == "__main__":
main()

View File

@ -1,22 +0,0 @@
{
# from dream2nix
externalSources,
translators,
# from nixpkgs
python3,
writeScript,
...
}:
let
cliPython = python3.withPackages (ps: [ ps.networkx ]);
in
writeScript "cli" ''
export d2nExternalSources=${externalSources}
translatorsJsonFile=${translators.translatorsJsonFile} \
dream2nixSrc=${../../.} \
${cliPython}/bin/python ${./cli.py} "$@"
''

View File

@ -9,7 +9,7 @@ from cleo.helpers import option
import networkx as nx
dream2nix_src = "./src"
dream2nix_src = os.environ.get("dream2nixSrc")
@ -25,9 +25,9 @@ class PackageCommand(Command):
option(
"source",
None,
"source of the package",
"source of the package, can be a path or flake-like spec",
flag=False,
multiple=True
multiple=False
),
option("translator", None, "which translator to use", flag=False),
option("output", None, "output file/directory for the dream.lock", flag=False),
@ -59,6 +59,24 @@ class PackageCommand(Command):
)
}
# ensure output directory
output = self.option("output")
if not output:
output = './.'
if not os.path.isdir(output):
os.mkdir(output)
existingFiles = set(os.listdir(output))
if any(f in existingFiles for f in ('default.nix', 'dream.lock')):
print(
f"output directory {output} already contains a default.nix "
"or dream.lock. Delete first!",
file=sys.stderr,
)
exit(1)
output = os.path.realpath(output)
outputDreamLock = f"{output}/dream.lock"
outputDefaultNix = f"{output}/default.nix"
# verify source
source = self.option("source")
if not source:
@ -67,19 +85,20 @@ class PackageCommand(Command):
f"Source not specified. Defaulting to current directory: {source}",
file=sys.stderr,
)
# check if source is valid fetcher spec
sourceSpec = {}
if source.partition(':')[0] in os.environ.get("fetcherNames", None).split():
print(f"fetching source for '{source}'")
sourceSpec =\
callNixFunction("fetchers.translateShortcut", shortcut=source)
source =\
buildNixFunction("fetchers.fetchShortcut", shortcut=source)
else:
# check if source path exists
if not os.path.exists(source):
raise print(f"Input path '{path}' does not exist", file=sys.stdout)
exit(1)
# determine output file
output = self.option("output")
if not output:
output = './dream.lock'
if os.path.isdir(output):
output = f"{output}/dream.lock"
output = os.path.realpath(output)
# select translator
translatorsSorted = sorted(
list_translators_for_source(source),
@ -93,14 +112,14 @@ class PackageCommand(Command):
chosen = self.choice(
'Select translator',
list(map(
lambda t: f"{t['subsystem']}.{t['type']}.{t['name']}",
lambda t: f"{t['subsystem']}.{t['type']}.{t['name']}{' (compatible)' if t['compatible'] else ''}",
translatorsSorted
)),
0
)
translator = chosen
translator = list(filter(
lambda t: [t['subsystem'], t['type'], t['name']] == translator.split('.'),
lambda t: [t['subsystem'], t['type'], t['name']] == translator.split(' (')[0].split('.'),
translatorsSorted,
))[0]
@ -151,18 +170,13 @@ class PackageCommand(Command):
translator_input = dict(
inputFiles=[],
inputDirectories=[source],
outputFile=output,
outputFile=outputDreamLock,
)
translator_input.update(specified_extra_args)
# remove output file if exists
if os.path.exists(output):
os.remove(output)
# build the translator bin
t = translator
translator_path = buildNixDerivation(
translator_path = buildNixAttribute(
f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
)
@ -177,17 +191,32 @@ class PackageCommand(Command):
)
# raise error if output wasn't produced
if not os.path.isfile(output):
if not os.path.isfile(outputDreamLock):
raise Exception(f"Translator failed to create dream.lock")
# read produced lock file
with open(output) as f:
with open(outputDreamLock) as f:
lock = json.load(f)
# write translator information to lock file
lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}"
lock['generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
lock['generic']['translatorParams'] = " ".join(sys.argv[2:])
# add main package source
mainPackage = lock['generic']['mainPackage']
if mainPackage:
mainSource = sourceSpec.copy()
if not mainSource:
mainSource = dict(
type="unknown",
version="unknown",
)
for field in ('versionField',):
if field in mainSource:
del mainSource[field]
mainSource['version'] = sourceSpec[sourceSpec['versionField']]
lock['sources'][mainPackage] = mainSource
# clean up dependency graph
# remove empty entries
depGraph = lock['generic']['dependencyGraph']
@ -224,21 +253,21 @@ class PackageCommand(Command):
lock['generic']['dependencyCyclesRemoved'] = True
# calculate combined hash if --combined was specified
if args.combined:
if self.option('combined'):
print("Building FOD of combined sources to retrieve output hash")
# remove hashes from lock file and init sourcesCombinedHash with emtpy string
strip_hashes_from_lock(lock)
lock['generic']['sourcesCombinedHash'] = ""
with open(output, 'w') as f:
with open(outputDreamLock, 'w') as f:
json.dump(lock, f, indent=2)
# compute FOD hash of combined sources
proc = sp.run(
[
"nix", "build", "--impure", "-L", "--expr",
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {output}; }}"
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {outputDreamLock}; }}"
],
capture_output=True,
)
@ -256,10 +285,20 @@ class PackageCommand(Command):
lock['generic']['sourcesCombinedHash'] = hash
# re-write dream.lock
with open(output, 'w') as f:
with open(outputDreamLock, 'w') as f:
json.dump(order_dict(lock), f, indent=2)
print(f"Created {output}")
# create default.nix
template = callNixFunction(
'apps.apps.cli2.templateDefaultNix',
dream2nixLocationRelative=os.path.relpath(dream2nix_src, output)
)
# with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
with open(outputDefaultNix, 'w') as defaultNix:
defaultNix.write(template)
print(f"Created {output}/{{dream.lock,default.nix}}")
def callNixFunction(function_path, **kwargs):
@ -274,8 +313,11 @@ def callNixFunction(function_path, **kwargs):
[
"nix", "eval", "--impure", "--raw", "--expr",
f'''
let
d2n = (import {dream2nix_src} {{}});
in
builtins.toJSON (
(import {dream2nix_src} {{}}).{function_path} {{}}
(d2n.utils.makeCallableViaEnv d2n.{function_path}) {{}}
)
''',
],
@ -283,15 +325,47 @@ def callNixFunction(function_path, **kwargs):
env=env
)
if proc.returncode:
print(f"Failed calling '{function_path}'", file=sys.stderr)
print(f"Failed calling nix function '{function_path}'", file=sys.stderr)
print(proc.stderr.decode(), file=sys.stderr)
exit(1)
# parse data for auto selected translator
# parse result data
return json.loads(proc.stdout)
def buildNixDerivation(attribute_path):
def buildNixFunction(function_path, **kwargs):
with tempfile.NamedTemporaryFile("w") as input_json_file:
json.dump(dict(**kwargs), input_json_file, indent=2)
input_json_file.seek(0) # flushes write cache
env = os.environ.copy()
env.update(dict(
FUNC_ARGS=input_json_file.name
))
proc = sp.run(
[
"nix", "build", "--impure", "-o", "tmp-result", "--expr",
f'''
let
d2n = (import {dream2nix_src} {{}});
in
(d2n.utils.makeCallableViaEnv d2n.{function_path}) {{}}
''',
],
capture_output=True,
env=env
)
if proc.returncode:
print(f"Failed calling nix function '{function_path}'", file=sys.stderr)
print(proc.stderr.decode(), file=sys.stderr)
exit(1)
# return store path of result
result = os.path.realpath("tmp-result")
os.remove("tmp-result")
return result
def buildNixAttribute(attribute_path):
proc = sp.run(
[
"nix", "build", "--impure", "-o", "tmp-result", "--expr",
@ -318,6 +392,11 @@ def list_translators_for_source(sourcePath):
return list(sorted(translatorsList, key=lambda t: t['compatible']))
def order_dict(d):
return {k: order_dict(v) if isinstance(v, dict) else v
for k, v in sorted(d.items())}
application = Application("package")
application.add(PackageCommand())

View File

@ -1,21 +1,44 @@
{
# from dream2nix
dream2nixWithExternals,
externalSources,
fetchers,
translators,
# from nixpkgs
lib,
python3,
writeScript,
...
}:
let
b = builtins;
cliPython = python3.withPackages (ps: [ ps.networkx ps.cleo ]);
in
writeScript "cli" ''
export d2nExternalSources=${externalSources}
dream2nixSrc=${../../.} \
{
program = writeScript "cli" ''
dream2nixSrc=${dream2nixWithExternals} \
fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
${cliPython}/bin/python ${./cli.py} "$@"
''
'';
templateDefaultNix =
{
dream2nixLocationRelative,
}:
''
{
dream2nix ? import ${dream2nixLocationRelative} {},
}:
(dream2nix.riseAndShine {
dreamLock = ./dream.lock;
}).package.overrideAttrs (old: {
})
'';
}

View File

@ -0,0 +1,9 @@
{
dream2nix,
}:
(dream2nix.riseAndShine {
dreamLock = ./dream.lock;
}).package.overrideAttrs (old: {
})

View File

@ -8,8 +8,9 @@
let
cliPython = python3.withPackages (ps: [ ps.cleo ]);
in
writeScript "cli" ''
{
program = writeScript "contribute" ''
dream2nixSrc=${../../.} \
${cliPython}/bin/python ${./contribute.py} contribute "$@"
''
'';
}

View File

@ -3,11 +3,12 @@
callPackageDream,
externalSources,
location,
dream2nixWithExternals,
translators,
...
}:
{
rec {
apps = { inherit cli cli2 contribute install; };
# the unified translator cli
cli = callPackageDream (import ./cli) {};
@ -17,29 +18,5 @@
contribute = callPackageDream (import ./contribute) {};
# install the framework to a specified location by copying the code
install = callPackageDream ({ writeScript, }:
writeScript
"install"
''
target="$1"
if [[ "$target" == "" ]]; then
echo "specify target"
exit 1
fi
mkdir -p "$target"
if [ -n "$(ls -A $target)" ]; then
echo "target directory not empty"
exit 1
fi
cp -r ${location}/* $target/
mkdir $target/external
cp -r ${externalSources}/* $target/external/
chmod -R +w $target
echo "Installed dream2nix successfully to '$target'."
echo "Please check/modify settings in '$target/config.json'"
''
) {};
install = callPackageDream (import ./install) {};
}

View File

@ -0,0 +1,31 @@
{
runCommand,
writeScript,
# dream2nix inputs
dream2nixWithExternals,
...
}:
{
program = writeScript "install"
''
target="$1"
if [[ "$target" == "" ]]; then
echo "specify target"
exit 1
fi
mkdir -p "$target"
if [ -n "$(ls -A $target)" ]; then
echo "target directory not empty"
exit 1
fi
cp -r ${dream2nixWithExternals}/* $target/
chmod -R +w $target
echo "Installed dream2nix successfully to '$target'."
echo "Please check/modify settings in '$target/config.json'"
'';
}

View File

@ -12,13 +12,16 @@
let
b = builtins;
utils = pkgs.callPackage ./utils.nix {};
callPackageDream = f: args: pkgs.callPackage f (args // {
inherit callPackageDream;
inherit externals;
inherit externalSources;
inherit location;
inherit fetchers;
inherit dream2nixWithExternals;
inherit translators;
inherit utils;
});
@ -48,13 +51,23 @@ let
translators = callPackageDream ./translators {};
# the location of the dream2nix framework for self references (update scripts, etc.)
location = ./.;
dream2nixWithExternals =
if b.pathExists (./. + "/external") then
./.
else
pkgs.runCommand "dream2nix-full-src" {} ''
cp -r ${./.} $out
chmod +w $out
mkdir $out/external
ls -lah ${externalSources}
cp -r ${externalSources}/* $out/external/
'';
in
rec {
inherit apps builders fetchers finders location translators;
inherit apps builders fetchers finders dream2nixWithExternals translators utils;
# automatically find a suitable builder for a given generic lock
findBuilder = dreamLock:
@ -123,8 +136,7 @@ rec {
# build package defined by dream.lock
# TODO: rename to riseAndShine
buildPackage =
riseAndShine =
{
dreamLock,
builder ? findBuilder (parseLock dreamLock),

View File

@ -18,8 +18,8 @@
fetchedSources = lib.mapAttrs (pname: source:
if source.type == "unknown" then
"unknown"
else if fetchers ? "${source.type}" then
fetchSource source
else if fetchers.fetchers ? "${source.type}" then
fetchSource { inherit source; }
else throw "unsupported source type '${source.type}'"
) sources;
}

View File

@ -25,14 +25,17 @@ rec {
combinedFetcher = callPackageDream ./combined-fetcher.nix { inherit defaultFetcher; };
fetchSource = source:
fetchSource = { source, }:
let
fetcher = fetchers."${source.type}";
fetcherOutputs = fetcher.outputs source;
in
fetcherOutputs.fetched (source.hash or null);
fetchViaShortcut = shortcut:
fetchShortcut = { shortcut, }:
fetchSource { source = translateShortcut { inherit shortcut; }; };
translateShortcut = { shortcut, }:
let
checkArgs = fetcherName: args:
@ -48,17 +51,19 @@ rec {
else
args;
fetchViaHttpUrl =
translateHttpUrl =
let
fetcher = fetchers.fetchurl;
fetcherOutputs = fetchers.http.outputs { url = shortcut; };
in
rec {
{
type = "fetchurl";
hash = fetcherOutputs.calcHash "sha256";
fetched = fetcherOutputs.fetched hash;
url = shortcut;
versionField = fetcher.versionField;
};
fetchViaGitShortcut =
translateGitShortcut =
let
urlAndParams = lib.elemAt (lib.splitString "+" shortcut) 1;
splitUrlParams = lib.splitString "?" urlAndParams;
@ -75,12 +80,14 @@ rec {
args = params // { inherit url; };
fetcherOutputs = fetcher.outputs (checkArgs "git" args);
in
rec {
{
type = "git";
hash = fetcherOutputs.calcHash "sha256";
fetched = fetcherOutputs.fetched hash;
inherit url;
versionField = fetcher.versionField;
};
fetchViaRegularShortcut =
translateRegularShortcut =
let
splitNameParams = lib.splitString ":" (lib.removeSuffix "/" shortcut);
fetcherName = lib.elemAt splitNameParams 0;
@ -104,15 +111,16 @@ rec {
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
''
else
rec {
args // {
type = fetcherName;
hash = fetcherOutputs.calcHash "sha256";
fetched = fetcherOutputs.fetched hash;
versionField = fetcher.versionField;
};
in
if lib.hasPrefix "git+" (lib.head (lib.splitString ":" shortcut)) then
fetchViaGitShortcut
translateGitShortcut
else if lib.hasPrefix "http://" shortcut || lib.hasPrefix "https://" shortcut then
fetchViaHttpUrl
translateHttpUrl
else
fetchViaRegularShortcut;
translateRegularShortcut;
}

View File

@ -5,7 +5,7 @@
externalSources,
externals,
location,
dream2nixWithExternals,
utils,
...
}:
@ -50,11 +50,10 @@ let
jsonInputFile=$(realpath $1)
outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile)
export d2nExternalSources=${externalSources}
nix eval --impure --raw --expr "
builtins.toJSON (
(import ${location} {}).translators.translators.${
(import ${dream2nixWithExternals} {}).translators.translators.${
lib.concatStringsSep "." translatorAttrPath
}.translate
(builtins.fromJSON (builtins.readFile '''$1'''))
@ -85,7 +84,7 @@ let
translatorsList = lib.collect (v: v ? translateBin) translators;
# json file exposing all existing translators to CLI including their special args
translatorsForInput = utils.makeCallableViaEnv (
translatorsForInput =
{
inputDirectories,
inputFiles,
@ -99,8 +98,7 @@ let
type
;
compatible = t.compatiblePaths args == args;
})
);
});
# pupulates a translators special args with defaults
getSpecialArgsDefaults = specialArgsDef:

View File

@ -17,7 +17,13 @@
...
}:
let
parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0);
packageLock =
if inputDirectories != [] then
"${lib.elemAt inputDirectories 0}/package-lock.json"
else
lib.elemAt inputFiles 0;
parsed = externals.npmlock2nix.readLockfile packageLock;
parseGithubDependency = dependency:
externals.npmlock2nix.parseGitHubRef dependency.version;
@ -87,6 +93,8 @@
)
dependencies;
in
# the dream lock
rec {
sources =
let
@ -145,9 +153,12 @@
inputFiles,
}@args:
{
inputDirectories = [];
inputDirectories = lib.filter
(utils.containsMatchingFile [ ''.*package-lock\.json'' ''.*package.json'' ])
args.inputDirectories;
inputFiles =
lib.filter (f: builtins.match ".*(package-lock\\.json)" f != null) args.inputFiles;
lib.filter (f: builtins.match ''.*package-lock\.json'' f != null) args.inputFiles;
};
specialArgs = {

View File

@ -60,7 +60,8 @@
}@args:
{
inputDirectories = [];
inputFiles = lib.filter (f: builtins.match ".*(requirements).*\\.txt" f != null) args.inputFiles;
inputFiles = lib.filter (f: builtins.match ''.*requirements.*\.txt'' f != null) args.inputFiles;
};
# define special args and provide defaults

View File

@ -11,39 +11,23 @@ let
in
rec {
basename = path: lib.last (lib.splitString "/" path);
dirname = path: builtins.concatStringsSep "/" (lib.init (lib.splitString "/" path));
isFile = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "regular";
isFile = path: (builtins.readDir (dirname path))."${basename path}" == "regular";
isDir = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "directory";
isDir = path: (builtins.readDir (dirname path))."${basename path}" == "directory";
listFiles = path: lib.filterAttrs (n: v: v == "regular") (builtins.listDir path);
listFiles = path: lib.attrNames (lib.filterAttrs (n: v: v == "regular") (builtins.readDir path));
# directory names of a given directory
dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
matchTopLevelFiles = pattern: path:
# is dir
if isDir path then
builtins.all (f: matchTopLevelFiles pattern f) (listFiles path)
# is file
else
let
match = (builtins.match pattern path);
in
if match == null then false else builtins.any (m: m != null) match;
compatibleTopLevelPaths = pattern: paths:
lib.filter
(path:
matchTopLevelFiles
pattern
path
)
paths;
# Returns true if every given pattern is satisfied by at least one file name
# inside the given directory.
# Sub-directories are not recursed.
containsMatchingFile = patterns: dir:
lib.all
(pattern: lib.any (file: b.match pattern file != null) (listFiles dir))
patterns;
# allow a function to receive its input from an environment variable
# whenever an empty set is passed
@ -53,7 +37,7 @@ rec {
else
func args;
# hash the contents of a path via `nix hash-path`
hashPath = algo: path:
let
hashFile = runCommand "hash-${algo}" {} ''
@ -62,6 +46,7 @@ rec {
in
b.readFile hashFile;
# builder to create a shell script that has it's own PATH
writePureShellScript = availablePrograms: script: writeScriptBin "run" ''
export PATH="${lib.makeBinPath availablePrograms}"
tmpdir=$(${coreutils}/bin/mktemp -d)