mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-09-19 02:17:14 +03:00
finalize new CLI
This commit is contained in:
DavHau
parent
ca22d7b59b
commit
7f19e0662f
@ -83,8 +83,8 @@
|
||||
#### Problem
|
||||
|
||||
- Design issues (FOD-impurity, Maintainability, etc.) cannot be fixed easily and lead to long term suffering of maintainers.
|
||||
- Innovation often happens on individual tools and are not adapted ecosystem wide
|
||||
- New nix features will not be easily adapted as this will require updating many individual tools.
|
||||
- Innovation often happens on individual tools and are not adopted ecosystem wide
|
||||
- New nix features will not be easily adopted as this will require updating many individual tools.
|
||||
|
||||
#### Solution
|
||||
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
|
||||
dream2nixFor = forAllSystems (system: pkgs: import ./src rec {
|
||||
externalSources = externalSourcesFor."${system}";
|
||||
inherit pkgs;
|
||||
inherit lib;
|
||||
});
|
||||
|
||||
@ -45,9 +46,9 @@
|
||||
lib.mapAttrs (appName: app:
|
||||
{
|
||||
type = "app";
|
||||
program = builtins.toString app;
|
||||
program = builtins.toString app.program;
|
||||
}
|
||||
) dream2nixFor."${system}".apps
|
||||
) dream2nixFor."${system}".apps.apps
|
||||
);
|
||||
|
||||
devShell = forAllSystems (system: pkgs: pkgs.mkShell {
|
||||
|
||||
@ -1,323 +0,0 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess as sp
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import networkx as nx
|
||||
|
||||
|
||||
with open (os.environ.get("translatorsJsonFile")) as f:
|
||||
translators = json.load(f)
|
||||
|
||||
|
||||
def strip_hashes_from_lock(lock):
|
||||
for source in lock['sources'].values():
|
||||
if 'hash' in source:
|
||||
del source['hash']
|
||||
|
||||
|
||||
def order_dict(d):
|
||||
return {k: order_dict(v) if isinstance(v, dict) else v
|
||||
for k, v in sorted(d.items())}
|
||||
|
||||
|
||||
def list_translators(args):
|
||||
out = "Available translators per build system"
|
||||
for subsystem, trans_types in translators.items():
|
||||
displayed = []
|
||||
for trans_type, translators_ in trans_types.items():
|
||||
for trans_name, translator in translators_.items():
|
||||
lines = tuple(
|
||||
f"{trans_type}.{trans_name}",
|
||||
)
|
||||
if translator:
|
||||
lines += (
|
||||
f"\n special args:",
|
||||
)
|
||||
for argName, argData in translator.items():
|
||||
if argData['type'] == 'argument':
|
||||
lines += (
|
||||
f"\n --arg_{argName} {{value}}",
|
||||
f"\n description: {argData['description']}",
|
||||
f"\n default: {argData['default']}",
|
||||
f"\n examples: {', '.join(argData['examples'])}",
|
||||
)
|
||||
elif argData['type'] == 'flag':
|
||||
lines += (
|
||||
f"\n --flag_{argName}",
|
||||
f"\n description: {argData['description']}",
|
||||
)
|
||||
else:
|
||||
raise Exception(f"Unknown type '{argData['type']}' of argument '{arg_Name}'")
|
||||
displayed.append(''.join(lines))
|
||||
nl = '\n'
|
||||
out += f"\n\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}"
|
||||
print(out)
|
||||
|
||||
|
||||
def translate(args):
|
||||
|
||||
dream2nix_src = os.environ.get("dream2nixSrc")
|
||||
|
||||
inputPaths = args.input
|
||||
|
||||
# collect special args
|
||||
specialArgs = {}
|
||||
for argName, argVal in vars(args).items():
|
||||
if argName.startswith("arg_"):
|
||||
specialArgs[argName[4:]] = argVal
|
||||
elif argName.startswith("flag_"):
|
||||
specialArgs[argName[5:]] = True
|
||||
|
||||
# check if all inputs exist
|
||||
for path in inputPaths:
|
||||
if not os.path.exists(path):
|
||||
raise Exception(f"Input path '{path}' does not exist")
|
||||
|
||||
inputFiles = list(filter(lambda p: os.path.isfile(p), inputPaths))
|
||||
inputFiles = list(map(lambda p:os.path.realpath(p), inputFiles))
|
||||
inputDirectories = list(filter(lambda p: os.path.isdir(p), inputPaths))
|
||||
inputDirectories = list(map(lambda p:os.path.realpath(p), inputDirectories))
|
||||
|
||||
# determine output directory
|
||||
if os.path.isdir(args.output):
|
||||
output = f"{args.output}/dream.lock"
|
||||
else:
|
||||
output = args.output
|
||||
output = os.path.realpath(output)
|
||||
|
||||
# translator arguments
|
||||
translatorInput = dict(
|
||||
inputFiles=inputFiles,
|
||||
inputDirectories=inputDirectories,
|
||||
outputFile=output,
|
||||
selector=args.translator or "",
|
||||
)
|
||||
translatorInput.update(specialArgs)
|
||||
|
||||
# remove output file if exists
|
||||
if os.path.exists(output):
|
||||
os.remove(output)
|
||||
|
||||
# dump translator arguments to json file and execute translator
|
||||
with tempfile.NamedTemporaryFile("w") as inputJsonFile:
|
||||
json.dump(translatorInput, inputJsonFile, indent=2)
|
||||
inputJsonFile.seek(0) # flushes write cache
|
||||
env = os.environ.copy()
|
||||
env.update(dict(
|
||||
FUNC_ARGS=inputJsonFile.name
|
||||
))
|
||||
procEval = sp.run(
|
||||
[
|
||||
"nix", "eval", "--impure", "--raw", "--expr",
|
||||
f"((import {dream2nix_src} {{}}).translators.selectTranslatorJSON {{}})",
|
||||
],
|
||||
capture_output=True,
|
||||
env=env
|
||||
)
|
||||
if procEval.returncode:
|
||||
print("Selecting translator failed", file=sys.stdout)
|
||||
print(procEval.stderr.decode(), file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# parse data for auto selected translator
|
||||
resultEval = json.loads(procEval.stdout)
|
||||
subsystem = resultEval['subsystem']
|
||||
trans_type = resultEval['type']
|
||||
trans_name = resultEval['name']
|
||||
|
||||
# include default values into input data
|
||||
translatorInputWithDefaults = resultEval['SpecialArgsDefaults']
|
||||
translatorInputWithDefaults.update(translatorInput)
|
||||
json.dump(translatorInputWithDefaults, inputJsonFile, indent=2)
|
||||
inputJsonFile.seek(0)
|
||||
|
||||
# build the translator bin
|
||||
procBuild = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-o", "translator", "--expr",
|
||||
f"(import {dream2nix_src} {{}}).translators.translators.{subsystem}.{trans_type}.{trans_name}.translateBin",
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
if procBuild.returncode:
|
||||
print("Building translator failed", file=sys.stdout)
|
||||
print(procBuild.stderr.decode(), file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# execute translator
|
||||
translatorPath = os.path.realpath("translator")
|
||||
os.remove("translator")
|
||||
sp.run(
|
||||
[f"{translatorPath}/bin/translate", inputJsonFile.name] + sys.argv[1:]
|
||||
)
|
||||
|
||||
# raise error if output wasn't produced
|
||||
if not os.path.isfile(output):
|
||||
raise Exception(f"Translator failed to create dream.lock")
|
||||
|
||||
# read produced lock file
|
||||
with open(output) as f:
|
||||
lock = json.load(f)
|
||||
|
||||
# write translator information to lock file
|
||||
lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}"
|
||||
lock['generic']['translatorParams'] = " ".join(sys.argv[2:])
|
||||
|
||||
# clean up dependency graph
|
||||
# remove empty entries
|
||||
depGraph = lock['generic']['dependencyGraph']
|
||||
if 'dependencyGraph' in lock['generic']:
|
||||
for pname, deps in depGraph.copy().items():
|
||||
if not deps:
|
||||
del depGraph[pname]
|
||||
|
||||
# remove cyclic dependencies
|
||||
edges = set()
|
||||
for pname, deps in depGraph.items():
|
||||
for dep in deps:
|
||||
edges.add((pname, dep))
|
||||
G = nx.DiGraph(sorted(list(edges)))
|
||||
cycle_count = 0
|
||||
removed_edges = []
|
||||
for pname in list(depGraph.keys()):
|
||||
try:
|
||||
while True:
|
||||
cycle = nx.find_cycle(G, pname)
|
||||
cycle_count += 1
|
||||
# remove_dependecy(indexed_pkgs, G, cycle[-1][0], cycle[-1][1])
|
||||
node_from, node_to = cycle[-1][0], cycle[-1][1]
|
||||
G.remove_edge(node_from, node_to)
|
||||
depGraph[node_from].remove(node_to)
|
||||
removed_edges.append((node_from, node_to))
|
||||
except nx.NetworkXNoCycle:
|
||||
continue
|
||||
if removed_edges:
|
||||
removed_cycles_text = 'Removed Cyclic dependencies:'
|
||||
for node, removed_node in removed_edges:
|
||||
removed_cycles_text += f"\n {node} -> {removed_node}"
|
||||
print(removed_cycles_text)
|
||||
lock['generic']['dependencyCyclesRemoved'] = True
|
||||
|
||||
# calculate combined hash if --combined was specified
|
||||
if args.combined:
|
||||
|
||||
print("Building FOD of combined sources to retrieve output hash")
|
||||
|
||||
# remove hashes from lock file and init sourcesCombinedHash with emtpy string
|
||||
strip_hashes_from_lock(lock)
|
||||
lock['generic']['sourcesCombinedHash'] = ""
|
||||
with open(output, 'w') as f:
|
||||
json.dump(lock, f, indent=2)
|
||||
|
||||
# compute FOD hash of combined sources
|
||||
proc = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-L", "--expr",
|
||||
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {output}; }}"
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
# read the output hash from the failed build log
|
||||
match = re.search(r"FOD_PATH=(.*=)", proc.stderr.decode())
|
||||
if not match:
|
||||
print(proc.stderr.decode())
|
||||
print(proc.stdout.decode())
|
||||
raise Exception("Could not find FOD hash in FOD log")
|
||||
hash = match.groups()[0]
|
||||
print(f"Computed FOD hash: {hash}")
|
||||
|
||||
# store the hash in the lock
|
||||
lock['generic']['sourcesCombinedHash'] = hash
|
||||
|
||||
# re-write dream.lock
|
||||
with open(output, 'w') as f:
|
||||
json.dump(order_dict(lock), f, indent=2)
|
||||
|
||||
print(f"Created {output}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="nix run dream2nix --",
|
||||
# description="translate projects to nix",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(
|
||||
title='actions',
|
||||
description='valid actions',
|
||||
help='which action to execute'
|
||||
)
|
||||
|
||||
list_parser = sub.add_parser(
|
||||
"list",
|
||||
description="list available translators"
|
||||
)
|
||||
|
||||
list_parser.set_defaults(func=list_translators)
|
||||
|
||||
|
||||
# PARSER FOR TRANSLATOR
|
||||
|
||||
translate_parser = sub.add_parser(
|
||||
"translate",
|
||||
prog="translate",
|
||||
description="translate projects to nix",
|
||||
)
|
||||
|
||||
translate_parser.set_defaults(func=translate)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-t", "--translator",
|
||||
help="select translator (list via: 'dream2nix list')",
|
||||
default=""
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-o", "--output",
|
||||
help="output file/directory for the dream.lock",
|
||||
default="./dream.lock"
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-c", "--combined",
|
||||
help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
|
||||
action="store_true"
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"input",
|
||||
help="input files or directories containing sources and metadata",
|
||||
nargs="+"
|
||||
)
|
||||
|
||||
# parse special args
|
||||
# (custom parameters required by individual translators)
|
||||
parsed, unknown = translate_parser.parse_known_args()
|
||||
for arg in unknown:
|
||||
if arg.startswith("--arg_"):
|
||||
translate_parser.add_argument(arg.split('=')[0])
|
||||
if arg.startswith("--flag_"):
|
||||
translate_parser.add_argument(arg.split('=')[0], action='store_true')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not hasattr(args, "func"):
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
args.func(args)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,22 +0,0 @@
|
||||
{
|
||||
# from dream2nix
|
||||
externalSources,
|
||||
translators,
|
||||
|
||||
# from nixpkgs
|
||||
python3,
|
||||
writeScript,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
cliPython = python3.withPackages (ps: [ ps.networkx ]);
|
||||
in
|
||||
|
||||
writeScript "cli" ''
|
||||
export d2nExternalSources=${externalSources}
|
||||
|
||||
translatorsJsonFile=${translators.translatorsJsonFile} \
|
||||
dream2nixSrc=${../../.} \
|
||||
${cliPython}/bin/python ${./cli.py} "$@"
|
||||
''
|
||||
@ -9,7 +9,7 @@ from cleo.helpers import option
|
||||
|
||||
import networkx as nx
|
||||
|
||||
dream2nix_src = "./src"
|
||||
dream2nix_src = os.environ.get("dream2nixSrc")
|
||||
|
||||
|
||||
|
||||
@ -25,9 +25,9 @@ class PackageCommand(Command):
|
||||
option(
|
||||
"source",
|
||||
None,
|
||||
"source of the package",
|
||||
"source of the package, can be a path or flake-like spec",
|
||||
flag=False,
|
||||
multiple=True
|
||||
multiple=False
|
||||
),
|
||||
option("translator", None, "which translator to use", flag=False),
|
||||
option("output", None, "output file/directory for the dream.lock", flag=False),
|
||||
@ -59,6 +59,24 @@ class PackageCommand(Command):
|
||||
)
|
||||
}
|
||||
|
||||
# ensure output directory
|
||||
output = self.option("output")
|
||||
if not output:
|
||||
output = './.'
|
||||
if not os.path.isdir(output):
|
||||
os.mkdir(output)
|
||||
existingFiles = set(os.listdir(output))
|
||||
if any(f in existingFiles for f in ('default.nix', 'dream.lock')):
|
||||
print(
|
||||
f"output directory {output} already contains a default.nix "
|
||||
"or dream.lock. Delete first!",
|
||||
file=sys.stderr,
|
||||
)
|
||||
exit(1)
|
||||
output = os.path.realpath(output)
|
||||
outputDreamLock = f"{output}/dream.lock"
|
||||
outputDefaultNix = f"{output}/default.nix"
|
||||
|
||||
# verify source
|
||||
source = self.option("source")
|
||||
if not source:
|
||||
@ -67,18 +85,19 @@ class PackageCommand(Command):
|
||||
f"Source not specified. Defaulting to current directory: {source}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
# check if source path exists
|
||||
if not os.path.exists(source):
|
||||
raise print(f"Input path '{path}' does not exist", file=sys.stdout)
|
||||
exit(1)
|
||||
|
||||
# determine output file
|
||||
output = self.option("output")
|
||||
if not output:
|
||||
output = './dream.lock'
|
||||
if os.path.isdir(output):
|
||||
output = f"{output}/dream.lock"
|
||||
output = os.path.realpath(output)
|
||||
# check if source is valid fetcher spec
|
||||
sourceSpec = {}
|
||||
if source.partition(':')[0] in os.environ.get("fetcherNames", None).split():
|
||||
print(f"fetching source for '{source}'")
|
||||
sourceSpec =\
|
||||
callNixFunction("fetchers.translateShortcut", shortcut=source)
|
||||
source =\
|
||||
buildNixFunction("fetchers.fetchShortcut", shortcut=source)
|
||||
else:
|
||||
# check if source path exists
|
||||
if not os.path.exists(source):
|
||||
raise print(f"Input path '{path}' does not exist", file=sys.stdout)
|
||||
exit(1)
|
||||
|
||||
# select translator
|
||||
translatorsSorted = sorted(
|
||||
@ -93,14 +112,14 @@ class PackageCommand(Command):
|
||||
chosen = self.choice(
|
||||
'Select translator',
|
||||
list(map(
|
||||
lambda t: f"{t['subsystem']}.{t['type']}.{t['name']}",
|
||||
lambda t: f"{t['subsystem']}.{t['type']}.{t['name']}{' (compatible)' if t['compatible'] else ''}",
|
||||
translatorsSorted
|
||||
)),
|
||||
0
|
||||
)
|
||||
translator = chosen
|
||||
translator = list(filter(
|
||||
lambda t: [t['subsystem'], t['type'], t['name']] == translator.split('.'),
|
||||
lambda t: [t['subsystem'], t['type'], t['name']] == translator.split(' (')[0].split('.'),
|
||||
translatorsSorted,
|
||||
))[0]
|
||||
|
||||
@ -151,18 +170,13 @@ class PackageCommand(Command):
|
||||
translator_input = dict(
|
||||
inputFiles=[],
|
||||
inputDirectories=[source],
|
||||
outputFile=output,
|
||||
outputFile=outputDreamLock,
|
||||
)
|
||||
translator_input.update(specified_extra_args)
|
||||
|
||||
|
||||
# remove output file if exists
|
||||
if os.path.exists(output):
|
||||
os.remove(output)
|
||||
|
||||
# build the translator bin
|
||||
t = translator
|
||||
translator_path = buildNixDerivation(
|
||||
translator_path = buildNixAttribute(
|
||||
f"translators.translators.{t['subsystem']}.{t['type']}.{t['name']}.translateBin"
|
||||
)
|
||||
|
||||
@ -177,17 +191,32 @@ class PackageCommand(Command):
|
||||
)
|
||||
|
||||
# raise error if output wasn't produced
|
||||
if not os.path.isfile(output):
|
||||
if not os.path.isfile(outputDreamLock):
|
||||
raise Exception(f"Translator failed to create dream.lock")
|
||||
|
||||
# read produced lock file
|
||||
with open(output) as f:
|
||||
with open(outputDreamLock) as f:
|
||||
lock = json.load(f)
|
||||
|
||||
# write translator information to lock file
|
||||
lock['generic']['translatedBy'] = f"{subsystem}.{trans_type}.{trans_name}"
|
||||
lock['generic']['translatedBy'] = f"{t['subsystem']}.{t['type']}.{t['name']}"
|
||||
lock['generic']['translatorParams'] = " ".join(sys.argv[2:])
|
||||
|
||||
# add main package source
|
||||
mainPackage = lock['generic']['mainPackage']
|
||||
if mainPackage:
|
||||
mainSource = sourceSpec.copy()
|
||||
if not mainSource:
|
||||
mainSource = dict(
|
||||
type="unknown",
|
||||
version="unknown",
|
||||
)
|
||||
for field in ('versionField',):
|
||||
if field in mainSource:
|
||||
del mainSource[field]
|
||||
mainSource['version'] = sourceSpec[sourceSpec['versionField']]
|
||||
lock['sources'][mainPackage] = mainSource
|
||||
|
||||
# clean up dependency graph
|
||||
# remove empty entries
|
||||
depGraph = lock['generic']['dependencyGraph']
|
||||
@ -224,21 +253,21 @@ class PackageCommand(Command):
|
||||
lock['generic']['dependencyCyclesRemoved'] = True
|
||||
|
||||
# calculate combined hash if --combined was specified
|
||||
if args.combined:
|
||||
if self.option('combined'):
|
||||
|
||||
print("Building FOD of combined sources to retrieve output hash")
|
||||
|
||||
# remove hashes from lock file and init sourcesCombinedHash with emtpy string
|
||||
strip_hashes_from_lock(lock)
|
||||
lock['generic']['sourcesCombinedHash'] = ""
|
||||
with open(output, 'w') as f:
|
||||
with open(outputDreamLock, 'w') as f:
|
||||
json.dump(lock, f, indent=2)
|
||||
|
||||
# compute FOD hash of combined sources
|
||||
proc = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-L", "--expr",
|
||||
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {output}; }}"
|
||||
f"(import {dream2nix_src} {{}}).fetchSources {{ dreamLock = {outputDreamLock}; }}"
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
@ -256,10 +285,20 @@ class PackageCommand(Command):
|
||||
lock['generic']['sourcesCombinedHash'] = hash
|
||||
|
||||
# re-write dream.lock
|
||||
with open(output, 'w') as f:
|
||||
with open(outputDreamLock, 'w') as f:
|
||||
json.dump(order_dict(lock), f, indent=2)
|
||||
|
||||
print(f"Created {output}")
|
||||
# create default.nix
|
||||
template = callNixFunction(
|
||||
'apps.apps.cli2.templateDefaultNix',
|
||||
dream2nixLocationRelative=os.path.relpath(dream2nix_src, output)
|
||||
)
|
||||
# with open(f"{dream2nix_src}/apps/cli2/templateDefault.nix") as template:
|
||||
with open(outputDefaultNix, 'w') as defaultNix:
|
||||
defaultNix.write(template)
|
||||
|
||||
print(f"Created {output}/{{dream.lock,default.nix}}")
|
||||
|
||||
|
||||
|
||||
def callNixFunction(function_path, **kwargs):
|
||||
@ -274,24 +313,59 @@ def callNixFunction(function_path, **kwargs):
|
||||
[
|
||||
"nix", "eval", "--impure", "--raw", "--expr",
|
||||
f'''
|
||||
builtins.toJSON (
|
||||
(import {dream2nix_src} {{}}).{function_path} {{}}
|
||||
)
|
||||
let
|
||||
d2n = (import {dream2nix_src} {{}});
|
||||
in
|
||||
builtins.toJSON (
|
||||
(d2n.utils.makeCallableViaEnv d2n.{function_path}) {{}}
|
||||
)
|
||||
''',
|
||||
],
|
||||
capture_output=True,
|
||||
env=env
|
||||
)
|
||||
if proc.returncode:
|
||||
print(f"Failed calling '{function_path}'", file=sys.stderr)
|
||||
print(f"Failed calling nix function '{function_path}'", file=sys.stderr)
|
||||
print(proc.stderr.decode(), file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# parse data for auto selected translator
|
||||
# parse result data
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def buildNixDerivation(attribute_path):
|
||||
def buildNixFunction(function_path, **kwargs):
|
||||
with tempfile.NamedTemporaryFile("w") as input_json_file:
|
||||
json.dump(dict(**kwargs), input_json_file, indent=2)
|
||||
input_json_file.seek(0) # flushes write cache
|
||||
env = os.environ.copy()
|
||||
env.update(dict(
|
||||
FUNC_ARGS=input_json_file.name
|
||||
))
|
||||
proc = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-o", "tmp-result", "--expr",
|
||||
f'''
|
||||
let
|
||||
d2n = (import {dream2nix_src} {{}});
|
||||
in
|
||||
(d2n.utils.makeCallableViaEnv d2n.{function_path}) {{}}
|
||||
''',
|
||||
],
|
||||
capture_output=True,
|
||||
env=env
|
||||
)
|
||||
if proc.returncode:
|
||||
print(f"Failed calling nix function '{function_path}'", file=sys.stderr)
|
||||
print(proc.stderr.decode(), file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# return store path of result
|
||||
result = os.path.realpath("tmp-result")
|
||||
os.remove("tmp-result")
|
||||
return result
|
||||
|
||||
|
||||
def buildNixAttribute(attribute_path):
|
||||
proc = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-o", "tmp-result", "--expr",
|
||||
@ -318,6 +392,11 @@ def list_translators_for_source(sourcePath):
|
||||
return list(sorted(translatorsList, key=lambda t: t['compatible']))
|
||||
|
||||
|
||||
def order_dict(d):
|
||||
return {k: order_dict(v) if isinstance(v, dict) else v
|
||||
for k, v in sorted(d.items())}
|
||||
|
||||
|
||||
application = Application("package")
|
||||
application.add(PackageCommand())
|
||||
|
||||
|
||||
@ -1,21 +1,44 @@
|
||||
{
|
||||
# from dream2nix
|
||||
dream2nixWithExternals,
|
||||
externalSources,
|
||||
fetchers,
|
||||
translators,
|
||||
|
||||
# from nixpkgs
|
||||
lib,
|
||||
python3,
|
||||
writeScript,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
|
||||
b = builtins;
|
||||
|
||||
cliPython = python3.withPackages (ps: [ ps.networkx ps.cleo ]);
|
||||
|
||||
in
|
||||
{
|
||||
program = writeScript "cli" ''
|
||||
dream2nixSrc=${dream2nixWithExternals} \
|
||||
fetcherNames="${b.toString (lib.attrNames fetchers.fetchers)}" \
|
||||
${cliPython}/bin/python ${./cli.py} "$@"
|
||||
'';
|
||||
|
||||
writeScript "cli" ''
|
||||
export d2nExternalSources=${externalSources}
|
||||
templateDefaultNix =
|
||||
{
|
||||
dream2nixLocationRelative,
|
||||
}:
|
||||
''
|
||||
{
|
||||
dream2nix ? import ${dream2nixLocationRelative} {},
|
||||
}:
|
||||
|
||||
dream2nixSrc=${../../.} \
|
||||
${cliPython}/bin/python ${./cli.py} "$@"
|
||||
''
|
||||
(dream2nix.riseAndShine {
|
||||
dreamLock = ./dream.lock;
|
||||
}).package.overrideAttrs (old: {
|
||||
|
||||
})
|
||||
'';
|
||||
}
|
||||
|
||||
9
src/apps/cli2/templateDefault.nix
Normal file
9
src/apps/cli2/templateDefault.nix
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
dream2nix,
|
||||
}:
|
||||
|
||||
(dream2nix.riseAndShine {
|
||||
dreamLock = ./dream.lock;
|
||||
}).package.overrideAttrs (old: {
|
||||
|
||||
})
|
||||
@ -8,8 +8,9 @@
|
||||
let
|
||||
cliPython = python3.withPackages (ps: [ ps.cleo ]);
|
||||
in
|
||||
|
||||
writeScript "cli" ''
|
||||
dream2nixSrc=${../../.} \
|
||||
${cliPython}/bin/python ${./contribute.py} contribute "$@"
|
||||
''
|
||||
{
|
||||
program = writeScript "contribute" ''
|
||||
dream2nixSrc=${../../.} \
|
||||
${cliPython}/bin/python ${./contribute.py} contribute "$@"
|
||||
'';
|
||||
}
|
||||
|
||||
@ -3,11 +3,12 @@
|
||||
|
||||
callPackageDream,
|
||||
externalSources,
|
||||
location,
|
||||
dream2nixWithExternals,
|
||||
translators,
|
||||
...
|
||||
}:
|
||||
{
|
||||
rec {
|
||||
apps = { inherit cli cli2 contribute install; };
|
||||
|
||||
# the unified translator cli
|
||||
cli = callPackageDream (import ./cli) {};
|
||||
@ -17,29 +18,5 @@
|
||||
contribute = callPackageDream (import ./contribute) {};
|
||||
|
||||
# install the framework to a specified location by copying the code
|
||||
install = callPackageDream ({ writeScript, }:
|
||||
writeScript
|
||||
"install"
|
||||
''
|
||||
target="$1"
|
||||
if [[ "$target" == "" ]]; then
|
||||
echo "specify target"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$target"
|
||||
if [ -n "$(ls -A $target)" ]; then
|
||||
echo "target directory not empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp -r ${location}/* $target/
|
||||
mkdir $target/external
|
||||
cp -r ${externalSources}/* $target/external/
|
||||
chmod -R +w $target
|
||||
|
||||
echo "Installed dream2nix successfully to '$target'."
|
||||
echo "Please check/modify settings in '$target/config.json'"
|
||||
''
|
||||
) {};
|
||||
install = callPackageDream (import ./install) {};
|
||||
}
|
||||
|
||||
31
src/apps/install/default.nix
Normal file
31
src/apps/install/default.nix
Normal file
@ -0,0 +1,31 @@
|
||||
{
|
||||
runCommand,
|
||||
writeScript,
|
||||
|
||||
# dream2nix inputs
|
||||
dream2nixWithExternals,
|
||||
...
|
||||
}:
|
||||
|
||||
{
|
||||
program = writeScript "install"
|
||||
''
|
||||
target="$1"
|
||||
if [[ "$target" == "" ]]; then
|
||||
echo "specify target"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$target"
|
||||
if [ -n "$(ls -A $target)" ]; then
|
||||
echo "target directory not empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp -r ${dream2nixWithExternals}/* $target/
|
||||
chmod -R +w $target
|
||||
|
||||
echo "Installed dream2nix successfully to '$target'."
|
||||
echo "Please check/modify settings in '$target/config.json'"
|
||||
'';
|
||||
}
|
||||
@ -12,13 +12,16 @@
|
||||
|
||||
let
|
||||
|
||||
b = builtins;
|
||||
|
||||
utils = pkgs.callPackage ./utils.nix {};
|
||||
|
||||
callPackageDream = f: args: pkgs.callPackage f (args // {
|
||||
inherit callPackageDream;
|
||||
inherit externals;
|
||||
inherit externalSources;
|
||||
inherit location;
|
||||
inherit fetchers;
|
||||
inherit dream2nixWithExternals;
|
||||
inherit translators;
|
||||
inherit utils;
|
||||
});
|
||||
@ -48,13 +51,23 @@ let
|
||||
translators = callPackageDream ./translators {};
|
||||
|
||||
# the location of the dream2nix framework for self references (update scripts, etc.)
|
||||
location = ./.;
|
||||
dream2nixWithExternals =
|
||||
if b.pathExists (./. + "/external") then
|
||||
./.
|
||||
else
|
||||
pkgs.runCommand "dream2nix-full-src" {} ''
|
||||
cp -r ${./.} $out
|
||||
chmod +w $out
|
||||
mkdir $out/external
|
||||
ls -lah ${externalSources}
|
||||
cp -r ${externalSources}/* $out/external/
|
||||
'';
|
||||
|
||||
in
|
||||
|
||||
rec {
|
||||
|
||||
inherit apps builders fetchers finders location translators;
|
||||
inherit apps builders fetchers finders dream2nixWithExternals translators utils;
|
||||
|
||||
# automatically find a suitable builder for a given generic lock
|
||||
findBuilder = dreamLock:
|
||||
@ -123,8 +136,7 @@ rec {
|
||||
|
||||
|
||||
# build package defined by dream.lock
|
||||
# TODO: rename to riseAndShine
|
||||
buildPackage =
|
||||
riseAndShine =
|
||||
{
|
||||
dreamLock,
|
||||
builder ? findBuilder (parseLock dreamLock),
|
||||
|
||||
@ -18,8 +18,8 @@
|
||||
fetchedSources = lib.mapAttrs (pname: source:
|
||||
if source.type == "unknown" then
|
||||
"unknown"
|
||||
else if fetchers ? "${source.type}" then
|
||||
fetchSource source
|
||||
else if fetchers.fetchers ? "${source.type}" then
|
||||
fetchSource { inherit source; }
|
||||
else throw "unsupported source type '${source.type}'"
|
||||
) sources;
|
||||
}
|
||||
|
||||
@ -25,14 +25,17 @@ rec {
|
||||
|
||||
combinedFetcher = callPackageDream ./combined-fetcher.nix { inherit defaultFetcher; };
|
||||
|
||||
fetchSource = source:
|
||||
fetchSource = { source, }:
|
||||
let
|
||||
fetcher = fetchers."${source.type}";
|
||||
fetcherOutputs = fetcher.outputs source;
|
||||
in
|
||||
fetcherOutputs.fetched (source.hash or null);
|
||||
|
||||
fetchShortcut = { shortcut, }:
|
||||
fetchSource { source = translateShortcut { inherit shortcut; }; };
|
||||
|
||||
fetchViaShortcut = shortcut:
|
||||
translateShortcut = { shortcut, }:
|
||||
let
|
||||
|
||||
checkArgs = fetcherName: args:
|
||||
@ -48,17 +51,19 @@ rec {
|
||||
else
|
||||
args;
|
||||
|
||||
fetchViaHttpUrl =
|
||||
translateHttpUrl =
|
||||
let
|
||||
fetcher = fetchers.fetchurl;
|
||||
fetcherOutputs = fetchers.http.outputs { url = shortcut; };
|
||||
in
|
||||
rec {
|
||||
{
|
||||
type = "fetchurl";
|
||||
hash = fetcherOutputs.calcHash "sha256";
|
||||
fetched = fetcherOutputs.fetched hash;
|
||||
url = shortcut;
|
||||
versionField = fetcher.versionField;
|
||||
};
|
||||
|
||||
fetchViaGitShortcut =
|
||||
translateGitShortcut =
|
||||
let
|
||||
urlAndParams = lib.elemAt (lib.splitString "+" shortcut) 1;
|
||||
splitUrlParams = lib.splitString "?" urlAndParams;
|
||||
@ -75,12 +80,14 @@ rec {
|
||||
args = params // { inherit url; };
|
||||
fetcherOutputs = fetcher.outputs (checkArgs "git" args);
|
||||
in
|
||||
rec {
|
||||
{
|
||||
type = "git";
|
||||
hash = fetcherOutputs.calcHash "sha256";
|
||||
fetched = fetcherOutputs.fetched hash;
|
||||
inherit url;
|
||||
versionField = fetcher.versionField;
|
||||
};
|
||||
|
||||
fetchViaRegularShortcut =
|
||||
translateRegularShortcut =
|
||||
let
|
||||
splitNameParams = lib.splitString ":" (lib.removeSuffix "/" shortcut);
|
||||
fetcherName = lib.elemAt splitNameParams 0;
|
||||
@ -104,15 +111,16 @@ rec {
|
||||
Should be ${fetcherName}:${lib.concatStringsSep "/" fetcher.inputs}
|
||||
''
|
||||
else
|
||||
rec {
|
||||
args // {
|
||||
type = fetcherName;
|
||||
hash = fetcherOutputs.calcHash "sha256";
|
||||
fetched = fetcherOutputs.fetched hash;
|
||||
versionField = fetcher.versionField;
|
||||
};
|
||||
in
|
||||
if lib.hasPrefix "git+" (lib.head (lib.splitString ":" shortcut)) then
|
||||
fetchViaGitShortcut
|
||||
translateGitShortcut
|
||||
else if lib.hasPrefix "http://" shortcut || lib.hasPrefix "https://" shortcut then
|
||||
fetchViaHttpUrl
|
||||
translateHttpUrl
|
||||
else
|
||||
fetchViaRegularShortcut;
|
||||
translateRegularShortcut;
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
externalSources,
|
||||
externals,
|
||||
location,
|
||||
dream2nixWithExternals,
|
||||
utils,
|
||||
...
|
||||
}:
|
||||
@ -50,11 +50,10 @@ let
|
||||
|
||||
jsonInputFile=$(realpath $1)
|
||||
outputFile=$(${pkgs.jq}/bin/jq '.outputFile' -c -r $jsonInputFile)
|
||||
export d2nExternalSources=${externalSources}
|
||||
|
||||
nix eval --impure --raw --expr "
|
||||
builtins.toJSON (
|
||||
(import ${location} {}).translators.translators.${
|
||||
(import ${dream2nixWithExternals} {}).translators.translators.${
|
||||
lib.concatStringsSep "." translatorAttrPath
|
||||
}.translate
|
||||
(builtins.fromJSON (builtins.readFile '''$1'''))
|
||||
@ -85,7 +84,7 @@ let
|
||||
translatorsList = lib.collect (v: v ? translateBin) translators;
|
||||
|
||||
# json file exposing all existing translators to CLI including their special args
|
||||
translatorsForInput = utils.makeCallableViaEnv (
|
||||
translatorsForInput =
|
||||
{
|
||||
inputDirectories,
|
||||
inputFiles,
|
||||
@ -99,8 +98,7 @@ let
|
||||
type
|
||||
;
|
||||
compatible = t.compatiblePaths args == args;
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
# pupulates a translators special args with defaults
|
||||
getSpecialArgsDefaults = specialArgsDef:
|
||||
|
||||
@ -17,7 +17,13 @@
|
||||
...
|
||||
}:
|
||||
let
|
||||
parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0);
|
||||
packageLock =
|
||||
if inputDirectories != [] then
|
||||
"${lib.elemAt inputDirectories 0}/package-lock.json"
|
||||
else
|
||||
lib.elemAt inputFiles 0;
|
||||
|
||||
parsed = externals.npmlock2nix.readLockfile packageLock;
|
||||
|
||||
parseGithubDependency = dependency:
|
||||
externals.npmlock2nix.parseGitHubRef dependency.version;
|
||||
@ -87,6 +93,8 @@
|
||||
)
|
||||
dependencies;
|
||||
in
|
||||
|
||||
# the dream lock
|
||||
rec {
|
||||
sources =
|
||||
let
|
||||
@ -145,9 +153,12 @@
|
||||
inputFiles,
|
||||
}@args:
|
||||
{
|
||||
inputDirectories = [];
|
||||
inputDirectories = lib.filter
|
||||
(utils.containsMatchingFile [ ''.*package-lock\.json'' ''.*package.json'' ])
|
||||
args.inputDirectories;
|
||||
|
||||
inputFiles =
|
||||
lib.filter (f: builtins.match ".*(package-lock\\.json)" f != null) args.inputFiles;
|
||||
lib.filter (f: builtins.match ''.*package-lock\.json'' f != null) args.inputFiles;
|
||||
};
|
||||
|
||||
specialArgs = {
|
||||
|
||||
@ -60,7 +60,8 @@
|
||||
}@args:
|
||||
{
|
||||
inputDirectories = [];
|
||||
inputFiles = lib.filter (f: builtins.match ".*(requirements).*\\.txt" f != null) args.inputFiles;
|
||||
|
||||
inputFiles = lib.filter (f: builtins.match ''.*requirements.*\.txt'' f != null) args.inputFiles;
|
||||
};
|
||||
|
||||
# define special args and provide defaults
|
||||
|
||||
@ -11,39 +11,23 @@ let
|
||||
in
|
||||
|
||||
rec {
|
||||
basename = path: lib.last (lib.splitString "/" path);
|
||||
|
||||
dirname = path: builtins.concatStringsSep "/" (lib.init (lib.splitString "/" path));
|
||||
isFile = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "regular";
|
||||
|
||||
isFile = path: (builtins.readDir (dirname path))."${basename path}" == "regular";
|
||||
isDir = path: (builtins.readDir (b.dirOf path))."${b.baseNameOf path}" == "directory";
|
||||
|
||||
isDir = path: (builtins.readDir (dirname path))."${basename path}" == "directory";
|
||||
|
||||
listFiles = path: lib.filterAttrs (n: v: v == "regular") (builtins.listDir path);
|
||||
listFiles = path: lib.attrNames (lib.filterAttrs (n: v: v == "regular") (builtins.readDir path));
|
||||
|
||||
# directory names of a given directory
|
||||
dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
|
||||
|
||||
matchTopLevelFiles = pattern: path:
|
||||
# is dir
|
||||
if isDir path then
|
||||
builtins.all (f: matchTopLevelFiles pattern f) (listFiles path)
|
||||
|
||||
# is file
|
||||
else
|
||||
let
|
||||
match = (builtins.match pattern path);
|
||||
in
|
||||
if match == null then false else builtins.any (m: m != null) match;
|
||||
|
||||
compatibleTopLevelPaths = pattern: paths:
|
||||
lib.filter
|
||||
(path:
|
||||
matchTopLevelFiles
|
||||
pattern
|
||||
path
|
||||
)
|
||||
paths;
|
||||
# Returns true if every given pattern is satisfied by at least one file name
|
||||
# inside the given directory.
|
||||
# Sub-directories are not recursed.
|
||||
containsMatchingFile = patterns: dir:
|
||||
lib.all
|
||||
(pattern: lib.any (file: b.match pattern file != null) (listFiles dir))
|
||||
patterns;
|
||||
|
||||
# allow a function to receive its input from an environment variable
|
||||
# whenever an empty set is passed
|
||||
@ -53,7 +37,7 @@ rec {
|
||||
else
|
||||
func args;
|
||||
|
||||
|
||||
# hash the contents of a path via `nix hash-path`
|
||||
hashPath = algo: path:
|
||||
let
|
||||
hashFile = runCommand "hash-${algo}" {} ''
|
||||
@ -62,6 +46,7 @@ rec {
|
||||
in
|
||||
b.readFile hashFile;
|
||||
|
||||
# builder to create a shell script that has it's own PATH
|
||||
writePureShellScript = availablePrograms: script: writeScriptBin "run" ''
|
||||
export PATH="${lib.makeBinPath availablePrograms}"
|
||||
tmpdir=$(${coreutils}/bin/mktemp -d)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user