mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-08-16 18:00:30 +03:00
improve handling of translator modules
This commit is contained in:
parent
d605ed3624
commit
d83d76d5ef
@ -10,6 +10,7 @@ It focuses on the following aspects:
|
||||
- Risk free opt-in FOD fetching
|
||||
- Common UI across 2nix tools
|
||||
- Reduce effort to develop new 2nix solutions
|
||||
- Exploration and adoption of new nix features
|
||||
|
||||
### Motivation
|
||||
2nix tools, or in other words, tools converting instructions of other build systems to nix build instructions, are an important part of the nix/nixos ecosystem. These tools make packaging workflows easier and often allow to manage complexity that would be hard or impossible to manage without.
|
||||
@ -64,7 +65,7 @@ Essential components like package update scripts or fetching and override logic
|
||||
Optionally, to save more storag space, individual hashes for source can be ommited and a single large FOD used instead.
|
||||
Due to a unified minimalistic fetching layer the risk of FOD hash breakages should be very low.
|
||||
|
||||
### Common UI across many 2nix tools
|
||||
### Common UI across many 2nix solutions
|
||||
2nix solutions which follow the dream2nix framework will have a unified UI for workflows like project initialization or code generation. This will allow quicker onboarding of new users by providing familiar workflows across different build systems.
|
||||
|
||||
### Reduced effort to develop new 2nix solutions
|
||||
|
@ -25,10 +25,12 @@
|
||||
{
|
||||
overlay = curr: prev: {};
|
||||
|
||||
defaultApp = forAllSystems (system: self.apps."${system}".cli);
|
||||
|
||||
apps = forAllSystems (system: {
|
||||
translate = {
|
||||
cli = {
|
||||
"type" = "app";
|
||||
"program" = builtins.toString (dream2nixFor."${system}".apps.translate);
|
||||
"program" = builtins.toString (dream2nixFor."${system}".apps.cli);
|
||||
};
|
||||
install = {
|
||||
"type" = "app";
|
||||
|
@ -11,82 +11,37 @@ with open (os.environ.get("translatorsJsonFile")) as f:
|
||||
translators = json.load(f)
|
||||
|
||||
|
||||
# TODO: detection translator automatically according to files
|
||||
def auto_detect_translator(files, subsystem):
|
||||
return list(translators[subsystem].keys())[0]
|
||||
|
||||
|
||||
def stripHashesFromLock(lock):
|
||||
for source in lock['sources'].values():
|
||||
del source['hash']
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="translate",
|
||||
description="translate projects to nix",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-s", "--subsystem",
|
||||
help="which subsystem to use, (eg: python, nodejs, ...)",
|
||||
choices=translators.keys()
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-t", "--translator",
|
||||
help="which specific translator to use",
|
||||
default="auto"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-o", "--output",
|
||||
help="output file/directory (generic lock)",
|
||||
default="./dream.lock"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-c", "--combined",
|
||||
help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
|
||||
action="store_true"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"input",
|
||||
help="input files containing relevant metadata",
|
||||
nargs="+"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# TODO: detection subsystem automatically according to files
|
||||
if not hasattr(args, "subsystem"):
|
||||
print("Please specify subsystem (-s, --subsystem)", file=sys.stderr)
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
return args
|
||||
def list(args):
|
||||
out = "Available translators per build system"
|
||||
for subsystem, trans_types in translators.items():
|
||||
displayed = []
|
||||
for trans_type, translators_ in trans_types.items():
|
||||
for translator in translators_:
|
||||
displayed.append(f"{trans_type}.{translator}")
|
||||
nl = '\n'
|
||||
out += f"\n - {subsystem}.{f'{nl} - {subsystem}.'.join(displayed)}"
|
||||
print(out)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
def translate(args):
|
||||
|
||||
dream2nix_src = os.environ.get("dream2nixSrc")
|
||||
|
||||
subsystem = args.subsystem
|
||||
files = args.input
|
||||
|
||||
# determine translator
|
||||
if args.translator == "auto":
|
||||
translator = auto_detect_translator(files, subsystem)
|
||||
else:
|
||||
translator = args.translator
|
||||
translator = args.translator
|
||||
|
||||
# determine output directory
|
||||
if os.path.isdir(args.output):
|
||||
output = f"{args.output}/dream.lock"
|
||||
else:
|
||||
output = args.output
|
||||
output = os.path.realpath(output)
|
||||
|
||||
# translator arguments
|
||||
translatorInput = dict(
|
||||
@ -95,11 +50,25 @@ def main():
|
||||
)
|
||||
|
||||
# dump translator arguments to json file and execute translator
|
||||
with tempfile.NamedTemporaryFile("w") as inputJson:
|
||||
json.dump(translatorInput, inputJson, indent=2)
|
||||
inputJson.seek(0) # flushes write cache
|
||||
with tempfile.NamedTemporaryFile("w") as inputJsonFile:
|
||||
json.dump(translatorInput, inputJsonFile, indent=2)
|
||||
inputJsonFile.seek(0) # flushes write cache
|
||||
procBuild = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "--expr",
|
||||
f"(import {dream2nix_src} {{}}).translators.translators.{translator}", "-o", "translator"
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
if procBuild.returncode:
|
||||
print("Building translator failed", file=sys.stdout)
|
||||
print(procBuild.stderr.decode(), file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
translatorPath = os.path.realpath("translator")
|
||||
os.remove("translator")
|
||||
sp.run(
|
||||
[f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:]
|
||||
[f"{translatorPath}/bin/translate", inputJsonFile.name] + sys.argv[1:]
|
||||
)
|
||||
|
||||
# raise error if output wasn't produced
|
||||
@ -113,7 +82,7 @@ def main():
|
||||
# calculate combined hash
|
||||
if args.combined:
|
||||
|
||||
print("Start building combined sourced FOD to get output hash")
|
||||
print("Start building FOD for combined sources to get output hash")
|
||||
|
||||
# remove hashes from lock file and init sourcesCombinedHash with emtpy string
|
||||
stripHashesFromLock(lock)
|
||||
@ -122,7 +91,6 @@ def main():
|
||||
json.dump(lock, f, indent=2)
|
||||
|
||||
# compute FOD hash of combined sources
|
||||
dream2nix_src = os.environ.get("dream2nixSrc")
|
||||
proc = sp.run(
|
||||
[
|
||||
"nix", "build", "--impure", "-L", "--expr",
|
||||
@ -149,5 +117,71 @@ def main():
|
||||
print(f"Created {output}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="nix run dream2nix --",
|
||||
# description="translate projects to nix",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(
|
||||
title='actions',
|
||||
description='valid actions',
|
||||
help='which action to execute'
|
||||
)
|
||||
|
||||
list_parser = sub.add_parser(
|
||||
"list",
|
||||
description="list available translators"
|
||||
)
|
||||
|
||||
list_parser.set_defaults(func=list)
|
||||
|
||||
translate_parser = sub.add_parser(
|
||||
"translate",
|
||||
prog="translate",
|
||||
description="translate projects to nix",
|
||||
)
|
||||
|
||||
translate_parser.set_defaults(func=translate)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-t", "--translator",
|
||||
help="select translator (list via: 'dream2nix list')",
|
||||
default="auto"
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-o", "--output",
|
||||
help="output file/directory for the generic lock",
|
||||
default="./dream.lock"
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"-c", "--combined",
|
||||
help="Store only one hash for all sources combined (smaller lock file -> larger FOD)",
|
||||
action="store_true"
|
||||
)
|
||||
|
||||
translate_parser.add_argument(
|
||||
"input",
|
||||
help="input files containing relevant metadata",
|
||||
nargs="+"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not hasattr(args, "func"):
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
args.func(args)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
@ -9,11 +9,11 @@ in
|
||||
{
|
||||
|
||||
# the unified translator cli
|
||||
translate = callPackage ({ python3, writeScript, ... }:
|
||||
cli = callPackage ({ python3, writeScript, ... }:
|
||||
writeScript "cli" ''
|
||||
translatorsJsonFile=${translators.translatorsJsonFile} \
|
||||
dream2nixSrc=${../.} \
|
||||
${python3}/bin/python ${./translators-cli.py} "$@"
|
||||
${python3}/bin/python ${./cli.py} "$@"
|
||||
''
|
||||
) {};
|
||||
|
||||
|
@ -1,52 +1,36 @@
|
||||
{ pkgs }:
|
||||
let
|
||||
|
||||
lib = pkgs.lib;
|
||||
|
||||
callPackage = pkgs.callPackage;
|
||||
|
||||
# every translator must provide 'bin/translate'
|
||||
translatorExec = translatorPkg: "${translatorPkg}/bin/translate";
|
||||
|
||||
# the list of all available translators
|
||||
translators = {
|
||||
dirNames = dir: lib.attrNames (lib.filterAttrs (name: type: type == "directory") (builtins.readDir dir));
|
||||
|
||||
python = {
|
||||
translators =
|
||||
lib.genAttrs (dirNames ./.) (subsystem:
|
||||
lib.genAttrs
|
||||
(lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ])
|
||||
(transType:
|
||||
lib.genAttrs (dirNames (./. + "/${subsystem}/${transType}")) (translatorName:
|
||||
callPackage (./. + "/${subsystem}/${transType}/${translatorName}") {}
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
# minimal POC python translator using pip. Type: 'external'
|
||||
external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; };
|
||||
external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; };
|
||||
external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; };
|
||||
external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; };
|
||||
external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; };
|
||||
|
||||
# TODO: add more translators
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
# Put all translator executables in a json file.
|
||||
# This will allow the cli to call the translators of different build systems
|
||||
# in a standardised way
|
||||
# TODO: This doesn't scale as it requires all translators being built.
|
||||
# Redesign this, to call the individual translators using nix run ...
|
||||
translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }:
|
||||
runCommand
|
||||
"translators.json"
|
||||
{
|
||||
buildInputs = lib.flatten
|
||||
(
|
||||
lib.mapAttrsToList
|
||||
(subsystem: translators:
|
||||
lib.attrValues translators
|
||||
)
|
||||
translators
|
||||
);
|
||||
}
|
||||
# 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs
|
||||
''
|
||||
#!${bash}/bin/bash
|
||||
cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out
|
||||
''
|
||||
) {};
|
||||
# dump the list of available translators to a json file so they can be listed in the CLI
|
||||
translatorsJsonFile = pkgs.writeText "translators.json" (builtins.toJSON (
|
||||
lib.genAttrs (dirNames ./.) (subsystem:
|
||||
lib.genAttrs
|
||||
(lib.filter (dir: builtins.pathExists (./. + "/${subsystem}/${dir}")) [ "impure" "ifd" "pure-nix" ])
|
||||
(transType:
|
||||
dirNames (./. + "/${subsystem}/${transType}")
|
||||
)
|
||||
)
|
||||
));
|
||||
|
||||
in
|
||||
{
|
||||
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
bash,
|
||||
jq,
|
||||
python,
|
||||
python3,
|
||||
writeScriptBin,
|
||||
...
|
||||
}:
|
||||
@ -22,7 +22,7 @@ writeScriptBin "translate" ''
|
||||
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
|
||||
|
||||
# pip executable
|
||||
pip=${python.pkgs.pip}/bin/pip
|
||||
pip=${python3.pkgs.pip}/bin/pip
|
||||
|
||||
# prepare temporary directory
|
||||
tmp=translateTmp
|
||||
@ -37,7 +37,7 @@ writeScriptBin "translate" ''
|
||||
-r ''${inputFiles/$'\n'/$' -r '}
|
||||
|
||||
# generate the generic lock from the downloaded list of files
|
||||
${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
|
||||
${python3}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
|
||||
|
||||
rm -rf $tmp
|
||||
''
|
Loading…
Reference in New Issue
Block a user