separate handling of input files and input dirs

This commit is contained in:
DavHau 2021-09-16 19:33:31 +01:00
parent b6dfa1dbd4
commit 7dccfd6e3b
6 changed files with 76 additions and 30 deletions

View File

@ -1,6 +1,10 @@
{
"inputPaths": [
"./some_project_src/requirements.txt",
"./some_project_src/requirements-dev.txt"
"inputDirectories": [
"./some_project_src"
],
"inputFiles": [
"./a/b/c/requirements.txt",
"./a/b/c/requirements-dev.txt"
]
}

View File

@ -16,7 +16,7 @@ def strip_hashes_from_lock(lock):
del source['hash']
def list(args):
def list_translators(args):
out = "Available translators per build system"
for subsystem, trans_types in translators.items():
displayed = []
@ -32,7 +32,15 @@ def translate(args):
dream2nix_src = os.environ.get("dream2nixSrc")
files = args.input
inputPaths = args.input
# check if all inputs exist
for path in inputPaths:
if not os.path.exists(path):
raise Exception(f"Input path '{path}' does not exist")
inputFiles = list(filter(lambda p: os.path.isfile(p), inputPaths))
inputDirectories = list(filter(lambda p: os.path.isdir(p), inputPaths))
# determine output directory
if os.path.isdir(args.output):
@ -43,7 +51,8 @@ def translate(args):
# translator arguments
translatorInput = dict(
inputPaths=files,
inputFiles=inputFiles,
inputDirectories=inputDirectories,
outputFile=output,
selector=args.translator or "",
)
@ -143,7 +152,7 @@ def parse_args():
description="list available translators"
)
list_parser.set_defaults(func=list)
list_parser.set_defaults(func=list_translators)
translate_parser = sub.add_parser(
"translate",
@ -173,7 +182,7 @@ def parse_args():
translate_parser.add_argument(
"input",
help="input files containing relevant metadata",
help="input files or directories containing sources and metadata",
nargs="+"
)

View File

@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage {
nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ];
unpackPhase = ''
mkdir dist
for file in $src; do
for file in ${lib.attrValues fetchedSources}; do
# pick right most element of path
fname=''${file##*/}
fname=$(stripHash $fname)

View File

@ -85,16 +85,24 @@ let
));
# filter translators by compatibility for the given input paths
compatibleTranslators = paths: translators_:
compatibleTranslators =
{
inputDirectories,
inputFiles,
translators,
}:
let
compatible =
lib.mapAttrs (subsystem: types:
lib.mapAttrs (type: translators:
lib.mapAttrs (type: translators_:
lib.filterAttrs (name: translator:
translator ? compatiblePaths && translator.compatiblePaths paths == paths
) translators
translator ? compatiblePaths
&&
translator.compatiblePaths { inherit inputDirectories inputFiles; }
== { inherit inputDirectories inputFiles; }
) translators_
) types
) translators_;
) translators;
in
# purge empty attrsets
lib.filterAttrsRecursive (k: v: v != {}) (lib.filterAttrsRecursive (k: v: v != {}) compatible);
@ -129,17 +137,24 @@ let
selectTranslatorBin = utils.makeCallableViaEnv (
{
selector, # like 'python.impure' or 'python.impure.pip'
inputPaths, # input paths to translate
inputDirectories, # input paths to translate
inputFiles, # input paths to translate
...
}:
let
selectedTranslators = reduceTranslatorsBySelector selector translators;
compatTranslators = compatibleTranslators inputPaths selectedTranslators;
compatTranslators = compatibleTranslators {
inherit inputDirectories inputFiles;
translators = selectedTranslators;
};
in
if selectedTranslators == {} then
throw "The selector '${selector}' does not select any known translators"
else if compatTranslators == {} then
throw "Could not find any translator which is compatible to the given inputs: ${builtins.toString inputPaths}"
throw ''
Could not find any translator which is compatible to the given inputs:
- ${builtins.concatStringsSep "\n - " (inputDirectories ++ inputFiles)}
''
else
(lib.head (lib.attrValues (lib.head (lib.attrValues (lib.head (lib.attrValues compatTranslators)))))).translateBin
);

View File

@ -1,18 +1,21 @@
{
lib,
externals,
translatorName,
utils,
...
}:
let
{
translate =
{
inputPaths,
inputDirectories,
inputFiles,
...
}:
let
parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputPaths 0);
parsed = externals.npmlock2nix.readLockfile (builtins.elemAt inputFiles 0);
in
{
sources = builtins.mapAttrs (pname: pdata:{
@ -34,10 +37,15 @@ let
};
};
compatiblePaths = paths: utils.compatibleTopLevelPaths ".*(package-lock\\.json)" paths;
in
{
inherit translate compatiblePaths;
compatiblePaths =
{
inputDirectories,
inputFiles,
}@args:
builtins.trace (lib.attrValues args)
{
inputDirectories = [];
inputFiles =
lib.filter (f: builtins.match ".*(package-lock\\.json)" f != null) args.inputFiles;
};
}

View File

@ -21,9 +21,11 @@
# accroding to the spec, the translator reads the input from a json file
jsonInput=$1
# extract the 'inputPaths' field from the json
inputPaths=$(${jq}/bin/jq '.inputPaths | .[]' -c -r $jsonInput)
# read the json input
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
inputDirectories=$(${jq}/bin/jq '.inputDirectories | .[]' -c -r $jsonInput)
inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput)
# pip executable
pip=${python3.pkgs.pip}/bin/pip
@ -38,7 +40,7 @@
--no-cache \
--dest $tmp \
--progress-bar off \
-r ''${inputPaths/$'\n'/$' -r '}
-r ''${inputFiles/$'\n'/$' -r '}
# generate the generic lock from the downloaded list of files
${python3}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
@ -48,5 +50,13 @@
# from a given list of paths, this function returns all paths which can be processed by this translator
compatiblePaths = paths: utils.compatibleTopLevelPaths ".*(requirements).*\\.txt" paths;
compatiblePaths =
{
inputDirectories,
inputFiles,
}@args:
{
inputDirectories = [];
inputFiles = lib.filter (f: builtins.match ".*(requirements).*\\.txt" f != null) args.inputFiles;
};
}