From baa442739f5c8d515e5f470c57cd3aa107e897ad Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 8 Nov 2021 11:47:15 +0700 Subject: [PATCH] add template and docs for contributing builder --- docs/contributors-guide.md | 57 +++----- src/apps/contribute/contribute.py | 164 ++++++++++++----------- src/builders/nodejs/granular/default.nix | 35 ++--- src/templates/builders/default.nix | 86 ++++++++++++ src/templates/translators/impure.nix | 45 ++++--- 5 files changed, 236 insertions(+), 151 deletions(-) create mode 100644 src/templates/builders/default.nix diff --git a/docs/contributors-guide.md b/docs/contributors-guide.md index d7926745..007e1f2f 100644 --- a/docs/contributors-guide.md +++ b/docs/contributors-guide.md @@ -22,55 +22,34 @@ In general there are 3 different types of translators ### Add a new translator -To add a new translator, execute the flakes app `contribute` which will generate a template for you. Then open the new `default.nix` file in an edtior +Clone repo and execute: +```shell +nix run .#contribute +``` +... then select `translator` and answer all questions. This will generate a template. The nix file must declare the following attributes: In case of a `pure` or `IFD` translator: -```nix -{ - # function which receives source files and returns an attribute set - # which follows the dream lock format - translate = ...; - - # function which receives source files and returns either true or false - # indicating if the current translator is capable of translating these files - compatiblePaths = ; - - # optionally specify additional arguments that the user can provide to the - # translator to customize its behavior - extraArgs = ...; -} -``` +See the template generated by the contribute app In case of an `impure` translator: -```nix -{ - # A derivation which outputs an executable at `/bin/run`. - # The executable will be called by dream2nix for translation - # - # The first arg `$1` will be a json file containing the input parameters - # like defined in /specifications/translator-call-example.json and the - # additional arguments required according to extraArgs - # - # The program is expected to create a file at the location specified - # by the input parameter `outFile`. - # The output file must contain the dream lock data encoded as json. - translateBin = ...; - - # A function which receives source files and returns either true or false - # indicating if the current translator is capable of translating these files - compatiblePaths = ; - - # optionally specify additional arguments that the user can provide to the - # translator to customize its behavior - extraArgs = ...; -} -``` +See the template generated by the contribute app Ways of debugging your translator: - run the dream2nix flake app and use the new translator - temporarily expose internal functions of your translator, then use nix repl `nix repl ./.` and invoke a function via `translators.translators.{subsystem}.{type}.{translator-name}.some_function` + + + +## Contribute Builder +### Add a new builder + +Clone repo and execute: +```shell +nix run .#contribute +``` +... then select `builder` and answer all questions. This will generate a template. \ No newline at end of file diff --git a/src/apps/contribute/contribute.py b/src/apps/contribute/contribute.py index 51d70878..28a2b7cd 100644 --- a/src/apps/contribute/contribute.py +++ b/src/apps/contribute/contribute.py @@ -1,4 +1,5 @@ import os +import pathlib import subprocess as sp from cleo import Application, Command from cleo.helpers import option @@ -9,96 +10,99 @@ dream2nix_src = "./src" class ContributeCommand(Command): - description = ( - "Add a new module to dream2nix by initializing a template" - ) + description = ( + "Add a new module to dream2nix by initializing a template" + ) - name = "contribute" + name = "contribute" - options = [ - option("module", None, "Which kind of module to contribute", flag=False), - option("subsystem", None, "which kind of subsystem", flag=False), - option("type", None, "pure or impure translator", flag=False), - option("name", None, "name of the new module", flag=False), - option( - "dependency", - None, - "Package to require, with an optional version constraint, " - "e.g. requests:^2.10.0 or requests=2.11.1.", - flag=False, - multiple=True, - ), - ] + options = [ + option("module", None, "Which kind of module to contribute", flag=False), + option("subsystem", None, "which kind of subsystem", flag=False), + option("type", None, "pure or impure translator", flag=False), + option("name", None, "name of the new module", flag=False), + option( + "dependency", + None, + "Package to require, with an optional version constraint, " + "e.g. requests:^2.10.0 or requests=2.11.1.", + flag=False, + multiple=True, + ), + ] - def handle(self): - if self.io.is_interactive(): - self.line("") - self.line( - "This command will initialize a template for adding a new module to dream2nix" - ) - self.line("") - - module = self.option("module") - if not module: - module = self.choice( - 'Select module type', - ['translator'], - 0 - ) - module = f"{module}s" - module_dir = dream2nix_src + f"/{module}/" + def handle(self): + if self.io.is_interactive(): + self.line("") + self.line( + "This command will initialize a template for adding a new module to dream2nix" + ) + self.line("") + + module = self.option("module") + if not module: + module = self.choice( + 'Select module type', + [ 'builder', 'translator'], + 0 + ) + module = f"{module}s" + module_dir = dream2nix_src + f"/{module}/" - subsystem = self.option('subsystem') - known_subsystems = list(dir for dir in os.listdir(module_dir) if os.path.isdir(module_dir + dir)) - if not subsystem: - subsystem = self.choice( - 'Select subsystem', - known_subsystems - + - [ - " -> add new" - ], - 0 - ) - if subsystem == " -> add new": - subsystem = self.ask('Please enter the name of a new subsystem:') - if subsystem in known_subsystems: - raise Exception(f"subsystem {subsystem} already exists") + subsystem = self.option('subsystem') + known_subsystems = list(dir for dir in os.listdir(module_dir) if os.path.isdir(module_dir + dir)) + if not subsystem: + subsystem = self.choice( + 'Select subsystem', + known_subsystems + + + [ + " -> add new" + ], + 0 + ) + if subsystem == " -> add new": + subsystem = self.ask('Please enter the name of a new subsystem:') + if subsystem in known_subsystems: + raise Exception(f"subsystem {subsystem} already exists") - - if module == 'translators': - type = self.option("type") - if not type: - type = self.choice( - f'Select {module} type', - ['impure', 'pure'], - 0 - ) - - name = self.option("name") - if not name: - name = self.ask('Specify name of new module:') - - for path in ( - module_dir + f"{subsystem}", - module_dir + f"{subsystem}/{type}", - module_dir + f"{subsystem}/{type}/{name}"): - if not os.path.isdir(path): - os.mkdir(path) - target_file = module_dir + f"{subsystem}/{type}/{name}/default.nix" - with open(dream2nix_src + f"/templates/{module}/{type}.nix") as template: - with open(target_file, 'w') as new_file: - new_file.write(template.read()) + + if module == 'translators': + type = self.option("type") + if not type: + type = self.choice( + f'Select {module} type', + ['impure', 'pure'], + 0 + ) + + name = self.option("name") + if not name: + name = self.ask('Specify name of new module:') - self.line(f"The template has been initialized in {target_file}") - if self.confirm('Would you like to open it in your default editor now?', True, '(?i)^(y|j)'): - sp.run(f"{os.environ.get('EDITOR')} {target_file}", shell=True) - + if module == 'translators': + new_path = module_dir + f"{subsystem}/{type}/{name}" + template_file = dream2nix_src + f"/templates/{module}/{type}.nix" + else: + new_path = module_dir + f"{subsystem}/{name}" + template_file = dream2nix_src + f"/templates/{module}/default.nix" + + pathlib.Path(new_path).mkdir(parents=True) + + target_file = f"{new_path}/default.nix" + with open(template_file) as template: + with open(target_file, 'w') as new_file: + new_file.write(template.read()) + + self.line(f"The template has been initialized in {target_file}") + if self.confirm('Would you like to open it in your default editor now?', True, '(?i)^(y|j)'): + sp.run(f"{os.environ.get('EDITOR')} {target_file}", shell=True) + application = Application("contribute") application.add(ContributeCommand()) if __name__ == '__main__': - application.run() + application.run() diff --git a/src/builders/nodejs/granular/default.nix b/src/builders/nodejs/granular/default.nix index f5b3e7a8..3dd93607 100644 --- a/src/builders/nodejs/granular/default.nix +++ b/src/builders/nodejs/granular/default.nix @@ -15,23 +15,31 @@ }: { - # funcs - getDependencies, - getSource, - buildPackageWithOtherBuilder, + # Funcs - # attributes - subsystemAttrs, - getCyclicDependencies, - mainPackageName, - mainPackageVersion, + # AttrSet -> Bool) -> AttrSet -> [x] + getCyclicDependencies, # name: version: -> [ {name=; version=; } ] + getDependencies, # name: version: -> [ {name=; version=; } ] + getSource, # name: version: -> store-path + buildPackageWithOtherBuilder, # { builder, name, version }: -> drv + + # Attributes + subsystemAttrs, # attrset + mainPackageName, # string + mainPackageVersion, # string + + # attrset of pname -> versions, + # where versions is a list of version strings packageVersions, - - # overrides + # Overrides + # Those must be applied by the builder to each individual derivation + # using `utils.applyOverridesToPackage` packageOverrides ? {}, - # custom opts: + # Custom Options: (parametrize builder behavior) + # These can be passed by the user via `builderArgs`. + # All options must provide default standalonePackageNames ? [], ... }@args: @@ -45,9 +53,6 @@ let isCyclic = name: version: (getCyclicDependencies name version) != []; - mainPackageKey = - "${mainPackageName}#${mainPackageVersion}"; - nodejsVersion = subsystemAttrs.nodejsVersion; nodejs = diff --git a/src/templates/builders/default.nix b/src/templates/builders/default.nix new file mode 100644 index 00000000..701d4758 --- /dev/null +++ b/src/templates/builders/default.nix @@ -0,0 +1,86 @@ +{ + lib, + pkgs, + stdenv, + + # dream2nix inputs + builders, + externals, + utils, + ... +}: + +{ + # Funcs + + # AttrSet -> Bool) -> AttrSet -> [x] + getCyclicDependencies, # name: version: -> [ {name=; version=; } ] + getDependencies, # name: version: -> [ {name=; version=; } ] + getSource, # name: version: -> store-path + buildPackageWithOtherBuilder, # { builder, name, version }: -> drv + + # Attributes + subsystemAttrs, # attrset + mainPackageName, # string + mainPackageVersion, # string + + # attrset of pname -> versions, + # where versions is a list of version strings + packageVersions, + + # Overrides + # Those must be applied by the builder to each individual derivation + # using `utils.applyOverridesToPackage` + packageOverrides ? {}, + + # Custom Options: (parametrize builder behavior) + # These can be passed by the user via `builderArgs`. + # All options must provide default + standalonePackageNames ? [], + ... +}@args: + +let + + b = builtins; + + # the main package + defaultPackage = packages."${mainPackageName}"."${mainPackageVersion}"; + + # manage pakcages in attrset to prevent duplicated evaluation + packages = + lib.mapAttrs + (name: versions: + lib.genAttrs + versions + (version: makeOnePackage name version)) + packageVersions; + + # Generates a derivation for a specific package name + version + makeOnePackage = name: version: + let + pkg = + stdenv.mkDerivation rec { + + pname = utils.sanitizeDerivationName name; + inherit version; + + src = getSource name version; + + buildInputs = + map + (dep: packages."${dep.name}"."${dep.version}") + (getDependencies name version); + + # Implement build phases + + }; + in + # apply packageOverrides to current derivation + (utils.applyOverridesToPackage packageOverrides pkg name); + + +in +{ + inherit defaultPackage packages; +} diff --git a/src/templates/translators/impure.nix b/src/templates/translators/impure.nix index a0310754..d954dff3 100644 --- a/src/templates/translators/impure.nix +++ b/src/templates/translators/impure.nix @@ -12,25 +12,36 @@ { - # the input format is specified in /specifications/translator-call-example.json - # this script receives a json file including the input paths and extraArgs - translateBin = writeScriptBin "translate" '' - #!${bash}/bin/bash + # A derivation which outputs an executable at `/bin/run`. + # The executable will be called by dream2nix for translation + # The input format is specified in /specifications/translator-call-example.json. + # The first arg `$1` will be a json file containing the input parameters + # like defined in /specifications/translator-call-example.json and the + # additional arguments required according to extraArgs + # + # The program is expected to create a file at the location specified + # by the input parameter `outFile`. + # The output file must contain the dream lock data encoded as json. + translateBin = utils.writePureShellScript + [ + bash + coreutils + jq + nix + ] + '' + # accroding to the spec, the translator reads the input from a json file + jsonInput=$1 - set -Eeuo pipefail + # read the json input + outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) + inputDirectories=$(${jq}/bin/jq '.inputDirectories | .[]' -c -r $jsonInput) + inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput) - # accroding to the spec, the translator reads the input from a json file - jsonInput=$1 - - # read the json input - outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) - inputDirectories=$(${jq}/bin/jq '.inputDirectories | .[]' -c -r $jsonInput) - inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput) - - # TODO: - # read input files/dirs and produce a json file at $outputFile - # containing the dream lock similar to /specifications/dream-lock-example.json - ''; + # TODO: + # read input files/dirs and produce a json file at $outputFile + # containing the dream lock similar to /specifications/dream-lock-example.json + ''; # From a given list of paths, this function returns all paths which can be processed by this translator.