diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..6f61a514 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.*/ +result diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..307f6571 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1631015389, + "narHash": "sha256-9SKTwRbp7OQGa4wKTo1wB6ctC2VlaIeKFZ+flqzPuoI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "bc06c93905f60a82d6ebbb78f78cf289257860cc", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..46d27ffc --- /dev/null +++ b/flake.nix @@ -0,0 +1,39 @@ +{ + description = "dream2nix: A generic framework for 2nix tools"; + + inputs = { + nixpkgs.url = "nixpkgs/nixos-unstable"; + }; + + outputs = { self, nixpkgs }: + let + + lib = nixpkgs.lib; + + supportedSystems = [ "x86_64-linux" ]; + + forAllSystems = f: lib.genAttrs supportedSystems (system: f system); + + nixpkgsFor = forAllSystems (system: import nixpkgs { + inherit system; + overlays = [ self.overlay ]; + }); + + dream2nixFor = forAllSystems (system: import ./src { pkgs = nixpkgsFor."${system}"; } ); + + in + { + overlay = curr: prev: {}; + + apps = forAllSystems (system: { + translate = { + "type" = "app"; + "program" = builtins.toString (dream2nixFor."${system}".apps.translate); + }; + install = { + "type" = "app"; + "program" = builtins.toString (dream2nixFor."${system}".apps.install); + }; + }); + }; +} diff --git a/specifications/generic-lock-example.json b/specifications/generic-lock-example.json index 25b3b9fc..87f54fa9 100644 --- a/specifications/generic-lock-example.json +++ b/specifications/generic-lock-example.json @@ -3,12 +3,12 @@ "requests": { "url": "https://download.pypi.org/requests/2.28.0", "hash": "000000000000000000000000000000000000000", - "type": "tarball" + "type": "fetchurl" }, "certifi": { "url": "https://download.pypi.org/certifi/2.0", "hash": "000000000000000000000000000000000000000", - "type": "tarball" + "type": "fetchurl" } }, @@ -16,7 +16,6 @@ "buildSystem": "python", "buildSystemFormatVersion": 1, "producedBy": "translator-poetry-1", - "system": "x86_64-linux", "dependencyGraph": { "requests": [ "certifi" @@ -25,6 +24,7 @@ }, "buildSystem": { + "pythonAttr": "python38", "sourceFormats": { "requests": "sdist", "certifi": "wheel" diff --git a/specifications/generic-lock-schema.json b/specifications/generic-lock-schema.json index 565f5482..2af27a24 100644 --- a/specifications/generic-lock-schema.json +++ b/specifications/generic-lock-schema.json @@ -16,8 +16,7 @@ "fetchurl", "git", "github", - "gitlab", - "tarball" + "gitlab" ] }, "url": { @@ -58,14 +57,6 @@ "rev": { "type": "string" } } } - }, - { - "if": { - "properties": { "type": { "const": "tarball" } } - }, - "then": { - "properties": { "url": { "type": "string" } } - } } ] } @@ -79,7 +70,6 @@ "buildSystem": { "type": "string" }, "buildSystemFormatVersion": { "type": "integer" }, "producedBy": { "type": "string" }, - "system": { "type": "string" }, "dependencyGraph": { "type": "object", "properties": { diff --git a/specifications/translator-call-example.json b/specifications/translator-call-example.json new file mode 100644 index 00000000..d7de07eb --- /dev/null +++ b/specifications/translator-call-example.json @@ -0,0 +1,6 @@ +{ + "inputFiles": [ + "./some_project_src/requirements.txt", + "./some_project_src/requirements-dev.txt" + ] +} diff --git a/src/apps/default.nix b/src/apps/default.nix new file mode 100644 index 00000000..764e8a2c --- /dev/null +++ b/src/apps/default.nix @@ -0,0 +1,36 @@ +{ + pkgs, + location, +}: +let + callPackage = pkgs.callPackage; +in +{ + # translate cli + translate = callPackage ({ writeScript, }: + writeScript + "translate" + ''${import ../translators { inherit pkgs; }}/bin/cli "$@"'' + ) {}; + + install = callPackage ({ writeScript, }: + writeScript + "install" + '' + target="$1" + if [[ "$target" == "" ]]; then + echo "specify target" + exit 1 + fi + + mkdir -p "$target" + if [ -n "$(ls -A $target)" ]; then + echo "target directory not empty" + exit 1 + fi + + cp -r ${location}/* $target/ + chmod -R +w $target + '' + ) {}; +} diff --git a/src/builders/default.nix b/src/builders/default.nix new file mode 100644 index 00000000..c08fc746 --- /dev/null +++ b/src/builders/default.nix @@ -0,0 +1,12 @@ +{ + callPackage, +}: +rec { + python = rec { + + default = simpleBuilder; + + simpleBuilder = callPackage ./python/simple-builder {}; + }; + +} diff --git a/src/builders/python/simple-builder/default.nix b/src/builders/python/simple-builder/default.nix new file mode 100644 index 00000000..9f0afba4 --- /dev/null +++ b/src/builders/python/simple-builder/default.nix @@ -0,0 +1,40 @@ +{ + lib, + pkgs, +}: + +{ + fetchedSources, + genericLock, +}: + +let + python = pkgs."${genericLock.buildSystem.pythonAttr}"; +in + +python.pkgs.buildPythonPackage { + name = "python-environment"; + format = ""; + src = lib.attrValues fetchedSources; + buildInputs = pkgs.pythonManylinuxPackages.manylinux1; + nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ]; + unpackPhase = '' + mkdir dist + for file in $src; do + fname=$(echo $file | cut -d "-" -f 2-) + cp $file dist/$fname + done + ''; + installPhase = '' + runHook preInstall + mkdir -p "$out/${python.sitePackages}" + export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" + ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} \ + --no-index \ + --no-warn-script-location \ + --prefix="$out" \ + --no-cache $pipInstallFlags \ + --ignore-installed + runHook postInstall + ''; +} diff --git a/src/default.nix b/src/default.nix new file mode 100644 index 00000000..1e218dfe --- /dev/null +++ b/src/default.nix @@ -0,0 +1,44 @@ +{ + pkgs ? import {}, +}: + +let + callPackage = pkgs.callPackage; +in + +rec { + + apps = callPackage ./apps { inherit location; }; + + builders = callPackage ./builders {}; + + fetchers = callPackage ./fetchers {}; + + translators = callPackage ./translators {}; + + + # the location of the dream2nix framework for self references (update scripts, etc.) + location = ./.; + + + # automatically find a suitable builder for a given generic lock + findBuilder = genericLock: + let + buildSystem = genericLock.generic.buildSystem; + in + builders."${buildSystem}".default; + + + # automatically build package defined by generic lock + buildPackage = + { + genericLock, + builder ? findBuilder genericLock, + fetcher ? fetchers.defaultFetcher + }: + builder { + inherit genericLock; + fetchedSources = fetcher { sources = genericLock.sources; }; + }; + +} \ No newline at end of file diff --git a/src/fetchers/default-fetcher.nix b/src/fetchers/default-fetcher.nix new file mode 100644 index 00000000..14d483cb --- /dev/null +++ b/src/fetchers/default-fetcher.nix @@ -0,0 +1,37 @@ +{ + # fetchers + fetchFromGitHub, + fetchFromGitLab, + fetchgit, + fetchurl, + + lib, + ... +}: +{ + # sources attrset from generic lock + sources, +}: +lib.mapAttrs (pname: source: + if source.type == "github" then + fetchFromGitHub { + inherit (source) url owner repo rev; + sha256 = source.hash; + } + else if source.type == "gitlab" then + fetchFromGitLab { + inherit (source) url owner repo rev; + sha256 = source.hash; + } + else if source.type == "git" then + fetchgit { + inherit (source) url rev; + sha256 = source.hash; + } + else if source.type == "fetchurl" then + fetchurl { + inherit (source) url; + sha256 = source.hash; + } + else throw "unsupported source type '${source.type}'" +) sources diff --git a/src/fetchers/default.nix b/src/fetchers/default.nix new file mode 100644 index 00000000..78611556 --- /dev/null +++ b/src/fetchers/default.nix @@ -0,0 +1,6 @@ +{ + callPackage, +}: +{ + defaultFetcher = callPackage ./default-fetcher.nix {}; +} diff --git a/src/translators/cli.py b/src/translators/cli.py new file mode 100644 index 00000000..972c9c63 --- /dev/null +++ b/src/translators/cli.py @@ -0,0 +1,100 @@ +import argparse +import json +import os +import subprocess as sp +import sys +import tempfile + + +with open (os.environ.get("translatorsJsonFile")) as f: + translators = json.load(f) + + +# TODO: detection translator automatically according to files +def auto_detect_translator(files, subsystem): + return list(translators[subsystem].keys())[0] + + +def parse_args(): + + parser = argparse.ArgumentParser( + prog="translate", + description="translate projects to nix", + ) + + parser.add_argument( + "-s", "--subsystem", + help="which subsystem to use, (eg: python, nodejs, ...)", + choices=translators.keys() + ) + + parser.add_argument( + "-t", "--translator", + help="which specific translator to use", + default="auto" + ) + + parser.add_argument( + "-o", "--output", + help="output file/directory (generic lock)", + default="./dream.lock" + ) + + parser.add_argument( + "input", + help="input files containing relevant metadata", + nargs="+" + ) + + args = parser.parse_args() + + # TODO: detection subsystem automatically according to files + if not hasattr(args, "subsystem"): + print("Please specify subsystem (-s, --subsystem)", file=sys.stderr) + parser.print_help() + exit(1) + + return args + + +def main(): + + args = parse_args() + + subsystem = args.subsystem + files = args.input + + # determine translator + if args.translator == "auto": + translator = auto_detect_translator(files, subsystem) + else: + translator = args.translator + + # determine output directory + if os.path.isdir(args.output): + output = f"{args.output}/dream.lock" + else: + output = args.output + + # translator arguments + translatorInput = dict( + inputFiles=files, + outputFile=output, + ) + + # dump translator arguments to json file and execute translator + with tempfile.NamedTemporaryFile("w") as inputJson: + json.dump(translatorInput, inputJson, indent=2) + inputJson.seek(0) # flushes write cache + sp.run( + [f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:] + ) + + if not os.path.isfile(output): + raise Exception(f"Translator '{translator}' failed to create dream.lock") + + print(f"Created {output}") + + +if __name__ == "__main__": + main() diff --git a/src/translators/default.nix b/src/translators/default.nix new file mode 100644 index 00000000..747247d0 --- /dev/null +++ b/src/translators/default.nix @@ -0,0 +1,56 @@ +{ pkgs }: +let + + callPackage = pkgs.callPackage; + + # every translator must provide 'bin/translate' + translatorExec = translatorPkg: "${translatorPkg}/bin/translate"; + + # the list of all available translators + translators = { + + python = { + + # minimal POC python translator using pip. Type: 'external' + external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; }; + external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; }; + external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; }; + external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; }; + external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; }; + + # TODO: add more translators + + }; + }; + + # Put all translator executables in a json file. + # This will allow the cli to call the translators of different build systems + # in a standardised way + translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }: + runCommand + "translators.json" + { + buildInputs = lib.flatten + ( + lib.mapAttrsToList + (subsystem: translators: + lib.attrValues translators + ) + translators + ); + } + # 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs + '' + #!${bash}/bin/bash + cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out + '' + ) {}; + +in + +# the unified translator cli +callPackage ({ python3, writeScriptBin, ... }: + writeScriptBin "cli" '' + translatorsJsonFile=${translatorsJsonFile} ${python3}/bin/python ${./cli.py} "$@" + '' +) {} diff --git a/src/translators/python/external-pip/default.nix b/src/translators/python/external-pip/default.nix new file mode 100644 index 00000000..99e3f8b7 --- /dev/null +++ b/src/translators/python/external-pip/default.nix @@ -0,0 +1,43 @@ +{ + bash, + jq, + python, + writeScriptBin, + ... +}: + +# +# the input format is specified in /specifications/translator-call-example.json + +writeScriptBin "translate" '' + #!${bash}/bin/bash + + set -Eeuo pipefail + + # accroding to the spec, the translator reads the input from a json file + jsonInput=$1 + + # extract the 'inputFiles' field from the json + inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput) + outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput) + + # pip executable + pip=${python.pkgs.pip}/bin/pip + + # prepare temporary directory + tmp=translateTmp + rm -rf $tmp + mkdir $tmp + + # download files according to requirements + $pip download \ + --no-cache \ + --dest $tmp \ + --progress-bar off \ + -r ''${inputFiles/$'\n'/$' -r '} + + # generate the generic lock from the downloaded list of files + ${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile + + rm -rf $tmp +'' diff --git a/src/translators/python/external-pip/generate-generic-lock.py b/src/translators/python/external-pip/generate-generic-lock.py new file mode 100644 index 00000000..9a33e9ff --- /dev/null +++ b/src/translators/python/external-pip/generate-generic-lock.py @@ -0,0 +1,78 @@ +from glob import glob +import base64 +import hashlib +import json +import sys + + +def main(): + direcotry = sys.argv[1] + output_file = sys.argv[2] + + packages = {} + + # loop over the downloaded files and compute: + # - url + # - sha256 + # - format (sdist/wheel) + for path in list(glob(direcotry + '/*')): + _, _, file = path.rpartition('/') + + print(f"processing file: {file}") + + # example: charset_normalizer-2.0.4-py3-none-any.whl + if file.endswith('.whl'): + format = 'wheel' + pname, _, pyver, _, _ = file.split('-') + # example: requests-2.26.0.tar.gz + else: + format = 'sdist' + pname, _, _ = file.rpartition('-') + pyver = 'source' + + url = f"https://files.pythonhosted.org/packages/{pyver}/{pname[0]}/{pname}/{file}" + + with open(path, 'rb') as f: + sha256 = f"sha256-{base64.b64encode(hashlib.sha256(f.read()).digest()).decode()}" + + packages[pname] = dict( + url=url, + sha256=sha256, + format=format + ) + + # create generic lock + generic_lock = dict( + sources={}, + generic={ + "buildSystem": "python", + "buildSystemFormatVersion": 1, + "producedBy": "translator-external-pip", + + # This translator is not aware of the exact dependency graph. + # This restricts us to use a single derivation builder later, + # which will install all packages at once + "dependencyGraph": None, + }, + buildSystem={ + "pythonAttr": f"python{sys.version_info.major}{sys.version_info.minor}", + "sourceFormats": + {pname: data['format'] for pname, data in packages.items()} + } + ) + + # populate sources of generic lock + for pname, data in packages.items(): + generic_lock['sources'][pname] = dict( + url=data['url'], + hash=data['sha256'], + type='fetchurl', + ) + + # dump generic lock to stdout (json) + with open(output_file, 'w') as lock: + json.dump(generic_lock, lock, indent=2)2 + + +if __name__ == "__main__": + main()