implement UI and examples:

- generic translator CLI exposed as flake app
 - flake app 'install' to install the framework
 - generic fetcher
 - simple translator for python
 - simple builer for python

.gitignore

@@ -0,0 +1,2 @@
+.*/
+result

flake.lock

@@ -0,0 +1,26 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1631015389,
+        "narHash": "sha256-9SKTwRbp7OQGa4wKTo1wB6ctC2VlaIeKFZ+flqzPuoI=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "bc06c93905f60a82d6ebbb78f78cf289257860cc",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,39 @@
+{
+  description = "dream2nix: A generic framework for 2nix tools";
+
+  inputs = {
+    nixpkgs.url = "nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+
+      lib = nixpkgs.lib; 
+
+      supportedSystems = [ "x86_64-linux" ];
+
+      forAllSystems = f: lib.genAttrs supportedSystems (system: f system);
+
+      nixpkgsFor = forAllSystems (system: import nixpkgs {
+        inherit system;
+        overlays = [ self.overlay ];
+      });
+
+      dream2nixFor = forAllSystems (system: import ./src { pkgs = nixpkgsFor."${system}"; } );
+
+    in
+      {
+        overlay = curr: prev: {};
+
+        apps = forAllSystems (system: {
+          translate = {
+            "type" = "app";
+            "program" = builtins.toString (dream2nixFor."${system}".apps.translate);
+          };
+          install = {
+            "type" = "app";
+            "program" = builtins.toString (dream2nixFor."${system}".apps.install);
+          };
+        });
+      };
+}

specifications/generic-lock-example.json

@@ -3,12 +3,12 @@
     "requests": {
       "url": "https://download.pypi.org/requests/2.28.0",
       "hash": "000000000000000000000000000000000000000",
-      "type": "tarball"
+      "type": "fetchurl"
     },
     "certifi": {
       "url": "https://download.pypi.org/certifi/2.0",
       "hash": "000000000000000000000000000000000000000",
-      "type": "tarball"
+      "type": "fetchurl"
     }
   },
 
@@ -16,7 +16,6 @@
     "buildSystem": "python",
     "buildSystemFormatVersion": 1,
     "producedBy": "translator-poetry-1",
-    "system": "x86_64-linux",
     "dependencyGraph": {
       "requests": [
         "certifi"
@@ -25,6 +24,7 @@
   },
 
   "buildSystem": {
+    "pythonAttr": "python38",
     "sourceFormats": {
       "requests": "sdist",
       "certifi": "wheel"

specifications/generic-lock-schema.json

@@ -16,8 +16,7 @@
                 "fetchurl",
                 "git",
                 "github",
-                "gitlab",
-                "tarball"
+                "gitlab"
               ]
             },
             "url": {
@@ -58,14 +57,6 @@
                   "rev": { "type": "string" }
                 }
               }
-            },
-            {
-              "if": {
-                "properties": { "type": { "const": "tarball" } }
-              },
-              "then": {
-                "properties": { "url": { "type": "string" } }
-              }
             }
           ]
         }
@@ -79,7 +70,6 @@
         "buildSystem": { "type": "string" },
         "buildSystemFormatVersion": { "type": "integer" },
         "producedBy": { "type": "string" },
-        "system": { "type": "string" },
         "dependencyGraph": {
           "type": "object",
           "properties": {

specifications/translator-call-example.json

@@ -0,0 +1,6 @@
+{
+  "inputFiles": [
+    "./some_project_src/requirements.txt",
+    "./some_project_src/requirements-dev.txt"
+  ]
+}

src/apps/default.nix

@@ -0,0 +1,36 @@
+{
+  pkgs,
+  location,
+}:
+let
+  callPackage = pkgs.callPackage;
+in
+{
+  # translate cli
+  translate = callPackage ({ writeScript, }:
+    writeScript
+      "translate"
+      ''${import ../translators { inherit pkgs; }}/bin/cli "$@"''
+  ) {};
+
+  install = callPackage ({ writeScript, }:
+    writeScript
+      "install"
+      ''
+        target="$1"
+        if [[ "$target" == "" ]]; then
+          echo "specify target"
+          exit 1
+        fi
+
+        mkdir -p "$target"
+        if [ -n "$(ls -A $target)" ]; then
+          echo "target directory not empty"
+          exit 1
+        fi
+
+        cp -r ${location}/* $target/
+        chmod -R +w $target
+      ''
+  ) {};
+}

src/builders/default.nix

@@ -0,0 +1,12 @@
+{
+  callPackage,
+}:
+rec {
+  python = rec {
+
+    default = simpleBuilder;
+
+    simpleBuilder = callPackage ./python/simple-builder {};
+  };
+  
+}

src/builders/python/simple-builder/default.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  pkgs,
+}:
+
+{
+  fetchedSources,
+  genericLock,
+}:
+
+let
+  python = pkgs."${genericLock.buildSystem.pythonAttr}";
+in
+
+python.pkgs.buildPythonPackage {
+  name = "python-environment";
+  format = "";
+  src = lib.attrValues fetchedSources;
+  buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
+  nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ];
+  unpackPhase = ''
+    mkdir dist 
+    for file in $src; do
+      fname=$(echo $file | cut -d "-" -f 2-)
+      cp $file dist/$fname
+    done
+  '';
+  installPhase = ''
+    runHook preInstall
+    mkdir -p "$out/${python.sitePackages}"
+    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
+    ${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} \
+      --no-index \
+      --no-warn-script-location \
+      --prefix="$out" \
+      --no-cache $pipInstallFlags \
+      --ignore-installed
+    runHook postInstall
+  '';
+}

src/default.nix

@@ -0,0 +1,44 @@
+{
+  pkgs ? import <nixpkgs> {},
+}:
+
+let
+  callPackage = pkgs.callPackage;
+in
+
+rec {
+
+  apps = callPackage ./apps { inherit location; };
+
+  builders = callPackage ./builders {};
+
+  fetchers = callPackage ./fetchers {};
+
+  translators = callPackage ./translators {};
+
+
+  # the location of the dream2nix framework for self references (update scripts, etc.)
+  location = ./.;
+
+
+  # automatically find a suitable builder for a given generic lock
+  findBuilder = genericLock:
+    let 
+      buildSystem = genericLock.generic.buildSystem;
+    in
+      builders."${buildSystem}".default;
+
+
+  # automatically build package defined by generic lock
+  buildPackage = 
+    {
+      genericLock,
+      builder ? findBuilder genericLock,
+      fetcher ? fetchers.defaultFetcher
+    }:
+    builder {
+      inherit genericLock;
+      fetchedSources = fetcher { sources = genericLock.sources; };
+    };
+   
+}

src/fetchers/default-fetcher.nix

@@ -0,0 +1,37 @@
+{
+  # fetchers
+  fetchFromGitHub,
+  fetchFromGitLab,
+  fetchgit,
+  fetchurl,
+
+  lib,
+  ...
+}:
+{
+  # sources attrset from generic lock
+  sources,
+}:
+lib.mapAttrs (pname: source:
+  if source.type == "github" then
+    fetchFromGitHub {
+      inherit (source) url owner repo rev;
+      sha256 = source.hash;
+    }
+  else if source.type == "gitlab" then 
+    fetchFromGitLab {
+      inherit (source) url owner repo rev;
+      sha256 = source.hash;
+    }
+  else if source.type == "git" then
+    fetchgit {
+      inherit (source) url rev;
+      sha256 = source.hash;
+    }
+  else if source.type == "fetchurl" then
+    fetchurl {
+      inherit (source) url;
+      sha256 = source.hash;
+    }
+  else throw "unsupported source type '${source.type}'"
+) sources

src/fetchers/default.nix

@@ -0,0 +1,6 @@
+{
+  callPackage,
+}:
+{
+  defaultFetcher = callPackage ./default-fetcher.nix {};
+}

src/translators/cli.py

@@ -0,0 +1,100 @@
+import argparse
+import json
+import os
+import subprocess as sp
+import sys
+import tempfile
+
+
+with open (os.environ.get("translatorsJsonFile")) as f:
+  translators = json.load(f)
+
+
+# TODO: detection translator automatically according to files
+def auto_detect_translator(files, subsystem):
+  return list(translators[subsystem].keys())[0]
+
+
+def parse_args():
+
+  parser = argparse.ArgumentParser(
+    prog="translate",
+    description="translate projects to nix",
+  )
+
+  parser.add_argument(
+    "-s", "--subsystem",
+    help="which subsystem to use, (eg: python, nodejs, ...)",
+    choices=translators.keys()
+  )
+
+  parser.add_argument(
+    "-t", "--translator",
+    help="which specific translator to use",
+    default="auto"
+  )
+
+  parser.add_argument(
+    "-o", "--output",
+    help="output file/directory (generic lock)",
+    default="./dream.lock"
+  )
+
+  parser.add_argument(
+    "input",
+    help="input files containing relevant metadata",
+    nargs="+"
+  )
+
+  args = parser.parse_args()
+
+  # TODO: detection subsystem automatically according to files
+  if not hasattr(args, "subsystem"):
+    print("Please specify subsystem (-s, --subsystem)", file=sys.stderr)
+    parser.print_help()
+    exit(1)
+
+  return args
+
+
+def main():
+
+  args = parse_args()
+
+  subsystem = args.subsystem
+  files = args.input
+
+  # determine translator
+  if args.translator == "auto":
+    translator = auto_detect_translator(files, subsystem)
+  else:
+    translator = args.translator
+
+  # determine output directory
+  if os.path.isdir(args.output):
+    output = f"{args.output}/dream.lock"
+  else:
+    output = args.output
+
+  # translator arguments
+  translatorInput = dict(
+    inputFiles=files,
+    outputFile=output,
+  )
+
+  # dump translator arguments to json file and execute translator
+  with tempfile.NamedTemporaryFile("w") as inputJson:
+    json.dump(translatorInput, inputJson, indent=2)
+    inputJson.seek(0) # flushes write cache
+    sp.run(
+      [f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:]
+    )
+
+  if not os.path.isfile(output):
+    raise Exception(f"Translator '{translator}' failed to create dream.lock")
+
+  print(f"Created {output}")
+
+
+if __name__ == "__main__":
+  main()

src/translators/default.nix

@@ -0,0 +1,56 @@
+{ pkgs }: 
+let
+
+  callPackage = pkgs.callPackage;
+
+  # every translator must provide 'bin/translate'
+  translatorExec = translatorPkg: "${translatorPkg}/bin/translate";
+
+  # the list of all available translators
+  translators = {
+
+    python = {
+
+      # minimal POC python translator using pip. Type: 'external'
+      external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; };
+      external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; };
+      external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; };
+      external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; };
+      external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; };
+
+      # TODO: add more translators
+
+    };
+  };
+
+  # Put all translator executables in a json file.
+  # This will allow the cli to call the translators of different build systems
+  # in a standardised way
+  translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }:
+    runCommand
+      "translators.json"
+      {
+        buildInputs = lib.flatten 
+          (
+            lib.mapAttrsToList
+              (subsystem: translators:
+                lib.attrValues translators
+              )
+              translators
+          );
+      }
+      # 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs
+      ''
+        #!${bash}/bin/bash
+        cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out
+      ''
+  ) {};
+
+in
+
+# the unified translator cli
+callPackage ({ python3, writeScriptBin, ... }:
+  writeScriptBin "cli" ''
+    translatorsJsonFile=${translatorsJsonFile} ${python3}/bin/python ${./cli.py} "$@"
+  ''
+) {}

src/translators/python/external-pip/default.nix

@@ -0,0 +1,43 @@
+{
+  bash,
+  jq,
+  python,
+  writeScriptBin,
+  ...
+}:
+
+#
+# the input format is specified in /specifications/translator-call-example.json
+
+writeScriptBin "translate" ''
+  #!${bash}/bin/bash
+
+  set -Eeuo pipefail
+
+  # accroding to the spec, the translator reads the input from a json file
+  jsonInput=$1
+
+  # extract the 'inputFiles' field from the json
+  inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput)
+  outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
+
+  # pip executable
+  pip=${python.pkgs.pip}/bin/pip
+
+  # prepare temporary directory
+  tmp=translateTmp
+  rm -rf $tmp
+  mkdir $tmp
+
+  # download files according to requirements
+  $pip download \
+    --no-cache \
+    --dest $tmp \
+    --progress-bar off \
+    -r ''${inputFiles/$'\n'/$' -r '}
+
+  # generate the generic lock from the downloaded list of files
+  ${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
+
+  rm -rf $tmp
+''

src/translators/python/external-pip/generate-generic-lock.py

@@ -0,0 +1,78 @@
+from glob import glob
+import base64
+import hashlib
+import json
+import sys
+
+
+def main():
+  direcotry = sys.argv[1]
+  output_file = sys.argv[2]
+
+  packages = {}
+
+  # loop over the downloaded files and compute:
+  #  - url
+  #  - sha256
+  #  - format (sdist/wheel)
+  for path in list(glob(direcotry + '/*')):
+    _, _, file = path.rpartition('/')
+
+    print(f"processing file: {file}")
+
+    # example: charset_normalizer-2.0.4-py3-none-any.whl
+    if file.endswith('.whl'):
+      format = 'wheel'
+      pname, _, pyver, _, _ = file.split('-')
+    # example: requests-2.26.0.tar.gz
+    else:
+      format = 'sdist'
+      pname, _, _ = file.rpartition('-')
+      pyver = 'source'
+
+    url = f"https://files.pythonhosted.org/packages/{pyver}/{pname[0]}/{pname}/{file}"
+
+    with open(path, 'rb') as f:
+      sha256 = f"sha256-{base64.b64encode(hashlib.sha256(f.read()).digest()).decode()}"
+
+    packages[pname] = dict(
+      url=url,
+      sha256=sha256,
+      format=format
+    )
+
+  # create generic lock
+  generic_lock = dict(
+    sources={},
+    generic={
+      "buildSystem": "python",
+      "buildSystemFormatVersion": 1,
+      "producedBy": "translator-external-pip",
+
+      # This translator is not aware of the exact dependency graph.
+      # This restricts us to use a single derivation builder later,
+      # which will install all packages at once
+      "dependencyGraph": None,
+    },
+    buildSystem={
+      "pythonAttr": f"python{sys.version_info.major}{sys.version_info.minor}",
+      "sourceFormats":
+        {pname: data['format'] for pname, data in packages.items()}
+    }
+  )
+
+  # populate sources of generic lock
+  for pname, data in packages.items():
+    generic_lock['sources'][pname] = dict(
+      url=data['url'],
+      hash=data['sha256'],
+      type='fetchurl',
+    )
+
+  # dump generic lock to stdout (json)
+  with open(output_file, 'w') as lock:
+    json.dump(generic_lock, lock, indent=2)2
+
+
+if __name__ == "__main__":
+  main()