mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-08-16 18:00:30 +03:00
implement UI and examples:
- generic translator CLI exposed as flake app - flake app 'install' to install the framework - generic fetcher - simple translator for python - simple builer for python
This commit is contained in:
parent
094b4c92fa
commit
2ce1ddb6f3
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
.*/
|
||||
result
|
26
flake.lock
Normal file
26
flake.lock
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1631015389,
|
||||
"narHash": "sha256-9SKTwRbp7OQGa4wKTo1wB6ctC2VlaIeKFZ+flqzPuoI=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "bc06c93905f60a82d6ebbb78f78cf289257860cc",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
"ref": "nixos-unstable",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
39
flake.nix
Normal file
39
flake.nix
Normal file
@ -0,0 +1,39 @@
|
||||
{
|
||||
description = "dream2nix: A generic framework for 2nix tools";
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "nixpkgs/nixos-unstable";
|
||||
};
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
let
|
||||
|
||||
lib = nixpkgs.lib;
|
||||
|
||||
supportedSystems = [ "x86_64-linux" ];
|
||||
|
||||
forAllSystems = f: lib.genAttrs supportedSystems (system: f system);
|
||||
|
||||
nixpkgsFor = forAllSystems (system: import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ self.overlay ];
|
||||
});
|
||||
|
||||
dream2nixFor = forAllSystems (system: import ./src { pkgs = nixpkgsFor."${system}"; } );
|
||||
|
||||
in
|
||||
{
|
||||
overlay = curr: prev: {};
|
||||
|
||||
apps = forAllSystems (system: {
|
||||
translate = {
|
||||
"type" = "app";
|
||||
"program" = builtins.toString (dream2nixFor."${system}".apps.translate);
|
||||
};
|
||||
install = {
|
||||
"type" = "app";
|
||||
"program" = builtins.toString (dream2nixFor."${system}".apps.install);
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
@ -3,12 +3,12 @@
|
||||
"requests": {
|
||||
"url": "https://download.pypi.org/requests/2.28.0",
|
||||
"hash": "000000000000000000000000000000000000000",
|
||||
"type": "tarball"
|
||||
"type": "fetchurl"
|
||||
},
|
||||
"certifi": {
|
||||
"url": "https://download.pypi.org/certifi/2.0",
|
||||
"hash": "000000000000000000000000000000000000000",
|
||||
"type": "tarball"
|
||||
"type": "fetchurl"
|
||||
}
|
||||
},
|
||||
|
||||
@ -16,7 +16,6 @@
|
||||
"buildSystem": "python",
|
||||
"buildSystemFormatVersion": 1,
|
||||
"producedBy": "translator-poetry-1",
|
||||
"system": "x86_64-linux",
|
||||
"dependencyGraph": {
|
||||
"requests": [
|
||||
"certifi"
|
||||
@ -25,6 +24,7 @@
|
||||
},
|
||||
|
||||
"buildSystem": {
|
||||
"pythonAttr": "python38",
|
||||
"sourceFormats": {
|
||||
"requests": "sdist",
|
||||
"certifi": "wheel"
|
||||
|
@ -16,8 +16,7 @@
|
||||
"fetchurl",
|
||||
"git",
|
||||
"github",
|
||||
"gitlab",
|
||||
"tarball"
|
||||
"gitlab"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
@ -58,14 +57,6 @@
|
||||
"rev": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": { "type": { "const": "tarball" } }
|
||||
},
|
||||
"then": {
|
||||
"properties": { "url": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -79,7 +70,6 @@
|
||||
"buildSystem": { "type": "string" },
|
||||
"buildSystemFormatVersion": { "type": "integer" },
|
||||
"producedBy": { "type": "string" },
|
||||
"system": { "type": "string" },
|
||||
"dependencyGraph": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
6
specifications/translator-call-example.json
Normal file
6
specifications/translator-call-example.json
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
"inputFiles": [
|
||||
"./some_project_src/requirements.txt",
|
||||
"./some_project_src/requirements-dev.txt"
|
||||
]
|
||||
}
|
36
src/apps/default.nix
Normal file
36
src/apps/default.nix
Normal file
@ -0,0 +1,36 @@
|
||||
{
|
||||
pkgs,
|
||||
location,
|
||||
}:
|
||||
let
|
||||
callPackage = pkgs.callPackage;
|
||||
in
|
||||
{
|
||||
# translate cli
|
||||
translate = callPackage ({ writeScript, }:
|
||||
writeScript
|
||||
"translate"
|
||||
''${import ../translators { inherit pkgs; }}/bin/cli "$@"''
|
||||
) {};
|
||||
|
||||
install = callPackage ({ writeScript, }:
|
||||
writeScript
|
||||
"install"
|
||||
''
|
||||
target="$1"
|
||||
if [[ "$target" == "" ]]; then
|
||||
echo "specify target"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$target"
|
||||
if [ -n "$(ls -A $target)" ]; then
|
||||
echo "target directory not empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp -r ${location}/* $target/
|
||||
chmod -R +w $target
|
||||
''
|
||||
) {};
|
||||
}
|
12
src/builders/default.nix
Normal file
12
src/builders/default.nix
Normal file
@ -0,0 +1,12 @@
|
||||
{
|
||||
callPackage,
|
||||
}:
|
||||
rec {
|
||||
python = rec {
|
||||
|
||||
default = simpleBuilder;
|
||||
|
||||
simpleBuilder = callPackage ./python/simple-builder {};
|
||||
};
|
||||
|
||||
}
|
40
src/builders/python/simple-builder/default.nix
Normal file
40
src/builders/python/simple-builder/default.nix
Normal file
@ -0,0 +1,40 @@
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
}:
|
||||
|
||||
{
|
||||
fetchedSources,
|
||||
genericLock,
|
||||
}:
|
||||
|
||||
let
|
||||
python = pkgs."${genericLock.buildSystem.pythonAttr}";
|
||||
in
|
||||
|
||||
python.pkgs.buildPythonPackage {
|
||||
name = "python-environment";
|
||||
format = "";
|
||||
src = lib.attrValues fetchedSources;
|
||||
buildInputs = pkgs.pythonManylinuxPackages.manylinux1;
|
||||
nativeBuildInputs = [ pkgs.autoPatchelfHook python.pkgs.wheelUnpackHook ];
|
||||
unpackPhase = ''
|
||||
mkdir dist
|
||||
for file in $src; do
|
||||
fname=$(echo $file | cut -d "-" -f 2-)
|
||||
cp $file dist/$fname
|
||||
done
|
||||
'';
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
mkdir -p "$out/${python.sitePackages}"
|
||||
export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
|
||||
${python}/bin/python -m pip install ./dist/*.{whl,tar.gz,zip} \
|
||||
--no-index \
|
||||
--no-warn-script-location \
|
||||
--prefix="$out" \
|
||||
--no-cache $pipInstallFlags \
|
||||
--ignore-installed
|
||||
runHook postInstall
|
||||
'';
|
||||
}
|
44
src/default.nix
Normal file
44
src/default.nix
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
pkgs ? import <nixpkgs> {},
|
||||
}:
|
||||
|
||||
let
|
||||
callPackage = pkgs.callPackage;
|
||||
in
|
||||
|
||||
rec {
|
||||
|
||||
apps = callPackage ./apps { inherit location; };
|
||||
|
||||
builders = callPackage ./builders {};
|
||||
|
||||
fetchers = callPackage ./fetchers {};
|
||||
|
||||
translators = callPackage ./translators {};
|
||||
|
||||
|
||||
# the location of the dream2nix framework for self references (update scripts, etc.)
|
||||
location = ./.;
|
||||
|
||||
|
||||
# automatically find a suitable builder for a given generic lock
|
||||
findBuilder = genericLock:
|
||||
let
|
||||
buildSystem = genericLock.generic.buildSystem;
|
||||
in
|
||||
builders."${buildSystem}".default;
|
||||
|
||||
|
||||
# automatically build package defined by generic lock
|
||||
buildPackage =
|
||||
{
|
||||
genericLock,
|
||||
builder ? findBuilder genericLock,
|
||||
fetcher ? fetchers.defaultFetcher
|
||||
}:
|
||||
builder {
|
||||
inherit genericLock;
|
||||
fetchedSources = fetcher { sources = genericLock.sources; };
|
||||
};
|
||||
|
||||
}
|
37
src/fetchers/default-fetcher.nix
Normal file
37
src/fetchers/default-fetcher.nix
Normal file
@ -0,0 +1,37 @@
|
||||
{
|
||||
# fetchers
|
||||
fetchFromGitHub,
|
||||
fetchFromGitLab,
|
||||
fetchgit,
|
||||
fetchurl,
|
||||
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# sources attrset from generic lock
|
||||
sources,
|
||||
}:
|
||||
lib.mapAttrs (pname: source:
|
||||
if source.type == "github" then
|
||||
fetchFromGitHub {
|
||||
inherit (source) url owner repo rev;
|
||||
sha256 = source.hash;
|
||||
}
|
||||
else if source.type == "gitlab" then
|
||||
fetchFromGitLab {
|
||||
inherit (source) url owner repo rev;
|
||||
sha256 = source.hash;
|
||||
}
|
||||
else if source.type == "git" then
|
||||
fetchgit {
|
||||
inherit (source) url rev;
|
||||
sha256 = source.hash;
|
||||
}
|
||||
else if source.type == "fetchurl" then
|
||||
fetchurl {
|
||||
inherit (source) url;
|
||||
sha256 = source.hash;
|
||||
}
|
||||
else throw "unsupported source type '${source.type}'"
|
||||
) sources
|
6
src/fetchers/default.nix
Normal file
6
src/fetchers/default.nix
Normal file
@ -0,0 +1,6 @@
|
||||
{
|
||||
callPackage,
|
||||
}:
|
||||
{
|
||||
defaultFetcher = callPackage ./default-fetcher.nix {};
|
||||
}
|
100
src/translators/cli.py
Normal file
100
src/translators/cli.py
Normal file
@ -0,0 +1,100 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess as sp
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
|
||||
with open (os.environ.get("translatorsJsonFile")) as f:
|
||||
translators = json.load(f)
|
||||
|
||||
|
||||
# TODO: detection translator automatically according to files
|
||||
def auto_detect_translator(files, subsystem):
|
||||
return list(translators[subsystem].keys())[0]
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="translate",
|
||||
description="translate projects to nix",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-s", "--subsystem",
|
||||
help="which subsystem to use, (eg: python, nodejs, ...)",
|
||||
choices=translators.keys()
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-t", "--translator",
|
||||
help="which specific translator to use",
|
||||
default="auto"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-o", "--output",
|
||||
help="output file/directory (generic lock)",
|
||||
default="./dream.lock"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"input",
|
||||
help="input files containing relevant metadata",
|
||||
nargs="+"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# TODO: detection subsystem automatically according to files
|
||||
if not hasattr(args, "subsystem"):
|
||||
print("Please specify subsystem (-s, --subsystem)", file=sys.stderr)
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
|
||||
subsystem = args.subsystem
|
||||
files = args.input
|
||||
|
||||
# determine translator
|
||||
if args.translator == "auto":
|
||||
translator = auto_detect_translator(files, subsystem)
|
||||
else:
|
||||
translator = args.translator
|
||||
|
||||
# determine output directory
|
||||
if os.path.isdir(args.output):
|
||||
output = f"{args.output}/dream.lock"
|
||||
else:
|
||||
output = args.output
|
||||
|
||||
# translator arguments
|
||||
translatorInput = dict(
|
||||
inputFiles=files,
|
||||
outputFile=output,
|
||||
)
|
||||
|
||||
# dump translator arguments to json file and execute translator
|
||||
with tempfile.NamedTemporaryFile("w") as inputJson:
|
||||
json.dump(translatorInput, inputJson, indent=2)
|
||||
inputJson.seek(0) # flushes write cache
|
||||
sp.run(
|
||||
[f"{translators[subsystem][translator]}/bin/translate", inputJson.name] + sys.argv[1:]
|
||||
)
|
||||
|
||||
if not os.path.isfile(output):
|
||||
raise Exception(f"Translator '{translator}' failed to create dream.lock")
|
||||
|
||||
print(f"Created {output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
56
src/translators/default.nix
Normal file
56
src/translators/default.nix
Normal file
@ -0,0 +1,56 @@
|
||||
{ pkgs }:
|
||||
let
|
||||
|
||||
callPackage = pkgs.callPackage;
|
||||
|
||||
# every translator must provide 'bin/translate'
|
||||
translatorExec = translatorPkg: "${translatorPkg}/bin/translate";
|
||||
|
||||
# the list of all available translators
|
||||
translators = {
|
||||
|
||||
python = {
|
||||
|
||||
# minimal POC python translator using pip. Type: 'external'
|
||||
external-pip-python36 = callPackage ./python/external-pip { python = pkgs.python36; };
|
||||
external-pip-python37 = callPackage ./python/external-pip { python = pkgs.python37; };
|
||||
external-pip-python38 = callPackage ./python/external-pip { python = pkgs.python38; };
|
||||
external-pip-python39 = callPackage ./python/external-pip { python = pkgs.python39; };
|
||||
external-pip-python310 = callPackage ./python/external-pip { python = pkgs.python310; };
|
||||
|
||||
# TODO: add more translators
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
# Put all translator executables in a json file.
|
||||
# This will allow the cli to call the translators of different build systems
|
||||
# in a standardised way
|
||||
translatorsJsonFile = callPackage ({ bash, lib, runCommand, ... }:
|
||||
runCommand
|
||||
"translators.json"
|
||||
{
|
||||
buildInputs = lib.flatten
|
||||
(
|
||||
lib.mapAttrsToList
|
||||
(subsystem: translators:
|
||||
lib.attrValues translators
|
||||
)
|
||||
translators
|
||||
);
|
||||
}
|
||||
# 'unsafeDiscardStringContext' is safe in thix context because all store paths are declared as buildInputs
|
||||
''
|
||||
#!${bash}/bin/bash
|
||||
cp ${builtins.toFile "translators.json" (builtins.unsafeDiscardStringContext (builtins.toJSON translators))} $out
|
||||
''
|
||||
) {};
|
||||
|
||||
in
|
||||
|
||||
# the unified translator cli
|
||||
callPackage ({ python3, writeScriptBin, ... }:
|
||||
writeScriptBin "cli" ''
|
||||
translatorsJsonFile=${translatorsJsonFile} ${python3}/bin/python ${./cli.py} "$@"
|
||||
''
|
||||
) {}
|
43
src/translators/python/external-pip/default.nix
Normal file
43
src/translators/python/external-pip/default.nix
Normal file
@ -0,0 +1,43 @@
|
||||
{
|
||||
bash,
|
||||
jq,
|
||||
python,
|
||||
writeScriptBin,
|
||||
...
|
||||
}:
|
||||
|
||||
#
|
||||
# the input format is specified in /specifications/translator-call-example.json
|
||||
|
||||
writeScriptBin "translate" ''
|
||||
#!${bash}/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
# accroding to the spec, the translator reads the input from a json file
|
||||
jsonInput=$1
|
||||
|
||||
# extract the 'inputFiles' field from the json
|
||||
inputFiles=$(${jq}/bin/jq '.inputFiles | .[]' -c -r $jsonInput)
|
||||
outputFile=$(${jq}/bin/jq '.outputFile' -c -r $jsonInput)
|
||||
|
||||
# pip executable
|
||||
pip=${python.pkgs.pip}/bin/pip
|
||||
|
||||
# prepare temporary directory
|
||||
tmp=translateTmp
|
||||
rm -rf $tmp
|
||||
mkdir $tmp
|
||||
|
||||
# download files according to requirements
|
||||
$pip download \
|
||||
--no-cache \
|
||||
--dest $tmp \
|
||||
--progress-bar off \
|
||||
-r ''${inputFiles/$'\n'/$' -r '}
|
||||
|
||||
# generate the generic lock from the downloaded list of files
|
||||
${python}/bin/python ${./generate-generic-lock.py} $tmp $outputFile
|
||||
|
||||
rm -rf $tmp
|
||||
''
|
78
src/translators/python/external-pip/generate-generic-lock.py
Normal file
78
src/translators/python/external-pip/generate-generic-lock.py
Normal file
@ -0,0 +1,78 @@
|
||||
from glob import glob
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
direcotry = sys.argv[1]
|
||||
output_file = sys.argv[2]
|
||||
|
||||
packages = {}
|
||||
|
||||
# loop over the downloaded files and compute:
|
||||
# - url
|
||||
# - sha256
|
||||
# - format (sdist/wheel)
|
||||
for path in list(glob(direcotry + '/*')):
|
||||
_, _, file = path.rpartition('/')
|
||||
|
||||
print(f"processing file: {file}")
|
||||
|
||||
# example: charset_normalizer-2.0.4-py3-none-any.whl
|
||||
if file.endswith('.whl'):
|
||||
format = 'wheel'
|
||||
pname, _, pyver, _, _ = file.split('-')
|
||||
# example: requests-2.26.0.tar.gz
|
||||
else:
|
||||
format = 'sdist'
|
||||
pname, _, _ = file.rpartition('-')
|
||||
pyver = 'source'
|
||||
|
||||
url = f"https://files.pythonhosted.org/packages/{pyver}/{pname[0]}/{pname}/{file}"
|
||||
|
||||
with open(path, 'rb') as f:
|
||||
sha256 = f"sha256-{base64.b64encode(hashlib.sha256(f.read()).digest()).decode()}"
|
||||
|
||||
packages[pname] = dict(
|
||||
url=url,
|
||||
sha256=sha256,
|
||||
format=format
|
||||
)
|
||||
|
||||
# create generic lock
|
||||
generic_lock = dict(
|
||||
sources={},
|
||||
generic={
|
||||
"buildSystem": "python",
|
||||
"buildSystemFormatVersion": 1,
|
||||
"producedBy": "translator-external-pip",
|
||||
|
||||
# This translator is not aware of the exact dependency graph.
|
||||
# This restricts us to use a single derivation builder later,
|
||||
# which will install all packages at once
|
||||
"dependencyGraph": None,
|
||||
},
|
||||
buildSystem={
|
||||
"pythonAttr": f"python{sys.version_info.major}{sys.version_info.minor}",
|
||||
"sourceFormats":
|
||||
{pname: data['format'] for pname, data in packages.items()}
|
||||
}
|
||||
)
|
||||
|
||||
# populate sources of generic lock
|
||||
for pname, data in packages.items():
|
||||
generic_lock['sources'][pname] = dict(
|
||||
url=data['url'],
|
||||
hash=data['sha256'],
|
||||
type='fetchurl',
|
||||
)
|
||||
|
||||
# dump generic lock to stdout (json)
|
||||
with open(output_file, 'w') as lock:
|
||||
json.dump(generic_lock, lock, indent=2)2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Loading…
Reference in New Issue
Block a user