Merge pull request #212 from a-kenji/init/debian-binary-builder

init: debian-binary builder

examples/debian-binary/flake.nix

@@ -0,0 +1,20 @@
+{
+  inputs = {
+    dream2nix.url = "github:nix-community/dream2nix";
+  };
+
+  outputs = {
+    self,
+    dream2nix,
+  } @ inp: (dream2nix.lib.makeFlakeOutputs {
+    systems = ["x86_64-linux"];
+    config.projectRoot = ./.;
+    source = ./.;
+    projects = {
+      htop = {
+        name = "htop";
+        translator = "debian-binary";
+      };
+    };
+  });
+}

src/subsystems/debian/builders/default/default.nix

@@ -0,0 +1,114 @@
+{...}: {
+  type = "pure";
+
+  build = {
+    lib,
+    pkgs,
+    stdenv,
+    # dream2nix inputs
+    externals,
+    ...
+  }: {
+    ### FUNCTIONS
+    # AttrSet -> Bool) -> AttrSet -> [x]
+    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
+    getDependencies, # name: version: -> [ {name=; version=; } ]
+    getSource, # name: version: -> store-path
+    # to get information about the original source spec
+    getSourceSpec, # name: version: -> {type="git"; url=""; hash="";}
+    ### ATTRIBUTES
+    subsystemAttrs, # attrset
+    defaultPackageName, # string
+    defaultPackageVersion, # string
+    # all exported (top-level) package names and versions
+    # attrset of pname -> version,
+    packages,
+    # all existing package names and versions
+    # attrset of pname -> versions,
+    # where versions is a list of version strings
+    packageVersions,
+    # function which applies overrides to a package
+    # It must be applied by the builder to each individual derivation
+    # Example:
+    #   produceDerivation name (mkDerivation {...})
+    produceDerivation,
+    ...
+  } @ args: let
+    l = lib // builtins;
+
+    allDependencySources' =
+      l.flatten
+      (l.mapAttrsToList
+        (name: versions:
+          if
+            l.elem name [
+              defaultPackageName
+              "libc6"
+            ]
+          then []
+          else l.map (ver: getSource name ver) versions)
+        packageVersions);
+
+    allDependencySources =
+      l.map
+      (src: src.original or src)
+      allDependencySources';
+
+    package = produceDerivation defaultPackageName (stdenv.mkDerivation {
+      name = defaultPackageName;
+      src = ":";
+      dontUnpack = true;
+      buildInputs = [pkgs.unzip];
+      nativeBuildInputs = [pkgs.autoPatchelfHook];
+      doCheck = false;
+      dontStrip = true;
+      buildPhase = ''
+        runHook preBuild
+
+        mkdir -p $out/bin
+        mkdir -p $out/share
+        mkdir -p $out/lib
+        mkdir -p $out/etc
+
+        for file in ${toString allDependencySources};do
+          mkdir -p $TMP/unpack
+          # unzip -d $TMP/unpack $file
+          cd $TMP/unpack
+          ar vx $file
+          tar xvf $TMP/unpack/data.tar.xz
+
+          echo $file
+
+          for variant in "bin" "sbin" "games"; do
+            if [[ -d $TMP/unpack/usr/$variant && -n "$(ls -A $TMP/unpack/usr/$variant)" ]]; then
+            echo "Copying usr/$variant"
+            cp -r $TMP/unpack/usr/$variant/* $out/bin
+            fi
+          done
+
+          echo "Copying usr/share"
+          if [ -d $TMP/unpack/usr/share ]; then
+          cp -r $TMP/unpack/usr/share/* $out/share
+          fi
+
+
+          for variant in "/usr/lib" "/usr/lib64" "/lib" "/lib64"; do
+            for file in $(find $TMP/unpack/$variant -type f -or -type l);do
+              cp -r $file $out/lib
+            done
+          done
+
+          mkdir -p $TMP/unpack/etc
+          cp -r $TMP/unpack/etc $out
+          rm -rf $TMP/unpack
+        done
+
+        runHook postBuild
+      '';
+      installPhase = ":";
+      # autoPatchelfIgnoreMissingDeps = true;
+    });
+  in {
+    packages.${defaultPackageName}.${defaultPackageVersion} = package;
+  };
+}

src/subsystems/debian/translators/debian-binary/aptdream/apt.patch

@@ -0,0 +1,13 @@
+diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc
+index c9c6a7e..0c7fb03 100644
+--- a/apt-pkg/deb/debsystem.cc
++++ b/apt-pkg/deb/debsystem.cc
+@@ -320,7 +320,7 @@ APT_PURE bool debSystem::ArchiveSupported(const char *Type)
+    System.. */
+ signed debSystem::Score(Configuration const &Cnf)
+ {
+-   signed Score = 0;
++   signed Score = 10;
+    if (FileExists(Cnf.FindFile("Dir::State::status",getDpkgStatusLocation(Cnf).c_str())) == true)
+        Score += 10;
+    if (FileExists(Cnf.Find("Dir::Bin::dpkg",BIN_DIR"/dpkg")) == true)

src/subsystems/debian/translators/debian-binary/aptdream/default.nix

@@ -0,0 +1,7 @@
+/*
+This apt is patched so it can run inside the nix build sandbox.
+By default apt refuses to execute if it does not detect a debian-like system.
+The patch is a one-liner bypassing that check.
+*/
+{pkgs, ...}:
+pkgs.apt.overrideDerivation (oldAttrs: {patches = [./apt.patch];})

src/subsystems/debian/translators/debian-binary/default.nix

@@ -0,0 +1,72 @@
+{
+  dlib,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+in {
+  type = "impure";
+
+  # A derivation which outputs a single executable at `$out`.
+  # The executable will be called by dream2nix for translation
+  # The input format is specified in /specifications/translator-call-example.json.
+  # The first arg `$1` will be a json file containing the input parameters
+  # like defined in /src/specifications/translator-call-example.json and the
+  # additional arguments required according to extraArgs
+  #
+  # The program is expected to create a file at the location specified
+  # by the input parameter `outFile`.
+  # The output file must contain the dream lock data encoded as json.
+  # See /src/specifications/dream-lock-example.json
+  translateBin = {
+    # dream2nix utils
+    utils,
+    # nixpkgs dependenies
+    bash,
+    coreutils,
+    jq,
+    writeScriptBin,
+    nix,
+    callPackage,
+    python3,
+    ...
+  }:
+    utils.writePureShellScript
+    [
+      bash
+      coreutils
+      jq
+      nix
+      (callPackage ./aptdream {})
+      python3
+    ]
+    ''
+      # accroding to the spec, the translator reads the input from a json file
+      jsonInput=$1
+
+      # read the json input
+      export outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInput))
+
+      pkgsName=$(jq '.project.name' -c -r $jsonInput)
+
+      cd $TMPDIR
+
+      mkdir ./state
+      touch ./status
+      mkdir ./download
+
+      mkdir -p ./etc/apt
+      echo "deb http://deb.debian.org/debian bullseye main" >> ./etc/apt/sources.list
+
+      export NAME=$pkgsName
+      python3 ${./generate_dream_lock.py}
+    '';
+
+  # If the translator requires additional arguments, specify them here.
+  # When users run the CLI, they will be asked to specify these arguments.
+  # There are only two types of arguments:
+  #   - string argument (type = "argument")
+  #   - boolean flag (type = "flag")
+  # String arguments contain a default value and examples. Flags do not.
+  extraArgs = {};
+}

src/subsystems/debian/translators/debian-binary/generate_dream_lock.py

@@ -0,0 +1,107 @@
+import base64
+import hashlib
+import json
+import os
+import pathlib
+import subprocess
+
+# for initialization
+def update_apt():
+    subprocess.run(
+        ["apt",
+        "-o", "Acquire::AllowInsecureRepositories=1",
+            "-o", "Dir::State::status=./status",
+            "-o", "Dir::Etc=./etc/apt",
+            "-o" ,"Dir::State=./state",
+            "update"
+        ])
+
+def get_package_info_apt(name):
+    result = subprocess.run(
+        ["apt",
+        "-o Acquire::AllowInsecureRepositories=1",
+            "-o", "Dir::State::status=./status",
+            "-o", "Dir::Etc=./etc/apt",
+            "-o" "Dir::State=./state",
+            "install", f"{name}", "--print-uris",
+        ],
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    print(f"result {result.stdout}")
+    with open('./deb-uris', 'w') as f:
+        f.write(result.stdout)
+
+    subprocess.run(
+        ["apt",
+        "-o", "Acquire::AllowInsecureRepositories=1",
+            "-o", "Dir::State::status=./status",
+            "-o", "Dir::Etc=./etc/apt",
+            "-o", "Dir::Cache=./download",
+            "-o", "Dir::State=./state",
+            "install", f"{name}", "--download-only", "-y" ,"--allow-unauthenticated",
+        ])
+
+def main():
+    update_apt()
+    get_package_info_apt(os.environ.get("NAME"))
+
+    default_package_src_name = f"{os.environ.get('NAME')}-binary"
+
+    with open("./deb-uris") as f:
+        uris = f.readlines()
+
+        dream_lock = dict(
+            sources={},
+            _generic={
+                "subsystem": "debian",
+                "defaultPackage": os.environ.get("NAME"),
+                "packages": {
+                    os.environ.get("NAME"): os.environ.get("VERSION"),
+                },
+                "sourcesAggregatedHash": None,
+                "location": "",
+            },
+            _subsystem={},
+        )
+
+        for line in uris:
+            # print(line)
+            split_lines = line.split(" ")
+            if len(split_lines) == 4:
+                (uri, deb, _, _) = split_lines
+                with open(f"./download/archives/{deb}", "rb") as f:
+                    bin = f.read()
+                    hash = hashlib.sha256(bin)
+                    digest = hash.digest()
+                    base = base64.b64encode(digest)
+                    decode = base.decode()
+                    sha256 = f"sha256-{decode}"
+                print(f"uri {uri}, deb: {deb}")
+                (name, version, _) = deb.split("_")
+
+                if name == os.environ.get("NAME"):
+                    name = default_package_src_name
+                dream_lock["sources"][name] = {
+                    version: dict(
+                        type="http",
+                        url=uri.replace("http:", "https:").replace("'", ""),
+                        hash=sha256,
+                    )
+                }
+
+    # add the version of the root package
+    dream_lock["_generic"]["packages"][os.environ.get("NAME")] = list(
+        dream_lock["sources"][default_package_src_name].keys()
+    )[0]
+
+    # dump dream lock to $outputFile
+    outputFile = (os.environ.get("outputFile"))
+    dirPath = pathlib.Path(os.path.dirname(outputFile))
+    dirPath.mkdir(parents=True, exist_ok=True)
+    with open(outputFile, "w") as lock:
+        json.dump(dream_lock, lock, indent=2)
+
+
+if __name__ == "__main__":
+    main()