Merge pull request #262 from tinybeachthor/php-translator-composer-json

php: impure translator `composer-json` + fix `composer-lock`

docs/src/SUMMARY.md

@@ -10,6 +10,7 @@
 - [Python](./subsystems/python.md)
 - [Node.js](./subsystems/node.md)
 - [Haskell](./subsystems/haskell.md)
+- [PHP](./subsystems/php.md)
 
 # Concepts
 - [Architectural Considerations](./intro/architectural-considerations.md)

docs/src/subsystems/php.md

@@ -0,0 +1,23 @@
+# PHP subsystem
+
+> !!! PHP support is a work in progress, and it is not yet usable (a
+> builder is missing). You can track the progress in
+> [nix-community/dream2nix#240](https://github.com/nix-community/dream2nix/issues/240).
+
+This section documents the PHP subsystem.
+
+## Translators
+
+### composer-lock (pure)
+
+Translates `composer.lock` into a dream2nix lockfile.
+
+### composer-json (impure)
+
+Resolves dependencies in `composer.json` using `composer` to generate a
+`composer.lock` lockfile, then invokes the `composer-lock` translator to
+generate a dream2nix lockfile.
+
+## Builders
+
+None so far.

src/subsystems/php/translators/composer-json/default.nix

@@ -0,0 +1,93 @@
+{
+  dlib,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+in {
+  type = "impure";
+
+  /*
+  Allow dream2nix to detect if a given directory contains a project
+  which can be translated with this translator.
+  Usually this can be done by checking for the existence of specific
+  file names or file endings.
+
+  Alternatively a fully featured discoverer can be implemented under
+  `src/subsystems/{subsystem}/discoverers`.
+  This is recommended if more complex project structures need to be
+  discovered like, for example, workspace projects spanning over multiple
+  sub-directories
+
+  If a fully featured discoverer exists, do not define `discoverProject`.
+  */
+  discoverProject = tree: (l.pathExists "${tree.fullPath}/composer.json");
+
+  # A derivation which outputs a single executable at `$out`.
+  # The executable will be called by dream2nix for translation
+  # The input format is specified in /specifications/translator-call-example.json.
+  # The first arg `$1` will be a json file containing the input parameters
+  # like defined in /src/specifications/translator-call-example.json and the
+  # additional arguments required according to extraArgs
+  #
+  # The program is expected to create a file at the location specified
+  # by the input parameter `outFile`.
+  # The output file must contain the dream lock data encoded as json.
+  # See /src/specifications/dream-lock-example.json
+  translateBin = {
+    # dream2nix utils
+    subsystems,
+    utils,
+    # nixpkgs dependenies
+    bash,
+    coreutils,
+    jq,
+    phpPackages,
+    writeScriptBin,
+    ...
+  }:
+    utils.writePureShellScript
+    [
+      bash
+      coreutils
+      jq
+      phpPackages.composer
+    ]
+    ''
+      # accroding to the spec, the translator reads the input from a json file
+      jsonInput=$1
+
+      # read the json input
+      outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInput))
+      source=$(jq '.source' -c -r $jsonInput)
+      relPath=$(jq '.project.relPath' -c -r $jsonInput)
+
+      pushd $TMPDIR
+      cp -r $source/* ./
+      chmod -R +w ./
+      newSource=$(pwd)
+
+      cd ./$relPath
+      rm -f composer.lock
+
+      echo "translating in temp dir: $(pwd)"
+
+      # create lockfile
+      if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
+        echo "excluding dev dependencies"
+        jq '.require-dev = {}' ./composer.json > composer.json.mod
+        mv composer.json.mod composer.json
+        composer update --no-install --no-dev
+      else
+        composer update --no-install
+      fi
+
+      jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
+
+      popd
+      ${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput
+    '';
+
+  # inherit options from composer-lock translator
+  extraArgs = dlib.translators.translators.php.composer-lock.extraArgs;
+}

src/subsystems/php/translators/composer-lock/default.nix

@@ -17,8 +17,8 @@ in {
   */
   generateUnitTestsForProjects = [
     (builtins.fetchTarball {
-      url = "https://code.castopod.org/adaures/castopod/-/archive/v1.0.0-alpha.80/castopod-v1.0.0-alpha.80.tar.gz";
-      sha256 = "sha256:0lv75pxzhs6q9w22czbgbnc48n6zhaajw9bag2sscaqnvfvfhcsf";
+      url = "https://github.com/tinybeachthor/dream2nix-php-composer-lock/archive/refs/tags/complex.tar.gz";
+      sha256 = "sha256:1xa5paafhwv4bcn2jsmbp1v2afh729r2h153g871zxdmsxsgwrn1";
     })
   ];
 
@@ -120,59 +120,113 @@ in {
     composerJson = (projectTree.getNodeFromPath "composer.json").jsonContent;
     composerLock = (projectTree.getNodeFromPath "composer.lock").jsonContent;
 
-    inherit (callPackageDream ../../utils.nix {}) satisfiesSemver;
+    inherit
+      (callPackageDream ../../utils.nix {})
+      satisfiesSemver
+      multiSatisfiesSemver
+      ;
 
-    # all the pinned packages
+    # all the packages
     packages =
       composerLock.packages
       ++ (
         if noDev
         then []
-        else composerLock."packages-dev"
+        else composerLock.packages-dev
       );
 
+    # packages with replacements applied
+    resolvedPackages = let
+      getProvide = pkg: (pkg.provide or {});
+      getReplace = pkg: let
+        resolveVersion = _: version:
+          if version == "self.version"
+          then pkg.version
+          else version;
+      in
+        l.mapAttrs resolveVersion (pkg.replace or {});
+      provide = pkg: dep: let
+        requirements = getDependencies pkg;
+        providements = getProvide dep;
+        cleanRequirements =
+          l.filterAttrs (
+            name: semver:
+              !((providements ? "${name}")
+                && (multiSatisfiesSemver providements."${name}" semver))
+          )
+          requirements;
+      in
+        pkg
+        // {
+          require =
+            cleanRequirements
+            // (
+              if requirements != cleanRequirements
+              then {"${dep.name}" = "${dep.version}";}
+              else {}
+            );
+        };
+      replace = pkg: dep: let
+        requirements = getDependencies pkg;
+        replacements = getReplace dep;
+        cleanRequirements =
+          l.filterAttrs (
+            name: semver:
+              !((replacements ? "${name}")
+                && (satisfiesSemver replacements."${name}" semver))
+          )
+          requirements;
+      in
+        pkg
+        // {
+          require =
+            cleanRequirements
+            // (
+              if requirements != cleanRequirements
+              then {"${dep.name}" = "${dep.version}";}
+              else {}
+            );
+        };
+      doReplace = pkg: l.foldl replace pkg packages;
+      doProvide = pkg: l.foldl provide pkg packages;
+      resolve = pkg: doProvide (doReplace pkg);
+    in
+      map resolve packages;
+
     # toplevel php semver
-    phpSemver = composerJson.require."php";
+    phpSemver = composerJson.require."php" or "*";
     # all the php extensions
     phpExtensions = let
-      all = map (pkg: l.attrsets.attrNames (getRequire pkg)) packages;
+      all = map (pkg: l.attrsets.attrNames (getDependencies pkg)) resolvedPackages;
       flat = l.lists.flatten all;
       extensions = l.filter (l.strings.hasPrefix "ext-") flat;
     in
-      l.lists.unique extensions;
+      map (l.strings.removePrefix "ext-") (l.lists.unique extensions);
 
-    # get require (and require-dev)
-    getRequire = pkg:
-      (
-        if noDev
-        then []
-        else (pkg."require-dev" or {})
-      )
-      // (pkg.require or {});
-    # strip php version & php extensions
-    cleanRequire = deps:
-      l.filterAttrs
-      (name: _: (name != "php") && !(l.strings.hasPrefix "ext-" name))
-      deps;
+    # get dependencies
+    getDependencies = pkg: (pkg.require or {});
 
     # resolve semvers into exact versions
-    pinRequires = dep: let
-      pin = name: semver:
+    pinPackages = pkgs: let
+      clean = requires:
+        l.filterAttrs
+        (name: _:
+          (l.all (x: name != x) ["php" "composer/composer" "composer-runtime-api"])
+          && !(l.strings.hasPrefix "ext-" name))
+        requires;
+      doPin = name: semver:
         (l.head
           (l.filter (dep: satisfiesSemver dep.version semver)
             (l.filter (dep: dep.name == name)
-              packages)))
+              resolvedPackages)))
         .version;
-      pinAttr = attr:
-        if attr ? dep
-        then l.mapAttrs pin dep."${attr}"
-        else {};
+      doPins = pkg:
+        pkg
+        // {
+          require = l.mapAttrs doPin (clean pkg.require);
+        };
     in
-      dep
-      // {
-        require = pinAttr "require";
-        "require-dev" = pinAttr "require-dev";
-      };
+      map doPins pkgs;
   in
     dlib.simpleTranslate2.translate
     ({objectsByKey, ...}: rec {
@@ -200,7 +254,7 @@ in {
       Users will not be interested in all individual dependencies.
       */
       exportedPackages = {
-        "${defaultPackage}" = composerJson.version;
+        "${defaultPackage}" = composerJson.version or "0.0.0";
       };
 
       /*
@@ -208,9 +262,8 @@ in {
       If the upstream format is a deep attrset, this list should contain
       a flattened representation of all entries.
       */
-      serializedRawObjects =
-        (map pinRequires composerLock.packages)
-        ++ [
+      serializedRawObjects = pinPackages (
+        [
           # Add the top-level package, this is not written in composer.lock
           {
             name = defaultPackage;
@@ -219,10 +272,17 @@ in {
               type = "path";
               path = projectSource;
             };
-            require = (pinRequires composerJson).require;
-            "require-dev" = (pinRequires composerJson)."require-dev";
+            require =
+              (
+                if noDev
+                then {}
+                else composerJson.require-dev
+              )
+              // composerJson.require;
           }
-        ];
+        ]
+        ++ resolvedPackages
+      );
 
       /*
       Define extractor functions which each extract one property from
@@ -242,7 +302,7 @@ in {
         dependencies = rawObj: finalObj:
           l.attrsets.mapAttrsToList
           (name: version: {inherit name version;})
-          (cleanRequire (getRequire rawObj));
+          (getDependencies rawObj);
 
         sourceSpec = rawObj: finalObj:
           if rawObj.source.type == "path"

src/subsystems/php/utils.nix

@@ -1,14 +1,73 @@
-{utils, ...}: {
+{
+  utils,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+
   # composer.lock uses a less strict semver interpretation
   # ~1.2 -> >=1.2 <2.0.0 (instead of >=1.2.0 <1.3.0)
+  # ~1   -> >=1.0 <2.0.0
   # this is identical with ^1.2 in the semver standard
-  satisfiesSemver = version: constraint: let
-    minorTilde = l.match "^[~]([[:d:]]+[.][[:d:]]+)$" constraint;
-    cleanConstraint =
-      if minorTilde != null && l.length minorTilde >= 0
-      then "^${l.head minorTilde}"
-      else constraint;
-    cleanVersion = l.removePrefix "v" version;
+  #
+  # remove v from version strings: ^v1.2.3 -> ^1.2.3
+  #
+  # remove branch suffix: ^1.2.x-dev -> ^1.2
+  #
+  satisfiesSemverSingle = version: constraint: let
+    removeSuffix = c: let
+      m = l.match "^(.*)[-][[:alpha:]]+$" c;
+    in
+      if m != null && l.length m >= 0
+      then l.head m
+      else c;
+    removeX = l.strings.removeSuffix ".x";
+    tilde = c: let
+      m = l.match "^[~]([[:d:]]+.*)$" c;
+    in
+      if m != null && l.length m >= 0
+      then "^${l.head m}"
+      else c;
+    wildcard = c: let
+      m = l.match "^([[:d:]]+.*)[.][*x]$" c;
+    in
+      if m != null && l.length m >= 0
+      then "^${l.head m}"
+      else c;
+    removeV = c: let
+      m = l.match "^(.)v([[:d:]]+[.].*)$" c;
+    in
+      if m != null && l.length m > 0
+      then l.concatStrings m
+      else c;
+    cleanConstraint = removeV (wildcard (tilde (removeSuffix constraint)));
+    cleanVersion = removeX (l.removePrefix "v" (removeSuffix version));
   in
-    utils.satisfiesSemver cleanVersion cleanConstraint;
+    (version == constraint)
+    || (
+      utils.satisfiesSemver
+      cleanVersion
+      cleanConstraint
+    );
+
+  splitAlternatives = v: let
+    # handle version alternatives: ^1.2 || ^2.0
+    trim = s: l.head (l.match "^[[:space:]]*([^[:space:]]*)[[:space:]]*$" s);
+    clean = l.replaceStrings ["||"] ["|"] v;
+  in
+    map trim (l.splitString "|" clean);
+in {
+  # 1.0.2 ~1.0.1
+  # matching a version with semver
+  satisfiesSemver = version: constraint:
+    l.any (satisfiesSemverSingle version) (splitAlternatives constraint);
+
+  # 1.0|2.0 ^2.0
+  # matching multiversion like the one in `provide` with semver
+  multiSatisfiesSemver = multiversion: constraint: let
+    satisfies = v: c: (v == "") || (v == "*") || (satisfiesSemverSingle v c);
+  in
+    l.any
+    (c: l.any (v: satisfies v c) (splitAlternatives multiversion))
+    (splitAlternatives constraint);
 }