Merge pull request #300 from tinybeachthor/php-indexer

php: indexer + fixes
This commit is contained in:
DavHau 2022-10-01 18:40:24 +02:00 committed by GitHub
commit ea3bb39310
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 386 additions and 89 deletions

View File

@ -26,6 +26,12 @@ Resolves dependencies in `composer.json` using `composer` to generate a
`composer.lock` lockfile, then invokes the `composer-lock` translator to
generate a dream2nix lockfile.
### packagist (impure)
Downloads a package version from [Packagist](https://packagist.org/).
Then uses `composer-lock` if a `composer.lock` file is present,
or `composer-json` otherwise to translate the package.
## Builders
### granular (pure) (default)

27
overrides/php/default.nix Normal file
View File

@ -0,0 +1,27 @@
{
lib,
pkgs,
...
}: let
l = lib // builtins;
in {
"rector/rector" = {
skip-composer-install = {
dontConfigure = true;
dontBuild = true;
};
};
"components/jquery" = {
fix-authors = {
postPatch = ''
jq \
".authors |= map(with_entries(
if .key == \"url\" \
then .key = \"homepage\" \
else . end \
))" \
composer.json | sponge composer.json
'';
};
};
}

View File

@ -20,6 +20,7 @@ in {
}: {
url,
rev,
submodules ? true,
...
} @ inp: let
isRevGitRef = isGitRef rev;
@ -58,7 +59,7 @@ in {
inherit url;
# disable fetching all refs if the source specifies a ref
allRefs = ! hasGitRef;
submodules = true;
inherit submodules;
}));
# git can either be verified via revision or hash.
@ -74,14 +75,14 @@ in {
// {
inherit url;
allRefs = true;
submodules = true;
inherit submodules;
})
else
fetchgit
(refAndRev
// {
inherit url;
fetchSubmodules = true;
fetchSubmodules = submodules;
sha256 = hash;
});
};

View File

@ -29,6 +29,7 @@ libraries.io also supports other interesting popularity metrics:
npm = "npm";
crates-io = "cargo";
pypi = "pypi";
packagist = "packagist";
};
in
utils.writePureShellScript

View File

@ -52,6 +52,7 @@
"url": { "type": "string" },
"rev": { "type": "string" },
"ref": { "type": "string" },
"submodules": { "type": "boolean" },
"hash": { "type": "string" },
"type": { "type": "string" },
"dir": { "type": "string" }

View File

@ -4,7 +4,7 @@
build = {
lib,
pkgs,
stdenv,
stdenvNoCC,
# dream2nix inputs
externals,
callPackageDream,
@ -89,6 +89,8 @@
# Generates a derivation for a specific package name + version
makeOnePackage = name: version: let
isTopLevel = packages ? name && packages.name == version;
dependencies = getDependencies name version;
repositories = let
transform = dep: let
@ -97,7 +99,7 @@
url = "${root}/vendor/${name}";
options = {
versions = {
"${name}" = "${version}";
"${l.strings.toLower name}" = "${version}";
};
symlink = false;
};
@ -124,10 +126,15 @@
(repositories ++ [{packagist = false;}]);
dependenciesString = l.toJSON (l.listToAttrs (
map (dep: {
inherit (dep) name;
name = l.strings.toLower dep.name;
value = dep.version;
})
dependencies
(dependencies
++ l.optional (subsystemAttrs.composerPluginApiSemver ? "${name}@${version}")
{
name = "composer-plugin-api";
version = subsystemAttrs.composerPluginApiSemver."${name}@${version}";
})
));
versionString =
@ -135,15 +142,15 @@
then "0.0.0"
else version;
pkg = stdenv.mkDerivation rec {
pkg = stdenvNoCC.mkDerivation rec {
pname = l.strings.sanitizeDerivationName name;
inherit version;
src = getSource name version;
nativeBuildInputs = with pkgs; [
jq
composer
moreutils
];
buildInputs = with pkgs;
[
@ -153,59 +160,91 @@
++ map (dep: allPackages."${dep.name}"."${dep.version}")
dependencies;
dontConfigure = true;
buildPhase = ''
inherit repositoriesString dependenciesString;
passAsFile = ["repositoriesString" "dependenciesString"];
unpackPhase = ''
runHook preUnpack
mkdir -p $out/lib/vendor/${name}
cd $out/lib/vendor/${name}
# copy source
PKG_OUT=$out/lib/vendor/${name}
mkdir -p $PKG_OUT
pushd $PKG_OUT
cp -r ${src}/* .
chmod -R +w .
# create composer.json if does not exist
if [ ! -f composer.json ]; then
echo "{}" > composer.json
fi
# save the original composer.json for reference
cp composer.json composer.json.orig
# set name & version
jq \
"(.name = \"${name}\") | \
(.version = \"${versionString}\")" \
composer.json | sponge composer.json
runHook postUnpack
'';
patchPhase = ''
runHook prePatch
# fixup composer.json
jq \
"(.extra.patches = {})" \
composer.json | sponge composer.json
runHook postPatch
'';
configurePhase = ''
runHook preConfigure
# disable packagist, set path repositories
jq \
--slurpfile repositories $repositoriesStringPath \
--slurpfile dependencies $dependenciesStringPath \
"(.repositories = \$repositories[0]) | \
(.require = \$dependencies[0]) | \
(.\"require-dev\" = {})" \
composer.json | sponge composer.json
runHook postConfigure
'';
composerInstallFlags =
[
"--no-scripts"
"--no-plugins"
]
++ l.optional (subsystemAttrs.noDev || !isTopLevel) "--no-dev";
buildPhase = ''
runHook preBuild
# remove composer.lock if exists
rm -f composer.lock
# disable packagist, set path repositories
mv composer.json composer.json.orig
cat <<EOF >> $out/repositories.json
${repositoriesString}
EOF
cat <<EOF >> $out/dependencies.json
${dependenciesString}
EOF
jq \
--slurpfile repositories $out/repositories.json \
--slurpfile dependencies $out/dependencies.json \
"(.repositories = \$repositories[0]) | \
(.require = \$dependencies[0]) | \
(.\"require-dev\" = {}) | \
(.version = \"${versionString}\")" \
composer.json.orig > composer.json
# build
composer install --no-scripts
composer install ${l.strings.concatStringsSep " " composerInstallFlags}
runHook postBuild
rm -rfv vendor/*/*/vendor
# cleanup
popd
rm $out/repositories.json
rm $out/dependencies.json
'';
installPhase = ''
pushd $PKG_OUT
runHook preInstall
BINS=$(jq -rcM "(.bin // [])[]" composer.json)
for bin in $BINS
do
mkdir -p $out/bin
pushd $out/bin
ln -s $PKG_OUT/$bin
ln -s $out/lib/vendor/${name}/$bin
popd
done
popd
runHook postInstall
'';
passthru.devShell = import ./devShell.nix {

View File

@ -4,7 +4,7 @@
build = {
lib,
pkgs,
stdenv,
stdenvNoCC,
# dream2nix inputs
externals,
callPackageDream,
@ -76,6 +76,31 @@
args.packages
);
# Unpack dependency
cleanDependency = name: version:
stdenvNoCC.mkDerivation rec {
pname = "${l.strings.sanitizeDerivationName name}-source";
inherit version;
versionString =
if version == "unknown"
then "0.0.0"
else version;
src = getSource name version;
dontConfigure = true;
buildPhase = ''
mkdir $out
cp -r ${src} $out
if [ ! -f $out/composer.json ]
then
echo \
{\"name\":\"${name}\",\"version\":\"${versionString}\"} \
> $out/composer.json
fi
'';
dontInstall = true;
dontFixup = true;
};
# Generates a derivation for a specific package name + version
makePackage = name: version: let
dependencies = getDependencies name version;
@ -89,10 +114,10 @@
intoRepository = dep: {
type = "path";
url = "${getSource dep.name dep.version}";
url = "${cleanDependency dep.name dep.version}";
options = {
versions = {
"${dep.name}" = "${dep.version}";
"${l.strings.toLower dep.name}" = "${dep.version}";
};
symlink = false;
};
@ -102,70 +127,117 @@
l.toJSON
(repositories ++ [{packagist = false;}]);
dependenciesString = l.toJSON (l.listToAttrs (
map (dep: {
name = l.strings.toLower dep.name;
value = dep.version;
})
dependencies
));
versionString =
if version == "unknown"
then "0.0.0"
else version;
pkg = stdenv.mkDerivation rec {
pkg = stdenvNoCC.mkDerivation rec {
pname = l.strings.sanitizeDerivationName name;
inherit version;
src = getSource name version;
nativeBuildInputs = with pkgs; [
jq
composer
moreutils
];
buildInputs = with pkgs; [
php
composer
];
dontConfigure = true;
buildPhase = ''
inherit repositoriesString dependenciesString;
passAsFile = ["repositoriesString" "dependenciesString"];
unpackPhase = ''
runHook preUnpack
mkdir -p $out/lib/vendor/${name}
cd $out/lib/vendor/${name}
# copy source
PKG_OUT=$out/lib/vendor/${name}
mkdir -p $PKG_OUT
pushd $PKG_OUT
cp -r ${src}/* .
chmod -R +w .
# create composer.json if does not exist
if [ ! -f composer.json ]; then
echo "{}" > composer.json
fi
# save the original composer.json for reference
cp composer.json composer.json.orig
# set name & version
jq \
"(.name = \"${name}\") | \
(.version = \"${versionString}\")" \
composer.json | sponge composer.json
runHook postUnpack
'';
patchPhase = ''
runHook prePatch
# fixup composer.json
jq \
"(.extra.patches = {})" \
composer.json | sponge composer.json
runHook postPatch
'';
configurePhase = ''
runHook preConfigure
# disable packagist, set path repositories
jq \
--slurpfile repositories $repositoriesStringPath \
--slurpfile dependencies $dependenciesStringPath \
"(.repositories = \$repositories[0]) | \
(.require = \$dependencies[0]) | \
(.\"require-dev\" = {})" \
composer.json | sponge composer.json
runHook postConfigure
'';
composerInstallFlags =
[
"--no-scripts"
"--no-plugins"
]
++ l.optional (subsystemAttrs.noDev) "--no-dev";
buildPhase = ''
runHook preBuild
# remove composer.lock if exists
rm -f composer.lock
# disable packagist, set path repositories
mv composer.json composer.json.orig
cat <<EOF >> $out/repositories.json
${repositoriesString}
EOF
jq \
--slurpfile repositories $out/repositories.json \
"(.repositories = \$repositories[0]) | \
(.version = \"${versionString}\")" \
composer.json.orig > composer.json
# build
composer install --no-scripts
composer install ${l.strings.concatStringsSep " " composerInstallFlags}
# cleanup
rm $out/repositories.json
popd
runHook postBuild
'';
installPhase = ''
pushd $PKG_OUT
runHook preInstall
BINS=$(jq -rcM "(.bin // [])[]" composer.json)
for bin in $BINS
do
mkdir -p $out/bin
pushd $out/bin
ln -s $PKG_OUT/$bin
ln -s $out/lib/vendor/${name}/$bin
popd
done
popd
runHook postInstall
'';
passthru.devShell = import ./devShell.nix {

View File

@ -53,7 +53,7 @@
re = {
operators = "([=><!~^]+)";
version = "((0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)|(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)|(0|[1-9][0-9]*)){0,1}([.x*]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*)){0,1}(\\+([0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*)){0,1}";
version = "((0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)|(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)|(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)|(0|[1-9][0-9]*))?(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\\+([0-9a-zA-Z-]+(\\.[0-9a-zA-Z-]+)*))?";
};
reLengths = {
@ -120,6 +120,12 @@
# remove v from version strings: ^v1.2.3 -> ^1.2.3
# remove branch suffix: ^1.2.x-dev -> ^1.2
satisfiesSingle = version: constraint: let
removeStability = c: let
m = l.match "^(.*)[@][[:alpha:]]+$" c;
in
if m != null && l.length m >= 0
then l.head m
else c;
removeSuffix = c: let
m = l.match "^(.*)[-][[:alpha:]]+$" c;
in
@ -130,7 +136,7 @@
m = l.match "^([[:d:]]+.*)[.][*x]$" c;
in
if m != null && l.length m >= 0
then "^${l.head m}"
then "~${l.head m}.0"
else c;
removeV = c: let
m = l.match "^(.)*v([[:d:]]+[.].*)$" c;
@ -142,10 +148,10 @@
m = l.match "^([0-9><=!-^~*]*)$" c;
in
m != null && l.length m > 0;
cleanConstraint = removeV (wildcard (removeSuffix (l.removePrefix "dev-" constraint)));
cleanConstraint = removeV (wildcard (removeSuffix (removeStability (l.removePrefix "dev-" constraint))));
cleanVersion = l.removePrefix "v" (wildcard (removeSuffix version));
in
(l.elem constraint ["" "*" "@dev" "@master" "@dev-master"])
(l.elem (removeStability constraint) ["" "*"])
|| (version == constraint)
|| ((isVersionLike cleanConstraint) && (satisfiesSingleInternal cleanVersion cleanConstraint));
@ -168,7 +174,9 @@
then l.head m
else v;
in
map (x: trim (cleanInlineAlias x)) (l.splitString " " clean);
map
(x: trim (cleanInlineAlias x))
(l.filter (x: x != "") (l.splitString " " clean));
in rec {
# matching a version with semver
# 1.0.2 (~1.0.1 || >=2.1 <2.4)

View File

@ -23,6 +23,7 @@
# nixpkgs dependenies
bash,
coreutils,
moreutils,
jq,
phpPackages,
...
@ -31,6 +32,7 @@
[
bash
coreutils
moreutils
jq
phpPackages.composer
]
@ -54,17 +56,25 @@
echo "translating in temp dir: $(pwd)"
# create lockfile
mv composer.json composer.json.orig
jq \
"(.config.lock = true) | \
(.config.\"platform-check\" = false) | \
(.authors = []) | \
(.require = ((.require // {}) | with_entries(.key |= ascii_downcase))) | \
(.\"require-dev\" = ((.\"require-dev\" // {}) | with_entries(.key |= ascii_downcase)))" \
composer.json.orig > composer.json
if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
echo "excluding dev dependencies"
jq '.require-dev = {}' ./composer.json > composer.json.mod
mv composer.json.mod composer.json
composer update --no-install --no-dev
else
composer update --no-install
jq \
'.require-dev = {}' \
composer.json | sponge composer.json
fi
composer update --ignore-platform-reqs --no-scripts --no-plugins --no-install
jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
popd
${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput
'';

View File

@ -117,6 +117,19 @@ in {
in
map (l.removePrefix "ext-") extensions;
composerPluginApiSemver = l.listToAttrs (l.flatten (map
(
pkg: let
requires = getRequire pkg;
in
l.optional (requires ? "composer-plugin-api")
{
name = "${pkg.name}@${pkg.version}";
value = requires."composer-plugin-api";
}
)
packages));
# get cleaned pkg attributes
getRequire = pkg:
l.mapAttrs
@ -142,11 +155,11 @@ in {
version = composerJson.version or "unknown";
source = {
type = "path";
path = projectSource;
path = rootSource;
};
require =
(l.optionalAttrs (!noDev) (composerJson.require-dev or {}))
// composerJson.require;
// (composerJson.require or {});
};
# all the packages
packages =
@ -204,7 +217,7 @@ in {
clean = requires:
l.filterAttrs
(name: _:
!(l.elem name ["php" "composer/composer" "composer-runtime-api"])
!(l.elem name ["php" "composer-plugin-api" "composer-runtime-api"])
&& !(l.strings.hasPrefix "ext-" name))
requires;
doPin = name: semver:
@ -235,7 +248,9 @@ in {
# The structure of this should be defined in:
# ./src/specifications/{subsystem}
subsystemAttrs = {
inherit noDev;
inherit phpSemver phpExtensions;
inherit composerPluginApiSemver;
};
# name of the default package
@ -278,14 +293,30 @@ in {
(getRequire rawObj);
sourceSpec = rawObj: finalObj:
if rawObj.source.type == "path"
if rawObj ? "source" && rawObj.source.type == "path"
then {
inherit (rawObj.source) type path;
rootName = finalObj.name;
rootVersion = finalObj.version;
}
else {
else if rawObj ? "source" && rawObj.source.type == "git"
then {
inherit (rawObj.source) type url;
rev = rawObj.source.reference;
};
submodules = false;
}
else if rawObj ? "dist" && rawObj.dist.type == "path"
then {
inherit (rawObj.dist) type;
path = rawObj.dist.url;
rootName = finalObj.name;
rootVersion = finalObj.version;
}
else
l.abort ''
Cannot find source for ${finalObj.name}@${finalObj.version},
rawObj: ${l.toJSON rawObj}
'';
};
/*

View File

@ -0,0 +1,101 @@
{
dlib,
lib,
...
}: {
type = "impure";
# the input format is specified in /specifications/translator-call-example.json
# this script receives a json file including the input paths and specialArgs
translateBin = {
# dream2nix utils
apps,
subsystems,
utils,
# nixpkgs dependenies
coreutils,
curl,
jq,
git,
moreutils,
...
}:
utils.writePureShellScript
[
coreutils
curl
jq
git
moreutils
]
''
# according to the spec, the translator reads the input from a json file
jsonInput=$1
# read the json input
outputFile=$(realpath -m $(jq '.outputFile' -c -r $jsonInput))
name=$(jq '.project.name' -c -r $jsonInput)
version=$(jq '.project.version' -c -r $jsonInput)
pushd $TMPDIR
# download and unpack package source
curl https://packagist.org/packages/$name.json | \
jq -rcM ".package.versions.\"$version\".source" \
> source_manifest
SOURCE_URL=$(jq -rcM ".url" source_manifest)
SOURCE_REV=$(jq -rcM ".reference" source_manifest)
mkdir source
pushd source
git init
git remote add origin $SOURCE_URL
git fetch --depth 1 origin $SOURCE_REV
git checkout FETCH_HEAD
mv composer.json composer.json.orig
jq ".version = \"$version\"" composer.json.orig > composer.json
popd
# generate arguments for package-lock translator
echo "{
\"source\": \"$TMPDIR/source\",
\"outputFile\": \"$outputFile\",
\"project\": {
\"name\": \"$name\",
\"relPath\": \"\"
}
}" > $TMPDIR/newJsonInput
popd
if [ -f $TMPDIR/source/composer.lock ]
then
echo 'Translating with composer-lock'
${subsystems.php.translators.composer-lock.translateBin} $TMPDIR/newJsonInput
else
echo 'Translating with composer-json'
${subsystems.php.translators.composer-json.translateBin} $TMPDIR/newJsonInput
fi
# add main package source info to dream-lock.json
echo "
{
\"type\": \"git\",
\"url\": \"$SOURCE_URL\",
\"rev\": \"$SOURCE_REV\"
}
" > $TMPDIR/sourceInfo.json
${apps.callNixWithD2N} eval --json "
with dream2nix.utils.dreamLock;
replaceRootSources {
dreamLock = l.fromJSON (l.readFile \"$outputFile\");
newSourceRoot = l.fromJSON (l.readFile \"$TMPDIR/sourceInfo.json\");
}
" \
| sponge "$outputFile"
'';
# inherit options from composer-json translator
extraArgs = dlib.translators.translators.php.composer-json.extraArgs;
}