Merge pull request #172838 from adisbladis/poetry2nix-1_29_0

poetry2nix: 1.28.0 -> 1.29.0
This commit is contained in:
adisbladis 2022-05-13 18:20:12 +08:00 committed by GitHub
commit 7c1068b08c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 236 additions and 163 deletions

View File

@ -5,7 +5,7 @@
}:
let
# Poetry2nix version
version = "1.28.0";
version = "1.29.0";
inherit (poetryLib) isCompatible readTOML moduleName;

View File

@ -3,48 +3,46 @@
, makeSetupHook
, wheel
, pip
, pkgs
}:
let
callPackage = python.pythonForBuild.pkgs.callPackage;
pythonInterpreter = python.pythonForBuild.interpreter;
pythonSitePackages = python.sitePackages;
nonOverlayedPython = pkgs.python3.pythonForBuild.withPackages (ps: [ ps.tomlkit ]);
makeRemoveSpecialDependenciesHook = { fields, kind }:
nonOverlayedPython.pkgs.callPackage
(
{}:
makeSetupHook
{
name = "remove-path-dependencies.sh";
deps = [ ];
substitutions = {
# NOTE: We have to use a non-overlayed Python here because otherwise we run into an infinite recursion
# because building of tomlkit and its dependencies also use these hooks.
pythonPath = nonOverlayedPython.pkgs.makePythonPath [ nonOverlayedPython ];
pythonInterpreter = nonOverlayedPython.interpreter;
pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
fields = fields;
kind = kind;
};
} ./remove-special-dependencies.sh
)
{ };
in
{
removePathDependenciesHook = makeRemoveSpecialDependenciesHook {
fields = [ "path" ];
kind = "path";
};
removePathDependenciesHook = callPackage
(
{}:
makeSetupHook
{
name = "remove-path-dependencies.sh";
deps = [ ];
substitutions = {
inherit pythonInterpreter;
yj = "${buildPackages.yj}/bin/yj";
pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
fields = [ "path" ];
kind = "path";
};
} ./remove-special-dependencies.sh
)
{ };
removeGitDependenciesHook = makeRemoveSpecialDependenciesHook {
fields = [ "git" "branch" "rev" "tag" ];
kind = "git";
};
removeGitDependenciesHook = callPackage
({}:
makeSetupHook
{
name = "remove-git-dependencies.sh";
deps = [ ];
substitutions = {
inherit pythonInterpreter;
yj = "${buildPackages.yj}/bin/yj";
pyprojectPatchScript = "${./pyproject-without-special-deps.py}";
fields = [ "git" "branch" "rev" "tag" ];
kind = "git";
};
} ./remove-special-dependencies.sh
)
{ };
pipBuildHook = callPackage
(
@ -89,6 +87,4 @@ in
} ./wheel-unpack-hook.sh
)
{ };
}

View File

@ -1,13 +1,14 @@
#!/usr/bin/env python
# Patch out special dependencies (git and path) from a pyproject.json file
# Patch out special dependencies (git and path) from a pyproject.toml file
import argparse
import json
import sys
import tomlkit
def main(input, output, fields_to_remove):
data = json.load(input)
data = tomlkit.loads(input.read())
try:
deps = data["tool"]["poetry"]["dependencies"]
@ -22,12 +23,7 @@ def main(input, output, fields_to_remove):
if any_removed:
dep["version"] = "*"
# Set ensure_ascii to False because TOML is valid UTF-8 so text that can't
# be represented in ASCII is perfectly legitimate
# HACK: Setting ensure_asscii to False breaks Python2 for some dependencies (like cachy==0.3.0)
json.dump(
data, output, separators=(",", ":"), ensure_ascii=sys.version_info.major < 3
)
output.write(tomlkit.dumps(data))
if __name__ == "__main__":
@ -37,20 +33,20 @@ if __name__ == "__main__":
"--input",
type=argparse.FileType("r"),
default=sys.stdin,
help="Location from which to read input JSON",
help="Location from which to read input TOML",
)
p.add_argument(
"-o",
"--output",
type=argparse.FileType("w"),
default=sys.stdout,
help="Location to write output JSON",
help="Location to write output TOML",
)
p.add_argument(
"-f",
"--fields-to-remove",
nargs="+",
help="The fields to remove from the dependency's JSON",
help="The fields to remove from the dependency's TOML",
)
args = p.parse_args()

View File

@ -1,18 +1,21 @@
remove-@kind@-dependencies-hook() {
# Tell poetry not to resolve special dependencies. Any version is fine!
if ! test -f pyproject.toml; then
return
fi
echo "Removing @kind@ dependencies"
# Tell poetry not to resolve special dependencies. Any version is fine!
@yj@ -tj < pyproject.toml | \
@pythonInterpreter@ \
@pyprojectPatchScript@ \
--fields-to-remove @fields@ > pyproject.json
@yj@ -jt < pyproject.json > pyproject.toml
# NOTE: We have to reset PYTHONPATH to avoid having propagatedBuildInputs
# from the currently building derivation leaking into our unrelated Python
# environment.
PYTHONPATH=@pythonPath@ \
@pythonInterpreter@ \
@pyprojectPatchScript@ \
--fields-to-remove @fields@ < pyproject.toml > pyproject.formatted.toml
rm pyproject.json
mv pyproject.formatted.toml pyproject.toml
echo "Finished removing @kind@ dependencies"
}

View File

@ -50,6 +50,9 @@
"aiokafka": [
"cython"
],
"aiolimiter": [
"poetry-core"
],
"aiomultiprocess": [
"flit-core"
],
@ -229,6 +232,9 @@
"flit-core",
"flitBuildHook"
],
"copier": [
"poetry-core"
],
"cpyparsing": [
"cython"
],
@ -239,6 +245,9 @@
"cython",
"poetry-core"
],
"cruft": [
"poetry-core"
],
"cssselect2": [
"flit",
"flit-core"
@ -350,6 +359,9 @@
"enumatch": [
"poetry"
],
"envs": [
"poetry-core"
],
"exceptiongroup": [
"flit-core",
"flitBuildHook"
@ -366,6 +378,9 @@
"fastavro": [
"cython"
],
"fastbencode": [
"cython"
],
"fastdtw": [
"cython"
],
@ -429,6 +444,9 @@
"gensim": [
"cython"
],
"gidgethub": [
"flitBuildHook"
],
"glances-api": [
"poetry-core"
],
@ -480,6 +498,9 @@
"hashids": [
"flit-core"
],
"hatch-vcs": [
"hatchling"
],
"hdate": [
"poetry-core"
],
@ -711,6 +732,9 @@
"monosat": [
"cython"
],
"more-itertools": [
"flit-core"
],
"motioneye-client": [
"poetry-core"
],
@ -720,6 +744,9 @@
"msgpack-numpy": [
"cython"
],
"msgpack-types": [
"poetry"
],
"msoffcrypto-tool": [
"poetry-core"
],
@ -735,6 +762,9 @@
"mypy-boto3-builder": [
"poetry-core"
],
"myst-parser": [
"flit-core"
],
"nats-python": [
"poetry-core"
],
@ -807,6 +837,9 @@
"openstacksdk": [
"pbr"
],
"openstep-plist": [
"cython"
],
"openvino": [
"cython"
],
@ -889,6 +922,9 @@
"pixelmatch": [
"poetry-core"
],
"pkgconfig": [
"poetry-core"
],
"poetry": [
"poetry-core"
],
@ -925,6 +961,12 @@
"prometheus-fastapi-instrumentator": [
"poetry"
],
"prospector": [
"poetry-core"
],
"protoletariat": [
"poetry-core"
],
"ptyprocess": [
"flit-core"
],
@ -1234,6 +1276,9 @@
"python-swiftclient": [
"pbr"
],
"python-trovo": [
"poetry"
],
"python_openzwave": [
"cython"
],
@ -1276,6 +1321,9 @@
"qiskit-terra": [
"cython"
],
"qstylizer": [
"pbr"
],
"questionary": [
"poetry-core"
],
@ -1330,6 +1378,9 @@
"rich": [
"poetry-core"
],
"rio-tiler": [
"flit-core"
],
"ripser": [
"cython"
],
@ -1367,6 +1418,9 @@
"sarif-om": [
"pbr"
],
"sat-search": [
"pytest-runner"
],
"scikit-bio": [
"cython"
],
@ -1430,6 +1484,9 @@
"single-version": [
"poetry-core"
],
"skia-pathops": [
"cython"
],
"slowapi": [
"poetry-core"
],
@ -1602,6 +1659,9 @@
"toggl-cli": [
"pbr"
],
"toml-cli": [
"poetry"
],
"toml-sort": [
"poetry"
],
@ -1620,6 +1680,9 @@
"traceback2": [
"pbr"
],
"traitlets": [
"flit-core"
],
"transmission-rpc": [
"poetry-core"
],

View File

@ -74,6 +74,12 @@ lib.composeManyExtensions [
inherit (pkgs) buildPackages;
pyBuildPackages = self.python.pythonForBuild.pkgs;
selectQt5 = version:
let
selector = builtins.concatStringsSep "" (lib.take 2 (builtins.splitVersion version));
in
pkgs."qt${selector}" or pkgs.qt5;
in
{
@ -287,6 +293,7 @@ lib.composeManyExtensions [
"36.0.0" = "sha256-Y6TuW7AryVgSvZ6G8WNoDIvi+0tvx8ZlEYF5qB0jfNk=";
"36.0.1" = "sha256-kozYXkqt1Wpqyo9GYCwN08J+zV92ZWFJY/f+rulxmeQ=";
"36.0.2" = "1a0ni1a3dbv2dvh6gx2i54z8v5j9m6asqg97kkv7gqb1ivihsbp8";
"37.0.2" = "sha256-qvrxvneoBXjP96AnUPyrtfmCnZo+IriHR5HbtWQ5Gk8=";
}.${version} or null;
sha256 = getCargoHash super.cryptography.version;
scrypto =
@ -517,6 +524,14 @@ lib.composeManyExtensions [
}
);
file-magic = super.file-magic.overridePythonAttrs (
old: {
postPatch = ''
substituteInPlace magic.py --replace "find_library('magic')" "'${pkgs.file}/lib/libmagic${pkgs.stdenv.hostPlatform.extensions.sharedLibrary}'"
'';
}
);
fiona = super.fiona.overridePythonAttrs (
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.gdal_2 ];
@ -735,6 +750,14 @@ lib.composeManyExtensions [
}
);
jinja2-ansible-filters = super.jinja2-ansible-filters.overridePythonAttrs (
old: {
preBuild = (old.preBuild or "") + ''
echo "${old.version}" > VERSION
'';
}
);
jira = super.jira.overridePythonAttrs (
old: {
inherit (pkgs.python3Packages.jira) patches;
@ -1056,6 +1079,18 @@ lib.composeManyExtensions [
mypy = super.mypy.overridePythonAttrs (
old: {
buildInputs = (old.buildInputs or [ ]) ++ [
self.types-typed-ast
];
# Compile mypy with mypyc, which makes mypy about 4 times faster. The compiled
# version is also the default in the wheels on Pypi that include binaries.
# is64bit: unfortunately the build would exhaust all possible memory on i686-linux.
MYPY_USE_MYPYC = stdenv.buildPlatform.is64bit;
# when testing reduce optimisation level to drastically reduce build time
# (default is 3)
# MYPYC_OPT_LEVEL = 1;
} // lib.optionalAttrs (old.format != "wheel") {
# FIXME: Remove patch after upstream has decided the proper solution.
# https://github.com/python/mypy/pull/11143
patches = (old.patches or [ ]) ++ lib.optionals ((lib.strings.versionAtLeast old.version "0.900") && lib.strings.versionOlder old.version "0.940") [
@ -1069,17 +1104,6 @@ lib.composeManyExtensions [
sha256 = "sha256-waIZ+m3tfvYE4HJ8kL6rN/C4fMjvLEe9UoPbt9mHWIM=";
})
];
buildInputs = (old.buildInputs or [ ]) ++ [
self.types-typed-ast
];
# Compile mypy with mypyc, which makes mypy about 4 times faster. The compiled
# version is also the default in the wheels on Pypi that include binaries.
# is64bit: unfortunately the build would exhaust all possible memory on i686-linux.
MYPY_USE_MYPYC = stdenv.buildPlatform.is64bit;
# when testing reduce optimisation level to drastically reduce build time
# (default is 3)
# MYPYC_OPT_LEVEL = 1;
}
);
@ -1566,106 +1590,35 @@ lib.composeManyExtensions [
pyqt5 =
let
drv = super.pyqt5;
withConnectivity = drv.passthru.args.withConnectivity or false;
withMultimedia = drv.passthru.args.withMultimedia or false;
withWebKit = drv.passthru.args.withWebKit or false;
withWebSockets = drv.passthru.args.withWebSockets or false;
qt5 = selectQt5 super.pyqt5.version;
in
super.pyqt5.overridePythonAttrs (
old: {
format = "other";
dontConfigure = true;
dontWrapQtApps = true;
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
pkg-config
pkgs.qt5.qmake
pkgs.xorg.lndir
pkgs.qt5.qtbase
pkgs.qt5.qtsvg
pkgs.qt5.qtdeclarative
pkgs.qt5.qtwebchannel
pkgs.qt5.qt3d
# self.pyqt5-sip
nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [
self.pyqt-builder
self.sip
]
++ lib.optional withConnectivity pkgs.qt5.qtconnectivity
++ lib.optional withMultimedia pkgs.qt5.qtmultimedia
++ lib.optional withWebKit pkgs.qt5.qtwebkit
++ lib.optional withWebSockets pkgs.qt5.qtwebsockets
;
qt5.full
];
}
);
buildInputs = (old.buildInputs or [ ]) ++ [
pkgs.dbus
pkgs.qt5.qtbase
pkgs.qt5.qtsvg
pkgs.qt5.qtdeclarative
self.sip
]
++ lib.optional withConnectivity pkgs.qt5.qtconnectivity
++ lib.optional withWebKit pkgs.qt5.qtwebkit
++ lib.optional withWebSockets pkgs.qt5.qtwebsockets
;
# Fix dbus mainloop
patches = pkgs.python3.pkgs.pyqt5.patches or [ ];
configurePhase = ''
runHook preConfigure
export PYTHONPATH=$PYTHONPATH:$out/${self.python.sitePackages}
mkdir -p $out/${self.python.sitePackages}/dbus/mainloop
${self.python.executable} configure.py -w \
--confirm-license \
--no-qml-plugin \
--bindir=$out/bin \
--destdir=$out/${self.python.sitePackages} \
--stubsdir=$out/${self.python.sitePackages}/PyQt5 \
--sipdir=$out/share/sip/PyQt5 \
--designer-plugindir=$out/plugins/designer
runHook postConfigure
'';
postInstall = ''
ln -s ${self.pyqt5-sip}/${self.python.sitePackages}/PyQt5/sip.* $out/${self.python.sitePackages}/PyQt5/
for i in $out/bin/*; do
wrapProgram $i --prefix PYTHONPATH : "$PYTHONPATH"
done
# Let's make it a namespace package
cat << EOF > $out/${self.python.sitePackages}/PyQt5/__init__.py
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
EOF
'';
installCheckPhase =
let
modules = [
"PyQt5"
"PyQt5.QtCore"
"PyQt5.QtQml"
"PyQt5.QtWidgets"
"PyQt5.QtGui"
]
++ lib.optional withWebSockets "PyQt5.QtWebSockets"
++ lib.optional withWebKit "PyQt5.QtWebKit"
++ lib.optional withMultimedia "PyQt5.QtMultimedia"
++ lib.optional withConnectivity "PyQt5.QtConnectivity"
;
imports = lib.concatMapStrings (module: "import ${module};") modules;
in
''
echo "Checking whether modules can be imported..."
${self.python.interpreter} -c "${imports}"
'';
doCheck = true;
enableParallelBuilding = true;
pyqt5-qt5 =
let
qt5 = selectQt5 super.pyqt5-qt5.version;
in
super.pyqt5-qt5.overridePythonAttrs (
old: {
dontWrapQtApps = true;
propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [
qt5.full
qt5.qtgamepad # As of 2022-05-13 not a port of qt5.full
pkgs.gtk3
pkgs.speechd
pkgs.postgresql
pkgs.unixODBC
];
}
);
@ -1681,7 +1634,10 @@ lib.composeManyExtensions [
old: {
# Fixes https://github.com/pytest-dev/pytest/issues/7891
postPatch = old.postPatch or "" + ''
sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg
# sometimes setup.cfg doesn't exist
if [ -f setup.cfg ]; then
sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg
fi
'';
}
);
@ -1758,6 +1714,10 @@ lib.composeManyExtensions [
}
);
python-twitter = super.python-twitter.overridePythonAttrs (old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ];
});
pythran = super.pythran.overridePythonAttrs (old: {
buildInputs = (old.buildInputs or [ ]) ++ [ self.pytest-runner ];
});
@ -1925,7 +1885,14 @@ lib.composeManyExtensions [
shapely = super.shapely.overridePythonAttrs (
old: {
buildInputs = (old.buildInputs or [ ]) ++ [ pkgs.geos ];
inherit (pkgs.python3.pkgs.shapely) patches GEOS_LIBRARY_PATH;
inherit (pkgs.python3.pkgs.shapely) GEOS_LIBRARY_PATH;
GEOS_LIBC = lib.optionalString (!stdenv.isDarwin) "${stdenv.cc.libc}/lib/libc${stdenv.hostPlatform.extensions.sharedLibrary}.6";
# Fix library paths
postPatch = old.postPatch or "" + ''
${self.python.pythonForBuild.interpreter} ${./shapely-rewrite.py} shapely/geos.py
'';
}
);

View File

@ -0,0 +1,46 @@
"""
Rewrite libc/library path references to Nix store paths
Nixpkgs uses a normal patch for this but we need to be less
sensitive to changes between versions.
"""
from textwrap import dedent
import sys
import ast
import os
with open(sys.argv[1]) as f:
mod = ast.parse(f.read(), "geos.py")
class LibTransformer(ast.NodeTransformer):
_lgeos_replaced = False
def visit_If(self, node):
if ast.unparse(node).startswith("if sys.platform.startswith('linux')"):
return ast.parse(
dedent(
"""
free = CDLL(%s).free
free.argtypes = [c_void_p]
free.restype = None
"""
)
% (lambda x: "'" + x + "'" if x else None)(os.environ.get("GEOS_LIBC"))
)
return node
def visit_Assign(self, node):
_target = node.targets[0]
if (
not self._lgeos_replaced
and isinstance(_target, ast.Name)
and _target.id == "_lgeos"
):
self._lgeos_replaced = True
return ast.parse("_lgeos = CDLL('%s')" % os.environ["GEOS_LIBRARY_PATH"])
return node
with open(sys.argv[1], "w") as f:
f.write(ast.unparse(LibTransformer().visit(mod)))

View File

@ -23,7 +23,9 @@ let
let
entries' = splitString "-" str;
# Hack: Remove version "suffixes" like 2.11.4-1
entries = builtins.filter (x: builtins.match "[0-9]" x == null) entries';
# Some wheels have build tag with more than one digit
# like openvino-2022.1.0-7019-cp36-cp36m-manylinux_2_27_x86_64.whl
entries = builtins.filter (x: builtins.match "[0-9]*" x == null) entries';
p = removeSuffix ".whl" (builtins.elemAt entries 4);
in
{