mirror of
https://github.com/nix-community/dream2nix.git
synced 2024-12-25 23:41:42 +03:00
Merge branch 'main' into fetch-pip-metadata
This commit is contained in:
Paul Haerle
GitHub
commit
c9e6067304
@ -71,7 +71,11 @@
|
||||
# Usually references to buildInputs would get lost in the dist output.
|
||||
# Patch wheels to ensure build inputs remain dependencies of the `dist` output
|
||||
# Those references are needed for the final autoPatchelfHook to find the required deps.
|
||||
patchedWheels = mapAttrs substitutions (name: dist: dist.overridePythonAttrs (old: {postFixup = "ln -s $out $dist/out";}));
|
||||
linkOutToDistOverride = old: {
|
||||
linkOutToDist = "ln -s $out $dist/out";
|
||||
postPhases = ["linkOutToDist"];
|
||||
};
|
||||
patchedWheels = mapAttrs substitutions (name: dist: dist.overridePythonAttrs linkOutToDistOverride);
|
||||
in {inherit patchedWheels downloadedWheels builtWheels;};
|
||||
|
||||
# The final dists we want to install.
|
||||
@ -98,7 +102,7 @@
|
||||
../nixpkgs-overrides
|
||||
];
|
||||
config = {
|
||||
nixpkgs-overrides.enable = true;
|
||||
nixpkgs-overrides.enable = l.mkDefault true;
|
||||
deps = {nixpkgs, ...}:
|
||||
l.mapAttrs (_: l.mkDefault) {
|
||||
inherit python;
|
||||
@ -106,6 +110,7 @@
|
||||
(nixpkgs)
|
||||
autoPatchelfHook
|
||||
stdenv
|
||||
unzip
|
||||
;
|
||||
};
|
||||
|
||||
@ -117,16 +122,19 @@
|
||||
(map (distDir: "--find-links ${distDir}") manualSetupDeps.${name} or [])
|
||||
++ (
|
||||
map (dep: "--find-links ${finalDistsPaths.${dep}}")
|
||||
config.eval-cache.content.mach-nix.dependencyTree.${name}.dependencies or []
|
||||
(getTransitiveDeps name)
|
||||
);
|
||||
};
|
||||
mkDerivation = {
|
||||
# distDir will contain a single file which is the src
|
||||
preUnpack = ''export src="${distDir}"/*'';
|
||||
nativeBuildInputs = [config.deps.autoPatchelfHook];
|
||||
nativeBuildInputs = [
|
||||
config.deps.unzip
|
||||
];
|
||||
# ensure build inputs are propagated for autopPatchelfHook
|
||||
postFixup = "ln -s $out $dist/out";
|
||||
postPhases = ["linkOutToDist"];
|
||||
};
|
||||
env.linkOutToDist = "ln -s $out $dist/out";
|
||||
# TODO If setup deps have been specified manually, we need to remove the
|
||||
# propagatedBuildInputs from nixpkgs to prevent collisions.
|
||||
#// lib.optionalAttrs (manualSetupDeps ? ${name}) {
|
||||
@ -138,6 +146,14 @@
|
||||
dependenciesFile = "${cfg.pythonSources}/metadata.json";
|
||||
dependencyTree = l.fromJSON (l.readFile dependenciesFile);
|
||||
|
||||
getTransitiveDeps' = name: let
|
||||
directDeps = dependencyTree.${name}.dependencies or [];
|
||||
in
|
||||
directDeps
|
||||
++ (l.concatMap getTransitiveDeps' directDeps);
|
||||
|
||||
getTransitiveDeps = name: l.unique (getTransitiveDeps' name);
|
||||
|
||||
makeModuleFromDerivation = _name: drv:
|
||||
drv-parts.lib.makeModule {
|
||||
packageFunc = drv;
|
||||
|
||||
@ -7,8 +7,8 @@
|
||||
l = lib // builtins;
|
||||
cfg = config.nixpkgs-overrides;
|
||||
|
||||
excludedNixpkgsAttrs =
|
||||
l.genAttrs cfg.excludedNixpkgsAttrs (name: null);
|
||||
exclude =
|
||||
l.genAttrs cfg.exclude (name: null);
|
||||
|
||||
extractOverrideAttrs = overrideFunc:
|
||||
(overrideFunc (old: {passthru.old = old;}))
|
||||
@ -17,7 +17,7 @@
|
||||
extractPythonAttrs = pythonPackage: let
|
||||
pythonAttrs = extractOverrideAttrs pythonPackage.overridePythonAttrs;
|
||||
in
|
||||
l.filterAttrs (name: _: ! excludedNixpkgsAttrs ? ${name}) pythonAttrs;
|
||||
l.filterAttrs (name: _: ! exclude ? ${name}) pythonAttrs;
|
||||
|
||||
extracted =
|
||||
if config.deps.python.pkgs ? ${config.name}
|
||||
@ -58,6 +58,16 @@ in {
|
||||
})
|
||||
{
|
||||
nixpkgs-overrides.lib = {inherit extractOverrideAttrs extractPythonAttrs;};
|
||||
nixpkgs-overrides.exclude = [
|
||||
"all"
|
||||
"args"
|
||||
"builder"
|
||||
"name"
|
||||
"pname"
|
||||
"version"
|
||||
"src"
|
||||
"outputs"
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
@ -7,21 +7,11 @@
|
||||
t = l.types;
|
||||
in {
|
||||
options.nixpkgs-overrides = {
|
||||
enable = l.mkEnableOption "Whether to copy attributes, except those in `excludedNixpkgsAttrs` from nixpkgs";
|
||||
enable = l.mkEnableOption "Whether to copy attributes, except those in `excluded` from nixpkgs";
|
||||
|
||||
excludedNixpkgsAttrs = l.mkOption {
|
||||
exclude = l.mkOption {
|
||||
type = t.listOf t.str;
|
||||
description = "Attributes we do not want to copy from nixpkgs";
|
||||
default = [
|
||||
"all"
|
||||
"args"
|
||||
"builder"
|
||||
"name"
|
||||
"pname"
|
||||
"version"
|
||||
"src"
|
||||
"outputs"
|
||||
];
|
||||
};
|
||||
|
||||
lib.extractOverrideAttrs = l.mkOption {
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
{
|
||||
"fetchPipHash": "sha256-i+GRzWBaX7DnTKVsUnVW2zU+wANpJwMKVFCS7jtHZrM="
|
||||
"fetchPipHash": "sha256-pAI3j43S4HedxsXIbe2+LQtpzcp1gjwlneohlD3V3LA="
|
||||
}
|
||||
@ -444,4 +444,5 @@
|
||||
"version": "6.0"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,3 +1,3 @@
|
||||
{
|
||||
"fetchPipHash": "sha256-eS81pqSjU6mgBL6tXadSxkGdafsVFThByOQcOf8FkF0="
|
||||
"fetchPipHash": "sha256-9kedXL+AOrMo3oue5IFRrfMlNZQ3UGp7AmhzlPPENsI="
|
||||
}
|
||||
@ -1,3 +1,3 @@
|
||||
{
|
||||
"fetchPipHash": "sha256-PDUrECFjoPznqXwqi2e1djx63t+kn/kAyM9JqQrTmd0="
|
||||
"fetchPipHash": "sha256-9TDHKU4cb7UorC3nT5stvraf1HbDvHXmD39P39aPCN0="
|
||||
}
|
||||
@ -32,6 +32,7 @@
|
||||
];
|
||||
specialArgs =
|
||||
args.specialArgs
|
||||
or {}
|
||||
// {
|
||||
inherit packageSets;
|
||||
dream2nix.modules.drv-parts = self.modules.drv-parts;
|
||||
|
||||
@ -29,6 +29,7 @@ ONLY_BINARY_FLAGS = os.getenv("onlyBinaryFlags")
|
||||
REQUIREMENTS_LIST = os.getenv("requirementsList")
|
||||
REQUIREMENTS_FILES = os.getenv("requirementsFiles")
|
||||
WRITE_METADATA = os.getenv("writeMetaData")
|
||||
TMPDIR = os.getenv("TMPDIR")
|
||||
|
||||
|
||||
def get_max_date():
|
||||
@ -84,7 +85,10 @@ def wait_for_proxy(proxy_port, cafile):
|
||||
# as we only proxy *some* calls, we need to combine upstream
|
||||
# ca certificates and the one from mitm proxy
|
||||
def generate_ca_bundle(path):
|
||||
with open(HOME / ".mitmproxy/mitmproxy-ca-cert.pem", "r") as f:
|
||||
proxy_cert = HOME / ".mitmproxy/mitmproxy-ca-cert.pem"
|
||||
while not os.path.exists(proxy_cert):
|
||||
time.sleep(0.1)
|
||||
with open(proxy_cert, "r") as f:
|
||||
mitmproxy_cacert = f.read()
|
||||
with open(certifi.where(), "r") as f:
|
||||
certifi_cacert = f.read()
|
||||
@ -111,7 +115,7 @@ if __name__ == "__main__":
|
||||
names_path = OUT / "names"
|
||||
dist_path.mkdir()
|
||||
names_path.mkdir()
|
||||
cache_path = Path("/build/pip_cache")
|
||||
cache_path = Path(f"{TMPDIR}/pip_cache")
|
||||
cache_path.mkdir()
|
||||
|
||||
print(f"selected maximum release date for python packages: {get_max_date()}")
|
||||
@ -121,6 +125,10 @@ if __name__ == "__main__":
|
||||
|
||||
venv_path = Path(".venv").absolute()
|
||||
create_venv(venv_path)
|
||||
|
||||
cafile = generate_ca_bundle(HOME / ".ca-cert.pem")
|
||||
wait_for_proxy(proxy_port, cafile)
|
||||
|
||||
pip(
|
||||
venv_path,
|
||||
"install",
|
||||
@ -128,8 +136,17 @@ if __name__ == "__main__":
|
||||
f"pip=={PIP_VERSION}",
|
||||
)
|
||||
|
||||
cafile = generate_ca_bundle(HOME / ".ca-cert.pem")
|
||||
wait_for_proxy(proxy_port, cafile)
|
||||
# some legacy setup.py based packages require wheel in order to be inspected
|
||||
pip(
|
||||
venv_path,
|
||||
"install",
|
||||
"--proxy",
|
||||
f"https://localhost:{proxy_port}",
|
||||
"--cert",
|
||||
cafile,
|
||||
"--upgrade",
|
||||
f"wheel",
|
||||
)
|
||||
|
||||
flags = [
|
||||
PIP_FLAGS,
|
||||
@ -146,7 +163,7 @@ if __name__ == "__main__":
|
||||
if NO_BINARY:
|
||||
optional_flags += ["--no-binary " + " --no-binary ".join(NO_BINARY.split())]
|
||||
if WRITE_METADATA:
|
||||
metadata_flags = ["--report", "/build/report.json"]
|
||||
metadata_flags = ["--report", f"{TMPDIR}/report.json"]
|
||||
|
||||
for req in REQUIREMENTS_LIST.split(" "):
|
||||
if req:
|
||||
@ -188,7 +205,7 @@ if __name__ == "__main__":
|
||||
if WRITE_METADATA:
|
||||
packages = dict()
|
||||
|
||||
with open("/build/report.json", "r") as f:
|
||||
with open(f"{TMPDIR}/report.json", "r") as f:
|
||||
report = json.load(f)
|
||||
|
||||
for install in report["install"]:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user