pypy: merge 2.7 and 3.5 into a single expression

This commit merges the two expressions in a single one, using
the passthru function that is shared with CPython.

pkgs/development/interpreters/python/default.nix

@@ -98,9 +98,32 @@ in {
     inherit passthruFun;
   };
 
-#   pypy27 = callPackage ./pypy {
+  pypy27 = callPackage ./pypy {
-#     self = pypy27;
+    self = pypy27;
-#     implementation = "pypy";
+    sourceVersion = {
-#   };
+      major = "6";
+      minor = "0";
+      patch = "0";
+    };
+    sha256 = "1qjwpc8n68sxxlfg36s5vn1h2gdfvvd6lxvr4lzbvfwhzrgqahsw";
+    pythonVersion = "2.7";
+    db = db.override { dbmSupport = true; };
+    python = python27;
+    inherit passthruFun;
+  };
+
+  pypy3 = callPackage ./pypy {
+    self = pypy3;
+    sourceVersion = {
+      major = "6";
+      minor = "0";
+      patch = "0";
+    };
+    sha256 = "0lwq8nn0r5yj01bwmkk5p7xvvrp4s550l8184mkmn74d3gphrlwg";
+    pythonVersion = "3.5";
+    db = db.override { dbmSupport = true; };
+    python = python27;
+    inherit passthruFun;
+  };
 
 })

pkgs/development/interpreters/python/pypy/2.7/default.nix

@@ -1,136 +0,0 @@
-{ stdenv, substituteAll, fetchurl
-, zlib ? null, zlibSupport ? true, bzip2, pkgconfig, libffi
-, sqlite, openssl, ncurses, python, expat, tcl, tk, tix, xlibsWrapper, libX11
-, makeWrapper, callPackage, self, gdbm, db
-, python-setup-hook
-# For the Python package set
-, packageOverrides ? (self: super: {})
-}:
-
-assert zlibSupport -> zlib != null;
-
-let
-  version = "6.0.0";
-  pythonVersion = "2.7";
-  libPrefix = "pypy${pythonVersion}";
-  sitePackages = "site-packages";
-
-  pythonForPypy = python.withPackages (ppkgs: [ ppkgs.pycparser ]);
-
-in stdenv.mkDerivation rec {
-  name = "pypy-${version}";
-  inherit version pythonVersion;
-
-  src = fetchurl {
-    url = "https://bitbucket.org/pypy/pypy/get/release-pypy${pythonVersion}-v${version}.tar.bz2";
-    sha256 = "1qjwpc8n68sxxlfg36s5vn1h2gdfvvd6lxvr4lzbvfwhzrgqahsw";
-  };
-
-  nativeBuildInputs = [ pkgconfig makeWrapper ];
-  buildInputs = [
-    bzip2 openssl pythonForPypy libffi ncurses expat sqlite tk tcl xlibsWrapper libX11 gdbm db
-  ] ++ stdenv.lib.optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc
-    ++ stdenv.lib.optional zlibSupport zlib;
-
-  hardeningDisable = stdenv.lib.optional stdenv.isi686 "pic";
-
-  C_INCLUDE_PATH = stdenv.lib.makeSearchPathOutput "dev" "include" buildInputs;
-  LIBRARY_PATH = stdenv.lib.makeLibraryPath buildInputs;
-  LD_LIBRARY_PATH = stdenv.lib.makeLibraryPath (stdenv.lib.filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs);
-
-  patches = [
-    (substituteAll {
-      src = ./tk_tcl_paths.patch;
-      inherit tk tcl;
-      tk_dev = tk.dev;
-      tcl_dev = tcl;
-      tk_libprefix = tk.libPrefix;
-      tcl_libprefix = tcl.libPrefix;
-    })
-  ];
-
-  postPatch = ''
-    substituteInPlace "lib-python/2.7/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
-
-    # hint pypy to find nix ncurses
-    substituteInPlace pypy/module/_minimal_curses/fficurses.py \
-      --replace "/usr/include/ncurses/curses.h" "${ncurses.dev}/include/curses.h" \
-      --replace "ncurses/curses.h" "${ncurses.dev}/include/curses.h" \
-      --replace "ncurses/term.h" "${ncurses.dev}/include/term.h" \
-      --replace "libraries=['curses']" "libraries=['ncurses']"
-
-    sed -i "s@libraries=\['sqlite3'\]\$@libraries=['sqlite3'], include_dirs=['${sqlite.dev}/include'], library_dirs=['${sqlite.out}/lib']@" lib_pypy/_sqlite3_build.py
-  '';
-
-  buildPhase = ''
-    ${pythonForPypy.interpreter} rpython/bin/rpython \
-      --make-jobs="$NIX_BUILD_CORES" \
-      -Ojit \
-      --batch pypy/goal/targetpypystandalone.py
-  '';
-
-  setupHook = python-setup-hook sitePackages;
-
-  doCheck = true;
-  checkPhase = ''
-    export TERMINFO="${ncurses.out}/share/terminfo/";
-    export TERM="xterm";
-    export HOME="$TMPDIR";
-    # disable shutils because it assumes gid 0 exists
-    # disable socket because it has two actual network tests that fail
-    # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
-    ./pypy-c ./pypy/test_all.py --pypy=./pypy-c -k 'not ( test_urllib2net or test_urllibnet or test_urllib2_localnet or test_socket or test_shutil )' lib-python
-  '';
-
-  installPhase = ''
-    mkdir -p $out/{bin,include,lib,pypy-c}
-
-    cp -R {include,lib_pypy,lib-python,pypy-c} $out/pypy-c
-    cp libpypy-c.so $out/lib/
-    ln -s $out/pypy-c/pypy-c $out/bin/pypy
-    chmod +x $out/bin/pypy
-
-    # other packages expect to find stuff according to libPrefix
-    ln -s $out/pypy-c/include $out/include/${libPrefix}
-    ln -s $out/pypy-c/lib-python/${pythonVersion} $out/lib/${libPrefix}
-
-    # We must wrap the original, not the symlink.
-    # PyPy uses argv[0] to find its standard library, and while it knows
-    # how to follow symlinks, it doesn't know about wrappers. So, it
-    # will think the wrapper is the original. As long as the wrapper has
-    # the same path as the original, this is OK.
-    wrapProgram "$out/pypy-c/pypy-c" \
-      --set LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:$out/lib" \
-      --set LIBRARY_PATH "${LIBRARY_PATH}:$out/lib"
-
-    # verify cffi modules
-    $out/bin/pypy -c "import Tkinter;import sqlite3;import curses"
-
-    # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
-    echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
-  '';
-
-  passthru = let
-    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
-  in rec {
-    inherit zlibSupport libPrefix sitePackages;
-    executable = "pypy";
-    isPypy = true;
-    isPy2 = true;
-    isPy27 = true;
-    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-    interpreter = "${self}/bin/${executable}";
-    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-    pkgs = pythonPackages;
-  };
-
-  enableParallelBuilding = true;  # almost no parallelization without STM
-
-  meta = with stdenv.lib; {
-    homepage = http://pypy.org/;
-    description = "Fast, compliant alternative implementation of the Python language (2.7.13)";
-    license = licenses.mit;
-    platforms = [ "i686-linux" "x86_64-linux" ];
-    maintainers = with maintainers; [ ];
-  };
-}

pkgs/development/interpreters/python/pypy/3/tk_tcl_paths.patch

@@ -1,17 +0,0 @@
---- pypy-pypy-84a2f3e6a7f8.org/lib_pypy/_tkinter/tklib_build.py	2017-10-03 11:49:20.000000000 +0100
-+++ pypy-pypy-84a2f3e6a7f8/lib_pypy/_tkinter/tklib_build.py	2017-11-21 13:20:51.398607530 +0000
-@@ -24,11 +24,11 @@
- else:
-     # On some Linux distributions, the tcl and tk libraries are
-     # stored in /usr/include, so we must check this case also
--    libdirs = []
-+    libdirs = ["@tcl@/lib", "@tk@/lib"]
-     found = False
-     for _ver in ['', '8.6', '8.5']:
--        incdirs = ['/usr/include/tcl' + _ver]
--        linklibs = ['tcl' + _ver, 'tk' + _ver]
-+        incdirs = ['@tcl_dev@/include', '@tk_dev@/include']
-+        linklibs = ['@tcl_libprefix@', '@tk_libprefix@']
-         if os.path.isdir(incdirs[0]):
-             found = True
-             break

pkgs/development/interpreters/python/pypy/default.nix

@@ -5,38 +5,54 @@
 , python-setup-hook
 # For the Python package set
 , packageOverrides ? (self: super: {})
+, sourceVersion
+, pythonVersion
+, sha256
+, passthruFun
 }:
 
 assert zlibSupport -> zlib != null;
 
-let
+with stdenv.lib;
-  version = "6.0.0";
-  pythonVersion = "3.5";
-  libPrefix = "pypy${pythonVersion}";
-  sitePackages = "site-packages";
 
+let
+  isPy3k = substring 0 1 pythonVersion == "3";
+  passthru = passthruFun rec {
+    inherit self sourceVersion pythonVersion packageOverrides;
+    implementation = "pypy";
+    libPrefix = "pypy${pythonVersion}";
+    executable = "pypy${if isPy3k then "3" else ""}";
+    pythonForBuild = self; # No cross-compiling for now.
+    sitePackages = "site-packages";
+  };
+  pname = passthru.executable;
+  version = with sourceVersion; "${major}.${minor}.${patch}";
   pythonForPypy = python.withPackages (ppkgs: [ ppkgs.pycparser ]);
 
-in stdenv.mkDerivation rec {
+in with passthru; stdenv.mkDerivation rec {
-  name = "pypy3-${version}";
+  inherit pname version;
-  inherit version pythonVersion;
 
   src = fetchurl {
     url = "https://bitbucket.org/pypy/pypy/get/release-pypy${pythonVersion}-v${version}.tar.bz2";
-    sha256 = "0lwq8nn0r5yj01bwmkk5p7xvvrp4s550l8184mkmn74d3gphrlwg";
+    inherit sha256;
   };
 
   nativeBuildInputs = [ pkgconfig makeWrapper ];
   buildInputs = [
-    bzip2 openssl pythonForPypy libffi ncurses expat sqlite tk tcl xlibsWrapper libX11 gdbm db lzma
+    bzip2 openssl pythonForPypy libffi ncurses expat sqlite tk tcl xlibsWrapper libX11 gdbm db
-  ] ++ stdenv.lib.optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc
+  ]  ++ optionals isPy3k [
-    ++ stdenv.lib.optional zlibSupport zlib;
+    lzma
+  ] ++ optionals (stdenv ? cc && stdenv.cc.libc != null) [
+    stdenv.cc.libc
+  ] ++ optionals zlibSupport [
+    zlib
+  ];
 
-  hardeningDisable = stdenv.lib.optional stdenv.isi686 "pic";
+  hardeningDisable = optional stdenv.isi686 "pic";
 
-  C_INCLUDE_PATH = stdenv.lib.makeSearchPathOutput "dev" "include" buildInputs;
+  C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" buildInputs;
-  LIBRARY_PATH = stdenv.lib.makeLibraryPath buildInputs;
+  LIBRARY_PATH = makeLibraryPath buildInputs;
-  LD_LIBRARY_PATH = stdenv.lib.makeLibraryPath (stdenv.lib.filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs);
+  LD_LIBRARY_PATH = makeLibraryPath (filter (x : x.outPath != stdenv.cc.libc.outPath or "") buildInputs);
 
   patches = [
     (substituteAll {
@@ -50,7 +66,7 @@ in stdenv.mkDerivation rec {
   ];
 
   postPatch = ''
-    substituteInPlace "lib-python/3/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
+    substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
 
     # hint pypy to find nix ncurses
     substituteInPlace pypy/module/_minimal_curses/fficurses.py \
@@ -72,60 +88,63 @@ in stdenv.mkDerivation rec {
   setupHook = python-setup-hook sitePackages;
 
   doCheck = true;
-  checkPhase = ''
+  checkPhase = let
+    disabledTests = [
+      # disable shutils because it assumes gid 0 exists
+      "test_shutil"
+      # disable socket because it has two actual network tests that fail
+      "test_socket"
+    ] ++ optionals (!isPy3k) [
+      # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
+      "test_urllib2net"
+      "test_urllibnet"
+      "test_urllib2_localnet"
+    ] ++ optionals isPy3k [
+      # disable asyncio due to https://github.com/NixOS/nix/issues/1238
+      "test_asyncio"
+      # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
+      "test_os"
+      # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
+      "test_pathlib"
+      # disable tarfile because it assumes gid 0 exists
+      "test_tarfile"
+    ];
+  in ''
     export TERMINFO="${ncurses.out}/share/terminfo/";
     export TERM="xterm";
     export HOME="$TMPDIR";
-    # disable asyncio due to https://github.com/NixOS/nix/issues/1238
+
-    # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
+    ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${concatStringsSep " or " disabledTests})' lib-python
-    # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
-    # disable shutils because it assumes gid 0 exists
-    # disable socket because it has two actual network tests that fail
-    # disable tarfile because it assumes gid 0 exists
-    ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./pypy3-c -k 'not ( test_asyncio or test_os or test_pathlib or test_shutil or test_socket or test_tarfile )' lib-python
   '';
 
   installPhase = ''
-    mkdir -p $out/{bin,include,lib,pypy3-c}
+    mkdir -p $out/{bin,include,lib,${executable}-c}
 
-    cp -R {include,lib_pypy,lib-python,pypy3-c} $out/pypy3-c
+    cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c
-    cp libpypy3-c.so $out/lib/
+    cp lib${executable}-c.so $out/lib/
-    ln -s $out/pypy3-c/pypy3-c $out/bin/pypy3
+    ln -s $out/${executable}-c/${executable}-c $out/bin/${executable}
 
     # other packages expect to find stuff according to libPrefix
-    ln -s $out/pypy3-c/include $out/include/${libPrefix}
+    ln -s $out/${executable}/include $out/include/${libPrefix}
-    ln -s $out/pypy3-c/lib-python/3 $out/lib/${libPrefix}
+    ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
 
     # We must wrap the original, not the symlink.
     # PyPy uses argv[0] to find its standard library, and while it knows
     # how to follow symlinks, it doesn't know about wrappers. So, it
     # will think the wrapper is the original. As long as the wrapper has
     # the same path as the original, this is OK.
-    wrapProgram "$out/pypy3-c/pypy3-c" \
+    wrapProgram "$out/${executable}-c/${executable}-c" \
       --set LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:$out/lib" \
       --set LIBRARY_PATH "${LIBRARY_PATH}:$out/lib"
 
     # verify cffi modules
-    $out/bin/pypy3 -c "import tkinter;import sqlite3;import curses;import lzma"
+    $out/bin/${executable} -c ${if isPy3k then "'import tkinter;import sqlite3;import curses;import lzma'" else "'import Tkinter;import sqlite3;import curses'"}
 
     # Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
     echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
   '';
 
-  passthru = let
+  inherit passthru;
-    pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
-  in rec {
-    inherit zlibSupport libPrefix sitePackages;
-    executable = "pypy3";
-    isPypy = true;
-    isPy3 = true;
-    isPy35 = true;
-    buildEnv = callPackage ../../wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; };
-    interpreter = "${self}/bin/${executable}";
-    withPackages = import ../../with-packages.nix { inherit buildEnv pythonPackages;};
-    pkgs = pythonPackages;
-  };
-
   enableParallelBuilding = true;  # almost no parallelization without STM
 
   meta = with stdenv.lib; {

pkgs/top-level/all-packages.nix

@@ -7954,18 +7954,7 @@ in
   python3Packages = python3.pkgs;
 
   pythonInterpreters = callPackage ./../development/interpreters/python {};
-  inherit (pythonInterpreters) python27 python35 python36 python37;
+  inherit (pythonInterpreters) python27 python35 python36 python37 pypy27 pypy3;
-
-  pypy27 = callPackage ../development/interpreters/python/pypy/2.7 {
-    self = pypy27;
-    python = python27.override{x11Support=true;};
-    db = db.override { dbmSupport = true; };
-  };
-  pypy3 = callPackage ../development/interpreters/python/pypy/3 {
-    self = pypy3;
-    python = python27;
-    db = db.override { dbmSupport = true; };
-  };
 
   # Python package sets.
   python27Packages = lib.hiPrioSet (recurseIntoAttrs python27.pkgs);