Merge pull request #786 from NixOS/python3support

add python3 support to pythonPackages

pkgs/development/interpreters/pypy/2.0/default.nix

@@ -7,6 +7,8 @@ let
 
   majorVersion = "2.0";
   version = "${majorVersion}.2";
+  pythonVersion = "2.7";
+  libPrefix = "pypy${majorVersion}";
 
   pypy = stdenv.mkDerivation rec {
     name = "pypy-${version}";
@@ -57,17 +59,22 @@ let
     '';
 
     installPhase = ''
-       mkdir -p $out/bin
-       mkdir -p $out/pypy-c
+       mkdir -p $out/{bin,include,lib,pypy-c}
+
        cp -R {include,lib_pypy,lib-python,pypy-c} $out/pypy-c
        ln -s $out/pypy-c/pypy-c $out/bin/pypy
        chmod +x $out/bin/pypy
+
+       # other packages expect to find stuff according to libPrefix
+       ln -s $out/pypy-c/include $out/include/${libPrefix}
+       ln -s $out/pypy-c/lib-python/${pythonVersion} $out/lib/${libPrefix}
+
        # TODO: compile python files?
     '';
 
     passthru = {
-      inherit zlibSupport;
-      libPrefix = "pypy${majorVersion}";
+      inherit zlibSupport libPrefix;
+      executable = "pypy";
     };
 
     enableParallelBuilding = true;

pkgs/development/interpreters/python/2.6/default.nix

@@ -75,6 +75,7 @@ let
     passthru = {
       inherit zlibSupport;
       libPrefix = "python${majorVersion}";
+      executable = "python2.6";
     };
 
     enableParallelBuilding = true;

pkgs/development/interpreters/python/2.7/default.nix

@@ -85,6 +85,7 @@ let
     passthru = {
       inherit zlibSupport;
       libPrefix = "python${majorVersion}";
+      executable = "python2.7";
     };
 
     enableParallelBuilding = true;

pkgs/development/interpreters/python/3.2/default.nix

@@ -32,6 +32,8 @@ stdenv.mkDerivation {
     sha256 = "0pxs234g08v3lar09lvzxw4vqdpwkbqmvkv894j2w7aklskcjd6v";
   };
 
+  NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";
+
   preConfigure = ''
     for i in /usr /sw /opt /pkg; do	# improve purity
       substituteInPlace ./setup.py --replace $i /no-such-path
@@ -49,6 +51,7 @@ stdenv.mkDerivation {
 
   postInstall = ''
     rm -rf "$out/lib/python${majorVersion}/test"
+    ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
   '';
 
   passthru = {
@@ -58,7 +61,9 @@ stdenv.mkDerivation {
     readlineSupport = readline != null;
     opensslSupport = openssl != null;
     tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null);
-    libPrefix = "python${majorVersion}m";
+    libPrefix = "python${majorVersion}";
+    executable = "python3.2m";
+    is_py3k = true;
   };
 
   enableParallelBuilding = true;

pkgs/development/interpreters/python/3.3/default.nix

@@ -32,6 +32,8 @@ stdenv.mkDerivation {
     sha256 = "16myvina7nakyyg7r5gnjyydk8bzar988vmxsw2k485w5gz04wpp";
   };
 
+  NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";
+
   preConfigure = ''
     for i in /usr /sw /opt /pkg; do	# improve purity
       substituteInPlace ./setup.py --replace $i /no-such-path
@@ -49,6 +51,7 @@ stdenv.mkDerivation {
 
   postInstall = ''
     rm -rf "$out/lib/python${majorVersion}/test"
+    ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
   '';
 
   passthru = {
@@ -58,7 +61,9 @@ stdenv.mkDerivation {
     readlineSupport = readline != null;
     opensslSupport = openssl != null;
     tkSupport = (tk != null) && (tcl != null) && (libX11 != null) && (xproto != null);
-    libPrefix = "python${majorVersion}m";
+    libPrefix = "python${majorVersion}";
+    executable = "python3.3m";
+    is_py3k = true;
   };
 
   enableParallelBuilding = true;

pkgs/development/interpreters/python/3.3/setup-hook.sh

@@ -1,12 +1,12 @@
 addPythonPath() {
-    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.2/site-packages
+    addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.3/site-packages
 }
 
 toPythonPath() {
     local paths="$1"
     local result=
     for i in $paths; do
-        p="$i/lib/python3.2/site-packages"
+        p="$i/lib/python3.3/site-packages"
         result="${result}${result:+:}$p"
     done
     echo $result

pkgs/development/python-modules/generic/default.nix

@@ -5,7 +5,7 @@
 
 { python, setuptools, wrapPython, lib, offlineDistutils, recursivePthLoader }:
 
-{ name, namePrefix ? "python-"
+{ name, namePrefix ? python.libPrefix + "-"
 
 , buildInputs ? []
 
@@ -33,7 +33,7 @@
 , checkPhase ?
     ''
       runHook preCheck
-      python setup.py test
+      ${python}/bin/${python.executable} setup.py test
       runHook postCheck
     ''
 
@@ -55,7 +55,7 @@ python.stdenv.mkDerivation (attrs // {
   meta = {
     platforms = python.meta.platforms;
   } // meta // {
-    maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow ];
+    maintainers = (meta.maintainers or []) ++ [ lib.maintainers.chaoflow lib.maintainers.iElectric ];
   };
 
   # checkPhase after installPhase to run tests on installed packages

pkgs/development/python-modules/setuptools/default.nix

@@ -2,31 +2,37 @@
 
 stdenv.mkDerivation rec {
   shortName = "setuptools-${version}";
-  name = "python-${shortName}";
+  name = "${python.executable}-${shortName}";
 
-  version = "0.6c11";
+  version = "0.9.8";
 
   src = fetchurl {
     url = "http://pypi.python.org/packages/source/s/setuptools/${shortName}.tar.gz";
-    sha256 = "1lx1hwxkhipyh206bgl90ddnfcnb68bzcvyawczbf833fadyl3v3";
+    sha256 = "037b8x3fdhx8s6xafqndi3yr8x2vr42n1kzs7jxk6j9s9fd65gs2";
   };
 
+  patches = [
+    # https://bitbucket.org/pypa/setuptools/issue/55/1-failure-lc_all-c-python33m-setuppy-test
+    ./distribute-skip-sdist_with_utf8_encoded_filename.patch
+  ];
+
   buildInputs = [ python wrapPython ];
 
-  buildPhase = "python setup.py build --build-base $out";
+  buildPhase = "${python}/bin/${python.executable} setup.py build --build-base $out";
 
   installPhase =
     ''
       dst=$out/lib/${python.libPrefix}/site-packages
       mkdir -p $dst
-      PYTHONPATH=$dst:$PYTHONPATH
-      python setup.py install --prefix=$out
+      PYTHONPATH="$dst:$PYTHONPATH"
+      ${python}/bin/${python.executable} setup.py install --prefix=$out
       wrapPythonPrograms
     '';
 
-  doCheck = false; # doesn't work with Python 2.7
-
-  checkPhase = "python setup.py test";
+  doCheck = true;
+  checkPhase = ''
+    ${python}/bin/${python.executable} setup.py test
+  '';
 
   meta = {
     description = "Utilities to facilitate the installation of Python packages";

pkgs/development/python-modules/setuptools/distribute-skip-sdist_with_utf8_encoded_filename.patch

@@ -0,0 +1,28 @@
+diff -r f5ac515f062a setuptools/tests/test_sdist.py
+--- a/setuptools/tests/test_sdist.py	Fri Jul 26 09:52:26 2013 +0200
++++ b/setuptools/tests/test_sdist.py	Sat Jul 27 20:22:17 2013 +0200
+@@ -3,12 +3,14 @@
+ 
+ 
+ import os
++import locale
+ import shutil
+ import sys
+ import tempfile
+ import unittest
+ import unicodedata
+ 
++from setuptools.tests.py26compat import skipIf
+ from setuptools.compat import StringIO, unicode
+ from setuptools.command.sdist import sdist
+ from setuptools.command.egg_info import manifest_maker
+@@ -318,6 +320,9 @@
+             filename = filename.decode('latin-1')
+             self.assertFalse(filename in cmd.filelist.files)
+ 
++
++    @skipIf(sys.version_info >= (3,) and locale.getpreferredencoding() != 'UTF-8',
++            'Unittest fails if locale is not utf-8 but the manifests is recorded correctly')
+     def test_sdist_with_utf8_encoded_filename(self):
+         # Test for #303.
+         dist = Distribution(SETUP_ATTRS)

pkgs/top-level/all-packages.nix

@@ -5736,6 +5736,7 @@ let
 
   ### DEVELOPMENT / PYTHON MODULES
 
+  # python function with default python interpreter
   buildPythonPackage = pythonPackages.buildPythonPackage;
 
   pythonPackages = python27Packages;
@@ -5750,12 +5751,32 @@ let
     python = python26;
   };
 
+  python3Packages = python33Packages;
+
+  python33Packages = recurseIntoAttrs (import ./python-packages.nix {
+    inherit pkgs;
+    inherit (lib) lowPrio;
+    python = python33;
+  });
+
+  python32Packages = import ./python-packages.nix {
+    inherit pkgs;
+    inherit (lib) lowPrio;
+    python = python32;
+  };
+
   python27Packages = recurseIntoAttrs (import ./python-packages.nix {
     inherit pkgs;
     inherit (lib) lowPrio;
     python = python27;
   });
 
+  pypyPackages = recurseIntoAttrs (import ./python-packages.nix {
+    inherit pkgs;
+    inherit (lib) lowPrio;
+    python = pypy;
+  });
+
   plone41Packages = recurseIntoAttrs (import ../development/web/plone/4.1.nix {
     inherit pkgs;
     pythonPackages = python26Packages;

pkgs/top-level/python-packages.nix

@@ -5,8 +5,9 @@ isPy26 = python.majorVersion == "2.6";
 isPy27 = python.majorVersion == "2.7";
 optional = pkgs.lib.optional;
 optionals = pkgs.lib.optionals;
+modules = python.modules or { readline = null; sqlite3 = null; curses = null; ssl = null; };
 
-pythonPackages = python.modules // rec {
+pythonPackages = modules // rec {
 
   inherit python;
   inherit (pkgs) fetchurl fetchsvn fetchgit stdenv;
@@ -301,8 +302,6 @@ pythonPackages = python.modules // rec {
       sha256 = "b0c12b8c48ed9180c7475fab18de50d63e1b517cfb46da4d2c66fc406fe902bc";
     };
 
-    installCommand = "python setup.py install --prefix=$out";
-
     # error: invalid command 'test'
     doCheck = false;
 
@@ -444,8 +443,8 @@ pythonPackages = python.modules // rec {
         pythonPackages.mutagen
         pythonPackages.munkres
         pythonPackages.musicbrainzngs
-        python.modules.sqlite3
-        python.modules.readline
+        modules.sqlite3
+        modules.readline
       ];
 
     meta = {
@@ -500,13 +499,13 @@ pythonPackages = python.modules // rec {
   };
 
   bpython = buildPythonPackage rec {
-     name = "bpython-0.11";
+     name = "bpython-0.12";
      src = fetchurl {
-       url = "http://www.bpython-interpreter.org/releases/bpython-0.11.tar.gz";
-       sha256 = "02dkmsmgy04l33nyw54rlxkjwff0yf3cy2kvdx8s5w344mqkkkv0";
+       url = "http://www.bpython-interpreter.org/releases/bpython-0.12.tar.gz";
+       sha256 = "1ilf58qq7sazmcgg4f1wswbhcn2gb8qbbrpgm6gf0j2lbm60gabl";
      };
 
-     propagatedBuildInputs = [ python.modules.curses pygments ];
+     propagatedBuildInputs = [ modules.curses pygments ];
      doCheck = false;
 
      meta = {
@@ -606,7 +605,7 @@ pythonPackages = python.modules // rec {
   #     rev = "refs/tags/0.9.3";
   #   };
   #
-  #   propagatedBuildInputs = [ pythonPackages.argparse python.modules.ssl ];
+  #   propagatedBuildInputs = [ pythonPackages.argparse modules.ssl ];
   #
   #   doCheck = false;
   #
@@ -1490,7 +1489,7 @@ pythonPackages = python.modules // rec {
       PYTHONPATH="${offlineDistutils}/lib/${python.libPrefix}/site-packages:$PYTHONPATH"
       export PYTHONPATH="$dst:$PYTHONPATH"
 
-      python setup.py install --prefix="$out"
+      ${python}/bin/${python.executable} setup.py install --prefix="$out"
 
       eapth="$out/lib/${python.libPrefix}"/site-packages/easy-install.pth
       if [ -e "$eapth" ]; then
@@ -1691,7 +1690,7 @@ pythonPackages = python.modules // rec {
 
     buildPhase = "make build";
     installCommand = ''
-      python setup.py install --prefix="$out" --root=/ --record="$out/lib/${python.libPrefix}/site-packages/dulwich/list.txt" --single-version-externally-managed
+      ${python}/bin/${python.executable} setup.py install --prefix="$out" --root=/ --record="$out/lib/${python.libPrefix}/site-packages/dulwich/list.txt" --single-version-externally-managed
     '';
 
     # For some reason "python setup.py test" doesn't work with Python 2.6.
@@ -2387,7 +2386,7 @@ pythonPackages = python.modules // rec {
       sha256 = "0xfaa6h8css3yhsmx5vcffizrz6mvmgm46q7449z3hq7g3793184";
     };
 
-    propagatedBuildInputs = [ python.modules.sqlite3 ];
+    propagatedBuildInputs = [ modules.sqlite3 ];
 
     doCheck = false;
 
@@ -2476,10 +2475,10 @@ pythonPackages = python.modules // rec {
 
     configurePhase = "cd python";
 
-    buildPhase = "python setup.py build";
+    buildPhase = "${python}/bin/${python.executable} setup.py build";
 
     installPhase = ''
-      python setup.py install --prefix=$out
+      ${python}/bin/${python.executable} setup.py install --prefix=$out
     '';
 
     meta = {
@@ -2500,7 +2499,7 @@ pythonPackages = python.modules // rec {
 
     buildInputs = [ pkgs.swig pkgs.openssl ];
 
-    buildPhase = "python setup.py build_ext --openssl=${pkgs.openssl}";
+    buildPhase = "${python}/bin/${python.executable} setup.py build_ext --openssl=${pkgs.openssl}";
 
     doCheck = false; # another test that depends on the network.
 
@@ -2754,7 +2753,7 @@ pythonPackages = python.modules // rec {
 
     buildInputs = [ pkgs.unzip ];
 
-    propagatedBuildInputs = [ argparse jinja2 six python.modules.readline ] ++
+    propagatedBuildInputs = [ argparse jinja2 six modules.readline ] ++
                             (optionals isPy26 [ importlib ordereddict ]);
 
     meta = {
@@ -3124,8 +3123,8 @@ pythonPackages = python.modules // rec {
     # TODO: add ATLAS=${pkgs.atlas}
     installCommand = ''
       export BLAS=${pkgs.blas} LAPACK=${pkgs.liblapack}
-      python setup.py build --fcompiler="gnu95"
-      python setup.py install --prefix=$out
+      ${python}/bin/${python.executable} setup.py build --fcompiler="gnu95"
+      ${python}/bin/${python.executable} setup.py install --prefix=$out
     '';
 
     # error: invalid command 'test'
@@ -3273,7 +3272,7 @@ pythonPackages = python.modules // rec {
     };
 
     buildInputs = [ nose ];
-    propagatedBuildInputs = [ dateutil numpy pytz python.modules.sqlite3 ];
+    propagatedBuildInputs = [ dateutil numpy pytz modules.sqlite3 ];
 
     # Tests require networking to pass
     doCheck = false;
@@ -3703,12 +3702,12 @@ pythonPackages = python.modules // rec {
     buildInputs = [ python pkgs.portaudio ];
 
     buildPhase = if stdenv.isDarwin then ''
-      PORTAUDIO_PATH="${pkgs.portaudio}" python setup.py build --static-link
+      PORTAUDIO_PATH="${pkgs.portaudio}" ${python}/bin/${python.executable} setup.py build --static-link
     '' else ''
-      python setup.py build
+      ${python}/bin/${python.executable} setup.py build
     '';
 
-    installPhase = "python setup.py install --prefix=$out";
+    installPhase = "${python}/bin/${python.executable} setup.py install --prefix=$out";
 
     meta = {
       description = "Python bindings for PortAudio";
@@ -3964,7 +3963,7 @@ pythonPackages = python.modules // rec {
     buildInputs = [ python ];
 
     installPhase = ''
-      python setup.py install --prefix=$out
+      ${python}/bin/${python.executable} setup.py install --prefix=$out
     '';
 
     meta = {
@@ -3993,7 +3992,7 @@ pythonPackages = python.modules // rec {
     propagatedBuildInputs = [ urlgrabber ];
 
     checkPhase = ''
-      python tests/baseclass.py -vv
+      ${python}/bin/${python.executable} tests/baseclass.py -vv
     '';
 
     meta = {
@@ -4071,7 +4070,7 @@ pythonPackages = python.modules // rec {
     propagatedBuildInputs = [ pkgs.parted ];
 
     checkPhase = ''
-      python -m unittest discover -v
+      ${python}/bin/${python.executable} -m unittest discover -v
     '';
 
     meta = {
@@ -4629,7 +4628,7 @@ pythonPackages = python.modules // rec {
 
     propagatedBuildInputs =
       [ recaptcha_client pytz memcached dateutil_1_5 paramiko flup pygments
-        djblets django_1_3 django_evolution pycrypto python.modules.sqlite3
+        djblets django_1_3 django_evolution pycrypto modules.sqlite3
         pysvn pil psycopg2
       ];
   };
@@ -4743,8 +4742,8 @@ pythonPackages = python.modules // rec {
     # TODO: add ATLAS=${pkgs.atlas}
     installCommand = ''
       export BLAS=${pkgs.blas} LAPACK=${pkgs.liblapack}
-      python setup.py build --fcompiler="gnu95"
-      python setup.py install --prefix=$out
+      ${python}/bin/${python.executable} setup.py build --fcompiler="gnu95"
+      ${python}/bin/${python.executable} setup.py install --prefix=$out
     '';
 
     meta = {
@@ -5022,7 +5021,7 @@ pythonPackages = python.modules // rec {
 
     buildInputs = [ nose ];
 
-    propagatedBuildInputs = [ python.modules.sqlite3 ];
+    propagatedBuildInputs = [ modules.sqlite3 ];
 
     meta = {
       homepage = http://www.sqlalchemy.org/;
@@ -5201,7 +5200,7 @@ pythonPackages = python.modules // rec {
 
     PYTHON_EGG_CACHE = "`pwd`/.egg-cache";
 
-    propagatedBuildInputs = [ genshi pkgs.setuptools python.modules.sqlite3 ];
+    propagatedBuildInputs = [ genshi pkgs.setuptools modules.sqlite3 ];
 
     meta = {
       description = "Enhanced wiki and issue tracking system for software development projects";
@@ -5322,12 +5321,18 @@ pythonPackages = python.modules // rec {
 
 
   unittest2 = buildPythonPackage rec {
-    name = "unittest2-0.5.1";
+    version = "0.5.1";
+    name = "unittest2-${version}";
 
-    src = fetchurl {
-      url = "http://pypi.python.org/packages/source/u/unittest2/${name}.tar.gz";
-      md5 = "a0af5cac92bbbfa0c3b0e99571390e0f";
-    };
+    src = if python.is_py3k or false
+       then fetchurl {
+           url = "http://pypi.python.org/packages/source/u/unittest2py3k/unittest2py3k-${version}.tar.gz";
+           sha256 = "00yl6lskygcrddx5zspkhr0ibgvpknl4678kkm6s626539grq93q";
+         }
+       else fetchurl {
+           url = "http://pypi.python.org/packages/source/u/unittest2/unittest2-${version}.tar.gz";
+           md5 = "a0af5cac92bbbfa0c3b0e99571390e0f";
+         };
 
     meta = {
       description = "A backport of the new features added to the unittest testing framework in Python 2.7";
@@ -5384,9 +5389,12 @@ pythonPackages = python.modules // rec {
       md5 = "9745c28256c70c76d36adb3767a00212";
     };
 
+    inherit recursivePthLoader;
+    pythonPath = [ recursivePthLoader ];
+
     patches = [ ../development/python-modules/virtualenv-change-prefix.patch ];
 
-    propagatedBuildInputs = [ python.modules.readline python.modules.sqlite3 ];
+    propagatedBuildInputs = [ modules.readline modules.sqlite3 modules.curses ];
 
     buildInputs = [ mock nose ];
 
@@ -5446,7 +5454,7 @@ pythonPackages = python.modules // rec {
       md5 = "11825b7074ba7043e157805e4e6e0f55";
     };
 
-    propagatedBuildInputs = [ nose python.modules.ssl ];
+    propagatedBuildInputs = [ nose modules.ssl ];
 
     meta = {
       description = "WSGI request and response object";
@@ -6100,13 +6108,14 @@ pythonPackages = python.modules // rec {
 
   zope_testing = buildPythonPackage rec {
     name = "zope.testing-${version}";
-    version = "4.1.1";
+    version = "4.1.2";
 
     src = fetchurl {
-      url = "http://pypi.python.org/packages/source/z/zope.testing/${name}.tar.gz";
-      md5 = "2e3829841090d6adff718b8b73c87b6b";
+      url = "http://pypi.python.org/packages/source/z/zope.testing/${name}.zip";
+      md5 = "01c30c342c6a18002a762bd5d320a6e9";
     };
 
+    buildInputs = [ pkgs.unzip ];
     propagatedBuildInputs = [ zope_interface zope_exceptions zope_location ];
 
     meta = {
@@ -6237,7 +6246,7 @@ pythonPackages = python.modules // rec {
     };
 
     buildInputs = [ pkgs.unzip unittest2 nose mock ];
-    propagatedBuildInputs = [ python.modules.curses libarchive ];
+    propagatedBuildInputs = [ modules.curses libarchive ];
 
     # two tests fail
     doCheck = false;
@@ -6478,7 +6487,7 @@ pythonPackages = python.modules // rec {
       sha256 = "1gj8i6j2i172cldqw98395235bn78ciagw6v17fgv01rmind3lag";
     };
 
-    buildInputs = [ django pkgs.pycairo ldap memcached python.modules.sqlite3 ];
+    buildInputs = [ django pkgs.pycairo ldap memcached modules.sqlite3 ];
 
     # error: invalid command 'test'
     doCheck = false;