mirror of
https://github.com/ilyakooo0/nixpkgs.git
synced 2024-12-28 22:32:58 +03:00
Merge pull request #22585 from FRidh/repr
Python: deterministic interpreters
This commit is contained in:
Frederik Rietdijk
GitHub
commit
04c41e753b
@ -641,6 +641,19 @@ community to help save time. No tool is preferred at the moment.
|
||||
- [pypi2nix](https://github.com/garbas/pypi2nix) by Rok Garbas
|
||||
- [pypi2nix](https://github.com/offlinehacker/pypi2nix) by Jaka Hudoklin
|
||||
|
||||
### Deterministic builds
|
||||
|
||||
Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
|
||||
Minor modifications had to be made to the interpreters in order to generate
|
||||
deterministic bytecode. This has security implications and is relevant for
|
||||
those using Python in a `nix-shell`.
|
||||
|
||||
When the environment variable `DETERMINISTIC_BUILD` is set, all bytecode will have timestamp 1.
|
||||
The `buildPythonPackage` function sets `DETERMINISTIC_BUILD` as well as
|
||||
[PYTHONHASHSEED](https://docs.python.org/3.5/using/cmdline.html#envvar-PYTHONHASHSEED).
|
||||
Both are also exported in `nix-shell`.
|
||||
|
||||
|
||||
## FAQ
|
||||
|
||||
### How can I install a working Python environment?
|
||||
|
||||
@ -253,6 +253,17 @@ following incompatible changes:</para>
|
||||
</para>
|
||||
</listitem>
|
||||
|
||||
<listitem>
|
||||
<para>
|
||||
Python 2.7, 3.5 and 3.6 are now built deterministically and 3.4 mostly.
|
||||
Minor modifications had to be made to the interpreters in order to generate
|
||||
deterministic bytecode. This has security implications and is relevant for
|
||||
those using Python in a <literal>nix-shell</literal>. See the Nixpkgs manual
|
||||
for details.
|
||||
</para>
|
||||
|
||||
</listitem>
|
||||
|
||||
</itemizedlist>
|
||||
|
||||
|
||||
|
||||
@ -178,6 +178,17 @@ in stdenv.mkDerivation {
|
||||
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
|
||||
|
||||
rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
|
||||
|
||||
# Determinism: Windows installers were not deterministic.
|
||||
# We're also not interested in building Windows installers.
|
||||
find "$out" -name 'wininst*.exe' | xargs -r rm -f
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
passthru = let
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
{ stdenv, fetchurl
|
||||
, bzip2
|
||||
, expat
|
||||
, libffi
|
||||
, gdbm
|
||||
, lzma
|
||||
, ncurses
|
||||
@ -50,21 +52,43 @@ in stdenv.mkDerivation {
|
||||
|
||||
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
|
||||
|
||||
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
|
||||
# This way python doesn't try to update them when we freeze timestamps in nix store.
|
||||
DETERMINISTIC_BUILD=1;
|
||||
# Determinism: We fix the hashes of str, bytes and datetime objects.
|
||||
PYTHONHASHSEED=0;
|
||||
|
||||
prePatch = optionalString stdenv.isDarwin ''
|
||||
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
|
||||
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
|
||||
'';
|
||||
|
||||
postPatch = optionalString (x11Support && (tix != null)) ''
|
||||
postPatch = ''
|
||||
# Determinism
|
||||
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
|
||||
# Determinism. This is done unconditionally
|
||||
substituteInPlace "Lib/importlib/_bootstrap.py" --replace "source_mtime = int(source_stats['mtime'])" "source_mtime = 1"
|
||||
'' + optionalString (x11Support && (tix != null)) ''
|
||||
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
|
||||
''
|
||||
# Avoid picking up getentropy() from glibc >= 2.25, as that would break
|
||||
# on older kernels. http://bugs.python.org/issue29157
|
||||
+ optionalString stdenv.isLinux
|
||||
''
|
||||
+ optionalString stdenv.isLinux ''
|
||||
substituteInPlace Python/random.c --replace 'defined(HAVE_GETENTROPY)' '0'
|
||||
cat Python/random.c
|
||||
'';
|
||||
'';
|
||||
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
|
||||
|
||||
configureFlags = [
|
||||
"--enable-shared"
|
||||
"--with-threads"
|
||||
"--without-ensurepip"
|
||||
"--with-system-expat"
|
||||
"--with-system-ffi"
|
||||
];
|
||||
|
||||
preConfigure = ''
|
||||
for i in /usr /sw /opt /pkg; do # improve purity
|
||||
@ -74,12 +98,6 @@ in stdenv.mkDerivation {
|
||||
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.6
|
||||
''}
|
||||
|
||||
configureFlagsArray=( --enable-shared --with-threads
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
|
||||
)
|
||||
'';
|
||||
|
||||
setupHook = ./setup-hook.sh;
|
||||
@ -102,6 +120,10 @@ in stdenv.mkDerivation {
|
||||
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
|
||||
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
|
||||
|
||||
# Determinism: Windows installers were not deterministic.
|
||||
# We're also not interested in building Windows installers.
|
||||
find "$out" -name 'wininst*.exe' | xargs -r rm -f
|
||||
|
||||
# Use Python3 as default python
|
||||
ln -s "$out/bin/idle3" "$out/bin/idle"
|
||||
ln -s "$out/bin/pip3" "$out/bin/pip"
|
||||
@ -109,6 +131,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3" "$out/bin/python"
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
{ stdenv, fetchurl, fetchpatch
|
||||
, bzip2
|
||||
, expat
|
||||
, libffi
|
||||
, gdbm
|
||||
, lzma
|
||||
, ncurses
|
||||
@ -32,7 +34,7 @@ let
|
||||
sitePackages = "lib/${libPrefix}/site-packages";
|
||||
|
||||
buildInputs = filter (p: p != null) [
|
||||
zlib bzip2 lzma gdbm sqlite readline ncurses openssl ]
|
||||
zlib bzip2 expat lzma libffi gdbm sqlite readline ncurses openssl ]
|
||||
++ optionals x11Support [ tcl tk libX11 xproto ]
|
||||
++ optionals stdenv.isDarwin [ CF configd ];
|
||||
|
||||
@ -50,6 +52,12 @@ in stdenv.mkDerivation {
|
||||
|
||||
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
|
||||
|
||||
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
|
||||
# This way python doesn't try to update them when we freeze timestamps in nix store.
|
||||
DETERMINISTIC_BUILD=1;
|
||||
# Determinism: We fix the hashes of str, bytes and datetime objects.
|
||||
PYTHONHASHSEED=0;
|
||||
|
||||
prePatch = optionalString stdenv.isDarwin ''
|
||||
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
|
||||
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
|
||||
@ -63,10 +71,27 @@ in stdenv.mkDerivation {
|
||||
})
|
||||
];
|
||||
|
||||
postPatch = optionalString (x11Support && (tix != null)) ''
|
||||
postPatch = ''
|
||||
# Determinism
|
||||
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
|
||||
# Determinism. This is done unconditionally
|
||||
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
|
||||
'' + optionalString (x11Support && (tix != null)) ''
|
||||
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
|
||||
'';
|
||||
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
|
||||
|
||||
configureFlags = [
|
||||
"--enable-shared"
|
||||
"--with-threads"
|
||||
"--without-ensurepip"
|
||||
"--with-system-expat"
|
||||
"--with-system-ffi"
|
||||
];
|
||||
|
||||
preConfigure = ''
|
||||
for i in /usr /sw /opt /pkg; do # improve purity
|
||||
substituteInPlace ./setup.py --replace $i /no-such-path
|
||||
@ -75,12 +100,6 @@ in stdenv.mkDerivation {
|
||||
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.6
|
||||
''}
|
||||
|
||||
configureFlagsArray=( --enable-shared --with-threads
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
|
||||
)
|
||||
'';
|
||||
|
||||
setupHook = ./setup-hook.sh;
|
||||
@ -103,6 +122,10 @@ in stdenv.mkDerivation {
|
||||
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
|
||||
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
|
||||
|
||||
# Determinism: Windows installers were not deterministic.
|
||||
# We're also not interested in building Windows installers.
|
||||
find "$out" -name 'wininst*.exe' | xargs -r rm -f
|
||||
|
||||
# Use Python3 as default python
|
||||
ln -s "$out/bin/idle3" "$out/bin/idle"
|
||||
ln -s "$out/bin/pip3" "$out/bin/pip"
|
||||
@ -110,6 +133,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3" "$out/bin/python"
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
{ stdenv, fetchurl, fetchpatch
|
||||
, glibc
|
||||
, bzip2
|
||||
, expat
|
||||
, libffi
|
||||
, gdbm
|
||||
, lzma
|
||||
, ncurses
|
||||
@ -50,6 +52,12 @@ in stdenv.mkDerivation {
|
||||
|
||||
NIX_LDFLAGS = optionalString stdenv.isLinux "-lgcc_s";
|
||||
|
||||
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
|
||||
# This way python doesn't try to update them when we freeze timestamps in nix store.
|
||||
DETERMINISTIC_BUILD=1;
|
||||
# Determinism: We fix the hashes of str, bytes and datetime objects.
|
||||
PYTHONHASHSEED=0;
|
||||
|
||||
prePatch = optionalString stdenv.isDarwin ''
|
||||
substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
|
||||
substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
|
||||
@ -63,10 +71,27 @@ in stdenv.mkDerivation {
|
||||
})
|
||||
];
|
||||
|
||||
postPatch = optionalString (x11Support && (tix != null)) ''
|
||||
postPatch = ''
|
||||
# Determinism
|
||||
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
|
||||
# Determinism. This is done unconditionally
|
||||
substituteInPlace "Lib/importlib/_bootstrap_external.py" --replace "source_mtime = int(st['mtime'])" "source_mtime = 1"
|
||||
'' + optionalString (x11Support && (tix != null)) ''
|
||||
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
|
||||
'';
|
||||
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}";
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}";
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
|
||||
|
||||
configureFlags = [
|
||||
"--enable-shared"
|
||||
"--with-threads"
|
||||
"--without-ensurepip"
|
||||
"--with-system-expat"
|
||||
"--with-system-ffi"
|
||||
];
|
||||
|
||||
preConfigure = ''
|
||||
for i in /usr /sw /opt /pkg; do # improve purity
|
||||
substituteInPlace ./setup.py --replace $i /no-such-path
|
||||
@ -75,12 +100,6 @@ in stdenv.mkDerivation {
|
||||
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.6
|
||||
''}
|
||||
|
||||
configureFlagsArray=( --enable-shared --with-threads
|
||||
CPPFLAGS="${concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs)}"
|
||||
LDFLAGS="${concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs)}"
|
||||
LIBS="${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}"
|
||||
)
|
||||
'';
|
||||
|
||||
setupHook = ./setup-hook.sh;
|
||||
@ -103,6 +122,10 @@ in stdenv.mkDerivation {
|
||||
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
|
||||
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
|
||||
|
||||
# Determinism: Windows installers were not deterministic.
|
||||
# We're also not interested in building Windows installers.
|
||||
find "$out" -name 'wininst*.exe' | xargs -r rm -f
|
||||
|
||||
# Use Python3 as default python
|
||||
ln -s "$out/bin/idle3" "$out/bin/idle"
|
||||
ln -s "$out/bin/pip3" "$out/bin/pip"
|
||||
@ -110,6 +133,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3" "$out/bin/python"
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
passthru = let
|
||||
|
||||
@ -57,9 +57,12 @@ python.stdenv.mkDerivation (builtins.removeAttrs attrs ["disabled"] // {
|
||||
|
||||
inherit pythonPath;
|
||||
|
||||
# patch python interpreter to write null timestamps when compiling python files
|
||||
# this way python doesn't try to update them when we freeze timestamps in nix store
|
||||
|
||||
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
|
||||
# This way python doesn't try to update them when we freeze timestamps in nix store.
|
||||
DETERMINISTIC_BUILD=1;
|
||||
# Determinism: We fix the hashes of str, bytes and datetime objects.
|
||||
PYTHONHASHSEED = 0;
|
||||
|
||||
buildInputs = [ wrapPython ] ++ buildInputs ++ pythonPath
|
||||
++ [ (ensureNewerSourcesHook { year = "1980"; }) ]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user