Merge pull request #84072 from gnprice/python-build

cpython: Use optimizations, for a 25% speedup.

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -12,6 +12,7 @@
 , zlib
 , self
 , configd, coreutils
+, autoreconfHook
 , python-setup-hook
 # Some proprietary libs assume UCS2 unicode, especially on darwin :(
 , ucsEncoding ? 4
@@ -85,6 +86,9 @@ let
       # backported in debian since 2013.
       # https://bugs.python.org/issue13146
       ./atomic_pyc.patch
+
+      # Backport from CPython 3.8 of a good list of tests to run for PGO.
+      ./profile-task.patch
     ] ++ optionals (x11Support && stdenv.isDarwin) [
       ./use-correct-tcl-tk-on-darwin.patch
     ] ++ optionals stdenv.isLinux [
@@ -96,6 +100,13 @@ let
       # libuuid, slowing down program startup a lot).
       ./no-ldconfig.patch
 
+      # Optimize symbol tables for the sake of dynamic linking.
+      # Significant for Python because of extension modules.
+      (fetchpatch {
+        url = "https://salsa.debian.org/cpython-team/python3/-/raw/27103a32e/debian/patches/link-opt.diff";
+        sha256 = "0vp36276ndbrwr7882vg7vjd61c8mv7bqgal6bbh2fimp6zlkdhv";
+      })
+
     ] ++ optionals stdenv.hostPlatform.isCygwin [
       ./2.5.2-ctypes-util-find_library.patch
       ./2.5.2-tkinter-x11.patch
@@ -135,6 +146,7 @@ let
     '';
 
   configureFlags = [
+    "--enable-optimizations"
     "--enable-shared"
     "--with-threads"
     "--enable-unicode=ucs${toString ucsEncoding}"
@@ -182,8 +194,9 @@ let
     ++ optionals x11Support [ tcl tk xlibsWrapper libX11 ]
     ++ optional (stdenv.isDarwin && configd != null) configd;
   nativeBuildInputs =
-    optionals (stdenv.hostPlatform != stdenv.buildPlatform)
+    [ autoreconfHook ]
-    [ buildPackages.stdenv.cc buildPackages.python ];
+    ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform)
+      [ buildPackages.stdenv.cc buildPackages.python ];
 
   mkPaths = paths: {
     C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" paths;

pkgs/development/interpreters/python/cpython/2.7/profile-task.patch

@@ -0,0 +1,21 @@
+Backport from CPython 3.8 of a good list of tests to run for PGO.
+
+Upstream commit:
+  https://github.com/python/cpython/commit/4e16a4a31
+
+Upstream discussion:
+  https://bugs.python.org/issue36044
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index 00fdd21ce..713dc1e53 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -259,7 +259,7 @@ TCLTK_LIBS=
+ # The task to run while instrumented when building the profile-opt target.
+ # We exclude unittests with -x that take a rediculious amount of time to
+ # run in the instrumented training build or do not provide much value.
+-PROFILE_TASK=-m test.regrtest --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess
++PROFILE_TASK=-m test.regrtest --pgo test_array test_base64 test_binascii test_binop test_bisect test_bytes test_bz2 test_cmath test_codecs test_collections test_complex test_dataclasses test_datetime test_decimal test_difflib test_embed test_float test_fstring test_functools test_generators test_hashlib test_heapq test_int test_itertools test_json test_long test_lzma test_math test_memoryview test_operator test_ordered_dict test_pickle test_pprint test_re test_set test_sqlite test_statistics test_struct test_tabnanny test_time test_unicode test_xml_etree test_xml_etree_c
+ 
+ # report files for gcov / lcov coverage report
+ COVERAGE_INFO= $(abs_builddir)/coverage.info

pkgs/development/interpreters/python/cpython/3.5/profile-task.patch

@@ -0,0 +1,21 @@
+Backport from CPython 3.8 of a good list of tests to run for PGO.
+
+Upstream commit:
+  https://github.com/python/cpython/commit/4e16a4a31
+
+Upstream discussion:
+  https://bugs.python.org/issue36044
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index 00fdd21ce..713dc1e53 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -259,7 +259,7 @@ TCLTK_LIBS=
+ # The task to run while instrumented when building the profile-opt target.
+ # We exclude unittests with -x that take a rediculious amount of time to
+ # run in the instrumented training build or do not provide much value.
+-PROFILE_TASK=-m test.regrtest --pgo -x test_asyncore test_gdb test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_main_handling test_multiprocessing_spawn test_subprocess
++PROFILE_TASK=-m test.regrtest --pgo test_array test_base64 test_binascii test_binop test_bisect test_bytes test_bz2 test_cmath test_codecs test_collections test_complex test_dataclasses test_datetime test_decimal test_difflib test_embed test_float test_fstring test_functools test_generators test_hashlib test_heapq test_int test_itertools test_json test_long test_lzma test_math test_memoryview test_operator test_ordered_dict test_pickle test_pprint test_re test_set test_sqlite test_statistics test_struct test_tabnanny test_time test_unicode test_xml_etree test_xml_etree_c
+ 
+ # report files for gcov / lcov coverage report
+ COVERAGE_INFO= $(abs_builddir)/coverage.info

pkgs/development/interpreters/python/cpython/3.6/profile-task.patch

@@ -0,0 +1,21 @@
+Backport from CPython 3.8 of a good list of tests to run for PGO.
+
+Upstream commit:
+  https://github.com/python/cpython/commit/4e16a4a31
+
+Upstream discussion:
+  https://bugs.python.org/issue36044
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index 00fdd21ce..713dc1e53 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -259,7 +259,7 @@ TCLTK_LIBS=
+ # The task to run while instrumented when building the profile-opt target.
+ # We exclude unittests with -x that take a rediculious amount of time to
+ # run in the instrumented training build or do not provide much value.
+-PROFILE_TASK=-m test.regrtest --pgo
++PROFILE_TASK=-m test.regrtest --pgo test_array test_base64 test_binascii test_binop test_bisect test_bytes test_bz2 test_cmath test_codecs test_collections test_complex test_dataclasses test_datetime test_decimal test_difflib test_embed test_float test_fstring test_functools test_generators test_hashlib test_heapq test_int test_itertools test_json test_long test_lzma test_math test_memoryview test_operator test_ordered_dict test_pickle test_pprint test_re test_set test_sqlite test_statistics test_struct test_tabnanny test_time test_unicode test_xml_etree test_xml_etree_c
+ 
+ # report files for gcov / lcov coverage report
+ COVERAGE_INFO= $(abs_builddir)/coverage.info

pkgs/development/interpreters/python/cpython/default.nix

@@ -12,6 +12,7 @@
 , zlib
 , self
 , configd
+, autoreconfHook
 , python-setup-hook
 , nukeReferences
 # For the Python package set
@@ -53,6 +54,7 @@ let
   version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
 
   nativeBuildInputs = [
+    autoreconfHook
     nukeReferences
   ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
     buildPackages.stdenv.cc
@@ -97,6 +99,31 @@ in with passthru; stdenv.mkDerivation {
     # (since it will do a futile invocation of gcc (!) to find
     # libuuid, slowing down program startup a lot).
     (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch")
+  ] ++ optionals stdenv.isLinux [
+    # Optimize symbol tables for the sake of dynamic linking.
+    # Significant for Python because of extension modules.
+    (
+      if pythonAtLeast "3.8" then
+        fetchpatch {
+          url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.8.3rc1-1/debian/patches/link-opt.diff";
+          sha256 = "0va85318nahnqgydwjs7723h8gx41inbdawdy6v4hiykzgc8s7vs";
+        }
+      else if isPy37 then
+        fetchurl {
+          url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.7.6-1/debian/patches/link-opt.diff";
+          sha256 = "1aqvsc0p3sxnfsi8jz7537wl6v95v26ba4nflwvmn5lxlc3y3g13";
+        }
+      else if isPy36 then
+        fetchpatch {
+          url = "https://salsa.debian.org/cpython-team/python3/-/raw/3.6.8-1/debian/patches/link-opt.diff";
+          sha256 = "1nhdrgla75ily9gk7xx0crxa7ynqzks0djxk36sa3lgg5w8vjvyr";
+        }
+      else
+        fetchpatch {
+          url = "https://salsa.debian.org/cpython-team/python3/-/raw/27103a32e/debian/patches/link-opt.diff";
+          sha256 = "0vp36276ndbrwr7882vg7vjd61c8mv7bqgal6bbh2fimp6zlkdhv";
+        }
+    )
   ] ++ optionals (isPy35 || isPy36) [
     # Determinism: Write null timestamps when compiling python files.
     ./3.5/force_bytecode_determinism.patch
@@ -110,6 +137,14 @@ in with passthru; stdenv.mkDerivation {
   ] ++ optionals (isPy37 || isPy38) [
     # Fix darwin build https://bugs.python.org/issue34027
     ./3.7/darwin-libutil.patch
+  ] ++ optionals (pythonOlder "3.8") [
+    # Backport from CPython 3.8 of a good list of tests to run for PGO.
+    (
+      if isPy36 || isPy37 then
+        ./3.6/profile-task.patch
+      else
+        ./3.5/profile-task.patch
+    )
   ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [
     # Fix for http://bugs.python.org/issue1222585
     # Upstream distutils is calling C compiler to compile C++ code, which
@@ -141,11 +176,14 @@ in with passthru; stdenv.mkDerivation {
   PYTHONHASHSEED=0;
 
   configureFlags = [
+    "--enable-optimizations"
     "--enable-shared"
-    "--with-threads"
     "--without-ensurepip"
     "--with-system-expat"
     "--with-system-ffi"
+  ] ++ optionals (pythonOlder "3.7") [
+    # This is unconditionally true starting in CPython 3.7.
+    "--with-threads"
   ] ++ optionals (sqlite != null && isPy3k) [
     "--enable-loadable-sqlite-extensions"
   ] ++ optionals (openssl != null) [