From 38aba237ad5fc27f5ffa66f3ff1ba648ee37799c Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Thu, 20 Jan 2022 00:28:51 +0100 Subject: [PATCH 1/7] python3Packages.tensorflow-probability: 0.8 -> 0.15.0 tensorflow-probability 0.8 was broken and marked as such (in #108977) because of a dependency on dm-tree, which seems not to have been available. Since dm-tree is now available in nixpkgs (#152971), tensorflow-probability is easy to fix and to upgrade. --- .../tensorflow-probability/default.nix | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow-probability/default.nix b/pkgs/development/python-modules/tensorflow-probability/default.nix index 1daaac6ae7eb..760e9a0e9d1c 100644 --- a/pkgs/development/python-modules/tensorflow-probability/default.nix +++ b/pkgs/development/python-modules/tensorflow-probability/default.nix @@ -1,7 +1,5 @@ { lib , fetchFromGitHub -, fetchpatch -, bazel_0_26 , buildBazelPackage , buildPythonPackage , python @@ -11,47 +9,33 @@ , tensorflow , six , numpy +, dm-tree +, keras , decorator , cloudpickle , gast , hypothesis , scipy +, pandas +, mpmath , matplotlib , mock , pytest }: let - version = "0.8.0"; + version = "0.15.0"; pname = "tensorflow_probability"; # first build all binaries and generate setup.py using bazel bazel-wheel = buildBazelPackage { - bazel = bazel_0_26; - name = "${pname}-${version}-py2.py3-none-any.whl"; - src = fetchFromGitHub { owner = "tensorflow"; repo = "probability"; - rev = version; - sha256 = "07cm8zba8n0ihzdm3k4a4rsg5v62xxsfvcw4h0niz91c0parqjqy"; + rev = "v" + version; + sha256 = "155fgmra90s08vjnp61qxdrpzq74xa3kdzhgdkavwgc25pvxn3mi"; }; - - patches = [ - (fetchpatch { - name = "gast-0.3.patch"; - url = "https://github.com/tensorflow/probability/commit/ae7a9d9771771ec1e7755a3588b9325f050a84cc.patch"; - sha256 = "0kfhx30gshm8f3945na9yjjik71r20qmjzifbigaj4l8dwd9dz1a"; - excludes = ["testing/*"]; - }) - (fetchpatch { - name = "cloudpickle-1.2.patch"; - url = "https://github.com/tensorflow/probability/commit/78ef12b5afe3f567d16c70b74015ed1ddff1b0c8.patch"; - sha256 = "12ms2xcljvvrnig0j78s3wfv4yf3bm5ps4rgfgv5lg2a8mzpc1ga"; - }) - ]; - nativeBuildInputs = [ # needed to create the output wheel in installPhase python @@ -64,7 +48,7 @@ let bazelTarget = ":pip_pkg"; fetchAttrs = { - sha256 = "1qw7vkwnxy45z4vm94isq5m96xiz35sigag7vjg1xb2sklbymxh8"; + sha256 = "0sgxdlw5x3dydy53l10vbrj8smh78b7r1wff8jxcgp4w69mk8zfm"; }; buildAttrs = { @@ -98,34 +82,31 @@ in buildPythonPackage { decorator cloudpickle gast + dm-tree + keras ]; # Listed here: - # https://github.com/tensorflow/probability/blob/f01d27a6f256430f03b14beb14d37def726cb257/testing/run_tests.sh#L58 + # https://github.com/tensorflow/probability/blob/f3777158691787d3658b5e80883fe1a933d48989/testing/dependency_install_lib.sh#L83 checkInputs = [ hypothesis pytest scipy + pandas + mpmath matplotlib mock ]; - # actual checks currently fail because for some reason - # tf.enable_eager_execution is called too late. Probably because upstream - # intents these tests to be run by bazel, not plain pytest. - # checkPhase = '' - # # tests need to import from other test files - # export PYTHONPATH="$PWD/tensorflow-probability:$PYTHONPATH" - # py.test - # ''; + # Ideally, we run unit tests with pytest, but in checkPhase, only the Bazel-build wheel is available. + # But it seems not guaranteed that running the tests with pytest will even work, see + # https://github.com/tensorflow/probability/blob/c2a10877feb2c4c06a4dc58281e69c37a11315b9/CONTRIBUTING.md?plain=1#L69 + # Ideally, tests would be run using Bazel. For now, lets's do a... # sanity check - checkPhase = '' - python -c 'import tensorflow_probability' - ''; + pythonImportsCheck = [ "tensorflow_probability" ]; meta = with lib; { - broken = true; # tf-probability 0.8.0 is not compatible with tensorflow 2.3.2 description = "Library for probabilistic reasoning and statistical analysis"; homepage = "https://www.tensorflow.org/probability/"; license = licenses.asl20; From a43876176fe7df59c793db8b367df26bec8b7084 Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Thu, 20 Jan 2022 13:59:11 +0100 Subject: [PATCH 2/7] python3Packages.tensorflow-tensorboard: disable for Python 3.10 tensorboard is currently not compatible with Python 3.10, but will be in the next release (https://github.com/tensorflow/tensorboard/issues/5478). --- .../python-modules/tensorflow-tensorboard/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix b/pkgs/development/python-modules/tensorflow-tensorboard/default.nix index 123c339fea48..ceebad2a4769 100644 --- a/pkgs/development/python-modules/tensorflow-tensorboard/default.nix +++ b/pkgs/development/python-modules/tensorflow-tensorboard/default.nix @@ -2,6 +2,7 @@ , fetchPypi , buildPythonPackage , pythonOlder +, pythonAtLeast , numpy , wheel , werkzeug @@ -24,7 +25,7 @@ buildPythonPackage rec { pname = "tensorflow-tensorboard"; version = "2.6.0"; format = "wheel"; - disabled = pythonOlder "3.6"; + disabled = pythonOlder "3.6" || pythonAtLeast "3.10"; src = fetchPypi { pname = "tensorboard"; From c7fa9455967f9e8b68fc42d8eb878aab24abed22 Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Thu, 20 Jan 2022 19:46:09 +0100 Subject: [PATCH 3/7] python3Packages.arviz: disable tests requiring TensorFlow Probability The tests in this version of arviz require an older version of TensorFlow Probability than the one packaged in this PR. This also fixes a few missing dependencies. --- .../python-modules/arviz/default.nix | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pkgs/development/python-modules/arviz/default.nix b/pkgs/development/python-modules/arviz/default.nix index 5aa10201917d..61b82d619fde 100644 --- a/pkgs/development/python-modules/arviz/default.nix +++ b/pkgs/development/python-modules/arviz/default.nix @@ -9,11 +9,12 @@ , numpy , pandas , pytest +, cloudpickle , scipy , setuptools -, tensorflow-probability +# , tensorflow-probability (incompatible version) , xarray -#, h5py (used by disabled tests) +, h5py #, pymc3 (broken) #, pyro-ppl (broken) #, pystan (not packaged) @@ -50,23 +51,25 @@ buildPythonPackage rec { emcee numba pytest - tensorflow-probability - #h5py (used by disabled tests) - #pymc3 (broken) - #pyro-ppl (broken) + cloudpickle + #tensorflow-probability (used by disabled tests) + h5py + #pymc3 (broken, used by disabled tests) + #pyro-ppl (broken, used by disabled tests) #pystan (not packaged) - #numpyro (not packaged) + #numpyro (not packaged, used by disabled tests) ]; # check requires pymc3 and pyro-ppl, which are currently broken, and pystan - # and numpyro, which are not yet packaged, some checks also need to make + # and numpyro, which are not yet packaged, and an incompatible (old) version + # of tensorflow-probability. some checks also need to make # directories and do not have permission to do so. So we can only check part # of the package # Additionally, there are some failures with the plots test, which revolve # around attempting to output .mp4 files through an interface that only wants # to output .html files. # The following test have been disabled as a result: data_cmdstanpy, - # data_numpyro, data_pyro, data_pystan, and plots. + # data_numpyro, data_pyro, data_pystan, data_tfp, data_pymc3 and plots. checkPhase = '' cd arviz/tests/ export HOME=$TMPDIR @@ -79,8 +82,7 @@ buildPythonPackage rec { base_tests/test_stats_utils.py \ base_tests/test_utils.py \ external_tests/test_data_cmdstan.py \ - external_tests/test_data_emcee.py \ - external_tests/test_data_tfp.py + external_tests/test_data_emcee.py ''; meta = with lib; { From 7ad9c1316390e4b24cb79af5ab84f5ea191506af Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Thu, 20 Jan 2022 19:49:50 +0100 Subject: [PATCH 4/7] python3Packages.dm-sonnet: mark as broken This version of dm-sonnet depends on an older TensorFlow version than currently packaged. --- pkgs/development/python-modules/dm-sonnet/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/development/python-modules/dm-sonnet/default.nix b/pkgs/development/python-modules/dm-sonnet/default.nix index f465ac200d42..3820fdd1e36c 100644 --- a/pkgs/development/python-modules/dm-sonnet/default.nix +++ b/pkgs/development/python-modules/dm-sonnet/default.nix @@ -86,5 +86,6 @@ in buildPythonPackage { license = licenses.asl20; maintainers = with maintainers; [ timokau ]; platforms = platforms.linux; + broken = true; # depends on older TensorFlow version than is currently packaged }; } From 61642e8b065235ac8adf08e5ff59789ff3f818dd Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Thu, 20 Jan 2022 19:51:13 +0100 Subject: [PATCH 5/7] python3Packages.pymc3: mark as broken Several dependencies are missing and finally, this version requires the theano-pymc3 Python package, which is currently not packaged. --- pkgs/development/python-modules/pymc3/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/development/python-modules/pymc3/default.nix b/pkgs/development/python-modules/pymc3/default.nix index 095f3baf97ae..5cf2615ed031 100644 --- a/pkgs/development/python-modules/pymc3/default.nix +++ b/pkgs/development/python-modules/pymc3/default.nix @@ -68,5 +68,8 @@ buildPythonPackage rec { homepage = "https://github.com/pymc-devs/pymc3"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ ilya-kolpakov ]; + # several dependencies are not declared and in the end it requires theano-pymc3 + # instead of Theano. The former is currently not packaged. + broken = true; }; } From 823b1ec53f795c0e2941ec0a7ba5fbacf1cf542f Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Fri, 21 Jan 2022 15:34:57 +0100 Subject: [PATCH 6/7] python3Packages.arviz: enable numba tests --- pkgs/development/python-modules/arviz/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/arviz/default.nix b/pkgs/development/python-modules/arviz/default.nix index 61b82d619fde..2cde4c6321f9 100644 --- a/pkgs/development/python-modules/arviz/default.nix +++ b/pkgs/development/python-modules/arviz/default.nix @@ -79,8 +79,10 @@ buildPythonPackage rec { base_tests/test_plot_utils.py \ base_tests/test_rcparams.py \ base_tests/test_stats.py \ + base_tests/test_stats_numba.py \ base_tests/test_stats_utils.py \ base_tests/test_utils.py \ + base_tests/test_utils_numba.py \ external_tests/test_data_cmdstan.py \ external_tests/test_data_emcee.py ''; From ecc46f194cdf3e7184622f10af6d8447c46580a5 Mon Sep 17 00:00:00 2001 From: Simeon Carstens Date: Sat, 29 Jan 2022 15:57:53 +0100 Subject: [PATCH 7/7] python3Packages.arviz: patch requirements.txt and add zarr tests arviz v0.11.4 has, according to its requirements.txt, a dependency on typing-extensions < 4. Currently, only typing-extensions v.4.0.1 are available in nixpkgs, while the above version constraint seems unnecessary, see this arviz PR: https://github.com/arviz-devs/arviz/pull/1948/files. This also activates tests that use zarr and to which changes are made in above PR. I thought those changes might be related to unconstraining the typing-extensions version, but looks like they are not. --- pkgs/development/python-modules/arviz/default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkgs/development/python-modules/arviz/default.nix b/pkgs/development/python-modules/arviz/default.nix index 2cde4c6321f9..22d545f04f50 100644 --- a/pkgs/development/python-modules/arviz/default.nix +++ b/pkgs/development/python-modules/arviz/default.nix @@ -12,8 +12,10 @@ , cloudpickle , scipy , setuptools +, typing-extensions # , tensorflow-probability (incompatible version) , xarray +, zarr , h5py #, pymc3 (broken) #, pyro-ppl (broken) @@ -46,12 +48,18 @@ buildPythonPackage rec { scipy ]; + postPatch = '' + substituteInPlace requirements.txt \ + --replace "typing_extensions>=3.7.4.3,<4" "typing_extensions>=3.7.4.3" + ''; + checkInputs = [ bokeh emcee numba pytest cloudpickle + zarr #tensorflow-probability (used by disabled tests) h5py #pymc3 (broken, used by disabled tests) @@ -83,6 +91,7 @@ buildPythonPackage rec { base_tests/test_stats_utils.py \ base_tests/test_utils.py \ base_tests/test_utils_numba.py \ + base_tests/test_data_zarr.py \ external_tests/test_data_cmdstan.py \ external_tests/test_data_emcee.py '';