From 82568e67187dcfbdd9536e4fb82cebac90c01ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Mancilla?= Date: Sat, 25 Sep 2021 17:41:16 -0300 Subject: [PATCH] shogun: refactor build Main changes are: - Build with default stdenv instead of forcing GCC 8, by applying upstream patches that fix building with newer compilers. Together with #134390 and #134393 (and the blas library fixes on #135493 and #136535), this enables building the derivation on darwin. - Use multiple outputs. - Move build-only inputs to nativeBuildInputs. - Generate the meta examples but do not compile them (they are compiled as part of the tests), to not make the build take more time than already is. Install the example source files into $doc (but do not install the example binaries, only the sources). - Enable testing by setting the CMake variable CMAKE_SKIP_BUILD_RPATH to false (it should not be needed if #108496 is merged). This allows the tests to locate the shogun library and run without problems. Disable integration and meta tests to speed up the build. With a few other minor changes: - Use upstream URL for the json-c patch. - Be explicit about which optional, commercial dependencies are not used, to stop CMake searching for them. - Do not set NIX_CFLAGS_COMPILE with '-faligned-new' because it causes a warning when compiling C files. Set CXXFLAGS instead. - Remove unnecessary ccache variables from the environment. - Fix wrong Google Mock download location, which was forcing CMake to need network access to download the tarball to the right location (which fails within the sandbox). --- .../machine-learning/shogun/default.nix | 62 ++++++++++++++----- pkgs/top-level/all-packages.nix | 11 +--- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index 55bf4378dde2..06a799b5182e 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -6,6 +6,7 @@ # build , cmake , ctags +, pythonPackages , swig # math , eigen @@ -30,7 +31,6 @@ , colpack # extra support , pythonSupport ? true -, pythonPackages ? null , opencvSupport ? false , opencv ? null , withSvmLight ? false @@ -56,7 +56,8 @@ let sha256 = "05s9dclmk7x5d7wnnj4qr6r6c827m72a44gizcv09lxr28pr9inz"; fetchSubmodules = true; }; - # we need the packed archive + + # The CMake external projects expect the packed archives rxcpp = fetchurl { url = "https://github.com/Reactive-Extensions/RxCpp/archive/v${rxcppVersion}.tar.gz"; sha256 = "0y2isr8dy2n1yjr9c5570kpc9lvdlch6jv0jvw000amwn5d3krsh"; @@ -71,16 +72,33 @@ in stdenv.mkDerivation rec { inherit pname version; + outputs = [ "out" "dev" "doc" ]; + src = srcs.toolbox; patches = [ + # Fix compile errors with json-c + # https://github.com/shogun-toolbox/shogun/pull/4104 (fetchpatch { - url = "https://github.com/awild82/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch"; + url = "https://github.com/shogun-toolbox/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch"; sha256 = "158hqv4xzw648pmjbwrhxjp7qcppqa7kvriif87gn3zdn711c49s"; }) + + # Fix compile errors with GCC 9+ + # https://github.com/shogun-toolbox/shogun/pull/4811 + (fetchpatch { + url = "https://github.com/shogun-toolbox/shogun/commit/c8b670be4790e0f06804b048a6f3d77c17c3ee95.patch"; + sha256 = "sha256-MxsR3Y2noFQevfqWK3nmX5iK4OVWeKBl5tfeDNgjcXk="; + }) + (fetchpatch { + url = "https://github.com/shogun-toolbox/shogun/commit/5aceefd9fb0e2132c354b9a0c0ceb9160cc9b2f7.patch"; + sha256 = "sha256-AgJJKQA8vc5oKaTQDqMdwBR4hT4sn9+uW0jLe7GteJw="; + }) + ] ++ lib.optional (!withSvmLight) ./svmlight-scrubber.patch; - nativeBuildInputs = [ cmake ]; + nativeBuildInputs = [ cmake swig ctags ] + ++ (with pythonPackages; [ python jinja2 ply ]); buildInputs = [ eigen @@ -100,34 +118,37 @@ stdenv.mkDerivation rec { nlopt lp_solve colpack - ctags - swig - ] ++ lib.optionals pythonSupport (with pythonPackages; [ python ply numpy ]) + ] ++ lib.optionals pythonSupport (with pythonPackages; [ python numpy ]) ++ lib.optional opencvSupport opencv; cmakeFlags = let enableIf = cond: if cond then "ON" else "OFF"; in [ - "-DBUILD_META_EXAMPLES=${enableIf doCheck}" - "-DCMAKE_VERBOSE_MAKEFILE=${enableIf doCheck}" + "-DBUILD_META_EXAMPLES=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ARPACK=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ARPREC=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_CPLEX=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_Mosek=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_TFLogger=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ViennaCL=ON" + "-DCMAKE_SKIP_BUILD_RPATH=OFF" + "-DCMAKE_CTEST_ARGUMENTS='--exclude-regex;TrainedModelSerialization'" # Sporadic segfault "-DENABLE_TESTING=${enableIf doCheck}" + "-DDISABLE_META_INTEGRATION_TESTS=ON" + "-DTRAVIS_DISABLE_META_CPP=ON" "-DPythonModular=${enableIf pythonSupport}" "-DOpenCV=${enableIf opencvSupport}" "-DUSE_SVMLIGHT=${enableIf withSvmLight}" ]; - CCACHE_DISABLE="1"; - CCACHE_DIR=".ccache"; + CXXFLAGS = "-faligned-new"; - NIX_CFLAGS_COMPILE="-faligned-new"; - - # broken - doCheck = false; + doCheck = true; postUnpack = '' - mkdir -p $sourceRoot/third_party/{rxcpp,gtest} + mkdir -p $sourceRoot/third_party/{rxcpp,GoogleMock} ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz - ln -s ${srcs.gtest} $sourceRoot/third_party/gtest/release-${gtestVersion}.tar.gz + ln -s ${srcs.gtest} $sourceRoot/third_party/GoogleMock/release-${gtestVersion}.tar.gz ''; postPatch = '' @@ -146,6 +167,13 @@ stdenv.mkDerivation rec { ./scripts/light-scrubber.sh ''; + postInstall = '' + mkdir -p $doc/share/doc/shogun/examples + mv $out/share/shogun/examples/cpp $doc/share/doc/shogun/examples + cp ../examples/undocumented/libshogun/*.cpp $doc/share/doc/shogun/examples/cpp + rm -r $out/share + ''; + meta = with lib; { description = "A toolbox which offers a wide range of efficient and unified machine learning methods"; homepage = "http://shogun-toolbox.org/"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 56c15d52b3d5..397532262356 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -26918,16 +26918,7 @@ with pkgs; shotcut = libsForQt5.callPackage ../applications/video/shotcut { }; shogun = callPackage ../applications/science/machine-learning/shogun { - stdenv = gcc8Stdenv; - - # Workaround for the glibc abi version mismatch. - # Please note that opencv builds are by default disabled. - opencv = opencv3.override { - stdenv = gcc8Stdenv; - openexr = openexr.override { - stdenv = gcc8Stdenv; - }; - }; + opencv = opencv3; }; smplayer = libsForQt5.callPackage ../applications/video/smplayer { };