diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index d49a997918c5..650664efcc55 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -636,6 +636,13 @@ work. + + + hbase version 0.98.24 has been removed. The + package now defaults to version 2.4.11. Versions 1.7.1 and + 3.0.0-alpha-2 are also available. + + services.paperless-ng was renamed to diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index a5bcd09211c0..1c6011f14721 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -227,6 +227,8 @@ In addition to numerous new and upgraded packages, this release has the followin - `services.ipfs.extraFlags` is now escaped with `utils.escapeSystemdExecArgs`. If you rely on systemd interpolating `extraFlags` in the service `ExecStart`, this will no longer work. +- `hbase` version 0.98.24 has been removed. The package now defaults to version 2.4.11. Versions 1.7.1 and 3.0.0-alpha-2 are also available. + - `services.paperless-ng` was renamed to `services.paperless`. Accordingly, the `paperless-ng-manage` script (located in `dataDir`) was renamed to `paperless-manage`. `services.paperless` now uses `paperless-ngx`. - The `matrix-synapse` service (`services.matrix-synapse`) has been converted to use the `settings` option defined in RFC42. diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index cb68ef685300..9848487fd134 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -199,6 +199,9 @@ in haste-server = handleTest ./haste-server.nix {}; haproxy = handleTest ./haproxy.nix {}; hardened = handleTest ./hardened.nix {}; + hbase1 = handleTest ./hbase.nix { package=pkgs.hbase1; }; + hbase2 = handleTest ./hbase.nix { package=pkgs.hbase2; }; + hbase3 = handleTest ./hbase.nix { package=pkgs.hbase3; }; hedgedoc = handleTest ./hedgedoc.nix {}; herbstluftwm = handleTest ./herbstluftwm.nix {}; installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {}); diff --git a/nixos/tests/hbase.nix b/nixos/tests/hbase.nix new file mode 100644 index 000000000000..a449d24dd6fd --- /dev/null +++ b/nixos/tests/hbase.nix @@ -0,0 +1,30 @@ +import ./make-test-python.nix ({ pkgs, lib, package ? pkgs.hbase, ... }: +{ + name = "hbase"; + + meta = with lib.maintainers; { + maintainers = [ illustris ]; + }; + + nodes = { + hbase = { pkgs, ... }: { + services.hbase = { + enable = true; + inherit package; + # Needed for standalone mode in hbase 2+ + # This setting and standalone mode are not suitable for production + settings."hbase.unsafe.stream.capability.enforce" = "false"; + }; + environment.systemPackages = with pkgs; [ + package + ]; + }; + }; + + testScript = '' + start_all() + hbase.wait_for_unit("hbase.service") + hbase.wait_until_succeeds("echo \"create 't1','f1'\" | sudo -u hbase hbase shell -n") + assert "NAME => 'f1'" in hbase.succeed("echo \"describe 't1'\" | sudo -u hbase hbase shell -n") + ''; +}) diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix index adb46540cba7..202b9fd5e68d 100644 --- a/pkgs/applications/networking/cluster/hadoop/default.nix +++ b/pkgs/applications/networking/cluster/hadoop/default.nix @@ -17,6 +17,8 @@ , openssl , glibc , nixosTests +, sparkSupport ? true +, spark }: with lib; @@ -52,6 +54,9 @@ let --prefix PATH : "${makeBinPath [ bash coreutils which]}"\ --prefix JAVA_LIBRARY_PATH : "${makeLibraryPath buildInputs}" done + '' + optionalString sparkSupport '' + # Add the spark shuffle service jar to YARN + cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/lib/${untarDir}/share/hadoop/yarn/ '' + libPatches; passthru = { inherit tests; }; diff --git a/pkgs/applications/networking/cluster/spark/default.nix b/pkgs/applications/networking/cluster/spark/default.nix index 057be75aeb34..a3ca42dff342 100644 --- a/pkgs/applications/networking/cluster/spark/default.nix +++ b/pkgs/applications/networking/cluster/spark/default.nix @@ -6,6 +6,7 @@ , python3Packages , extraPythonPackages ? [ ] , coreutils +, hadoopSupport ? true , hadoop , RSupport ? true , R @@ -17,12 +18,13 @@ let spark = { pname, version, sha256, extraMeta ? {} }: stdenv.mkDerivation rec { inherit pname version; + jdk = if hadoopSupport then hadoop.jdk else jdk8; src = fetchzip { url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz"; sha256 = sha256; }; nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ jdk8 python3Packages.python ] + buildInputs = [ jdk python3Packages.python ] ++ extraPythonPackages ++ optional RSupport R; @@ -34,9 +36,11 @@ let cp $out/lib/${untarDir}/conf/log4j.properties{.template,} cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF - export JAVA_HOME="${jdk8}" + export JAVA_HOME="${jdk}" export SPARK_HOME="$out/lib/${untarDir}" + '' + optionalString hadoopSupport '' export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath) + '' + '' export PYSPARK_PYTHON="${python3Packages.python}/bin/${python3Packages.python.executable}" export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH" ${optionalString RSupport '' diff --git a/pkgs/build-support/fetchurl/mirrors.nix b/pkgs/build-support/fetchurl/mirrors.nix index 954c88d1bafa..2c6ba2b02448 100644 --- a/pkgs/build-support/fetchurl/mirrors.nix +++ b/pkgs/build-support/fetchurl/mirrors.nix @@ -18,6 +18,7 @@ # Apache apache = [ + "https://dlcdn.apache.org/" "https://www-eu.apache.org/dist/" "https://ftp.wayne.edu/apache/" "https://www.apache.org/dist/" diff --git a/pkgs/servers/hbase/default.nix b/pkgs/servers/hbase/default.nix index 32d2c09de97f..46521da88539 100644 --- a/pkgs/servers/hbase/default.nix +++ b/pkgs/servers/hbase/default.nix @@ -1,22 +1,55 @@ -{ lib, stdenv, fetchurl, makeWrapper }: -stdenv.mkDerivation rec { - pname = "hbase"; - version = "0.98.24"; +{ lib +, stdenv +, fetchurl +, makeWrapper +, jdk8_headless +, jdk11_headless +, nixosTests +}: - src = fetchurl { - url = "mirror://apache/hbase/${version}/hbase-${version}-hadoop2-bin.tar.gz"; - sha256 = "0kz72wqsii09v9hxkw10mzyvjhji5sx3l6aijjalgbybavpcxglb"; +let common = { version, hash, jdk ? jdk11_headless, tests }: + stdenv.mkDerivation rec { + pname = "hbase"; + inherit version; + + src = fetchurl { + url = "mirror://apache/hbase/${version}/hbase-${version}-bin.tar.gz"; + inherit hash; + }; + + nativeBuildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out + cp -R * $out + wrapProgram $out/bin/hbase --set-default JAVA_HOME ${jdk.home} + ''; + + passthru = { inherit tests; }; + + meta = with lib; { + description = "A distributed, scalable, big data store"; + homepage = "https://hbase.apache.org"; + license = licenses.asl20; + maintainers = with lib.maintainers; [ illustris ]; + platforms = lib.platforms.linux; + }; }; - - nativeBuildInputs = [ makeWrapper ]; - installPhase = '' - mkdir -p $out - cp -R * $out - ''; - meta = with lib; { - description = "A distributed, scalable, big data store"; - homepage = "https://hbase.apache.org"; - license = licenses.asl20; - platforms = lib.platforms.linux; +in +{ + hbase_1_7 = common { + version = "1.7.1"; + hash = "sha256-DrH2G79QLT8L0YTTmAGC9pUWU8semSaTOsrsQRCI2rY="; + jdk = jdk8_headless; + tests.standalone = nixosTests.hbase1; + }; + hbase_2_4 = common { + version = "2.4.11"; + hash = "sha256-m0vjUtPaj8czHHh+rQNJJgrFAM744cHd06KE0ut7QeU="; + tests.standalone = nixosTests.hbase2; + }; + hbase_3_0 = common { + version = "3.0.0-alpha-2"; + hash = "sha256-QPvgO1BeFWvMT5PdUm/SL92ZgvSvYIuJbzolbBTenz4="; + tests.standalone = nixosTests.hbase3; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 71198968e779..b674e03b1aab 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -21414,7 +21414,11 @@ with pkgs; hasura-cli = callPackage ../servers/hasura/cli.nix { }; - hbase = callPackage ../servers/hbase {}; + inherit (callPackage ../servers/hbase {}) hbase_1_7 hbase_2_4 hbase_3_0; + hbase1 = hbase_1_7; + hbase2 = hbase_2_4; + hbase3 = hbase_3_0; + hbase = hbase2; # when updating, point to the latest stable release headphones = callPackage ../servers/headphones {};