Merge pull request #166135 from illustris/hadoop

hbase: 0.98.24 -> 2.4.11, spark & hadoop: improve interoperability

nixos/doc/manual/from_md/release-notes/rl-2205.section.xml

@@ -636,6 +636,13 @@
           work.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <literal>hbase</literal> version 0.98.24 has been removed. The
+          package now defaults to version 2.4.11. Versions 1.7.1 and
+          3.0.0-alpha-2 are also available.
+        </para>
+      </listitem>
       <listitem>
         <para>
           <literal>services.paperless-ng</literal> was renamed to

nixos/doc/manual/release-notes/rl-2205.section.md

@@ -227,6 +227,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - `services.ipfs.extraFlags` is now escaped with `utils.escapeSystemdExecArgs`. If you rely on systemd interpolating `extraFlags` in the service `ExecStart`, this will no longer work.
 
+- `hbase` version 0.98.24 has been removed. The package now defaults to version 2.4.11. Versions 1.7.1 and 3.0.0-alpha-2 are also available.
+
 - `services.paperless-ng` was renamed to `services.paperless`. Accordingly, the `paperless-ng-manage` script (located in `dataDir`) was renamed to `paperless-manage`. `services.paperless` now uses `paperless-ngx`.
 
 - The `matrix-synapse` service (`services.matrix-synapse`) has been converted to use the `settings` option defined in RFC42.

nixos/tests/all-tests.nix

@@ -199,6 +199,9 @@ in
   haste-server = handleTest ./haste-server.nix {};
   haproxy = handleTest ./haproxy.nix {};
   hardened = handleTest ./hardened.nix {};
+  hbase1 = handleTest ./hbase.nix { package=pkgs.hbase1; };
+  hbase2 = handleTest ./hbase.nix { package=pkgs.hbase2; };
+  hbase3 = handleTest ./hbase.nix { package=pkgs.hbase3; };
   hedgedoc = handleTest ./hedgedoc.nix {};
   herbstluftwm = handleTest ./herbstluftwm.nix {};
   installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {});

nixos/tests/hbase.nix

@@ -0,0 +1,30 @@
+import ./make-test-python.nix ({ pkgs, lib, package ? pkgs.hbase, ... }:
+{
+  name = "hbase";
+
+  meta = with lib.maintainers; {
+    maintainers = [ illustris ];
+  };
+
+  nodes = {
+    hbase = { pkgs, ... }: {
+      services.hbase = {
+        enable = true;
+        inherit package;
+        # Needed for standalone mode in hbase 2+
+        # This setting and standalone mode are not suitable for production
+        settings."hbase.unsafe.stream.capability.enforce" = "false";
+      };
+      environment.systemPackages = with pkgs; [
+        package
+      ];
+    };
+  };
+
+  testScript = ''
+    start_all()
+    hbase.wait_for_unit("hbase.service")
+    hbase.wait_until_succeeds("echo \"create 't1','f1'\" | sudo -u hbase hbase shell -n")
+    assert "NAME => 'f1'" in hbase.succeed("echo \"describe 't1'\" | sudo -u hbase hbase shell -n")
+  '';
+})

pkgs/applications/networking/cluster/hadoop/default.nix

@@ -17,6 +17,8 @@
 , openssl
 , glibc
 , nixosTests
+, sparkSupport ? true
+, spark
 }:
 
 with lib;
@@ -52,6 +54,9 @@ let
             --prefix PATH : "${makeBinPath [ bash coreutils which]}"\
             --prefix JAVA_LIBRARY_PATH : "${makeLibraryPath buildInputs}"
         done
+      '' + optionalString sparkSupport ''
+        # Add the spark shuffle service jar to YARN
+        cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/lib/${untarDir}/share/hadoop/yarn/
       '' + libPatches;
 
       passthru = { inherit tests; };

pkgs/applications/networking/cluster/spark/default.nix

@@ -6,6 +6,7 @@
 , python3Packages
 , extraPythonPackages ? [ ]
 , coreutils
+, hadoopSupport ? true
 , hadoop
 , RSupport ? true
 , R
@@ -17,12 +18,13 @@ let
   spark = { pname, version, sha256, extraMeta ? {} }:
     stdenv.mkDerivation rec {
       inherit pname version;
+      jdk = if hadoopSupport then hadoop.jdk else jdk8;
       src = fetchzip {
         url = "mirror://apache/spark/${pname}-${version}/${pname}-${version}-bin-without-hadoop.tgz";
         sha256 = sha256;
       };
       nativeBuildInputs = [ makeWrapper ];
-      buildInputs = [ jdk8 python3Packages.python ]
+      buildInputs = [ jdk python3Packages.python ]
         ++ extraPythonPackages
         ++ optional RSupport R;
 
@@ -34,9 +36,11 @@ let
         cp $out/lib/${untarDir}/conf/log4j.properties{.template,}
 
         cat > $out/lib/${untarDir}/conf/spark-env.sh <<- EOF
-        export JAVA_HOME="${jdk8}"
+        export JAVA_HOME="${jdk}"
         export SPARK_HOME="$out/lib/${untarDir}"
+      '' + optionalString hadoopSupport ''
         export SPARK_DIST_CLASSPATH=$(${hadoop}/bin/hadoop classpath)
+      '' + ''
         export PYSPARK_PYTHON="${python3Packages.python}/bin/${python3Packages.python.executable}"
         export PYTHONPATH="\$PYTHONPATH:$PYTHONPATH"
         ${optionalString RSupport ''

pkgs/build-support/fetchurl/mirrors.nix

@@ -18,6 +18,7 @@
 
   # Apache
   apache = [
+    "https://dlcdn.apache.org/"
     "https://www-eu.apache.org/dist/"
     "https://ftp.wayne.edu/apache/"
     "https://www.apache.org/dist/"

pkgs/servers/hbase/default.nix

@@ -1,22 +1,55 @@
-{ lib, stdenv, fetchurl, makeWrapper }:
-stdenv.mkDerivation rec {
-  pname = "hbase";
-  version = "0.98.24";
+{ lib
+, stdenv
+, fetchurl
+, makeWrapper
+, jdk8_headless
+, jdk11_headless
+, nixosTests
+}:
 
-  src = fetchurl {
-    url = "mirror://apache/hbase/${version}/hbase-${version}-hadoop2-bin.tar.gz";
-    sha256 = "0kz72wqsii09v9hxkw10mzyvjhji5sx3l6aijjalgbybavpcxglb";
+let common = { version, hash, jdk ? jdk11_headless, tests }:
+  stdenv.mkDerivation rec {
+    pname = "hbase";
+    inherit version;
+
+    src = fetchurl {
+      url = "mirror://apache/hbase/${version}/hbase-${version}-bin.tar.gz";
+      inherit hash;
+    };
+
+    nativeBuildInputs = [ makeWrapper ];
+    installPhase = ''
+      mkdir -p $out
+      cp -R * $out
+      wrapProgram $out/bin/hbase --set-default JAVA_HOME ${jdk.home}
+    '';
+
+    passthru = { inherit tests; };
+
+    meta = with lib; {
+      description = "A distributed, scalable, big data store";
+      homepage = "https://hbase.apache.org";
+      license = licenses.asl20;
+      maintainers = with lib.maintainers; [ illustris ];
+      platforms = lib.platforms.linux;
+    };
   };
-
-  nativeBuildInputs = [ makeWrapper ];
-  installPhase = ''
-    mkdir -p $out
-    cp -R * $out
-  '';
-  meta = with lib; {
-    description = "A distributed, scalable, big data store";
-    homepage = "https://hbase.apache.org";
-    license = licenses.asl20;
-    platforms = lib.platforms.linux;
+in
+{
+  hbase_1_7 = common {
+    version = "1.7.1";
+    hash = "sha256-DrH2G79QLT8L0YTTmAGC9pUWU8semSaTOsrsQRCI2rY=";
+    jdk = jdk8_headless;
+    tests.standalone = nixosTests.hbase1;
+  };
+  hbase_2_4 = common {
+    version = "2.4.11";
+    hash = "sha256-m0vjUtPaj8czHHh+rQNJJgrFAM744cHd06KE0ut7QeU=";
+    tests.standalone = nixosTests.hbase2;
+  };
+  hbase_3_0 = common {
+    version = "3.0.0-alpha-2";
+    hash = "sha256-QPvgO1BeFWvMT5PdUm/SL92ZgvSvYIuJbzolbBTenz4=";
+    tests.standalone = nixosTests.hbase3;
   };
 }

pkgs/top-level/all-packages.nix

@@ -21414,7 +21414,11 @@ with pkgs;
 
   hasura-cli = callPackage ../servers/hasura/cli.nix { };
 
-  hbase = callPackage ../servers/hbase {};
+  inherit (callPackage ../servers/hbase {}) hbase_1_7 hbase_2_4 hbase_3_0;
+  hbase1 = hbase_1_7;
+  hbase2 = hbase_2_4;
+  hbase3 = hbase_3_0;
+  hbase = hbase2; # when updating, point to the latest stable release
 
   headphones = callPackage ../servers/headphones {};