Merge pull request #296549 from TomaSajt/strip-java-archives-hook

add stripJavaArchivesHook and use treewide
2024-09-19 20:09:01 +02:00 · 2024-03-19 05:41:08 +01:00 · 2024-03-19 05:41:08 +01:00 · 186c3e34a7
parent ced4e31b4a e08f26cdb1
commit 186c3e34a7
28 changed files with 89 additions and 129 deletions
--- a/doc/languages-frameworks/java.section.md
+++ b/doc/languages-frameworks/java.section.md
@ -4,12 +4,31 @@ Ant-based Java packages are typically built from source as follows:

 ```nix
 stdenv.mkDerivation {
-  name = "...";
+  pname = "...";
+  version = "...";
+
  src = fetchurl { ... };

-  nativeBuildInputs = [ jdk ant ];
+  nativeBuildInputs = [
+    ant
+    jdk
+    stripJavaArchivesHook # removes timestamp metadata from jar files
+  ];

-  buildPhase = "ant";
+  buildPhase = ''
+    runHook preBuild
+    ant # build the project using ant
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    # copy generated jar file(s) to an appropriate location in $out
+    install -Dm644 build/foo.jar $out/share/java/foo.jar
+
+    runHook postInstall
+  '';
 }
 ```

@ -17,6 +36,10 @@ Note that `jdk` is an alias for the OpenJDK (self-built where available,
 or pre-built via Zulu). Platforms with OpenJDK not (yet) in Nixpkgs
 (`Aarch32`, `Aarch64`) point to the (unfree) `oraclejdk`.

+Also note that not using `stripJavaArchivesHook` will likely cause the
+generated `.jar` files to be non-deterministic, which is not optimal.
+Using it, however, does not always guarantee reproducibility.
+
 JAR files that are intended to be used by other packages should be
 installed in `$out/share/java`. JDKs have a stdenv setup hook that add
 any JARs in the `share/java` directories of the build inputs to the
--- a/pkgs/applications/blockchains/bisq-desktop/default.nix
+++ b/pkgs/applications/blockchains/bisq-desktop/default.nix
@ -9,7 +9,7 @@
 , dpkg
 , writeScript
 , bash
-, strip-nondeterminism
+, stripJavaArchivesHook
 , tor
 , zip
 , xz
@ -50,7 +50,7 @@ stdenv.mkDerivation rec {
    dpkg
    imagemagick
    makeWrapper
-    strip-nondeterminism
+    stripJavaArchivesHook
    xz
    zip
    findutils
@ -89,7 +89,6 @@ stdenv.mkDerivation rec {
    tar --sort=name --mtime="@$SOURCE_DATE_EPOCH" -cJf native/linux/x64/tor.tar.xz tor
    tor_jar_file=$(find ./opt/bisq/lib/app -name "tor-binary-linux64-*.jar")
    zip -r $tor_jar_file native
-    strip-nondeterminism ./opt/bisq/lib/app/*.jar
  '';

  installPhase = ''
--- a/pkgs/applications/misc/calcoo/default.nix
+++ b/pkgs/applications/misc/calcoo/default.nix
@ -2,7 +2,7 @@
 , stdenv
 , fetchzip
 , ant
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , jdk
 , makeWrapper
 }:
@ -18,7 +18,7 @@ stdenv.mkDerivation (finalAttrs: {

  nativeBuildInputs = [
    ant
-    canonicalize-jars-hook
+    stripJavaArchivesHook
    jdk
    makeWrapper
  ];
--- a/pkgs/applications/misc/mkgmap/default.nix
+++ b/pkgs/applications/misc/mkgmap/default.nix
@ -6,6 +6,7 @@
 , jre
 , ant
 , makeWrapper
+, stripJavaArchivesHook
 , doCheck ? true
 , withExamples ? false
 }:
@ -30,10 +31,6 @@ stdenv.mkDerivation rec {
  ];

  postPatch = with deps; ''
-    # Fix the output jar timestamps for reproducibility
-    substituteInPlace build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-
    # Manually create version properties file for reproducibility
    mkdir -p build/classes
    cat > build/classes/mkgmap-version.properties << EOF
@ -61,7 +58,7 @@ stdenv.mkDerivation rec {
    '') testInputs}
  '';

-  nativeBuildInputs = [ jdk ant makeWrapper ];
+  nativeBuildInputs = [ jdk ant makeWrapper stripJavaArchivesHook ];

  buildPhase = ''
    runHook preBuild
--- a/pkgs/applications/misc/mkgmap/splitter/default.nix
+++ b/pkgs/applications/misc/mkgmap/splitter/default.nix
@ -6,6 +6,7 @@
 , jre
 , ant
 , makeWrapper
+, stripJavaArchivesHook
 , doCheck ? true
 }:
 let
@ -30,10 +31,6 @@ stdenv.mkDerivation rec {
  ];

  postPatch = with deps; ''
-    # Fix the output jar timestamps for reproducibility
-    substituteInPlace build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-
    # Manually create version properties file for reproducibility
    mkdir -p build/classes
    cat > build/classes/splitter-version.properties << EOF
@ -58,7 +55,7 @@ stdenv.mkDerivation rec {
    '') testInputs}
  '';

-  nativeBuildInputs = [ jdk ant makeWrapper ];
+  nativeBuildInputs = [ jdk ant makeWrapper stripJavaArchivesHook ];

  buildPhase = ''
    runHook preBuild
--- a/pkgs/applications/misc/pattypan/default.nix
+++ b/pkgs/applications/misc/pattypan/default.nix
@ -7,7 +7,7 @@
 , wrapGAppsHook
 , makeDesktopItem
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -27,7 +27,7 @@ stdenv.mkDerivation (finalAttrs: {
    makeWrapper
    wrapGAppsHook
    copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  dontWrapGApps = true;
--- a/pkgs/applications/networking/remote/dayon/default.nix
+++ b/pkgs/applications/networking/remote/dayon/default.nix
@ -6,7 +6,7 @@
 , jre
 , makeWrapper
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -25,7 +25,7 @@ stdenv.mkDerivation (finalAttrs: {
    jdk
    makeWrapper
    copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/applications/office/jameica/default.nix
+++ b/pkgs/applications/office/jameica/default.nix
@ -4,6 +4,7 @@
 , makeDesktopItem
 , makeWrapper
 , wrapGAppsHook
+, stripJavaArchivesHook
 , ant
 , jdk
 , jre
@ -46,13 +47,7 @@ stdenv.mkDerivation rec {
    hash = "sha256-MSVSd5DyVL+dcfTDv1M99hxickPwT2Pt6QGNsu6DGZI=";
  };

-  postPatch = ''
-    # Fix jar timestamps for reproducibility
-    substituteInPlace build/build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
-  nativeBuildInputs = [ ant jdk wrapGAppsHook makeWrapper ];
+  nativeBuildInputs = [ ant jdk wrapGAppsHook makeWrapper stripJavaArchivesHook ];
  buildInputs = lib.optionals stdenv.isLinux [ gtk2 glib libXtst ]
    ++ lib.optional stdenv.isDarwin Cocoa;

--- a/pkgs/applications/science/biology/trimmomatic/default.nix
+++ b/pkgs/applications/science/biology/trimmomatic/default.nix
@ -5,7 +5,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -29,7 +29,7 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/build-support/java/canonicalize-jar.nix
+++ b/pkgs/build-support/java/canonicalize-jar.nix
@ -1,9 +0,0 @@
-{ substituteAll, unzip, zip }:
-
-substituteAll {
-  name = "canonicalize-jar";
-  src = ./canonicalize-jar.sh;
-
-  unzip = "${unzip}/bin/unzip";
-  zip = "${zip}/bin/zip";
-}
--- a/pkgs/build-support/java/canonicalize-jar.sh
+++ b/pkgs/build-support/java/canonicalize-jar.sh
@ -1,29 +0,0 @@
-# Canonicalize the manifest & repack with deterministic timestamps.
-canonicalizeJar() {
-    local input='' outer=''
-    input="$(realpath -sm -- "$1")"
-    outer="$(pwd)"
-    # -qq: even quieter
-    @unzip@ -qq "$input" -d "$input-tmp"
-    canonicalizeJarManifest "$input-tmp/META-INF/MANIFEST.MF"
-    # Sets all timestamps to Jan 1 1980, the earliest mtime zips support.
-    find -- "$input-tmp" -exec touch -t 198001010000.00 {} +
-    rm "$input"
-    pushd "$input-tmp" 2>/dev/null
-    # -q|--quiet, -r|--recurse-paths
-    # -o|--latest-time: canonicalizes overall archive mtime
-    # -X|--no-extra: don't store platform-specific extra file attribute fields
-    @zip@ -qroX "$outer/tmp-out.jar" . 2> /dev/null
-    popd 2>/dev/null
-    rm -rf "$input-tmp"
-    mv "$outer/tmp-out.jar" "$input"
-}
-
-# See also the Java specification's JAR requirements:
-# https://docs.oracle.com/javase/8/docs/technotes/guides/jar/jar.html#Notes_on_Manifest_and_Signature_Files
-canonicalizeJarManifest() {
-    local input=''
-    input="$(realpath -sm -- "$1")"
-    (head -n 1 "$input" && tail -n +2 "$input" | sort | grep -v '^\s*$') > "$input-tmp"
-    mv "$input-tmp" "$input"
-}
--- a/pkgs/build-support/setup-hooks/canonicalize-jars.sh
+++ b/pkgs/build-support/setup-hooks/canonicalize-jars.sh
@ -1,16 +0,0 @@
-# This setup hook causes the fixup phase to repack all JAR files in a
-# canonical & deterministic fashion, e.g. resetting mtimes (like with normal
-# store files) and avoiding impure metadata.
-
-fixupOutputHooks+=('if [ -z "$dontCanonicalizeJars" -a -e "$prefix" ]; then canonicalizeJarsIn "$prefix"; fi')
-
-canonicalizeJarsIn() {
-  local dir="$1"
-  echo "canonicalizing jars in $dir"
-  dir="$(realpath -sm -- "$dir")"
-  while IFS= read -rd '' f; do
-    canonicalizeJar "$f"
-  done < <(find -- "$dir" -type f -name '*.jar' -print0)
-}
-
-source @canonicalize_jar@
--- a/pkgs/build-support/setup-hooks/strip-java-archives.sh
+++ b/pkgs/build-support/setup-hooks/strip-java-archives.sh
@ -0,0 +1,16 @@
+# This setup hook makes the fixup phase to repack all java archives in a
+# deterministic fashion. The most important change being done is the resetting
+# of the modification times of the archive entries
+
+fixupOutputHooks+=('stripJavaArchivesIn $prefix')
+
+stripJavaArchivesIn() {
+    local dir="$1"
+    echo "stripping java archives in $dir"
+    find $dir -type f -regextype posix-egrep -regex ".*\.(jar|war|hpi|apk)$" -print0 |
+    while IFS= read -rd '' f; do
+        echo "stripping java archive $f"
+        strip-nondeterminism --type jar "$f"
+    done
+}
+
--- a/pkgs/by-name/jo/jogl/package.nix
+++ b/pkgs/by-name/jo/jogl/package.nix
@ -5,6 +5,7 @@
 , jdk11
 , git
 , xmlstarlet
+, stripJavaArchivesHook
 , xcbuild
 , udev
 , xorg
@ -42,13 +43,6 @@ stdenv.mkDerivation {
    substituteInPlace gluegen/src/java/com/jogamp/common/util/IOUtil.java \
      --replace-fail '#!/bin/true' '#!${coreutils}/bin/true'
  ''
-  # set timestamp of files in jar to a fixed point in time
-  + ''
-    xmlstarlet ed --inplace \
-      --append //jar --type attr -n modificationtime --value 1980-01-01T00:00Z \
-      gluegen/make/{build.xml,gluegen-cpptasks-base.xml} \
-      jogl/make/{build.xml,build-nativewindow.xml,build-jogl.xml}
-  ''
  # prevent looking for native libraries in /usr/lib
  + ''
    substituteInPlace jogl/make/build-*.xml \
@ -72,6 +66,7 @@ stdenv.mkDerivation {
    jdk11
    git
    xmlstarlet
+    stripJavaArchivesHook
  ] ++ lib.optionals stdenv.isDarwin [
    xcbuild
  ];
--- a/pkgs/development/compilers/abcl/default.nix
+++ b/pkgs/development/compilers/abcl/default.nix
@ -6,7 +6,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 let
@ -30,7 +30,7 @@ stdenv.mkDerivation (finalAttrs: {
    jdk
    fakeHostname
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/development/compilers/jasmin/default.nix
+++ b/pkgs/development/compilers/jasmin/default.nix
@ -5,7 +5,7 @@
 , ant
 , jdk8
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , callPackage
 }:

@ -27,7 +27,7 @@ in stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/development/libraries/freetts/default.nix
+++ b/pkgs/development/libraries/freetts/default.nix
@ -4,6 +4,7 @@
 , ant
 , jdk8
 , sharutils
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -20,16 +21,11 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk8
    sharutils
+    stripJavaArchivesHook
  ];

  sourceRoot = "${finalAttrs.src.name}/freetts-${finalAttrs.version}";

-  postPatch = ''
-    # Fix jar timestamps for reproducibility
-    substituteInPlace build.xml demo.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
  buildPhase = ''
    runHook preBuild

--- a/pkgs/development/libraries/java/cup/default.nix
+++ b/pkgs/development/libraries/java/cup/default.nix
@ -4,7 +4,7 @@
 , ant
 , jdk
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -24,7 +24,7 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/development/libraries/java/hydra-ant-logger/default.nix
+++ b/pkgs/development/libraries/java/hydra-ant-logger/default.nix
@ -3,7 +3,7 @@
 , fetchFromGitHub
 , ant
 , jdk
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation {
@ -20,7 +20,7 @@ stdenv.mkDerivation {
  nativeBuildInputs = [
    ant
    jdk
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/development/libraries/java/swt/default.nix
+++ b/pkgs/development/libraries/java/swt/default.nix
@ -1,6 +1,6 @@
 { lib
 , stdenv
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , fetchzip
 , pkg-config
 , atk
@ -58,7 +58,7 @@ in stdenv.mkDerivation rec {
  };

  nativeBuildInputs = [
-    canonicalize-jars-hook
+    stripJavaArchivesHook
    pkg-config
  ];
  buildInputs = [
--- a/pkgs/development/tools/analysis/jdepend/default.nix
+++ b/pkgs/development/tools/analysis/jdepend/default.nix
@ -4,7 +4,7 @@
 , ant
 , jdk
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -22,7 +22,7 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/development/tools/parsing/javacc/default.nix
+++ b/pkgs/development/tools/parsing/javacc/default.nix
@ -5,7 +5,7 @@
 , jdk
 , jre
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -23,7 +23,7 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/games/prismlauncher/default.nix
+++ b/pkgs/games/prismlauncher/default.nix
@ -1,7 +1,7 @@
 { lib
 , stdenv
 , fetchFromGitHub
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 , cmake
 , cmark
 , Cocoa
@ -40,7 +40,7 @@ stdenv.mkDerivation (finalAttrs: {
    hash = "sha256-4VsoxZzi/EfEsnDvvwzg2xhj7j5B+k3gvaSqwJFDweE=";
  };

-  nativeBuildInputs = [ extra-cmake-modules cmake jdk17 ninja canonicalize-jars-hook ];
+  nativeBuildInputs = [ extra-cmake-modules cmake jdk17 ninja stripJavaArchivesHook ];
  buildInputs =
    [
      qtbase
--- a/pkgs/tools/games/jpsxdec/default.nix
+++ b/pkgs/tools/games/jpsxdec/default.nix
@ -7,7 +7,7 @@
 , makeWrapper
 , makeDesktopItem
 , copyDesktopItems
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -28,7 +28,7 @@ stdenv.mkDerivation (finalAttrs: {
    jdk8
    makeWrapper
    copyDesktopItems
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  buildPhase = ''
--- a/pkgs/tools/misc/ili2c/default.nix
+++ b/pkgs/tools/misc/ili2c/default.nix
@ -5,7 +5,7 @@
 , jdk8
 , jre8
 , makeWrapper
-, canonicalize-jars-hook
+, stripJavaArchivesHook
 }:

 let
@ -20,7 +20,7 @@ stdenv.mkDerivation (finalAttrs: {
    ant
    jdk
    makeWrapper
-    canonicalize-jars-hook
+    stripJavaArchivesHook
  ];

  src = fetchFromGitHub {
--- a/pkgs/tools/typesetting/fop/default.nix
+++ b/pkgs/tools/typesetting/fop/default.nix
@ -5,6 +5,7 @@
 , jdk
 , jre
 , makeWrapper
+, stripJavaArchivesHook
 }:

 stdenv.mkDerivation (finalAttrs: {
@ -16,16 +17,11 @@ stdenv.mkDerivation (finalAttrs: {
    hash = "sha256-b7Av17wu6Ar/npKOiwYqzlvBFSIuXTpqTacM1sxtBvc=";
  };

-  postPatch = ''
-    # Fix jar timestamps for reproducibility
-    substituteInPlace fop/build.xml \
-        --replace-fail '<jar ' '<jar modificationtime="0" '
-  '';
-
  nativeBuildInputs = [
    ant
    jdk
    makeWrapper
+    stripJavaArchivesHook
  ];

  # Note: not sure if this is needed anymore
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@ -147,6 +147,7 @@ mapAliases ({
  callPackage_i686 = pkgsi686Linux.callPackage;
  cadence = throw "cadence has been removed from nixpkgs, as it was archived upstream"; # Added 2023-10-28
  cask = emacs.pkgs.cask; # Added 2022-11-12
+  canonicalize-jars-hook = stripJavaArchivesHook; # Added 2024-03-17
  cargo-embed = throw "cargo-embed is now part of the probe-rs package"; # Added 2023-07-03
  cargo-espflash = espflash;
  cargo-flash = throw "cargo-flash is now part of the probe-rs package"; # Added 2023-07-03
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -212,11 +212,10 @@ with pkgs;
    inherit (darwin.apple_sdk.frameworks) Security;
  };

-  canonicalize-jar = callPackage ../build-support/java/canonicalize-jar.nix { };
-  canonicalize-jars-hook = makeSetupHook {
-    name = "canonicalize-jars-hook";
-    substitutions = { canonicalize_jar = canonicalize-jar; };
-  } ../build-support/setup-hooks/canonicalize-jars.sh;
+  stripJavaArchivesHook = makeSetupHook {
+    name = "strip-java-archives-hook";
+    propagatedBuildInputs = [ strip-nondeterminism ];
+  } ../build-support/setup-hooks/strip-java-archives.sh;

  ensureNewerSourcesHook = { year }: makeSetupHook {
    name = "ensure-newer-sources-hook";