From e9fd4389d2cc2289150d067623c908870ee556e3 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 04:35:36 +0800 Subject: [PATCH 1/7] writeClosure: init, replacing writeReferencesToFile Replace writeReferencesToFile with writeClosure. Make writeClosure accept a list of paths instead of a path. Re-implement with JSON-based exportReferencesGraph interface provided by __structuredAttrs = true. Reword the documentation. Co-authored-by: Valentin Gagarin Co-authored-by: Robert Hensing Co-authored-by: Someone Serge --- .../trivial-build-helpers.chapter.md | 10 +++++--- .../manual/release-notes/rl-2405.section.md | 2 ++ .../trivial-builders/default.nix | 24 +++++++++++-------- .../writeReferenceClosureToFile-mixed.nix | 23 ++++++++++++++++++ pkgs/top-level/stage.nix | 3 ++- 5 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix diff --git a/doc/build-helpers/trivial-build-helpers.chapter.md b/doc/build-helpers/trivial-build-helpers.chapter.md index 02d0a8682bf7..0880fccadc1e 100644 --- a/doc/build-helpers/trivial-build-helpers.chapter.md +++ b/doc/build-helpers/trivial-build-helpers.chapter.md @@ -557,14 +557,18 @@ This creates a derivation with a directory structure like the following: ## `writeReferencesToFile` {#trivial-builder-writeReferencesToFile} -Writes the closure of transitive dependencies to a file. +Deprecated. Use [`writeClosure`](#trivial-builder-writeClosure) instead. -This produces the equivalent of `nix-store -q --requisites`. +## `writeClosure` {#trivial-builder-writeClosure} + +Given a list of [store paths](https://nixos.org/manual/nix/stable/glossary#gloss-store-path) (or string-like expressions coercible to store paths), write their collective [closure](https://nixos.org/manual/nix/stable/glossary#gloss-closure) to a text file. + +The result is equivalent to the output of `nix-store -q --requisites`. For example, ```nix -writeReferencesToFile (writeScriptBin "hi" ''${hello}/bin/hello'') +writeClosure [ (writeScriptBin "hi" ''${hello}/bin/hello'') ] ``` produces an output path `/nix/store/-runtime-deps` containing diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md index 22689868cf02..8bce8dcda737 100644 --- a/nixos/doc/manual/release-notes/rl-2405.section.md +++ b/nixos/doc/manual/release-notes/rl-2405.section.md @@ -167,6 +167,8 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m - Invidious has changed its default database username from `kemal` to `invidious`. Setups involving an externally provisioned database (i.e. `services.invidious.database.createLocally == false`) should adjust their configuration accordingly. The old `kemal` user will not be removed automatically even when the database is provisioned automatically.(https://github.com/NixOS/nixpkgs/pull/265857) +- `writeReferencesToFile` is deprecated in favour of the new trivial build helper `writeClosure`. The latter accepts a list of paths and has an unambiguous name and cleaner implementation. + - `inetutils` now has a lower priority to avoid shadowing the commonly used `util-linux`. If one wishes to restore the default priority, simply use `lib.setPrio 5 inetutils` or override with `meta.priority = 5`. - `paperless`' `services.paperless.extraConfig` setting has been removed and converted to the freeform type and option named `services.paperless.settings`. diff --git a/pkgs/build-support/trivial-builders/default.nix b/pkgs/build-support/trivial-builders/default.nix index df81d67d868d..d7438923a54b 100644 --- a/pkgs/build-support/trivial-builders/default.nix +++ b/pkgs/build-support/trivial-builders/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }: +{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }: let inherit (lib) @@ -625,18 +625,22 @@ rec { # Docs in doc/build-helpers/trivial-build-helpers.chapter.md # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile - writeReferencesToFile = path: runCommand "runtime-deps" + # TODO: Convert to throw after Nixpkgs 24.05 branch-off. + writeReferencesToFile = (if config.allowAliases then lib.warn else throw) + "writeReferencesToFile is deprecated in favour of writeClosure" + (path: writeClosure [ path ]); + + # Docs in doc/build-helpers/trivial-build-helpers.chapter.md + # See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure + writeClosure = paths: runCommand "runtime-deps" { - exportReferencesGraph = [ "graph" path ]; + # Get the cleaner exportReferencesGraph interface + __structuredAttrs = true; + exportReferencesGraph.graph = paths; + nativeBuildInputs = [ jq ]; } '' - touch $out - while read path; do - echo $path >> $out - read dummy - read nrRefs - for ((i = 0; i < nrRefs; i++)); do read ref; done - done < graph + jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out" ''; # Docs in doc/build-helpers/trivial-build-helpers.chapter.md diff --git a/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix b/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix new file mode 100644 index 000000000000..fed3a4f2adbc --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix @@ -0,0 +1,23 @@ +{ lib +, runCommandLocal + # Test targets +, writeClosure +, samples +}: +runCommandLocal "test-trivial-builders-writeClosure-mixed" { + __structuredAttrs = true; + references = lib.mapAttrs (n: v: writeClosure [ v ]) samples; + allRefs = writeClosure (lib.attrValues samples); + inherit samples; + meta.maintainers = with lib.maintainers; [ + ShamrockLee + ]; +} '' + set -eu -o pipefail + echo >&2 Testing mixed closures... + echo >&2 Checking all samples "(''${samples[*]})" "$allRefs" + diff -U3 \ + <(sort <"$allRefs") \ + <(cat "''${references[@]}" | sort | uniq) + touch "$out" +'' diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index d8bf53af86aa..3b84026b4696 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -110,8 +110,9 @@ let trivialBuilders = self: super: import ../build-support/trivial-builders { inherit lib; + inherit (self) config; inherit (self) runtimeShell stdenv stdenvNoCC; - inherit (self.pkgsBuildHost) shellcheck-minimal; + inherit (self.pkgsBuildHost) jq shellcheck-minimal; inherit (self.pkgsBuildHost.xorg) lndir; }; From 8a8d07bc15387c1789e1630e3c65da3258bad382 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Wed, 29 Nov 2023 01:24:22 +0800 Subject: [PATCH 2/7] tests.trivial-builders.references: test writeClosure Test writeClosure instead of writeReferencesToFile. Add multiple-path test for writeReferenceClosureToFile. Rename variables: - references -> closures (passthru affected) - REFERENCES -> CLOSURES --- .../trivial-builders/test/default.nix | 3 ++ .../test/references/default.nix | 16 ++++++---- .../test/references/references-test.sh | 29 ++++++++++++------- .../test/writeClosure-union.nix | 23 +++++++++++++++ .../writeReferenceClosureToFile-mixed.nix | 23 --------------- 5 files changed, 54 insertions(+), 40 deletions(-) create mode 100644 pkgs/build-support/trivial-builders/test/writeClosure-union.nix delete mode 100644 pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix diff --git a/pkgs/build-support/trivial-builders/test/default.nix b/pkgs/build-support/trivial-builders/test/default.nix index 59dbba3f1841..f41372d922bb 100644 --- a/pkgs/build-support/trivial-builders/test/default.nix +++ b/pkgs/build-support/trivial-builders/test/default.nix @@ -26,6 +26,9 @@ recurseIntoAttrs { then references else {}; writeCBin = callPackage ./writeCBin.nix {}; + writeClosure-union = callPackage ./writeClosure-union.nix { + inherit (references) samples; + }; writeShellApplication = callPackage ./writeShellApplication.nix {}; writeScriptBin = callPackage ./writeScriptBin.nix {}; writeShellScript = callPackage ./write-shell-script.nix {}; diff --git a/pkgs/build-support/trivial-builders/test/references/default.nix b/pkgs/build-support/trivial-builders/test/references/default.nix index 3e21c905321e..928cc1d9461f 100644 --- a/pkgs/build-support/trivial-builders/test/references/default.nix +++ b/pkgs/build-support/trivial-builders/test/references/default.nix @@ -12,7 +12,7 @@ , cleanSamples ? lib.filterAttrs (n: lib.isStringLike) # Test targets , writeDirectReferencesToFile -, writeReferencesToFile +, writeClosure }: # -------------------------------------------------------------------------- # @@ -46,8 +46,9 @@ let samplesToString = attrs: lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs); - references = lib.mapAttrs (n: v: writeReferencesToFile v) samples; + closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples; directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples; + collectiveClosure = writeClosure (lib.attrValues samples); testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: { name = "references-test"; @@ -61,8 +62,9 @@ let mkdir -p "$out/bin" substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \ --replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \ - --replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \ - --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} + --replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \ + --replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \ + --replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure} runHook postInstall chmod +x "$out/bin/${finalAttrs.meta.mainProgram}" ''; @@ -79,8 +81,9 @@ let passthru = { inherit + collectiveClosure directReferences - references + closures samples ; }; @@ -109,8 +112,9 @@ testers.nixosTest { ''; passthru = { inherit + collectiveClosure directReferences - references + closures samples testScriptBin ; diff --git a/pkgs/build-support/trivial-builders/test/references/references-test.sh b/pkgs/build-support/trivial-builders/test/references/references-test.sh index 1b8f8e1504ec..92e4467287ed 100755 --- a/pkgs/build-support/trivial-builders/test/references/references-test.sh +++ b/pkgs/build-support/trivial-builders/test/references/references-test.sh @@ -33,16 +33,17 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" # nixpkgs root - # Injected by Nix (to avoid evaluating in a derivation) - # turn them into arrays - # shellcheck disable=SC2206 # deliberately unquoted - declare -A samples=( @SAMPLES@ ) - # shellcheck disable=SC2206 # deliberately unquoted - declare -A directRefs=( @DIRECT_REFS@ ) - # shellcheck disable=SC2206 # deliberately unquoted - declare -A references=( @REFERENCES@ ) + # Inject the path to compare from the Nix expression -echo >&2 Testing direct references... + # Associative Arrays + declare -A samples=( @SAMPLES@ ) + declare -A directRefs=( @DIRECT_REFS@ ) + declare -A closures=( @CLOSURES@ ) + + # Path string + collectiveClosure=@COLLECTIVE_CLOSURE@ + +echo >&2 Testing direct closures... for i in "${!samples[@]}"; do echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}" diff -U3 \ @@ -52,10 +53,16 @@ done echo >&2 Testing closure... for i in "${!samples[@]}"; do - echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}" + echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}" diff -U3 \ - <(sort <"${references[$i]}") \ + <(sort <"${closures[$i]}") \ <(nix-store -q --requisites "${samples[$i]}" | sort) done +echo >&2 Testing mixed closures... +echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure" +diff -U3 \ + <(sort <"$collectiveClosure") \ + <(nix-store -q --requisites "${samples[@]}" | sort) + echo 'OK!' diff --git a/pkgs/build-support/trivial-builders/test/writeClosure-union.nix b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix new file mode 100644 index 000000000000..92a2bf9f0988 --- /dev/null +++ b/pkgs/build-support/trivial-builders/test/writeClosure-union.nix @@ -0,0 +1,23 @@ +{ lib +, runCommandLocal + # Test targets +, writeClosure +, samples +}: +runCommandLocal "test-trivial-builders-writeClosure-union" { + __structuredAttrs = true; + closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples; + collectiveClosure = writeClosure (lib.attrValues samples); + inherit samples; + meta.maintainers = with lib.maintainers; [ + ShamrockLee + ]; +} '' + set -eu -o pipefail + echo >&2 Testing mixed closures... + echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure" + diff -U3 \ + <(sort <"$collectiveClosure") \ + <(cat "''${closures[@]}" | sort | uniq) + touch "$out" +'' diff --git a/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix b/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix deleted file mode 100644 index fed3a4f2adbc..000000000000 --- a/pkgs/build-support/trivial-builders/test/writeReferenceClosureToFile-mixed.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ lib -, runCommandLocal - # Test targets -, writeClosure -, samples -}: -runCommandLocal "test-trivial-builders-writeClosure-mixed" { - __structuredAttrs = true; - references = lib.mapAttrs (n: v: writeClosure [ v ]) samples; - allRefs = writeClosure (lib.attrValues samples); - inherit samples; - meta.maintainers = with lib.maintainers; [ - ShamrockLee - ]; -} '' - set -eu -o pipefail - echo >&2 Testing mixed closures... - echo >&2 Checking all samples "(''${samples[*]})" "$allRefs" - diff -U3 \ - <(sort <"$allRefs") \ - <(cat "''${references[@]}" | sort | uniq) - touch "$out" -'' From 97a7ce7aa63aee1e3d5991a7db4c41e06a28db24 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 08:24:31 +0800 Subject: [PATCH 3/7] nixosTests.nixops: remove dead code from Nix expression --- nixos/tests/nixops/default.nix | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/nixos/tests/nixops/default.nix b/nixos/tests/nixops/default.nix index 8477e5059fca..6468b8c38224 100644 --- a/nixos/tests/nixops/default.nix +++ b/nixos/tests/nixops/default.nix @@ -93,23 +93,5 @@ let inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey; - /* - Return a store path with a closure containing everything including - derivations and all build dependency outputs, all the way down. - */ - allDrvOutputs = pkg: - let name = "allDrvOutputs-${pkg.pname or pkg.name or "unknown"}"; - in - pkgs.runCommand name { refs = pkgs.writeReferencesToFile pkg.drvPath; } '' - touch $out - while read ref; do - case $ref in - *.drv) - cat $ref >>$out - ;; - esac - done <$refs - ''; - in tests From e28ad1a0a31395783c5f813f52346b7405ae5d87 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 08:28:24 +0800 Subject: [PATCH 4/7] referenceByPopularity: rename in comment writeReferencesToFile -> writeClosure --- pkgs/build-support/references-by-popularity/closure-graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/references-by-popularity/closure-graph.py b/pkgs/build-support/references-by-popularity/closure-graph.py index 579f3b041fa8..4f8efd42ed81 100644 --- a/pkgs/build-support/references-by-popularity/closure-graph.py +++ b/pkgs/build-support/references-by-popularity/closure-graph.py @@ -8,8 +8,8 @@ # and how deep in the tree they live. Equally-"popular" paths are then # sorted by name. # -# The existing writeReferencesToFile prints the paths in a simple -# ascii-based sorting of the paths. +# The existing writeClosure prints the paths in a simple ascii-based +# sorting of the paths. # # Sorting the paths by graph improves the chances that the difference # between two builds appear near the end of the list, instead of near From 5c1a3b619db5a664977f60cddd3280355d3888a5 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 08:33:22 +0800 Subject: [PATCH 5/7] singularity-tools.buildImages: use writeClosure --- pkgs/build-support/singularity-tools/default.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkgs/build-support/singularity-tools/default.nix b/pkgs/build-support/singularity-tools/default.nix index 8d7ad9e742a1..c9e53a4cb706 100644 --- a/pkgs/build-support/singularity-tools/default.nix +++ b/pkgs/build-support/singularity-tools/default.nix @@ -4,7 +4,7 @@ , storeDir ? builtins.storeDir , writeScript , singularity -, writeReferencesToFile +, writeClosure , bash , vmTools , gawk @@ -50,18 +50,13 @@ rec { }: let projectName = singularity.projectName or "singularity"; - layer = mkLayer { - inherit name; - contents = contents ++ [ bash runScriptFile ]; - inherit projectName; - }; runAsRootFile = shellScript "run-as-root.sh" runAsRoot; runScriptFile = shellScript "run-script.sh" runScript; result = vmTools.runInLinuxVM ( runCommand "${projectName}-image-${name}.img" { buildInputs = [ singularity e2fsprogs util-linux gawk ]; - layerClosure = writeReferencesToFile layer; + layerClosure = writeClosure contents; preVM = vmTools.createEmptyImage { size = diskSize; fullName = "${projectName}-run-disk"; From 67ec1a7d7b88f6afdb2cfadb0f2c909d40a41627 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 08:38:19 +0800 Subject: [PATCH 6/7] dockerTools.buildImage: writeReferencesToFile -> writeClosure --- pkgs/build-support/docker/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index ea464ebea18c..8e04944bc810 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -29,7 +29,7 @@ , tarsum , util-linux , vmTools -, writeReferencesToFile +, writeClosure , writeScript , writeShellScriptBin , writeText @@ -630,7 +630,7 @@ rec { imageName = lib.toLower name; imageTag = lib.optionalString (tag != null) tag; inherit fromImage baseJson; - layerClosure = writeReferencesToFile layer; + layerClosure = writeClosure [ layer ]; passthru.buildArgs = args; passthru.layer = layer; passthru.imageTag = From 112c3d5ecd3ab41e3f60de90d7d559503d516423 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Fri, 15 Mar 2024 08:40:18 +0800 Subject: [PATCH 7/7] ociTools.buildContainer: writeReferencesToFile -> writeClosure --- pkgs/build-support/oci-tools/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix index 67e081522d64..1f5507f2eb75 100644 --- a/pkgs/build-support/oci-tools/default.nix +++ b/pkgs/build-support/oci-tools/default.nix @@ -1,4 +1,4 @@ -{ lib, writeText, runCommand, writeReferencesToFile }: +{ lib, writeText, runCommand, writeClosure }: { buildContainer = @@ -72,7 +72,7 @@ set -o pipefail mkdir -p $out/rootfs/{dev,proc,sys} cp ${config} $out/config.json - xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/ + xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/ ''; }