Merge pull request #178717 from ShamrockLee/write-multiple-references

trivial-builders: replace writeReferencesToFile with writeClosure
This commit is contained in:
Someone 2024-03-19 08:57:20 +00:00 committed by GitHub
commit 63709965b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 87 additions and 62 deletions

View file

@ -658,14 +658,18 @@ This creates a derivation with a directory structure like the following:
## `writeReferencesToFile` {#trivial-builder-writeReferencesToFile}
Writes the closure of transitive dependencies to a file.
Deprecated. Use [`writeClosure`](#trivial-builder-writeClosure) instead.
This produces the equivalent of `nix-store -q --requisites`.
## `writeClosure` {#trivial-builder-writeClosure}
Given a list of [store paths](https://nixos.org/manual/nix/stable/glossary#gloss-store-path) (or string-like expressions coercible to store paths), write their collective [closure](https://nixos.org/manual/nix/stable/glossary#gloss-closure) to a text file.
The result is equivalent to the output of `nix-store -q --requisites`.
For example,
```nix
writeReferencesToFile (writeScriptBin "hi" ''${hello}/bin/hello'')
writeClosure [ (writeScriptBin "hi" ''${hello}/bin/hello'') ]
```
produces an output path `/nix/store/<hash>-runtime-deps` containing

View file

@ -171,6 +171,8 @@ The pre-existing [services.ankisyncd](#opt-services.ankisyncd.enable) has been m
- Invidious has changed its default database username from `kemal` to `invidious`. Setups involving an externally provisioned database (i.e. `services.invidious.database.createLocally == false`) should adjust their configuration accordingly. The old `kemal` user will not be removed automatically even when the database is provisioned automatically.(https://github.com/NixOS/nixpkgs/pull/265857)
- `writeReferencesToFile` is deprecated in favour of the new trivial build helper `writeClosure`. The latter accepts a list of paths and has an unambiguous name and cleaner implementation.
- `inetutils` now has a lower priority to avoid shadowing the commonly used `util-linux`. If one wishes to restore the default priority, simply use `lib.setPrio 5 inetutils` or override with `meta.priority = 5`.
- `paperless`' `services.paperless.extraConfig` setting has been removed and converted to the freeform type and option named `services.paperless.settings`.

View file

@ -93,23 +93,5 @@ let
inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey;
/*
Return a store path with a closure containing everything including
derivations and all build dependency outputs, all the way down.
*/
allDrvOutputs = pkg:
let name = "allDrvOutputs-${pkg.pname or pkg.name or "unknown"}";
in
pkgs.runCommand name { refs = pkgs.writeReferencesToFile pkg.drvPath; } ''
touch $out
while read ref; do
case $ref in
*.drv)
cat $ref >>$out
;;
esac
done <$refs
'';
in
tests

View file

@ -29,7 +29,7 @@
, tarsum
, util-linux
, vmTools
, writeReferencesToFile
, writeClosure
, writeScript
, writeShellScriptBin
, writeText
@ -630,7 +630,7 @@ rec {
imageName = lib.toLower name;
imageTag = lib.optionalString (tag != null) tag;
inherit fromImage baseJson;
layerClosure = writeReferencesToFile layer;
layerClosure = writeClosure [ layer ];
passthru.buildArgs = args;
passthru.layer = layer;
passthru.imageTag =

View file

@ -1,4 +1,4 @@
{ lib, writeText, runCommand, writeReferencesToFile }:
{ lib, writeText, runCommand, writeClosure }:
{
buildContainer =
@ -72,7 +72,7 @@
set -o pipefail
mkdir -p $out/rootfs/{dev,proc,sys}
cp ${config} $out/config.json
xargs tar c < ${writeReferencesToFile args} | tar -xC $out/rootfs/
xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/
'';
}

View file

@ -8,8 +8,8 @@
# and how deep in the tree they live. Equally-"popular" paths are then
# sorted by name.
#
# The existing writeReferencesToFile prints the paths in a simple
# ascii-based sorting of the paths.
# The existing writeClosure prints the paths in a simple ascii-based
# sorting of the paths.
#
# Sorting the paths by graph improves the chances that the difference
# between two builds appear near the end of the list, instead of near

View file

@ -4,7 +4,7 @@
, storeDir ? builtins.storeDir
, writeScript
, singularity
, writeReferencesToFile
, writeClosure
, bash
, vmTools
, gawk
@ -50,18 +50,13 @@ rec {
}:
let
projectName = singularity.projectName or "singularity";
layer = mkLayer {
inherit name;
contents = contents ++ [ bash runScriptFile ];
inherit projectName;
};
runAsRootFile = shellScript "run-as-root.sh" runAsRoot;
runScriptFile = shellScript "run-script.sh" runScript;
result = vmTools.runInLinuxVM (
runCommand "${projectName}-image-${name}.img"
{
buildInputs = [ singularity e2fsprogs util-linux gawk ];
layerClosure = writeReferencesToFile layer;
layerClosure = writeClosure contents;
preVM = vmTools.createEmptyImage {
size = diskSize;
fullName = "${projectName}-run-disk";

View file

@ -1,4 +1,4 @@
{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck-minimal }:
{ lib, config, stdenv, stdenvNoCC, jq, lndir, runtimeShell, shellcheck-minimal }:
let
inherit (lib)
@ -625,18 +625,22 @@ rec {
# Docs in doc/build-helpers/trivial-build-helpers.chapter.md
# See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeReferencesToFile
writeReferencesToFile = path: runCommand "runtime-deps"
# TODO: Convert to throw after Nixpkgs 24.05 branch-off.
writeReferencesToFile = (if config.allowAliases then lib.warn else throw)
"writeReferencesToFile is deprecated in favour of writeClosure"
(path: writeClosure [ path ]);
# Docs in doc/build-helpers/trivial-build-helpers.chapter.md
# See https://nixos.org/manual/nixpkgs/unstable/#trivial-builder-writeClosure
writeClosure = paths: runCommand "runtime-deps"
{
exportReferencesGraph = [ "graph" path ];
# Get the cleaner exportReferencesGraph interface
__structuredAttrs = true;
exportReferencesGraph.graph = paths;
nativeBuildInputs = [ jq ];
}
''
touch $out
while read path; do
echo $path >> $out
read dummy
read nrRefs
for ((i = 0; i < nrRefs; i++)); do read ref; done
done < graph
jq -r ".graph | map(.path) | sort | .[]" "$NIX_ATTRS_JSON_FILE" > "$out"
'';
# Docs in doc/build-helpers/trivial-build-helpers.chapter.md

View file

@ -26,6 +26,9 @@ recurseIntoAttrs {
then references
else {};
writeCBin = callPackage ./writeCBin.nix {};
writeClosure-union = callPackage ./writeClosure-union.nix {
inherit (references) samples;
};
writeShellApplication = callPackage ./writeShellApplication.nix {};
writeScriptBin = callPackage ./writeScriptBin.nix {};
writeShellScript = callPackage ./write-shell-script.nix {};

View file

@ -12,7 +12,7 @@
, cleanSamples ? lib.filterAttrs (n: lib.isStringLike)
# Test targets
, writeDirectReferencesToFile
, writeReferencesToFile
, writeClosure
}:
# -------------------------------------------------------------------------- #
@ -46,8 +46,9 @@ let
samplesToString = attrs:
lib.concatMapStringsSep " " (name: "[${name}]=${lib.escapeShellArg "${attrs.${name}}"}") (builtins.attrNames attrs);
references = lib.mapAttrs (n: v: writeReferencesToFile v) samples;
closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
directReferences = lib.mapAttrs (n: v: writeDirectReferencesToFile v) samples;
collectiveClosure = writeClosure (lib.attrValues samples);
testScriptBin = stdenvNoCC.mkDerivation (finalAttrs: {
name = "references-test";
@ -61,8 +62,9 @@ let
mkdir -p "$out/bin"
substitute "$src" "$out/bin/${finalAttrs.meta.mainProgram}" \
--replace "@SAMPLES@" ${lib.escapeShellArg (samplesToString samples)} \
--replace "@REFERENCES@" ${lib.escapeShellArg (samplesToString references)} \
--replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)}
--replace "@CLOSURES@" ${lib.escapeShellArg (samplesToString closures)} \
--replace "@DIRECT_REFS@" ${lib.escapeShellArg (samplesToString directReferences)} \
--replace "@COLLECTIVE_CLOSURE@" ${lib.escapeShellArg collectiveClosure}
runHook postInstall
chmod +x "$out/bin/${finalAttrs.meta.mainProgram}"
'';
@ -79,8 +81,9 @@ let
passthru = {
inherit
collectiveClosure
directReferences
references
closures
samples
;
};
@ -109,8 +112,9 @@ testers.nixosTest {
'';
passthru = {
inherit
collectiveClosure
directReferences
references
closures
samples
testScriptBin
;

View file

@ -33,16 +33,17 @@ set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")" # nixpkgs root
# Injected by Nix (to avoid evaluating in a derivation)
# turn them into arrays
# shellcheck disable=SC2206 # deliberately unquoted
declare -A samples=( @SAMPLES@ )
# shellcheck disable=SC2206 # deliberately unquoted
declare -A directRefs=( @DIRECT_REFS@ )
# shellcheck disable=SC2206 # deliberately unquoted
declare -A references=( @REFERENCES@ )
# Inject the path to compare from the Nix expression
echo >&2 Testing direct references...
# Associative Arrays
declare -A samples=( @SAMPLES@ )
declare -A directRefs=( @DIRECT_REFS@ )
declare -A closures=( @CLOSURES@ )
# Path string
collectiveClosure=@COLLECTIVE_CLOSURE@
echo >&2 Testing direct closures...
for i in "${!samples[@]}"; do
echo >&2 Checking "$i" "${samples[$i]}" "${directRefs[$i]}"
diff -U3 \
@ -52,10 +53,16 @@ done
echo >&2 Testing closure...
for i in "${!samples[@]}"; do
echo >&2 Checking "$i" "${samples[$i]}" "${references[$i]}"
echo >&2 Checking "$i" "${samples[$i]}" "${closures[$i]}"
diff -U3 \
<(sort <"${references[$i]}") \
<(sort <"${closures[$i]}") \
<(nix-store -q --requisites "${samples[$i]}" | sort)
done
echo >&2 Testing mixed closures...
echo >&2 Checking all samples "(${samples[*]})" "$collectiveClosure"
diff -U3 \
<(sort <"$collectiveClosure") \
<(nix-store -q --requisites "${samples[@]}" | sort)
echo 'OK!'

View file

@ -0,0 +1,23 @@
{ lib
, runCommandLocal
# Test targets
, writeClosure
, samples
}:
runCommandLocal "test-trivial-builders-writeClosure-union" {
__structuredAttrs = true;
closures = lib.mapAttrs (n: v: writeClosure [ v ]) samples;
collectiveClosure = writeClosure (lib.attrValues samples);
inherit samples;
meta.maintainers = with lib.maintainers; [
ShamrockLee
];
} ''
set -eu -o pipefail
echo >&2 Testing mixed closures...
echo >&2 Checking all samples "(''${samples[*]})" "$collectiveClosure"
diff -U3 \
<(sort <"$collectiveClosure") \
<(cat "''${closures[@]}" | sort | uniq)
touch "$out"
''

View file

@ -110,8 +110,9 @@ let
trivialBuilders = self: super:
import ../build-support/trivial-builders {
inherit lib;
inherit (self) config;
inherit (self) runtimeShell stdenv stdenvNoCC;
inherit (self.pkgsBuildHost) shellcheck-minimal;
inherit (self.pkgsBuildHost) jq shellcheck-minimal;
inherit (self.pkgsBuildHost.xorg) lndir;
};