Merge pull request #289584 from athre0z/docker-zstd

dockerTools: configurable compression schema
This commit is contained in:
Robert Hensing 2024-02-19 18:06:54 +01:00 committed by GitHub
commit d2dfcfcfad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 115 additions and 14 deletions

View file

@ -178,6 +178,13 @@ Similarly, if you encounter errors similar to `Error_Protocol ("certificate has
_Default value:_ 0.
`compressor` (String; _optional_)
: Selects the algorithm used to compress the image.
_Default value:_ `"gz"`.\
_Possible values:_ `"none"`, `"gz"`, `"zstd"`.
`contents` **DEPRECATED**
: This attribute is deprecated, and users are encouraged to use `copyToRoot` instead.

View file

@ -155,6 +155,15 @@ in {
docker.succeed("docker images --format '{{.Tag}}' | grep -F '${examples.nixLayered.imageTag}'")
docker.succeed("docker rmi ${examples.nixLayered.imageName}")
with subtest("Check that images with alternative compression schemas load"):
docker.succeed(
"docker load --input='${examples.bashZstdCompressed}'",
"docker rmi ${examples.bashZstdCompressed.imageName}",
)
docker.succeed(
"docker load --input='${examples.bashUncompressed}'",
"docker rmi ${examples.bashUncompressed.imageName}",
)
with subtest(
"Check if the nix store is correctly initialized by listing "
@ -476,6 +485,18 @@ in {
"docker run --rm ${examples.layeredImageWithFakeRootCommands.imageName} /hello/bin/layeredImageWithFakeRootCommands-hello"
)
with subtest("mergeImage correctly deals with varying compression schemas in inputs"):
docker.succeed("docker load --input='${examples.mergeVaryingCompressor}'")
for sub_image, tag in [
("${examples.redis.imageName}", "${examples.redis.imageTag}"),
("${examples.bashUncompressed.imageName}", "${examples.bashUncompressed.imageTag}"),
("${examples.bashZstdCompressed.imageName}", "${examples.bashZstdCompressed.imageTag}"),
]:
docker.succeed(f"docker images --format '{{{{.Repository}}}}-{{{{.Tag}}}}' | grep -F '{sub_image}-{tag}'")
docker.succeed(f"docker rmi {sub_image}")
with subtest("exportImage produces a valid tarball"):
docker.succeed(
"tar -tf ${examples.exportBash} | grep '\./bin/bash' > /dev/null"

View file

@ -8,6 +8,7 @@
, proot
, fakeNss
, fakeroot
, file
, go
, jq
, jshon
@ -34,6 +35,7 @@
, writeText
, writeTextDir
, writePython3
, zstd
}:
let
@ -76,6 +78,30 @@ let
# mapping from the go package.
defaultArchitecture = go.GOARCH;
compressors = {
none = {
ext = "";
nativeInputs = [ ];
compress = "cat";
decompress = "cat";
};
gz = {
ext = ".gz";
nativeInputs = [ pigz ];
compress = "pigz -p$NIX_BUILD_CORES -nTR";
decompress = "pigz -d -p$NIX_BUILD_CORES";
};
zstd = {
ext = ".zst";
nativeInputs = [ zstd ];
compress = "zstd -T$NIX_BUILD_CORES";
decompress = "zstd -d -T$NIX_BUILD_CORES";
};
};
compressorForImage = compressor: imageName: compressors.${compressor} or
(throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]");
in
rec {
examples = callPackage ./examples.nix {
@ -487,16 +513,17 @@ rec {
'';
};
buildLayeredImage = lib.makeOverridable ({ name, ... }@args:
buildLayeredImage = lib.makeOverridable ({ name, compressor ? "gz", ... }@args:
let
stream = streamLayeredImage args;
compress = compressorForImage compressor name;
in
runCommand "${baseNameOf name}.tar.gz"
runCommand "${baseNameOf name}.tar${compress.ext}"
{
inherit (stream) imageName;
passthru = { inherit (stream) imageTag; };
nativeBuildInputs = [ pigz ];
} "${stream} | pigz -nTR > $out"
nativeBuildInputs = compress.nativeInputs;
} "${stream} | ${compress.compress} > $out"
);
# 1. extract the base image
@ -539,6 +566,8 @@ rec {
buildVMMemorySize ? 512
, # Time of creation of the image.
created ? "1970-01-01T00:00:01Z"
, # Compressor to use. One of: none, gz, zstd.
compressor ? "gz"
, # Deprecated.
contents ? null
,
@ -574,6 +603,8 @@ rec {
in
if created == "now" then impure else pure;
compress = compressorForImage compressor name;
layer =
if runAsRoot == null
then
@ -590,9 +621,9 @@ rec {
extraCommands;
copyToRoot = rootContents;
};
result = runCommand "docker-image-${baseName}.tar.gz"
result = runCommand "docker-image-${baseName}.tar${compress.ext}"
{
nativeBuildInputs = [ jshon pigz jq moreutils ];
nativeBuildInputs = [ jshon jq moreutils ] ++ compress.nativeInputs;
# Image name must be lowercase
imageName = lib.toLower name;
imageTag = lib.optionalString (tag != null) tag;
@ -746,7 +777,7 @@ rec {
chmod -R a-w image
echo "Cooking the image..."
tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out
tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | ${compress.compress} > $out
echo "Finished."
'';
@ -761,16 +792,28 @@ rec {
mergeImages = images: runCommand "merge-docker-images"
{
inherit images;
nativeBuildInputs = [ pigz jq ];
nativeBuildInputs = [ file jq ]
++ compressors.none.nativeInputs
++ compressors.gz.nativeInputs
++ compressors.zstd.nativeInputs;
} ''
mkdir image inputs
# Extract images
repos=()
manifests=()
last_image_mime="application/gzip"
for item in $images; do
name=$(basename $item)
mkdir inputs/$name
tar -I pigz -xf $item -C inputs/$name
last_image_mime=$(file --mime-type -b $item)
case $last_image_mime in
"application/x-tar") ${compressors.none.decompress};;
"application/zstd") ${compressors.zstd.decompress};;
"application/gzip") ${compressors.gz.decompress};;
*) echo "error: unexpected layer type $last_image_mime" >&2; exit 1;;
esac < $item | tar -xC inputs/$name
if [ -f inputs/$name/repositories ]; then
repos+=(inputs/$name/repositories)
fi
@ -787,7 +830,14 @@ rec {
mv repositories image/repositories
mv manifest.json image/manifest.json
# Create tarball and gzip
tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | pigz -nTR > $out
tar -C image --hard-dereference --sort=name --mtime="@$SOURCE_DATE_EPOCH" --owner=0 --group=0 --xform s:'^./':: -c . | (
case $last_image_mime in
"application/x-tar") ${compressors.none.compress};;
"application/zstd") ${compressors.zstd.compress};;
"application/gzip") ${compressors.gz.compress};;
# `*)` not needed; already checked.
esac
) > $out
'';
@ -1239,14 +1289,15 @@ rec {
};
# Wrapper around streamNixShellImage to build an image from the result
buildNixShellImage = { drv, ... }@args:
buildNixShellImage = { drv, compressor ? "gz", ... }@args:
let
stream = streamNixShellImage args;
compress = compressorForImage compressor drv.name;
in
runCommand "${drv.name}-env.tar.gz"
runCommand "${drv.name}-env.tar${compress.ext}"
{
inherit (stream) imageName;
passthru = { inherit (stream) imageTag; };
nativeBuildInputs = [ pigz ];
} "${stream} | pigz -nTR > $out";
nativeBuildInputs = compress.nativeInputs;
} "${stream} | ${compress.compress} > $out";
}

View file

@ -480,6 +480,22 @@ rec {
layerC = layerOnTopOf layerB "c";
in layerC;
bashUncompressed = pkgs.dockerTools.buildImage {
name = "bash-uncompressed";
tag = "latest";
compressor = "none";
# Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
copyToRoot = pkgs.bashInteractive;
};
bashZstdCompressed = pkgs.dockerTools.buildImage {
name = "bash-zstd";
tag = "latest";
compressor = "zstd";
# Not recommended. Use `buildEnv` between copy and packages to avoid file duplication.
copyToRoot = pkgs.bashInteractive;
};
# buildImage without explicit tag
bashNoTag = pkgs.dockerTools.buildImage {
name = "bash-no-tag";
@ -614,6 +630,12 @@ rec {
layeredImageWithFakeRootCommands
];
mergeVaryingCompressor = pkgs.dockerTools.mergeImages [
redis
bashUncompressed
bashZstdCompressed
];
helloOnRoot = pkgs.dockerTools.streamLayeredImage {
name = "hello";
tag = "latest";