nixos/iso-image: extremely cursed performance optimization for Hydra

Right now the worst case chain of events for building an ISO on Hydra is

    - copy everything to squashfs builder
    - run squashfs builder
    - download squashfs from builder
    - compress squashfs
    - upload squashfs to S3
    - copy squashfs to ISO builder
    - run ISO builder
    - download ISO from builder
    - compress ISO
    - upload ISO to S3

This inlines the squashfs build into the ISO build, which makes it

    - copy everything to ISO builder
    - run ISO builder
    - download ISO from builder
    - compress ISO
    - upload ISO to S3

Which should reduce queue runner load by $alot per ISO, which we have four of on small channels
(one release, one test per arch) and a lot more than four of on large channels (with various desktops)
This commit is contained in:
K900 2024-03-11 22:27:48 +03:00
parent 587a40f999
commit 77536af43b
3 changed files with 30 additions and 11 deletions

View file

@ -1,4 +1,4 @@
{ stdenv, closureInfo, xorriso, syslinux, libossp_uuid { lib, stdenv, callPackage, closureInfo, xorriso, syslinux, libossp_uuid, squashfsTools
, # The file name of the resulting ISO image. , # The file name of the resulting ISO image.
isoName ? "cd.iso" isoName ? "cd.iso"
@ -16,6 +16,17 @@
# symlink to `object' that will be added to the CD. # symlink to `object' that will be added to the CD.
storeContents ? [] storeContents ? []
, # In addition to `contents', the closure of the store paths listed
# in `squashfsContents' is compressed as squashfs and the result is
# placed in /nix-store.squashfs on the CD.
# FIXME: This is a performance optimization to avoid Hydra copying
# the squashfs between builders and should be removed when Hydra
# is smarter about scheduling.
squashfsContents ? []
, # Compression settings for squashfs
squashfsCompression ? "xz -Xdict-size 100%"
, # Whether this should be an El-Torito bootable CD. , # Whether this should be an El-Torito bootable CD.
bootable ? false bootable ? false
@ -45,12 +56,20 @@ assert bootable -> bootImage != "";
assert efiBootable -> efiBootImage != ""; assert efiBootable -> efiBootImage != "";
assert usbBootable -> isohybridMbrImage != ""; assert usbBootable -> isohybridMbrImage != "";
let
needSquashfs = squashfsContents != [];
makeSquashfsDrv = callPackage ./make-squashfs.nix {
storeContents = squashfsContents;
comp = squashfsCompression;
};
in
stdenv.mkDerivation { stdenv.mkDerivation {
name = isoName; name = isoName;
__structuredAttrs = true; __structuredAttrs = true;
buildCommandPath = ./make-iso9660-image.sh; buildCommandPath = ./make-iso9660-image.sh;
nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ]; nativeBuildInputs = [ xorriso syslinux zstd libossp_uuid ]
++ lib.optionals needSquashfs makeSquashfsDrv.nativeBuildInputs;
inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable; inherit isoName bootable bootImage compressImage volumeID efiBootImage efiBootable isohybridMbrImage usbBootable;
@ -60,6 +79,8 @@ stdenv.mkDerivation {
objects = map (x: x.object) storeContents; objects = map (x: x.object) storeContents;
symlinks = map (x: x.symlink) storeContents; symlinks = map (x: x.symlink) storeContents;
squashfsCommand = lib.optionalString needSquashfs makeSquashfsDrv.buildCommand;
# For obtaining the closure of `storeContents'. # For obtaining the closure of `storeContents'.
closureInfo = closureInfo { rootPaths = map (x: x.object) storeContents; }; closureInfo = closureInfo { rootPaths = map (x: x.object) storeContents; };
} }

View file

@ -68,6 +68,11 @@ for i in $(< $closureInfo/store-paths); do
addPath "${i:1}" "$i" addPath "${i:1}" "$i"
done done
# If needed, build a squashfs and add that
if [[ -n "$squashfsCommand" ]]; then
(out="nix-store.squashfs" eval "$squashfsCommand")
addPath "nix-store.squashfs" "nix-store.squashfs"
fi
# Also include a manifest of the closures in a format suitable for # Also include a manifest of the closures in a format suitable for
# nix-store --load-db. # nix-store --load-db.

View file

@ -811,12 +811,6 @@ in
optional config.isoImage.includeSystemBuildDependencies optional config.isoImage.includeSystemBuildDependencies
config.system.build.toplevel.drvPath; config.system.build.toplevel.drvPath;
# Create the squashfs image that contains the Nix store.
system.build.squashfsStore = pkgs.callPackage ../../../lib/make-squashfs.nix {
storeContents = config.isoImage.storeContents;
comp = config.isoImage.squashfsCompression;
};
# Individual files to be included on the CD, outside of the Nix # Individual files to be included on the CD, outside of the Nix
# store on the CD. # store on the CD.
isoImage.contents = isoImage.contents =
@ -827,9 +821,6 @@ in
{ source = config.system.build.initialRamdisk + "/" + config.system.boot.loader.initrdFile; { source = config.system.build.initialRamdisk + "/" + config.system.boot.loader.initrdFile;
target = "/boot/" + config.system.boot.loader.initrdFile; target = "/boot/" + config.system.boot.loader.initrdFile;
} }
{ source = config.system.build.squashfsStore;
target = "/nix-store.squashfs";
}
{ source = pkgs.writeText "version" config.system.nixos.label; { source = pkgs.writeText "version" config.system.nixos.label;
target = "/version.txt"; target = "/version.txt";
} }
@ -878,6 +869,8 @@ in
bootable = config.isoImage.makeBiosBootable; bootable = config.isoImage.makeBiosBootable;
bootImage = "/isolinux/isolinux.bin"; bootImage = "/isolinux/isolinux.bin";
syslinux = if config.isoImage.makeBiosBootable then pkgs.syslinux else null; syslinux = if config.isoImage.makeBiosBootable then pkgs.syslinux else null;
squashfsContents = config.isoImage.storeContents;
squashfsCompression = config.isoImage.squashfsCompression;
} // optionalAttrs (config.isoImage.makeUsbBootable && config.isoImage.makeBiosBootable) { } // optionalAttrs (config.isoImage.makeUsbBootable && config.isoImage.makeBiosBootable) {
usbBootable = true; usbBootable = true;
isohybridMbrImage = "${pkgs.syslinux}/share/syslinux/isohdpfx.bin"; isohybridMbrImage = "${pkgs.syslinux}/share/syslinux/isohdpfx.bin";