From eb435897a6de9c1924b0fc45147470bb557ae105 Mon Sep 17 00:00:00 2001 From: Julien Malka Date: Sat, 6 Jan 2024 00:30:23 +0000 Subject: [PATCH 01/10] nixos/systemd-boot: init boot counting --- .../manual/release-notes/rl-2405.section.md | 2 + .../boot/loader/systemd-boot/boot-counting.md | 38 ++++ .../systemd-boot/systemd-boot-builder.py | 214 +++++++++++++----- .../boot/loader/systemd-boot/systemd-boot.nix | 16 +- nixos/modules/system/boot/systemd.nix | 4 + nixos/tests/systemd-boot.nix | 157 ++++++++++++- 6 files changed, 366 insertions(+), 65 deletions(-) create mode 100644 nixos/modules/system/boot/loader/systemd-boot/boot-counting.md diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md index d238e92271c5..9db2b4897791 100644 --- a/nixos/doc/manual/release-notes/rl-2405.section.md +++ b/nixos/doc/manual/release-notes/rl-2405.section.md @@ -14,6 +14,8 @@ In addition to numerous new and upgraded packages, this release has the followin - This can be disabled through the `environment.stub-ld.enable` option. - If you use `programs.nix-ld.enable`, no changes are needed. The stub will be disabled automatically. +- NixOS now has support for *automatic boot assessment* (see [here](https://systemd.io/AUTOMATIC_BOOT_ASSESSMENT/)) for detailed description of the feature) for `systemd-boot` users. Available as [boot.loader.systemd-boot.bootCounting](#opt-boot.loader.systemd-boot.bootCounting.enable). + - Julia environments can now be built with arbitrary packages from the ecosystem using the `.withPackages` function. For example: `julia.withPackages ["Plots"]`. ## New Services {#sec-release-24.05-new-services} diff --git a/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md b/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md new file mode 100644 index 000000000000..736c54228452 --- /dev/null +++ b/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md @@ -0,0 +1,38 @@ +# Automatic boot assessment with systemd-boot {#sec-automatic-boot-assessment} + +## Overview {#sec-automatic-boot-assessment-overview} + +Automatic boot assessment (or boot-counting) is a feature of `systemd-boot` that allows for automatically detecting invalid boot entries. +When the feature is active, each boot entry has an associated counter with a user defined number of trials. Whenever `system-boot` boots an entry, its counter is decreased by one, ultimately being marked as *bad* if the counter ever reaches zero. However, if an entry is successfully booted, systemd will permanently mark it as *good* and remove the counter altogether. Whenever an entry is marked as *bad*, it is sorted last in the systemd-boot menu. +A complete explanation of how that feature works can be found [here](https://systemd.io/AUTOMATIC_BOOT_ASSESSMENT/). + +## Enabling the feature {#sec-automatic-boot-assessment-enable} + +The feature can be enabled by toogling the [boot.loader.systemd-boot.bootCounting](#opt-boot.loader.systemd-boot.bootCounting.enable) option. + +## The boot-complete.target unit {#sec-automatic-boot-assessment-boot-complete-target} + +A *successful boot* for an entry is defined in terms of the `boot-complete.target` synchronisation point. It is up to the user to schedule all necessary units for the machine to be considered successfully booted before that synchronisation point. +For example, if you are running `nsd`, an authoritative DNS server on a machine and you want to be sure that a *good* entry is an entry where that DNS server is started successfully. A configuration for that NixOS machine could look like that: + +``` +boot.loader.systemd-boot.bootCounting.enable = true; +services.nsd.enable = true; +/* rest of nsd configuration omitted */ + +systemd.services.nsd = { + before = [ "boot-complete.target" ]; + wantedBy = [ "boot-complete.target" ]; + unitConfig.FailureAction = "reboot"; +}; + +``` + +## Interaction with specialisations {#sec-automatic-boot-assessment-specialisations} + +When the boot-counting feature is enabled, `systemd-boot` will still try the boot entries in the same order as they are displayed in the boot menu. This means that the specialisations of a given generation will be tried directly after that generation. A generation being marked as *bad* do not mean that its specialisations will also be marked as *bad* (as its specialisations could very well be booting successfully). + + +## Limitations {#sec-automatic-boot-assessment-limitations} + +This feature has to be used wisely to not risk any data integrity issues. Rollbacking into past generations can sometimes be dangerous, for example if some of the services may have undefined behaviors in the presence of unrecognized data migrations from future versions of themselves. diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py index 6cd46f30373b..75f32459e6b5 100644 --- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py +++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py @@ -12,8 +12,9 @@ import subprocess import sys import warnings import json -from typing import NamedTuple, Dict, List +from typing import NamedTuple, Dict, List, Type, Generator, Iterable from dataclasses import dataclass +from pathlib import Path @dataclass @@ -28,7 +29,114 @@ class BootSpec: specialisations: Dict[str, "BootSpec"] initrdSecrets: str | None = None +@dataclass +class Entry: + profile: str | None + generation_number: int + specialisation: str | None + @classmethod + def from_path(cls: Type["Entry"], path: Path) -> "Entry": + filename = path.name + # Matching nixos-$profile-generation-*.conf + rex_profile = re.compile(r"^nixos-(.*)-generation-.*\.conf$") + # Matching nixos*-generation-$number*.conf + rex_generation = re.compile(r"^nixos.*-generation-([0-9]+).*\.conf$") + # Matching nixos*-generation-$number-specialisation-$specialisation_name*.conf + rex_specialisation = re.compile(r"^nixos.*-generation-([0-9]+)-specialisation-([a-zA-Z0-9]+).*\.conf$") + profile = rex_profile.sub(r"\1", filename) if rex_profile.match(filename) else None + specialisation = rex_specialisation.sub(r"\2", filename) if rex_specialisation.match(filename) else None + try: + generation_number = int(rex_generation.sub(r"\1", filename)) + except ValueError: + raise + return cls(profile, generation_number, specialisation) + + +BOOT_ENTRY = """title {title} +version Generation {generation} {description} +linux {kernel} +initrd {initrd} +options {kernel_params} +machine-id {machine_id} +sort-key {sort_key} +""" + +@dataclass +class DiskEntry(): + entry: Entry + default: bool + counters: str | None + title: str + description: str + kernel: str + initrd: str + kernel_params: str + machine_id: str + + @classmethod + def from_path(cls: Type["DiskEntry"], path: Path) -> "DiskEntry": + entry = Entry.from_path(path) + with open(path, 'r') as f: + data = f.read().splitlines() + if '' in data: + data.remove('') + entry_map = dict(l.split(' ', 1) for l in data) + assert "title" in entry_map + assert "version" in entry_map + version_splitted = entry_map["version"].split(" ", 2) + assert version_splitted[0] == "Generation" + assert version_splitted[1].isdigit() + assert "linux" in entry_map + assert "initrd" in entry_map + assert "options" in entry_map + assert "machine-id" in entry_map + assert "sort-key" in entry_map + filename = path.name + # Matching nixos*-generation-*$counters.conf + rex_counters = re.compile(r"^nixos.*-generation-.*(\+\d(-\d)?)\.conf$") + counters = rex_counters.sub(r"\1", filename) if rex_counters.match(filename) else None + disk_entry = cls( + entry=entry, + default=(entry_map["sort-key"] == "default"), + counters=counters, + title=entry_map["title"], + description=entry_map["version"], + kernel=entry_map["linux"], + initrd=entry_map["initrd"], + kernel_params=entry_map["options"], + machine_id=entry_map["machine-id"]) + return disk_entry + + def write(self) -> None: + tmp_path = self.path.with_suffix(".tmp") + with tmp_path.open('w') as f: + # We use "sort-key" to sort the default generation first. + # The "default" string is sorted before "non-default" (alphabetically) + f.write(BOOT_ENTRY.format(title=self.title, + generation=self.entry.generation_number, + kernel=self.kernel, + initrd=self.initrd, + kernel_params=self.kernel_params, + machine_id=self.machine_id, + description=self.description, + sort_key="default" if self.default else "non-default")) + f.flush() + os.fsync(f.fileno()) + tmp_path.rename(self.path) + + + @property + def path(self) -> Path: + pieces = [ + "nixos", + self.entry.profile or None, + "generation", + str(self.entry.generation_number), + f"specialisation-{self.entry.specialisation}" if self.entry.specialisation else None, + ] + prefix = "-".join(p for p in pieces if p) + return Path(f"@efiSysMountPoint@/loader/entries/{prefix}{self.counters if self.counters else ''}.conf") libc = ctypes.CDLL("libc.so.6") @@ -56,29 +164,14 @@ def system_dir(profile: str | None, generation: int, specialisation: str | None) else: return d -BOOT_ENTRY = """title {title} -version Generation {generation} {description} -linux {kernel} -initrd {initrd} -options {kernel_params} -""" - -def generation_conf_filename(profile: str | None, generation: int, specialisation: str | None) -> str: - pieces = [ - "nixos", - profile or None, - "generation", - str(generation), - f"specialisation-{specialisation}" if specialisation else None, - ] - return "-".join(p for p in pieces if p) + ".conf" - - -def write_loader_conf(profile: str | None, generation: int, specialisation: str | None) -> None: +def write_loader_conf(profile: str | None) -> None: with open("@efiSysMountPoint@/loader/loader.conf.tmp", 'w') as f: if "@timeout@" != "": f.write("timeout @timeout@\n") - f.write("default %s\n" % generation_conf_filename(profile, generation, specialisation)) + if profile: + f.write("default nixos-%s-generation-*\n" % profile) + else: + f.write("default nixos-generation-*\n") if not @editor@: f.write("editor 0\n") f.write("console-mode @consoleMode@\n") @@ -86,6 +179,17 @@ def write_loader_conf(profile: str | None, generation: int, specialisation: str os.fsync(f.fileno()) os.rename("@efiSysMountPoint@/loader/loader.conf.tmp", "@efiSysMountPoint@/loader/loader.conf") +def scan_entries() -> Generator[DiskEntry, None, None]: + """ + Scan all entries in $ESP/loader/entries/* + Does not support Type 2 entries as we do not support them for now. + Returns a generator of Entry. + """ + for path in Path("@efiSysMountPoint@/loader/entries/").glob("nixos*-generation-[1-9]*.conf"): + try: + yield DiskEntry.from_path(path) + except ValueError: + continue def get_bootspec(profile: str | None, generation: int) -> BootSpec: system_directory = system_dir(profile, generation, None) @@ -120,7 +224,7 @@ def copy_from_file(file: str, dry_run: bool = False) -> str: return efi_file_path def write_entry(profile: str | None, generation: int, specialisation: str | None, - machine_id: str, bootspec: BootSpec, current: bool) -> None: + machine_id: str, bootspec: BootSpec, entries: Iterable[DiskEntry], current: bool) -> None: if specialisation: bootspec = bootspec.specialisations[specialisation] kernel = copy_from_file(bootspec.kernel) @@ -142,28 +246,30 @@ def write_entry(profile: str | None, generation: int, specialisation: str | None f'for "{title} - Configuration {generation}", an older generation', file=sys.stderr) print("note: this is normal after having removed " "or renamed a file in `boot.initrd.secrets`", file=sys.stderr) - entry_file = "@efiSysMountPoint@/loader/entries/%s" % ( - generation_conf_filename(profile, generation, specialisation)) - tmp_path = "%s.tmp" % (entry_file) kernel_params = "init=%s " % bootspec.init - kernel_params = kernel_params + " ".join(bootspec.kernelParams) build_time = int(os.path.getctime(system_dir(profile, generation, specialisation))) build_date = datetime.datetime.fromtimestamp(build_time).strftime('%F') + counters = "+@bootCountingTrials@" if @bootCounting@ else "" + entry = Entry(profile, generation, specialisation) + # We check if the entry we are writing is already on disk + # and we update its "default entry" status + for entry_on_disk in entries: + if entry == entry_on_disk.entry: + entry_on_disk.default = current + entry_on_disk.write() + return - with open(tmp_path, 'w') as f: - f.write(BOOT_ENTRY.format(title=title, - generation=generation, - kernel=kernel, - initrd=initrd, - kernel_params=kernel_params, - description=f"{bootspec.label}, built on {build_date}")) - if machine_id is not None: - f.write("machine-id %s\n" % machine_id) - f.flush() - os.fsync(f.fileno()) - os.rename(tmp_path, entry_file) - + DiskEntry( + entry=entry, + title=title, + kernel=kernel, + initrd=initrd, + counters=counters, + kernel_params=kernel_params, + machine_id=machine_id, + description=f"{bootspec.label}, built on {build_date}", + default=current).write() def get_generations(profile: str | None = None) -> list[SystemIdentifier]: gen_list = subprocess.check_output([ @@ -188,30 +294,19 @@ def get_generations(profile: str | None = None) -> list[SystemIdentifier]: return configurations[-configurationLimit:] -def remove_old_entries(gens: list[SystemIdentifier]) -> None: - rex_profile = re.compile(r"^@efiSysMountPoint@/loader/entries/nixos-(.*)-generation-.*\.conf$") - rex_generation = re.compile(r"^@efiSysMountPoint@/loader/entries/nixos.*-generation-([0-9]+)(-specialisation-.*)?\.conf$") +def remove_old_entries(gens: list[SystemIdentifier], disk_entries: Iterable[DiskEntry]) -> None: known_paths = [] for gen in gens: bootspec = get_bootspec(gen.profile, gen.generation) known_paths.append(copy_from_file(bootspec.kernel, True)) known_paths.append(copy_from_file(bootspec.initrd, True)) - for path in glob.iglob("@efiSysMountPoint@/loader/entries/nixos*-generation-[1-9]*.conf"): - if rex_profile.match(path): - prof = rex_profile.sub(r"\1", path) - else: - prof = None - try: - gen_number = int(rex_generation.sub(r"\1", path)) - except ValueError: - continue - if not (prof, gen_number, None) in gens: - os.unlink(path) + for disk_entry in disk_entries: + if (disk_entry.entry.profile, disk_entry.entry.generation_number, None) not in gens: + os.unlink(disk_entry.path) for path in glob.iglob("@efiSysMountPoint@/efi/nixos/*"): - if not path in known_paths and not os.path.isdir(path): + if path not in known_paths and not os.path.isdir(path): os.unlink(path) - def get_profiles() -> list[str]: if os.path.isdir("/nix/var/nix/profiles/system-profiles/"): return [x @@ -284,16 +379,17 @@ def install_bootloader(args: argparse.Namespace) -> None: gens = get_generations() for profile in get_profiles(): gens += get_generations(profile) - remove_old_entries(gens) + entries = scan_entries() + remove_old_entries(gens, entries) for gen in gens: try: bootspec = get_bootspec(gen.profile, gen.generation) is_default = os.path.dirname(bootspec.init) == args.default_config - write_entry(*gen, machine_id, bootspec, current=is_default) + write_entry(*gen, machine_id, bootspec, entries, current=is_default) for specialisation in bootspec.specialisations.keys(): - write_entry(gen.profile, gen.generation, specialisation, machine_id, bootspec, current=is_default) + write_entry(gen.profile, gen.generation, specialisation, machine_id, bootspec, entries, current=is_default) if is_default: - write_loader_conf(*gen) + write_loader_conf(gen.profile) except OSError as e: # See https://github.com/NixOS/nixpkgs/issues/114552 if e.errno == errno.EINVAL: diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix index 9d55c21077d1..3a4173210f5f 100644 --- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix +++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix @@ -49,6 +49,8 @@ let ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/loader/entries/"${escapeShellArg n} '') cfg.extraEntries)} ''; + bootCountingTrials = cfg.bootCounting.trials; + bootCounting = if cfg.bootCounting.enable then "True" else "False"; }; checkedSystemdBootBuilder = pkgs.runCommand "systemd-boot" { @@ -69,7 +71,10 @@ let ''; in { - meta.maintainers = with lib.maintainers; [ julienmalka ]; + meta = { + maintainers = with lib.maintainers; [ julienmalka ]; + doc = ./boot-counting.md; + }; imports = [ (mkRenamedOptionModule [ "boot" "loader" "gummiboot" "enable" ] [ "boot" "loader" "systemd-boot" "enable" ]) @@ -238,6 +243,15 @@ in { ''; }; + bootCounting = { + enable = mkEnableOption (lib.mdDoc "automatic boot assessment"); + trials = mkOption { + default = 3; + type = types.int; + description = lib.mdDoc "number of trials each entry should start with"; + }; + }; + }; config = mkIf cfg.enable { diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix index 87333999313e..b315fc91e3d0 100644 --- a/nixos/modules/system/boot/systemd.nix +++ b/nixos/modules/system/boot/systemd.nix @@ -101,6 +101,10 @@ let "systemd-rfkill.service" "systemd-rfkill.socket" + # Boot counting + "boot-complete.target" + ] ++ lib.optional config.boot.loader.systemd-boot.bootCounting.enable "systemd-bless-boot.service" ++ [ + # Hibernate / suspend. "hibernate.target" "suspend.target" diff --git a/nixos/tests/systemd-boot.nix b/nixos/tests/systemd-boot.nix index c0b37a230df0..6e956ab74fbe 100644 --- a/nixos/tests/systemd-boot.nix +++ b/nixos/tests/systemd-boot.nix @@ -13,9 +13,11 @@ let boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; environment.systemPackages = [ pkgs.efibootmgr ]; + # Needed for machine-id to be persisted between reboots + environment.etc."machine-id".text = "00000000000000000000000000000000"; }; in -{ +rec { basic = makeTest { name = "systemd-boot"; meta.maintainers = with pkgs.lib.maintainers; [ danielfullmer julienmalka ]; @@ -252,15 +254,15 @@ in ''; }; - garbage-collect-entry = makeTest { - name = "systemd-boot-garbage-collect-entry"; + garbage-collect-entry = { withBootCounting ? false, ... }: makeTest { + name = "systemd-boot-garbage-collect-entry" + optionalString withBootCounting "-with-boot-counting"; meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; nodes = { inherit common; machine = { pkgs, nodes, ... }: { imports = [ common ]; - + boot.loader.systemd-boot.bootCounting.enable = withBootCounting; # These are configs for different nodes, but we'll use them here in `machine` system.extraDependencies = [ nodes.common.system.build.toplevel @@ -275,8 +277,12 @@ in '' machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${baseSystem}") machine.succeed("nix-env -p /nix/var/nix/profiles/system --delete-generations 1") + # At this point generation 1 has already been marked as good so we reintroduce counters artificially + ${optionalString withBootCounting '' + machine.succeed("mv /boot/loader/entries/nixos-generation-1.conf /boot/loader/entries/nixos-generation-1+3.conf") + ''} machine.succeed("${baseSystem}/bin/switch-to-configuration boot") - machine.fail("test -e /boot/loader/entries/nixos-generation-1.conf") + machine.fail("test -e /boot/loader/entries/nixos-generation-1*") machine.succeed("test -e /boot/loader/entries/nixos-generation-2.conf") ''; }; @@ -322,4 +328,145 @@ in machine.wait_for_unit("multi-user.target") ''; }; + + # Check that we are booting the default entry and not the generation with largest version number + defaultEntry = { withBootCounting ? false, ... }: makeTest { + name = "systemd-boot-default-entry" + optionalString withBootCounting "-with-boot-counting"; + meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; + + nodes = { + machine = { pkgs, lib, nodes, ... }: { + imports = [ common ]; + system.extraDependencies = [ nodes.other_machine.system.build.toplevel ]; + boot.loader.systemd-boot.bootCounting.enable = withBootCounting; + }; + + other_machine = { pkgs, lib, ... }: { + imports = [ common ]; + boot.loader.systemd-boot.bootCounting.enable = withBootCounting; + environment.systemPackages = [ pkgs.hello ]; + }; + }; + testScript = { nodes, ... }: + let + orig = nodes.machine.system.build.toplevel; + other = nodes.other_machine.system.build.toplevel; + in + '' + orig = "${orig}" + other = "${other}" + + def check_current_system(system_path): + machine.succeed(f'test $(readlink -f /run/current-system) = "{system_path}"') + + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + check_current_system(orig) + + # Switch to other configuration + machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${other}") + machine.succeed(f"{other}/bin/switch-to-configuration boot") + # Rollback, default entry is now generation 1 + machine.succeed("nix-env -p /nix/var/nix/profiles/system --rollback") + machine.succeed(f"{orig}/bin/switch-to-configuration boot") + + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + ${if withBootCounting + then ''machine.succeed("test -e /boot/loader/entries/nixos-generation-2+3.conf")'' + else ''machine.succeed("test -e /boot/loader/entries/nixos-generation-2.conf")''} + machine.shutdown() + + machine.start() + machine.wait_for_unit("multi-user.target") + # Check that we booted generation 1 (default) + # even though generation 2 comes first in alphabetical order + check_current_system(orig) + ''; + }; + + + bootCounting = + let + baseConfig = { pkgs, lib, ... }: { + imports = [ common ]; + boot.loader.systemd-boot.bootCounting.enable = true; + boot.loader.systemd-boot.bootCounting.trials = 2; + }; + in + makeTest { + name = "systemd-boot-counting"; + meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; + + nodes = { + machine = { pkgs, lib, nodes, ... }: { + imports = [ baseConfig ]; + system.extraDependencies = [ nodes.bad_machine.system.build.toplevel ]; + }; + + bad_machine = { pkgs, lib, ... }: { + imports = [ baseConfig ]; + + systemd.services."failing" = { + script = "exit 1"; + requiredBy = [ "boot-complete.target" ]; + before = [ "boot-complete.target" ]; + serviceConfig.Type = "oneshot"; + }; + }; + }; + testScript = { nodes, ... }: + let + orig = nodes.machine.system.build.toplevel; + bad = nodes.bad_machine.system.build.toplevel; + in + '' + orig = "${orig}" + bad = "${bad}" + + def check_current_system(system_path): + machine.succeed(f'test $(readlink -f /run/current-system) = "{system_path}"') + + # Ensure we booted using an entry with counters enabled + machine.succeed( + "test -e /sys/firmware/efi/efivars/LoaderBootCountPath-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" + ) + + # systemd-bless-boot should have already removed the "+2" suffix from the boot entry + machine.wait_for_unit("systemd-bless-boot.service") + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + check_current_system(orig) + + # Switch to bad configuration + machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${bad}") + machine.succeed(f"{bad}/bin/switch-to-configuration boot") + + # Ensure new bootloader entry has initialized counter + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + machine.succeed("test -e /boot/loader/entries/nixos-generation-2+2.conf") + machine.shutdown() + + machine.start() + machine.wait_for_unit("multi-user.target") + check_current_system(bad) + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + machine.succeed("test -e /boot/loader/entries/nixos-generation-2+1-1.conf") + machine.shutdown() + + machine.start() + machine.wait_for_unit("multi-user.target") + check_current_system(bad) + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + machine.succeed("test -e /boot/loader/entries/nixos-generation-2+0-2.conf") + machine.shutdown() + + # Should boot back into original configuration + machine.start() + check_current_system(orig) + machine.wait_for_unit("multi-user.target") + machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") + machine.succeed("test -e /boot/loader/entries/nixos-generation-2+0-2.conf") + machine.shutdown() + ''; + }; + defaultEntryWithBootCounting = defaultEntry { withBootCounting = true; }; + garbageCollectEntryWithBootCounting = garbage-collect-entry { withBootCounting = true; }; } From 0e1746ad578321d7a977dce44072341dd9bf04fd Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Tue, 9 Jan 2024 21:54:32 +0100 Subject: [PATCH 02/10] python311Packages.dm-haiku: 0.10.0 -> 0.11.0 Changelog: https://github.com/google-deepmind/dm-haiku/releases/tag/v0.0.11 --- .../python-modules/dm-haiku/default.nix | 18 +++++++++--------- .../python-modules/dm-haiku/tests.nix | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/dm-haiku/default.nix b/pkgs/development/python-modules/dm-haiku/default.nix index 08c1716867a7..6ff6ff412b7e 100644 --- a/pkgs/development/python-modules/dm-haiku/default.nix +++ b/pkgs/development/python-modules/dm-haiku/default.nix @@ -1,6 +1,9 @@ { buildPythonPackage , fetchFromGitHub , fetchpatch +, absl-py +, flax +, numpy , callPackage , lib , jmp @@ -10,23 +13,17 @@ buildPythonPackage rec { pname = "dm-haiku"; - version = "0.0.10"; + version = "0.0.11"; format = "setuptools"; src = fetchFromGitHub { owner = "deepmind"; - repo = pname; + repo = "dm-haiku"; rev = "refs/tags/v${version}"; - hash = "sha256-EZx3o6PgTeFjTwI9Ko9H39EqPSE0yLWWpsdqX6ALlo4="; + hash = "sha256-xve1vNsVOC6/HVtzmzswM/Sk3uUNaTtqNAKheFb/tmI="; }; patches = [ - # https://github.com/deepmind/dm-haiku/issues/717 - (fetchpatch { - name = "remove-typing-extensions.patch"; - url = "https://github.com/deepmind/dm-haiku/commit/c22867db1a3314a382bd2ce36511e2b756dc32a8.patch"; - hash = "sha256-SxJc8FrImwMqTJ5OuJ1f4T+HfHgW/sGqXeIqlxEatlE="; - }) # https://github.com/deepmind/dm-haiku/pull/672 (fetchpatch { name = "fix-find-namespace-packages.patch"; @@ -41,8 +38,11 @@ buildPythonPackage rec { ]; propagatedBuildInputs = [ + absl-py + flax jaxlib jmp + numpy tabulate ]; diff --git a/pkgs/development/python-modules/dm-haiku/tests.nix b/pkgs/development/python-modules/dm-haiku/tests.nix index dec909729dcf..3a99bd3ac85a 100644 --- a/pkgs/development/python-modules/dm-haiku/tests.nix +++ b/pkgs/development/python-modules/dm-haiku/tests.nix @@ -20,6 +20,7 @@ buildPythonPackage { pname = "dm-haiku-tests"; + format = "other"; inherit (dm-haiku) version; src = dm-haiku.testsout; From ee1c02123689dbcf322e1074a0d9aab55b50d1e4 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Tue, 9 Jan 2024 22:04:54 +0100 Subject: [PATCH 03/10] python311Packages.bsuite: use propagatedBuildInputs for dependencies --- pkgs/development/python-modules/bsuite/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/bsuite/default.nix b/pkgs/development/python-modules/bsuite/default.nix index bf85d8fe7a09..f6560941f3d6 100644 --- a/pkgs/development/python-modules/bsuite/default.nix +++ b/pkgs/development/python-modules/bsuite/default.nix @@ -35,7 +35,7 @@ let bsuite = buildPythonPackage rec { hash = "sha256-ak9McvXl7Nz5toUaPaRaJek9lurxiQiIW209GnZEjX0="; }; - buildInputs = [ + propagatedBuildInputs = [ absl-py dm-env dm-tree From 53b8616b2d1d608c1c07ea0c883112dfc37a8a01 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 10 Jan 2024 09:16:38 +0100 Subject: [PATCH 04/10] python311Packages.bsuite: include patch that fixes deprecated use of np.int --- pkgs/development/python-modules/bsuite/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/development/python-modules/bsuite/default.nix b/pkgs/development/python-modules/bsuite/default.nix index f6560941f3d6..3c11353bb101 100644 --- a/pkgs/development/python-modules/bsuite/default.nix +++ b/pkgs/development/python-modules/bsuite/default.nix @@ -1,6 +1,7 @@ { lib , fetchPypi , buildPythonPackage +, fetchpatch , frozendict , termcolor , matplotlib @@ -35,6 +36,13 @@ let bsuite = buildPythonPackage rec { hash = "sha256-ak9McvXl7Nz5toUaPaRaJek9lurxiQiIW209GnZEjX0="; }; + patches = [ + (fetchpatch { # Convert np.int -> np.int32 since np.int is deprecated (https://github.com/google-deepmind/bsuite/pull/48) + url = "https://github.com/google-deepmind/bsuite/pull/48/commits/f8d81b2f1c27ef2c8c71ae286001ed879ea306ab.patch"; + hash = "sha256-FXtvVS+U8brulq8Z27+yWIimB+kigGiUOIv1SHb1TA8="; + }) + ]; + propagatedBuildInputs = [ absl-py dm-env From 175f71a0f352df6d5ef98816ff22245704674787 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 10 Jan 2024 09:30:56 +0100 Subject: [PATCH 05/10] python311Packages.dm-haiku: fix tests --- .../python-modules/dm-haiku/default.nix | 87 ++++++++++++++----- .../python-modules/dm-haiku/tests.nix | 69 --------------- 2 files changed, 66 insertions(+), 90 deletions(-) delete mode 100644 pkgs/development/python-modules/dm-haiku/tests.nix diff --git a/pkgs/development/python-modules/dm-haiku/default.nix b/pkgs/development/python-modules/dm-haiku/default.nix index 6ff6ff412b7e..cb97e2f837af 100644 --- a/pkgs/development/python-modules/dm-haiku/default.nix +++ b/pkgs/development/python-modules/dm-haiku/default.nix @@ -1,17 +1,27 @@ -{ buildPythonPackage +{ lib +, buildPythonPackage , fetchFromGitHub , fetchpatch , absl-py , flax -, numpy -, callPackage -, lib -, jmp -, tabulate , jaxlib +, jmp +, numpy +, tabulate +, pytest-xdist +, pytestCheckHook +, bsuite +, chex +, cloudpickle +, dill +, dm-env +, dm-tree +, optax +, rlax +, tensorflow }: -buildPythonPackage rec { +let dm-haiku = buildPythonPackage rec { pname = "dm-haiku"; version = "0.0.11"; format = "setuptools"; @@ -32,11 +42,6 @@ buildPythonPackage rec { }) ]; - outputs = [ - "out" - "testsout" - ]; - propagatedBuildInputs = [ absl-py flax @@ -50,17 +55,56 @@ buildPythonPackage rec { "haiku" ]; - postInstall = '' - mkdir $testsout - cp -R examples $testsout/examples - ''; + nativeCheckInputs = [ + bsuite + chex + cloudpickle + dill + dm-env + dm-haiku + dm-tree + jaxlib + optax + pytest-xdist + pytestCheckHook + rlax + tensorflow + ]; + + disabledTests = [ + # See https://github.com/deepmind/dm-haiku/issues/366. + "test_jit_Recurrent" + + # Assertion errors + "testShapeChecking0" + "testShapeChecking1" + + # This test requires a more recent version of tensorflow. The current one (2.13) is not enough. + "test_reshape_convert" + + # This test requires JAX support for double precision (64bit), but enabling this causes several + # other tests to fail. + # https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision + "test_doctest_haiku.experimental" + ]; + + disabledTestPaths = [ + # Those tests requires a more recent version of tensorflow. The current one (2.13) is not enough. + "haiku/_src/integration/jax2tf_test.py" + ]; - # check in passthru.tests.pytest to escape infinite recursion with bsuite doCheck = false; - passthru.tests = { - pytest = callPackage ./tests.nix { }; - }; + # check in passthru.tests.pytest to escape infinite recursion with bsuite + passthru.tests.pytest = dm-haiku.overridePythonAttrs (_: { + pname = "${pname}-tests"; + doCheck = true; + + # We don't have to install because the only purpose + # of this passthru test is to, well, test. + # This fixes having to set `catchConflicts` to false. + dontInstall = true; + }); meta = with lib; { description = "Haiku is a simple neural network library for JAX developed by some of the authors of Sonnet."; @@ -68,4 +112,5 @@ buildPythonPackage rec { license = licenses.asl20; maintainers = with maintainers; [ ndl ]; }; -} +}; +in dm-haiku diff --git a/pkgs/development/python-modules/dm-haiku/tests.nix b/pkgs/development/python-modules/dm-haiku/tests.nix deleted file mode 100644 index 3a99bd3ac85a..000000000000 --- a/pkgs/development/python-modules/dm-haiku/tests.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ buildPythonPackage -, dm-haiku -, chex -, cloudpickle -, dill -, dm-tree -, jaxlib -, pytest-xdist -, pytestCheckHook -, tensorflow -, bsuite -, frozendict -, dm-env -, scikit-image -, rlax -, distrax -, tensorflow-probability -, optax -}: - -buildPythonPackage { - pname = "dm-haiku-tests"; - format = "other"; - inherit (dm-haiku) version; - - src = dm-haiku.testsout; - - dontBuild = true; - dontInstall = true; - - nativeCheckInputs = [ - bsuite - chex - cloudpickle - dill - distrax - dm-env - dm-haiku - dm-tree - frozendict - jaxlib - pytest-xdist - pytestCheckHook - optax - rlax - scikit-image - tensorflow - tensorflow-probability - ]; - - disabledTests = [ - # See https://github.com/deepmind/dm-haiku/issues/366. - "test_jit_Recurrent" - # Assertion errors - "test_connect_conv_padding_function_same0" - "test_connect_conv_padding_function_valid0" - "test_connect_conv_padding_function_same1" - "test_connect_conv_padding_function_same2" - "test_connect_conv_padding_function_valid1" - "test_connect_conv_padding_function_valid2" - "test_invalid_axis_ListString" - "test_invalid_axis_String" - "test_simple_case" - "test_simple_case_with_scale" - "test_slice_axis" - "test_zero_inputs" - ]; - -} From 6fe5e5a54ac122f9f7f3235fac8e82e28a286638 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 10 Jan 2024 11:40:12 +0100 Subject: [PATCH 06/10] python311Packages.distrax: use pytest-xdist --- pkgs/development/python-modules/distrax/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/distrax/default.nix b/pkgs/development/python-modules/distrax/default.nix index 616dbae7a4fd..cc667cc6bf19 100644 --- a/pkgs/development/python-modules/distrax/default.nix +++ b/pkgs/development/python-modules/distrax/default.nix @@ -7,6 +7,7 @@ , numpy , tensorflow-probability , dm-haiku +, pytest-xdist , pytestCheckHook }: @@ -33,6 +34,7 @@ buildPythonPackage rec { nativeCheckInputs = [ dm-haiku + pytest-xdist pytestCheckHook ]; From c92778b9e5a5d23a1f5bab3b987fc9577eaff43d Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Wed, 10 Jan 2024 11:50:07 +0100 Subject: [PATCH 07/10] python311Packages.rlax: use pytest-xdist --- pkgs/development/python-modules/rlax/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/development/python-modules/rlax/default.nix b/pkgs/development/python-modules/rlax/default.nix index 1bb7189d6957..ceb8e9758619 100644 --- a/pkgs/development/python-modules/rlax/default.nix +++ b/pkgs/development/python-modules/rlax/default.nix @@ -12,6 +12,7 @@ , tensorflow-probability , dm-haiku , optax +, pytest-xdist , pytestCheckHook }: @@ -49,6 +50,7 @@ buildPythonPackage rec { nativeCheckInputs = [ dm-haiku optax + pytest-xdist pytestCheckHook ]; From 9d594179ad78c1a9edc701895288965480601e92 Mon Sep 17 00:00:00 2001 From: Florian Brandes Date: Mon, 21 Aug 2023 21:26:41 +0200 Subject: [PATCH 08/10] kodiPackages.pvr-vdr-vnsi: init at 20.4.1 Signed-off-by: Florian Brandes --- .../kodi/addons/pvr-vdr-vnsi/default.nix | 23 +++++++++++++++++++ pkgs/top-level/kodi-packages.nix | 2 ++ 2 files changed, 25 insertions(+) create mode 100644 pkgs/applications/video/kodi/addons/pvr-vdr-vnsi/default.nix diff --git a/pkgs/applications/video/kodi/addons/pvr-vdr-vnsi/default.nix b/pkgs/applications/video/kodi/addons/pvr-vdr-vnsi/default.nix new file mode 100644 index 000000000000..4b5e8c6a7094 --- /dev/null +++ b/pkgs/applications/video/kodi/addons/pvr-vdr-vnsi/default.nix @@ -0,0 +1,23 @@ +{ lib, rel, buildKodiBinaryAddon, fetchFromGitHub, libGL }: +buildKodiBinaryAddon rec { + pname = "pvr-vdr-vnsi"; + namespace = "pvr.vdr.vnsi"; + version = "20.4.1"; + + src = fetchFromGitHub { + owner = "kodi-pvr"; + repo = "pvr.vdr.vnsi"; + rev = "${version}-${rel}"; + sha256 = "sha256-QooWK+LwlN5RAISjAQ2YiyDAjQQMzod8fFXpI0ll+hc="; + }; + + extraBuildInputs = [ libGL ]; + + meta = with lib; { + homepage = "https://github.com/kodi-pvr/pvr.vdr.vnsi"; + description = "Kodi's VDR VNSI PVR client addon"; + platforms = platforms.all; + license = licenses.gpl2Only; + maintainers = teams.kodi.members; + }; +} diff --git a/pkgs/top-level/kodi-packages.nix b/pkgs/top-level/kodi-packages.nix index b16f5acc99ac..80c080eff7ba 100644 --- a/pkgs/top-level/kodi-packages.nix +++ b/pkgs/top-level/kodi-packages.nix @@ -106,6 +106,8 @@ let self = rec { pvr-iptvsimple = callPackage ../applications/video/kodi/addons/pvr-iptvsimple { }; + pvr-vdr-vnsi = callPackage ../applications/video/kodi/addons/pvr-vdr-vnsi { }; + osmc-skin = callPackage ../applications/video/kodi/addons/osmc-skin { }; vfs-libarchive = callPackage ../applications/video/kodi/addons/vfs-libarchive { }; From 49d6a6ed8d8839c4d4d43f1ff16c011f638ad8dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 10 Jan 2024 14:31:28 -0500 Subject: [PATCH 09/10] readline: Support building for Windows with MinGW This commit is specifically designed to avoid a mass rebuild. I'll make a follow-up PR to staging which will make the patches unconditional, etc. Co-Authored-By: Weijia Wang <9713184+wegank@users.noreply.github.com> --- pkgs/development/libraries/readline/8.2.nix | 49 +++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/pkgs/development/libraries/readline/8.2.nix b/pkgs/development/libraries/readline/8.2.nix index 1c53da3cdfa4..72e3370576e7 100644 --- a/pkgs/development/libraries/readline/8.2.nix +++ b/pkgs/development/libraries/readline/8.2.nix @@ -1,4 +1,10 @@ -{ fetchurl, stdenv, lib, ncurses +{ lib, stdenv +, fetchpatch, fetchurl +, ncurses, termcap +, curses-library ? + if stdenv.hostPlatform.isWindows + then termcap + else ncurses }: stdenv.mkDerivation rec { @@ -13,7 +19,7 @@ stdenv.mkDerivation rec { outputs = [ "out" "dev" "man" "doc" "info" ]; strictDeps = true; - propagatedBuildInputs = [ ncurses ]; + propagatedBuildInputs = [ curses-library ]; patchFlags = [ "-p0" ]; @@ -27,11 +33,38 @@ stdenv.mkDerivation rec { in import ./readline-8.2-patches.nix patch); - patches = - [ ./link-against-ncurses.patch - ./no-arch_only-8.2.patch - ] - ++ upstreamPatches; + patches = lib.optionals (curses-library.pname == "ncurses") [ + ./link-against-ncurses.patch + ] ++ [ + ./no-arch_only-8.2.patch + ] + ++ upstreamPatches + ++ lib.optionals stdenv.hostPlatform.isWindows [ + (fetchpatch { + name = "0001-sigwinch.patch"; + url = "https://github.com/msys2/MINGW-packages/raw/90e7536e3b9c3af55c336d929cfcc32468b2f135/mingw-w64-readline/0001-sigwinch.patch"; + stripLen = 1; + hash = "sha256-sFK6EJrSNl0KLWqFv5zBXaQRuiQoYIZVoZfa8BZqfKA="; + }) + (fetchpatch { + name = "0002-event-hook.patch"; + url = "https://github.com/msys2/MINGW-packages/raw/3476319d2751a676b911f3de9e1ec675081c03b8/mingw-w64-readline/0002-event-hook.patch"; + stripLen = 1; + hash = "sha256-F8ytYuIjBtH83ZCJdf622qjwSw+wZEVyu53E/mPsoAo="; + }) + (fetchpatch { + name = "0003-fd_set.patch"; + url = "https://github.com/msys2/MINGW-packages/raw/35830ab27e5ed35c2a8d486961ab607109f5af50/mingw-w64-readline/0003-fd_set.patch"; + stripLen = 1; + hash = "sha256-UiaXZRPjKecpSaflBMCphI2kqOlcz1JkymlCrtpMng4="; + }) + (fetchpatch { + name = "0004-locale.patch"; + url = "https://github.com/msys2/MINGW-packages/raw/f768c4b74708bb397a77e3374cc1e9e6ef647f20/mingw-w64-readline/0004-locale.patch"; + stripLen = 1; + hash = "sha256-dk4343KP4EWXdRRCs8GRQlBgJFgu1rd79RfjwFD/nJc="; + }) + ]; meta = with lib; { description = "Library for interactive line editing"; @@ -57,7 +90,7 @@ stdenv.mkDerivation rec { maintainers = with maintainers; [ dtzWill ]; - platforms = platforms.unix; + platforms = platforms.unix ++ platforms.windows; branch = "8.2"; }; } From 176e84d44faa4dec9f5e0aad74e0dcd455a278d8 Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Thu, 11 Jan 2024 01:00:43 +0100 Subject: [PATCH 10/10] Revert "nixos/systemd-boot: init boot counting" --- .../manual/release-notes/rl-2405.section.md | 2 - .../boot/loader/systemd-boot/boot-counting.md | 38 --- .../systemd-boot/systemd-boot-builder.py | 216 +++++------------- .../boot/loader/systemd-boot/systemd-boot.nix | 16 +- nixos/modules/system/boot/systemd.nix | 4 - nixos/tests/systemd-boot.nix | 157 +------------ 6 files changed, 66 insertions(+), 367 deletions(-) delete mode 100644 nixos/modules/system/boot/loader/systemd-boot/boot-counting.md diff --git a/nixos/doc/manual/release-notes/rl-2405.section.md b/nixos/doc/manual/release-notes/rl-2405.section.md index 8387b1f77e3d..648064643930 100644 --- a/nixos/doc/manual/release-notes/rl-2405.section.md +++ b/nixos/doc/manual/release-notes/rl-2405.section.md @@ -16,8 +16,6 @@ In addition to numerous new and upgraded packages, this release has the followin - This can be disabled through the `environment.stub-ld.enable` option. - If you use `programs.nix-ld.enable`, no changes are needed. The stub will be disabled automatically. -- NixOS now has support for *automatic boot assessment* (see [here](https://systemd.io/AUTOMATIC_BOOT_ASSESSMENT/)) for detailed description of the feature) for `systemd-boot` users. Available as [boot.loader.systemd-boot.bootCounting](#opt-boot.loader.systemd-boot.bootCounting.enable). - - Julia environments can now be built with arbitrary packages from the ecosystem using the `.withPackages` function. For example: `julia.withPackages ["Plots"]`. ## New Services {#sec-release-24.05-new-services} diff --git a/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md b/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md deleted file mode 100644 index 736c54228452..000000000000 --- a/nixos/modules/system/boot/loader/systemd-boot/boot-counting.md +++ /dev/null @@ -1,38 +0,0 @@ -# Automatic boot assessment with systemd-boot {#sec-automatic-boot-assessment} - -## Overview {#sec-automatic-boot-assessment-overview} - -Automatic boot assessment (or boot-counting) is a feature of `systemd-boot` that allows for automatically detecting invalid boot entries. -When the feature is active, each boot entry has an associated counter with a user defined number of trials. Whenever `system-boot` boots an entry, its counter is decreased by one, ultimately being marked as *bad* if the counter ever reaches zero. However, if an entry is successfully booted, systemd will permanently mark it as *good* and remove the counter altogether. Whenever an entry is marked as *bad*, it is sorted last in the systemd-boot menu. -A complete explanation of how that feature works can be found [here](https://systemd.io/AUTOMATIC_BOOT_ASSESSMENT/). - -## Enabling the feature {#sec-automatic-boot-assessment-enable} - -The feature can be enabled by toogling the [boot.loader.systemd-boot.bootCounting](#opt-boot.loader.systemd-boot.bootCounting.enable) option. - -## The boot-complete.target unit {#sec-automatic-boot-assessment-boot-complete-target} - -A *successful boot* for an entry is defined in terms of the `boot-complete.target` synchronisation point. It is up to the user to schedule all necessary units for the machine to be considered successfully booted before that synchronisation point. -For example, if you are running `nsd`, an authoritative DNS server on a machine and you want to be sure that a *good* entry is an entry where that DNS server is started successfully. A configuration for that NixOS machine could look like that: - -``` -boot.loader.systemd-boot.bootCounting.enable = true; -services.nsd.enable = true; -/* rest of nsd configuration omitted */ - -systemd.services.nsd = { - before = [ "boot-complete.target" ]; - wantedBy = [ "boot-complete.target" ]; - unitConfig.FailureAction = "reboot"; -}; - -``` - -## Interaction with specialisations {#sec-automatic-boot-assessment-specialisations} - -When the boot-counting feature is enabled, `systemd-boot` will still try the boot entries in the same order as they are displayed in the boot menu. This means that the specialisations of a given generation will be tried directly after that generation. A generation being marked as *bad* do not mean that its specialisations will also be marked as *bad* (as its specialisations could very well be booting successfully). - - -## Limitations {#sec-automatic-boot-assessment-limitations} - -This feature has to be used wisely to not risk any data integrity issues. Rollbacking into past generations can sometimes be dangerous, for example if some of the services may have undefined behaviors in the presence of unrecognized data migrations from future versions of themselves. diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py index 75f32459e6b5..6cd46f30373b 100644 --- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py +++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py @@ -12,9 +12,8 @@ import subprocess import sys import warnings import json -from typing import NamedTuple, Dict, List, Type, Generator, Iterable +from typing import NamedTuple, Dict, List from dataclasses import dataclass -from pathlib import Path @dataclass @@ -29,114 +28,7 @@ class BootSpec: specialisations: Dict[str, "BootSpec"] initrdSecrets: str | None = None -@dataclass -class Entry: - profile: str | None - generation_number: int - specialisation: str | None - @classmethod - def from_path(cls: Type["Entry"], path: Path) -> "Entry": - filename = path.name - # Matching nixos-$profile-generation-*.conf - rex_profile = re.compile(r"^nixos-(.*)-generation-.*\.conf$") - # Matching nixos*-generation-$number*.conf - rex_generation = re.compile(r"^nixos.*-generation-([0-9]+).*\.conf$") - # Matching nixos*-generation-$number-specialisation-$specialisation_name*.conf - rex_specialisation = re.compile(r"^nixos.*-generation-([0-9]+)-specialisation-([a-zA-Z0-9]+).*\.conf$") - profile = rex_profile.sub(r"\1", filename) if rex_profile.match(filename) else None - specialisation = rex_specialisation.sub(r"\2", filename) if rex_specialisation.match(filename) else None - try: - generation_number = int(rex_generation.sub(r"\1", filename)) - except ValueError: - raise - return cls(profile, generation_number, specialisation) - - -BOOT_ENTRY = """title {title} -version Generation {generation} {description} -linux {kernel} -initrd {initrd} -options {kernel_params} -machine-id {machine_id} -sort-key {sort_key} -""" - -@dataclass -class DiskEntry(): - entry: Entry - default: bool - counters: str | None - title: str - description: str - kernel: str - initrd: str - kernel_params: str - machine_id: str - - @classmethod - def from_path(cls: Type["DiskEntry"], path: Path) -> "DiskEntry": - entry = Entry.from_path(path) - with open(path, 'r') as f: - data = f.read().splitlines() - if '' in data: - data.remove('') - entry_map = dict(l.split(' ', 1) for l in data) - assert "title" in entry_map - assert "version" in entry_map - version_splitted = entry_map["version"].split(" ", 2) - assert version_splitted[0] == "Generation" - assert version_splitted[1].isdigit() - assert "linux" in entry_map - assert "initrd" in entry_map - assert "options" in entry_map - assert "machine-id" in entry_map - assert "sort-key" in entry_map - filename = path.name - # Matching nixos*-generation-*$counters.conf - rex_counters = re.compile(r"^nixos.*-generation-.*(\+\d(-\d)?)\.conf$") - counters = rex_counters.sub(r"\1", filename) if rex_counters.match(filename) else None - disk_entry = cls( - entry=entry, - default=(entry_map["sort-key"] == "default"), - counters=counters, - title=entry_map["title"], - description=entry_map["version"], - kernel=entry_map["linux"], - initrd=entry_map["initrd"], - kernel_params=entry_map["options"], - machine_id=entry_map["machine-id"]) - return disk_entry - - def write(self) -> None: - tmp_path = self.path.with_suffix(".tmp") - with tmp_path.open('w') as f: - # We use "sort-key" to sort the default generation first. - # The "default" string is sorted before "non-default" (alphabetically) - f.write(BOOT_ENTRY.format(title=self.title, - generation=self.entry.generation_number, - kernel=self.kernel, - initrd=self.initrd, - kernel_params=self.kernel_params, - machine_id=self.machine_id, - description=self.description, - sort_key="default" if self.default else "non-default")) - f.flush() - os.fsync(f.fileno()) - tmp_path.rename(self.path) - - - @property - def path(self) -> Path: - pieces = [ - "nixos", - self.entry.profile or None, - "generation", - str(self.entry.generation_number), - f"specialisation-{self.entry.specialisation}" if self.entry.specialisation else None, - ] - prefix = "-".join(p for p in pieces if p) - return Path(f"@efiSysMountPoint@/loader/entries/{prefix}{self.counters if self.counters else ''}.conf") libc = ctypes.CDLL("libc.so.6") @@ -164,14 +56,29 @@ def system_dir(profile: str | None, generation: int, specialisation: str | None) else: return d -def write_loader_conf(profile: str | None) -> None: +BOOT_ENTRY = """title {title} +version Generation {generation} {description} +linux {kernel} +initrd {initrd} +options {kernel_params} +""" + +def generation_conf_filename(profile: str | None, generation: int, specialisation: str | None) -> str: + pieces = [ + "nixos", + profile or None, + "generation", + str(generation), + f"specialisation-{specialisation}" if specialisation else None, + ] + return "-".join(p for p in pieces if p) + ".conf" + + +def write_loader_conf(profile: str | None, generation: int, specialisation: str | None) -> None: with open("@efiSysMountPoint@/loader/loader.conf.tmp", 'w') as f: if "@timeout@" != "": f.write("timeout @timeout@\n") - if profile: - f.write("default nixos-%s-generation-*\n" % profile) - else: - f.write("default nixos-generation-*\n") + f.write("default %s\n" % generation_conf_filename(profile, generation, specialisation)) if not @editor@: f.write("editor 0\n") f.write("console-mode @consoleMode@\n") @@ -179,17 +86,6 @@ def write_loader_conf(profile: str | None) -> None: os.fsync(f.fileno()) os.rename("@efiSysMountPoint@/loader/loader.conf.tmp", "@efiSysMountPoint@/loader/loader.conf") -def scan_entries() -> Generator[DiskEntry, None, None]: - """ - Scan all entries in $ESP/loader/entries/* - Does not support Type 2 entries as we do not support them for now. - Returns a generator of Entry. - """ - for path in Path("@efiSysMountPoint@/loader/entries/").glob("nixos*-generation-[1-9]*.conf"): - try: - yield DiskEntry.from_path(path) - except ValueError: - continue def get_bootspec(profile: str | None, generation: int) -> BootSpec: system_directory = system_dir(profile, generation, None) @@ -224,7 +120,7 @@ def copy_from_file(file: str, dry_run: bool = False) -> str: return efi_file_path def write_entry(profile: str | None, generation: int, specialisation: str | None, - machine_id: str, bootspec: BootSpec, entries: Iterable[DiskEntry], current: bool) -> None: + machine_id: str, bootspec: BootSpec, current: bool) -> None: if specialisation: bootspec = bootspec.specialisations[specialisation] kernel = copy_from_file(bootspec.kernel) @@ -246,30 +142,28 @@ def write_entry(profile: str | None, generation: int, specialisation: str | None f'for "{title} - Configuration {generation}", an older generation', file=sys.stderr) print("note: this is normal after having removed " "or renamed a file in `boot.initrd.secrets`", file=sys.stderr) + entry_file = "@efiSysMountPoint@/loader/entries/%s" % ( + generation_conf_filename(profile, generation, specialisation)) + tmp_path = "%s.tmp" % (entry_file) kernel_params = "init=%s " % bootspec.init + kernel_params = kernel_params + " ".join(bootspec.kernelParams) build_time = int(os.path.getctime(system_dir(profile, generation, specialisation))) build_date = datetime.datetime.fromtimestamp(build_time).strftime('%F') - counters = "+@bootCountingTrials@" if @bootCounting@ else "" - entry = Entry(profile, generation, specialisation) - # We check if the entry we are writing is already on disk - # and we update its "default entry" status - for entry_on_disk in entries: - if entry == entry_on_disk.entry: - entry_on_disk.default = current - entry_on_disk.write() - return - DiskEntry( - entry=entry, - title=title, - kernel=kernel, - initrd=initrd, - counters=counters, - kernel_params=kernel_params, - machine_id=machine_id, - description=f"{bootspec.label}, built on {build_date}", - default=current).write() + with open(tmp_path, 'w') as f: + f.write(BOOT_ENTRY.format(title=title, + generation=generation, + kernel=kernel, + initrd=initrd, + kernel_params=kernel_params, + description=f"{bootspec.label}, built on {build_date}")) + if machine_id is not None: + f.write("machine-id %s\n" % machine_id) + f.flush() + os.fsync(f.fileno()) + os.rename(tmp_path, entry_file) + def get_generations(profile: str | None = None) -> list[SystemIdentifier]: gen_list = subprocess.check_output([ @@ -294,18 +188,29 @@ def get_generations(profile: str | None = None) -> list[SystemIdentifier]: return configurations[-configurationLimit:] -def remove_old_entries(gens: list[SystemIdentifier], disk_entries: Iterable[DiskEntry]) -> None: +def remove_old_entries(gens: list[SystemIdentifier]) -> None: + rex_profile = re.compile(r"^@efiSysMountPoint@/loader/entries/nixos-(.*)-generation-.*\.conf$") + rex_generation = re.compile(r"^@efiSysMountPoint@/loader/entries/nixos.*-generation-([0-9]+)(-specialisation-.*)?\.conf$") known_paths = [] for gen in gens: bootspec = get_bootspec(gen.profile, gen.generation) known_paths.append(copy_from_file(bootspec.kernel, True)) known_paths.append(copy_from_file(bootspec.initrd, True)) - for disk_entry in disk_entries: - if (disk_entry.entry.profile, disk_entry.entry.generation_number, None) not in gens: - os.unlink(disk_entry.path) - for path in glob.iglob("@efiSysMountPoint@/efi/nixos/*"): - if path not in known_paths and not os.path.isdir(path): + for path in glob.iglob("@efiSysMountPoint@/loader/entries/nixos*-generation-[1-9]*.conf"): + if rex_profile.match(path): + prof = rex_profile.sub(r"\1", path) + else: + prof = None + try: + gen_number = int(rex_generation.sub(r"\1", path)) + except ValueError: + continue + if not (prof, gen_number, None) in gens: os.unlink(path) + for path in glob.iglob("@efiSysMountPoint@/efi/nixos/*"): + if not path in known_paths and not os.path.isdir(path): + os.unlink(path) + def get_profiles() -> list[str]: if os.path.isdir("/nix/var/nix/profiles/system-profiles/"): @@ -379,17 +284,16 @@ def install_bootloader(args: argparse.Namespace) -> None: gens = get_generations() for profile in get_profiles(): gens += get_generations(profile) - entries = scan_entries() - remove_old_entries(gens, entries) + remove_old_entries(gens) for gen in gens: try: bootspec = get_bootspec(gen.profile, gen.generation) is_default = os.path.dirname(bootspec.init) == args.default_config - write_entry(*gen, machine_id, bootspec, entries, current=is_default) + write_entry(*gen, machine_id, bootspec, current=is_default) for specialisation in bootspec.specialisations.keys(): - write_entry(gen.profile, gen.generation, specialisation, machine_id, bootspec, entries, current=is_default) + write_entry(gen.profile, gen.generation, specialisation, machine_id, bootspec, current=is_default) if is_default: - write_loader_conf(gen.profile) + write_loader_conf(*gen) except OSError as e: # See https://github.com/NixOS/nixpkgs/issues/114552 if e.errno == errno.EINVAL: diff --git a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix index 3a4173210f5f..9d55c21077d1 100644 --- a/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix +++ b/nixos/modules/system/boot/loader/systemd-boot/systemd-boot.nix @@ -49,8 +49,6 @@ let ${pkgs.coreutils}/bin/install -D $empty_file "${efi.efiSysMountPoint}/efi/nixos/.extra-files/loader/entries/"${escapeShellArg n} '') cfg.extraEntries)} ''; - bootCountingTrials = cfg.bootCounting.trials; - bootCounting = if cfg.bootCounting.enable then "True" else "False"; }; checkedSystemdBootBuilder = pkgs.runCommand "systemd-boot" { @@ -71,10 +69,7 @@ let ''; in { - meta = { - maintainers = with lib.maintainers; [ julienmalka ]; - doc = ./boot-counting.md; - }; + meta.maintainers = with lib.maintainers; [ julienmalka ]; imports = [ (mkRenamedOptionModule [ "boot" "loader" "gummiboot" "enable" ] [ "boot" "loader" "systemd-boot" "enable" ]) @@ -243,15 +238,6 @@ in { ''; }; - bootCounting = { - enable = mkEnableOption (lib.mdDoc "automatic boot assessment"); - trials = mkOption { - default = 3; - type = types.int; - description = lib.mdDoc "number of trials each entry should start with"; - }; - }; - }; config = mkIf cfg.enable { diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix index b315fc91e3d0..87333999313e 100644 --- a/nixos/modules/system/boot/systemd.nix +++ b/nixos/modules/system/boot/systemd.nix @@ -101,10 +101,6 @@ let "systemd-rfkill.service" "systemd-rfkill.socket" - # Boot counting - "boot-complete.target" - ] ++ lib.optional config.boot.loader.systemd-boot.bootCounting.enable "systemd-bless-boot.service" ++ [ - # Hibernate / suspend. "hibernate.target" "suspend.target" diff --git a/nixos/tests/systemd-boot.nix b/nixos/tests/systemd-boot.nix index 6e956ab74fbe..c0b37a230df0 100644 --- a/nixos/tests/systemd-boot.nix +++ b/nixos/tests/systemd-boot.nix @@ -13,11 +13,9 @@ let boot.loader.systemd-boot.enable = true; boot.loader.efi.canTouchEfiVariables = true; environment.systemPackages = [ pkgs.efibootmgr ]; - # Needed for machine-id to be persisted between reboots - environment.etc."machine-id".text = "00000000000000000000000000000000"; }; in -rec { +{ basic = makeTest { name = "systemd-boot"; meta.maintainers = with pkgs.lib.maintainers; [ danielfullmer julienmalka ]; @@ -254,15 +252,15 @@ rec { ''; }; - garbage-collect-entry = { withBootCounting ? false, ... }: makeTest { - name = "systemd-boot-garbage-collect-entry" + optionalString withBootCounting "-with-boot-counting"; + garbage-collect-entry = makeTest { + name = "systemd-boot-garbage-collect-entry"; meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; nodes = { inherit common; machine = { pkgs, nodes, ... }: { imports = [ common ]; - boot.loader.systemd-boot.bootCounting.enable = withBootCounting; + # These are configs for different nodes, but we'll use them here in `machine` system.extraDependencies = [ nodes.common.system.build.toplevel @@ -277,12 +275,8 @@ rec { '' machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${baseSystem}") machine.succeed("nix-env -p /nix/var/nix/profiles/system --delete-generations 1") - # At this point generation 1 has already been marked as good so we reintroduce counters artificially - ${optionalString withBootCounting '' - machine.succeed("mv /boot/loader/entries/nixos-generation-1.conf /boot/loader/entries/nixos-generation-1+3.conf") - ''} machine.succeed("${baseSystem}/bin/switch-to-configuration boot") - machine.fail("test -e /boot/loader/entries/nixos-generation-1*") + machine.fail("test -e /boot/loader/entries/nixos-generation-1.conf") machine.succeed("test -e /boot/loader/entries/nixos-generation-2.conf") ''; }; @@ -328,145 +322,4 @@ rec { machine.wait_for_unit("multi-user.target") ''; }; - - # Check that we are booting the default entry and not the generation with largest version number - defaultEntry = { withBootCounting ? false, ... }: makeTest { - name = "systemd-boot-default-entry" + optionalString withBootCounting "-with-boot-counting"; - meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; - - nodes = { - machine = { pkgs, lib, nodes, ... }: { - imports = [ common ]; - system.extraDependencies = [ nodes.other_machine.system.build.toplevel ]; - boot.loader.systemd-boot.bootCounting.enable = withBootCounting; - }; - - other_machine = { pkgs, lib, ... }: { - imports = [ common ]; - boot.loader.systemd-boot.bootCounting.enable = withBootCounting; - environment.systemPackages = [ pkgs.hello ]; - }; - }; - testScript = { nodes, ... }: - let - orig = nodes.machine.system.build.toplevel; - other = nodes.other_machine.system.build.toplevel; - in - '' - orig = "${orig}" - other = "${other}" - - def check_current_system(system_path): - machine.succeed(f'test $(readlink -f /run/current-system) = "{system_path}"') - - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - check_current_system(orig) - - # Switch to other configuration - machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${other}") - machine.succeed(f"{other}/bin/switch-to-configuration boot") - # Rollback, default entry is now generation 1 - machine.succeed("nix-env -p /nix/var/nix/profiles/system --rollback") - machine.succeed(f"{orig}/bin/switch-to-configuration boot") - - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - ${if withBootCounting - then ''machine.succeed("test -e /boot/loader/entries/nixos-generation-2+3.conf")'' - else ''machine.succeed("test -e /boot/loader/entries/nixos-generation-2.conf")''} - machine.shutdown() - - machine.start() - machine.wait_for_unit("multi-user.target") - # Check that we booted generation 1 (default) - # even though generation 2 comes first in alphabetical order - check_current_system(orig) - ''; - }; - - - bootCounting = - let - baseConfig = { pkgs, lib, ... }: { - imports = [ common ]; - boot.loader.systemd-boot.bootCounting.enable = true; - boot.loader.systemd-boot.bootCounting.trials = 2; - }; - in - makeTest { - name = "systemd-boot-counting"; - meta.maintainers = with pkgs.lib.maintainers; [ julienmalka ]; - - nodes = { - machine = { pkgs, lib, nodes, ... }: { - imports = [ baseConfig ]; - system.extraDependencies = [ nodes.bad_machine.system.build.toplevel ]; - }; - - bad_machine = { pkgs, lib, ... }: { - imports = [ baseConfig ]; - - systemd.services."failing" = { - script = "exit 1"; - requiredBy = [ "boot-complete.target" ]; - before = [ "boot-complete.target" ]; - serviceConfig.Type = "oneshot"; - }; - }; - }; - testScript = { nodes, ... }: - let - orig = nodes.machine.system.build.toplevel; - bad = nodes.bad_machine.system.build.toplevel; - in - '' - orig = "${orig}" - bad = "${bad}" - - def check_current_system(system_path): - machine.succeed(f'test $(readlink -f /run/current-system) = "{system_path}"') - - # Ensure we booted using an entry with counters enabled - machine.succeed( - "test -e /sys/firmware/efi/efivars/LoaderBootCountPath-4a67b082-0a4c-41cf-b6c7-440b29bb8c4f" - ) - - # systemd-bless-boot should have already removed the "+2" suffix from the boot entry - machine.wait_for_unit("systemd-bless-boot.service") - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - check_current_system(orig) - - # Switch to bad configuration - machine.succeed("nix-env -p /nix/var/nix/profiles/system --set ${bad}") - machine.succeed(f"{bad}/bin/switch-to-configuration boot") - - # Ensure new bootloader entry has initialized counter - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - machine.succeed("test -e /boot/loader/entries/nixos-generation-2+2.conf") - machine.shutdown() - - machine.start() - machine.wait_for_unit("multi-user.target") - check_current_system(bad) - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - machine.succeed("test -e /boot/loader/entries/nixos-generation-2+1-1.conf") - machine.shutdown() - - machine.start() - machine.wait_for_unit("multi-user.target") - check_current_system(bad) - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - machine.succeed("test -e /boot/loader/entries/nixos-generation-2+0-2.conf") - machine.shutdown() - - # Should boot back into original configuration - machine.start() - check_current_system(orig) - machine.wait_for_unit("multi-user.target") - machine.succeed("test -e /boot/loader/entries/nixos-generation-1.conf") - machine.succeed("test -e /boot/loader/entries/nixos-generation-2+0-2.conf") - machine.shutdown() - ''; - }; - defaultEntryWithBootCounting = defaultEntry { withBootCounting = true; }; - garbageCollectEntryWithBootCounting = garbage-collect-entry { withBootCounting = true; }; }