From b0c781cc4136e4678db1864875750c916b78ad33 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 15 Jun 2022 16:59:21 +0200 Subject: [PATCH] nixos/testing: Move entrypoint to nixos/lib + doc --- .../writing-nixos-tests.section.md | 73 +++++- .../writing-nixos-tests.section.xml | 217 ++++++++++++------ nixos/lib/default.nix | 8 + nixos/lib/testing-python.nix | 31 +-- nixos/lib/testing/default.nix | 24 ++ 5 files changed, 256 insertions(+), 97 deletions(-) create mode 100644 nixos/lib/testing/default.nix diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md index 6934bb0face7..8dd3e6fb7597 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.section.md +++ b/nixos/doc/manual/development/writing-nixos-tests.section.md @@ -1,9 +1,9 @@ # Writing Tests {#sec-writing-nixos-tests} -A NixOS test is a Nix expression that has the following structure: +A NixOS test is a module that has the following structure: ```nix -import ./make-test-python.nix { +{ # One or more machines: nodes = @@ -21,7 +21,10 @@ import ./make-test-python.nix { } ``` -The attribute `testScript` is a bit of Python code that executes the +We refer to the whole test above as a test module, whereas the values +in `nodes.` are NixOS modules. (A NixOS configuration is a module.) + +The option `testScript` is a bit of Python code that executes the test (described below). During the test, it will start one or more virtual machines, the configuration of which is described by the attribute `nodes`. @@ -34,7 +37,64 @@ when switching between consoles, and so on. An interesting multi-node test is [`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix). It uses two client nodes to test correct locking across server crashes. -There are a few special NixOS configuration options for test VMs: +## Calling a test {#sec-calling-nixos-tests} + +Tests are invoked a bit differently depending on whether the test lives in NixOS or in another project. + +### Testing within NixOS {#sec-call-nixos-test-in-nixos} + +Test modules can be instantiated into derivations in multiple ways. + +Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix). + +```nix + hostname = runTest ./hostname.nix; +``` + +Overrides can be added by defining an anonymous module in `all-tests.nix`. +For the purpose of constructing a test matrix, use the `matrix` options instead. + +```nix + hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; }; +``` + +You can run a test with attribute name `mytest` in `all-tests.nix` by invoking: + +```shell +nix-build -A nixosTests.mytest +``` + +### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos} + +Outside the `nixpkgs` repository, you can instantiate the test by first acquiring the NixOS library, + +```nix +# regular nix +let nixos-lib = import (nixpkgs + "/nixos/lib") { }; +in +``` + +```nix +# flake +let nixos-lib = nixpkgs.lib.nixos; +in +``` + +... and then invoking `runTest`, for example: + +```nix +nixos-lib.runTest { + imports = [ ./test.nix ]; + hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs + defaults.services.foo.package = mypkg; +} +``` + +`runTest` returns a derivation that runs the test. + +## Configuring the nodes {#sec-nixos-test-nodes} + +There are a few special NixOS options for test VMs: `virtualisation.memorySize` @@ -304,7 +364,7 @@ For faster dev cycles it\'s also possible to disable the code-linters (this shouldn\'t be commited though): ```nix -import ./make-test-python.nix { +{ skipLint = true; nodes.machine = { config, pkgs, ... }: @@ -336,7 +396,7 @@ Similarly, the type checking of test scripts can be disabled in the following way: ```nix -import ./make-test-python.nix { +{ skipTypeCheck = true; nodes.machine = { config, pkgs, ... }: @@ -400,7 +460,6 @@ added using the parameter `extraPythonPackages`. For example, you could add `numpy` like this: ```nix -import ./make-test-python.nix { extraPythonPackages = p: [ p.numpy ]; diff --git a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml index d6f4f61c0645..6d0465e4230b 100644 --- a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml +++ b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml @@ -1,10 +1,10 @@
Writing Tests - A NixOS test is a Nix expression that has the following structure: + A NixOS test is a module that has the following structure: -import ./make-test-python.nix { +{ # One or more machines: nodes = @@ -22,7 +22,12 @@ import ./make-test-python.nix { } - The attribute testScript is a bit of Python code + We refer to the whole test above as a test module, whereas the + values in nodes.<name> are NixOS modules. + (A NixOS configuration is a module.) + + + The option testScript is a bit of Python code that executes the test (described below). During the test, it will start one or more virtual machines, the configuration of which is described by the attribute nodes. @@ -38,78 +43,149 @@ import ./make-test-python.nix { It uses two client nodes to test correct locking across server crashes. - - There are a few special NixOS configuration options for test VMs: - - - - - virtualisation.memorySize - - - - The memory of the VM in megabytes. - - - - - - virtualisation.vlans - - - - The virtual networks to which the VM is connected. See - nat.nix - for an example. - - - - - - virtualisation.writableStore - - - - By default, the Nix store in the VM is not writable. If you - enable this option, a writable union file system is mounted on - top of the Nix store to make it appear writable. This is - necessary for tests that run Nix operations that modify the - store. - - - - - - For more options, see the module - qemu-vm.nix. - - - The test script is a sequence of Python statements that perform - various actions, such as starting VMs, executing commands in the - VMs, and so on. Each virtual machine is represented as an object - stored in the variable name if this is also the - identifier of the machine in the declarative config. If you - specified a node nodes.machine, the following - example starts the machine, waits until it has finished booting, - then executes a command and checks that the output is more-or-less - correct: - - +
+ Calling a test + + Tests are invoked a bit differently depending on whether the test + lives in NixOS or in another project. + +
+ Testing within NixOS + + Test modules can be instantiated into derivations in multiple + ways. + + + Tests that are part of NixOS are added to + nixos/tests/all-tests.nix. + + + hostname = runTest ./hostname.nix; + + + Overrides can be added by defining an anonymous module in + all-tests.nix. For the purpose of + constructing a test matrix, use the matrix + options instead. + + + hostname = runTest { imports = [ ./hostname.nix ]; defaults.networking.firewall.enable = false; }; + + + You can run a test with attribute name mytest + in all-tests.nix by invoking: + + +nix-build -A nixosTests.mytest + +
+
+ Testing outside the NixOS project + + Outside the nixpkgs repository, you can + instantiate the test by first acquiring the NixOS library, + + +# regular nix +let nixos-lib = import (nixpkgs + "/nixos/lib") { }; +in + + +# flake +let nixos-lib = nixpkgs.lib.nixos; +in + + + … and then invoking runTest, for example: + + +nixos-lib.runTest { + imports = [ ./test.nix ]; + hostPkgs = pkgs; # the Nixpkgs package set used outside the VMs + defaults.services.foo.package = mypkg; +} + + + runTest returns a derivation that runs the + test. + +
+
+
+ Configuring the nodes + + There are a few special NixOS options for test VMs: + + + + + virtualisation.memorySize + + + + The memory of the VM in megabytes. + + + + + + virtualisation.vlans + + + + The virtual networks to which the VM is connected. See + nat.nix + for an example. + + + + + + virtualisation.writableStore + + + + By default, the Nix store in the VM is not writable. If you + enable this option, a writable union file system is mounted + on top of the Nix store to make it appear writable. This is + necessary for tests that run Nix operations that modify the + store. + + + + + + For more options, see the module + qemu-vm.nix. + + + The test script is a sequence of Python statements that perform + various actions, such as starting VMs, executing commands in the + VMs, and so on. Each virtual machine is represented as an object + stored in the variable name if this is also the + identifier of the machine in the declarative config. If you + specified a node nodes.machine, the following + example starts the machine, waits until it has finished booting, + then executes a command and checks that the output is more-or-less + correct: + + machine.start() machine.wait_for_unit("default.target") if not "Linux" in machine.succeed("uname"): raise Exception("Wrong OS") - - The first line is technically unnecessary; machines are implicitly - started when you first execute an action on them (such as - wait_for_unit or succeed). If - you have multiple machines, you can speed up the test by starting - them in parallel: - - + + The first line is technically unnecessary; machines are implicitly + started when you first execute an action on them (such as + wait_for_unit or succeed). + If you have multiple machines, you can speed up the test by + starting them in parallel: + + start_all() +
Machine objects @@ -563,7 +639,7 @@ machine.wait_for_unit("xautolock.service", "x-session-user") code-linters (this shouldn't be commited though): -import ./make-test-python.nix { +{ skipLint = true; nodes.machine = { config, pkgs, ... }: @@ -595,7 +671,7 @@ import ./make-test-python.nix { the following way: -import ./make-test-python.nix { +{ skipTypeCheck = true; nodes.machine = { config, pkgs, ... }: @@ -669,7 +745,6 @@ def foo_running(): numpy like this: -import ./make-test-python.nix { extraPythonPackages = p: [ p.numpy ]; diff --git a/nixos/lib/default.nix b/nixos/lib/default.nix index 2b3056e01457..65d91342d4d1 100644 --- a/nixos/lib/default.nix +++ b/nixos/lib/default.nix @@ -21,6 +21,8 @@ let seqAttrsIf = cond: a: lib.mapAttrs (_: v: seqIf cond a v); eval-config-minimal = import ./eval-config-minimal.nix { inherit lib; }; + + testing-lib = import ./testing/default.nix { inherit lib; }; in /* This attribute set appears as lib.nixos in the flake, or can be imported @@ -30,4 +32,10 @@ in inherit (seqAttrsIf (!featureFlags?minimalModules) minimalModulesWarning eval-config-minimal) evalModules ; + + inherit (testing-lib) + evalTest + runTest + ; + } diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 1c331a3c5162..e72e5d476bfa 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -12,6 +12,10 @@ with pkgs; +let + nixos-lib = import ./default.nix { inherit (pkgs) lib; }; +in + rec { inherit pkgs; @@ -166,26 +170,15 @@ rec { ${lib.optionalString (interactive) "--add-flags --interactive"} ''); - evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; - runTest = module: (evalTest module).config.run; + evalTest = module: nixos-lib.evalTest { imports = [ extraTestModule module ]; }; + runTest = module: nixos-lib.runTest { imports = [ extraTestModule module ]; }; - testModules = [ - ./testing/driver.nix - ./testing/interactive.nix - ./testing/legacy.nix - ./testing/meta.nix - ./testing/name.nix - ./testing/network.nix - ./testing/nodes.nix - ./testing/pkgs.nix - ./testing/run.nix - ./testing/testScript.nix - { - config = { - hostPkgs = pkgs; - }; - } - ]; + extraTestModule = { + config = { + hostPkgs = pkgs; + minimalResult = hydra; + }; + }; # Make a full-blown test makeTest = diff --git a/nixos/lib/testing/default.nix b/nixos/lib/testing/default.nix new file mode 100644 index 000000000000..676d52f5c3fb --- /dev/null +++ b/nixos/lib/testing/default.nix @@ -0,0 +1,24 @@ +{ lib }: +let + + evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; }; + runTest = module: (evalTest module).config.result; + + testModules = [ + ./call-test.nix + ./driver.nix + ./interactive.nix + ./legacy.nix + ./meta.nix + ./name.nix + ./network.nix + ./nodes.nix + ./pkgs.nix + ./run.nix + ./testScript.nix + ]; + +in +{ + inherit evalTest runTest testModules; +}