Merge staging-next into staging

nixos/modules/services/system/cloud-init.nix

@@ -81,7 +81,8 @@ in
            - write-files
            - growpart
            - resizefs
-           - update_etc_hosts
+           - update_hostname
+           - resolv_conf
            - ca-certs
            - rsyslog
            - users-groups

nixos/modules/system/boot/networkd.nix

@@ -584,6 +584,7 @@ let
           "Label"
           "PreferredLifetime"
           "Scope"
+          "RouteMetric"
           "HomeAddress"
           "DuplicateAddressDetection"
           "ManageTemporaryAddress"
@@ -592,6 +593,7 @@ let
         ])
         (assertHasField "Address")
         (assertValueOneOf "PreferredLifetime" ["forever" "infinity" "0" 0])
+        (assertInt "RouteMetric")
         (assertValueOneOf "HomeAddress" boolValues)
         (assertValueOneOf "DuplicateAddressDetection" ["ipv4" "ipv6" "both" "none"])
         (assertValueOneOf "ManageTemporaryAddress" boolValues)

nixos/tests/all-tests.nix

@@ -125,6 +125,7 @@ in {
   cjdns = handleTest ./cjdns.nix {};
   clickhouse = handleTest ./clickhouse.nix {};
   cloud-init = handleTest ./cloud-init.nix {};
+  cloud-init-hostname = handleTest ./cloud-init-hostname.nix {};
   cntr = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cntr.nix {};
   cockroachdb = handleTestOn ["x86_64-linux"] ./cockroachdb.nix {};
   collectd = handleTest ./collectd.nix {};

nixos/tests/cloud-init-hostname.nix

@@ -0,0 +1,46 @@
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
+
+let
+  # Hostname can also be set through "hostname" in user-data.
+  # This is how proxmox configures hostname through cloud-init.
+  metadataDrive = pkgs.stdenv.mkDerivation {
+    name = "metadata";
+    buildCommand = ''
+      mkdir -p $out/iso
+
+      cat << EOF > $out/iso/user-data
+      #cloud-config
+      hostname: testhostname
+      EOF
+
+      cat << EOF > $out/iso/meta-data
+      instance-id: iid-local02
+      EOF
+
+      ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso
+    '';
+  };
+
+in makeTest {
+  name = "cloud-init-hostname";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ lewo illustris ];
+  };
+
+  nodes.machine2 = { ... }: {
+    virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ];
+    services.cloud-init.enable = true;
+    networking.hostName = "";
+  };
+
+  testScript = ''
+    unnamed.wait_for_unit("cloud-final.service")
+    assert "testhostname" in unnamed.succeed("hostname")
+  '';
+}

nixos/tests/cloud-init.nix

@@ -49,19 +49,17 @@ let
               gateway: '12.34.56.9'
           - type: nameserver
             address:
-            - '8.8.8.8'
+            - '6.7.8.9'
             search:
             - 'example.com'
       EOF
       ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso
       '';
   };
+
 in makeTest {
   name = "cloud-init";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ lewo ];
-    broken = true; # almost always times out after spending many hours
-  };
+  meta.maintainers = with pkgs.lib.maintainers; [ lewo illustris ];
   nodes.machine = { ... }:
   {
     virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ];
@@ -90,21 +88,27 @@ in makeTest {
 
     # we should be able to log in as the root user, as well as the created nixos user
     unnamed.succeed(
-        "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil root@localhost 'true'"
+        "timeout 10 ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil root@localhost 'true'"
     )
     unnamed.succeed(
-        "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'true'"
+        "timeout 10 ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'true'"
     )
 
     # test changing hostname via cloud-init worked
     assert (
         unnamed.succeed(
-            "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'hostname'"
+            "timeout 10 ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'hostname'"
         ).strip()
         == "test"
     )
 
+    # check IP and route configs
     assert "default via 12.34.56.9 dev eth0 proto static" in unnamed.succeed("ip route")
     assert "12.34.56.0/24 dev eth0 proto kernel scope link src 12.34.56.78" in unnamed.succeed("ip route")
+
+    # check nameserver and search configs
+    assert "6.7.8.9" in unnamed.succeed("resolvectl status")
+    assert "example.com" in unnamed.succeed("resolvectl status")
+
   '';
 }

pkgs/applications/networking/browsers/librewolf/default.nix

@@ -9,7 +9,7 @@ in
   binaryName = "librewolf";
   version = librewolf-src.packageVersion;
   src = librewolf-src.firefox;
-  inherit (librewolf-src) extraConfigureFlags extraPostPatch extraPassthru;
+  inherit (librewolf-src) extraConfigureFlags extraPatches extraPostPatch extraPassthru;
 
   meta = {
     description = "A fork of Firefox, focused on privacy, security and freedom";

pkgs/applications/networking/browsers/librewolf/librewolf.nix

@@ -1,4 +1,4 @@
-{ callPackage }:
+{ callPackage, lib, stdenv, fetchpatch }:
 let
   src = callPackage ./src.nix { };
 in
@@ -6,7 +6,13 @@ rec {
 
   inherit (src) packageVersion firefox source;
 
-  extraPatches = [ ./verify-telemetry-macros.patch ];
+  extraPatches = lib.optionals stdenv.isAarch64 [
+    (fetchpatch { # https://bugzilla.mozilla.org/show_bug.cgi?id=1791275
+      name = "no-sysctl-aarch64.patch";
+      url = "https://hg.mozilla.org/mozilla-central/raw-rev/0efaf5a00aaceeed679885e4cd393bd9a5fcd0ff";
+      hash = "sha256-wS/KufeLFxCexQalGGNg8+vnQhzDiL79OLt8FtL/JJ8=";
+    })
+  ];
 
   extraConfigureFlags = [
     "--with-app-name=librewolf"

pkgs/applications/version-management/git-and-tools/git-remote-codecommit/default.nix

@@ -1,11 +1,19 @@
-{ lib, buildPythonApplication, fetchFromGitHub, isPy3k, botocore, pytest, mock
-, flake8, tox, awscli }:
+{ lib, fetchFromGitHub, python3Packages, awscli }:
+
+with python3Packages;
 
 buildPythonApplication rec {
   pname = "git-remote-codecommit";
   version = "1.15.1";
   disabled = !isPy3k;
 
+  # The check dependency awscli has some overrides
+  # which yield a different botocore.
+  # This results in a duplicate version during installation
+  # of the wheel, even though it does not matter
+  # because it is only a test dependency.
+  catchConflicts = false;
+
   src = fetchFromGitHub {
     owner = "aws";
     repo = pname;

pkgs/applications/version-management/smartgithg/default.nix

@@ -34,7 +34,10 @@ stdenv.mkDerivation rec {
       --prefix JRE_HOME : ${jre} \
       --prefix JAVA_HOME : ${jre} \
       --prefix SMARTGITHG_JAVA_HOME : ${jre} \
-    ) \
+    )
+    # add missing shebang for start script
+    sed -i $out/bin/smartgit \
+      -e '1i#!/bin/bash'
   '';
 
   installPhase = ''

pkgs/development/interpreters/python/default.nix

@@ -31,13 +31,19 @@
     , pythonAttr ? null
     , self # is pythonOnHostForTarget
     }: let
-      pythonPackages = callPackage
+      pythonPackages = let
+        ensurePythonModules = items: let
+          providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
+          notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
+          func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
+        in lib.mapAttrs func items;
+      in ensurePythonModules (callPackage
         # Function that when called
         # - imports python-packages.nix
         # - adds spliced package sets to the package set
         # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
         ({ pkgs, stdenv, python, overrides }: let
-          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+          pythonPackagesFun = import ./python-packages-base.nix {
             inherit stdenv pkgs lib;
             python = self;
           };
@@ -48,47 +54,19 @@
             selfHostHost = pythonOnHostForHost.pkgs;
             selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
           };
-          keep = self: {
-            # TODO maybe only define these here so nothing is needed to be kept in sync.
-            inherit (self)
-              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
-              python bootstrapped-pip buildPythonPackage buildPythonApplication
-              fetchPypi
-              hasPythonModule requiredPythonModules makePythonPath disabledIf
-              toPythonModule toPythonApplication
-              buildSetupcfg
-
-              condaInstallHook
-              condaUnpackHook
-              eggUnpackHook
-              eggBuildHook
-              eggInstallHook
-              flitBuildHook
-              pipBuildHook
-              pipInstallHook
-              pytestCheckHook
-              pythonCatchConflictsHook
-              pythonImportsCheckHook
-              pythonNamespacesHook
-              pythonRecompileBytecodeHook
-              pythonRemoveBinBytecodeHook
-              pythonRemoveTestsDirHook
-              setuptoolsBuildHook
-              setuptoolsCheckHook
-              venvShellHook
-              wheelUnpackHook
-
-              wrapPython
-
-              pythonPackages
-
-              recursivePthLoader
-            ;
-          };
+          hooks = import ./hooks/default.nix;
+          keep = lib.extends hooks pythonPackagesFun;
           extra = _: {};
           optionalExtensions = cond: as: if cond then as else [];
+          pythonExtension = import ../../../top-level/python-packages.nix;
           python2Extension = import ../../../top-level/python2-packages.nix;
-          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
+          extensions = lib.composeManyExtensions ([
+            pythonExtension
+          ] ++ (optionalExtensions (!self.isPy3k) [
+            python2Extension
+          ]) ++ pythonPackagesExtensions ++ [
+            overrides
+          ]);
           aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
         in lib.makeScopeWithSplicing
           splicePackages
@@ -96,11 +74,11 @@
           otherSplices
           keep
           extra
-          (lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
+          (lib.extends (lib.composeExtensions aliases extensions) keep))
         {
           overrides = packageOverrides;
           python = self;
-        };
+        });
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";

pkgs/development/interpreters/python/hooks/default.nix

@@ -1,21 +1,15 @@
-# Hooks for building Python packages.
-{ python
-, lib
-, makeSetupHook
-, disabledIf
-, isPy3k
-}:
+self: super: with self;
 
 let
-  callPackage = python.pythonForBuild.pkgs.callPackage;
-  pythonInterpreter = python.pythonForBuild.interpreter;
-  pythonSitePackages = python.sitePackages;
-  pythonCheckInterpreter = python.interpreter;
+  pythonInterpreter = super.python.pythonForBuild.interpreter;
+  pythonSitePackages = super.python.sitePackages;
+  pythonCheckInterpreter = super.python.interpreter;
   setuppy = ../run_setup.py;
-in rec {
+in {
+  makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);
 
-  condaInstallHook = callPackage ({ gnutar, lbzip2 }:
-    makeSetupHook {
+  condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
+    makePythonHook {
       name = "conda-install-hook";
       deps = [ gnutar lbzip2 ];
       substitutions = {
@@ -23,20 +17,20 @@ in rec {
       };
     } ./conda-install-hook.sh) {};
 
-  condaUnpackHook = callPackage ({}:
-    makeSetupHook {
+  condaUnpackHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "conda-unpack-hook";
       deps = [];
     } ./conda-unpack-hook.sh) {};
 
-  eggBuildHook = callPackage ({ }:
-    makeSetupHook {
+  eggBuildHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "egg-build-hook.sh";
       deps = [ ];
     } ./egg-build-hook.sh) {};
 
-  eggInstallHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  eggInstallHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "egg-install-hook.sh";
       deps = [ setuptools ];
       substitutions = {
@@ -44,14 +38,14 @@ in rec {
       };
     } ./egg-install-hook.sh) {};
 
-  eggUnpackHook = callPackage ({ }:
-    makeSetupHook {
+  eggUnpackHook = callPackage ({ makePythonHook, }:
+    makePythonHook {
       name = "egg-unpack-hook.sh";
       deps = [ ];
     } ./egg-unpack-hook.sh) {};
 
-  flitBuildHook = callPackage ({ flit }:
-    makeSetupHook {
+  flitBuildHook = callPackage ({ makePythonHook, flit }:
+    makePythonHook {
       name = "flit-build-hook";
       deps = [ flit ];
       substitutions = {
@@ -59,8 +53,8 @@ in rec {
       };
     } ./flit-build-hook.sh) {};
 
-  pipBuildHook = callPackage ({ pip, wheel }:
-    makeSetupHook {
+  pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
+    makePythonHook {
       name = "pip-build-hook.sh";
       deps = [ pip wheel ];
       substitutions = {
@@ -68,8 +62,8 @@ in rec {
       };
     } ./pip-build-hook.sh) {};
 
-  pipInstallHook = callPackage ({ pip }:
-    makeSetupHook {
+  pipInstallHook = callPackage ({ makePythonHook, pip }:
+    makePythonHook {
       name = "pip-install-hook";
       deps = [ pip ];
       substitutions = {
@@ -77,8 +71,8 @@ in rec {
       };
     } ./pip-install-hook.sh) {};
 
-  pytestCheckHook = callPackage ({ pytest }:
-    makeSetupHook {
+  pytestCheckHook = callPackage ({ makePythonHook, pytest }:
+    makePythonHook {
       name = "pytest-check-hook";
       deps = [ pytest ];
       substitutions = {
@@ -86,8 +80,8 @@ in rec {
       };
     } ./pytest-check-hook.sh) {};
 
-  pythonCatchConflictsHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "python-catch-conflicts-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages setuptools;
@@ -95,29 +89,29 @@ in rec {
       };
     } ./python-catch-conflicts-hook.sh) {};
 
-  pythonImportsCheckHook = callPackage ({}:
-    makeSetupHook {
+  pythonImportsCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-imports-check-hook.sh";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./python-imports-check-hook.sh) {};
 
-  pythonNamespacesHook = callPackage ({ findutils }:
-    makeSetupHook {
+  pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
+    makePythonHook {
       name = "python-namespaces-hook.sh";
       substitutions = {
         inherit pythonSitePackages findutils;
       };
     } ./python-namespaces-hook.sh) {};
 
-  pythonOutputDistHook = callPackage ({ }:
-    makeSetupHook {
+  pythonOutputDistHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-output-dist-hook";
   } ./python-output-dist-hook.sh ) {};
 
-  pythonRecompileBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-recompile-bytecode-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
@@ -126,8 +120,8 @@ in rec {
       };
     } ./python-recompile-bytecode-hook.sh ) {};
 
-  pythonRelaxDepsHook = callPackage ({ wheel }:
-    makeSetupHook {
+  pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "python-relax-deps-hook";
       deps = [ wheel ];
       substitutions = {
@@ -135,21 +129,21 @@ in rec {
       };
     } ./python-relax-deps-hook.sh) {};
 
-  pythonRemoveBinBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-bin-bytecode-hook";
     } ./python-remove-bin-bytecode-hook.sh) {};
 
-  pythonRemoveTestsDirHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-tests-dir-hook";
       substitutions = {
         inherit pythonSitePackages;
       };
     } ./python-remove-tests-dir-hook.sh) {};
 
-  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
-    makeSetupHook {
+  setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
+    makePythonHook {
       name = "setuptools-setup-hook";
       deps = [ setuptools wheel ];
       substitutions = {
@@ -157,8 +151,8 @@ in rec {
       };
     } ./setuptools-build-hook.sh) {};
 
-  setuptoolsCheckHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "setuptools-check-hook";
       deps = [ setuptools ];
       substitutions = {
@@ -166,16 +160,16 @@ in rec {
       };
     } ./setuptools-check-hook.sh) {};
 
-  unittestCheckHook = callPackage ({ }:
-    makeSetupHook {
+  unittestCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "unittest-check-hook";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./unittest-check-hook.sh) {};
 
-  venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
-    makeSetupHook {
+  venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
+    makePythonHook {
       name = "venv-shell-hook";
       deps = [ ensureNewerSourcesForZipFilesHook ];
       substitutions = {
@@ -183,14 +177,18 @@ in rec {
       };
     } ./venv-shell-hook.sh) {});
 
-  wheelUnpackHook = callPackage ({ wheel }:
-    makeSetupHook {
+  wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "wheel-unpack-hook.sh";
       deps = [ wheel ];
     } ./wheel-unpack-hook.sh) {};
 
-  sphinxHook = callPackage ({ sphinx, installShellFiles }:
-    makeSetupHook {
+  wrapPython = callPackage ../wrap-python.nix {
+    inherit (pkgs.buildPackages) makeWrapper;
+  };
+
+  sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
+    makePythonHook {
       name = "python${python.pythonVersion}-sphinx-hook";
       deps = [ sphinx installShellFiles ];
     } ./sphinx-hook.sh) {};

pkgs/development/interpreters/python/python-packages-base.nix

@@ -0,0 +1,104 @@
+{ pkgs
+, stdenv
+, lib
+, python
+}:
+
+self:
+
+let
+  inherit (self) callPackage;
+  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
+
+  namePrefix = python.libPrefix + "-";
+
+  # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
+  # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
+  makeOverridablePythonPackage = f: origArgs:
+    let
+      ff = f origArgs;
+      overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
+    in
+      if builtins.isAttrs ff then (ff // {
+        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
+      })
+      else if builtins.isFunction ff then {
+        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
+        __functor = self: ff;
+      }
+      else ff;
+
+  buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+    inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
+    inherit toPythonModule; # Libraries provide modules
+  }));
+
+  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+    namePrefix = "";        # Python applications should not have any prefix
+    toPythonModule = x: x;  # Application does not provide modules.
+  }));
+
+  # See build-setupcfg/default.nix for documentation.
+  buildSetupcfg = import ../../../build-support/build-setupcfg self;
+
+  fetchPypi = callPackage ./fetchpypi.nix { };
+
+  # Check whether a derivation provides a Python module.
+  hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
+
+  # Get list of required Python modules given a list of derivations.
+  requiredPythonModules = drvs: let
+    modules = lib.filter hasPythonModule drvs;
+  in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
+
+  # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
+  # providing Python modules.
+  makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
+
+  removePythonPrefix = lib.removePrefix namePrefix;
+
+  # Convert derivation to a Python module.
+  toPythonModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        pythonModule = python;
+        pythonPath = [ ]; # Deprecated, for compatibility.
+        requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs;
+      };
+    });
+
+  # Convert a Python library to an application.
+  toPythonApplication = drv:
+    drv.overrideAttrs( oldAttrs: {
+      passthru = (oldAttrs.passthru or {}) // {
+        # Remove Python prefix from name so we have a "normal" name.
+        # While the prefix shows up in the store path, it won't be
+        # used by `nix-env`.
+        name = removePythonPrefix oldAttrs.name;
+        pythonModule = false;
+      };
+    });
+
+  disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
+
+  disabledIf = x: drv: if x then disabled drv else drv;
+
+in {
+
+  inherit lib pkgs stdenv;
+  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
+  inherit buildPythonPackage buildPythonApplication;
+  inherit fetchPypi;
+  inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
+  inherit toPythonModule toPythonApplication;
+  inherit buildSetupcfg;
+
+  python = toPythonModule python;
+  # Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions
+  pythonPackages = self;
+
+  # Remove?
+  recursivePthLoader = toPythonModule (callPackage ../../../development/python-modules/recursive-pth-loader { });
+
+}

pkgs/development/interpreters/python/wrap-python.nix

@@ -1,11 +1,11 @@
 { lib
 , python
-, makeSetupHook
+, makePythonHook
 , makeWrapper }:
 
 with lib;
 
-makeSetupHook {
+makePythonHook {
       deps = makeWrapper;
       substitutions.sitePackages = python.sitePackages;
       substitutions.executable = python.interpreter;

pkgs/development/python-modules/geopandas/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "geopandas";
-  version = "0.11.1";
+  version = "0.12.0";
   format = "setuptools";
 
   disabled = pythonOlder "3.8";
@@ -22,8 +22,8 @@ buildPythonPackage rec {
   src = fetchFromGitHub {
     owner = "geopandas";
     repo = "geopandas";
-    rev = "v${version}";
-    hash = "sha256-vL+zC8q7bif5pheq6pz7XRfzMKLaLQ0xDceTz0imw/E=";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-pMboKhwlueRjpbukF9u+yy9jlu0ikA1vQ08n7V2C7wo=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/gspread/default.nix

@@ -9,14 +9,14 @@
 
 buildPythonPackage rec {
   pname = "gspread";
-  version = "5.6.0";
+  version = "5.6.2";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-n8qFUXP9suZIs9qee7/7g2Ab/XxxMdRPp4HfhMaJ5/w=";
+    hash = "sha256-07v/S3qtD8LJhkWOFIU3oC/ntG5xYvQfOkI5K/oq24k=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/mailchecker/default.nix

@@ -6,14 +6,14 @@
 
 buildPythonPackage rec {
   pname = "mailchecker";
-  version = "5.0.1";
+  version = "5.0.3";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-fJoV/mNImdcTpJC6c2zmYWZCXlBWLOP+5W5Hsmw2yOQ=";
+    hash = "sha256-D8xyXBUDRaXLempcJ7G/Ybe7A3FiAaZ8kgm4jgKhkSI=";
   };
 
   # Module has no tests

pkgs/development/python-modules/nbclassic/default.nix

@@ -26,14 +26,14 @@
 
 buildPythonPackage rec {
   pname = "nbclassic";
-  version = "0.4.6";
+  version = "0.4.7";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-PBjTQ7KS+TjvyTIFdh5nTyDsoG6tJeDu3Bf3riUr9W0=";
+    hash = "sha256-HgRwWDtVCJxCeUDtMbioZv/vfMqxAUlOQJ7+Wse6mJc=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/neo/default.nix

@@ -10,14 +10,14 @@
 
 buildPythonPackage rec {
   pname = "neo";
-  version = "0.11.0";
+  version = "0.11.1";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-zfjhMko/u9Hv1WGNzTfPxJexmXkjvXELWYRywdhGZ0o=";
+    sha256 = "sha256-9KIGBEszKtALEAcrDcenCzWfo2XseG+Sq3V+9K5YhHQ=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/pox/default.nix

@@ -6,14 +6,14 @@
 
 buildPythonPackage rec {
   pname = "pox";
-  version = "0.3.1";
+  version = "0.3.2";
   format = "setuptools";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-y7DArNZQwP+2IJmdphHpOq5RBcRqCExM6vL3BO1wjB4=";
+    sha256 = "sha256-6CUiUpdjjW49SUFfjPtlQHpdFeVvL7f+nZueMFDGXuE=";
   };
 
   # Test sare failing the sandbox

pkgs/development/python-modules/py3status/default.nix

@@ -24,11 +24,11 @@
 
 buildPythonPackage rec {
   pname = "py3status";
-  version = "3.46";
+  version = "3.47";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-T7zaNepat9RQReCeww+kJOBK0vjfdahkjRgx27AWpcE=";
+    sha256 = "sha256-e2UTTD8J1GDg43FdzU8Xiaj2bL/gHLIT2lzwbwarIyI=";
   };
 
   doCheck = false;

pkgs/development/python-modules/xattr/default.nix

@@ -7,11 +7,11 @@
 
 buildPythonPackage rec {
   pname = "xattr";
-  version = "0.9.9";
+  version = "0.10.0";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "09cb7e1efb3aa1b4991d6be4eb25b73dc518b4fe894f0915f5b0dcede972f346";
+    sha256 = "sha256-ciZS0qUyTheJHEFtTHbZHM+YgwqPUWoN6FM86GfzrK8=";
   };
 
   propagatedBuildInputs = [ cffi ];

pkgs/development/python-modules/zcs/default.nix

@@ -8,11 +8,11 @@
 
 buildPythonPackage rec {
   pname = "zcs";
-  version = "0.1.21";
+  version = "0.1.22";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-Zs2aK+RR84uKjh+ZF/3gulS78zbb+XahTVSTJAArKHA=";
+    sha256 = "sha256-+0lG2OirfXj55IFA9GMERVWtrWwULfVfdbIg8ebH+7M=";
   };
 
   patches = [

pkgs/development/tools/buildah/default.nix

@@ -46,7 +46,7 @@ buildGoModule rec {
     runHook preBuild
     patchShebangs .
     make bin/buildah
-    make -C docs GOMD2MAN="${go-md2man}/bin/go-md2man"
+    make -C docs GOMD2MAN="go-md2man"
     runHook postBuild
   '';
 

pkgs/servers/web-apps/outline/default.nix

@@ -9,13 +9,13 @@
 
 stdenv.mkDerivation rec {
   pname = "outline";
-  version = "0.66.1";
+  version = "0.66.2";
 
   src = fetchFromGitHub {
     owner = "outline";
     repo = "outline";
     rev = "v${version}";
-    sha256 = "sha256-pK/rrGAL9JKt52jQRVtbQgPPe644anOARUNOhPCUCqE=";
+    sha256 = "sha256-jRnw6UIUA3gAgyqQg6R1GOI4O8HXKnVfTH3d3SFBa9A=";
   };
 
   nativeBuildInputs = [ makeWrapper yarn2nix-moretea.fixup_yarn_lock ];

pkgs/tools/admin/awscli/default.nix

@@ -30,6 +30,7 @@ let
         '';
       });
     };
+    self = py;
   };
 
 in

pkgs/tools/misc/lifecycled/default.nix

@@ -4,13 +4,13 @@
 }:
 buildGoModule rec {
   pname = "lifecycled";
-  version = "3.2.0";
+  version = "3.3.0";
 
   src = fetchFromGitHub {
     owner = "buildkite";
     repo = "lifecycled";
     rev = "v${version}";
-    sha256 = "sha256-+Ts2ERoEZcBdxMXQlxPVtQe3pst5NXWKU3rmS5CgR7A=";
+    sha256 = "sha256-zskN2T0+1xZPjppggeGpPFuQ8/AgPNyN77F33rDoghc=";
   };
 
   vendorSha256 = "sha256-q5wYKSLHRzL+UGn29kr8+mUupOPR1zohTscbzjMRCS0=";

pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch

@@ -1,27 +1,12 @@
-From 269cc4c9558549f340ec186d9246654564b2f633 Mon Sep 17 00:00:00 2001
-From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
-Date: Tue, 18 Aug 2020 10:22:36 +0100
-Subject: [PATCH] add nixos support
-MIME-Version: 1.0
-Content-Type: text/plain; charset=UTF-8
-Content-Transfer-Encoding: 8bit
-
-Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
----
- cloudinit/distros/__init__.py |   1 +
- cloudinit/distros/nixos.py    | 103 ++++++++++++++++++++++++++++++++++
- 2 files changed, 104 insertions(+)
- create mode 100644 cloudinit/distros/nixos.py
-
 diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
-index 2537608f..c533b585 100755
+index 4a468cf8..c60c899b 100644
 --- a/cloudinit/distros/__init__.py
 +++ b/cloudinit/distros/__init__.py
-@@ -47,6 +47,7 @@ OSFAMILIES = {
-     'gentoo': ['gentoo'],
-     'redhat': ['amazon', 'centos', 'fedora', 'rhel'],
-     'suse': ['opensuse', 'sles'],
-+    'nixos': ['nixos'],
+@@ -55,6 +55,7 @@ OSFAMILIES = {
+         "virtuozzo",
+     ],
+     "suse": ["opensuse", "sles"],
++    "nixos": ["nixos"],
  }
  
  LOG = logging.getLogger(__name__)
@@ -134,6 +119,3 @@ index 00000000..d53d2a61
 +
 +    def update_package_sources(self):
 +        raise NotImplementedError()
--- 
-2.28.0
-

pkgs/tools/virtualization/cloud-init/default.nix

@@ -9,18 +9,19 @@
 , python3
 , shadow
 , systemd
+, coreutils
 }:
 
 python3.pkgs.buildPythonApplication rec {
   pname = "cloud-init";
-  version = "21.4";
+  version = "22.3.3";
   namePrefix = "";
 
   src = fetchFromGitHub {
     owner = "canonical";
     repo = "cloud-init";
     rev = version;
-    sha256 = "09413qz9y2csvhjb4krjnkfj97vlykx79j912p27jjcrg82f1nib";
+    hash = "sha256-9vdFPSmkkdJDlVfA9DgqczRoOBMmSMezdl3D/0OSbsQ=";
   };
 
   patches = [ ./0001-add-nixos-support.patch ];
@@ -30,11 +31,14 @@ python3.pkgs.buildPythonApplication rec {
       --replace /lib/systemd $out/lib/systemd
 
     substituteInPlace cloudinit/net/networkd.py \
-      --replace "['/usr/sbin', '/bin']" "['/usr/sbin', '/bin', '${iproute2}/bin', '${systemd}/bin']"
+      --replace '["/usr/sbin", "/bin"]' '["/usr/sbin", "/bin", "${iproute2}/bin", "${systemd}/bin"]'
 
     substituteInPlace tests/unittests/test_net_activators.py \
-      --replace "['/usr/sbin', '/bin']" \
-        "['/usr/sbin', '/bin', '${iproute2}/bin', '${systemd}/bin']"
+      --replace '["/usr/sbin", "/bin"]' \
+        '["/usr/sbin", "/bin", "${iproute2}/bin", "${systemd}/bin"]'
+
+    substituteInPlace tests/unittests/cmd/test_clean.py \
+      --replace "/bin/bash" "/bin/sh"
   '';
 
   postInstall = ''
@@ -62,6 +66,9 @@ python3.pkgs.buildPythonApplication rec {
     dmidecode
     # needed for tests; at runtime we rather want the setuid wrapper
     shadow
+    responses
+    pytest-mock
+    coreutils
   ];
 
   makeWrapperArgs = [
@@ -96,20 +103,6 @@ python3.pkgs.buildPythonApplication rec {
     "test_install_with_version"
   ];
 
-  disabledTestPaths = [
-    # Oracle tests are not passing
-    "cloudinit/sources/tests/test_oracle.py"
-    # Disable the integration tests. pycloudlib would be required
-    "tests/unittests/test_datasource/test_aliyun.py"
-    "tests/unittests/test_datasource/test_azure.py"
-    "tests/unittests/test_datasource/test_ec2.py"
-    "tests/unittests/test_datasource/test_exoscale.py"
-    "tests/unittests/test_datasource/test_gce.py"
-    "tests/unittests/test_datasource/test_openstack.py"
-    "tests/unittests/test_datasource/test_scaleway.py"
-    "tests/unittests/test_ec2_util.py"
-  ];
-
   preCheck = ''
     # TestTempUtils.test_mkdtemp_default_non_root does not like TMPDIR=/build
     export TMPDIR=/tmp
@@ -119,13 +112,13 @@ python3.pkgs.buildPythonApplication rec {
     "cloudinit"
   ];
 
-  passthru.tests.cloud-init = nixosTests.cloud-init;
+  passthru.tests = { inherit (nixosTests) cloud-init cloud-init-hostname; };
 
   meta = with lib; {
     homepage = "https://cloudinit.readthedocs.org";
     description = "Provides configuration and customization of cloud instance";
     license = with licenses; [ asl20 gpl3Plus ];
-    maintainers = with maintainers; [ madjar phile314 ];
+    maintainers = with maintainers; [ madjar phile314 illustris ];
     platforms = platforms.all;
   };
 }

pkgs/top-level/python-packages.nix

@@ -6,146 +6,9 @@
 #
 # For more details, please see the Python section in the Nixpkgs manual.
 
-{ pkgs
-, stdenv
-, lib
-, python
-}:
+self: super: with self; {
 
-self:
-
-let
-  inherit (self) callPackage;
-  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
-
-  namePrefix = python.libPrefix + "-";
-
-  bootstrapped-pip = callPackage ../development/python-modules/bootstrapped-pip { };
-
-  # Derivations built with `buildPythonPackage` can already be overriden with `override`, `overrideAttrs`, and `overrideDerivation`.
-  # This function introduces `overridePythonAttrs` and it overrides the call to `buildPythonPackage`.
-  makeOverridablePythonPackage = f: origArgs:
-    let
-      ff = f origArgs;
-      overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
-    in
-      if builtins.isAttrs ff then (ff // {
-        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
-      })
-      else if builtins.isFunction ff then {
-        overridePythonAttrs = newArgs: makeOverridablePythonPackage f (overrideWith newArgs);
-        __functor = self: ff;
-      }
-      else ff;
-
-  buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
-    inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
-    inherit toPythonModule; # Libraries provide modules
-  }));
-
-  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ../development/interpreters/python/mk-python-derivation.nix {
-    namePrefix = "";        # Python applications should not have any prefix
-    toPythonModule = x: x;  # Application does not provide modules.
-  }));
-
-  # See build-setupcfg/default.nix for documentation.
-  buildSetupcfg = import ../build-support/build-setupcfg self;
-
-  fetchPypi = callPackage ../development/interpreters/python/fetchpypi.nix { };
-
-  # Check whether a derivation provides a Python module.
-  hasPythonModule = drv: drv?pythonModule && drv.pythonModule == python;
-
-  # Get list of required Python modules given a list of derivations.
-  requiredPythonModules = drvs: let
-    modules = lib.filter hasPythonModule drvs;
-  in lib.unique ([python] ++ modules ++ lib.concatLists (lib.catAttrs "requiredPythonModules" modules));
-
-  # Create a PYTHONPATH from a list of derivations. This function recurses into the items to find derivations
-  # providing Python modules.
-  makePythonPath = drvs: lib.makeSearchPath python.sitePackages (requiredPythonModules drvs);
-
-  removePythonPrefix = lib.removePrefix namePrefix;
-
-  # Convert derivation to a Python module.
-  toPythonModule = drv:
-    drv.overrideAttrs( oldAttrs: {
-      # Use passthru in order to prevent rebuilds when possible.
-      passthru = (oldAttrs.passthru or {})// {
-        pythonModule = python;
-        pythonPath = [ ]; # Deprecated, for compatibility.
-        requiredPythonModules = requiredPythonModules drv.propagatedBuildInputs;
-      };
-    });
-
-  # Convert a Python library to an application.
-  toPythonApplication = drv:
-    drv.overrideAttrs( oldAttrs: {
-      passthru = (oldAttrs.passthru or {}) // {
-        # Remove Python prefix from name so we have a "normal" name.
-        # While the prefix shows up in the store path, it won't be
-        # used by `nix-env`.
-        name = removePythonPrefix oldAttrs.name;
-        pythonModule = false;
-      };
-    });
-
-  disabled = drv: throw "${removePythonPrefix (drv.pname or drv.name)} not supported for interpreter ${python.executable}";
-
-  disabledIf = x: drv: if x then disabled drv else drv;
-
-in {
-
-  inherit pkgs stdenv;
-
-  inherit (python.passthru) isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy311 isPy3k isPyPy pythonAtLeast pythonOlder;
-  inherit python bootstrapped-pip buildPythonPackage buildPythonApplication;
-  inherit fetchPypi;
-  inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
-  inherit toPythonModule toPythonApplication;
-  inherit buildSetupcfg;
-
-  inherit (callPackage ../development/interpreters/python/hooks { })
-    sphinxHook
-    condaInstallHook
-    condaUnpackHook
-    eggUnpackHook
-    eggBuildHook
-    eggInstallHook
-    flitBuildHook
-    pipBuildHook
-    pipInstallHook
-    pytestCheckHook
-    pythonCatchConflictsHook
-    pythonImportsCheckHook
-    pythonNamespacesHook
-    pythonOutputDistHook
-    pythonRecompileBytecodeHook
-    pythonRelaxDepsHook
-    pythonRemoveBinBytecodeHook
-    pythonRemoveTestsDirHook
-    setuptoolsBuildHook
-    setuptoolsCheckHook
-    unittestCheckHook
-    venvShellHook
-    wheelUnpackHook;
-
-  # helpers
-
-  # We use build packages because we are making a setup hook to be used as a
-  # native build input. The script itself references both the build-time
-  # (build) and run-time (host) python from the explicitly passed in `python`
-  # attribute, so the `buildPackages` doesn't effect that.
-  wrapPython = pkgs.buildPackages.callPackage ../development/interpreters/python/wrap-python.nix {
-    inherit python;
-  };
-
-  # Dont take pythonPackages from "global" pkgs scope to avoid mixing python versions
-  pythonPackages = self;
-
-  # specials
-
-  recursivePthLoader = callPackage ../development/python-modules/recursive-pth-loader { };
+  bootstrapped-pip = toPythonModule (callPackage ../development/python-modules/bootstrapped-pip { });
 
   setuptools = callPackage ../development/python-modules/setuptools { };
 

pkgs/top-level/python2-packages.nix

@@ -7,7 +7,7 @@ self: super:
 with self; with super; {
   attrs = callPackage ../development/python2-modules/attrs { };
 
-  bootstrapped-pip = callPackage ../development/python2-modules/bootstrapped-pip { };
+  bootstrapped-pip = toPythonModule (callPackage ../development/python2-modules/bootstrapped-pip { });
 
   boto3 = callPackage ../development/python2-modules/boto3 {};