From 3a4459a30508419f2498bbcbf85782dabf0178df Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 6 Nov 2018 00:26:55 +0100 Subject: [PATCH 1/4] nixos/rspamd: Support multiple workers When the workers option for rspamd was originally implemented it was based on a flawed understanding of how workers are configured in rspamd. This meant that while rspamd supports configuring multiple workers of the same type, so that different controller workers could have different passwords, the NixOS module did not support this because it would write an invalid configuration file if you tried. Specifically a configuration like the one below: ``` workers.controller = {}; workers.controller2 = { type = "controller"; }; ``` Would result in a rspamd configuration of: ``` worker { type = "controller"; count = 1; .include "$CONFDIR/worker-controller.inc" } worker "controller2" { type = "controller"; count = 1; } ``` While to get multiple controller workers it should instead be: ``` worker "controller" { type = "controller"; count = 1; .include "$CONFDIR/worker-controller.inc" } worker "controller" { type = "controller"; count = 1; } ``` --- nixos/modules/services/mail/rspamd.nix | 2 +- nixos/tests/rspamd.nix | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/mail/rspamd.nix b/nixos/modules/services/mail/rspamd.nix index d83d6f1f750c..78d2e7c262bc 100644 --- a/nixos/modules/services/mail/rspamd.nix +++ b/nixos/modules/services/mail/rspamd.nix @@ -139,7 +139,7 @@ let } ${concatStringsSep "\n" (mapAttrsToList (name: value: '' - worker ${optionalString (value.name != "normal" && value.name != "controller") "${value.name}"} { + worker "${value.type}" { type = "${value.type}"; ${optionalString (value.enable != null) "enabled = ${if value.enable != false then "yes" else "no"};"} diff --git a/nixos/tests/rspamd.nix b/nixos/tests/rspamd.nix index af765f37b91b..f7c27137be98 100644 --- a/nixos/tests/rspamd.nix +++ b/nixos/tests/rspamd.nix @@ -28,6 +28,8 @@ let ${checkSocket "/run/rspamd/rspamd.sock" "rspamd" "rspamd" "660" } sleep 10; $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf")); $machine->log($machine->succeed("systemctl cat rspamd.service")); $machine->log($machine->succeed("curl http://localhost:11334/auth")); $machine->log($machine->succeed("curl http://127.0.0.1:11334/auth")); @@ -56,6 +58,8 @@ in ${checkSocket "/run/rspamd.sock" "root" "root" "600" } ${checkSocket "/run/rspamd-worker.sock" "root" "root" "666" } $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf")); $machine->log($machine->succeed("rspamc -h /run/rspamd-worker.sock stat")); $machine->log($machine->succeed("curl --unix-socket /run/rspamd-worker.sock http://localhost/ping")); ''; @@ -78,6 +82,15 @@ in owner = "root"; group = "root"; }]; + workers.controller2 = { + type = "controller"; + bindSockets = [ "0.0.0.0:11335" ]; + extraConfig = '' + static_dir = "''${WWWDIR}"; + secure_ip = null; + password = "verysecretpassword"; + ''; + }; }; }; @@ -87,8 +100,13 @@ in ${checkSocket "/run/rspamd.sock" "root" "root" "600" } ${checkSocket "/run/rspamd-worker.sock" "root" "root" "666" } $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'verysecretpassword' /etc/rspamd/rspamd.conf")); + $machine->waitUntilSucceeds("journalctl -u rspamd | grep -i 'starting controller process' >&2"); $machine->log($machine->succeed("rspamc -h /run/rspamd-worker.sock stat")); $machine->log($machine->succeed("curl --unix-socket /run/rspamd-worker.sock http://localhost/ping")); + $machine->log($machine->succeed("curl http://localhost:11335/ping")); ''; }; customLuaRules = makeTest { From 46ef075e7daad1bcaab1d4d1258c7d6c64a87b63 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 6 Nov 2018 00:32:14 +0100 Subject: [PATCH 2/4] nixos/rspamd: Add defaults for rspamd_proxy worker --- nixos/modules/services/mail/rspamd.nix | 30 ++++++++++++++------------ 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nixos/modules/services/mail/rspamd.nix b/nixos/modules/services/mail/rspamd.nix index 78d2e7c262bc..3489227f0833 100644 --- a/nixos/modules/services/mail/rspamd.nix +++ b/nixos/modules/services/mail/rspamd.nix @@ -58,7 +58,7 @@ let }; type = mkOption { type = types.nullOr (types.enum [ - "normal" "controller" "fuzzy_storage" "proxy" "lua" + "normal" "controller" "fuzzy_storage" "rspamd_proxy" "lua" ]); description = "The type of this worker"; }; @@ -99,19 +99,21 @@ let description = "Additional entries to put verbatim into worker section of rspamd config file."; }; }; - config = mkIf (name == "normal" || name == "controller" || name == "fuzzy") { + config = mkIf (name == "normal" || name == "controller" || name == "fuzzy" || name == "rspamd_proxy") { type = mkDefault name; - includes = mkDefault [ "$CONFDIR/worker-${name}.inc" ]; - bindSockets = mkDefault (if name == "normal" - then [{ - socket = "/run/rspamd/rspamd.sock"; - mode = "0660"; - owner = cfg.user; - group = cfg.group; - }] - else if name == "controller" - then [ "localhost:11334" ] - else [] ); + includes = mkDefault [ "$CONFDIR/worker-${if name == "rspamd_proxy" then "proxy" else name}.inc" ]; + bindSockets = + let + unixSocket = name: { + mode = "0660"; + socket = "/run/rspamd/${name}.sock"; + owner = cfg.user; + group = cfg.group; + }; + in mkDefault (if name == "normal" then [(unixSocket "rspamd")] + else if name == "controller" then [ "localhost:11334" ] + else if name == "rspamd_proxy" then [ (unixSocket "proxy") ] + else [] ); }; }; @@ -284,7 +286,7 @@ in description = '' User to use when no root privileges are required. ''; - }; + }; group = mkOption { type = types.string; From fba69f388bbb7ab9f79b646e55ef1ef78daa1213 Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 6 Nov 2018 00:34:23 +0100 Subject: [PATCH 3/4] nixos/rspamd: Put extraConfig in included files The lines stored in `extraConfig` and `worker..extraConfig` should take precedent over values from included files but in order to do this in rspamd UCL they need to be stored in a file that then gets included with a high priority. This commit uses the overrides option to store the value of the two `extraConfig` options in `extra-config.inc` and `worker-.inc` respectively. --- nixos/modules/services/mail/rspamd.nix | 22 +++++++++++++++++++--- nixos/tests/rspamd.nix | 3 ++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/mail/rspamd.nix b/nixos/modules/services/mail/rspamd.nix index 3489227f0833..927fc4d6a356 100644 --- a/nixos/modules/services/mail/rspamd.nix +++ b/nixos/modules/services/mail/rspamd.nix @@ -140,7 +140,10 @@ let .include(try=true; priority=10) "$LOCAL_CONFDIR/override.d/logging.inc" } - ${concatStringsSep "\n" (mapAttrsToList (name: value: '' + ${concatStringsSep "\n" (mapAttrsToList (name: value: let + includeName = if name == "rspamd_proxy" then "proxy" else name; + tryOverride = if value.extraConfig == "" then "true" else "false"; + in '' worker "${value.type}" { type = "${value.type}"; ${optionalString (value.enable != null) @@ -148,11 +151,14 @@ let ${mkBindSockets value.enable value.bindSockets} ${optionalString (value.count != null) "count = ${toString value.count};"} ${concatStringsSep "\n " (map (each: ".include \"${each}\"") value.includes)} - ${value.extraConfig} + .include(try=true; priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/worker-${includeName}.inc" + .include(try=${tryOverride}; priority=10) "$LOCAL_CONFDIR/override.d/worker-${includeName}.inc" } '') cfg.workers)} - ${cfg.extraConfig} + ${optionalString (cfg.extraConfig != "") '' + .include(priority=10) "$LOCAL_CONFDIR/override.d/extra-config.inc" + ''} ''; rspamdDir = pkgs.linkFarm "etc-rspamd-dir" ( @@ -190,6 +196,15 @@ let in mkDefault (pkgs.writeText name' config.text)); }; }; + + configOverrides = + (mapAttrs' (n: v: nameValuePair "worker-${if n == "rspamd_proxy" then "proxy" else n}.inc" { + text = v.extraConfig; + }) + (filterAttrs (n: v: v.extraConfig != "") cfg.workers)) + // (if cfg.extraConfig == "" then {} else { + "extra-config.inc".text = cfg.extraConfig; + }); in { @@ -302,6 +317,7 @@ in ###### implementation config = mkIf cfg.enable { + services.rspamd.overrides = configOverrides; # Allow users to run 'rspamc' and 'rspamadm'. environment.systemPackages = [ pkgs.rspamd ]; diff --git a/nixos/tests/rspamd.nix b/nixos/tests/rspamd.nix index f7c27137be98..ccfe8f7bb0d8 100644 --- a/nixos/tests/rspamd.nix +++ b/nixos/tests/rspamd.nix @@ -102,7 +102,8 @@ in $machine->log($machine->succeed("cat /etc/rspamd/rspamd.conf")); $machine->log($machine->succeed("grep 'CONFDIR/worker-controller.inc' /etc/rspamd/rspamd.conf")); $machine->log($machine->succeed("grep 'CONFDIR/worker-normal.inc' /etc/rspamd/rspamd.conf")); - $machine->log($machine->succeed("grep 'verysecretpassword' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'LOCAL_CONFDIR/override.d/worker-controller2.inc' /etc/rspamd/rspamd.conf")); + $machine->log($machine->succeed("grep 'verysecretpassword' /etc/rspamd/override.d/worker-controller2.inc")); $machine->waitUntilSucceeds("journalctl -u rspamd | grep -i 'starting controller process' >&2"); $machine->log($machine->succeed("rspamc -h /run/rspamd-worker.sock stat")); $machine->log($machine->succeed("curl --unix-socket /run/rspamd-worker.sock http://localhost/ping")); From e01605be153906394cd35cd728e8e7c7fbce473e Mon Sep 17 00:00:00 2001 From: Brian Olsen Date: Tue, 6 Nov 2018 00:40:52 +0100 Subject: [PATCH 4/4] nixos/rspamd: Add options for postfix integration The `rmilter` module has options for configuring `postfix` to use it but since that module is deprecated because rspamd now has a builtin worker that supports the milter protocol this commit adds similar `postfix` integration options directly to the `rspamd` module. --- nixos/modules/services/mail/rspamd.nix | 44 +++++++++++++++++++++++++- nixos/tests/rspamd.nix | 44 ++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/mail/rspamd.nix b/nixos/modules/services/mail/rspamd.nix index 927fc4d6a356..1a2a9eca96a2 100644 --- a/nixos/modules/services/mail/rspamd.nix +++ b/nixos/modules/services/mail/rspamd.nix @@ -6,6 +6,7 @@ let cfg = config.services.rspamd; opts = options.services.rspamd; + postfixCfg = config.services.postfix; bindSocketOpts = {options, config, ... }: { options = { @@ -309,7 +310,30 @@ in description = '' Group to use when no root privileges are required. ''; - }; + }; + + postfix = { + enable = mkOption { + type = types.bool; + default = false; + description = "Add rspamd milter to postfix main.conf"; + }; + + config = mkOption { + type = with types; attrsOf (either bool (either str (listOf str))); + description = '' + Addon to postfix configuration + ''; + default = { + smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"]; + non_smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"]; + }; + example = { + smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"]; + non_smtpd_milters = ["unix:/run/rspamd/rspamd-milter.sock"]; + }; + }; + }; }; }; @@ -318,6 +342,24 @@ in config = mkIf cfg.enable { services.rspamd.overrides = configOverrides; + services.rspamd.workers = mkIf cfg.postfix.enable { + controller = {}; + rspamd_proxy = { + bindSockets = [ { + mode = "0660"; + socket = "/run/rspamd/rspamd-milter.sock"; + owner = cfg.user; + group = postfixCfg.group; + } ]; + extraConfig = '' + upstream "local" { + default = yes; # Self-scan upstreams are always default + self_scan = yes; # Enable self-scan + } + ''; + }; + }; + services.postfix.config = mkIf cfg.postfix.enable cfg.postfix.config; # Allow users to run 'rspamc' and 'rspamadm'. environment.systemPackages = [ pkgs.rspamd ]; diff --git a/nixos/tests/rspamd.nix b/nixos/tests/rspamd.nix index ccfe8f7bb0d8..076632a70bab 100644 --- a/nixos/tests/rspamd.nix +++ b/nixos/tests/rspamd.nix @@ -181,4 +181,48 @@ in $machine->log($machine->succeed("cat /etc/tests/muh.eml | rspamc -h 127.0.0.1:11334 symbols | grep NO_MUH")); ''; }; + postfixIntegration = makeTest { + name = "rspamd-postfix-integration"; + machine = { + environment.systemPackages = with pkgs; [ msmtp ]; + environment.etc."tests/gtube.eml".text = '' + From: Sheep1 + To: Sheep2 + Subject: Evil cows + + I find cows to be evil don't you? + + XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X + ''; + environment.etc."tests/example.eml".text = '' + From: Sheep1 + To: Sheep2 + Subject: Evil cows + + I find cows to be evil don't you? + ''; + users.users.tester.password = "test"; + services.postfix = { + enable = true; + destination = ["example.com"]; + }; + services.rspamd = { + enable = true; + postfix.enable = true; + }; + }; + testScript = '' + ${initMachine} + $machine->waitForOpenPort(11334); + $machine->waitForOpenPort(25); + ${checkSocket "/run/rspamd/rspamd-milter.sock" "rspamd" "postfix" "660" } + $machine->log($machine->succeed("rspamc -h 127.0.0.1:11334 stat")); + $machine->log($machine->succeed("msmtp --host=localhost -t --read-envelope-from < /etc/tests/example.eml")); + $machine->log($machine->fail("msmtp --host=localhost -t --read-envelope-from < /etc/tests/gtube.eml")); + + $machine->waitUntilFails('[ "$(postqueue -p)" != "Mail queue is empty" ]'); + $machine->fail("journalctl -u postfix | grep -i error >&2"); + $machine->fail("journalctl -u postfix | grep -i warning >&2"); + ''; + }; }