diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c89bbcc71acf..4032df9fb91a 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -137,6 +137,7 @@ ./services/computing/torque/server.nix ./services/computing/torque/mom.nix ./services/computing/slurm/slurm.nix + ./services/continuous-integration/buildbot/master.nix ./services/continuous-integration/buildkite-agent.nix ./services/continuous-integration/hydra/default.nix ./services/continuous-integration/gitlab-runner.nix diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix new file mode 100644 index 000000000000..a40be4f546ea --- /dev/null +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -0,0 +1,250 @@ +# NixOS module for Buildbot continous integration server. + +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.buildbot-master; + escapeStr = s: escape ["'"] s; + masterCfg = pkgs.writeText "master.cfg" '' + from buildbot.plugins import * + factory = util.BuildFactory() + c = BuildmasterConfig = dict( + workers = [${concatStringsSep "," cfg.workers}], + protocols = { 'pb': {'port': ${cfg.bpPort} } }, + title = '${escapeStr cfg.title}', + titleURL = '${escapeStr cfg.titleUrl}', + buildbotURL = '${escapeStr cfg.buildbotUrl}', + db = dict(db_url='${escapeStr cfg.dbUrl}'), + www = dict(port=${toString cfg.port}), + change_source = [ ${concatStringsSep "," cfg.changeSource} ], + schedulers = [ ${concatStringsSep "," cfg.schedulers} ], + builders = [ ${concatStringsSep "," cfg.builders} ], + status = [ ${concatStringsSep "," cfg.status} ], + ) + for step in [ ${concatStringsSep "," cfg.factorySteps} ]: + factory.addStep(step) + + ${cfg.extraConfig} + ''; + + configFile = if cfg.masterCfg == null then masterCfg else cfg.masterCfg; + +in { + options = { + services.buildbot-master = { + + factorySteps = mkOption { + type = types.listOf types.str; + description = "Factory Steps"; + default = []; + example = [ + "steps.Git(repourl='git://github.com/buildbot/pyflakes.git', mode='incremental')" + "steps.ShellCommand(command=['trial', 'pyflakes'])" + ]; + }; + + changeSource = mkOption { + type = types.listOf types.str; + description = "List of Change Sources."; + default = []; + example = [ + "changes.GitPoller('git://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + ]; + }; + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Buildbot continuous integration server."; + }; + + extraConfig = mkOption { + type = types.str; + description = "Extra configuration to append to master.cfg"; + default = ""; + }; + + masterCfg = mkOption { + type = with types; nullOr path; + description = '' + Optionally pass path to raw master.cfg file. + Other options in this configuration will be ignored. + ''; + default = null; + example = literalExample '' + pkgs.writeText "master.cfg" "BuildmasterConfig = c = {}" + ''; + }; + + schedulers = mkOption { + type = types.listOf types.str; + description = "List of Schedulers."; + default = [ + "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])" + "schedulers.ForceScheduler(name='force',builderNames=['runtests'])" + ]; + }; + + builders = mkOption { + type = types.listOf types.str; + description = "List of Builders."; + default = [ + "util.BuilderConfig(name='runtests',workernames=['default-worker'],factory=factory)" + ]; + }; + + workers = mkOption { + type = types.listOf types.str; + description = "List of Workers."; + default = [ + "worker.Worker('default-worker', 'password')" + ]; + example = [ "worker.LocalWorker('default-worker')" ]; + }; + + status = mkOption { + default = []; + type = types.listOf types.str; + description = "List of status notification endpoints."; + }; + + user = mkOption { + default = "buildbot"; + type = types.str; + description = "User the buildbot server should execute under."; + }; + + group = mkOption { + default = "buildbot"; + type = types.str; + description = "Primary group of buildbot user."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = [ "nixbld" ]; + description = "List of extra groups that the buildbot user should be a part of."; + }; + + home = mkOption { + default = "/home/buildbot"; + type = types.path; + description = "Buildbot home directory."; + }; + + buildbotDir = mkOption { + default = "${cfg.home}/master"; + type = types.path; + description = "Specifies the Buildbot directory."; + }; + + bpPort = mkOption { + default = "9989"; + type = types.string; + example = "tcp:10000:interface=127.0.0.1"; + description = "Port where the master will listen to Buildbot Worker."; + }; + + listenAddress = mkOption { + default = "0.0.0.0"; + type = types.str; + description = "Specifies the bind address on which the buildbot HTTP interface listens."; + }; + + buildbotUrl = mkOption { + default = "http://localhost:8010/"; + type = types.str; + description = "Specifies the Buildbot URL."; + }; + + title = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot Title."; + }; + + titleUrl = mkOption { + default = "Buildbot"; + type = types.str; + description = "Specifies the Buildbot TitleURL."; + }; + + dbUrl = mkOption { + default = "sqlite:///state.sqlite"; + type = types.str; + description = "Specifies the database connection string."; + }; + + port = mkOption { + default = 8010; + type = types.int; + description = "Specifies port number on which the buildbot HTTP interface listens."; + }; + + package = mkOption { + type = types.package; + default = pkgs.buildbot-ui; + description = '' + Package to use for buildbot. + buildbot-full is required in order to use local workers. + ''; + example = pkgs.buildbot-full; + }; + + packages = mkOption { + default = [ ]; + example = [ pkgs.git ]; + type = types.listOf types.package; + description = "Packages to add to PATH for the buildbot process."; + }; + }; + }; + + config = mkIf cfg.enable { + users.extraGroups = optional (cfg.group == "buildbot") { + name = "buildbot"; + }; + + users.extraUsers = optional (cfg.user == "buildbot") { + name = "buildbot"; + description = "buildbot user"; + isNormalUser = true; + createHome = true; + home = cfg.home; + group = cfg.group; + extraGroups = cfg.extraGroups; + useDefaultShell = true; + }; + + systemd.services.buildbot-master = { + description = "Buildbot Continuous Integration Server"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + path = cfg.packages; + + serviceConfig = { + Type = "forking"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = cfg.home; + ExecStart = "${cfg.package}/bin/buildbot start ${cfg.buildbotDir}"; + }; + + preStart = '' + mkdir -vp ${cfg.buildbotDir} + chown -c ${cfg.user}:${cfg.group} ${cfg.buildbotDir} + ln -sf ${configFile} ${cfg.buildbotDir}/master.cfg + ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} + ''; + + postStart = '' + until [[ $(${pkgs.curl}/bin/curl -s --head -w '\n%{http_code}' http://localhost:${toString cfg.port} | tail -n1) =~ ^(200|403)$ ]]; do + sleep 1 + done + ''; + }; + }; + +} diff --git a/pkgs/development/tools/build-managers/buildbot/default.nix b/pkgs/development/tools/build-managers/buildbot/default.nix index d16b83a2831c..8e85c645e2e0 100644 --- a/pkgs/development/tools/build-managers/buildbot/default.nix +++ b/pkgs/development/tools/build-managers/buildbot/default.nix @@ -1,12 +1,21 @@ -{ stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }: +{ stdenv, + lib, + pythonPackages, + fetchurl, + coreutils, + openssh, + buildbot-worker, + plugins ? [], + enableLocalWorker ? false +}: pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "16bnrr5qkfpnby9sw9azcagnw0ybi7d8bpdlga2a4c61jg2d5dnc"; + sha256 = "18rnsp691cnmbymlch6czx3mrcmifmf6dk97h9nslgfkkyf25n5g"; }; buildInputs = with pythonPackages; [ @@ -22,7 +31,7 @@ pythonPackages.buildPythonApplication (rec { pylint astroid pyflakes - ]; + ] ++ lib.optionals (enableLocalWorker) [openssh]; propagatedBuildInputs = with pythonPackages; [ @@ -52,14 +61,17 @@ pythonPackages.buildPythonApplication (rec { ramlfications sphinx-jinja - ] ++ plugins; + ] ++ plugins ++ + lib.optionals (enableLocalWorker) [buildbot-worker]; preInstall = '' # writes out a file that can't be read properly sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py + ''; + postPatch = '' # re-hardcode path to tail - sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py + sed -i 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py ''; postFixup = '' diff --git a/pkgs/development/tools/build-managers/buildbot/plugins.nix b/pkgs/development/tools/build-managers/buildbot/plugins.nix index 09f8b1e750a5..2875f6942a9e 100644 --- a/pkgs/development/tools/build-managers/buildbot/plugins.nix +++ b/pkgs/development/tools/build-managers/buildbot/plugins.nix @@ -4,11 +4,11 @@ let buildbot-pkg = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-pkg"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "0dfdyc3x0926dynzdl9w7z0p84w287l362mxdl3r6wl87gkisr10"; + sha256 = "0frmnc73dsyc9mjnrnpm4vdrwb7c63gc6maq6xvlp486v7sdhjbi"; }; propagatedBuildInputs = with pythonPackages; [ setuptools ]; @@ -23,22 +23,19 @@ let }; in { - www = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot_www"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; # NOTE: wheel is used due to buildbot circular dependency format = "wheel"; src = fetchurl { - url = "https://pypi.python.org/packages/78/45/b43bd85695cd0178f8bac9c3b394062e9eb46f489b655c11e950e54278a2/${name}-py2-none-any.whl"; - sha256 = "0ixi0y0jhbql55swsvy0jin1v6xf4q4mw9p5n9sll2h10lyp9h0p"; + url = "https://pypi.python.org/packages/02/d0/fc56ee27a09498638a47dcc5637ee5412ab7a67bfb4b3ff47e041f3d7b66/${name}-py2-none-any.whl"; + sha256 = "14ghch67k6090736n89l401swz7r9hnk2zlmdb59niq8lg7dyg9q"; }; - propagatedBuildInputs = [ buildbot-pkg ]; - meta = with stdenv.lib; { homepage = http://buildbot.net/; description = "Buildbot UI"; @@ -51,11 +48,11 @@ in { console-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-console-view"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "1fig635yg5dgn239g9wzfpw9wc3p91lcl9nnig9k7fijz85pwrva"; + sha256 = "0dc7rb7mrpva5gj7l57i96a78d6yj28pkkj9hfim1955z9dgn58l"; }; propagatedBuildInputs = [ buildbot-pkg ]; @@ -72,11 +69,11 @@ in { waterfall-view = pythonPackages.buildPythonPackage rec { name = "${pname}-${version}"; pname = "buildbot-waterfall-view"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "08kh966grj9b4mif337vv7bqy5ixz8xz31ml63wysjb65djnjbk8"; + sha256 = "0x9vvw15zzgj4w3qcxh8r10rb36ni0qh1215y7wbawh5lggnjm0g"; }; propagatedBuildInputs = [ buildbot-pkg ]; diff --git a/pkgs/development/tools/build-managers/buildbot/worker.nix b/pkgs/development/tools/build-managers/buildbot/worker.nix index 01b2051aaa6d..7d7ecc1c52d3 100644 --- a/pkgs/development/tools/build-managers/buildbot/worker.nix +++ b/pkgs/development/tools/build-managers/buildbot/worker.nix @@ -3,11 +3,11 @@ pythonPackages.buildPythonApplication (rec { name = "${pname}-${version}"; pname = "buildbot-worker"; - version = "0.9.0rc4"; + version = "0.9.0.post1"; src = fetchurl { url = "mirror://pypi/b/${pname}/${name}.tar.gz"; - sha256 = "1fv40pki1awv5f2z9vd7phjk7dlsy1cp4blsy2vdhqwbc7112a8c"; + sha256 = "1f8ij3y62r9z7qv92x21rg9h9whhakkwv59rgniq09j64ggjz8lx"; }; buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 3992f2382964..9b6f913e1ac0 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5952,6 +5952,7 @@ in }; buildbot-full = self.buildbot.override { plugins = with self.buildbot-plugins; [ www console-view waterfall-view ]; + enableLocalWorker = true; }; buildkite-agent = callPackage ../development/tools/continuous-integration/buildkite-agent { };