Merge pull request #51986 from ivan/add-grab-site

grab-site: init at 2.1.11

pkgs/development/python-modules/fb-re2/default.nix

@@ -0,0 +1,30 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, re2
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "fb-re2";
+  version = "1.0.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0wd97qdcafcca90s6692g2dmzl5n6cdvm20vn7pmag3l9gvx395c";
+  };
+
+  buildInputs = [ re2 ];
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = {
+    description = "Python wrapper for Google's RE2";
+    homepage = https://github.com/facebook/pyre2;
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/ludios_wpull/default.nix

@@ -0,0 +1,39 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, isPy3k
+, chardet
+, dnspython
+, html5-parser
+, lxml
+, namedlist
+, sqlalchemy
+, tornado_4
+, Yapsy
+}:
+
+buildPythonPackage rec {
+  pname = "ludios_wpull";
+  version = "3.0.7";
+
+  disabled = !isPy3k;
+
+  src = fetchFromGitHub {
+    rev = "${version}";
+    owner = "ludios";
+    repo = "wpull";
+    sha256 = "1j96avm0ynbazypzp766wh26n4qc73y7wgsiqfrdfl6x7rx20wgf";
+  };
+
+  propagatedBuildInputs = [ chardet dnspython html5-parser lxml namedlist sqlalchemy tornado_4 Yapsy ];
+
+  # Test suite has tests that fail on all platforms
+  doCheck = false;
+
+  meta = {
+    description = "Web crawler; fork of wpull used by grab-site";
+    homepage = https://github.com/ludios/wpull;
+    license = lib.licenses.gpl3;
+    maintainers = with lib.maintainers; [ ivan ];
+  };
+}

pkgs/development/python-modules/namedlist/default.nix

@@ -0,0 +1,33 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, pytest
+}:
+
+buildPythonPackage rec {
+  pname = "namedlist";
+  version = "1.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "11n9c4a5ak9971awkf1g92m6mcmiprhrw98ik2cmjsqxmz73j2qr";
+  };
+
+  # Test file has a `unittest.main()` at the bottom that fails the tests;
+  # py.test can run the tests without it.
+  postPatch = ''
+    substituteInPlace test/test_namedlist.py --replace "unittest.main()" ""
+  '';
+
+  checkInputs = [ pytest ];
+  checkPhase = ''
+    py.test
+  '';
+
+  meta = {
+    description = "Similar to namedtuple, but instances are mutable";
+    homepage = https://bitbucket.org/ericvsmith/namedlist;
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ ivan ];
+  };
+}

pkgs/tools/backup/grab-site/default.nix

@@ -0,0 +1,30 @@
+{ stdenv, python3Packages, fetchFromGitHub }:
+
+python3Packages.buildPythonApplication rec {
+  version = "2.1.11";
+  name = "grab-site-${version}";
+
+  src = fetchFromGitHub {
+    rev = "${version}";
+    owner = "ludios";
+    repo = "grab-site";
+    sha256 = "0w24ngr2b7nipqiwkxpql2467b5aq2vbknkb0sry6a457kb5ppsl";
+  };
+
+  propagatedBuildInputs = with python3Packages; [
+    click ludios_wpull manhole lmdb autobahn fb-re2 websockets cchardet
+  ];
+
+  checkPhase = ''
+    export PATH=$PATH:$out/bin
+    bash ./tests/offline-tests
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Crawler for web archiving with WARC output";
+    homepage = https://github.com/ludios/grab-site;
+    license = licenses.mit;
+    maintainers = with maintainers; [ ivan ];
+    platforms = platforms.all;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -10004,6 +10004,8 @@ in
 
   gperftools = callPackage ../development/libraries/gperftools { };
 
+  grab-site = callPackage ../tools/backup/grab-site { };
+
   grib-api = callPackage ../development/libraries/grib-api { };
 
   grpc = callPackage ../development/libraries/grpc { };

pkgs/top-level/python-packages.nix

@@ -1671,6 +1671,8 @@ in {
     then callPackage ../development/python-modules/faulthandler {}
     else throw "faulthandler is built into ${python.executable}";
 
+  fb-re2 = callPackage ../development/python-modules/fb-re2 { };
+
   flit = callPackage ../development/python-modules/flit { };
 
   flowlogs_reader = callPackage ../development/python-modules/flowlogs_reader { };
@@ -1872,6 +1874,8 @@ in {
 
   lpod = callPackage ../development/python-modules/lpod { };
 
+  ludios_wpull = callPackage ../development/python-modules/ludios_wpull { };
+
   luftdaten = callPackage ../development/python-modules/luftdaten { };
 
   m2r = callPackage ../development/python-modules/m2r { };
@@ -3078,6 +3082,8 @@ in {
 
   namebench = callPackage ../development/python-modules/namebench { };
 
+  namedlist = callPackage ../development/python-modules/namedlist { };
+
   nameparser = callPackage ../development/python-modules/nameparser { };
 
   nbconvert = callPackage ../development/python-modules/nbconvert { };