lua: add withPackages function (#54460)

* lua: add withPackages function

First step towards more automation similar to the haskell backend.
Follow up of https://github.com/NixOS/nixpkgs/pull/33903

pkgs/development/interpreters/lua-5/5.1.nix

@@ -1,4 +1,8 @@
-{ stdenv, fetchurl, readline }:
+{ stdenv, fetchurl, readline
+, self
+, callPackage
+, packageOverrides ? (self: super: {})
+}:
 
 let
   dsoPatch = fetchurl {
@@ -6,6 +10,7 @@ let
     sha256 = "11fcyb4q55p4p7kdb8yp85xlw8imy14kzamp2khvcyxss4vw8ipw";
     name = "lua-arch.patch";
   };
+  luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
 in
 stdenv.mkDerivation rec {
   name = "lua-${version}";
@@ -17,6 +22,10 @@ stdenv.mkDerivation rec {
     sha256 = "2640fc56a795f29d28ef15e13c34a47e223960b0240e8cb0a82d9b0738695333";
   };
 
+  LuaPathSearchPaths    = luaPackages.getLuaPathList luaversion;
+  LuaCPathSearchPaths   = luaPackages.getLuaCPathList luaversion;
+  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+
   buildInputs = [ readline ];
 
   patches = (if stdenv.isDarwin then [ ./5.1.darwin.patch ] else [ dsoPatch ])
@@ -39,6 +48,16 @@ stdenv.mkDerivation rec {
     rmdir $out/{share,lib}/lua/5.1 $out/{share,lib}/lua
   '';
 
+  passthru = rec {
+    buildEnv = callPackage ./wrapper.nix {
+      lua=self;
+      inherit (luaPackages) requiredLuaModules;
+    };
+    withPackages = import ./with-packages.nix { inherit buildEnv luaPackages;};
+    pkgs = luaPackages;
+    interpreter = "${self}/bin/lua";
+  };
+
   meta = {
     homepage = http://www.lua.org;
     description = "Powerful, fast, lightweight, embeddable scripting language";
@@ -51,6 +70,7 @@ stdenv.mkDerivation rec {
       for configuration, scripting, and rapid prototyping.
     '';
     license = stdenv.lib.licenses.mit;
+    platforms = with stdenv.lib.platforms; linux ++ darwin;
     hydraPlatforms = stdenv.lib.platforms.linux;
   };
 }

pkgs/development/interpreters/lua-5/5.2.nix

@@ -1,4 +1,10 @@
-{ stdenv, fetchurl, readline, compat ? false }:
+{ stdenv, fetchurl, readline
+# compiles compatibility layer with lua5.1
+, compat ? false
+, callPackage
+, self
+, packageOverrides ? (self: super: {})
+}:
 
 let
   dsoPatch = fetchurl {
@@ -6,12 +12,17 @@ let
     sha256 = "1by1dy4ql61f5c6njq9ibf9kaqm3y633g2q8j54iyjr4cxvqwqz9";
     name = "lua-arch.patch";
   };
+  luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
 in
 stdenv.mkDerivation rec {
   name = "lua-${version}";
   luaversion = "5.2";
   version = "${luaversion}.4";
 
+  LuaPathSearchPaths    = luaPackages.getLuaPathList luaversion;
+  LuaCPathSearchPaths   = luaPackages.getLuaCPathList luaversion;
+  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+
   src = fetchurl {
     url = "https://www.lua.org/ftp/${name}.tar.gz";
     sha256 = "0jwznq0l8qg9wh5grwg07b5cy3lzngvl5m2nl1ikp6vqssmf9qmr";
@@ -21,6 +32,19 @@ stdenv.mkDerivation rec {
 
   patches = if stdenv.isDarwin then [ ./5.2.darwin.patch ] else [ dsoPatch ];
 
+
+  passthru = rec {
+    buildEnv = callPackage ./wrapper.nix {
+      lua = self;
+      inherit (luaPackages) requiredLuaModules;
+    };
+    withPackages = import ./with-packages.nix { inherit buildEnv luaPackages;};
+    pkgs = luaPackages;
+    interpreter = "${self}/bin/lua";
+  };
+
+  enableParallelBuilding = true;
+
   configurePhase =
     if stdenv.isDarwin
     then ''

pkgs/development/interpreters/lua-5/5.3.nix

@@ -1,5 +1,11 @@
-{ stdenv, fetchurl, readline, compat ? false }:
-
+{ stdenv, fetchurl, readline, compat ? false
+, callPackage
+, self
+, packageOverrides ? (self: super: {})
+}:
+let
+  luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
+in
 stdenv.mkDerivation rec {
   name = "lua-${version}";
   luaversion = "5.3";
@@ -10,6 +16,10 @@ stdenv.mkDerivation rec {
     sha256 = "0c2eed3f960446e1a3e4b9a1ca2f3ff893b6ce41942cf54d5dd59ab4b3b058ac";
   };
 
+  LuaPathSearchPaths    = luaPackages.getLuaPathList luaversion;
+  LuaCPathSearchPaths   = luaPackages.getLuaCPathList luaversion;
+  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+
   buildInputs = [ readline ];
 
   patches = if stdenv.isDarwin then [ ./5.2.darwin.patch ] else [];
@@ -54,6 +64,16 @@ stdenv.mkDerivation rec {
     ln -s "$out/lib/pkgconfig/lua.pc" "$out/lib/pkgconfig/lua${luaversion}.pc"
   '';
 
+  passthru = rec {
+    buildEnv = callPackage ./wrapper.nix {
+      lua = self;
+      inherit (luaPackages) requiredLuaModules;
+    };
+    withPackages = import ./with-packages.nix { inherit buildEnv luaPackages;};
+    pkgs = luaPackages;
+    interpreter = "${self}/bin/lua";
+  };
+
   meta = {
     homepage = http://www.lua.org;
     description = "Powerful, fast, lightweight, embeddable scripting language";

pkgs/development/interpreters/lua-5/setup-hook.nix

@@ -0,0 +1,15 @@
+{ runCommand, lib, }:
+
+LuaPathSearchPaths: LuaCPathSearchPaths:
+
+let
+  hook = ./setup-hook.sh;
+in runCommand "lua-setup-hook.sh" {
+  # hum doesn't seem to like caps !! BUG ?
+  luapathsearchpaths=lib.escapeShellArgs LuaPathSearchPaths;
+  luacpathsearchpaths=lib.escapeShellArgs LuaCPathSearchPaths;
+} ''
+  cp ${hook} hook.sh
+  substituteAllInPlace hook.sh
+  mv hook.sh $out
+''

pkgs/development/interpreters/lua-5/setup-hook.sh

@@ -0,0 +1,47 @@
+# set -e
+
+nix_print() {
+    if (( "${NIX_DEBUG:-0}" >= $1 )); then
+        echo "$2"
+    fi
+}
+
+nix_debug() {
+    nix_print 3 "$1"
+}
+
+addToLuaSearchPathWithCustomDelimiter() {
+    local varName="$1"
+    local absPattern="$2"
+    # delete longest match starting from the lua placeholder '?'
+    local topDir="${absPattern%%\?*}"
+
+    # export only if the folder exists else LUA_PATH grows too big
+    if  [ ! -d "$topDir" ]; then return; fi
+
+    export "${varName}=${!varName:+${!varName};}${absPattern}"
+}
+
+addToLuaPath() {
+    local dir="$1"
+
+    if [[ ! -d "$dir" ]]; then
+        nix_debug "$dir not a directory abort"
+        return 0
+    fi
+    cd "$dir"
+    for pattern in @luapathsearchpaths@;
+    do
+        addToLuaSearchPathWithCustomDelimiter LUA_PATH "$PWD/$pattern"
+    done
+
+    # LUA_CPATH
+    for pattern in @luacpathsearchpaths@;
+    do
+        addToLuaSearchPathWithCustomDelimiter LUA_CPATH "$PWD/$pattern"
+    done
+    cd -
+}
+
+addEnvHooks "$hostOffset" addToLuaPath
+

pkgs/development/interpreters/lua-5/with-packages.nix

@@ -0,0 +1,4 @@
+{ buildEnv, luaPackages }:
+
+# this is a function that returns a function that returns an environment
+f: let packages = f luaPackages; in buildEnv.override { extraLibs = packages; }

pkgs/development/interpreters/lua-5/wrapper.nix

@@ -0,0 +1,73 @@
+{ stdenv, lua, buildEnv, makeWrapper
+, extraLibs ? []
+, extraOutputsToInstall ? []
+, postBuild ? ""
+, ignoreCollisions ? false
+, lib
+, requiredLuaModules
+, makeWrapperArgs ? []
+}:
+
+# Create a lua executable that knows about additional packages.
+let
+  env = let
+    paths =  requiredLuaModules (extraLibs ++ [ lua ] );
+  in buildEnv {
+    name = "${lua.name}-env";
+
+    inherit paths;
+    inherit ignoreCollisions;
+    extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall;
+
+    # we create wrapper for the binaries in the different packages
+    postBuild = ''
+
+      . "${makeWrapper}/nix-support/setup-hook"
+
+      # get access to lua functions
+      . ${lua}/nix-support/setup-hook
+
+      if [ -L "$out/bin" ]; then
+          unlink "$out/bin"
+      fi
+      mkdir -p "$out/bin"
+
+      addToLuaPath "$out"
+
+      # take every binary from lua packages and put them into the env
+      for path in ${stdenv.lib.concatStringsSep " " paths}; do
+        nix_debug "looking for binaries in path = $path"
+        if [ -d "$path/bin" ]; then
+          cd "$path/bin"
+          for prg in *; do
+            if [ -f "$prg" ]; then
+              rm -f "$out/bin/$prg"
+              if [ -x "$prg" ]; then
+                nix_debug "Making wrapper $prg"
+                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --suffix LUA_PATH ';' "$LUA_PATH"   --suffix LUA_CPATH ';' "$LUA_CPATH" ${stdenv.lib.concatStringsSep " " makeWrapperArgs}
+              fi
+            fi
+          done
+        fi
+      done
+    '' + postBuild;
+
+    inherit (lua) meta;
+
+    passthru = lua.passthru // {
+      interpreter = "${env}/bin/lua";
+      inherit lua;
+      env = stdenv.mkDerivation {
+        name = "interactive-${lua.name}-environment";
+        nativeBuildInputs = [ env ];
+
+        buildCommand = ''
+          echo >&2 ""
+          echo >&2 "*** lua 'env' attributes are intended for interactive nix-shell sessions, not for building! ***"
+          echo >&2 ""
+          exit 1
+        '';
+    };
+    };
+  };
+in env

pkgs/development/interpreters/luajit/2.0.nix

@@ -0,0 +1,10 @@
+{ self, callPackage, lib }:
+callPackage ./default.nix {
+  inherit self;
+  version = "2.0.5";
+  isStable = true;
+  sha256 = "0yg9q4q6v028bgh85317ykc9whgxgysp76qzaqgq55y6jy11yjw7";
+  extraMeta = {
+    platforms = lib.filter (p: p != "aarch64-linux") lib.meta.platforms;
+  };
+}

pkgs/development/interpreters/luajit/2.1.nix

@@ -0,0 +1,7 @@
+{ self, callPackage, lib }:
+callPackage ./default.nix {
+  inherit self;
+  version = "2.1.0-beta3";
+  isStable = false;
+  sha256 = "1hyrhpkwjqsv54hnnx4cl8vk44h9d6c9w0fz1jfjz00w255y7lhs";
+}

pkgs/development/interpreters/luajit/default.nix

@@ -1,71 +1,74 @@
-{ stdenv, lib, fetchurl }:
-rec {
+{ stdenv, lib, fetchurl
+, name ? "luajit-${version}"
+, isStable
+, sha256
+, version
+, extraMeta ? {}
+, callPackage
+, self
+, packageOverrides ? (self: super: {})
+}:
+let
+  luaPackages = callPackage ../../lua-modules {lua=self; overrides=packageOverrides;};
+in
+stdenv.mkDerivation rec {
+  inherit name version;
+  src = fetchurl {
+    url = "http://luajit.org/download/LuaJIT-${version}.tar.gz";
+    inherit sha256;
+  };
 
-  luajit = luajit_2_1;
+  luaversion = "5.1";
 
-  luajit_2_0 = generic {
-    version = "2.0.5";
-    isStable = true;
-    sha256 = "0yg9q4q6v028bgh85317ykc9whgxgysp76qzaqgq55y6jy11yjw7";
-    meta = genericMeta // {
-      platforms = lib.filter (p: p != "aarch64-linux") genericMeta.platforms;
+  patchPhase = ''
+    substituteInPlace Makefile \
+      --replace /usr/local "$out"
+
+    substituteInPlace src/Makefile --replace gcc cc
+  '' + stdenv.lib.optionalString (stdenv.cc.libc != null)
+  ''
+    substituteInPlace Makefile \
+      --replace ldconfig ${stdenv.cc.libc.bin or stdenv.cc.libc}/bin/ldconfig
+  '';
+
+  configurePhase = false;
+
+  buildFlags = [ "amalg" ]; # Build highly optimized version
+  enableParallelBuilding = true;
+
+  installPhase   = ''
+      make install PREFIX="$out"
+      ( cd "$out/include"; ln -s luajit-*/* . )
+      ln -s "$out"/bin/luajit-* "$out"/bin/lua
+    ''
+    + stdenv.lib.optionalString (!isStable) ''
+      ln -s "$out"/bin/luajit-* "$out"/bin/luajit
+    '';
+
+  LuaPathSearchPaths = [
+    "lib/lua/${luaversion}/?.lua" "share/lua/${luaversion}/?.lua"
+    "share/lua/${luaversion}/?/init.lua" "lib/lua/${luaversion}/?/init.lua"
+    "share/${name}/?.lua"
+  ];
+  LuaCPathSearchPaths = [ "lib/lua/${luaversion}/?.so" "share/lua/${luaversion}/?.so" ];
+  setupHook = luaPackages.lua-setup-hook LuaPathSearchPaths LuaCPathSearchPaths;
+
+  passthru = rec {
+    buildEnv = callPackage ../lua-5/wrapper.nix {
+      lua = self;
+      inherit (luaPackages) requiredLuaModules;
     };
+    withPackages = import ../lua-5/with-packages.nix { inherit buildEnv luaPackages;};
+    pkgs = luaPackages;
+    interpreter = "${self}/bin/lua";
   };
 
-  luajit_2_1 = generic {
-    version = "2.1.0-beta3";
-    isStable = false;
-    sha256 = "1hyrhpkwjqsv54hnnx4cl8vk44h9d6c9w0fz1jfjz00w255y7lhs";
-  };
-
-  genericMeta = with stdenv.lib; {
+  meta = with stdenv.lib; extraMeta // {
     description = "High-performance JIT compiler for Lua 5.1";
     homepage    = http://luajit.org;
     license     = licenses.mit;
     platforms   = platforms.linux ++ platforms.darwin;
     maintainers = with maintainers; [ thoughtpolice smironov vcunat andir ];
   };
-
-  generic =
-    { version, sha256 ? null, isStable
-    , name ? "luajit-${version}"
-    , src ?
-      (fetchurl {
-        url = "http://luajit.org/download/LuaJIT-${version}.tar.gz";
-        inherit sha256;
-      })
-    , meta ? genericMeta
-    }:
-
-    stdenv.mkDerivation rec {
-      inherit name version src meta;
-
-      luaversion = "5.1";
-
-      patchPhase = ''
-        substituteInPlace Makefile \
-          --replace /usr/local "$out"
-
-        substituteInPlace src/Makefile --replace gcc cc
-      '' + stdenv.lib.optionalString (stdenv.cc.libc != null)
-      ''
-        substituteInPlace Makefile \
-          --replace ldconfig ${stdenv.cc.libc.bin or stdenv.cc.libc}/bin/ldconfig
-      '';
-
-      configurePhase = false;
-
-      buildFlags = [ "amalg" ]; # Build highly optimized version
-      enableParallelBuilding = true;
-
-      installPhase   = ''
-        make install PREFIX="$out"
-        ( cd "$out/include"; ln -s luajit-*/* . )
-        ln -s "$out"/bin/luajit-* "$out"/bin/lua
-      ''
-        + stdenv.lib.optionalString (!isStable)
-          ''
-            ln -s "$out"/bin/luajit-* "$out"/bin/luajit
-          '';
-    };
 }
+

pkgs/development/lua-modules/default.nix

@@ -0,0 +1,17 @@
+# inspired by pkgs/development/haskell-modules/default.nix
+{ pkgs, stdenv, lib
+, lua
+, overrides ? (self: super: {})
+}:
+
+let
+
+  inherit (lib) extends makeExtensible;
+
+  initialPackages = (pkgs.callPackage ../../top-level/lua-packages.nix {
+    inherit lua;
+  });
+
+  extensible-self = makeExtensible initialPackages;
+in
+  extensible-self

pkgs/development/lua-modules/generic/default.nix

@@ -1,11 +1,11 @@
-{ lua, writeText }:
+{ lua, writeText, toLuaModule }:
 
 { buildInputs ? [], disabled ? false, ... } @ attrs:
 
 if disabled then
   throw "${attrs.name} not supported by interpreter lua-${lua.luaversion}"
 else
-  lua.stdenv.mkDerivation (
+  toLuaModule( lua.stdenv.mkDerivation (
     {
       makeFlags = [
         "PREFIX=$(out)"
@@ -51,4 +51,4 @@ else
         addEnvHooks "$hostOffset" addLuaLibCPath
       '';
     }
-  )
+  ) )

pkgs/top-level/all-packages.nix

@@ -7750,28 +7750,43 @@ in
   wabt = callPackage ../development/tools/wabt { };
 
   ### LUA MODULES
-
-  lua5_1 = callPackage ../development/interpreters/lua-5/5.1.nix { };
-  lua5_2 = callPackage ../development/interpreters/lua-5/5.2.nix { };
+  lua5_1 = callPackage ../development/interpreters/lua-5/5.1.nix {
+    self = lua5_1;
+  };
+  lua5_2 = callPackage ../development/interpreters/lua-5/5.2.nix {
+    self = lua5_2;
+  };
   lua5_2_compat = callPackage ../development/interpreters/lua-5/5.2.nix {
     compat = true;
+    self = lua5_2_compat;
+  };
+  lua5_3 = callPackage ../development/interpreters/lua-5/5.3.nix {
+    self = lua5_3;
   };
-  lua5_3 = callPackage ../development/interpreters/lua-5/5.3.nix { };
   lua5_3_compat = callPackage ../development/interpreters/lua-5/5.3.nix {
     compat = true;
+    self = lua5_3_compat;
   };
   lua5 = lua5_2_compat;
   lua = lua5;
 
-  lua51Packages = recurseIntoAttrs (callPackage ./lua-packages.nix { lua = lua5_1; });
-  lua52Packages = recurseIntoAttrs (callPackage ./lua-packages.nix { lua = lua5_2; });
-  lua53Packages = recurseIntoAttrs (callPackage ./lua-packages.nix { lua = lua5_3; });
-  luajitPackages = recurseIntoAttrs (callPackage ./lua-packages.nix { lua = luajit; });
+  lua51Packages = recurseIntoAttrs lua5_1.pkgs;
+  lua52Packages = recurseIntoAttrs lua5_2.pkgs;
+  lua53Packages = recurseIntoAttrs lua5_3.pkgs;
+  luajitPackages = recurseIntoAttrs luajit.pkgs;
 
   luaPackages = lua52Packages;
 
-  inherit (callPackages ../development/interpreters/luajit {})
-    luajit luajit_2_0 luajit_2_1;
+  # override instead ?
+  luajit_2_0 = callPackage ../development/interpreters/luajit/2.0.nix {
+    self = luajit_2_0;
+  };
+
+  luajit_2_1 = callPackage ../development/interpreters/luajit/2.1.nix {
+    self = luajit_2_1;
+  };
+
+  luajit = luajit_2_1;
 
   luarocks = luaPackages.luarocks;
   luarocks-nix = luaPackages.luarocks-nix;

pkgs/top-level/lua-packages.nix

@@ -10,13 +10,45 @@
 , glib, gobject-introspection, libevent, zlib, autoreconfHook, gnum4
 , mysql, postgresql, cyrus_sasl
 , fetchFromGitHub, libmpack, which, fetchpatch, writeText
+, pkgs
+, fetchgit
+, overrides ? (self: super: {})
+, lib
 }:
 
 let
-  isLua52 = lua.luaversion == "5.2";
+  packages = ( self:
+
+let
+  luaAtLeast = lib.versionAtLeast lua.luaversion;
+  luaOlder = lib.versionOlder lua.luaversion;
+  isLua51 = (lib.versions.majorMinor lua.version) == "5.1";
+  isLua52 = (lib.versions.majorMinor lua.version) == "5.2";
   isLua53 = lua.luaversion == "5.3";
   isLuaJIT = (builtins.parseDrvName lua.name).name == "luajit";
 
+  lua-setup-hook = callPackage ../development/interpreters/lua-5/setup-hook.nix { };
+
+  # Check whether a derivation provides a lua module.
+  hasLuaModule = drv: drv ? luaModule ;
+
+  callPackage = pkgs.newScope self;
+
+  requiredLuaModules = drvs: with stdenv.lib; let
+    modules =  filter hasLuaModule drvs;
+  in unique ([lua] ++ modules ++ concatLists (catAttrs "requiredLuaModules" modules));
+
+  # Convert derivation to a lua module.
+  toLuaModule = drv:
+    drv.overrideAttrs( oldAttrs: {
+      # Use passthru in order to prevent rebuilds when possible.
+      passthru = (oldAttrs.passthru or {})// {
+        luaModule = lua;
+        requiredLuaModules = requiredLuaModules drv.propagatedBuildInputs;
+      };
+    });
+
+
   platformString =
     if stdenv.isDarwin then "macosx"
     else if stdenv.isFreeBSD then "freebsd"
@@ -24,10 +56,16 @@ let
     else if stdenv.isSunOS then "solaris"
     else throw "unsupported platform";
 
-  self = _self;
-  _self = with self; {
-  inherit lua;
-  inherit (stdenv.lib) maintainers;
+in
+with self; {
+
+  getLuaPathList = majorVersion: [
+     "lib/lua/${majorVersion}/?.lua" "share/lua/${majorVersion}/?.lua"
+    "share/lua/${majorVersion}/?/init.lua" "lib/lua/${majorVersion}/?/init.lua"
+  ];
+  getLuaCPathList = majorVersion: [
+     "lib/lua/${majorVersion}/?.so" "share/lua/${majorVersion}/?.so" "share/lua/${majorVersion}/?/init.so"
+  ];
 
   # helper functions for dealing with LUA_PATH and LUA_CPATH
   getPath       = lib : type : "${lib}/lib/lua/${lua.luaversion}/?.${type};${lib}/share/lua/${lua.luaversion}/?.${type}";
@@ -39,6 +77,11 @@ let
     inherit lua writeText;
   };
 
+
+  inherit toLuaModule lua-setup-hook;
+  inherit requiredLuaModules luaOlder luaAtLeast
+    isLua51 isLua52 isLuaJIT lua callPackage;
+
   luarocks = callPackage ../development/tools/misc/luarocks {
     inherit lua;
   };
@@ -1078,4 +1121,5 @@ let
     };
   };
 
-}; in self
+});
+in (lib.extends overrides packages)