nixpkgs/pkgs/development/lisp-modules-new/lisp-packages.nix
2022-11-24 19:50:30 +01:00

418 lines
14 KiB
Nix

# TODO:
# - faster build by using lisp with preloaded asdf?
# - dont include java libs unless abcl?
# - dont use build-asdf-system to build lispWithPackages?
# - make the lisp packages overridable? (e.g. buildInputs glibc->musl)
# - build asdf with nix and use that instead of one shipped with impls
# (e.g. to fix build with clisp - does anyone use clisp?)
# - claspPackages ? (gotta package clasp with nix first)
# - hard one: remove unrelated sources ( of systems not being built)
# - figure out a less awkward way to patch sources
# (have to build from src directly for SLIME to work, so can't just patch sources in place)
{ pkgs, lib, stdenv, ... }:
let
inherit (lib)
length
filter
foldl
unique
id
concat
concatMap
mutuallyExclusive
findFirst
remove
setAttr
getAttr
hasAttr
attrNames
attrValues
filterAttrs
mapAttrs
splitString
concatStringsSep
concatMapStringsSep
replaceStrings
removeSuffix
hasInfix
optionalString
makeBinPath
makeLibraryPath
makeSearchPath
recurseIntoAttrs
dontRecurseIntoAttrs
;
inherit (builtins)
head
tail
elem
split
storeDir;
# Returns a flattened dependency tree without duplicates
# This is probably causing performance problems...
flattenedDeps = lispLibs:
let
toSet = list: builtins.listToAttrs (map (d: { name = d.pname; value = d; }) list);
toList = attrValues;
walk = acc: node:
if length node.lispLibs == 0
then acc
else builtins.foldl' walk (acc // toSet node.lispLibs) node.lispLibs;
in toList (walk {} { inherit lispLibs; });
# Stolen from python-packages.nix
# Actually no idea how this works
makeOverridableLispPackage = f: origArgs:
let
ff = f origArgs;
overrideWith = newArgs: origArgs // (if pkgs.lib.isFunction newArgs then newArgs origArgs else newArgs);
in
if builtins.isAttrs ff then (ff // {
overrideLispAttrs = newArgs: makeOverridableLispPackage f (overrideWith newArgs);
})
else if builtins.isFunction ff then {
overrideLispAttrs = newArgs: makeOverridableLispPackage f (overrideWith newArgs);
__functor = self: ff;
}
else ff;
#
# Wrapper around stdenv.mkDerivation for building ASDF systems.
#
build-asdf-system = makeOverridableLispPackage (
{ pname,
version,
src ? null,
patches ? [],
# Native libraries, will be appended to the library path
nativeLibs ? [],
# Java libraries for ABCL, will be appended to the class path
javaLibs ? [],
# Lisp dependencies
# these should be packages built with `build-asdf-system`
lispLibs ? [],
# Lisp command to run buildScript
lisp,
# Some libraries have multiple systems under one project, for
# example, cffi has cffi-grovel, cffi-toolchain etc. By
# default, only the `pname` system is build.
#
# .asd's not listed in `systems` are removed in
# installPhase. This prevents asdf from referring to uncompiled
# systems on run time.
#
# Also useful when the pname is differrent than the system name,
# such as when using reverse domain naming.
systems ? [ pname ],
# The .asd files that this package provides
asds ? systems,
# Other args to mkDerivation
...
} @ args:
let
# A little slow for big dependency graphs.
# How to make it faster?
# Maybe do it in the buildScript in Common Lisp or Bash, instead
# of here with Nix?
libsFlat = flattenedDeps lispLibs;
in stdenv.mkDerivation (rec {
inherit pname version nativeLibs javaLibs lispLibs lisp systems asds;
# When src is null, we are building a lispWithPackages and only
# want to make use of the dependency environment variables
# generated by build-asdf-system
dontUnpack = src == null;
# TODO: Do the propagation like for lisp, native and java like this:
# https://github.com/teu5us/nix-lisp-overlay/blob/e30dafafa5c1b9a5b0ccc9aaaef9d285d9f0c46b/pkgs/development/lisp-modules/setup-hook.sh
# Then remove the "echo >> nix-drvs" from buildScript
# Tell asdf where to find system definitions of lisp dependencies.
#
# The "//" ending is important as it makes asdf recurse into
# subdirectories when searching for .asd's. This is to support
# projects where .asd's aren't in the root directory.
CL_SOURCE_REGISTRY = makeSearchPath "/" libsFlat;
# Tell lisp where to find native dependencies
#
# Normally generated from lispLibs, but LD_LIBRARY_PATH as a
# derivation attr itself can be used as an extension point when
# the libs are not in a '/lib' subdirectory
LD_LIBRARY_PATH =
let
libs = concatMap (x: x.nativeLibs) libsFlat;
paths = filter (x: x != "") (map (x: x.LD_LIBRARY_PATH) libsFlat);
path =
makeLibraryPath libs
+ optionalString (length paths != 0) ":"
+ concatStringsSep ":" paths;
in concatStringsSep ":" (unique (splitString ":" path));
# Java libraries For ABCL
CLASSPATH = makeSearchPath "share/java/*" (concatMap (x: x.javaLibs) libsFlat);
# Portable script to build the systems.
#
# `lisp` must evaluate this file then exit immediately. For
# example, SBCL's --script flag does just that.
#
# NOTE:
# Every other library worked fine with asdf:compile-system in
# buildScript.
#
# cl-syslog, for some reason, signals that CL-SYSLOG::VALID-SD-ID-P
# is undefined with compile-system, but works perfectly with
# load-system. Strange.
buildScript = pkgs.writeText "build-${pname}.lisp" ''
(require :asdf)
(dolist (s '(${concatStringsSep " " systems}))
(asdf:load-system s))
'';
buildPhase = optionalString (src != null) ''
# In addition to lisp dependencies, make asdf see the .asd's
# of the systems being built
#
# *Append* src since `lispLibs` can provide .asd's that are
# also in `src` but are not in `systems` (that is, the .asd's
# that will be deleted in installPhase). We don't want to
# rebuild them, but to load them from lispLibs.
#
# NOTE: It's important to read files from `src` instead of
# from pwd to get go-to-definition working with SLIME
export CL_SOURCE_REGISTRY=$CL_SOURCE_REGISTRY:$src//
# Similiarily for native deps
export LD_LIBRARY_PATH=${makeLibraryPath nativeLibs}:$LD_LIBRARY_PATH
export CLASSPATH=${makeSearchPath "share/java/*" javaLibs}:$CLASSPATH
# Make asdf compile from `src` to pwd and load `lispLibs`
# from storeDir. Otherwise it could try to recompile lisp deps.
export ASDF_OUTPUT_TRANSLATIONS="$src:$(pwd):${storeDir}:${storeDir}"
# track lisp dependencies for graph generation
# TODO: Do the propagation like for lisp, native and java like this:
# https://github.com/teu5us/nix-lisp-overlay/blob/e30dafafa5c1b9a5b0ccc9aaaef9d285d9f0c46b/pkgs/development/lisp-modules/setup-hook.sh
# Then remove the "echo >> nix-drvs" from buildScript
echo $lispLibs >> __nix-drvs
# Finally, compile the systems
${lisp} $buildScript
'';
# Copy compiled files to store
#
# Make sure to include '$' in regex to prevent skipping
# stuff like 'iolib.asdf.asd' for system 'iolib.asd'
#
# Same with '/': `local-time.asd` for system `cl-postgres+local-time.asd`
installPhase =
let
mkSystemsRegex = systems:
concatMapStringsSep "\\|" (replaceStrings ["." "+"] ["[.]" "[+]"]) systems;
in
''
mkdir -pv $out
cp -r * $out
# Remove all .asd files except for those in `systems`.
find $out -name "*.asd" \
| grep -v "/\(${mkSystemsRegex systems}\)\.asd$" \
| xargs rm -fv || true
'';
# Not sure if it's needed, but caused problems with SBCL
# save-lisp-and-die binaries in the past
dontStrip = true;
dontFixup = true;
} // (args // {
src = if builtins.length (args.patches or []) > 0
then pkgs.applyPatches { inherit (args) src patches; }
else args.src;
patches = [];
# make sure that propagated build-inputs from lispLibs are propagated
propagatedBuildInputs = lib.unique
(builtins.concatLists
(lib.catAttrs "propagatedBuildInputs"
(builtins.concatLists [[args] lispLibs nativeLibs javaLibs])));
})));
# Build the set of lisp packages using `lisp`
# These packages are defined manually for one reason or another:
# - The library is not in quicklisp
# - The library that is in quicklisp is broken
# - Special build procedure such as cl-unicode, asdf
#
# These Probably could be done even in ql.nix
# - Want to pin a specific commit
# - Want to apply custom patches
#
# They can use the auto-imported quicklisp packages as dependencies,
# but some of those don't work out of the box.
#
# E.g if a QL package depends on cl-unicode it won't build out of
# the box. The dependency has to be rewritten using the manually
# fixed cl-unicode.
#
# This is done by generating a 'fixed' set of Quicklisp packages by
# calling quicklispPackagesFor with the right `fixup`.
commonLispPackagesFor = lisp:
let
build-asdf-system' = body: build-asdf-system (body // { inherit lisp; });
in import ./packages.nix {
inherit pkgs;
inherit lisp;
inherit quicklispPackagesFor;
inherit fixupFor;
build-asdf-system = build-asdf-system';
};
# Build the set of packages imported from quicklisp using `lisp`
quicklispPackagesFor = { lisp, fixup ? lib.id, build ? build-asdf-system }:
let
build-asdf-system' = body: build (body // {
inherit lisp;
});
in import ./ql.nix {
inherit pkgs;
inherit fixup;
build-asdf-system = build-asdf-system';
};
# Rewrite deps of pkg to use manually defined packages
#
# The purpose of manual packages is to customize one package, but
# then it has to be propagated everywhere for it to make sense and
# have consistency in the package tree.
fixupFor = manualPackages: qlPkg:
assert (lib.isAttrs qlPkg && !lib.isDerivation qlPkg);
let
# Make it possible to reuse generated attrs without recursing into oblivion
packages = (lib.filterAttrs (n: v: n != qlPkg.pname) manualPackages);
substituteLib = pkg:
if lib.hasAttr pkg.pname packages
then packages.${pkg.pname}
else pkg;
pkg = substituteLib qlPkg;
in pkg // { lispLibs = map substituteLib pkg.lispLibs; };
makeAttrName = str:
removeSuffix
"_"
(replaceStrings
["+" "." "/"]
["_plus_" "_dot_" "_slash_"]
str);
oldMakeWrapper = pkgs.runCommand "make-wrapper.sh" {} ''
substitute ${./old-make-wrapper.sh} $out \
--replace @shell@ ${pkgs.bash}/bin/bash
'';
# Creates a lisp wrapper with `packages` installed
#
# `packages` is a function that takes `clpkgs` - a set of lisp
# packages - as argument and returns the list of packages to be
# installed
lispWithPackagesInternal = clpkgs: packages:
# FIXME just use flattenedDeps instead
(build-asdf-system rec {
lisp = (head (lib.attrValues clpkgs)).lisp;
# See dontUnpack in build-asdf-system
src = null;
pname = baseNameOf (head (split " " lisp));
version = "with-packages";
lispLibs = packages clpkgs;
systems = [];
}).overrideAttrs(o: {
installPhase = ''
# The recent version of makeWrapper causes breakage. For more info see
# https://github.com/Uthar/nix-cl/issues/2
source ${oldMakeWrapper}
mkdir -pv $out/bin
makeWrapper \
${head (split " " o.lisp)} \
$out/bin/${baseNameOf (head (split " " o.lisp))} \
--prefix CL_SOURCE_REGISTRY : "${o.CL_SOURCE_REGISTRY}" \
--prefix ASDF_OUTPUT_TRANSLATIONS : ${concatStringsSep "::" (flattenedDeps o.lispLibs)}: \
--prefix LD_LIBRARY_PATH : "${o.LD_LIBRARY_PATH}" \
--prefix LD_LIBRARY_PATH : "${makeLibraryPath o.nativeLibs}" \
--prefix CLASSPATH : "${o.CLASSPATH}" \
--prefix CLASSPATH : "${makeSearchPath "share/java/*" o.javaLibs}" \
--prefix PATH : "${makeBinPath (o.buildInputs or [])}" \
--prefix PATH : "${makeBinPath (o.propagatedBuildInputs or [])}"
'';
});
lispWithPackages = lisp:
let
packages = lispPackagesFor lisp;
in lispWithPackagesInternal packages;
lispPackagesFor = lisp:
let
packages = commonLispPackagesFor lisp;
qlPackages = quicklispPackagesFor {
inherit lisp;
fixup = fixupFor packages;
};
in qlPackages // packages;
commonLispPackages = rec {
inherit
build-asdf-system
lispWithPackagesInternal
lispPackagesFor
lispWithPackages;
# TODO: uncomment clasp when clasp 1.0.0 is packaged
# There's got to be a better way than this...
# The problem was that with --load everywhere, some
# implementations didn't exit with 0 on compilation failure
# Maybe a handler-case in buildScript?
sbcl = "${pkgs.sbcl}/bin/sbcl --script";
ecl = "${pkgs.ecl}/bin/ecl --shell";
abcl = ''${pkgs.abcl}/bin/abcl --batch --eval "(load \"$buildScript\")"'';
ccl = ''${pkgs.ccl}/bin/ccl --batch --eval "(load \"$buildScript\")" --'';
# clasp = ''${pkgs.clasp}/bin/clasp --non-interactive --quit --load'';
# Manually defined packages shadow the ones imported from quicklisp
sbclPackages = recurseIntoAttrs (lispPackagesFor sbcl);
eclPackages = dontRecurseIntoAttrs (lispPackagesFor ecl);
abclPackages = dontRecurseIntoAttrs (lispPackagesFor abcl);
cclPackages = dontRecurseIntoAttrs (lispPackagesFor ccl);
# claspPackages = lispPackagesFor clasp;
sbclWithPackages = lispWithPackages sbcl;
eclWithPackages = lispWithPackages ecl;
abclWithPackages = lispWithPackages abcl;
cclWithPackages = lispWithPackages ccl;
# claspWithPackages = lispWithPackages clasp;
};
in commonLispPackages