diff --git a/doc/default.nix b/doc/default.nix
index 60c613878c72..ec458634a42c 100644
--- a/doc/default.nix
+++ b/doc/default.nix
@@ -81,6 +81,10 @@ pkgs.stdenv.mkDerivation {
inputFile = ./languages-frameworks/vim.md;
outputFile = "./languages-frameworks/vim.xml";
}
+ + toDocbook {
+ inputFile = ./languages-frameworks/emscripten.md;
+ outputFile = "./languages-frameworks/emscripten.xml";
+ }
+ ''
echo ${lib.nixpkgsVersion} > .version
diff --git a/doc/languages-frameworks/emscripten.md b/doc/languages-frameworks/emscripten.md
new file mode 100644
index 000000000000..24c49ec1409c
--- /dev/null
+++ b/doc/languages-frameworks/emscripten.md
@@ -0,0 +1,185 @@
+# User's Guide to Emscripten in Nixpkgs
+
+[Emscripten](https://github.com/kripken/emscripten): An LLVM-to-JavaScript Compiler
+
+This section of the manual covers how to use `emscripten` in nixpkgs.
+
+Minimal requirements:
+
+* nix
+* nixpkgs
+
+Modes of use of `emscripten`:
+
+* **Imperative usage** (on the command line):
+
+ If you want to work with `emcc`, `emconfigure` and `emmake` as you are used to from Ubuntu and similar distributions you can use these commands:
+
+ * `nix-env -i emscripten`
+ * `nix-shell -p emscripten`
+
+* **Declarative usage**:
+
+ This mode is far more power full since this makes use of `nix` for dependency management of emscripten libraries and targets by using the `mkDerivation` which is implemented by `pkgs.emscriptenStdenv` and `pkgs.buildEmscriptenPackage`. The source for the packages is in `pkgs/top-level/emscripten-packages.nix` and the abstraction behind it in `pkgs/development/em-modules/generic/default.nix`.
+ * build and install all packages:
+ * `nix-env -iA emscriptenPackages`
+
+ * dev-shell for zlib implementation hacking:
+ * `nix-shell -A emscriptenPackages.zlib`
+
+
+## Imperative usage
+
+A few things to note:
+
+* `export EMCC_DEBUG=2` is nice for debugging
+* `~/.emscripten`, the build artifact cache sometimes creates issues and needs to be removed from time to time
+
+
+## Declarative usage
+
+Let's see two different examples from `pkgs/top-level/emscripten-packages.nix`:
+
+* `pkgs.zlib.override`
+* `pkgs.buildEmscriptenPackage`
+
+Both are interesting concepts.
+
+A special requirement of the `pkgs.buildEmscriptenPackage` is the `doCheck = true` is a default meaning that each emscriptenPackage requires a `checkPhase` implemented.
+
+* Use `export EMCC_DEBUG=2` from within a emscriptenPackage's `phase` to get more detailed debug output what is going wrong.
+* ~/.emscripten cache is requiring us to set `HOME=$TMPDIR` in individual phases. This makes compilation slower but also makes it more deterministic.
+
+### Usage 1: pkgs.zlib.override
+
+This example uses `zlib` from nixpkgs but instead of compiling **C** to **ELF** it compiles **C** to **JS** since we were using `pkgs.zlib.override` and changed stdenv to `pkgs.emscriptenStdenv`. A few adaptions and hacks were set in place to make it working. One advantage is that when `pkgs.zlib` is updated, it will automatically update this package as well. However, this can also be the downside...
+
+See the `zlib` example:
+
+ zlib = (pkgs.zlib.override {
+ stdenv = pkgs.emscriptenStdenv;
+ }).overrideDerivation
+ (old: rec {
+ buildInputs = old.buildInputs ++ [ pkgconfig ];
+ # we need to reset this setting!
+ NIX_CFLAGS_COMPILE="";
+ configurePhase = ''
+ # FIXME: Some tests require writing at $HOME
+ HOME=$TMPDIR
+ runHook preConfigure
+
+ #export EMCC_DEBUG=2
+ emconfigure ./configure --prefix=$out --shared
+
+ runHook postConfigure
+ '';
+ dontStrip = true;
+ outputs = [ "out" ];
+ buildPhase = ''
+ emmake make
+ '';
+ installPhase = ''
+ emmake make install
+ '';
+ checkPhase = ''
+ echo "================= testing zlib using node ================="
+
+ echo "Compiling a custom test"
+ set -x
+ emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \
+ libz.so.${old.version} -I . -o example.js
+
+ echo "Using node to execute the test"
+ ${pkgs.nodejs}/bin/node ./example.js
+
+ set +x
+ if [ $? -ne 0 ]; then
+ echo "test failed for some reason"
+ exit 1;
+ else
+ echo "it seems to work! very good."
+ fi
+ echo "================= /testing zlib using node ================="
+ '';
+
+ postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin ''
+ substituteInPlace configure \
+ --replace '/usr/bin/libtool' 'ar' \
+ --replace 'AR="libtool"' 'AR="ar"' \
+ --replace 'ARFLAGS="-o"' 'ARFLAGS="-r"'
+ '';
+ });
+
+### Usage 2: pkgs.buildEmscriptenPackage
+
+This `xmlmirror` example features a emscriptenPackage which is defined completely from this context and no `pkgs.zlib.override` is used.
+
+ xmlmirror = pkgs.buildEmscriptenPackage rec {
+ name = "xmlmirror";
+
+ buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ];
+ nativeBuildInputs = [ pkgconfig zlib ];
+
+ src = pkgs.fetchgit {
+ url = "https://gitlab.com/odfplugfest/xmlmirror.git";
+ rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd";
+ sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b";
+ };
+
+ configurePhase = ''
+ rm -f fastXmlLint.js*
+ # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/8
+ sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv
+ # https://github.com/kripken/emscripten/issues/6344
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/9
+ sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/11
+ sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv
+ '';
+
+ buildPhase = ''
+ HOME=$TMPDIR
+ make -f Makefile.emEnv
+ '';
+
+ outputs = [ "out" "doc" ];
+
+ installPhase = ''
+ mkdir -p $out/share
+ mkdir -p $doc/share/${name}
+
+ cp Demo* $out/share
+ cp -R codemirror-5.12 $out/share
+ cp fastXmlLint.js* $out/share
+ cp *.xsd $out/share
+ cp *.js $out/share
+ cp *.xhtml $out/share
+ cp *.html $out/share
+ cp *.json $out/share
+ cp *.rng $out/share
+ cp README.md $doc/share/${name}
+ '';
+ checkPhase = ''
+
+ '';
+ };
+
+### Declarative debugging
+
+Use `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz` and from there you can go trough the individual steps. This makes it easy to build a good `unit test` or list the files of the project.
+
+1. `nix-shell -I nixpkgs=/some/dir/nixpkgs -A emscriptenPackages.libz`
+2. `cd /tmp/`
+3. `unpackPhase`
+4. cd libz-1.2.3
+5. `configurePhase`
+6. `buildPhase`
+7. ... happy hacking...
+
+## Summary
+
+Using this toolchain makes it easy to leverage `nix` from NixOS, MacOSX or even Windows (WSL+ubuntu+nix). This toolchain is reproducible, behaves like the rest of the packages from nixpkgs and contains a set of well working examples to learn and adapt from.
+
+If in trouble, ask the maintainers.
+
diff --git a/doc/languages-frameworks/index.xml b/doc/languages-frameworks/index.xml
index fc15d847d15f..6743c131201f 100644
--- a/doc/languages-frameworks/index.xml
+++ b/doc/languages-frameworks/index.xml
@@ -30,6 +30,7 @@ such as Perl or Haskell. These are described in this chapter.
+
diff --git a/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix b/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix
index f35ca26584f1..5317cf25d41b 100644
--- a/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix
+++ b/pkgs/development/compilers/emscripten-fastcomp/emscripten-fastcomp.nix
@@ -1,7 +1,7 @@
-{ stdenv, fetchFromGitHub, cmake, python, ... }:
+{ emscriptenVersion, stdenv, llvm, fetchFromGitHub, cmake, python, gtest, ... }:
let
- rev = "1.37.16";
+ rev = emscriptenVersion;
gcc = if stdenv.cc.isGNU then stdenv.cc.cc else stdenv.cc.cc.gcc;
in
stdenv.mkDerivation rec {
@@ -10,18 +10,18 @@ stdenv.mkDerivation rec {
src = fetchFromGitHub {
owner = "kripken";
repo = "emscripten-fastcomp";
- sha256 = "0wj9sc0gciaiidcjv6wb0qn6ks06xds7q34351masc7qpvd217by";
+ sha256 = "04j698gmp686b5lricjakm5hyh2z2kh28m1ffkghmkyz4zkzmx98";
inherit rev;
};
srcFL = fetchFromGitHub {
owner = "kripken";
repo = "emscripten-fastcomp-clang";
- sha256 = "1akdgxzxhzjbhp4d14ajcrp9jrf39x004a726ly2gynqc185l4j7";
+ sha256 = "1ici51mmpgg80xk3y8f376nbbfak6rz27qdy98l8lxkrymklp5g5";
inherit rev;
};
- nativeBuildInputs = [ cmake python ];
+ nativeBuildInputs = [ cmake python gtest ];
preConfigure = ''
cp -Lr ${srcFL} tools/clang
chmod +w -R tools/clang
@@ -30,9 +30,10 @@ stdenv.mkDerivation rec {
"-DCMAKE_BUILD_TYPE=Release"
"-DLLVM_TARGETS_TO_BUILD='X86;JSBackend'"
"-DLLVM_INCLUDE_EXAMPLES=OFF"
- "-DLLVM_INCLUDE_TESTS=OFF"
- # "-DCLANG_INCLUDE_EXAMPLES=OFF"
- "-DCLANG_INCLUDE_TESTS=OFF"
+ "-DLLVM_INCLUDE_TESTS=ON"
+ #"-DLLVM_CONFIG=${llvm}/bin/llvm-config"
+ "-DLLVM_BUILD_TESTS=ON"
+ "-DCLANG_INCLUDE_TESTS=ON"
] ++ (stdenv.lib.optional stdenv.isLinux
# necessary for clang to find crtend.o
"-DGCC_INSTALL_PREFIX=${gcc}"
diff --git a/pkgs/development/compilers/emscripten/default.nix b/pkgs/development/compilers/emscripten/default.nix
index ae80ad6fc808..ac972c99a959 100644
--- a/pkgs/development/compilers/emscripten/default.nix
+++ b/pkgs/development/compilers/emscripten/default.nix
@@ -1,9 +1,9 @@
-{ stdenv, fetchFromGitHub, emscriptenfastcomp, python, nodejs, closurecompiler
-, jre, binaryen, enableWasm ? true
+{ emscriptenVersion, stdenv, fetchFromGitHub, emscriptenfastcomp, python, nodejs, closurecompiler, pkgs
+, jre, binaryen, enableWasm ? true , python2Packages, cmake
}:
let
- rev = "1.37.16";
+ rev = emscriptenVersion;
appdir = "share/emscripten";
in
@@ -13,10 +13,12 @@ stdenv.mkDerivation {
src = fetchFromGitHub {
owner = "kripken";
repo = "emscripten";
- sha256 = "1qyhjx5zza01vnwmj6qzxbkagxknn4kzb6gw12fqw5q8pa8fy4zy";
+ sha256 = "02p0cp86vd1mydlpq544xbydggpnrq9dhbxx7h08j235frjm5cdc";
inherit rev;
};
+ buildInputs = [ nodejs cmake python ];
+
buildCommand = ''
mkdir -p $out/${appdir}
cp -r $src/* $out/${appdir}
@@ -38,9 +40,23 @@ stdenv.mkDerivation {
echo "COMPILER_ENGINE = NODE_JS" >> $out/${appdir}/config
echo "CLOSURE_COMPILER = '${closurecompiler}/share/java/closure-compiler-v${closurecompiler.version}.jar'" >> $out/${appdir}/config
echo "JAVA = '${jre}/bin/java'" >> $out/${appdir}/config
+ # to make the test(s) below work
+ echo "SPIDERMONKEY_ENGINE = []" >> $out/${appdir}/config
''
+ stdenv.lib.optionalString enableWasm ''
echo "BINARYEN_ROOT = '${binaryen}'" >> $out/share/emscripten/config
+ ''
+ +
+ ''
+ echo "--------------- running test -----------------"
+ # quick hack to get the test working
+ HOME=$TMPDIR
+ cp $out/${appdir}/config $HOME/.emscripten
+ export PATH=$PATH:$out/bin
+
+ #export EMCC_DEBUG=2
+ ${python}/bin/python $src/tests/runner.py test_hello_world
+ echo "--------------- /running test -----------------"
'';
meta = with stdenv.lib; {
diff --git a/pkgs/development/em-modules/generic/default.nix b/pkgs/development/em-modules/generic/default.nix
index 16b8f1df595e..332fab8e14aa 100644
--- a/pkgs/development/em-modules/generic/default.nix
+++ b/pkgs/development/em-modules/generic/default.nix
@@ -1,4 +1,4 @@
-{ pkgs, lib, emscripten }:
+{ pkgs, lib, emscripten, python }:
{ buildInputs ? [], nativeBuildInputs ? []
@@ -11,8 +11,8 @@ pkgs.stdenv.mkDerivation (
{
name = "emscripten-${args.name}";
- buildInputs = [ emscripten ] ++ buildInputs;
- nativeBuildInputs = [ emscripten ] ++ nativeBuildInputs;
+ buildInputs = [ emscripten python ] ++ buildInputs;
+ nativeBuildInputs = [ emscripten python ] ++ nativeBuildInputs;
# fake conftest results with emscripten's python magic
EMCONFIGURE_JS=2;
@@ -38,9 +38,18 @@ pkgs.stdenv.mkDerivation (
runHook postBuild
'';
+ doCheck = true;
+
checkPhase = args.checkPhase or ''
runHook preCheck
+ echo "Please provide a test for your emscripten based library/tool, see libxml2 as an exmple on how to use emcc/node to verify your build"
+ echo ""
+ echo "In normal C 'unresolved symbols' would yield an error and a breake of execution. In contrast, in emscripten they are only a warning which is ok given that emscripten assumptions about shared libraries."
+ echo " -> https://github.com/kripken/emscripten/wiki/Linking"
+ echo "So just assume the dependencies were built using hydra, then YOU WILL NEVER see the warning and your code depending on a library will always fail!"
+ exit 1
+
runHook postCheck
'';
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 0676d59d25a3..054febb62231 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2079,6 +2079,8 @@ with pkgs;
buildEmscriptenPackage = callPackage ../development/em-modules/generic { };
+ emscriptenVersion = "1.37.36";
+
emscripten = callPackage ../development/compilers/emscripten { };
emscriptenfastcompPackages = callPackage ../development/compilers/emscripten-fastcomp { };
diff --git a/pkgs/top-level/emscripten-packages.nix b/pkgs/top-level/emscripten-packages.nix
index 7e63041867de..5b5b32b89da5 100644
--- a/pkgs/top-level/emscripten-packages.nix
+++ b/pkgs/top-level/emscripten-packages.nix
@@ -1,49 +1,92 @@
{ pkgs }:
+with pkgs;
+
# emscripten toolchain abstraction for nix
# https://github.com/NixOS/nixpkgs/pull/16208
-with pkgs; rec {
- json_c = pkgs.json_c.override {
- stdenv = emscriptenStdenv;
- };
+rec {
+ json_c = (pkgs.json_c.override {
+ stdenv = pkgs.emscriptenStdenv;
+ }).overrideDerivation
+ (old: {
+ nativeBuildInputs = [ autoreconfHook pkgconfig ];
+ propagatedBuildInputs = [ zlib ];
+ buildInputs = old.buildInputs ++ [ automake autoconf ];
+ configurePhase = ''
+ HOME=$TMPDIR
+ emconfigure ./configure --prefix=$out
+ '';
+ checkPhase = ''
+ echo "================= testing json_c using node ================="
+
+ echo "Compiling a custom test"
+ set -x
+ emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 tests/test1.c \
+ `pkg-config zlib --cflags` \
+ `pkg-config zlib --libs` \
+ -I . \
+ .libs/libjson-c.so \
+ -o ./test1.js
+
+ echo "Using node to execute the test which basically outputs an error on stderr which we grep for"
+ ${pkgs.nodejs}/bin/node ./test1.js
+
+ set +x
+ if [ $? -ne 0 ]; then
+ echo "test1.js execution failed -> unit test failed, please fix"
+ exit 1;
+ else
+ echo "test1.js execution seems to work! very good."
+ fi
+ echo "================= /testing json_c using node ================="
+ '';
+ });
libxml2 = (pkgs.libxml2.override {
stdenv = emscriptenStdenv;
pythonSupport = false;
}).overrideDerivation
(old: {
- nativeBuildInputs = old.nativeBuildInputs ++ [ autoreconfHook pkgconfig ];
- buildInputs = old.buildInputs ++ [ zlib nodejs ];
+ propagatedBuildInputs = [ zlib ];
+ buildInputs = old.buildInputs ++ [ pkgconfig ];
+
# just override it with nothing so it does not fail
autoreconfPhase = "echo autoreconfPhase not used...";
+ configurePhase = ''
+ HOME=$TMPDIR
+ emconfigure ./configure --prefix=$out --without-python
+ '';
checkPhase = ''
- echo "================= testing xmllint using node ================="
+ echo "================= testing libxml2 using node ================="
+
+ echo "Compiling a custom test"
+ set -x
emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 xmllint.o \
./.libs/libxml2.a `pkg-config zlib --cflags` `pkg-config zlib --libs` -o ./xmllint.test.js \
--embed-file ./test/xmlid/id_err1.xml
- # test/xmlid/id_err2.xml:3: validity error : xml:id : attribute type should be ID
- #
- # ^
- node ./xmllint.test.js --noout test/xmlid/id_err1.xml 2>&1 | grep 0bar
+
+ echo "Using node to execute the test which basically outputs an error on stderr which we grep for"
+ ${pkgs.nodejs}/bin/node ./xmllint.test.js --noout test/xmlid/id_err1.xml 2>&1 | grep 0bar
+
+ set +x
if [ $? -ne 0 ]; then
echo "xmllint unit test failed, please fix this package"
exit 1;
else
echo "since there is no stupid text containing 'foo xml:id' it seems to work! very good."
fi
- echo "================= /testing xmllint using node ================="
+ echo "================= /testing libxml2 using node ================="
'';
});
- xmlmirror = buildEmscriptenPackage rec {
+ xmlmirror = pkgs.buildEmscriptenPackage rec {
name = "xmlmirror";
- nativeBuildInputs = [ pkgconfig ];
- buildInputs = [ autoconf automake libtool gnumake libxml2 nodejs
- python openjdk json_c zlib ];
+ buildInputs = [ pkgconfig autoconf automake libtool gnumake libxml2 nodejs openjdk json_c ];
+ nativeBuildInputs = [ pkgconfig zlib ];
- src = fetchgit {
+ src = pkgs.fetchgit {
url = "https://gitlab.com/odfplugfest/xmlmirror.git";
rev = "4fd7e86f7c9526b8f4c1733e5c8b45175860a8fd";
sha256 = "1jasdqnbdnb83wbcnyrp32f36w3xwhwp0wq8lwwmhqagxrij1r4b";
@@ -51,6 +94,14 @@ with pkgs; rec {
configurePhase = ''
rm -f fastXmlLint.js*
+ # a fix for ERROR:root:For asm.js, TOTAL_MEMORY must be a multiple of 16MB, was 234217728
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/8
+ sed -e "s/TOTAL_MEMORY=234217728/TOTAL_MEMORY=268435456/g" -i Makefile.emEnv
+ # https://github.com/kripken/emscripten/issues/6344
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/9
+ sed -e "s/\$(JSONC_LDFLAGS) \$(ZLIB_LDFLAGS) \$(LIBXML20_LDFLAGS)/\$(JSONC_LDFLAGS) \$(LIBXML20_LDFLAGS) \$(ZLIB_LDFLAGS) /g" -i Makefile.emEnv
+ # https://gitlab.com/odfplugfest/xmlmirror/issues/11
+ sed -e "s/-o fastXmlLint.js/-s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -o fastXmlLint.js/g" -i Makefile.emEnv
'';
buildPhase = ''
@@ -75,25 +126,58 @@ with pkgs; rec {
cp *.rng $out/share
cp README.md $doc/share/${name}
'';
-
- postInstall = ''
+ checkPhase = ''
+
'';
};
zlib = (pkgs.zlib.override {
- stdenv = emscriptenStdenv;
+ stdenv = pkgs.emscriptenStdenv;
}).overrideDerivation
- (old: {
+ (old: rec {
+ buildInputs = old.buildInputs ++ [ pkgconfig ];
+ # we need to reset this setting!
+ NIX_CFLAGS_COMPILE="";
configurePhase = ''
# FIXME: Some tests require writing at $HOME
HOME=$TMPDIR
runHook preConfigure
- emconfigure ./configure --prefix=$out
+ #export EMCC_DEBUG=2
+ emconfigure ./configure --prefix=$out --shared
runHook postConfigure
'';
- postPatch = stdenv.lib.optionalString stdenv.isDarwin ''
+ dontStrip = true;
+ outputs = [ "out" ];
+ buildPhase = ''
+ emmake make
+ '';
+ installPhase = ''
+ emmake make install
+ '';
+ checkPhase = ''
+ echo "================= testing zlib using node ================="
+
+ echo "Compiling a custom test"
+ set -x
+ emcc -O2 -s EMULATE_FUNCTION_POINTER_CASTS=1 test/example.c -DZ_SOLO \
+ libz.so.${old.version} -I . -o example.js
+
+ echo "Using node to execute the test"
+ ${pkgs.nodejs}/bin/node ./example.js
+
+ set +x
+ if [ $? -ne 0 ]; then
+ echo "test failed for some reason"
+ exit 1;
+ else
+ echo "it seems to work! very good."
+ fi
+ echo "================= /testing zlib using node ================="
+ '';
+
+ postPatch = pkgs.stdenv.lib.optionalString pkgs.stdenv.isDarwin ''
substituteInPlace configure \
--replace '/usr/bin/libtool' 'ar' \
--replace 'AR="libtool"' 'AR="ar"' \