Merge pull request #289231 from adisbladis/fetchnpmlock

importNpmLock: init
This commit is contained in:
adisbladis 2024-03-08 15:14:37 +13:00 committed by GitHub
commit 434df3c94b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 316 additions and 3 deletions

View file

@ -233,6 +233,37 @@ sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
It returns a derivation with all `package-lock.json` dependencies downloaded into `$out/`, usable as an npm cache.
#### importNpmLock {#javascript-buildNpmPackage-importNpmLock}
`importNpmLock` is a Nix function that requires the following optional arguments:
- `npmRoot`: Path to package directory containing the source tree
- `package`: Parsed contents of `package.json`
- `packageLock`: Parsed contents of `package-lock.json`
- `pname`: Package name
- `version`: Package version
It returns a derivation with a patched `package.json` & `package-lock.json` with all dependencies resolved to Nix store paths.
This function is analogous to using `fetchNpmDeps`, but instead of specifying `hash` it uses metadata from `package.json` & `package-lock.json`.
Note that `npmHooks.npmConfigHook` cannot be used with `importNpmLock`. You will instead need to use `importNpmLock.npmConfigHook`:
```nix
{ buildNpmPackage, importNpmLock }:
buildNpmPackage {
pname = "hello";
version = "0.1.0";
npmDeps = importNpmLock {
npmRoot = ./.;
};
npmConfigHook = importNpmLock.npmConfigHook;
}
```
### corepack {#javascript-corepack}
This package puts the corepack wrappers for pnpm and yarn in your PATH, and they will honor the `packageManager` setting in the `package.json`.

View file

@ -49,6 +49,12 @@
name = "${name}-npm-deps";
hash = npmDepsHash;
}
# Custom npmConfigHook
, npmConfigHook ? null
# Custom npmBuildHook
, npmBuildHook ? null
# Custom npmInstallHook
, npmInstallHook ? null
, ...
} @ args:
@ -57,14 +63,19 @@ let
npmHooks = buildPackages.npmHooks.override {
inherit nodejs;
};
inherit (npmHooks) npmConfigHook npmBuildHook npmInstallHook;
in
stdenv.mkDerivation (args // {
inherit npmDeps npmBuildScript;
nativeBuildInputs = nativeBuildInputs
++ [ nodejs npmConfigHook npmBuildHook npmInstallHook nodejs.python ]
++ [
nodejs
# Prefer passed hooks
(if npmConfigHook != null then npmConfigHook else npmHooks.npmConfigHook)
(if npmBuildHook != null then npmBuildHook else npmHooks.npmBuildHook)
(if npmInstallHook != null then npmInstallHook else npmHooks.npmInstallHook)
nodejs.python
]
++ lib.optionals stdenv.isDarwin [ darwin.cctools ];
buildInputs = buildInputs ++ [ nodejs ];

View file

@ -0,0 +1,134 @@
{ lib
, fetchurl
, stdenv
, callPackages
, runCommand
}:
let
inherit (builtins) match elemAt toJSON removeAttrs;
inherit (lib) importJSON mapAttrs;
matchGitHubReference = match "github(.com)?:.+";
getName = package: package.name or "unknown";
getVersion = package: package.version or "0.0.0";
# Fetch a module from package-lock.json -> packages
fetchModule =
{ module
, npmRoot ? null
}: (
if module ? "resolved" then
(
let
# Parse scheme from URL
mUrl = match "(.+)://(.+)" module.resolved;
scheme = elemAt mUrl 0;
in
(
if mUrl == null then
(
assert npmRoot != null; {
outPath = npmRoot + "/${module.resolved}";
}
)
else if (scheme == "http" || scheme == "https") then
(
fetchurl {
url = module.resolved;
hash = module.integrity;
}
)
else if lib.hasPrefix "git" module.resolved then
(
builtins.fetchGit {
url = module.resolved;
}
)
else throw "Unsupported URL scheme: ${scheme}"
)
)
else null
);
# Manage node_modules outside of the store with hooks
hooks = callPackages ./hooks { };
in
{
importNpmLock =
{ npmRoot ? null
, package ? importJSON (npmRoot + "/package.json")
, packageLock ? importJSON (npmRoot + "/package-lock.json")
, pname ? getName package
, version ? getVersion package
}:
let
mapLockDependencies =
mapAttrs
(name: version: (
# Substitute the constraint with the version of the dependency from the top-level of package-lock.
if (
# if the version is `latest`
version == "latest"
||
# Or if it's a github reference
matchGitHubReference version != null
) then packageLock'.packages.${"node_modules/${name}"}.version
# But not a regular version constraint
else version
));
packageLock' = packageLock // {
packages =
mapAttrs
(_: module:
let
src = fetchModule {
inherit module npmRoot;
};
in
(removeAttrs module [
"link"
"funding"
]) // lib.optionalAttrs (src != null) {
resolved = "file:${src}";
} // lib.optionalAttrs (module ? dependencies) {
dependencies = mapLockDependencies module.dependencies;
} // lib.optionalAttrs (module ? optionalDependencies) {
optionalDependencies = mapLockDependencies module.optionalDependencies;
})
packageLock.packages;
};
mapPackageDependencies = mapAttrs (name: _: packageLock'.packages.${"node_modules/${name}"}.resolved);
# Substitute dependency references in package.json with Nix store paths
packageJSON' = package // lib.optionalAttrs (package ? dependencies) {
dependencies = mapPackageDependencies package.dependencies;
} // lib.optionalAttrs (package ? devDependencies) {
devDependencies = mapPackageDependencies package.devDependencies;
};
pname = package.name or "unknown";
in
runCommand "${pname}-${version}-sources"
{
inherit pname version;
passAsFile = [ "package" "packageLock" ];
package = toJSON packageJSON';
packageLock = toJSON packageLock';
} ''
mkdir $out
cp "$packagePath" $out/package.json
cp "$packageLockPath" $out/package-lock.json
'';
inherit hooks;
inherit (hooks) npmConfigHook;
__functor = self: self.importNpmLock;
}

View file

@ -0,0 +1,52 @@
#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
// When installing files rewritten to the Nix store with npm
// npm writes the symlinks relative to the build directory.
//
// This makes relocating node_modules tricky when refering to the store.
// This script walks node_modules and canonicalizes symlinks.
async function canonicalize(storePrefix, root) {
console.log(storePrefix, root)
const entries = await fs.promises.readdir(root);
const paths = entries.map((entry) => path.join(root, entry));
const stats = await Promise.all(
paths.map(async (path) => {
return {
path: path,
stat: await fs.promises.lstat(path),
};
})
);
const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink());
const dirs = stats.filter((stat) => stat.stat.isDirectory());
// Canonicalize symlinks to their real path
await Promise.all(
symlinks.map(async (stat) => {
const target = await fs.promises.realpath(stat.path);
if (target.startsWith(storePrefix)) {
await fs.promises.unlink(stat.path);
await fs.promises.symlink(target, stat.path);
}
})
);
// Recurse into directories
await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path)));
}
async function main() {
const args = process.argv.slice(2);
const storePrefix = args[0];
if (fs.existsSync("node_modules")) {
await canonicalize(storePrefix, "node_modules");
}
}
main();

View file

@ -0,0 +1,13 @@
{ callPackage, lib, makeSetupHook, srcOnly, nodejs }:
{
npmConfigHook = makeSetupHook
{
name = "npm-config-hook";
substitutions = {
nodeSrc = srcOnly nodejs;
nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
canonicalizeSymlinksScript = ./canonicalize-symlinks.js;
storePrefix = builtins.storeDir;
};
} ./npm-config-hook.sh;
}

View file

@ -0,0 +1,70 @@
# shellcheck shell=bash
npmConfigHook() {
echo "Executing npmConfigHook"
if [ -n "${npmRoot-}" ]; then
pushd "$npmRoot"
fi
if [ -z "${npmDeps-}" ]; then
echo "Error: 'npmDeps' should be set when using npmConfigHook."
exit 1
fi
echo "Configuring npm"
export HOME="$TMPDIR"
export npm_config_nodedir="@nodeSrc@"
export npm_config_node_gyp="@nodeGyp@"
npm config set offline true
npm config set progress false
npm config set fund false
echo "Installing patched package.json/package-lock.json"
# Save original package.json/package-lock.json for closure size reductions.
# The patched one contains store paths we don't want at runtime.
mv package.json .package.json.orig
if test -f package-lock.json; then # Not all packages have package-lock.json.
mv package-lock.json .package-lock.json.orig
fi
cp --no-preserve=mode "${npmDeps}/package.json" package.json
cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json
echo "Installing dependencies"
if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
echo
echo "ERROR: npm failed to install dependencies"
echo
echo "Here are a few things you can try, depending on the error:"
echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`'
echo
exit 1
fi
patchShebangs node_modules
npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
patchShebangs node_modules
# Canonicalize symlinks from relative paths to the Nix store.
node @canonicalizeSymlinksScript@ @storePrefix@
# Revert to pre-patched package.json/package-lock.json for closure size reductions
mv .package.json.orig package.json
if test -f ".package-lock.json.orig"; then
mv .package-lock.json.orig package-lock.json
fi
if [ -n "${npmRoot-}" ]; then
popd
fi
echo "Finished npmConfigHook"
}
postConfigureHooks+=(npmConfigHook)

View file

@ -10262,6 +10262,8 @@ with pkgs;
inherit (callPackages ../build-support/node/fetch-npm-deps { })
fetchNpmDeps prefetch-npm-deps;
importNpmLock = callPackages ../build-support/node/import-npm-lock { };
nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs // { __attrsFailEvaluation = true; };
nodePackages = dontRecurseIntoAttrs nodejs.pkgs // { __attrsFailEvaluation = true; };