stdenv: Fix handling of dependencies and hooks

4 far-reaching changes: Smaller PATH, New vars, different propagation
logic, and different hook logic

Smaller PATH
------------

`buildInputs` no longer go on the PATH at build time, as they cannot be
run when cross compiling and we don't want to special case. Simply make
a `nativeBuildInput` too if one needs them on the PATH. Fixes #21191.

Many new depedendency variables
-------------------------------

See the stdenv chapter of the nixpkgs manual. I pulled out the existing
documentation of dependency specification into a new section, and added
language for these two (and their propagated equivalents) along side
the others'.

More complex propagation logic
------------------------------

Before a propagated*XXX*Input always acted as if it was specified
directly as a *XXX*Input downstream. That's simple enough, but violates
the intended roles of each sort of dep, which has functional and not
just stylistic consequences.

The new algorithm is detailed in the manual, and ensures everything
ends up in the right place. I tried to give both an informal and formal
description, but I suspect in practice it will not make much sense
until one tries cross compiling, after which it will immediately make
sense as the only sane option.

Simplified hook logic
---------------------

Rather than `envHook` and `crossEnvHook`, whose behavior differs
depending on whether we are cross compiling or not, there is now one
hook per sort (or rather non-propagated and propagated pair of sorts)
of dependency. These new hooks have the same meaning regardless of
cross compilation. See the setup hook section of stdenv chapter of the
Nixpkgs manual for more details.
This commit is contained in:
John Ericson 2017-09-11 12:20:20 -04:00
parent bb18a3b573
commit 7f3ca3e21a
2 changed files with 291 additions and 91 deletions

View file

@ -14,11 +14,27 @@ rec {
mkDerivation = mkDerivation =
{ name ? "" { name ? ""
, nativeBuildInputs ? [] # These types of dependencies are all exhaustively documented in
, buildInputs ? [] # the "Specifying Dependencies" section of the "Standard
# Environment" chapter of the Nixpkgs manual.
, propagatedNativeBuildInputs ? [] # TODO(@Ericson2314): Stop using legacy dep attribute names
, propagatedBuildInputs ? []
# host offset -> target offset
, depsBuildBuild ? [] # -1 -> -1
, depsBuildBuildPropagated ? [] # -1 -> -1
, nativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, propagatedNativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, depsBuildTarget ? [] # -1 -> 1
, depsBuildTargetPropagated ? [] # -1 -> 1
, depsHostHost ? [] # 0 -> 0
, depsHostHostPropagated ? [] # 0 -> 0
, buildInputs ? [] # 0 -> 1 N.B. Legacy name
, propagatedBuildInputs ? [] # 0 -> 1 N.B. Legacy name
, depsTargetTarget ? [] # 1 -> 1
, depsTargetTargetPropagated ? [] # 1 -> 1
, configureFlags ? [] , configureFlags ? []
, # Target is not included by default because most programs don't care. , # Target is not included by default because most programs don't care.
@ -56,15 +72,35 @@ rec {
inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags; inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
}) })
else let else let
dependencies = map lib.chooseDevOutputs [ dependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
(map (drv: drv.nativeDrv or drv) nativeBuildInputs (map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh ++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh) ++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHost)
(map (drv: drv.crossDrv or drv) buildInputs) (map (drv: drv.crossDrv or drv) buildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
]
]; ];
propagatedDependencies = map lib.chooseDevOutputs [ propagatedDependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs) (map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs) (map (drv: drv.crossDrv or drv) propagatedBuildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
]
]; ];
outputs' = outputs' =
@ -105,11 +141,19 @@ rec {
userHook = config.stdenv.userHook or null; userHook = config.stdenv.userHook or null;
__ignoreNulls = true; __ignoreNulls = true;
nativeBuildInputs = lib.elemAt dependencies 0; depsBuildBuild = lib.elemAt (lib.elemAt dependencies 0) 0;
buildInputs = lib.elemAt dependencies 1; nativeBuildInputs = lib.elemAt (lib.elemAt dependencies 0) 1;
depsBuildTarget = lib.elemAt (lib.elemAt dependencies 0) 2;
depsHostBuild = lib.elemAt (lib.elemAt dependencies 1) 0;
buildInputs = lib.elemAt (lib.elemAt dependencies 1) 1;
depsTargetTarget = lib.elemAt (lib.elemAt dependencies 2) 0;
propagatedNativeBuildInputs = lib.elemAt propagatedDependencies 0; depsBuildBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;
propagatedBuildInputs = lib.elemAt propagatedDependencies 1; propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;
depsBuildTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;
depsHostBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
propagatedBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;
depsTargetTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;
# This parameter is sometimes a string, sometimes null, and sometimes a list, yuck # This parameter is sometimes a string, sometimes null, and sometimes a list, yuck
configureFlags = let inherit (lib) optional elem; in configureFlags = let inherit (lib) optional elem; in

View file

@ -300,11 +300,72 @@ runHook preHook
runHook addInputsHook runHook addInputsHook
# Recursively find all build inputs. # Package accumulators
# shellcheck disable=SC2034
declare -a pkgsBuildBuild pkgsBuildHost pkgsBuildTarget
declare -a pkgsHostHost pkgsHostTarget
declare -a pkgsTargetTarget
declare -ra pkgBuildAccumVars=(pkgsBuildBuild pkgsBuildHost pkgsBuildTarget)
declare -ra pkgHostAccumVars=(pkgsHostHost pkgsHostTarget)
declare -ra pkgTargetAccumVars=(pkgsTargetTarget)
declare -ra pkgAccumVarVars=(pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars)
# Hooks
declare -a envBuildBuildHooks envBuildHostHooks envBuildTargetHooks
declare -a envHostHostHooks envHostTargetHooks
declare -a envTargetTargetHooks
declare -ra pkgBuildHookVars=(envBuildBuildHook envBuildHostHook envBuildTargetHook)
declare -ra pkgHostHookVars=(envHostHostHook envHostTargetHook)
declare -ra pkgTargetHookVars=(envTargetTargetHook)
declare -ra pkgHookVarVars=(pkgBuildHookVars pkgHostHookVars pkgTargetHookVars)
# Propagated dep files
declare -ra propagatedBuildDepFiles=(
propagated-build-build-deps
propagated-native-build-inputs # Legacy name for back-compat
propagated-build-target-deps
)
declare -ra propagatedHostDepFiles=(
propagated-host-host-deps
propagated-build-inputs # Legacy name for back-compat
)
declare -ra propagatedTargetDepFiles=(
propagated-target-target-deps
)
declare -ra propagatedDepFilesVars=(
propagatedBuildDepFiles
propagatedHostDepFiles
propagatedTargetDepFiles
)
# Platform offsets: build = -1, host = 0, target = 1
declare -ra allPlatOffsets=(-1 0 1)
# Mutually-recursively find all build inputs. See the dependency section of the
# stdenv chapter of the Nixpkgs manual for the specification this algorithm
# implements.
findInputs() { findInputs() {
local pkg="$1"; shift local -r pkg="$1"
local var="$1"; shift local -ri hostOffset="$2"
local propagatedBuildInputsFiles=("$@") local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
local varVar="${pkgAccumVarVars[$hostOffset + 1]}"
local varRef="$varVar[\$targetOffset - \$hostOffset]"
local var="${!varRef}"
unset -v varVar varRef
# TODO(@Ericson2314): Restore using associative array once Darwin # TODO(@Ericson2314): Restore using associative array once Darwin
# nix-shell doesn't use impure bash. This should replace the O(n) # nix-shell doesn't use impure bash. This should replace the O(n)
@ -324,21 +385,106 @@ findInputs() {
exit 1 exit 1
fi fi
local file # The current package's host and target offset together
for file in "${propagatedBuildInputsFiles[@]}"; do # provide a <=-preserving homomorphism from the relative
file="$pkg/nix-support/$file" # offsets to current offset
[[ -f "$file" ]] || continue function mapOffset() {
local -ri inputOffset="$1"
if (( "$inputOffset" <= 0 )); then
local -ri outputOffset="$inputOffset + $hostOffset"
else
local -ri outputOffset="$inputOffset - 1 + $targetOffset"
fi
echo "$outputOffset"
}
# Host offset relative to that of the package whose immediate
# dependencies we are currently exploring.
local -i relHostOffset
for relHostOffset in "${allPlatOffsets[@]}"; do
# `+ 1` so we start at 0 for valid index
local files="${propagatedDepFilesVars[$relHostOffset + 1]}"
# Host offset relative to the package currently being
# built---as absolute an offset as will be used.
local -i hostOffsetNext
hostOffsetNext="$(mapOffset relHostOffset)"
# Ensure we're in bounds relative to the package currently
# being built.
[[ "${allPlatOffsets[*]}" = *"$hostOffsetNext"* ]] || continue
# Target offset relative to the *host* offset of the package
# whose immediate dependencies we are currently exploring.
local -i relTargetOffset
for relTargetOffset in "${allPlatOffsets[@]}"; do
(( "$relHostOffset" <= "$relTargetOffset" )) || continue
local fileRef="${files}[$relTargetOffset - $relHostOffset]"
local file="${!fileRef}"
unset -v fileRef
# Target offset relative to the package currently being
# built.
local -i targetOffsetNext
targetOffsetNext="$(mapOffset relTargetOffset)"
# Once again, ensure we're in bounds relative to the
# package currently being built.
[[ "${allPlatOffsets[*]}" = *"$targetOffsetNext"* ]] || continue
[[ -f "$pkg/nix-support/$file" ]] || continue
local pkgNext local pkgNext
for pkgNext in $(< "$file"); do for pkgNext in $(< "$pkg/nix-support/$file"); do
findInputs "$pkgNext" "$var" "${propagatedBuildInputsFiles[@]}" findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"
done
done done
done done
} }
# Make sure all are at least defined as empty
: ${depsBuildBuild=} ${depsBuildBuildPropagated=}
: ${nativeBuildInputs=} ${propagatedNativeBuildInputs=} ${defaultNativeBuildInputs=}
: ${depsBuildTarget=} ${depsBuildTargetPropagated=}
: ${depsHostHost=} ${depsHostHostPropagated=}
: ${buildInputs=} ${propagatedBuildInputs=} ${defaultBuildInputs=}
: ${depsTargetTarget=} ${depsTargetTargetPropagated=}
for pkg in $depsBuildBuild $depsBuildBuildPropagated; do
findInputs "$pkg" -1 -1
done
for pkg in $nativeBuildInputs $propagatedNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $depsBuildTarget $depsBuildTargetPropagated; do
findInputs "$pkg" -1 1
done
for pkg in $depsHostHost $depsHostHostPropagated; do
findInputs "$pkg" 0 0
done
for pkg in $buildInputs $propagatedBuildInputs ; do
findInputs "$pkg" 0 1
done
for pkg in $depsTargetTarget $depsTargetTargetPropagated; do
findInputs "$pkg" 1 1
done
# Default inputs must be processed last
for pkg in $defaultNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $defaultBuildInputs; do
findInputs "$pkg" 0 1
done
# Add package to the future PATH and run setup hooks # Add package to the future PATH and run setup hooks
activatePackage() { activatePackage() {
local pkg="$1" local pkg="$1"
local -ri hostOffset="$2"
local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
if [ -f "$pkg" ]; then if [ -f "$pkg" ]; then
local oldOpts="$(shopt -po nounset)" local oldOpts="$(shopt -po nounset)"
@ -347,11 +493,16 @@ activatePackage() {
eval "$oldOpts" eval "$oldOpts"
fi fi
if [ -d "$pkg/bin" ]; then # Only dependencies whose host platform is guaranteed to match the
# build platform are included here. That would be `depsBuild*`,
# and legacy `nativeBuildInputs`. Other aren't because of cross
# compiling, and we want to have consistent rules whether or not
# we are cross compiling.
if [[ "$hostOffset" -le -1 && -d "$pkg/bin" ]]; then
addToSearchPath _PATH "$pkg/bin" addToSearchPath _PATH "$pkg/bin"
fi fi
if [ -f "$pkg/nix-support/setup-hook" ]; then if [[ -f "$pkg/nix-support/setup-hook" ]]; then
local oldOpts="$(shopt -po nounset)" local oldOpts="$(shopt -po nounset)"
set +u set +u
source "$pkg/nix-support/setup-hook" source "$pkg/nix-support/setup-hook"
@ -359,67 +510,55 @@ activatePackage() {
fi fi
} }
declare -a nativePkgs crossPkgs _activatePkgs() {
if [ -z "${crossConfig:-}" ]; then local -i hostOffset targetOffset
# Not cross-compiling - both buildInputs (and variants like propagatedBuildInputs) local pkg
# are handled identically to nativeBuildInputs
for i in ${nativeBuildInputs:-} ${buildInputs:-} \
${defaultNativeBuildInputs:-} ${defaultBuildInputs:-} \
${propagatedNativeBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs propagated-build-inputs
done
else
for i in ${nativeBuildInputs:-} ${defaultNativeBuildInputs:-} ${propagatedNativeBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs
done
for i in ${buildInputs:-} ${defaultBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" crossPkgs propagated-build-inputs
done
fi
for i in ${nativePkgs+"${nativePkgs[@]}"} ${crossPkgs+"${crossPkgs[@]}"}; do for hostOffset in "${allPlatOffsets[@]}"; do
activatePackage "$i" local pkgsVar="${pkgAccumVarVars[$hostOffset + 1]}"
done for targetOffset in "${allPlatOffsets[@]}"; do
(( "$hostOffset" <= "$targetOffset" )) || continue
local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
activatePackage "$pkg" "$hostOffset" "$targetOffset"
done
done
done
}
# Run the package setup hooks and build _PATH
_activatePkgs
# Set the relevant environment variables to point to the build inputs # Set the relevant environment variables to point to the build inputs
# found above. # found above.
# #
# These `depOffset`s tell the env hook what sort of dependency # These `depOffset`s, beyond indexing the arrays, also tell the env
# (ignoring propagatedness) is being passed to the env hook. In a real # hook what sort of dependency (ignoring propagatedness) is being
# language, we'd append a closure with this information to the # passed to the env hook. In a real language, we'd append a closure
# relevant env hook array, but bash doesn't have closures, so it's # with this information to the relevant env hook array, but bash
# easier to just pass this in. # doesn't have closures, so it's easier to just pass this in.
_addToEnv() {
local -i depHostOffset depTargetOffset
local pkg
_addToNativeEnv() { for depHostOffset in "${allPlatOffsets[@]}"; do
local pkg="$1" local hookVar="${pkgHookVarVars[$depHostOffset + 1]}"
if [[ -n "${crossConfig:-}" ]]; then local pkgsVar="${pkgAccumVarVars[$depHostOffset + 1]}"
local -i depOffset=-1 for depTargetOffset in "${allPlatOffsets[@]}"; do
else (( "$depHostOffset" <= "$depTargetOffset" )) || continue
local -i depOffset=0 local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"
fi local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
# Run the package-specific hooks set by the setup-hook scripts. for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
runHook envHook "$pkg" runHook "${!hookRef}" "$pkg"
done
done
done
} }
# Old bash empty array hack # Run the package-specific hooks set by the setup-hook scripts.
for i in ${nativePkgs+"${nativePkgs[@]}"}; do _addToEnv
_addToNativeEnv "$i"
done
_addToCrossEnv() {
local pkg="$1"
local -i depOffset=0
# Run the package-specific hooks set by the setup-hook scripts.
runHook crossEnvHook "$pkg"
}
# Old bash empty array hack
for i in ${crossPkgs+"${crossPkgs[@]}"}; do
_addToCrossEnv "$i"
done
_addRpathPrefix "$out" _addRpathPrefix "$out"
@ -882,6 +1021,7 @@ installPhase() {
# propagated-build-inputs. # propagated-build-inputs.
fixupPhase() { fixupPhase() {
# Make sure everything is writable so "strip" et al. work. # Make sure everything is writable so "strip" et al. work.
local output
for output in $outputs; do for output in $outputs; do
if [ -e "${!output}" ]; then chmod -R u+w "${!output}"; fi if [ -e "${!output}" ]; then chmod -R u+w "${!output}"; fi
done done
@ -895,19 +1035,35 @@ fixupPhase() {
done done
# Propagate build inputs and setup hook into the development output. # Propagate dependencies & setup hook into the development output.
declare -ra flatVars=(
# Build
depsBuildBuildPropagated
propagatedNativeBuildInputs
depsBuildTargetPropagated
# Host
depsHostHostPropagated
propagatedBuildInputs
# Target
depsTargetTargetPropagated
)
declare -ra flatFiles=(
"${propagatedBuildDepFiles[@]}"
"${propagatedHostDepFiles[@]}"
"${propagatedTargetDepFiles[@]}"
)
local propagatedInputsIndex
for propagatedInputsIndex in "${!flatVars[@]}"; do
local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"
local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"
[[ "${!propagatedInputsSlice}" ]] || continue
if [ -n "${propagatedBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support" mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
printWords $propagatedBuildInputs > "${!outputDev}/nix-support/propagated-build-inputs" printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"
fi done
if [ -n "${propagatedNativeBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086
printWords $propagatedNativeBuildInputs > "${!outputDev}/nix-support/propagated-native-build-inputs"
fi
if [ -n "${setupHook:-}" ]; then if [ -n "${setupHook:-}" ]; then