mirror of
https://github.com/SebastianWendel/nixpkgs.git
synced 2024-09-20 04:19:00 +02:00
ollama: refactor
This commit is contained in:
abysssol
parent
c75dd8bff7
commit
70e834c19b
|
|
@ -17,19 +17,47 @@
|
|||
, linuxPackages
|
||||
, darwin
|
||||
|
||||
, enableRocm ? false
|
||||
, enableCuda ? false
|
||||
# one of `[ null "rocm" "cuda" ]`
|
||||
, acceleration ? null
|
||||
}:
|
||||
|
||||
let
|
||||
pname = "ollama";
|
||||
version = "0.1.26";
|
||||
|
||||
warnIfNotLinux = warning: (lib.warnIfNot stdenv.isLinux warning stdenv.isLinux);
|
||||
gpuWarning = api: "building ollama with ${api} is only supported on linux; falling back to cpu";
|
||||
rocmIsEnabled = enableRocm && (warnIfNotLinux (gpuWarning "rocm"));
|
||||
cudaIsEnabled = enableCuda && (warnIfNotLinux (gpuWarning "cuda"));
|
||||
enableLinuxGpu = rocmIsEnabled || cudaIsEnabled;
|
||||
validAccel = lib.assertOneOf "ollama.acceleration" acceleration [ null "rocm" "cuda" ];
|
||||
|
||||
warnIfNotLinux = api: (lib.warnIfNot stdenv.isLinux
|
||||
"building ollama with `${api}` is only supported on linux; falling back to cpu"
|
||||
stdenv.isLinux);
|
||||
enableRocm = validAccel && (acceleration == "rocm") && (warnIfNotLinux "rocm");
|
||||
enableCuda = validAccel && (acceleration == "cuda") && (warnIfNotLinux "cuda");
|
||||
|
||||
rocmClang = linkFarm "rocm-clang" {
|
||||
llvm = rocmPackages.llvm.clang;
|
||||
};
|
||||
rocmPath = buildEnv {
|
||||
name = "rocm-path";
|
||||
paths = [
|
||||
rocmPackages.rocm-device-libs
|
||||
rocmClang
|
||||
];
|
||||
};
|
||||
|
||||
cudaToolkit = buildEnv {
|
||||
name = "cuda-toolkit";
|
||||
ignoreCollisions = true; # FIXME: find a cleaner way to do this without ignoring collisions
|
||||
paths = [
|
||||
cudaPackages.cudatoolkit
|
||||
cudaPackages.cuda_cudart
|
||||
];
|
||||
};
|
||||
|
||||
runtimeLibs = lib.optionals enableRocm [
|
||||
rocmPackages.rocm-smi
|
||||
] ++ lib.optionals enableCuda [
|
||||
linuxPackages.nvidia_x11
|
||||
];
|
||||
|
||||
appleFrameworks = darwin.apple_sdk_11_0.frameworks;
|
||||
metalFrameworks = [
|
||||
|
|
@ -39,6 +67,13 @@ let
|
|||
appleFrameworks.MetalPerformanceShaders
|
||||
];
|
||||
|
||||
|
||||
goBuild =
|
||||
if enableCuda then
|
||||
buildGoModule.override { stdenv = overrideCC stdenv gcc12; }
|
||||
else
|
||||
buildGoModule;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "jmorganca";
|
||||
repo = "ollama";
|
||||
|
|
@ -53,130 +88,87 @@ let
|
|||
extraPrefix = "llm/llama.cpp/";
|
||||
};
|
||||
inherit (lib) licenses platforms maintainers;
|
||||
ollama = {
|
||||
inherit pname version src;
|
||||
vendorHash = "sha256-zTrBighPBqZ9hhkEV3UawJZUYyPRay7+P6wkhDtpY7M=";
|
||||
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
] ++ lib.optionals enableLinuxGpu [
|
||||
makeWrapper
|
||||
] ++ lib.optionals stdenv.isDarwin
|
||||
metalFrameworks;
|
||||
|
||||
patches = [
|
||||
# remove uses of `git` in the `go generate` script
|
||||
# instead use `patch` where necessary
|
||||
./remove-git.patch
|
||||
# replace a hardcoded use of `g++` with `$CXX`
|
||||
./replace-gcc.patch
|
||||
|
||||
# ollama's patches of llama.cpp's example server
|
||||
# `ollama/llm/generate/gen_common.sh` -> "apply temporary patches until fix is upstream"
|
||||
(preparePatch "01-cache.diff" "sha256-MTTln2G0G8dntihUzEjPM1ruTsApb4ZToBczJb8EG68=")
|
||||
(preparePatch "02-cudaleaks.diff" "sha256-Cu7E9iEcvddPL9mPPI5Z96qmwWigi3f0WgSpPRjGc88=")
|
||||
];
|
||||
postPatch = ''
|
||||
# use a patch from the nix store in the `go generate` script
|
||||
substituteInPlace llm/generate/gen_common.sh \
|
||||
--subst-var-by cmakeIncludePatch '${./cmake-include.patch}'
|
||||
# `ollama/llm/generate/gen_common.sh` -> "avoid duplicate main symbols when we link into the cgo binary"
|
||||
substituteInPlace llm/llama.cpp/examples/server/server.cpp \
|
||||
--replace-fail 'int main(' 'int __main('
|
||||
# replace inaccurate version number with actual release version
|
||||
substituteInPlace version/version.go --replace-fail 0.0.0 '${version}'
|
||||
'';
|
||||
preBuild = ''
|
||||
export OLLAMA_SKIP_PATCHING=true
|
||||
# build llama.cpp libraries for ollama
|
||||
go generate ./...
|
||||
'';
|
||||
|
||||
ldflags = [
|
||||
"-s"
|
||||
"-w"
|
||||
"-X=github.com/jmorganca/ollama/version.Version=${version}"
|
||||
"-X=github.com/jmorganca/ollama/server.mode=release"
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Get up and running with large language models locally";
|
||||
homepage = "https://github.com/jmorganca/ollama";
|
||||
license = licenses.mit;
|
||||
platforms = platforms.unix;
|
||||
mainProgram = "ollama";
|
||||
maintainers = with maintainers; [ abysssol dit7ya elohmeier ];
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
rocmClang = linkFarm "rocm-clang" {
|
||||
llvm = rocmPackages.llvm.clang;
|
||||
};
|
||||
rocmPath = buildEnv {
|
||||
name = "rocm-path";
|
||||
paths = [
|
||||
rocmPackages.rocm-device-libs
|
||||
rocmClang
|
||||
];
|
||||
};
|
||||
rocmVars = {
|
||||
ROCM_PATH = rocmPath;
|
||||
CLBlast_DIR = "${clblast}/lib/cmake/CLBlast";
|
||||
};
|
||||
|
||||
cudaToolkit = buildEnv {
|
||||
name = "cuda-toolkit";
|
||||
ignoreCollisions = true; # FIXME: find a cleaner way to do this without ignoring collisions
|
||||
paths = [
|
||||
cudaPackages.cudatoolkit
|
||||
cudaPackages.cuda_cudart
|
||||
];
|
||||
};
|
||||
cudaVars = {
|
||||
CUDA_LIB_DIR = "${cudaToolkit}/lib";
|
||||
CUDACXX = "${cudaToolkit}/bin/nvcc";
|
||||
CUDAToolkit_ROOT = cudaToolkit;
|
||||
};
|
||||
|
||||
linuxGpuLibs = {
|
||||
buildInputs = lib.optionals rocmIsEnabled [
|
||||
rocmPackages.clr
|
||||
rocmPackages.hipblas
|
||||
rocmPackages.rocblas
|
||||
rocmPackages.rocsolver
|
||||
rocmPackages.rocsparse
|
||||
libdrm
|
||||
] ++ lib.optionals cudaIsEnabled [
|
||||
cudaPackages.cuda_cudart
|
||||
];
|
||||
};
|
||||
|
||||
appleGpuLibs = { buildInputs = metalFrameworks; };
|
||||
|
||||
runtimeLibs = lib.optionals rocmIsEnabled [
|
||||
rocmPackages.rocm-smi
|
||||
] ++ lib.optionals cudaIsEnabled [
|
||||
linuxPackages.nvidia_x11
|
||||
];
|
||||
runtimeLibWrapper = {
|
||||
postFixup = ''
|
||||
mv "$out/bin/${pname}" "$out/bin/.${pname}-unwrapped"
|
||||
makeWrapper "$out/bin/.${pname}-unwrapped" "$out/bin/${pname}" \
|
||||
--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath runtimeLibs}'
|
||||
'';
|
||||
};
|
||||
|
||||
goBuild =
|
||||
if cudaIsEnabled then
|
||||
buildGoModule.override { stdenv = overrideCC stdenv gcc12; }
|
||||
else
|
||||
buildGoModule;
|
||||
in
|
||||
goBuild (ollama
|
||||
// (lib.optionalAttrs rocmIsEnabled rocmVars)
|
||||
// (lib.optionalAttrs cudaIsEnabled cudaVars)
|
||||
// (lib.optionalAttrs enableLinuxGpu linuxGpuLibs)
|
||||
// (lib.optionalAttrs enableLinuxGpu runtimeLibWrapper)
|
||||
goBuild ((lib.optionalAttrs enableRocm {
|
||||
ROCM_PATH = rocmPath;
|
||||
CLBlast_DIR = "${clblast}/lib/cmake/CLBlast";
|
||||
}) // (lib.optionalAttrs enableCuda {
|
||||
CUDA_LIB_DIR = "${cudaToolkit}/lib";
|
||||
CUDACXX = "${cudaToolkit}/bin/nvcc";
|
||||
CUDAToolkit_ROOT = cudaToolkit;
|
||||
}) // {
|
||||
inherit pname version src;
|
||||
vendorHash = "sha256-zTrBighPBqZ9hhkEV3UawJZUYyPRay7+P6wkhDtpY7M=";
|
||||
|
||||
// (lib.optionalAttrs stdenv.isDarwin appleGpuLibs))
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
] ++ lib.optionals (enableRocm || enableCuda) [
|
||||
makeWrapper
|
||||
] ++ lib.optionals stdenv.isDarwin
|
||||
metalFrameworks;
|
||||
|
||||
buildInputs = lib.optionals enableRocm [
|
||||
rocmPackages.clr
|
||||
rocmPackages.hipblas
|
||||
rocmPackages.rocblas
|
||||
rocmPackages.rocsolver
|
||||
rocmPackages.rocsparse
|
||||
libdrm
|
||||
] ++ lib.optionals enableCuda [
|
||||
cudaPackages.cuda_cudart
|
||||
] ++ lib.optionals stdenv.isDarwin
|
||||
metalFrameworks;
|
||||
|
||||
patches = [
|
||||
# remove uses of `git` in the `go generate` script
|
||||
# instead use `patch` where necessary
|
||||
./remove-git.patch
|
||||
# replace a hardcoded use of `g++` with `$CXX`
|
||||
./replace-gcc.patch
|
||||
|
||||
# ollama's patches of llama.cpp's example server
|
||||
# `ollama/llm/generate/gen_common.sh` -> "apply temporary patches until fix is upstream"
|
||||
(preparePatch "01-cache.diff" "sha256-MTTln2G0G8dntihUzEjPM1ruTsApb4ZToBczJb8EG68=")
|
||||
(preparePatch "02-cudaleaks.diff" "sha256-Cu7E9iEcvddPL9mPPI5Z96qmwWigi3f0WgSpPRjGc88=")
|
||||
];
|
||||
postPatch = ''
|
||||
# use a patch from the nix store in the `go generate` script
|
||||
substituteInPlace llm/generate/gen_common.sh \
|
||||
--subst-var-by cmakeIncludePatch '${./cmake-include.patch}'
|
||||
# `ollama/llm/generate/gen_common.sh` -> "avoid duplicate main symbols when we link into the cgo binary"
|
||||
substituteInPlace llm/llama.cpp/examples/server/server.cpp \
|
||||
--replace-fail 'int main(' 'int __main('
|
||||
# replace inaccurate version number with actual release version
|
||||
substituteInPlace version/version.go --replace-fail 0.0.0 '${version}'
|
||||
'';
|
||||
preBuild = ''
|
||||
export OLLAMA_SKIP_PATCHING=true
|
||||
# build llama.cpp libraries for ollama
|
||||
go generate ./...
|
||||
'';
|
||||
postFixup = ''
|
||||
# the app doesn't appear functional at the moment, so hide it
|
||||
mv "$out/bin/app" "$out/bin/.ollama-app"
|
||||
'' + lib.optionalString (enableRocm || enableCuda) ''
|
||||
# expose runtime libraries necessary to use the gpu
|
||||
mv "$out/bin/ollama" "$out/bin/.ollama-unwrapped"
|
||||
makeWrapper "$out/bin/.ollama-unwrapped" "$out/bin/ollama" \
|
||||
--suffix LD_LIBRARY_PATH : '/run/opengl-driver/lib:${lib.makeLibraryPath runtimeLibs}'
|
||||
'';
|
||||
|
||||
ldflags = [
|
||||
"-s"
|
||||
"-w"
|
||||
"-X=github.com/jmorganca/ollama/version.Version=${version}"
|
||||
"-X=github.com/jmorganca/ollama/server.mode=release"
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Get up and running with large language models locally";
|
||||
homepage = "https://github.com/jmorganca/ollama";
|
||||
license = licenses.mit;
|
||||
platforms = platforms.unix;
|
||||
mainProgram = "ollama";
|
||||
maintainers = with maintainers; [ abysssol dit7ya elohmeier ];
|
||||
};
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in a new issue