mirror of
https://github.com/SebastianWendel/nixpkgs.git
synced 2024-09-20 04:19:00 +02:00
python311Packages.tokenizers: 0.15.0 -> 0.19.1
Diff: https://github.com/huggingface/tokenizers/compare/v0.15.0...v0.19.1
This commit is contained in:
parent
92d295f588
commit
5e766a1a10
566
pkgs/development/python-modules/tokenizers/Cargo.lock
generated
566
pkgs/development/python-modules/tokenizers/Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
@ -63,16 +63,16 @@ let
|
||||||
in
|
in
|
||||||
buildPythonPackage rec {
|
buildPythonPackage rec {
|
||||||
pname = "tokenizers";
|
pname = "tokenizers";
|
||||||
version = "0.15.0";
|
version = "0.19.1";
|
||||||
format = "pyproject";
|
pyproject = true;
|
||||||
|
|
||||||
disabled = pythonOlder "3.7";
|
disabled = pythonOlder "3.7";
|
||||||
|
|
||||||
src = fetchFromGitHub {
|
src = fetchFromGitHub {
|
||||||
owner = "huggingface";
|
owner = "huggingface";
|
||||||
repo = pname;
|
repo = "tokenizers";
|
||||||
rev = "v${version}";
|
rev = "refs/tags/v${version}";
|
||||||
hash = "sha256-+yfX12eKtgZV1OQvPOlMVTONbpFuigHcl4SjoCIZkSk=";
|
hash = "sha256-sKEAt46cdme821tzz9WSKnQb3hPmFJ4zvHgBNRxjEuk=";
|
||||||
};
|
};
|
||||||
|
|
||||||
cargoDeps = rustPlatform.importCargoLock {
|
cargoDeps = rustPlatform.importCargoLock {
|
||||||
|
@ -97,7 +97,13 @@ buildPythonPackage rec {
|
||||||
Security
|
Security
|
||||||
];
|
];
|
||||||
|
|
||||||
propagatedBuildInputs = [
|
# Cargo.lock is outdated
|
||||||
|
# TODO: remove at next release
|
||||||
|
preConfigure = ''
|
||||||
|
cargo update --offline
|
||||||
|
'';
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
numpy
|
numpy
|
||||||
];
|
];
|
||||||
|
|
||||||
|
@ -123,6 +129,8 @@ buildPythonPackage rec {
|
||||||
|
|
||||||
disabledTests = [
|
disabledTests = [
|
||||||
# Downloads data using the datasets module
|
# Downloads data using the datasets module
|
||||||
|
"test_encode_special_tokens"
|
||||||
|
"test_splitting"
|
||||||
"TestTrainFromIterators"
|
"TestTrainFromIterators"
|
||||||
# Those tests require more data
|
# Those tests require more data
|
||||||
"test_from_pretrained"
|
"test_from_pretrained"
|
||||||
|
|
Loading…
Reference in a new issue