git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #109532 from danieldk/tokenizers-0.10.0

python3Packages.tokenizers: 0.9.4 -> 0.10.0

commit4f337655757d998f87411b7abbdf9af65dac5e74 authorGuillaume Girol <symphorien@users.noreply.github.com> dateSat 23 Jan 2021, 16:56 (6 years ago) parent5f92e69cca0b01
1 file changed+14−3
pkgs/development/python-modules/tokenizers/default.nix+14−3
pkgs/development/python-modules/tokenizers/default.nix+14−3
@@ -7,6 +7,7 @@
77, wheel
88, numpy
99, python
10+, datasets
1011, pytestCheckHook
1112, requests
1213}:
@@ -50,16 +51,16 @@ let
5051 };
5152in rustPlatform.buildRustPackage rec {
5253 pname = "tokenizers";
53− version = "0.9.4";
54+ version = "0.10.0";
5455
5556 src = fetchFromGitHub {
5657 owner = "huggingface";
5758 repo = pname;
5859 rev = "python-v${version}";
59− hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg=";
60+ hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
6061 };
6162
62− cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M=";
63+ cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
6364
6465 sourceRoot = "source/bindings/python";
6566
@@ -75,6 +76,7 @@ in rustPlatform.buildRustPackage rec {
7576 ];
7677
7778 installCheckInputs = [
79+ datasets
7880 pytestCheckHook
7981 requests
8082 ];
@@ -105,6 +107,15 @@ in rustPlatform.buildRustPackage rec {
105107 pipInstallPhase
106108 '';
107109
110+ preCheck = ''
111+ HOME=$TMPDIR
112+ '';
113+
114+ disabledTests = [
115+ # Downloads data using the datasets module.
116+ "TestTrainFromIterators"
117+ ];
118+
108119 meta = with lib; {
109120 homepage = "https://github.com/huggingface/tokenizers";
110121 description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";