@@ -32,6 +32,14 @@ let
3232 url = "https://norvig.com/big.txt";
3333 sha256 = "0yz80icdly7na03cfpl0nfk5h3j3cam55rj486n03wph81ynq1ps";
3434 };
35+ docPipelineTokenizer = fetchurl {
36+ url = "https://s3.amazonaws.com/models.huggingface.co/bert/anthony/doc-pipeline/tokenizer.json";
37+ hash = "sha256-i533xC8J5CDMNxBjo+p6avIM8UOcui8RmGAmK0GmfBc=";
38+ };
39+ docQuicktourTokenizer = fetchurl {
40+ url = "https://s3.amazonaws.com/models.huggingface.co/bert/anthony/doc-quicktour/tokenizer.json";
41+ hash = "sha256-ipY9d5DR5nxoO6kj7rItueZ9AO5wq9+Nzr6GuEIfIBI=";
42+ };
3543 openaiVocab = fetchurl {
3644 url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-vocab.json";
3745 sha256 = "0y40gc9bixj5rxv674br1rxmxkd3ly29p80x1596h8yywwcrpx7x";
@@ -42,16 +50,16 @@ let
4250 };
4351in rustPlatform.buildRustPackage rec {
4452 pname = "tokenizers";
45− version = "0.9.2";
53+ version = "0.9.4";
4654
4755 src = fetchFromGitHub {
4856 owner = "huggingface";
4957 repo = pname;
5058 rev = "python-v${version}";
51− sha256 = "0rsm1g5zfq3ygdb3s8v9xqqpgfzvvkc4n5ik3ahy8sw7pyjljb4m";
59+ hash = "sha256-JXoH9yfhMIFg5qDY5zrF6iWb7XKugjMfk1NxSizfaWg=";
5260 };
5361
54− cargoSha256 = "0yn699dq9hdjh7fyci99ni8mmd5qdhzrsi80grzgf5cch8g38rbi";
62+ cargoSha256 = "sha256-u9qitrOxJSABs0VjwHUZgmw7VTQXNbp6l8fKKE/RQ7M=";
5563
5664 sourceRoot = "source/bindings/python";
5765
@@ -82,6 +90,8 @@ in rustPlatform.buildRustPackage rec {
8290 ln -s ${robertaMerges} roberta-base-merges.txt
8391 ln -s ${albertVocab} albert-base-v1-tokenizer.json
8492 ln -s ${bertVocab} bert-base-uncased-vocab.txt
93+ ln -s ${docPipelineTokenizer} bert-wiki.json
94+ ln -s ${docQuicktourTokenizer} tokenizer-wiki.json
8595 ln -s ${norvigBig} big.txt
8696 ln -s ${openaiVocab} openai-gpt-vocab.json
8797 ln -s ${openaiMerges} openai-gpt-merges.txt )