Updated docs & dependencies

Cargo.toml

@@ -53,20 +53,20 @@ all-tests = []
 features = ["doc-only"]
 
 [dependencies]
-rust_tokenizers = {version="~6.0.0", path="E:/Coding/backup-rust/rust-tokenizers/main"}
-tch = "~0.2.0"
-serde_json = "1.0.56"
-serde = { version = "1.0.114", features = ["derive"] }
+rust_tokenizers = "~6.0.0"
+tch = "~0.2.1"
+serde_json = "1.0.59"
+serde = { version = "1.0.117", features = ["derive"] }
 dirs = "3.0.1"
 itertools = "0.9.0"
 ordered-float = "2.0.0"
 cached-path = "0.4.5"
 lazy_static = "1.4.0"
 uuid = { version = "0.8.1", features = ["v4"] }
-thiserror = "1.0.20"
+thiserror = "1.0.21"
 
 [dev-dependencies]
-anyhow = "1.0.32"
+anyhow = "1.0.33"
 csv = "1.1.3"
 criterion = "0.3.3"
-torch-sys = "0.2.0"
+torch-sys = "0.2.1"

src/albert/mod.rs

@@ -22,12 +22,12 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::AlbertTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::albert::{AlbertConfig, AlbertForMaskedLM};
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::AlbertTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/bart/mod.rs

@@ -17,12 +17,12 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::RobertaTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::bart::{BartConfig, BartModel};
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::RobertaTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/bert/mod.rs

@@ -21,12 +21,12 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::BertTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::bert::{BertConfig, BertForMaskedLM};
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::BertTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/distilbert/mod.rs

@@ -20,7 +20,6 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::BertTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::distilbert::{
@@ -29,6 +28,7 @@
 //! };
 //! use rust_bert::resources::{LocalResource, RemoteResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::BertTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/electra/mod.rs

@@ -25,12 +25,12 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::BertTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::electra::{ElectraConfig, ElectraForMaskedLM};
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::BertTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/gpt2/mod.rs

@@ -16,12 +16,12 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::Gpt2Tokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::gpt2::{GPT2LMHeadModel, Gpt2Config};
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::Gpt2Tokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/marian/mod.rs

@@ -23,7 +23,7 @@
 //! use rust_bert::marian::MarianForConditionalGeneration;
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
-//! use rust_tokenizers::preprocessing::tokenizer::marian_tokenizer::MarianTokenizer;
+//! use rust_tokenizers::tokenizer::MarianTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/openai_gpt/mod.rs

@@ -15,13 +15,13 @@
 //!
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
-//! use rust_tokenizers::OpenAiGptTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::gpt2::Gpt2Config;
 //! use rust_bert::openai_gpt::OpenAiGptModel;
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::OpenAiGptTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/pipelines/token_classification.rs

@@ -48,7 +48,6 @@
 //! Output: \
 //! ```no_run
 //! # use rust_bert::pipelines::token_classification::Token;
-//! use rust_tokenizers::preprocessing::tokenizer::base_tokenizer::{Mask, Offset};
 //! use rust_tokenizers::{Mask, Offset};
 //! # let output =
 //! [

src/roberta/mod.rs

@@ -21,13 +21,13 @@
 //! ```no_run
 //! # fn main() -> anyhow::Result<()> {
 //! #
-//! use rust_tokenizers::RobertaTokenizer;
 //! use tch::{nn, Device};
 //! # use std::path::PathBuf;
 //! use rust_bert::bert::BertConfig;
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::roberta::RobertaForMaskedLM;
 //! use rust_bert::Config;
+//! use rust_tokenizers::tokenizer::RobertaTokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),

src/t5/mod.rs

@@ -22,7 +22,7 @@
 //! use rust_bert::resources::{LocalResource, Resource};
 //! use rust_bert::t5::{T5Config, T5ForConditionalGeneration};
 //! use rust_bert::Config;
-//! use rust_tokenizers::preprocessing::tokenizer::t5_tokenizer::T5Tokenizer;
+//! use rust_tokenizers::tokenizer::T5Tokenizer;
 //!
 //! let config_resource = Resource::Local(LocalResource {
 //!     local_path: PathBuf::from("path/to/config.json"),