mirror of
https://github.com/guillaume-be/rust-bert.git
synced 2024-10-26 22:19:05 +03:00
Updated docs & dependencies
This commit is contained in:
parent
851863482c
commit
3571058a92
14
Cargo.toml
14
Cargo.toml
@ -53,20 +53,20 @@ all-tests = []
|
||||
features = ["doc-only"]
|
||||
|
||||
[dependencies]
|
||||
rust_tokenizers = {version="~6.0.0", path="E:/Coding/backup-rust/rust-tokenizers/main"}
|
||||
tch = "~0.2.0"
|
||||
serde_json = "1.0.56"
|
||||
serde = { version = "1.0.114", features = ["derive"] }
|
||||
rust_tokenizers = "~6.0.0"
|
||||
tch = "~0.2.1"
|
||||
serde_json = "1.0.59"
|
||||
serde = { version = "1.0.117", features = ["derive"] }
|
||||
dirs = "3.0.1"
|
||||
itertools = "0.9.0"
|
||||
ordered-float = "2.0.0"
|
||||
cached-path = "0.4.5"
|
||||
lazy_static = "1.4.0"
|
||||
uuid = { version = "0.8.1", features = ["v4"] }
|
||||
thiserror = "1.0.20"
|
||||
thiserror = "1.0.21"
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = "1.0.32"
|
||||
anyhow = "1.0.33"
|
||||
csv = "1.1.3"
|
||||
criterion = "0.3.3"
|
||||
torch-sys = "0.2.0"
|
||||
torch-sys = "0.2.1"
|
||||
|
@ -22,12 +22,12 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::AlbertTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::albert::{AlbertConfig, AlbertForMaskedLM};
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::AlbertTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -17,12 +17,12 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::RobertaTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::bart::{BartConfig, BartModel};
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::RobertaTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -21,12 +21,12 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::BertTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::bert::{BertConfig, BertForMaskedLM};
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::BertTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -20,7 +20,6 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::BertTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::distilbert::{
|
||||
@ -29,6 +28,7 @@
|
||||
//! };
|
||||
//! use rust_bert::resources::{LocalResource, RemoteResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::BertTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -25,12 +25,12 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::BertTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::electra::{ElectraConfig, ElectraForMaskedLM};
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::BertTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -16,12 +16,12 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::Gpt2Tokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::gpt2::{GPT2LMHeadModel, Gpt2Config};
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::Gpt2Tokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -23,7 +23,7 @@
|
||||
//! use rust_bert::marian::MarianForConditionalGeneration;
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::preprocessing::tokenizer::marian_tokenizer::MarianTokenizer;
|
||||
//! use rust_tokenizers::tokenizer::MarianTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -15,13 +15,13 @@
|
||||
//!
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! use rust_tokenizers::OpenAiGptTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::gpt2::Gpt2Config;
|
||||
//! use rust_bert::openai_gpt::OpenAiGptModel;
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::OpenAiGptTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -48,7 +48,6 @@
|
||||
//! Output: \
|
||||
//! ```no_run
|
||||
//! # use rust_bert::pipelines::token_classification::Token;
|
||||
//! use rust_tokenizers::preprocessing::tokenizer::base_tokenizer::{Mask, Offset};
|
||||
//! use rust_tokenizers::{Mask, Offset};
|
||||
//! # let output =
|
||||
//! [
|
||||
|
@ -21,13 +21,13 @@
|
||||
//! ```no_run
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! #
|
||||
//! use rust_tokenizers::RobertaTokenizer;
|
||||
//! use tch::{nn, Device};
|
||||
//! # use std::path::PathBuf;
|
||||
//! use rust_bert::bert::BertConfig;
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::roberta::RobertaForMaskedLM;
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::tokenizer::RobertaTokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
@ -22,7 +22,7 @@
|
||||
//! use rust_bert::resources::{LocalResource, Resource};
|
||||
//! use rust_bert::t5::{T5Config, T5ForConditionalGeneration};
|
||||
//! use rust_bert::Config;
|
||||
//! use rust_tokenizers::preprocessing::tokenizer::t5_tokenizer::T5Tokenizer;
|
||||
//! use rust_tokenizers::tokenizer::T5Tokenizer;
|
||||
//!
|
||||
//! let config_resource = Resource::Local(LocalResource {
|
||||
//! local_path: PathBuf::from("path/to/config.json"),
|
||||
|
Loading…
Reference in New Issue
Block a user