hasktorch/tokenizers
1
1
Fork 0
mirror of https://github.com/hasktorch/tokenizers.git synced 2024-10-05 17:37:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow invalid_reference_casting

Browse Source
This commit is contained in:
Junji Hashimoto 2024-04-21 14:19:19 +09:00
parent f6e9f98995
commit 23944f1146
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tokenizers/src/models/bpe/trainer.rs
View File

@ -414,6 +414,7 @@ impl BpeTrainer {
word_counts: &HashMap<String, u32>,
model: &mut BPE,
) -> Result<Vec<AddedToken>> {
#![allow(invalid_reference_casting)]
let mut word_to_id: HashMap<String, u32> = HashMap::with_capacity(self.vocab_size);
let mut id_to_word: Vec<String> = Vec::with_capacity(self.vocab_size);