hasktorch/tokenizers
1
1
Fork 0
mirror of https://github.com/hasktorch/tokenizers.git synced 2024-08-16 00:30:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update training to include new lines

Browse Source
This commit is contained in:
Anthony MOI 2020-01-03 20:23:58 -05:00
parent a1891387ed
commit 805dc58949
No known key found for this signature in database
GPG Key ID: CB646B1164C636A0
1 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tokenizers/src/tokenizer/mod.rs
View File

@ -411,16 +411,24 @@ impl Tokenizer {
let mut words = HashMap::new();
let file: std::fs::File = File::open(file)?;
let file = BufReader::new(file);
let mut file = BufReader::new(file);
for line in file.lines() {
let line = line?;
let normalized = self.normalize(&line)?;
let pre_tokenized = self.pre_tokenize(normalized.get())?;
trainer.process_tokens(
&mut words,
pre_tokenized.into_iter().map(|(t, _)| t).collect(),
);
let mut buf = String::new();
loop {
buf.clear();
// We read new lines using this API instead of the Lines Iterator
// on purpose. We want to keep the `\n` and potential `\r` between each lines
match file.read_line(&mut buf)? {
0 => break,
_ => {
let normalized = self.normalize(&buf)?;
let pre_tokenized = self.pre_tokenize(normalized.get())?;
trainer.process_tokens(
&mut words,
pre_tokenized.into_iter().map(|(t, _)| t).collect(),
);
}
}
}
Ok(words)