mirror of
https://github.com/marian-nmt/marian.git
synced 2024-09-17 09:47:34 +03:00
move eos to vocab
This commit is contained in:
parent
04fb8734a2
commit
21bdc84e25
@ -112,9 +112,7 @@ int main(int argc, char** argv) {
|
||||
while (getline(source_file, source_line)) {
|
||||
getline(target_file, target_line);
|
||||
std::vector<size_t> source_ids = source_vocab.ProcessSentence(source_line);
|
||||
source_ids.push_back(source_vocab.GetEOS()); // Append EOS token.
|
||||
std::vector<size_t> target_ids = target_vocab.ProcessSentence(target_line);
|
||||
target_ids.push_back(target_vocab.GetEOS()); // Append EOS token.
|
||||
source_sentences.push_back(source_ids);
|
||||
target_sentences.push_back(target_ids);
|
||||
if (num_source_tokens < 0 || source_ids.size() > num_source_tokens) {
|
||||
|
@ -54,6 +54,7 @@ std::vector<size_t> Vocab::ProcessSentence(const std::string &sentence)
|
||||
size_t id = GetOrCreate(toks[i]);
|
||||
ret[i] = id;
|
||||
}
|
||||
ret.push_back(GetEOS()); // Append EOS token.
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user