marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-17 09:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

move eos to vocab

Browse Source
This commit is contained in:
Hieu Hoang 2016-09-17 23:09:33 +02:00
parent 04fb8734a2
commit 21bdc84e25
2 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/validate_encoder_decoder.cu
View File

@ -112,9 +112,7 @@ int main(int argc, char** argv) {
while (getline(source_file, source_line)) { while (getline(source_file, source_line)) {
getline(target_file, target_line); getline(target_file, target_line);
std::vector<size_t> source_ids = source_vocab.ProcessSentence(source_line); std::vector<size_t> source_ids = source_vocab.ProcessSentence(source_line);
source_ids.push_back(source_vocab.GetEOS()); // Append EOS token.
std::vector<size_t> target_ids = target_vocab.ProcessSentence(target_line); std::vector<size_t> target_ids = target_vocab.ProcessSentence(target_line);
target_ids.push_back(target_vocab.GetEOS()); // Append EOS token.
source_sentences.push_back(source_ids); source_sentences.push_back(source_ids);
target_sentences.push_back(target_ids); target_sentences.push_back(target_ids);
if (num_source_tokens < 0 || source_ids.size() > num_source_tokens) { if (num_source_tokens < 0 || source_ids.size() > num_source_tokens) {

1
src/vocab.cpp
View File

@ -54,6 +54,7 @@ std::vector<size_t> Vocab::ProcessSentence(const std::string &sentence)
size_t id = GetOrCreate(toks[i]); size_t id = GetOrCreate(toks[i]);
ret[i] = id; ret[i] = id;
} }
ret.push_back(GetEOS()); // Append EOS token.
return ret; return ret;
} }