From 21bdc84e253afd9c016fb75478e462c15a319a77 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Sat, 17 Sep 2016 23:09:33 +0200 Subject: [PATCH] move eos to vocab --- src/validate_encoder_decoder.cu | 2 -- src/vocab.cpp | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/validate_encoder_decoder.cu b/src/validate_encoder_decoder.cu index 15159ba6..f78ddce8 100644 --- a/src/validate_encoder_decoder.cu +++ b/src/validate_encoder_decoder.cu @@ -112,9 +112,7 @@ int main(int argc, char** argv) { while (getline(source_file, source_line)) { getline(target_file, target_line); std::vector source_ids = source_vocab.ProcessSentence(source_line); - source_ids.push_back(source_vocab.GetEOS()); // Append EOS token. std::vector target_ids = target_vocab.ProcessSentence(target_line); - target_ids.push_back(target_vocab.GetEOS()); // Append EOS token. source_sentences.push_back(source_ids); target_sentences.push_back(target_ids); if (num_source_tokens < 0 || source_ids.size() > num_source_tokens) { diff --git a/src/vocab.cpp b/src/vocab.cpp index 25c28386..e447f8cc 100644 --- a/src/vocab.cpp +++ b/src/vocab.cpp @@ -54,6 +54,7 @@ std::vector Vocab::ProcessSentence(const std::string &sentence) size_t id = GetOrCreate(toks[i]); ret[i] = id; } + ret.push_back(GetEOS()); // Append EOS token. return ret; }