moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-30 23:42:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

add GetVocabIds()

Browse Source
This commit is contained in:
Hieu Hoang 2016-06-15 18:21:04 +01:00
parent 73ac4e43cb
commit 30bd411e17
1 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
contrib/other-builds/moses2/legacy/ProbingPT/StoreVocab.h
View File

@ -8,6 +8,7 @@
#include <string>
#include <boost/unordered_map.hpp>
#include "../OutputFileStream.h"
#include "../Util2.h"
namespace Moses2
{
@ -41,6 +42,20 @@ public:
}
}
std::vector<VOCABID> GetVocabIds(const std::string &line)
{
std::vector<std::string> toks = Moses2::Tokenize(line, " ");
std::vector<VOCABID> ret(toks.size());
for (size_t i = 0; i < toks.size(); ++i) {
const std::string &tok = toks[i];
VOCABID id = GetVocabId(tok);
ret[i] = id;
}
return ret;
}
void Save()
{
OutputFileStream strme(m_path);