moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-30 23:42:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

separate out xml processing into separate method

Browse Source
This commit is contained in:
Hieu Hoang 2016-06-20 19:11:53 +01:00
parent fc02b00360
commit f40c6bd030
2 changed files with 26 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
contrib/other-builds/moses2/PhraseBased/Sentence.cpp
View File

@ -32,6 +32,28 @@ Sentence *Sentence::CreateFromString(MemPool &pool, FactorCollection &vocab,
if (system.options.input.xml_policy) {
// xml
ret = CreateFromStringXML(pool, vocab, system, str, translationId);
}
else {
// no xml
//cerr << "PB Sentence" << endl;
std::vector<std::string> toks = Tokenize(str);
size_t size = toks.size();
ret = new (pool.Allocate<Sentence>()) Sentence(translationId, pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
}
//cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl;
return ret;
}
Sentence *Sentence::CreateFromStringXML(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str, long translationId)
{
Sentence *ret;
vector<XMLOption*> xmlOptions;
pugi::xml_document doc;
@ -85,23 +107,10 @@ Sentence *Sentence::CreateFromString(MemPool &pool, FactorCollection &vocab,
for (size_t i = 0; i < xmlOptions.size(); ++i) {
delete xmlOptions[i];
}
}
else {
// no xml
//cerr << "PB Sentence" << endl;
std::vector<std::string> toks = Tokenize(str);
size_t size = toks.size();
ret = new (pool.Allocate<Sentence>()) Sentence(translationId, pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
}
//cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl;
return ret;
return ret;
}
void Sentence::XMLParse(
size_t depth,
const pugi::xml_node &parentNode,

3
contrib/other-builds/moses2/PhraseBased/Sentence.h
View File

@ -48,6 +48,9 @@ public:
{}
protected:
static Sentence *CreateFromStringXML(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str, long translationId);
static void XMLParse(
size_t depth,
const pugi::xml_node &parentNode,