mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
use properties to store lex ro data in pt
This commit is contained in:
parent
fdbae28115
commit
eefcd29bef
@ -65,7 +65,10 @@ LexicalReordering::~LexicalReordering()
|
||||
|
||||
void LexicalReordering::Load(System &system)
|
||||
{
|
||||
if (FileExists(m_path + ".minlexr") ) {
|
||||
if (!m_ptProperty.empty()) {
|
||||
// do nothing
|
||||
}
|
||||
else if (FileExists(m_path + ".minlexr") ) {
|
||||
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr", m_FactorsF,
|
||||
m_FactorsE, m_FactorsC);
|
||||
m_blank = new (system.systemPool.Allocate<PhraseImpl>()) PhraseImpl(system.systemPool, 0);
|
||||
@ -101,7 +104,7 @@ void LexicalReordering::SetParameter(const std::string& key, const std::string&
|
||||
m_path = value;
|
||||
}
|
||||
else if (key == "type") {
|
||||
|
||||
// ignore. only do 1 type
|
||||
}
|
||||
else if (key == "input-factor") {
|
||||
m_FactorsF = Tokenize<FactorType>(value);
|
||||
@ -109,7 +112,9 @@ void LexicalReordering::SetParameter(const std::string& key, const std::string&
|
||||
else if (key == "output-factor") {
|
||||
m_FactorsE = Tokenize<FactorType>(value);
|
||||
}
|
||||
|
||||
else if (key == "pt-property") {
|
||||
m_ptProperty = value;
|
||||
}
|
||||
else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
@ -153,7 +158,26 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrase &targetPhrase,
|
||||
const Phrase &sourcePhrase) const
|
||||
{
|
||||
if (m_compactModel) {
|
||||
if (!m_ptProperty.empty()) {
|
||||
string propStr = GetProperty(targetPhrase.properties, m_ptProperty);
|
||||
//cerr << "propStr=" << propStr << endl;
|
||||
if (!propStr.empty()) {
|
||||
vector<SCORE> values = Tokenize<SCORE>(propStr, " ");
|
||||
assert(values.size() == 6);
|
||||
|
||||
SCORE *scoreArr = pool.Allocate<SCORE>(6);
|
||||
for (size_t i = 0; i < 6; ++i) {
|
||||
scoreArr[i] = values[i];
|
||||
}
|
||||
|
||||
targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
|
||||
}
|
||||
else {
|
||||
targetPhrase.ffData[m_PhraseTableInd] = NULL;
|
||||
}
|
||||
}
|
||||
else if (m_compactModel) {
|
||||
// using external compact binary model
|
||||
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase, *m_blank);
|
||||
if (values.size()) {
|
||||
assert(values.size() == 6);
|
||||
@ -168,6 +192,7 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
}
|
||||
}
|
||||
else {
|
||||
// using external memory model
|
||||
assert(m_coll);
|
||||
|
||||
// cache data in target phrase
|
||||
@ -185,7 +210,6 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LexicalReordering::EvaluateWhenApplied(const Manager &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
@ -243,6 +267,29 @@ const LexicalReordering::Values *LexicalReordering::GetValues(const Phrase &sour
|
||||
}
|
||||
}
|
||||
|
||||
std::string LexicalReordering::GetProperty(const char *properties, const std::string &key) const
|
||||
{
|
||||
string ret;
|
||||
if (properties == NULL) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
string propStr(properties);
|
||||
|
||||
size_t start = propStr.find("{{" + key + " ");
|
||||
if (start == propStr.npos) {
|
||||
// do nothing
|
||||
}
|
||||
else {
|
||||
size_t keySize = key.size();
|
||||
size_t end = propStr.find("}}", start);
|
||||
assert(end != propStr.npos);
|
||||
ret = propStr.substr(start + keySize + 3, end - start - keySize - 3);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t LexicalReordering::GetOrientation(Range const& cur) const
|
||||
{
|
||||
return (cur.GetStartPos() == 0) ? 0 : 1;
|
||||
|
@ -57,10 +57,13 @@ public:
|
||||
|
||||
protected:
|
||||
std::string m_path;
|
||||
std::string m_ptProperty;
|
||||
FactorList m_FactorsF;
|
||||
FactorList m_FactorsE;
|
||||
FactorList m_FactorsC;
|
||||
|
||||
std::string GetProperty(const char *properties, const std::string &key) const;
|
||||
|
||||
virtual void
|
||||
EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrase &targetPhrase,
|
||||
|
@ -31,6 +31,7 @@ TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const System &system
|
||||
|
||||
TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size)
|
||||
:PhraseImpl(pool, size)
|
||||
,properties(NULL)
|
||||
{
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
|
||||
|
||||
@ -40,6 +41,7 @@ TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size)
|
||||
|
||||
TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©)
|
||||
:PhraseImpl(pool, copy)
|
||||
,properties(NULL)
|
||||
{
|
||||
// scores
|
||||
m_estimatedScore = copy.m_estimatedScore;
|
||||
|
@ -23,6 +23,7 @@ class TargetPhrase : public PhraseImpl
|
||||
friend std::ostream& operator<<(std::ostream &, const TargetPhrase &);
|
||||
public:
|
||||
mutable void **ffData;
|
||||
char *properties;
|
||||
|
||||
static TargetPhrase *CreateFromString(MemPool &pool, const System &system, const std::string &str);
|
||||
TargetPhrase(MemPool &pool, const System &system, size_t size);
|
||||
|
@ -132,6 +132,12 @@ void PhraseTableMemory::Load(System &system)
|
||||
target->GetScores().CreateFromString(toks[2], *this, system, true);
|
||||
//cerr << "created scores" << endl;
|
||||
|
||||
// properties
|
||||
if (toks.size() == 7) {
|
||||
target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
|
||||
strcpy(target->properties, toks[6].c_str());
|
||||
}
|
||||
|
||||
system.featureFunctions.EvaluateInIsolation(system.systemPool, system, *source, *target);
|
||||
m_root.AddRule(*source, target);
|
||||
}
|
||||
|
@ -131,18 +131,32 @@ int main(int argc, char** argv)
|
||||
//cerr << line << endl;
|
||||
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
|
||||
assert(toks.size() >= 2);
|
||||
toks[0] = Trim(toks[0]);
|
||||
toks[1] = Trim(toks[1]);
|
||||
|
||||
cerr << toks[0] << endl
|
||||
<< toks[1] << endl;
|
||||
for (size_t i = 0; i < toks.size(); ++i) {
|
||||
toks[i] = Trim(toks[i]);
|
||||
}
|
||||
|
||||
std::vector<float> scores = GetScore(toks[0], toks[1], "");
|
||||
|
||||
for (size_t i = 0; i < scores.size(); ++i) {
|
||||
cerr << scores[i] << " ";
|
||||
// output
|
||||
for (size_t i = 0; i < toks.size(); ++i) {
|
||||
cout << toks[i] << " ||| ";
|
||||
}
|
||||
cerr << endl << endl;
|
||||
|
||||
// blank columns
|
||||
for (size_t i = toks.size(); i < 6; ++i) {
|
||||
cout << "||| ";
|
||||
}
|
||||
|
||||
// key-value pairs
|
||||
cout << "{{LexRO ";
|
||||
for (size_t i = 0; i < scores.size(); ++i) {
|
||||
cout << scores[i] << " ";
|
||||
}
|
||||
cout << "}}";
|
||||
|
||||
cout << endl;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user