mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-26 11:28:48 +03:00
span length
This commit is contained in:
parent
d95bd5bdae
commit
e25e7b69be
@ -7,6 +7,7 @@
|
||||
#include "moses/StackVec.h"
|
||||
#include "moses/TargetPhrase.h"
|
||||
#include "moses/PP/PhraseProperty.h"
|
||||
#include "moses/PP/SpanLengthPhraseProperty.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -15,7 +16,9 @@ namespace Moses
|
||||
SpanLength::SpanLength(const std::string &line)
|
||||
:StatelessFeatureFunction(1, line)
|
||||
,m_smoothingMethod(None)
|
||||
,m_const(0)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
void SpanLength::Evaluate(const Phrase &source
|
||||
@ -42,19 +45,26 @@ void SpanLength::Evaluate(const InputType &input
|
||||
return;
|
||||
}
|
||||
|
||||
string str = property.get()->GetValueString();
|
||||
PhraseProperty *prop = property.get();
|
||||
SpanLengthPhraseProperty *slProp = static_cast<SpanLengthPhraseProperty*>(prop);
|
||||
assert(slProp);
|
||||
|
||||
string str = slProp->GetValueString();
|
||||
cerr << "str=" << str << endl;
|
||||
|
||||
const Phrase *ruleSource = targetPhrase.GetRuleSource();
|
||||
assert(ruleSource);
|
||||
cerr << *ruleSource << endl;
|
||||
|
||||
float score = 0;
|
||||
for (size_t i = 0; i < stackVec->size(); ++i) {
|
||||
const ChartCellLabel &cell = *stackVec->at(i);
|
||||
const WordsRange &ntRange = cell.GetCoverage();
|
||||
|
||||
|
||||
size_t sourceWidth = ntRange.GetNumWordsCovered();
|
||||
float prob = slProp->GetProb(i, sourceWidth, m_const);
|
||||
score += TransformScore(prob);
|
||||
}
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
|
||||
}
|
||||
|
||||
|
@ -18,6 +18,10 @@ SpanLengthPhraseProperty::SpanLengthPhraseProperty(const std::string &value)
|
||||
float count = Scan<float>(toks[i + 1]);
|
||||
Populate(span, count);
|
||||
}
|
||||
|
||||
// totals
|
||||
CalcTotals(m_source);
|
||||
CalcTotals(m_target);
|
||||
}
|
||||
|
||||
void SpanLengthPhraseProperty::Populate(const string &span, float count)
|
||||
@ -37,9 +41,44 @@ void SpanLengthPhraseProperty::Populate(const std::vector<size_t> &toks, float c
|
||||
m_source.resize(ntInd + 1);
|
||||
m_target.resize(ntInd + 1);
|
||||
}
|
||||
m_source[ntInd][sourceLength] = count;
|
||||
m_target[ntInd][targetLength] = count;
|
||||
m_source[ntInd].first[sourceLength] = count;
|
||||
m_target[ntInd].first[targetLength] = count;
|
||||
|
||||
}
|
||||
|
||||
void SpanLengthPhraseProperty::CalcTotals(Vec &vec)
|
||||
{
|
||||
for (size_t i = 0; i < vec.size(); i = i + 2) {
|
||||
float total = 0;
|
||||
|
||||
const Map &map = vec[i].first;
|
||||
Map::const_iterator iter;
|
||||
for (iter = map.begin(); iter != map.end(); ++iter) {
|
||||
float count = iter->second;
|
||||
total += count;
|
||||
}
|
||||
|
||||
vec[i].second = total;
|
||||
}
|
||||
}
|
||||
|
||||
float SpanLengthPhraseProperty::GetProb(size_t ntInd, size_t sourceWidth, float smoothing) const
|
||||
{
|
||||
float count;
|
||||
|
||||
const std::pair<Map, float> &data = m_source[ntInd];
|
||||
const Map &map = data.first;
|
||||
Map::const_iterator iter = map.find(sourceWidth);
|
||||
if (iter == map.end()) {
|
||||
count = 0;
|
||||
}
|
||||
else {
|
||||
count = iter->second;
|
||||
}
|
||||
count += smoothing;
|
||||
|
||||
float ret = count / (data.second + smoothing * map.size());
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -14,14 +14,16 @@ class SpanLengthPhraseProperty : public PhraseProperty
|
||||
public:
|
||||
SpanLengthPhraseProperty(const std::string &value);
|
||||
|
||||
float GetProb(size_t ntInd, size_t sourceWidth, float smoothing) const;
|
||||
protected:
|
||||
// fractional counts
|
||||
typedef std::map<size_t, float> Map;
|
||||
typedef std::vector<Map> Vec;
|
||||
typedef std::vector<std::pair<Map, float> > Vec;
|
||||
Vec m_source, m_target;
|
||||
|
||||
void Populate(const std::string &span, float count);
|
||||
void Populate(const std::vector<size_t> &toks, float count);
|
||||
void CalcTotals(Vec &vec);
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
Loading…
Reference in New Issue
Block a user