moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 05:14:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

start framework for evaluating target phrase score

Browse Source
This commit is contained in:
Hieu Hoang 2013-05-02 14:55:26 +01:00
parent c99e9f8c90
commit 47465f8afb
34 changed files with 89 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
moses/BleuScoreFeature.cpp
View File

@ -860,7 +860,7 @@ const FFState* BleuScoreFeature::EmptyHypothesisState(const InputType& input) co
void BleuScoreFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/BleuScoreFeature.h
View File

@ -120,7 +120,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
private:
bool m_enabled;

2
moses/DecodeFeature.h
View File

@ -61,7 +61,7 @@ class DecodeFeature : public StatelessFeatureFunction {
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool IsDecodeFeature() const

12
moses/DummyScoreProducers.cpp
View File

@ -103,9 +103,9 @@ FFState* DistortionScoreProducer::Evaluate(
void DistortionScoreProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
// no score
}
void WordPenaltyProducer::Evaluate(
@ -118,15 +118,17 @@ void WordPenaltyProducer::Evaluate(
void WordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
float score = - targetPhrase.GetSize();
scoreBreakdown.Assign(this, score);
}
void UnknownWordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
// shouldn't be called
CHECK(false);
}

15
moses/DummyScoreProducers.h
View File

@ -40,7 +40,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
};
@ -65,7 +65,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
};
@ -90,9 +90,14 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
virtual bool IsTuneable() const
{ return false; }
bool IsDecodeFeature() const
{ return true; }
virtual bool IsTuneable() const { return false; }
};
@ -115,7 +120,7 @@ class MetaFeatureProducer : public StatelessFeatureFunction
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
};
}

2
moses/FeatureFunction.h
View File

@ -134,7 +134,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const = 0;
, ScoreComponentCollection &estimatedFutureScore) const = 0;
virtual bool IsDecodeFeature() const
{ return false; }

2
moses/GlobalLexicalModel.cpp
View File

@ -188,7 +188,7 @@ void GlobalLexicalModel::Evaluate
void GlobalLexicalModel::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/GlobalLexicalModel.h
View File

@ -81,7 +81,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
virtual StatelessFeatureType GetStatelessFeatureType() const

2
moses/GlobalLexicalModelUnlimited.cpp
View File

@ -336,7 +336,7 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat
void GlobalLexicalModelUnlimited::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/GlobalLexicalModelUnlimited.h
View File

@ -91,7 +91,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }

26
moses/LM/Base.cpp
View File

@ -62,9 +62,31 @@ void LanguageModel::IncrementalCallback(Incremental::Manager &manager) const {
void LanguageModel::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
if (Useable(targetPhrase)) {
// contains factors used by this LM
float fullScore, nGramScore;
size_t oovCount;
CalcScore(targetPhrase, fullScore, nGramScore, oovCount);
float estimateScore = fullScore - nGramScore;
if (StaticData::Instance().GetLMEnableOOVFeature()) {
vector<float> scores(2), estimateScores(2);
scores[0] = nGramScore;
scores[1] = oovCount;
scoreBreakdown.Assign(this, scores);
estimateScores[0] = estimateScore;
estimateScores[1] = 0;
estimatedFutureScore.Assign(this, estimateScores);
} else {
scoreBreakdown.Assign(this, nGramScore);
estimatedFutureScore.Assign(this, estimateScore);
}
}
}
} // namespace Moses

2
moses/LM/Base.h
View File

@ -88,7 +88,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
};

2
moses/LexicalReordering.cpp
View File

@ -97,7 +97,7 @@ const FFState* LexicalReordering::EmptyHypothesisState(const InputType &input) c
void LexicalReordering::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/LexicalReordering.h
View File

@ -49,7 +49,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
private:
bool DecodeCondition(std::string s);

2
moses/PhraseBoundaryFeature.cpp
View File

@ -96,7 +96,7 @@ FFState* PhraseBoundaryFeature::Evaluate
void PhraseBoundaryFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/PhraseBoundaryFeature.h
View File

@ -48,7 +48,7 @@ public:
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }

2
moses/PhraseLengthFeature.cpp
View File

@ -42,7 +42,7 @@ void PhraseLengthFeature::Evaluate(
void PhraseLengthFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/PhraseLengthFeature.h
View File

@ -29,7 +29,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
};

2
moses/PhrasePairFeature.cpp
View File

@ -246,7 +246,7 @@ void PhrasePairFeature::Evaluate(
void PhrasePairFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/PhrasePairFeature.h
View File

@ -43,7 +43,7 @@ class PhrasePairFeature: public StatelessFeatureFunction {
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/);

2
moses/ScoreComponentCollectionTest.cpp
View File

@ -38,7 +38,7 @@ class MockStatelessFeatureFunction : public StatelessFeatureFunction {
virtual void EvaluateChart(const ChartBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{ }
};

2
moses/SourceWordDeletionFeature.cpp
View File

@ -82,7 +82,7 @@ void SourceWordDeletionFeature::EvaluateChart(
void SourceWordDeletionFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/SourceWordDeletionFeature.h
View File

@ -31,7 +31,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,

2
moses/TargetBigramFeature.cpp
View File

@ -114,7 +114,7 @@ FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
void TargetBigramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/TargetBigramFeature.h
View File

@ -44,7 +44,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
private:
FactorType m_factorType;

2
moses/TargetNgramFeature.cpp
View File

@ -396,7 +396,7 @@ FFState* TargetNgramFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int
void TargetNgramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/TargetNgramFeature.h
View File

@ -197,7 +197,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
private:
FactorType m_factorType;

21
moses/TargetPhrase.cpp
View File

@ -80,18 +80,30 @@ void TargetPhrase::WriteToRulePB(hgmert::Rule* pb) const
void TargetPhrase::Evaluate()
{
float totalEstFutureScore = 0;
return;
ScoreComponentCollection temp;
ScoreComponentCollection estimatedFutureScore;
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
cerr << ff.GetScoreProducerDescription() << endl;
if (!ff.IsDecodeFeature()) {
float estimatedFutureScore = 0;
ff.Evaluate(*this, m_scoreBreakdown, estimatedFutureScore);
totalEstFutureScore += estimatedFutureScore;
ff.Evaluate(*this, temp, estimatedFutureScore);
}
}
/*
cerr << *this << endl;
cerr << "total=" << (estimatedFutureScore.GetWeightedScore() + temp.GetWeightedScore()) << endl;
cerr << "estimatedFutureScore=" << estimatedFutureScore.GetWeightedScore()
<< " " << estimatedFutureScore << endl;
cerr << "temp=" << temp.GetWeightedScore() << " " << temp << endl;
cerr << "m_fullScore="<< m_fullScore<< " " << m_scoreBreakdown << endl;
cerr << flush;
*/
}
void TargetPhrase::SetScore(float score)
@ -197,6 +209,7 @@ void TargetPhrase::SetScoreChart(const FeatureFunction* translationScoreProducer
,const LMList &languageModels
,const WordPenaltyProducer* wpProducer)
{
//cerr << *this << endl;
CHECK(weightT.size() == scoreVector.size());
// calc average score if non-best

2
moses/TargetWordInsertionFeature.cpp
View File

@ -86,7 +86,7 @@ void TargetWordInsertionFeature::EvaluateChart(
void TargetWordInsertionFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/TargetWordInsertionFeature.h
View File

@ -31,7 +31,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,

1
moses/TranslationModel/RuleTable/LoaderStandard.cpp
View File

@ -247,6 +247,7 @@ bool RuleTableLoaderStandard::Load(FormatType format
}
targetPhrase->SetScoreChart(&ruleTable, scoreVector, weight, languageModels,wpProducer);
targetPhrase->Evaluate();
TargetPhraseCollection &phraseColl = GetOrCreateTargetPhraseCollection(ruleTable, targetPhrase->GetSourcePhrase(), *targetPhrase, sourceLHS);
phraseColl.Add(targetPhrase);

2
moses/TranslationOptionCollection.cpp
View File

@ -268,6 +268,8 @@ void TranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceWord,s
targetPhrase.SetScore(m_system,*inputScores);
}
targetPhrase.Evaluate();
transOpt = new TranslationOption(WordsRange(sourcePos, sourcePos + length - 1), targetPhrase, m_source
, StaticData::Instance().GetUnknownWordPenaltyProducer());
transOpt->CalcScore(m_system);

2
moses/WordTranslationFeature.cpp
View File

@ -530,7 +530,7 @@ void WordTranslationFeature::EvaluateChart(
void WordTranslationFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}

2
moses/WordTranslationFeature.h
View File

@ -52,7 +52,7 @@ public:
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
, ScoreComponentCollection &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }