moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

start framework for evaluating target phrase score

Browse Source
This commit is contained in:
Hieu Hoang 2013-05-02 12:15:26 +01:00
parent 0425083c12
commit c99e9f8c90
34 changed files with 227 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
moses/BleuScoreFeature.cpp
View File

@ -858,5 +858,12 @@ const FFState* BleuScoreFeature::EmptyHypothesisState(const InputType& input) co
return new BleuScoreState();
}
void BleuScoreFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
} // namespace.

4
moses/BleuScoreFeature.h
View File

@ -118,6 +118,10 @@ public:
float GetTargetLengthHistory() { return m_target_length_history; }
float GetAverageInputLength() { return m_avg_input_length; }
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
private:
bool m_enabled;
bool m_sentence_bleu;

8
moses/DecodeFeature.h
View File

@ -58,7 +58,15 @@ class DecodeFeature : public StatelessFeatureFunction {
const std::vector<FactorType>& GetInput() const;
const std::vector<FactorType>& GetOutput() const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{}
bool IsDecodeFeature() const
{ return true; }
protected:
std::vector<FactorType> m_input;
std::vector<FactorType> m_output;

21
moses/DummyScoreProducers.cpp
View File

@ -101,6 +101,13 @@ FFState* DistortionScoreProducer::Evaluate(
return res;
}
void DistortionScoreProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
void WordPenaltyProducer::Evaluate(
const PhraseBasedFeatureContext& context,
ScoreComponentCollection* out) const
@ -109,4 +116,18 @@ void WordPenaltyProducer::Evaluate(
out->PlusEquals(this, -static_cast<float>(tp.GetSize()));
}
void WordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
void UnknownWordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

18
moses/DummyScoreProducers.h
View File

@ -37,6 +37,11 @@ public:
CHECK(0); // feature function not valid in chart decoder
return NULL;
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
};
/** Doesn't do anything but provide a key into the global
@ -57,6 +62,11 @@ public:
{
//required but does nothing.
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
};
/** unknown word penalty */
@ -78,6 +88,10 @@ public:
//do nothing - not a real feature
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
virtual bool IsTuneable() const { return false; }
};
@ -98,6 +112,10 @@ class MetaFeatureProducer : public StatelessFeatureFunction
ScoreComponentCollection*) const {
//do nothing - not a real feature
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
};
}

7
moses/FeatureFunction.h
View File

@ -132,6 +132,13 @@ public:
const std::string &GetArgLine() const
{ return m_argLine; }
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const = 0;
virtual bool IsDecodeFeature() const
{ return false; }
};
/** base class for all stateless feature functions.

10
moses/GlobalLexicalModel.cpp
View File

@ -178,11 +178,19 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP
return score;
}
void GlobalLexicalModel::Evaluate
void GlobalLexicalModel::Evaluate
(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
accumulator->PlusEquals( this,
GetFromCacheOrScorePhrase(context.GetTargetPhrase()) );
}
void GlobalLexicalModel::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

5
moses/GlobalLexicalModel.h
View File

@ -79,6 +79,11 @@ public:
exit(1);
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return DependsOnSource; }

7
moses/GlobalLexicalModelUnlimited.cpp
View File

@ -334,4 +334,11 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat
}
void GlobalLexicalModelUnlimited::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

3
moses/GlobalLexicalModelUnlimited.h
View File

@ -89,6 +89,9 @@ public:
assert(0);
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }

7
moses/LM/Base.cpp
View File

@ -60,4 +60,11 @@ void LanguageModel::IncrementalCallback(Incremental::Manager &manager) const {
UTIL_THROW(util::Exception, "Incremental search is only supported by KenLM.");
}
void LanguageModel::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
} // namespace Moses

5
moses/LM/Base.h
View File

@ -85,6 +85,11 @@ public:
// KenLM only (others throw an exception): call incremental search with the model and mapping.
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
};
}

1
moses/LM/Implementation.cpp
View File

@ -222,7 +222,6 @@ FFState* LanguageModelImplementation::EvaluateChart(const ChartHypothesis& hypo,
float finalizedScore = 0.0; // finalized, has sufficient context
// get index map for underlying hypotheses
const TargetPhrase &target = hypo.GetCurrTargetPhrase();
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
hypo.GetCurrTargetPhrase().GetAlignNonTerm().GetNonTermIndexMap();

7
moses/LexicalReordering.cpp
View File

@ -95,5 +95,12 @@ const FFState* LexicalReordering::EmptyHypothesisState(const InputType &input) c
return m_configuration->CreateLexicalReorderingState(input);
}
void LexicalReordering::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

12
moses/LexicalReordering.h
View File

@ -28,10 +28,6 @@ public:
LexicalReordering(const std::string &line);
virtual ~LexicalReordering();
virtual FFState* Evaluate(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
void InitializeForInput(const InputType& i){
@ -40,6 +36,10 @@ public:
Scores GetProb(const Phrase& f, const Phrase& e) const;
virtual FFState* Evaluate(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(const ChartHypothesis&,
int /* featureID */,
ScoreComponentCollection*) const {
@ -47,6 +47,10 @@ public:
return NULL;
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
private:
bool DecodeCondition(std::string s);
bool DecodeDirection(std::string s);

7
moses/PhraseBoundaryFeature.cpp
View File

@ -94,5 +94,12 @@ FFState* PhraseBoundaryFeature::Evaluate
return new PhraseBoundaryState(endSourceWord,endTargetWord);
}
void PhraseBoundaryFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

5
moses/PhraseBoundaryFeature.h
View File

@ -46,7 +46,10 @@ public:
{
abort();
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }

7
moses/PhraseLengthFeature.cpp
View File

@ -40,4 +40,11 @@ void PhraseLengthFeature::Evaluate(
//cerr << nameSource.str() << " " << nameTarget.str() << " " << nameBoth.str() << endl;
}
void PhraseLengthFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

4
moses/PhraseLengthFeature.h
View File

@ -27,6 +27,10 @@ public:
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
};
}

7
moses/PhrasePairFeature.cpp
View File

@ -244,4 +244,11 @@ void PhrasePairFeature::Evaluate(
}
}
void PhrasePairFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

4
moses/PhrasePairFeature.h
View File

@ -41,6 +41,10 @@ class PhrasePairFeature: public StatelessFeatureFunction {
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/);
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }

5
moses/ScoreComponentCollectionTest.cpp
View File

@ -36,6 +36,10 @@ class MockStatelessFeatureFunction : public StatelessFeatureFunction {
StatelessFeatureFunction(desc,n, line) {}
virtual void Evaluate(const PhraseBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void EvaluateChart(const ChartBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{ }
};
class MockSingleFeature : public MockStatelessFeatureFunction {
@ -54,6 +58,7 @@ class MockSparseFeature : public MockStatelessFeatureFunction {
};
struct MockProducers {
MockProducers() {}

7
moses/SourceWordDeletionFeature.cpp
View File

@ -80,6 +80,13 @@ void SourceWordDeletionFeature::EvaluateChart(
ComputeFeatures(context.GetTargetPhrase(), accumulator, alignmentInfo);
}
void SourceWordDeletionFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const

4
moses/SourceWordDeletionFeature.h
View File

@ -29,6 +29,10 @@ public:
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;

8
moses/TargetBigramFeature.cpp
View File

@ -111,5 +111,13 @@ FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
}
return new TargetBigramState(targetPhrase.GetWord(targetPhrase.GetSize()-1));
}
void TargetBigramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

5
moses/TargetBigramFeature.h
View File

@ -41,6 +41,11 @@ public:
{
abort();
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
private:
FactorType m_factorType;
Word m_bos;

7
moses/TargetNgramFeature.cpp
View File

@ -394,6 +394,13 @@ FFState* TargetNgramFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int
return new TargetNgramChartState(cur_hypo, featureId, m_n);
}
void TargetNgramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const {
stringstream ngram;
size_t size = contextFactor.size();

4
moses/TargetNgramFeature.h
View File

@ -195,6 +195,10 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
private:
FactorType m_factorType;
Word m_bos;

14
moses/TargetPhrase.cpp
View File

@ -78,7 +78,21 @@ void TargetPhrase::WriteToRulePB(hgmert::Rule* pb) const
}
#endif
void TargetPhrase::Evaluate()
{
float totalEstFutureScore = 0;
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
if (!ff.IsDecodeFeature()) {
float estimatedFutureScore = 0;
ff.Evaluate(*this, m_scoreBreakdown, estimatedFutureScore);
totalEstFutureScore += estimatedFutureScore;
}
}
}
void TargetPhrase::SetScore(float score)
{

2
moses/TargetPhrase.h
View File

@ -61,6 +61,8 @@ public:
explicit TargetPhrase(std::string out_string);
explicit TargetPhrase(const Phrase &targetPhrase);
void Evaluate();
//!Set score for Sentence XML target options
void SetScore(float score);

7
moses/TargetWordInsertionFeature.cpp
View File

@ -84,6 +84,13 @@ void TargetWordInsertionFeature::EvaluateChart(
ComputeFeatures(targetPhrase, accumulator, alignmentInfo);
}
void TargetWordInsertionFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const

4
moses/TargetWordInsertionFeature.h
View File

@ -29,6 +29,10 @@ public:
void EvaluateChart( const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;

7
moses/WordTranslationFeature.cpp
View File

@ -528,4 +528,11 @@ void WordTranslationFeature::EvaluateChart(
}
void WordTranslationFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const
{
CHECK(false);
}
}

4
moses/WordTranslationFeature.h
View File

@ -50,6 +50,10 @@ public:
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, float &estimatedFutureScore) const;
void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; }
float GetSparseProducerWeight() const { return m_sparseProducerWeight; }