moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-06 19:49:41 +03:00
Code Issues Packages Projects Releases Wiki Activity

Introduce parameter --increase-BP

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mira-mtm5@3742 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
evahasler 2010-11-29 14:40:15 +00:00
parent 2feabd94fc
commit 4af767af60
5 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mira/Decoder.cpp
View File

@ -66,14 +66,14 @@ namespace Mira {
delete[] mosesargv; delete[] mosesargv;
} }
MosesDecoder::MosesDecoder(const vector<vector<string> >& refs, bool useScaledReference, bool scaleByInputLength) MosesDecoder::MosesDecoder(const vector<vector<string> >& refs, bool useScaledReference, bool scaleByInputLength, bool increaseBP)
: m_manager(NULL) { : m_manager(NULL) {
// force initialisation of the phrase dictionary // force initialisation of the phrase dictionary
const StaticData &staticData = StaticData::Instance(); const StaticData &staticData = StaticData::Instance();
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
// Add the bleu feature // Add the bleu feature
m_bleuScoreFeature = new BleuScoreFeature(useScaledReference, scaleByInputLength); m_bleuScoreFeature = new BleuScoreFeature(useScaledReference, scaleByInputLength, increaseBP);
(const_cast<TranslationSystem&>(system)).AddFeatureFunction(m_bleuScoreFeature); (const_cast<TranslationSystem&>(system)).AddFeatureFunction(m_bleuScoreFeature);
m_bleuScoreFeature->LoadReferences(refs); m_bleuScoreFeature->LoadReferences(refs);
} }

2
mira/Decoder.h
View File

@ -50,7 +50,7 @@ void initMoses(const std::string& inifile, int debuglevel, int argc=0, char** a
**/ **/
class MosesDecoder { class MosesDecoder {
public: public:
MosesDecoder(const std::vector<std::vector<std::string> >& refs, bool useScaledReference, bool scaleByInputLength); MosesDecoder(const std::vector<std::vector<std::string> >& refs, bool useScaledReference, bool scaleByInputLength, bool increaseBP);
//returns the best sentence //returns the best sentence
std::vector<const Moses::Word*> getNBest(const std::string& source, std::vector<const Moses::Word*> getNBest(const std::string& source,

8
mira/Main.cpp
View File

@ -90,6 +90,7 @@ int main(int argc, char** argv) {
bool accumulateWeights; bool accumulateWeights;
bool useScaledReference; bool useScaledReference;
bool scaleByInputLength; bool scaleByInputLength;
bool increaseBP;
float clipping; float clipping;
bool fixedClipping; bool fixedClipping;
po::options_description desc("Allowed options"); po::options_description desc("Allowed options");
@ -109,11 +110,12 @@ int main(int argc, char** argv) {
("margin-scale-factor,m", po::value<float>(&marginScaleFactor)->default_value(1.0), "Margin scale factor, regularises the update by scaling the enforced margin") ("margin-scale-factor,m", po::value<float>(&marginScaleFactor)->default_value(1.0), "Margin scale factor, regularises the update by scaling the enforced margin")
("nbest,n", po::value<size_t>(&n)->default_value(10), "Number of translations in nbest list") ("nbest,n", po::value<size_t>(&n)->default_value(10), "Number of translations in nbest list")
("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments") ("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments")
("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(0), "Use nbest list with distinct translations in inference step") ("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(false), "Use nbest list with distinct translations in inference step")
("only-violated-constraints", po::value<bool>(&onlyViolatedConstraints)->default_value(false), "Add only violated constraints to the optimisation problem") ("only-violated-constraints", po::value<bool>(&onlyViolatedConstraints)->default_value(false), "Add only violated constraints to the optimisation problem")
("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs") ("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
("use-scaled-reference", po::value<bool>(&useScaledReference)->default_value(true), "Use scaled reference length for comparing target and reference length of phrases") ("use-scaled-reference", po::value<bool>(&useScaledReference)->default_value(true), "Use scaled reference length for comparing target and reference length of phrases")
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale the BLEU score by a history of the input lengths") ("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale the BLEU score by a history of the input lengths")
("increase-BP", po::value<bool>(&increaseBP)->default_value(false), "Increase penalty for short translations")
("clipping", po::value<float>(&clipping)->default_value(0.01f), "Set a clipping threshold for SMO to regularise updates") ("clipping", po::value<float>(&clipping)->default_value(0.01f), "Set a clipping threshold for SMO to regularise updates")
("fixed-clipping", po::value<bool>(&fixedClipping)->default_value(false), "Use a fixed clipping threshold with SMO (instead of adaptive)"); ("fixed-clipping", po::value<bool>(&fixedClipping)->default_value(false), "Use a fixed clipping threshold with SMO (instead of adaptive)");
@ -169,7 +171,7 @@ int main(int argc, char** argv) {
// initialise Moses // initialise Moses
initMoses(mosesConfigFile, verbosity);//, argc, argv); initMoses(mosesConfigFile, verbosity);//, argc, argv);
MosesDecoder* decoder = new MosesDecoder(referenceSentences, useScaledReference, scaleByInputLength); MosesDecoder* decoder = new MosesDecoder(referenceSentences, useScaledReference, scaleByInputLength, increaseBP);
ScoreComponentCollection startWeights = decoder->getWeights(); ScoreComponentCollection startWeights = decoder->getWeights();
startWeights.L1Normalise(); startWeights.L1Normalise();
decoder->setWeights(startWeights); decoder->setWeights(startWeights);
@ -205,7 +207,7 @@ int main(int argc, char** argv) {
Optimiser* optimiser = NULL; Optimiser* optimiser = NULL;
cerr << "Nbest list size: " << n << endl; cerr << "Nbest list size: " << n << endl;
cerr << "Distinct translations in nbest list?: " << distinctNbest << endl; cerr << "Distinct translations in nbest list? " << distinctNbest << endl;
if (learner == "mira") { if (learner == "mira") {
cerr << "Optimising using Mira" << endl; cerr << "Optimising using Mira" << endl;
optimiser = new MiraOptimiser(n, hildreth, marginScaleFactor, onlyViolatedConstraints, clipping, fixedClipping); optimiser = new MiraOptimiser(n, hildreth, marginScaleFactor, onlyViolatedConstraints, clipping, fixedClipping);

29
moses/src/BleuScoreFeature.cpp
View File

@ -79,9 +79,10 @@ BleuScoreFeature::BleuScoreFeature():
m_target_length_history(0), m_target_length_history(0),
m_ref_length_history(0), m_ref_length_history(0),
m_use_scaled_reference(true), m_use_scaled_reference(true),
m_scale_by_input_length(true) {} m_scale_by_input_length(true),
m_increase_BP(false) {}
BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLength): BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLength, bool increaseBP):
StatefulFeatureFunction("BleuScore"), StatefulFeatureFunction("BleuScore"),
m_count_history(BleuScoreState::bleu_order), m_count_history(BleuScoreState::bleu_order),
m_match_history(BleuScoreState::bleu_order), m_match_history(BleuScoreState::bleu_order),
@ -89,7 +90,8 @@ BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLen
m_target_length_history(0), m_target_length_history(0),
m_ref_length_history(0), m_ref_length_history(0),
m_use_scaled_reference(useScaledReference), m_use_scaled_reference(useScaledReference),
m_scale_by_input_length(scaleByInputLength) {} m_scale_by_input_length(scaleByInputLength),
m_increase_BP(increaseBP) {}
void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs) void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs)
{ {
@ -331,7 +333,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_scaled_ref_length) { if (state->m_target_length < state->m_scaled_ref_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length; float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length; float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length)); if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
} }
} }
else { else {
@ -340,7 +347,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_scaled_ref_length) { if (state->m_target_length < state->m_scaled_ref_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length; float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length; float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length)); if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
} }
} }
else { else {
@ -348,7 +360,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_source_phrase_length) { if (state->m_target_length < state->m_source_phrase_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length; float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length; float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length)); if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
} }
} }
} }

5
moses/src/BleuScoreFeature.h
View File

@ -45,7 +45,7 @@ typedef std::map< Phrase, size_t > NGrams;
class BleuScoreFeature : public StatefulFeatureFunction { class BleuScoreFeature : public StatefulFeatureFunction {
public: public:
BleuScoreFeature(); BleuScoreFeature();
BleuScoreFeature(bool useScaledReference, bool scaleByInputLength); BleuScoreFeature(bool useScaledReference, bool scaleByInputLength, bool increaseBP);
std::string GetScoreProducerDescription() const std::string GetScoreProducerDescription() const
{ {
@ -91,6 +91,9 @@ private:
// whether or not to scale the BLEU score by a history of the input size // whether or not to scale the BLEU score by a history of the input size
bool m_scale_by_input_length; bool m_scale_by_input_length;
// increase penalty for short translations
bool m_increase_BP;
// counts for pseudo-document big_O // counts for pseudo-document big_O
std::vector< float > m_count_history; std::vector< float > m_count_history;
std::vector< float > m_match_history; std::vector< float > m_match_history;