moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-11-10 00:47:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Introduce parameter --increase-BP

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mira-mtm5@3742 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
evahasler 2010-11-29 14:40:15 +00:00
parent 2feabd94fc
commit 4af767af60
5 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mira/Decoder.cpp
View File

@ -66,14 +66,14 @@ namespace Mira {
delete[] mosesargv;
}
MosesDecoder::MosesDecoder(const vector<vector<string> >& refs, bool useScaledReference, bool scaleByInputLength)
MosesDecoder::MosesDecoder(const vector<vector<string> >& refs, bool useScaledReference, bool scaleByInputLength, bool increaseBP)
: m_manager(NULL) {
// force initialisation of the phrase dictionary
const StaticData &staticData = StaticData::Instance();
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
// Add the bleu feature
m_bleuScoreFeature = new BleuScoreFeature(useScaledReference, scaleByInputLength);
m_bleuScoreFeature = new BleuScoreFeature(useScaledReference, scaleByInputLength, increaseBP);
(const_cast<TranslationSystem&>(system)).AddFeatureFunction(m_bleuScoreFeature);
m_bleuScoreFeature->LoadReferences(refs);
}

2
mira/Decoder.h
View File

@ -50,7 +50,7 @@ void initMoses(const std::string& inifile, int debuglevel, int argc=0, char** a
**/
class MosesDecoder {
public:
MosesDecoder(const std::vector<std::vector<std::string> >& refs, bool useScaledReference, bool scaleByInputLength);
MosesDecoder(const std::vector<std::vector<std::string> >& refs, bool useScaledReference, bool scaleByInputLength, bool increaseBP);
//returns the best sentence
std::vector<const Moses::Word*> getNBest(const std::string& source,

8
mira/Main.cpp
View File

@ -90,6 +90,7 @@ int main(int argc, char** argv) {
bool accumulateWeights;
bool useScaledReference;
bool scaleByInputLength;
bool increaseBP;
float clipping;
bool fixedClipping;
po::options_description desc("Allowed options");
@ -109,11 +110,12 @@ int main(int argc, char** argv) {
("margin-scale-factor,m", po::value<float>(&marginScaleFactor)->default_value(1.0), "Margin scale factor, regularises the update by scaling the enforced margin")
("nbest,n", po::value<size_t>(&n)->default_value(10), "Number of translations in nbest list")
("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments")
("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(0), "Use nbest list with distinct translations in inference step")
("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(false), "Use nbest list with distinct translations in inference step")
("only-violated-constraints", po::value<bool>(&onlyViolatedConstraints)->default_value(false), "Add only violated constraints to the optimisation problem")
("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
("use-scaled-reference", po::value<bool>(&useScaledReference)->default_value(true), "Use scaled reference length for comparing target and reference length of phrases")
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale the BLEU score by a history of the input lengths")
("increase-BP", po::value<bool>(&increaseBP)->default_value(false), "Increase penalty for short translations")
("clipping", po::value<float>(&clipping)->default_value(0.01f), "Set a clipping threshold for SMO to regularise updates")
("fixed-clipping", po::value<bool>(&fixedClipping)->default_value(false), "Use a fixed clipping threshold with SMO (instead of adaptive)");
@ -169,7 +171,7 @@ int main(int argc, char** argv) {
// initialise Moses
initMoses(mosesConfigFile, verbosity);//, argc, argv);
MosesDecoder* decoder = new MosesDecoder(referenceSentences, useScaledReference, scaleByInputLength);
MosesDecoder* decoder = new MosesDecoder(referenceSentences, useScaledReference, scaleByInputLength, increaseBP);
ScoreComponentCollection startWeights = decoder->getWeights();
startWeights.L1Normalise();
decoder->setWeights(startWeights);
@ -205,7 +207,7 @@ int main(int argc, char** argv) {
Optimiser* optimiser = NULL;
cerr << "Nbest list size: " << n << endl;
cerr << "Distinct translations in nbest list?: " << distinctNbest << endl;
cerr << "Distinct translations in nbest list? " << distinctNbest << endl;
if (learner == "mira") {
cerr << "Optimising using Mira" << endl;
optimiser = new MiraOptimiser(n, hildreth, marginScaleFactor, onlyViolatedConstraints, clipping, fixedClipping);

29
moses/src/BleuScoreFeature.cpp
View File

@ -79,9 +79,10 @@ BleuScoreFeature::BleuScoreFeature():
m_target_length_history(0),
m_ref_length_history(0),
m_use_scaled_reference(true),
m_scale_by_input_length(true) {}
m_scale_by_input_length(true),
m_increase_BP(false) {}
BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLength):
BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLength, bool increaseBP):
StatefulFeatureFunction("BleuScore"),
m_count_history(BleuScoreState::bleu_order),
m_match_history(BleuScoreState::bleu_order),
@ -89,7 +90,8 @@ BleuScoreFeature::BleuScoreFeature(bool useScaledReference, bool scaleByInputLen
m_target_length_history(0),
m_ref_length_history(0),
m_use_scaled_reference(useScaledReference),
m_scale_by_input_length(scaleByInputLength) {}
m_scale_by_input_length(scaleByInputLength),
m_increase_BP(increaseBP) {}
void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs)
{
@ -331,7 +333,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_scaled_ref_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
}
}
else {
@ -340,7 +347,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_scaled_ref_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
}
}
else {
@ -348,7 +360,12 @@ float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const {
if (state->m_target_length < state->m_source_phrase_length) {
float smoothed_target_length = m_target_length_history + state->m_target_length;
float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length;
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
if (m_increase_BP) {
precision *= exp(1 - ((smoothed_ref_length + 1)/ smoothed_target_length));
}
else{
precision *= exp(1 - (smoothed_ref_length / smoothed_target_length));
}
}
}
}

5
moses/src/BleuScoreFeature.h
View File

@ -45,7 +45,7 @@ typedef std::map< Phrase, size_t > NGrams;
class BleuScoreFeature : public StatefulFeatureFunction {
public:
BleuScoreFeature();
BleuScoreFeature(bool useScaledReference, bool scaleByInputLength);
BleuScoreFeature(bool useScaledReference, bool scaleByInputLength, bool increaseBP);
std::string GetScoreProducerDescription() const
{
@ -91,6 +91,9 @@ private:
// whether or not to scale the BLEU score by a history of the input size
bool m_scale_by_input_length;
// increase penalty for short translations
bool m_increase_BP;
// counts for pseudo-document big_O
std::vector< float > m_count_history;
std::vector< float > m_match_history;