From 8833098925a9de6e444edd4e53d5729ad565f262 Mon Sep 17 00:00:00 2001 From: phkoehn Date: Tue, 26 May 2009 19:30:35 +0000 Subject: [PATCH] generalized n-best list reporting for feature functions, added experimental version of global lexical model git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@2343 1f5c12ca-751b-0410-a591-d2e778427230 --- moses-cmd/src/IOWrapper.cpp | 141 +++++++++++------- moses/src/DummyScoreProducers.cpp | 15 ++ moses/src/DummyScoreProducers.h | 3 + moses/src/GenerationDictionary.h | 4 + moses/src/LanguageModel.h | 5 + moses/src/LexicalReordering.h | 5 + moses/src/Makefile.am | 1 + moses/src/Manager.cpp | 2 +- moses/src/Parameter.cpp | 2 + moses/src/PhraseDictionary.h | 4 + moses/src/PhraseDictionaryTree.h | 4 + moses/src/PhraseDictionaryTreeAdaptor.h | 4 + moses/src/ScoreProducer.h | 3 + moses/src/Sentence.cpp | 2 +- moses/src/StaticData.cpp | 121 +++++++++------ moses/src/StaticData.h | 3 + moses/src/TargetPhraseCollection.h | 3 +- moses/src/TranslationOptionCollectionText.cpp | 1 - .../lattice-distortion/truth/results.txt | 72 ++++----- .../nbest-multi-factor/truth/results.txt | 4 +- scripts/training/mert-moses-new.pl | 8 +- scripts/training/mert-moses.pl | 7 +- 22 files changed, 265 insertions(+), 149 deletions(-) diff --git a/moses-cmd/src/IOWrapper.cpp b/moses-cmd/src/IOWrapper.cpp index 0e9121332..1f2e4059b 100755 --- a/moses-cmd/src/IOWrapper.cpp +++ b/moses-cmd/src/IOWrapper.cpp @@ -320,9 +320,10 @@ void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, b void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translationId) { - bool labeledOutput = StaticData::Instance().IsLabeledNBestList(); - bool includeAlignment = StaticData::Instance().NBestIncludesAlignment(); - bool includeWordAlignment = StaticData::Instance().PrintAlignmentInfoInNbest(); + const StaticData &staticData = StaticData::Instance(); + bool labeledOutput = staticData.IsLabeledNBestList(); + bool includeAlignment = staticData.NBestIncludesAlignment(); + bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest(); TrellisPathList::const_iterator iter; for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) @@ -337,55 +338,88 @@ void IOWrapper::OutputNBestList(const TrellisPathList &nBestList, long translati const Hypothesis &edge = *edges[currEdge]; OutputSurface(*m_nBestStream, edge.GetCurrTargetPhrase(), m_outputFactorOrder, false); // false for not reporting all factors } - *m_nBestStream << " ||| "; + *m_nBestStream << " |||"; + + std::string lastName = ""; + const vector& sff = + staticData.GetScoreIndexManager().GetStatefulFeatureFunctions(); + for( size_t i=0; iGetScoreProducerWeightShortName() ) + { + lastName = sff[i]->GetScoreProducerWeightShortName(); + *m_nBestStream << " " << lastName << ":"; + } + vector scores = path.GetScoreBreakdown().GetScoresForProducer( sff[i] ); + for (size_t j = 0; j& slf = + staticData.GetScoreIndexManager().GetStatelessFeatureFunctions(); + for( size_t i=0; iGetScoreProducerWeightShortName() ) + { + lastName = slf[i]->GetScoreProducerWeightShortName(); + *m_nBestStream << " " << lastName << ":"; + } + vector scores = path.GetScoreBreakdown().GetScoresForProducer( slf[i] ); + for (size_t j = 0; j rms = StaticData::Instance().GetReorderModels(); - if(rms.size() > 0) - { - vector::iterator iter; - for(iter = rms.begin(); iter != rms.end(); ++iter) - { - vector scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); - for (size_t j = 0; j 0) { - if (labeledOutput) - *m_nBestStream << "lm: "; - LMList::const_iterator lmi = lml.begin(); - for (; lmi != lml.end(); ++lmi) { - *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " "; - } - } - +// vector rms = StaticData::Instance().GetReorderModels(); +// if(rms.size() > 0) +// { +// vector::iterator iter; +// for(iter = rms.begin(); iter != rms.end(); ++iter) +// { +// vector scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); +// for (size_t j = 0; j 0) { +// if (labeledOutput) +// *m_nBestStream << "lm: "; +// LMList::const_iterator lmi = lml.begin(); +// for (; lmi != lml.end(); ++lmi) { +// *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(*lmi) << " "; +// } +// } +// // translation components if (StaticData::Instance().GetInputType()==SentenceInput){ // translation components for text input vector pds = StaticData::Instance().GetPhraseDictionaries(); if (pds.size() > 0) { if (labeledOutput) - *m_nBestStream << "tm: "; + *m_nBestStream << " tm:"; vector::iterator iter; for (iter = pds.begin(); iter != pds.end(); ++iter) { vector scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); for (size_t j = 0; jGetNumInputScores(); if (iter == pds.begin() && labeledOutput) - *m_nBestStream << "tm: "; + *m_nBestStream << " tm:"; for (size_t j = pd_numinputscore; j < scores.size() ; ++j) - *m_nBestStream << scores[j] << " "; + *m_nBestStream << " " << scores[j]; } } } - - - // word penalty - if (labeledOutput) - *m_nBestStream << "w: "; - *m_nBestStream << path.GetScoreBreakdown().GetScoreForProducer(StaticData::Instance().GetWordPenaltyProducer()) << " "; - // generation vector gds = StaticData::Instance().GetGenerationDictionaries(); - if (gds.size() > 0) { + if (gds.size() > 0) { if (labeledOutput) - *m_nBestStream << "g: "; - vector::iterator iter; - for (iter = gds.begin(); iter != gds.end(); ++iter) { - vector scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); - for (size_t j = 0; j::iterator iter; + for (iter = gds.begin(); iter != gds.end(); ++iter) { + vector scores = path.GetScoreBreakdown().GetScoresForProducer(*iter); + for (size_t j = 0; j= 0 ; currEdge--) { diff --git a/moses/src/DummyScoreProducers.cpp b/moses/src/DummyScoreProducers.cpp index 08d1244a8..9fa0c9764 100644 --- a/moses/src/DummyScoreProducers.cpp +++ b/moses/src/DummyScoreProducers.cpp @@ -46,6 +46,11 @@ std::string DistortionScoreProducer::GetScoreProducerDescription() const return "Distortion"; } +std::string DistortionScoreProducer::GetScoreProducerWeightShortName() const +{ + return "d"; +} + float DistortionScoreProducer::CalculateDistortionScore(const WordsRange &prev, const WordsRange &curr, const int FirstGap) const { const int USE_OLD = 1; @@ -106,6 +111,11 @@ std::string WordPenaltyProducer::GetScoreProducerDescription() const return "WordPenalty"; } +std::string WordPenaltyProducer::GetScoreProducerWeightShortName() const +{ + return "w"; +} + size_t WordPenaltyProducer::GetNumInputScores() const { return 0;} void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollection* out) const @@ -128,6 +138,11 @@ std::string UnknownWordPenaltyProducer::GetScoreProducerDescription() const return "!UnknownWordPenalty"; } +std::string UnknownWordPenaltyProducer::GetScoreProducerWeightShortName() const +{ + return "u"; +} + size_t UnknownWordPenaltyProducer::GetNumInputScores() const { return 0;} bool UnknownWordPenaltyProducer::ComputeValueInTranslationOption() const { diff --git a/moses/src/DummyScoreProducers.h b/moses/src/DummyScoreProducers.h index ce810b308..a1f80df57 100644 --- a/moses/src/DummyScoreProducers.h +++ b/moses/src/DummyScoreProducers.h @@ -20,6 +20,7 @@ public: size_t GetNumScoreComponents() const; std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const; size_t GetNumInputScores() const; virtual const FFState* EmptyHypothesisState() const; @@ -40,6 +41,7 @@ public: size_t GetNumScoreComponents() const; std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const; size_t GetNumInputScores() const; virtual void Evaluate( @@ -55,6 +57,7 @@ public: size_t GetNumScoreComponents() const; std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const; size_t GetNumInputScores() const; virtual bool ComputeValueInTranslationOption() const; diff --git a/moses/src/GenerationDictionary.h b/moses/src/GenerationDictionary.h index 3a6ebe759..b06185b97 100644 --- a/moses/src/GenerationDictionary.h +++ b/moses/src/GenerationDictionary.h @@ -71,6 +71,10 @@ public: size_t GetNumScoreComponents() const; std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const + { + return "g"; + } /** number of unique input entries in the generation table. * NOT the number of lines in the generation table diff --git a/moses/src/LanguageModel.h b/moses/src/LanguageModel.h index c55211939..d2745aa61 100644 --- a/moses/src/LanguageModel.h +++ b/moses/src/LanguageModel.h @@ -121,6 +121,11 @@ public: } virtual std::string GetScoreProducerDescription() const = 0; + + std::string GetScoreProducerWeightShortName() const + { + return "lm"; + } //! overrideable funtions for IRST LM to cleanup. Maybe something to do with on demand/cache loading/unloading virtual void InitializeBeforeSentenceProcessing(){}; diff --git a/moses/src/LexicalReordering.h b/moses/src/LexicalReordering.h index eccf2fdac..4526a561a 100644 --- a/moses/src/LexicalReordering.h +++ b/moses/src/LexicalReordering.h @@ -54,6 +54,11 @@ class LexicalReordering : public StatefulFeatureFunction { virtual std::string GetScoreProducerDescription() const { return "Generic Lexical Reordering Model... overwrite in subclass."; }; + + std::string GetScoreProducerWeightShortName() const { + return "d"; + }; + //new virtual int GetNumOrientationTypes() const = 0; virtual OrientationType GetOrientationType(Hypothesis*) const = 0; diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am index cc9975ae6..10b9d6f93 100644 --- a/moses/src/Makefile.am +++ b/moses/src/Makefile.am @@ -19,6 +19,7 @@ libmoses_a_SOURCES = \ FFState.cpp \ FloydWarshall.cpp \ GenerationDictionary.cpp \ + GlobalLexicalModel.cpp \ hash.cpp \ Hypothesis.cpp \ HypothesisStack.cpp \ diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp index 3922795e0..abfceaa60 100644 --- a/moses/src/Manager.cpp +++ b/moses/src/Manager.cpp @@ -62,7 +62,7 @@ Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm) Manager::~Manager() { - delete m_transOptColl; + delete m_transOptColl; delete m_search; StaticData::Instance().CleanUpAfterSentenceProcessing(); diff --git a/moses/src/Parameter.cpp b/moses/src/Parameter.cpp index 456957284..c38e6963b 100644 --- a/moses/src/Parameter.cpp +++ b/moses/src/Parameter.cpp @@ -44,6 +44,7 @@ Parameter::Parameter() AddParam("drop-unknown", "du", "drop unknown words instead of copying them"); AddParam("factor-delimiter", "fd", "specify a different factor delimiter than the default"); AddParam("generation-file", "location and properties of the generation table"); + AddParam("global-lexical-file", "gl", "discriminatively trained global lexical translation model file"); AddParam("input-factors", "list of factors in the input"); AddParam("input-file", "i", "location of the input file to be translated"); AddParam("inputtype", "text (0), confusion network (1), word lattice (2) (default = 0)"); @@ -74,6 +75,7 @@ Parameter::Parameter() AddParam("weight-generation", "g", "weight(s) for generation components"); AddParam("weight-i", "I", "weight(s) for word insertion - used for parameters from confusion network and lattice input links"); AddParam("weight-l", "lm", "weight(s) for language models"); + AddParam("weight-lex", "lex", "weight for global lexical model"); AddParam("weight-t", "tm", "weights for translation model components"); AddParam("weight-w", "w", "weight for word penalty"); AddParam("weight-u", "u", "weight for unknown word penalty"); diff --git a/moses/src/PhraseDictionary.h b/moses/src/PhraseDictionary.h index 8a045b0a0..3dcb3ae0c 100644 --- a/moses/src/PhraseDictionary.h +++ b/moses/src/PhraseDictionary.h @@ -58,6 +58,10 @@ class PhraseDictionary : public Dictionary, public StatelessFeatureFunction //! Overriden by load on demand phrase tables classes to load data for each input virtual void InitializeForInput(InputType const &/*source*/) {} std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const + { + return "tm"; + } size_t GetNumScoreComponents() const; size_t GetNumInputScores() const; diff --git a/moses/src/PhraseDictionaryTree.h b/moses/src/PhraseDictionaryTree.h index 50d255f3a..da708c372 100644 --- a/moses/src/PhraseDictionaryTree.h +++ b/moses/src/PhraseDictionaryTree.h @@ -116,6 +116,10 @@ public: // for debugging void PrintTargetCandidates(PrefixPtr p,std::ostream& out) const; std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const + { + return "tm"; + } }; diff --git a/moses/src/PhraseDictionaryTreeAdaptor.h b/moses/src/PhraseDictionaryTreeAdaptor.h index bec2a059d..4d161b0c7 100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.h +++ b/moses/src/PhraseDictionaryTreeAdaptor.h @@ -67,6 +67,10 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionary { void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase); std::string GetScoreProducerDescription() const; + std::string GetScoreProducerWeightShortName() const + { + return "tm"; + } size_t GetNumInputScores() const; diff --git a/moses/src/ScoreProducer.h b/moses/src/ScoreProducer.h index f0f1720d1..811ff1962 100644 --- a/moses/src/ScoreProducer.h +++ b/moses/src/ScoreProducer.h @@ -48,6 +48,9 @@ public: //! returns a string description of this producer virtual std::string GetScoreProducerDescription() const = 0; + //! returns the weight parameter name of this producer (used in n-best list) + virtual std::string GetScoreProducerWeightShortName() const = 0; + //! returns the number of scores gathered from the input (0 by default) virtual size_t GetNumInputScores() const { return 0; }; diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp index e3139480c..7c7c53793 100644 --- a/moses/src/Sentence.cpp +++ b/moses/src/Sentence.cpp @@ -148,7 +148,7 @@ bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const { { if (pos < m_xmlCoverageMap.size() && m_xmlCoverageMap[pos]) { return true; - } + } } return false; } diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp index d66aa592e..c4ccc8296 100644 --- a/moses/src/StaticData.cpp +++ b/moses/src/StaticData.cpp @@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "LanguageModelMultiFactor.h" #include "LanguageModelFactory.h" #include "LexicalReordering.h" +#include "GlobalLexicalModel.h" #include "SentenceStats.h" #include "PhraseDictionaryTreeAdaptor.h" #include "UserMessage.h" @@ -263,7 +264,7 @@ bool StaticData::LoadData(Parameter *parameter) m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(m_scoreIndexManager); m_allWeights.push_back(m_weightUnknownWord); - // reordering constraints + // reordering constraints m_maxDistortion = (m_parameter->GetParam("distortion-limit").size() > 0) ? Scan(m_parameter->GetParam("distortion-limit")[0]) : -1; @@ -272,28 +273,28 @@ bool StaticData::LoadData(Parameter *parameter) // settings for pruning m_maxHypoStackSize = (m_parameter->GetParam("stack").size() > 0) ? Scan(m_parameter->GetParam("stack")[0]) : DEFAULT_MAX_HYPOSTACK_SIZE; - m_minHypoStackDiversity = 0; - if (m_parameter->GetParam("stack-diversity").size() > 0) { - if (m_maxDistortion > 15) { - UserMessage::Add("stack diversity > 0 is not allowed for distortion limits larger than 15"); - return false; - } - if (m_inputType == WordLatticeInput) { - UserMessage::Add("stack diversity > 0 is not allowed for lattice input"); - return false; - } - m_minHypoStackDiversity = Scan(m_parameter->GetParam("stack-diversity")[0]); - } + m_minHypoStackDiversity = 0; + if (m_parameter->GetParam("stack-diversity").size() > 0) { + if (m_maxDistortion > 15) { + UserMessage::Add("stack diversity > 0 is not allowed for distortion limits larger than 15"); + return false; + } + if (m_inputType == WordLatticeInput) { + UserMessage::Add("stack diversity > 0 is not allowed for lattice input"); + return false; + } + m_minHypoStackDiversity = Scan(m_parameter->GetParam("stack-diversity")[0]); + } m_beamWidth = (m_parameter->GetParam("beam-threshold").size() > 0) ? TransformScore(Scan(m_parameter->GetParam("beam-threshold")[0])) : TransformScore(DEFAULT_BEAM_WIDTH); - m_earlyDiscardingThreshold = (m_parameter->GetParam("early-discarding-threshold").size() > 0) ? - TransformScore(Scan(m_parameter->GetParam("early-discarding-threshold")[0])) - : TransformScore(DEFAULT_EARLY_DISCARDING_THRESHOLD); - m_translationOptionThreshold = (m_parameter->GetParam("translation-option-threshold").size() > 0) ? - TransformScore(Scan(m_parameter->GetParam("translation-option-threshold")[0])) - : TransformScore(DEFAULT_TRANSLATION_OPTION_THRESHOLD); + m_earlyDiscardingThreshold = (m_parameter->GetParam("early-discarding-threshold").size() > 0) ? + TransformScore(Scan(m_parameter->GetParam("early-discarding-threshold")[0])) + : TransformScore(DEFAULT_EARLY_DISCARDING_THRESHOLD); + m_translationOptionThreshold = (m_parameter->GetParam("translation-option-threshold").size() > 0) ? + TransformScore(Scan(m_parameter->GetParam("translation-option-threshold")[0])) + : TransformScore(DEFAULT_TRANSLATION_OPTION_THRESHOLD); m_maxNoTransOptPerCoverage = (m_parameter->GetParam("max-trans-opt-per-coverage").size() > 0) ? Scan(m_parameter->GetParam("max-trans-opt-per-coverage")[0]) : DEFAULT_MAX_TRANS_OPT_SIZE; @@ -310,8 +311,7 @@ bool StaticData::LoadData(Parameter *parameter) m_cubePruningDiversity = (m_parameter->GetParam("cube-pruning-diversity").size() > 0) ? Scan(m_parameter->GetParam("cube-pruning-diversity")[0]) : DEFAULT_CUBE_PRUNING_DIVERSITY; - // Unknown Word Processing -- wade - //TODO replace this w/general word dropping -- EVH + // unknown word processing SetBooleanParameter( &m_dropUnknown, "drop-unknown", false ); // minimum Bayes risk decoding @@ -321,7 +321,7 @@ bool StaticData::LoadData(Parameter *parameter) m_mbrScale = (m_parameter->GetParam("mbr-scale").size() > 0) ? Scan(m_parameter->GetParam("mbr-scale")[0]) : 1.0f; - m_timeout_threshold = (m_parameter->GetParam("time-out").size() > 0) ? + m_timeout_threshold = (m_parameter->GetParam("time-out").size() > 0) ? Scan(m_parameter->GetParam("time-out")[0]) : -1; m_timeout = (GetTimeoutThreshold() == -1) ? false : true; @@ -374,13 +374,14 @@ bool StaticData::LoadData(Parameter *parameter) if (!LoadGenerationTables()) return false; if (!LoadPhraseTables()) return false; if (!LoadMapping()) return false; + if (!LoadGlobalLexicalModel()) return false; - m_scoreIndexManager.InitFeatureNames(); + m_scoreIndexManager.InitFeatureNames(); if (m_parameter->GetParam("weight-file").size() > 0) { - if (m_parameter->GetParam("weight-file").size() != 1) { - UserMessage::Add(string("ERROR: weight-file takes a single parameter")); - return false; - } + if (m_parameter->GetParam("weight-file").size() != 1) { + UserMessage::Add(string("ERROR: weight-file takes a single parameter")); + return false; + } string fnam = m_parameter->GetParam("weight-file")[0]; m_scoreIndexManager.InitWeightVectorFromFile(fnam, &m_allWeights); } @@ -419,6 +420,7 @@ StaticData::~StaticData() RemoveAllInColl(m_languageModel); RemoveAllInColl(m_decodeStepVL); RemoveAllInColl(m_reorderModels); + RemoveAllInColl(m_globalLexicalModels); // delete trans opt map, std::pair< TranslationOptionList*, clock_t > >::iterator iterCache; @@ -443,11 +445,7 @@ bool StaticData::LoadLexicalReorderingModel() std::cerr << "Loading lexical distortion models...\n"; const vector fileStr = m_parameter->GetParam("distortion-file"); const vector weightsStr = m_parameter->GetParam("weight-d"); - /*old code - const vector modelStr = m_parameter.GetParam("distortion-type"); //TODO check name? - const vector fileStr = m_parameter.GetParam("distortion-file"); - const vector weightsStr = m_parameter.GetParam("weight-d"); - */ + std::vector weights; size_t w = 1; //cur weight size_t f = 0; //cur file @@ -459,8 +457,6 @@ bool StaticData::LoadLexicalReorderingModel() //load all models for(size_t i = 0; i < fileStr.size(); ++i) { - //std::cerr << "Model " << i << ":"; - //Todo: 'else' should be 'else if(...)' to check it is a lexical model... vector spec = Tokenize(fileStr[f], " "); ++f; //mark file as consumed if(4 != spec.size()){ @@ -594,6 +590,39 @@ bool StaticData::LoadLexicalReorderingModel() return true; } +bool StaticData::LoadGlobalLexicalModel() +{ + const vector &weight = Scan(m_parameter->GetParam("weight-lex")); + const vector &file = m_parameter->GetParam("global-lexical-file"); + + if (weight.size() != file.size()) + { + std::cerr << "number of weights and models for the global lexical model does not match (" + << weight.size() << " != " << file.size() << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < weight.size(); i++ ) + { + vector spec = Tokenize(file[i], " "); + if ( spec.size() != 2 ) + { + std::cerr << "wrong global lexical model specification: " << file[i] << endl; + return false; + } + vector< string > factors = Tokenize(spec[0],"-"); + if ( factors.size() != 2 ) + { + std::cerr << "wrong factor definition for global lexical model: " << spec[0] << endl; + return false; + } + vector inputFactors = Tokenize(factors[0],","); + vector outputFactors = Tokenize(factors[1],","); + m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], weight[i], inputFactors, outputFactors ) ); + } + return true; +} + bool StaticData::LoadLanguageModels() { if (m_parameter->GetParam("lmodel-file").size() > 0) @@ -968,21 +997,23 @@ void StaticData::CleanUpAfterSentenceProcessing() const binary format is used) */ void StaticData::InitializeBeforeSentenceProcessing(InputType const& in) const { - m_input = ∈ - for(size_t i=0;iInitializeForInput(in); - } - for(size_t j=0;jInitializeForInput(in); - } - //something LMs could do before translating a sentence - LMList::const_iterator iterLM; + m_input = ∈ + for(size_t i=0;iInitializeForInput(in); + } + for(size_t i=0;iInitializeForInput(in); + } + for(size_t i=0;iInitializeForInput((Sentence const&)in); + } + //something LMs could do before translating a sentence + LMList::const_iterator iterLM; for (iterLM = m_languageModel.begin() ; iterLM != m_languageModel.end() ; ++iterLM) { LanguageModel &languageModel = **iterLM; - languageModel.InitializeBeforeSentenceProcessing(); + languageModel.InitializeBeforeSentenceProcessing(); } - } void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector& weights) diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h index 6320cab18..9b5dcb754 100644 --- a/moses/src/StaticData.h +++ b/moses/src/StaticData.h @@ -45,6 +45,7 @@ namespace Moses class InputType; class LexicalReordering; +class GlobalLexicalModel; class PhraseDictionary; class GenerationDictionary; class DistortionScoreProducer; @@ -69,6 +70,7 @@ protected: ScoreIndexManager m_scoreIndexManager; std::vector m_allWeights; std::vector m_reorderModels; + std::vector m_globalLexicalModels; // Initial = 0 = can be used when creating poss trans // Other = 1 = used to calculate LM score once all steps have been processed float @@ -176,6 +178,7 @@ protected: //! load decoding steps bool LoadMapping(); bool LoadLexicalReorderingModel(); + bool LoadGlobalLexicalModel(); public: diff --git a/moses/src/TargetPhraseCollection.h b/moses/src/TargetPhraseCollection.h index 7189d13ce..1ea2e7141 100644 --- a/moses/src/TargetPhraseCollection.h +++ b/moses/src/TargetPhraseCollection.h @@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -//! a list of target phrases that is trsnalated from the same source phrase +//! a list of target phrases that is translated from the same source phrase class TargetPhraseCollection { protected: @@ -72,4 +72,3 @@ public: } - diff --git a/moses/src/TranslationOptionCollectionText.cpp b/moses/src/TranslationOptionCollectionText.cpp index 92ae6420f..bbbc0aaae 100644 --- a/moses/src/TranslationOptionCollectionText.cpp +++ b/moses/src/TranslationOptionCollectionText.cpp @@ -51,7 +51,6 @@ void TranslationOptionCollectionText::ProcessUnknownWord(size_t sourcePos) bool TranslationOptionCollectionText::HasXmlOptionsOverlappingRange(size_t startPosition, size_t endPosition) const { Sentence const& source=dynamic_cast(m_source); return source.XmlOverlap(startPosition,endPosition); - } /** diff --git a/regression-testing/tests/lattice-distortion/truth/results.txt b/regression-testing/tests/lattice-distortion/truth/results.txt index 83474c538..674c093ff 100644 --- a/regression-testing/tests/lattice-distortion/truth/results.txt +++ b/regression-testing/tests/lattice-distortion/truth/results.txt @@ -14,40 +14,40 @@ SCORE_4 = 2.004 SCORE_5 = -0.233 SCORE_6 = -1.851 SCORE_7 = -196.549 -TRANSLATION_0_NBEST_1=1 2 ||| d: 0 lm: -4.30484 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_0_NBEST_2=2 1 ||| d: -3 lm: -9.70915 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_1_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_1_NBEST_2=12 3 ||| d: 0 lm: -7.98575 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_1_NBEST_3=2 3 1 ||| d: -2 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_1_NBEST_4=1 3 2 ||| d: -2 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_1_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_1_NBEST_6=2 1 3 ||| d: -1 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_1_NBEST_7=3 12 ||| d: -3 lm: -10.0318 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_1_NBEST_8=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_2=2 3 1 ||| d: -2 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_3=1 3 2 ||| d: -2 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_4=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_5=2 1 3 ||| d: -1 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_2_NBEST_6=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_2=1 3 2 ||| d: -1 lm: -10.7767 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_3=1 12 ||| d: 0 lm: -9.28642 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_3_NBEST_4=2 3 1 ||| d: -3 lm: -9.26988 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_6=12 1 ||| d: -3 lm: -8.59503 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_3_NBEST_7=3 2 1 ||| d: -4 lm: -12.57 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_3_NBEST_8=2 1 3 ||| d: -3 lm: -13.645 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_1=1234 3 4 5 ||| d: -3 lm: -8.02249 I: 0 tm: -9.21034 -9.21034 w: -4 -TRANSLATION_4_NBEST_2=4 1234 5 ||| d: -2 lm: -7.07413 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_3=5 4 1234 ||| d: -1 lm: -8.17776 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_4=3 4 5 1234 ||| d: 0 lm: -12.8014 I: 0 tm: -9.21034 -9.21034 w: -4 -TRANSLATION_4_NBEST_5=2 3 1234 ||| d: 0 lm: -10.4352 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_6=1234 4 5 ||| d: -3 lm: -7.84275 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_7=3 4 1234 5 ||| d: -2 lm: -11.6793 I: 0 tm: -9.21034 -9.21034 w: -4 -TRANSLATION_4_NBEST_8=1234 5 4 ||| d: -4 lm: -6.99714 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_4_NBEST_9=1 1234 ||| d: 0 lm: -8.4748 I: 0 tm: -4.60517 -4.60517 w: -2 -TRANSLATION_4_NBEST_10=4 5 1234 ||| d: 0 lm: -12.0545 I: 0 tm: -6.90776 -6.90776 w: -3 -TRANSLATION_5_NBEST_1=1 ||| d: 0 lm: -4.54836 I: 0 tm: -2.30259 -2.30259 w: -1 -TRANSLATION_6_NBEST_1=' ||| d: 0 lm: -101.55 I: 0 tm: 0 0 w: -1 +TRANSLATION_0_NBEST_1=1 2 ||| d: 0 lm: -4.30484 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_0_NBEST_2=2 1 ||| d: -3 lm: -9.70915 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_1_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_1_NBEST_2=12 3 ||| d: 0 lm: -7.98575 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_1_NBEST_3=2 3 1 ||| d: -2 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_1_NBEST_4=1 3 2 ||| d: -2 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_1_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_1_NBEST_6=2 1 3 ||| d: -1 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_1_NBEST_7=3 12 ||| d: -3 lm: -10.0318 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_1_NBEST_8=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_2=2 3 1 ||| d: -2 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_3=1 3 2 ||| d: -2 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_4=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_5=2 1 3 ||| d: -1 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_2_NBEST_6=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_1=1 2 3 ||| d: 0 lm: -6.08823 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_2=1 3 2 ||| d: -1 lm: -10.7767 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_3=1 12 ||| d: 0 lm: -9.28642 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_3_NBEST_4=2 3 1 ||| d: -3 lm: -9.26988 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_5=3 1 2 ||| d: -3 lm: -10.9714 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_6=12 1 ||| d: -3 lm: -8.59503 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_3_NBEST_7=3 2 1 ||| d: -4 lm: -12.57 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_3_NBEST_8=2 1 3 ||| d: -3 lm: -13.645 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_1=1234 3 4 5 ||| d: -3 lm: -8.02249 w: -4 I: 0 tm: -9.21034 -9.21034 +TRANSLATION_4_NBEST_2=4 1234 5 ||| d: -2 lm: -7.07413 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_3=5 4 1234 ||| d: -1 lm: -8.17776 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_4=3 4 5 1234 ||| d: 0 lm: -12.8014 w: -4 I: 0 tm: -9.21034 -9.21034 +TRANSLATION_4_NBEST_5=2 3 1234 ||| d: 0 lm: -10.4352 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_6=1234 4 5 ||| d: -3 lm: -7.84275 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_7=3 4 1234 5 ||| d: -2 lm: -11.6793 w: -4 I: 0 tm: -9.21034 -9.21034 +TRANSLATION_4_NBEST_8=1234 5 4 ||| d: -4 lm: -6.99714 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_4_NBEST_9=1 1234 ||| d: 0 lm: -8.4748 w: -2 I: 0 tm: -4.60517 -4.60517 +TRANSLATION_4_NBEST_10=4 5 1234 ||| d: 0 lm: -12.0545 w: -3 I: 0 tm: -6.90776 -6.90776 +TRANSLATION_5_NBEST_1=1 ||| d: 0 lm: -4.54836 w: -1 I: 0 tm: -2.30259 -2.30259 +TRANSLATION_6_NBEST_1=' ||| d: 0 lm: -101.55 w: -1 I: 0 tm: 0 0 TOTAL_WALLTIME ~ 0 diff --git a/regression-testing/tests/nbest-multi-factor/truth/results.txt b/regression-testing/tests/nbest-multi-factor/truth/results.txt index ff3d742ed..a595f78e1 100644 --- a/regression-testing/tests/nbest-multi-factor/truth/results.txt +++ b/regression-testing/tests/nbest-multi-factor/truth/results.txt @@ -2,6 +2,6 @@ TRANSLATION_0=there are various different opinions . LMLOAD_TIME ~ 11.00 PTLOAD_TIME ~ 23.00 SCORE_0= -113.734 -TRANSLATION_0_NBEST_1=there are various different opinions . ||| d: 0 lm: -21.6664 tm: -3.46226 -10.822 -2.82666 -9.32004 2.99969 -2.44147 -4.91188 -6.12017 -10.5033 2.99969 w: -6 g: -0.0625204 -18.098 -TRANSLATION_0_NBEST_2=there are various other opinions . ||| d: 0 lm: -25.3276 tm: -1.94591 -10.3833 -0.693147 -7.77959 1.99979 -0.924822 -4.91188 -3.33314 -10.5033 1.99979 w: -6 g: -0.149241 -16.9924 +TRANSLATION_0_NBEST_1=there are various different opinions . ||| d: 0 lm: -21.6664 w: -6 tm: -3.46226 -10.822 -2.82666 -9.32004 2.99969 -2.44147 -4.91188 -6.12017 -10.5033 2.99969 g: -0.0625204 -18.098 +TRANSLATION_0_NBEST_2=there are various other opinions . ||| d: 0 lm: -25.3276 w: -6 tm: -1.94591 -10.3833 -0.693147 -7.77959 1.99979 -0.924822 -4.91188 -3.33314 -10.5033 1.99979 g: -0.149241 -16.9924 TOTAL_WALLTIME ~ 23 diff --git a/scripts/training/mert-moses-new.pl b/scripts/training/mert-moses-new.pl index fe929046c..0bab4fed4 100755 --- a/scripts/training/mert-moses-new.pl +++ b/scripts/training/mert-moses-new.pl @@ -73,18 +73,19 @@ my $additional_triples = { [ 0.3, 0.0, 0.5 ], [ 0.2, 0.0, 0.5 ], [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty + "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model }; # moses.ini file uses FULL names for lambdas, while this training script internally (and on the command line) # uses ABBR names. -my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation"; +my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation lex=weight-lex"; my %ABBR2FULL = map {split/=/,$_,2} split /\s+/, $ABBR_FULL_MAP; my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} split /\s+/, $ABBR_FULL_MAP; # We parse moses.ini to figure out how many weights do we need to optimize. # For this, we must know the correspondence between options defining files # for models and options assigning weights to these models. -my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g"; +my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g global-lexical-file=lex"; my %TABLECONFIG2ABBR = map {split(/=/,$_,2)} split /\s+/, $TABLECONFIG_ABBR_MAP; # There are weights that do not correspond to any input file, they just increase the total number of lambdas we optimize @@ -415,6 +416,7 @@ if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) / || $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) / || $___DECODER_FLAGS =~ /(^|\s)-(generation-file) / || $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) / ) { die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files."; } @@ -1026,6 +1028,7 @@ sub scan_config { "generation-file" => 3, "lmodel-file" => 3, "distortion-file" => 3, + "global-lexical-file" => 1, ); # by default, each line of each section means one lambda, but some sections # explicitly state a custom number of lambdas @@ -1055,6 +1058,7 @@ sub scan_config { $defined_steps{$1}++ if /^([TG])/ || /^\d+ ([TG])/; } if (defined $section && defined $where_is_filename{$section}) { + print "$section -> $where_is_filename{$section}\n"; # this ini section is relevant to lambdas chomp; my @flds = split / +/; diff --git a/scripts/training/mert-moses.pl b/scripts/training/mert-moses.pl index 6c1019c72..158e9bb86 100755 --- a/scripts/training/mert-moses.pl +++ b/scripts/training/mert-moses.pl @@ -83,18 +83,19 @@ my $additional_triples = { [ 0.3, 0.0, 0.5 ], [ 0.2, 0.0, 0.5 ], [ 0.0,-1.0, 1.0 ] ], # ... last weight is phrase penalty + "lex"=> [ [ 0.1, 0.0, 0.2 ] ], # global lexical model }; # moses.ini file uses FULL names for lambdas, while this training script internally (and on the command line) # uses ABBR names. -my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation"; +my $ABBR_FULL_MAP = "d=weight-d lm=weight-l tm=weight-t w=weight-w g=weight-generation lex=weight-lex"; my %ABBR2FULL = map {split/=/,$_,2} split /\s+/, $ABBR_FULL_MAP; my %FULL2ABBR = map {my ($a, $b) = split/=/,$_,2; ($b, $a);} split /\s+/, $ABBR_FULL_MAP; # We parse moses.ini to figure out how many weights do we need to optimize. # For this, we must know the correspondence between options defining files # for models and options assigning weights to these models. -my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g"; +my $TABLECONFIG_ABBR_MAP = "ttable-file=tm lmodel-file=lm distortion-file=d generation-file=g global-lexical-file=lex"; my %TABLECONFIG2ABBR = map {split(/=/,$_,2)} split /\s+/, $TABLECONFIG_ABBR_MAP; # There are weights that do not correspond to any input file, they just increase the total number of lambdas we optimize @@ -407,6 +408,7 @@ if ($___DECODER_FLAGS =~ /(^|\s)-(config|f) / || $___DECODER_FLAGS =~ /(^|\s)-(distortion-file) / || $___DECODER_FLAGS =~ /(^|\s)-(generation-file) / || $___DECODER_FLAGS =~ /(^|\s)-(lmodel-file) / +|| $___DECODER_FLAGS =~ /(^|\s)-(global-lexical-file) / ) { die "It is forbidden to supply any of -config, -ttable-file, -distortion-file, -generation-file or -lmodel-file in the --decoder-flags.\nPlease use only the --config option to give the config file that lists all the supplementary files."; } @@ -1084,6 +1086,7 @@ sub scan_config { "generation-file" => 3, "lmodel-file" => 3, "distortion-file" => 3, + "global-lexical-file" => 1, ); # by default, each line of each section means one lambda, but some sections # explicitly state a custom number of lambdas