From f7e3f200137ea674130a91e84339b50abcd7871f Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 6 Feb 2013 20:05:00 +0000 Subject: [PATCH] each language model should have separate description, rather than just 'LM' --- moses/LM/Base.cpp | 4 ++-- moses/LM/Base.h | 2 +- moses/LM/Implementation.h | 6 +++--- moses/LM/Ken.cpp | 28 +++++++++++++-------------- moses/LM/Ken.h | 4 ++-- moses/Parameter.cpp | 20 +++++++++++++++++++- moses/Parameter.h | 2 ++ moses/StaticData.cpp | 40 +++++++++++++++++++++++++++++++++++++-- moses/StaticData.h | 2 ++ 9 files changed, 83 insertions(+), 25 deletions(-) diff --git a/moses/LM/Base.cpp b/moses/LM/Base.cpp index 1ef6acbc3..373ce557e 100644 --- a/moses/LM/Base.cpp +++ b/moses/LM/Base.cpp @@ -33,8 +33,8 @@ using namespace std; namespace Moses { -LanguageModel::LanguageModel(const std::string &line) : - StatefulFeatureFunction("LM", StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line ) +LanguageModel::LanguageModel(const std::string& description, const std::string &line) : + StatefulFeatureFunction(description, StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line ) { m_enableOOVFeature = StaticData::Instance().GetLMEnableOOVFeature(); } diff --git a/moses/LM/Base.h b/moses/LM/Base.h index 2953296e0..4ec8470f1 100644 --- a/moses/LM/Base.h +++ b/moses/LM/Base.h @@ -39,7 +39,7 @@ class Phrase; //! Abstract base class which represent a language model on a contiguous phrase class LanguageModel : public StatefulFeatureFunction { protected: - LanguageModel(const std::string &line); + LanguageModel(const std::string& description, const std::string &line); // This can't be in the constructor for virual function dispatch reasons diff --git a/moses/LM/Implementation.h b/moses/LM/Implementation.h index 321e19047..cfeb963e1 100644 --- a/moses/LM/Implementation.h +++ b/moses/LM/Implementation.h @@ -121,8 +121,8 @@ public: class LMRefCount : public LanguageModel { public: - LMRefCount(LanguageModelImplementation *impl, const std::string &line) - : LanguageModel(line) + LMRefCount(LanguageModelImplementation *impl, const std::string& description, const std::string &line) + : LanguageModel(description, line) , m_impl(impl) {} LanguageModel *Duplicate() const { @@ -162,7 +162,7 @@ class LMRefCount : public LanguageModel { private: LMRefCount(const LMRefCount ©_from) - : LanguageModel(copy_from.GetArgLine()) + : LanguageModel(copy_from.GetScoreProducerDescription(), copy_from.GetArgLine()) , m_impl(copy_from.m_impl) {} boost::shared_ptr m_impl; diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index 10f699c8d..365895a14 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -63,7 +63,7 @@ struct KenLMState : public FFState { */ template class LanguageModelKen : public LanguageModel { public: - LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy); + LanguageModelKen(const std::string &description, const std::string &line, const std::string &file, FactorType factorType, bool lazy); LanguageModel *Duplicate() const; @@ -138,8 +138,8 @@ private: std::vector &m_mapping; }; -template LanguageModelKen::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy) -:LanguageModel(line) +template LanguageModelKen::LanguageModelKen(const std::string &description, const std::string &line, const std::string &file, FactorType factorType, bool lazy) +:LanguageModel(description, line) ,m_factorType(factorType) { lm::ngram::Config config; @@ -163,7 +163,7 @@ template LanguageModel *LanguageModelKen::Duplicate() const } template LanguageModelKen::LanguageModelKen(const LanguageModelKen ©_from) -:LanguageModel(copy_from.GetArgLine()), +:LanguageModel(copy_from.GetScoreProducerDescription(), copy_from.GetArgLine()), m_ngram(copy_from.m_ngram), // TODO: don't copy this. m_lmIdLookup(copy_from.m_lmIdLookup), @@ -336,7 +336,7 @@ template FFState *LanguageModelKen::EvaluateChart(const Cha } // namespace -LanguageModel *ConstructKenLM(const std::string &line) +LanguageModel *ConstructKenLM(const std::string &description, const std::string &line) { cerr << "line=" << line << endl; FactorType factorType; @@ -367,32 +367,32 @@ LanguageModel *ConstructKenLM(const std::string &line) } } - return ConstructKenLM(line, filePath, factorType, lazy); + return ConstructKenLM(description, line, filePath, factorType, lazy); } -LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy) { +LanguageModel *ConstructKenLM(const std::string &description, const std::string &line, const std::string &file, FactorType factorType, bool lazy) { try { lm::ngram::ModelType model_type; if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) { switch(model_type) { case lm::ngram::PROBING: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); case lm::ngram::REST_PROBING: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); case lm::ngram::TRIE: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); case lm::ngram::QUANT_TRIE: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); case lm::ngram::ARRAY_TRIE: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); case lm::ngram::QUANT_ARRAY_TRIE: - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); default: std::cerr << "Unrecognized kenlm model type " << model_type << std::endl; abort(); } } else { - return new LanguageModelKen(line, file, factorType, lazy); + return new LanguageModelKen(description, line, file, factorType, lazy); } } catch (std::exception &e) { std::cerr << e.what() << std::endl; diff --git a/moses/LM/Ken.h b/moses/LM/Ken.h index 9d115bfbc..3c2ceb774 100644 --- a/moses/LM/Ken.h +++ b/moses/LM/Ken.h @@ -30,10 +30,10 @@ namespace Moses { class LanguageModel; -LanguageModel *ConstructKenLM(const std::string &line); +LanguageModel *ConstructKenLM(const std::string &description, const std::string &line); //! This will also load. Returns a templated KenLM class -LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy); +LanguageModel *ConstructKenLM(const std::string &description, const std::string &line, const std::string &file, FactorType factorType, bool lazy); } // namespace Moses diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp index d1c01945f..6bf044eb6 100644 --- a/moses/Parameter.cpp +++ b/moses/Parameter.cpp @@ -324,7 +324,14 @@ bool Parameter::LoadParam(int argc, char* argv[]) std::vector &Parameter::GetWeights(const std::string &name, size_t ind) { - return m_weights[name + SPrint(ind)]; + std::vector &ret = m_weights[name + SPrint(ind)]; + + cerr << "WEIGHT " << name << ind << "="; + for (size_t i = 0; i < ret.size(); ++i) { + cerr << ret[i] << ","; + } + cerr << endl; + return ret; } void Parameter::SetWeight(const std::string &name, size_t ind, float weight) @@ -1097,6 +1104,17 @@ void Parameter::OverwriteParam(const string ¶mName, PARAM_VEC values) } VERBOSE(2, std::endl); } + +std::set Parameter::GetWeightNames() const +{ + std::set ret; + std::map >::const_iterator iter; + for (iter = m_weights.begin(); iter != m_weights.end(); ++iter) { + const string &key = iter->first; + ret.insert(key); + } + return ret; +} } diff --git a/moses/Parameter.h b/moses/Parameter.h index eff961721..55a7924e9 100644 --- a/moses/Parameter.h +++ b/moses/Parameter.h @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_Parameter_h #include +#include #include #include #include "TypeDef.h" @@ -113,6 +114,7 @@ public: } std::vector &GetWeights(const std::string &name, size_t ind); + std::set GetWeightNames() const; const PARAM_MAP &GetParams() const { return m_setting; } diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index ea65e56f0..1793241e8 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -595,13 +595,13 @@ bool StaticData::LoadData(Parameter *parameter) SetWeights(model, weights); } else if (feature == "KENLM") { - LanguageModel *model = ConstructKenLM(line); + LanguageModel *model = ConstructKenLM(feature, line); const vector &weights = m_parameter->GetWeights(feature, featureIndex); SetWeights(model, weights); } else if (feature == "IRSTLM") { LanguageModelIRST *irstlm = new LanguageModelIRST(line); - LanguageModel *model = new LMRefCount(irstlm, line); + LanguageModel *model = new LMRefCount(irstlm, feature, line); const vector &weights = m_parameter->GetWeights(feature, featureIndex); SetWeights(model, weights); } @@ -661,6 +661,10 @@ bool StaticData::LoadData(Parameter *parameter) if (!LoadPhraseTables()) return false; if (!LoadDecodeGraphs()) return false; + if (!CheckWeights()) { + return false; + } + // report individual sparse features in n-best list if (m_parameter->GetParam("report-sparse-features").size() > 0) { for(size_t i=0; iGetParam("report-sparse-features").size(); i++) { @@ -702,6 +706,8 @@ bool StaticData::LoadData(Parameter *parameter) m_allWeights.PlusEquals(extraWeights); } + cerr << endl << "m_allWeights=" << m_allWeights << endl; + return true; } @@ -1267,6 +1273,36 @@ void StaticData::CollectFeatureFunctions() } +bool StaticData::CheckWeights() const +{ + set weightNames = m_parameter->GetWeightNames(); + + const std::vector &ffs = FeatureFunction::GetFeatureFunctions(); + for (size_t i = 0; i < ffs.size(); ++i) { + const FeatureFunction &ff = *ffs[i]; + const string &descr = ff.GetScoreProducerDescription(); + + set::iterator iter = weightNames.find(descr); + if (iter == weightNames.end()) { + cerr << "Can't find weights for feature function " << descr << endl; + } + else { + weightNames.erase(iter); + } + } + + if (!weightNames.empty()) { + cerr << "The following weights have no feature function. Maybe incorrectly spelt weights: "; + set::iterator iter; + for (iter = weightNames.begin(); iter != weightNames.end(); ++iter) { + cerr << *iter << ","; + } + return false; + } + + return true; +} + } // namespace diff --git a/moses/StaticData.h b/moses/StaticData.h index 5903f0a4d..cecce3231 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -711,6 +711,8 @@ public: void CleanUpAfterSentenceProcessing(const InputType& source) const; void CollectFeatureFunctions(); + bool CheckWeights() const; + }; }