diff --git a/contrib/other-builds/OnDiskPt/.cproject b/contrib/other-builds/OnDiskPt/.cproject index f551380fd..e32a5baea 100644 --- a/contrib/other-builds/OnDiskPt/.cproject +++ b/contrib/other-builds/OnDiskPt/.cproject @@ -11,12 +11,12 @@ - - + + @@ -72,13 +72,13 @@ - - + + diff --git a/contrib/other-builds/moses-chart-cmd/.cproject b/contrib/other-builds/moses-chart-cmd/.cproject index 811c61c58..d03866163 100644 --- a/contrib/other-builds/moses-chart-cmd/.cproject +++ b/contrib/other-builds/moses-chart-cmd/.cproject @@ -33,7 +33,6 @@ diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project index 70fe2c0be..0b65a40e6 100644 --- a/contrib/other-builds/moses/.project +++ b/contrib/other-builds/moses/.project @@ -1156,6 +1156,16 @@ 1 PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h + + FF/InternalStructStatelessFF.cpp + 1 + PARENT-3-PROJECT_LOC/moses/FF/InternalStructStatelessFF.cpp + + + FF/InternalStructStatelessFF.h + 1 + PARENT-3-PROJECT_LOC/moses/FF/InternalStructStatelessFF.h + FF/LexicalReordering 2 diff --git a/moses/FF/BleuScoreFeature.cpp b/moses/FF/BleuScoreFeature.cpp index 452555855..71620410d 100644 --- a/moses/FF/BleuScoreFeature.cpp +++ b/moses/FF/BleuScoreFeature.cpp @@ -76,7 +76,7 @@ void BleuScoreState::AddNgramCountAndMatches(std::vector< size_t >& counts, BleuScoreFeature::BleuScoreFeature(const std::string &line) - :StatefulFeatureFunction("BleuScoreFeature",1, line), + :StatefulFeatureFunction(1, line), m_enabled(true), m_sentence_bleu(true), m_simple_history_bleu(false), diff --git a/moses/FF/ConstrainedDecoding.h b/moses/FF/ConstrainedDecoding.h index 6d382abc9..f7228a86d 100644 --- a/moses/FF/ConstrainedDecoding.h +++ b/moses/FF/ConstrainedDecoding.h @@ -34,7 +34,7 @@ class ConstrainedDecoding : public StatefulFeatureFunction { public: ConstrainedDecoding(const std::string &line) - :StatefulFeatureFunction("ConstrainedDecoding", 1, line) + :StatefulFeatureFunction(1, line) ,m_maxUnknowns(0) { m_tuneable = false; ReadParameters(); diff --git a/moses/FF/ControlRecombination.h b/moses/FF/ControlRecombination.h index 2ac4aa8e3..3d1d0d987 100644 --- a/moses/FF/ControlRecombination.h +++ b/moses/FF/ControlRecombination.h @@ -45,7 +45,7 @@ class ControlRecombination : public StatefulFeatureFunction { public: ControlRecombination(const std::string &line) - :StatefulFeatureFunction("ControlRecombination", 0, line) + :StatefulFeatureFunction(0, line) ,m_type(SameOutput) { diff --git a/moses/FF/DecodeFeature.cpp b/moses/FF/DecodeFeature.cpp index e4e7d9960..7a8745066 100644 --- a/moses/FF/DecodeFeature.cpp +++ b/moses/FF/DecodeFeature.cpp @@ -29,27 +29,24 @@ using namespace std; namespace Moses { -DecodeFeature::DecodeFeature( const std::string& description - , const std::string &line) - : StatelessFeatureFunction(description, line) +DecodeFeature::DecodeFeature(const std::string &line) + : StatelessFeatureFunction(line) { VERBOSE(2,"DecodeFeature:" << std::endl); } -DecodeFeature::DecodeFeature( const std::string& description - , size_t numScoreComponents +DecodeFeature::DecodeFeature(size_t numScoreComponents , const std::string &line) - : StatelessFeatureFunction(description,numScoreComponents, line) + : StatelessFeatureFunction(numScoreComponents, line) { VERBOSE(2,"DecodeFeature: no factors yet" << std::endl); } -DecodeFeature::DecodeFeature(const std::string& description - , size_t numScoreComponents +DecodeFeature::DecodeFeature(size_t numScoreComponents , const std::vector &input , const std::vector &output , const std::string &line) - : StatelessFeatureFunction(description,numScoreComponents, line) + : StatelessFeatureFunction(numScoreComponents, line) , m_input(input), m_output(output) { m_inputFactors = FactorMask(input); diff --git a/moses/FF/DecodeFeature.h b/moses/FF/DecodeFeature.h index 71fffecfa..920a0240e 100644 --- a/moses/FF/DecodeFeature.h +++ b/moses/FF/DecodeFeature.h @@ -38,15 +38,12 @@ class DecodeFeature : public StatelessFeatureFunction { public: - DecodeFeature( const std::string& description + DecodeFeature(const std::string &line); + + DecodeFeature(size_t numScoreComponents , const std::string &line); - DecodeFeature( const std::string& description - , size_t numScoreComponents - , const std::string &line); - - DecodeFeature( const std::string& description - , size_t numScoreComponents + DecodeFeature(size_t numScoreComponents , const std::vector &input , const std::vector &output , const std::string &line); diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp index 54fb8ae95..303f35236 100644 --- a/moses/FF/DistortionScoreProducer.cpp +++ b/moses/FF/DistortionScoreProducer.cpp @@ -23,7 +23,7 @@ struct DistortionState_traditional : public FFState { }; DistortionScoreProducer::DistortionScoreProducer(const std::string &line) - : StatefulFeatureFunction("Distortion", 1, line) + : StatefulFeatureFunction(1, line) { ReadParameters(); } diff --git a/moses/FF/ExternalFeature.h b/moses/FF/ExternalFeature.h index 04414f0a8..f199ca99f 100644 --- a/moses/FF/ExternalFeature.h +++ b/moses/FF/ExternalFeature.h @@ -37,7 +37,7 @@ class ExternalFeature : public StatefulFeatureFunction { public: ExternalFeature(const std::string &line) - :StatefulFeatureFunction("ExternalFeature", line) { + :StatefulFeatureFunction(line) { ReadParameters(); } ~ExternalFeature(); diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp index 9a9bcdb82..78e703304 100644 --- a/moses/FF/Factory.cpp +++ b/moses/FF/Factory.cpp @@ -4,7 +4,6 @@ #include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h" #include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h" #include "moses/TranslationModel/PhraseDictionaryMemory.h" -#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h" #include "moses/TranslationModel/PhraseDictionaryMultiModel.h" #include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h" #include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h" @@ -38,6 +37,9 @@ #include "moses/LM/SkeletonLM.h" #include "moses/TranslationModel/SkeletonPT.h" +#ifdef HAVE_CMPH +#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h" +#endif #ifdef PT_UG #include "moses/TranslationModel/mmsapt.h" #endif @@ -144,7 +146,6 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME2("PhraseDictionaryBinary", PhraseDictionaryTreeAdaptor); MOSES_FNAME(PhraseDictionaryOnDisk); MOSES_FNAME(PhraseDictionaryMemory); - MOSES_FNAME(PhraseDictionaryCompact); MOSES_FNAME(PhraseDictionaryMultiModel); MOSES_FNAME(PhraseDictionaryMultiModelCounts); MOSES_FNAME(PhraseDictionaryALSuffixArray); @@ -161,6 +162,9 @@ FeatureRegistry::FeatureRegistry() MOSES_FNAME(SkeletonLM); MOSES_FNAME(SkeletonPT); +#ifdef HAVE_CMPH + MOSES_FNAME(PhraseDictionaryCompact); +#endif #ifdef PT_UG MOSES_FNAME(Mmsapt); #endif @@ -183,7 +187,9 @@ FeatureRegistry::FeatureRegistry() Add("KENLM", new KenFactory()); } -FeatureRegistry::~FeatureRegistry() {} +FeatureRegistry::~FeatureRegistry() +{ +} void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory) { @@ -203,4 +209,15 @@ void FeatureRegistry::Construct(const std::string &name, const std::string &line i->second->Create(line); } +void FeatureRegistry::PrintFF() const +{ + std::cerr << "Available feature functions:" << std::endl; + Map::const_iterator iter; + for (iter = registry_.begin(); iter != registry_.end(); ++iter) { + const string &ffName = iter->first; + std::cerr << ffName << std::endl; + } + +} + } // namespace Moses diff --git a/moses/FF/Factory.h b/moses/FF/Factory.h index 4fc93d061..1c3867c09 100644 --- a/moses/FF/Factory.h +++ b/moses/FF/Factory.h @@ -18,6 +18,7 @@ public: ~FeatureRegistry(); void Construct(const std::string &name, const std::string &line); + void PrintFF() const; private: void Add(const std::string &name, FeatureFactory *factory); diff --git a/moses/FF/FeatureFunction.cpp b/moses/FF/FeatureFunction.cpp index 14a7527cf..8dede9866 100644 --- a/moses/FF/FeatureFunction.cpp +++ b/moses/FF/FeatureFunction.cpp @@ -30,40 +30,27 @@ FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name) } FeatureFunction:: -FeatureFunction(const std::string& description, - const std::string& line) +FeatureFunction(const std::string& line) : m_tuneable(true) , m_numScoreComponents(1) { - Initialize(description, line); + Initialize(line); } FeatureFunction:: -FeatureFunction(const std::string& description, - size_t numScoreComponents, +FeatureFunction(size_t numScoreComponents, const std::string& line) : m_tuneable(true) , m_numScoreComponents(numScoreComponents) { - Initialize(description, line); + Initialize(line); } void FeatureFunction:: -Initialize(const std::string& description, const std::string &line) +Initialize(const std::string &line) { - ParseLine(description, line); - - if (m_description == "") { - size_t index = description_counts.count(description); - - ostringstream dstream; - dstream << description; - dstream << index; - - description_counts.insert(description); - m_description = dstream.str(); - } + ParseLine(line); ScoreComponentCollection::RegisterScoreProducer(this); m_producers.push_back(this); @@ -71,11 +58,13 @@ Initialize(const std::string& description, const std::string &line) FeatureFunction::~FeatureFunction() {} -void FeatureFunction::ParseLine(const std::string& description, const std::string &line) +void FeatureFunction::ParseLine(const std::string &line) { vector toks = Tokenize(line); CHECK(toks.size()); + string nameStub = toks[0]; + set keys; for (size_t i = 1; i < toks.size(); ++i) { @@ -93,6 +82,19 @@ void FeatureFunction::ParseLine(const std::string& description, const std::strin m_args.push_back(args); } } + + // name + if (m_description == "") { + size_t index = description_counts.count(nameStub); + + ostringstream dstream; + dstream << nameStub; + dstream << index; + + description_counts.insert(nameStub); + m_description = dstream.str(); + } + } void FeatureFunction::SetParameter(const std::string& key, const std::string& value) diff --git a/moses/FF/FeatureFunction.h b/moses/FF/FeatureFunction.h index 0eec06768..98a637223 100644 --- a/moses/FF/FeatureFunction.h +++ b/moses/FF/FeatureFunction.h @@ -36,8 +36,8 @@ protected: //In case there's multiple producers with the same description static std::multiset description_counts; - void Initialize(const std::string& description, const std::string &line); - void ParseLine(const std::string& description, const std::string &line); + void Initialize(const std::string &line); + void ParseLine(const std::string &line); public: static const std::vector& GetFeatureFunctions() { @@ -45,8 +45,8 @@ public: } static FeatureFunction &FindFeatureFunction(const std::string& name); - FeatureFunction(const std::string& description, const std::string &line); - FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line); + FeatureFunction(const std::string &line); + FeatureFunction(size_t numScoreComponents, const std::string &line); virtual bool IsStateless() const = 0; virtual ~FeatureFunction(); diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp index d4ce6c875..acffc76ee 100644 --- a/moses/FF/GlobalLexicalModel.cpp +++ b/moses/FF/GlobalLexicalModel.cpp @@ -11,7 +11,7 @@ using namespace std; namespace Moses { GlobalLexicalModel::GlobalLexicalModel(const std::string &line) - : StatelessFeatureFunction("GlobalLexicalModel",1, line) + : StatelessFeatureFunction(1, line) { std::cerr << "Creating global lexical model...\n"; ReadParameters(); diff --git a/moses/FF/GlobalLexicalModelUnlimited.cpp b/moses/FF/GlobalLexicalModelUnlimited.cpp index f1f4b693b..ffa6c08c4 100644 --- a/moses/FF/GlobalLexicalModelUnlimited.cpp +++ b/moses/FF/GlobalLexicalModelUnlimited.cpp @@ -11,7 +11,7 @@ using namespace std; namespace Moses { GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line) - :StatelessFeatureFunction("GlobalLexicalModelUnlimited", 0, line) + :StatelessFeatureFunction(0, line) { CHECK(false); // TODO need to update arguments to key=value diff --git a/moses/FF/InputFeature.cpp b/moses/FF/InputFeature.cpp index 14efa12f0..a6ba048e5 100644 --- a/moses/FF/InputFeature.cpp +++ b/moses/FF/InputFeature.cpp @@ -12,7 +12,7 @@ using namespace std; namespace Moses { InputFeature::InputFeature(const std::string &line) - :StatelessFeatureFunction("InputFeature", line) + :StatelessFeatureFunction(line) { ReadParameters(); } diff --git a/moses/FF/InternalStructStatelessFF.h b/moses/FF/InternalStructStatelessFF.h index e97473fe1..6802479ce 100644 --- a/moses/FF/InternalStructStatelessFF.h +++ b/moses/FF/InternalStructStatelessFF.h @@ -10,7 +10,7 @@ class InternalStructStatelessFF : public StatelessFeatureFunction { public: InternalStructStatelessFF(const std::string &line) - :StatelessFeatureFunction("InternalStructStatelessFF", line) + :StatelessFeatureFunction(line) {} bool IsUseable(const FactorMask &mask) const diff --git a/moses/FF/LexicalReordering/LexicalReordering.cpp b/moses/FF/LexicalReordering/LexicalReordering.cpp index 2486714a5..c73c0324b 100644 --- a/moses/FF/LexicalReordering/LexicalReordering.cpp +++ b/moses/FF/LexicalReordering/LexicalReordering.cpp @@ -10,7 +10,7 @@ using namespace std; namespace Moses { LexicalReordering::LexicalReordering(const std::string &line) - : StatefulFeatureFunction("LexicalReordering", line) + : StatefulFeatureFunction(line) { std::cerr << "Initializing LexicalReordering.." << std::endl; diff --git a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp index c33851899..27396aa5a 100644 --- a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp +++ b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp @@ -8,7 +8,7 @@ #include "moses/TargetPhrase.h" #include "moses/TargetPhraseCollection.h" -#ifndef WIN32 +#ifdef HAVE_CMPH #include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h" #endif @@ -52,8 +52,10 @@ void auxAppend(IPhrase& head, const IPhrase& tail) LexicalReorderingTable* LexicalReorderingTable::LoadAvailable(const std::string& filePath, const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors) { //decide use Compact or Tree or Memory table - LexicalReorderingTable *compactLexr = - LexicalReorderingTableCompact::CheckAndLoad(filePath + ".minlexr", f_factors, e_factors, c_factors); + LexicalReorderingTable *compactLexr = NULL; +#ifdef HAVE_CMPH + compactLexr = LexicalReorderingTableCompact::CheckAndLoad(filePath + ".minlexr", f_factors, e_factors, c_factors); +#endif if(compactLexr) return compactLexr; if(FileExists(filePath+".binlexr.idx")) { diff --git a/moses/FF/OSM-Feature/OpSequenceModel.cpp b/moses/FF/OSM-Feature/OpSequenceModel.cpp index e35e31a33..553196270 100644 --- a/moses/FF/OSM-Feature/OpSequenceModel.cpp +++ b/moses/FF/OSM-Feature/OpSequenceModel.cpp @@ -11,7 +11,7 @@ namespace Moses { OpSequenceModel::OpSequenceModel(const std::string &line) - :StatefulFeatureFunction("OpSequenceModel", 5, line ) + :StatefulFeatureFunction(5, line ) { sFactor = 0; tFactor = 0; diff --git a/moses/FF/PhraseBoundaryFeature.cpp b/moses/FF/PhraseBoundaryFeature.cpp index a398fdc6a..d82181b76 100644 --- a/moses/FF/PhraseBoundaryFeature.cpp +++ b/moses/FF/PhraseBoundaryFeature.cpp @@ -18,7 +18,7 @@ int PhraseBoundaryState::Compare(const FFState& other) const } PhraseBoundaryFeature::PhraseBoundaryFeature(const std::string &line) - : StatefulFeatureFunction("PhraseBoundaryFeature", 0, line) + : StatefulFeatureFunction(0, line) { std::cerr << "Initializing source word deletion feature.." << std::endl; ReadParameters(); diff --git a/moses/FF/PhraseLengthFeature.cpp b/moses/FF/PhraseLengthFeature.cpp index fb69a2de3..43e0d1b2d 100644 --- a/moses/FF/PhraseLengthFeature.cpp +++ b/moses/FF/PhraseLengthFeature.cpp @@ -10,7 +10,7 @@ namespace Moses using namespace std; PhraseLengthFeature::PhraseLengthFeature(const std::string &line) - :StatelessFeatureFunction("PhraseLengthFeature", 0, line) + :StatelessFeatureFunction(0, line) { ReadParameters(); } diff --git a/moses/FF/PhrasePairFeature.cpp b/moses/FF/PhrasePairFeature.cpp index 564d131ba..e122bf6ab 100644 --- a/moses/FF/PhrasePairFeature.cpp +++ b/moses/FF/PhrasePairFeature.cpp @@ -15,7 +15,7 @@ namespace Moses { PhrasePairFeature::PhrasePairFeature(const std::string &line) - :StatelessFeatureFunction("PhrasePairFeature", 0, line) + :StatelessFeatureFunction(0, line) { std::cerr << "Initializing PhrasePairFeature.." << std::endl; ReadParameters(); diff --git a/moses/FF/PhrasePenalty.cpp b/moses/FF/PhrasePenalty.cpp index 8b54939a7..b3e493707 100644 --- a/moses/FF/PhrasePenalty.cpp +++ b/moses/FF/PhrasePenalty.cpp @@ -5,7 +5,7 @@ namespace Moses { PhrasePenalty::PhrasePenalty(const std::string &line) - : StatelessFeatureFunction("PhrasePenalty",1, line) + : StatelessFeatureFunction(1, line) { ReadParameters(); } diff --git a/moses/FF/SkeletonStatefulFF.h b/moses/FF/SkeletonStatefulFF.h index c3bb10e37..5b0ca0f95 100644 --- a/moses/FF/SkeletonStatefulFF.h +++ b/moses/FF/SkeletonStatefulFF.h @@ -22,7 +22,7 @@ class SkeletonStatefulFF : public StatefulFeatureFunction { public: SkeletonStatefulFF(const std::string &line) - :StatefulFeatureFunction("SkeletonStatefulFF", 3, line) + :StatefulFeatureFunction(3, line) {} bool IsUseable(const FactorMask &mask) const { diff --git a/moses/FF/SkeletonStatelessFF.h b/moses/FF/SkeletonStatelessFF.h index f3c494a94..d17f324aa 100644 --- a/moses/FF/SkeletonStatelessFF.h +++ b/moses/FF/SkeletonStatelessFF.h @@ -10,7 +10,7 @@ class SkeletonStatelessFF : public StatelessFeatureFunction { public: SkeletonStatelessFF(const std::string &line) - :StatelessFeatureFunction("SkeletonStatelessFF", 2, line) + :StatelessFeatureFunction(2, line) {} bool IsUseable(const FactorMask &mask) const { diff --git a/moses/FF/SourceWordDeletionFeature.cpp b/moses/FF/SourceWordDeletionFeature.cpp index b920cc619..12918a9e9 100644 --- a/moses/FF/SourceWordDeletionFeature.cpp +++ b/moses/FF/SourceWordDeletionFeature.cpp @@ -18,7 +18,7 @@ namespace Moses using namespace std; SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line) - :StatelessFeatureFunction("SourceWordDeletionFeature", 0, line), + :StatelessFeatureFunction(0, line), m_unrestricted(true) { std::cerr << "Initializing source word deletion feature.." << std::endl; diff --git a/moses/FF/StatefulFeatureFunction.cpp b/moses/FF/StatefulFeatureFunction.cpp index 37b9c2dee..9e61ed05f 100644 --- a/moses/FF/StatefulFeatureFunction.cpp +++ b/moses/FF/StatefulFeatureFunction.cpp @@ -5,14 +5,14 @@ namespace Moses std::vector StatefulFeatureFunction::m_statefulFFs; -StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, const std::string &line) - : FeatureFunction(description, line) +StatefulFeatureFunction::StatefulFeatureFunction(const std::string &line) + : FeatureFunction(line) { m_statefulFFs.push_back(this); } -StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line) - : FeatureFunction(description,numScoreComponents, line) +StatefulFeatureFunction::StatefulFeatureFunction(size_t numScoreComponents, const std::string &line) + : FeatureFunction(numScoreComponents, line) { m_statefulFFs.push_back(this); } diff --git a/moses/FF/StatefulFeatureFunction.h b/moses/FF/StatefulFeatureFunction.h index e99355ba2..75b46d827 100644 --- a/moses/FF/StatefulFeatureFunction.h +++ b/moses/FF/StatefulFeatureFunction.h @@ -19,8 +19,8 @@ public: return m_statefulFFs; } - StatefulFeatureFunction(const std::string& description, const std::string &line); - StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line); + StatefulFeatureFunction(const std::string &line); + StatefulFeatureFunction(size_t numScoreComponents, const std::string &line); /** * \brief This interface should be implemented. diff --git a/moses/FF/StatelessFeatureFunction.cpp b/moses/FF/StatelessFeatureFunction.cpp index 51b98064e..ecad23e6f 100644 --- a/moses/FF/StatelessFeatureFunction.cpp +++ b/moses/FF/StatelessFeatureFunction.cpp @@ -5,14 +5,14 @@ namespace Moses std::vector StatelessFeatureFunction::m_statelessFFs; -StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, const std::string &line) - :FeatureFunction(description, line) +StatelessFeatureFunction::StatelessFeatureFunction(const std::string &line) + :FeatureFunction(line) { m_statelessFFs.push_back(this); } -StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line) - :FeatureFunction(description, numScoreComponents, line) +StatelessFeatureFunction::StatelessFeatureFunction(size_t numScoreComponents, const std::string &line) + :FeatureFunction(numScoreComponents, line) { m_statelessFFs.push_back(this); } diff --git a/moses/FF/StatelessFeatureFunction.h b/moses/FF/StatelessFeatureFunction.h index e2368cf58..fde740115 100644 --- a/moses/FF/StatelessFeatureFunction.h +++ b/moses/FF/StatelessFeatureFunction.h @@ -18,8 +18,8 @@ public: return m_statelessFFs; } - StatelessFeatureFunction(const std::string& description, const std::string &line); - StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line); + StatelessFeatureFunction(const std::string &line); + StatelessFeatureFunction(size_t numScoreComponents, const std::string &line); /** * This should be implemented for features that apply to phrase-based models. **/ diff --git a/moses/FF/TargetBigramFeature.cpp b/moses/FF/TargetBigramFeature.cpp index 083e16ac0..337e47050 100644 --- a/moses/FF/TargetBigramFeature.cpp +++ b/moses/FF/TargetBigramFeature.cpp @@ -18,7 +18,7 @@ int TargetBigramState::Compare(const FFState& other) const } TargetBigramFeature::TargetBigramFeature(const std::string &line) - :StatefulFeatureFunction("TargetBigramFeature", 0, line) + :StatefulFeatureFunction(0, line) { std::cerr << "Initializing target bigram feature.." << std::endl; ReadParameters(); diff --git a/moses/FF/TargetNgramFeature.cpp b/moses/FF/TargetNgramFeature.cpp index 8dc934ffd..a14082266 100644 --- a/moses/FF/TargetNgramFeature.cpp +++ b/moses/FF/TargetNgramFeature.cpp @@ -38,7 +38,7 @@ int TargetNgramState::Compare(const FFState& other) const } TargetNgramFeature::TargetNgramFeature(const std::string &line) - :StatefulFeatureFunction("TargetNgramFeature", 0, line) + :StatefulFeatureFunction(0, line) { std::cerr << "Initializing target ngram feature.." << std::endl; ReadParameters(); diff --git a/moses/FF/TargetWordInsertionFeature.cpp b/moses/FF/TargetWordInsertionFeature.cpp index f59e75bc6..6011e9365 100644 --- a/moses/FF/TargetWordInsertionFeature.cpp +++ b/moses/FF/TargetWordInsertionFeature.cpp @@ -16,7 +16,7 @@ namespace Moses using namespace std; TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line) - :StatelessFeatureFunction("TargetWordInsertionFeature", 0, line), + :StatelessFeatureFunction(0, line), m_unrestricted(true) { std::cerr << "Initializing target word insertion feature.." << std::endl; diff --git a/moses/FF/UnknownWordPenaltyProducer.cpp b/moses/FF/UnknownWordPenaltyProducer.cpp index fcb91a430..edc05dd27 100644 --- a/moses/FF/UnknownWordPenaltyProducer.cpp +++ b/moses/FF/UnknownWordPenaltyProducer.cpp @@ -7,7 +7,7 @@ using namespace std; namespace Moses { UnknownWordPenaltyProducer::UnknownWordPenaltyProducer(const std::string &line) - : StatelessFeatureFunction("UnknownWordPenalty",1, line) + : StatelessFeatureFunction(1, line) { m_tuneable = false; ReadParameters(); diff --git a/moses/FF/WordPenaltyProducer.cpp b/moses/FF/WordPenaltyProducer.cpp index 8d77d68fd..ac11824f5 100644 --- a/moses/FF/WordPenaltyProducer.cpp +++ b/moses/FF/WordPenaltyProducer.cpp @@ -7,7 +7,7 @@ using namespace std; namespace Moses { WordPenaltyProducer::WordPenaltyProducer(const std::string &line) - : StatelessFeatureFunction("WordPenalty",1, line) + : StatelessFeatureFunction(1, line) { ReadParameters(); } diff --git a/moses/FF/WordTranslationFeature.cpp b/moses/FF/WordTranslationFeature.cpp index e45659cde..0a5936203 100644 --- a/moses/FF/WordTranslationFeature.cpp +++ b/moses/FF/WordTranslationFeature.cpp @@ -18,7 +18,7 @@ namespace Moses { WordTranslationFeature::WordTranslationFeature(const std::string &line) - :StatelessFeatureFunction("WordTranslationFeature", 0, line) + :StatelessFeatureFunction(0, line) ,m_unrestricted(true) ,m_simple(true) ,m_sourceContext(false) diff --git a/moses/GenerationDictionary.cpp b/moses/GenerationDictionary.cpp index ee9dbcb06..efebd1ae2 100644 --- a/moses/GenerationDictionary.cpp +++ b/moses/GenerationDictionary.cpp @@ -36,7 +36,7 @@ namespace Moses { GenerationDictionary::GenerationDictionary(const std::string &line) - : DecodeFeature("Generation", line) + : DecodeFeature(line) { ReadParameters(); } diff --git a/moses/LM/Base.cpp b/moses/LM/Base.cpp index b77d64fef..f59b5e31b 100644 --- a/moses/LM/Base.cpp +++ b/moses/LM/Base.cpp @@ -34,8 +34,8 @@ using namespace std; namespace Moses { -LanguageModel::LanguageModel(const std::string& description, const std::string &line) : - StatefulFeatureFunction(description, StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line ) +LanguageModel::LanguageModel(const std::string &line) : + StatefulFeatureFunction(StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line ) { m_enableOOVFeature = StaticData::Instance().GetLMEnableOOVFeature(); } diff --git a/moses/LM/Base.h b/moses/LM/Base.h index 5c9616e46..fb24ba627 100644 --- a/moses/LM/Base.h +++ b/moses/LM/Base.h @@ -43,7 +43,7 @@ class Phrase; class LanguageModel : public StatefulFeatureFunction { protected: - LanguageModel(const std::string& description, const std::string &line); + LanguageModel(const std::string &line); // This can't be in the constructor for virual function dispatch reasons diff --git a/moses/LM/IRST.cpp b/moses/LM/IRST.cpp index bde78e1b8..44c5d8c4c 100644 --- a/moses/LM/IRST.cpp +++ b/moses/LM/IRST.cpp @@ -40,7 +40,7 @@ using namespace std; namespace Moses { LanguageModelIRST::LanguageModelIRST(const std::string &line) - :LanguageModelSingleFactor("IRSTLM", line) + :LanguageModelSingleFactor(line) { const StaticData &staticData = StaticData::Instance(); int threadCount = staticData.ThreadCount(); diff --git a/moses/LM/Implementation.cpp b/moses/LM/Implementation.cpp index 3f94fece8..267b94dd6 100644 --- a/moses/LM/Implementation.cpp +++ b/moses/LM/Implementation.cpp @@ -41,8 +41,8 @@ using namespace std; namespace Moses { -LanguageModelImplementation::LanguageModelImplementation(const std::string& description, const std::string &line) - :LanguageModel(description, line) +LanguageModelImplementation::LanguageModelImplementation(const std::string &line) + :LanguageModel(line) { } diff --git a/moses/LM/Implementation.h b/moses/LM/Implementation.h index e6a230ad8..a39f5e42b 100644 --- a/moses/LM/Implementation.h +++ b/moses/LM/Implementation.h @@ -60,7 +60,7 @@ protected: Word m_sentenceStartWord, m_sentenceEndWord; //! Contains factors which represents the beging and end words for this LM. //! Usually and - LanguageModelImplementation(const std::string& description, const std::string &line); + LanguageModelImplementation(const std::string &line); public: diff --git a/moses/LM/Joint.h b/moses/LM/Joint.h index e94c29b4f..1dbdb019e 100644 --- a/moses/LM/Joint.h +++ b/moses/LM/Joint.h @@ -50,7 +50,7 @@ protected: size_t m_implFactor; public: LanguageModelJoint(const std::string &line, LanguageModelSingleFactor *lmImpl) - :LanguageModelMultiFactor("JointLM", line) { + :LanguageModelMultiFactor(line) { m_lmImpl = lmImpl; } diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index ed0278029..2bb6a16f9 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -140,7 +140,7 @@ private: } // namespace template LanguageModelKen::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy) - :LanguageModel("KENLM", line) + :LanguageModel(line) ,m_factorType(factorType) { lm::ngram::Config config; @@ -161,7 +161,7 @@ template LanguageModelKen::LanguageModelKen(const std::stri } template LanguageModelKen::LanguageModelKen(const LanguageModelKen ©_from) - :LanguageModel(copy_from.GetScoreProducerDescription(), copy_from.GetArgLine()), + :LanguageModel(copy_from.GetArgLine()), m_ngram(copy_from.m_ngram), // TODO: don't copy this. m_lmIdLookup(copy_from.m_lmIdLookup), diff --git a/moses/LM/MultiFactor.h b/moses/LM/MultiFactor.h index 619ad94f3..df476f93f 100644 --- a/moses/LM/MultiFactor.h +++ b/moses/LM/MultiFactor.h @@ -41,8 +41,8 @@ class LanguageModelMultiFactor : public LanguageModelImplementation protected: FactorMask m_factorTypes; - LanguageModelMultiFactor(const std::string& description, const std::string &line) - :LanguageModelImplementation(description, line) { + LanguageModelMultiFactor(const std::string &line) + :LanguageModelImplementation(line) { } public: diff --git a/moses/LM/NeuralLMWrapper.cpp b/moses/LM/NeuralLMWrapper.cpp index 41ceded9d..467c41846 100644 --- a/moses/LM/NeuralLMWrapper.cpp +++ b/moses/LM/NeuralLMWrapper.cpp @@ -10,7 +10,7 @@ using namespace std; namespace Moses { NeuralLMWrapper::NeuralLMWrapper(const std::string &line) -:LanguageModelSingleFactor("NeuralLM", line) +:LanguageModelSingleFactor(line) { // This space intentionally left blank } diff --git a/moses/LM/ORLM.h b/moses/LM/ORLM.h index 4443b4732..dd2a675d8 100644 --- a/moses/LM/ORLM.h +++ b/moses/LM/ORLM.h @@ -22,7 +22,7 @@ class LanguageModelORLM : public LanguageModelSingleFactor public: typedef count_t T; // type for ORLM filter LanguageModelORLM(const std::string &line) - :LanguageModelSingleFactor("ORLM", line) + :LanguageModelSingleFactor(line) ,m_lm(0) { } bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder); diff --git a/moses/LM/ParallelBackoff.cpp b/moses/LM/ParallelBackoff.cpp index 3d67e9c90..063e9c818 100644 --- a/moses/LM/ParallelBackoff.cpp +++ b/moses/LM/ParallelBackoff.cpp @@ -85,7 +85,7 @@ private: public: LanguageModelParallelBackoff(const std::string &line) - :LanguageModelMultiFactor("ParallelBackoffLM", line) { + :LanguageModelMultiFactor(line) { } ~LanguageModelParallelBackoff(); diff --git a/moses/LM/Rand.cpp b/moses/LM/Rand.cpp index 0f11fea07..a3c53b483 100644 --- a/moses/LM/Rand.cpp +++ b/moses/LM/Rand.cpp @@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // The following ifdef prevents XCode and other non-bjam build systems // from attempting to compile this file when LM_RAND is disabled. // -#ifdef LM_RAND #include #include @@ -44,7 +43,7 @@ namespace Moses { LanguageModelRandLM::LanguageModelRandLM(const std::string &line) - :LanguageModelSingleFactor("RandLM", line) + :LanguageModelSingleFactor(line) , m_lm(0) { } @@ -147,4 +146,3 @@ void LanguageModelRandLM::CleanUpAfterSentenceProcessing(const InputType& source } -#endif diff --git a/moses/LM/SRI.cpp b/moses/LM/SRI.cpp index 6182bd22c..708374668 100644 --- a/moses/LM/SRI.cpp +++ b/moses/LM/SRI.cpp @@ -54,7 +54,7 @@ using namespace std; namespace Moses { LanguageModelSRI::LanguageModelSRI(const std::string &line) - :LanguageModelSingleFactor("SRILM", line) + :LanguageModelSingleFactor(line) ,m_srilmVocab(0) ,m_srilmModel(0) { diff --git a/moses/LM/SingleFactor.cpp b/moses/LM/SingleFactor.cpp index 04f410c98..d2c0f990a 100644 --- a/moses/LM/SingleFactor.cpp +++ b/moses/LM/SingleFactor.cpp @@ -38,8 +38,8 @@ using namespace std; namespace Moses { -LanguageModelSingleFactor::LanguageModelSingleFactor(const std::string& description, const std::string &line) - :LanguageModelImplementation(description, line) +LanguageModelSingleFactor::LanguageModelSingleFactor(const std::string &line) + :LanguageModelImplementation(line) { m_nullContextState = new PointerState(NULL); m_beginSentenceState = new PointerState(NULL); diff --git a/moses/LM/SingleFactor.h b/moses/LM/SingleFactor.h index af0edd4c6..eeb5cdbef 100644 --- a/moses/LM/SingleFactor.h +++ b/moses/LM/SingleFactor.h @@ -43,7 +43,7 @@ protected: FFState *m_nullContextState; FFState *m_beginSentenceState; - LanguageModelSingleFactor(const std::string& description, const std::string &line); + LanguageModelSingleFactor(const std::string &line); public: virtual ~LanguageModelSingleFactor(); diff --git a/moses/LM/SkeletonLM.cpp b/moses/LM/SkeletonLM.cpp index 6bf47e108..6c11512fe 100644 --- a/moses/LM/SkeletonLM.cpp +++ b/moses/LM/SkeletonLM.cpp @@ -7,7 +7,7 @@ using namespace std; namespace Moses { SkeletonLM::SkeletonLM(const std::string &line) - :LanguageModelSingleFactor("SkeletonLM", line) + :LanguageModelSingleFactor(line) { ReadParameters(); diff --git a/moses/OutputCollector.h b/moses/OutputCollector.h index 5f72433d8..22cfb46a6 100644 --- a/moses/OutputCollector.h +++ b/moses/OutputCollector.h @@ -62,7 +62,7 @@ public: } bool OutputIsCout() const { - return (m_outStream == std::cout); + return (m_outStream == &std::cout); } /** diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp index 902c5c610..3f34abeb9 100644 --- a/moses/Parameter.cpp +++ b/moses/Parameter.cpp @@ -270,6 +270,7 @@ bool Parameter::LoadParam(int argc, char* argv[]) && (configPath = FindParam("-config", argc, argv)) == "") { PrintCredit(); Explain(); + PrintFF(); cerr << endl; UserMessage::Add("No configuration file was specified. Use -config or -f"); @@ -1302,6 +1303,11 @@ void Parameter::OverwriteParam(const string ¶mName, PARAM_VEC values) VERBOSE(2, std::endl); } +void Parameter::PrintFF() const +{ + StaticData::Instance().GetFeatureRegistry().PrintFF(); +} + std::set Parameter::GetWeightNames() const { std::set ret; diff --git a/moses/Parameter.h b/moses/Parameter.h index f5016fb2b..973a1f09b 100644 --- a/moses/Parameter.h +++ b/moses/Parameter.h @@ -62,6 +62,7 @@ protected: void AddParam(const std::string ¶mName, const std::string &abbrevName, const std::string &description); void PrintCredit(); + void PrintFF() const; void SetWeight(const std::string &name, size_t ind, float weight); void SetWeight(const std::string &name, size_t ind, const std::vector &weights); diff --git a/moses/ScoreComponentCollectionTest.cpp b/moses/ScoreComponentCollectionTest.cpp index f44595aec..06f999e3d 100644 --- a/moses/ScoreComponentCollectionTest.cpp +++ b/moses/ScoreComponentCollectionTest.cpp @@ -32,8 +32,8 @@ BOOST_AUTO_TEST_SUITE(scc) class MockStatelessFeatureFunction : public StatelessFeatureFunction { public: - MockStatelessFeatureFunction(const string& desc, size_t n, const string &line) : - StatelessFeatureFunction(desc,n, line) {} + MockStatelessFeatureFunction(size_t n, const string &line) : + StatelessFeatureFunction(n, line) {} void Evaluate(const Hypothesis&, ScoreComponentCollection*) const {} void EvaluateChart(const ChartHypothesis&, ScoreComponentCollection*) const {} void Evaluate(const InputType &input @@ -52,7 +52,7 @@ public: class MockSingleFeature : public MockStatelessFeatureFunction { public: - MockSingleFeature(): MockStatelessFeatureFunction("MockSingle",1, "MockSingle") {} + MockSingleFeature(): MockStatelessFeatureFunction(1, "MockSingle") {} bool IsUseable(const FactorMask &mask) const { return true; @@ -62,7 +62,7 @@ public: class MockMultiFeature : public MockStatelessFeatureFunction { public: - MockMultiFeature(): MockStatelessFeatureFunction("MockMulti", 5, "MockMulti") {} + MockMultiFeature(): MockStatelessFeatureFunction(5, "MockMulti") {} bool IsUseable(const FactorMask &mask) const { return true; @@ -73,7 +73,7 @@ public: class MockSparseFeature : public MockStatelessFeatureFunction { public: - MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", 0, "MockSparse") {} + MockSparseFeature(): MockStatelessFeatureFunction(0, "MockSparse") {} bool IsUseable(const FactorMask &mask) const { return true; diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp index 4ca807888..1aad30254 100644 --- a/moses/StaticData.cpp +++ b/moses/StaticData.cpp @@ -24,7 +24,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "util/check.hh" #include "TypeDef.h" -#include "moses/FF/Factory.h" #include "moses/FF/WordPenaltyProducer.h" #include "moses/FF/UnknownWordPenaltyProducer.h" #include "moses/FF/InputFeature.h" @@ -510,7 +509,6 @@ bool StaticData::LoadData(Parameter *parameter) map featureIndexMap; const vector &features = m_parameter->GetParam("feature"); - FeatureRegistry registry; for (size_t i = 0; i < features.size(); ++i) { const string &line = Trim(features[i]); cerr << "line=" << line << endl; @@ -521,7 +519,7 @@ bool StaticData::LoadData(Parameter *parameter) const string &feature = toks[0]; - registry.Construct(feature, line); + m_registry.Construct(feature, line); } OverrideFeatures(); diff --git a/moses/StaticData.h b/moses/StaticData.h index 3e5f2921b..16136f849 100644 --- a/moses/StaticData.h +++ b/moses/StaticData.h @@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "Parameter.h" #include "SentenceStats.h" #include "ScoreComponentCollection.h" +#include "moses/FF/Factory.h" namespace Moses { @@ -206,6 +207,8 @@ protected: FactorType m_placeHolderFactor; bool m_useLegacyPT; + FeatureRegistry m_registry; + StaticData(); void LoadChartDecodingParameters(); @@ -752,6 +755,9 @@ public: return m_placeHolderFactor; } + const FeatureRegistry &GetFeatureRegistry() const + { return m_registry; } + /** check whether we should be using the old code to support binary phrase-table. ** eventually, we'll stop support the binary phrase-table and delete this legacy code **/ diff --git a/moses/TranslationModel/CompactPT/Jamfile b/moses/TranslationModel/CompactPT/Jamfile index 452c0048e..f42d63f64 100644 --- a/moses/TranslationModel/CompactPT/Jamfile +++ b/moses/TranslationModel/CompactPT/Jamfile @@ -5,12 +5,13 @@ if $(with-cmph) { lib cmph : : $(with-cmph)/lib $(with-cmph)/lib64 ; includes += $(with-cmph)/include ; current = "--with-cmph=$(with-cmph)" ; + fakelib CompactPT : [ glob *.cpp ] ../..//headers cmph : $(includes) $(PT-LOG) : : $(includes) ; } else { alias cmph ; + fakelib CompactPT ; } path-constant PT-LOG : bin/pt.log ; update-if-changed $(PT-LOG) $(current) ; -fakelib CompactPT : [ glob *.cpp ] ../..//headers cmph : $(includes) $(PT-LOG) : : $(includes) ; diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp index c60d0f944..876e334eb 100644 --- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp +++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp @@ -42,7 +42,7 @@ namespace Moses { PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line) - :PhraseDictionary("PhraseDictionaryCompact", line) + :PhraseDictionary(line) ,m_inMemory(true) ,m_useAlignmentInfo(true) ,m_hash(10, 16) diff --git a/moses/TranslationModel/DynSAInclude/vocab.cpp b/moses/TranslationModel/DynSAInclude/vocab.cpp index da1443f66..dfb3aabff 100644 --- a/moses/TranslationModel/DynSAInclude/vocab.cpp +++ b/moses/TranslationModel/DynSAInclude/vocab.cpp @@ -118,7 +118,7 @@ bool Vocab::Load(FileHandler* vcbin, const FactorDirection& direction, std::string line, word_str; wordID_t id; - void *ret = getline(*vcbin, line); + std::istream &ret = getline(*vcbin, line); CHECK(ret); std::istringstream first(line.c_str()); uint32_t vcbsize(0); diff --git a/moses/TranslationModel/PhraseDictionary.cpp b/moses/TranslationModel/PhraseDictionary.cpp index 6c7aeefc9..159e798a1 100644 --- a/moses/TranslationModel/PhraseDictionary.cpp +++ b/moses/TranslationModel/PhraseDictionary.cpp @@ -32,8 +32,8 @@ using namespace std; namespace Moses { -PhraseDictionary::PhraseDictionary(const std::string &description, const std::string &line) - :DecodeFeature(description, line) +PhraseDictionary::PhraseDictionary(const std::string &line) + :DecodeFeature(line) ,m_tableLimit(20) // default ,m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE) { diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h index 671fe19b2..f8477f70e 100644 --- a/moses/TranslationModel/PhraseDictionary.h +++ b/moses/TranslationModel/PhraseDictionary.h @@ -59,7 +59,7 @@ class ChartParser; class PhraseDictionary : public DecodeFeature { public: - PhraseDictionary(const std::string &description, const std::string &line); + PhraseDictionary(const std::string &line); virtual ~PhraseDictionary() { } diff --git a/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp b/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp index f9022eeee..ac5a817bb 100644 --- a/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp +++ b/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp @@ -10,7 +10,7 @@ namespace Moses { PhraseDictionaryDynSuffixArray:: PhraseDictionaryDynSuffixArray(const std::string &line) - : PhraseDictionary("PhraseDictionaryDynSuffixArray", line) + : PhraseDictionary(line) ,m_biSA(new BilingualDynSuffixArray()) { ReadParameters(); diff --git a/moses/TranslationModel/PhraseDictionaryMemory.cpp b/moses/TranslationModel/PhraseDictionaryMemory.cpp index 4ad3c5e87..e7f0174d0 100644 --- a/moses/TranslationModel/PhraseDictionaryMemory.cpp +++ b/moses/TranslationModel/PhraseDictionaryMemory.cpp @@ -41,7 +41,7 @@ using namespace std; namespace Moses { PhraseDictionaryMemory::PhraseDictionaryMemory(const std::string &line) - : RuleTableTrie("PhraseDictionaryMemory", line) + : RuleTableTrie(line) { ReadParameters(); diff --git a/moses/TranslationModel/PhraseDictionaryMemory.h b/moses/TranslationModel/PhraseDictionaryMemory.h index 44e57e7f4..a1a23eca3 100644 --- a/moses/TranslationModel/PhraseDictionaryMemory.h +++ b/moses/TranslationModel/PhraseDictionaryMemory.h @@ -39,8 +39,8 @@ class PhraseDictionaryMemory : public RuleTableTrie friend class RuleTableLoader; protected: - PhraseDictionaryMemory(const std::string &description, const std::string &line) - : RuleTableTrie(description, line) { + PhraseDictionaryMemory(int type, const std::string &line) + : RuleTableTrie(line) { } public: diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp index 923df4fe6..be3986929 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp @@ -26,7 +26,7 @@ namespace Moses { PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line) - :PhraseDictionary("PhraseDictionaryMultiModel", line) + :PhraseDictionary(line) { ReadParameters(); @@ -43,10 +43,11 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line) CHECK(m_pdStr.size() == m_multimodelweights.size() || m_pdStr.size()*numWeights == m_multimodelweights.size()); } -PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &description, const std::string &line) - :PhraseDictionary(description, line) +PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::string &line) + :PhraseDictionary(line) { - if (description == "PhraseDictionaryMultiModelCounts") { + if (type == 1) { + // PhraseDictionaryMultiModelCounts CHECK(m_pdStr.size() == m_multimodelweights.size() || m_pdStr.size()*4 == m_multimodelweights.size()); } } diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.h b/moses/TranslationModel/PhraseDictionaryMultiModel.h index cd385895b..4d7786534 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModel.h +++ b/moses/TranslationModel/PhraseDictionaryMultiModel.h @@ -61,7 +61,7 @@ class PhraseDictionaryMultiModel: public PhraseDictionary public: PhraseDictionaryMultiModel(const std::string &line); - PhraseDictionaryMultiModel(const std::string &description, const std::string &line); + PhraseDictionaryMultiModel(int type, const std::string &line); ~PhraseDictionaryMultiModel(); void Load(); virtual void CollectSufficientStatistics(const Phrase& src, std::map* allStats) const; diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp index e33c9e8a8..6f0f85466 100644 --- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp +++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp @@ -61,7 +61,7 @@ namespace Moses { PhraseDictionaryMultiModelCounts::PhraseDictionaryMultiModelCounts(const std::string &line) - :PhraseDictionaryMultiModel("PhraseDictionaryMultiModelCounts", line) + :PhraseDictionaryMultiModel(1, line) { m_mode = "instance_weighting"; m_combineFunction = InstanceWeighting; diff --git a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp index 7c5ae9a37..066fb4067 100644 --- a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp +++ b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp @@ -27,7 +27,7 @@ namespace Moses PhraseDictionaryTreeAdaptor:: PhraseDictionaryTreeAdaptor(const std::string &line) - : PhraseDictionary("PhraseDictionaryBinary", line) + : PhraseDictionary(line) { ReadParameters(); } diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp index 89356ad7a..a5625cb71 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp @@ -21,7 +21,7 @@ using namespace std; namespace Moses { PhraseDictionaryALSuffixArray::PhraseDictionaryALSuffixArray(const std::string &line) - : PhraseDictionaryMemory("PhraseDictionaryALSuffixArray", line) + : PhraseDictionaryMemory(1, line) { const StaticData &staticData = StaticData::Instance(); if (staticData.ThreadCount() > 1) { diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp index cfad3f830..542df66a9 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp @@ -54,7 +54,7 @@ namespace Moses { PhraseDictionaryFuzzyMatch::PhraseDictionaryFuzzyMatch(const std::string &line) - :PhraseDictionary("PhraseDictionaryFuzzyMatch", line) + :PhraseDictionary(line) ,m_FuzzyMatchWrapper(NULL) { CHECK(m_args.size() == 0); diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp index b8fc434fb..5998e24a5 100644 --- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp @@ -35,7 +35,7 @@ using namespace std; namespace Moses { PhraseDictionaryOnDisk::PhraseDictionaryOnDisk(const std::string &line) - : MyBase("PhraseDictionaryOnDisk", line) + : MyBase(line) { ReadParameters(); } diff --git a/moses/TranslationModel/RuleTable/Trie.h b/moses/TranslationModel/RuleTable/Trie.h index 0d363f747..c7cb7e991 100644 --- a/moses/TranslationModel/RuleTable/Trie.h +++ b/moses/TranslationModel/RuleTable/Trie.h @@ -40,8 +40,8 @@ class Word; class RuleTableTrie : public PhraseDictionary { public: - RuleTableTrie(const std::string &description, const std::string &line) - : PhraseDictionary(description, line) { + RuleTableTrie(const std::string &line) + : PhraseDictionary(line) { } virtual ~RuleTableTrie(); diff --git a/moses/TranslationModel/RuleTable/UTrie.h b/moses/TranslationModel/RuleTable/UTrie.h index 80a2e4b3d..8222bb632 100644 --- a/moses/TranslationModel/RuleTable/UTrie.h +++ b/moses/TranslationModel/RuleTable/UTrie.h @@ -46,7 +46,7 @@ class RuleTableUTrie : public RuleTableTrie { public: RuleTableUTrie(const std::string &line) - : RuleTableTrie("RuleTableUTrie", line) { + : RuleTableTrie(line) { } const UTrieNode &GetRootNode() const { diff --git a/moses/TranslationModel/SkeletonPT.cpp b/moses/TranslationModel/SkeletonPT.cpp index 322494d68..83496f352 100644 --- a/moses/TranslationModel/SkeletonPT.cpp +++ b/moses/TranslationModel/SkeletonPT.cpp @@ -7,7 +7,7 @@ using namespace std; namespace Moses { SkeletonPT::SkeletonPT(const std::string &line) - : PhraseDictionary("SkeletonPT", line) + : PhraseDictionary(line) { ReadParameters(); } diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp index 878ab7968..a932e2a99 100644 --- a/moses/TranslationOptionCollectionLattice.cpp +++ b/moses/TranslationOptionCollectionLattice.cpp @@ -29,6 +29,7 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice( const InputFeature *inputFeature = StaticData::Instance().GetInputFeature(); CHECK(inputFeature); + size_t maxPhraseLength = StaticData::Instance().GetMaxPhraseLength(); size_t size = input.GetSize(); // 1-word phrases @@ -76,6 +77,11 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice( } size_t startPos = prevPath.GetWordsRange().GetStartPos(); + + if (endPos - startPos + 1 > maxPhraseLength) { + continue; + } + WordsRange range(startPos, endPos); const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos); diff --git a/phrase-extract/extract-ghkm/Subgraph.cpp b/phrase-extract/extract-ghkm/Subgraph.cpp index 6c4af8acb..0c941218d 100644 --- a/phrase-extract/extract-ghkm/Subgraph.cpp +++ b/phrase-extract/extract-ghkm/Subgraph.cpp @@ -17,8 +17,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ***********************************************************************/ +#include #include "Subgraph.h" - #include "Node.h" namespace Moses diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index 374038b1b..bbf1bed0d 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -967,7 +967,7 @@ decode default-name: evaluation/output qsub-script: yes ignore-if: use-hiero - rerun-on-change: decoder decoder-settings nbest report-segmentation report-precision-by-coverage analyze-search-graph + rerun-on-change: decoder decoder-settings nbest report-segmentation report-precision-by-coverage analyze-search-graph wade error: Translation was not performed correctly not-error: trans: No such file or directory hiero-decode @@ -1031,6 +1031,13 @@ lowercase-reference pass-if: recaser multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl template: $output-lowercaser < IN > OUT +wade + in: filtered-dir truecased-input tokenized-reference alignment system-output + out: wade-analysis + default-name: evaluation/wade-analysis + ignore-unless: wade + rerun-on-change: wade + template: $moses-script-dir/ems/support/run-wade.perl $wade IN IN1 IN2 IN3 IN4 OUT nist-bleu in: wrapped-output reference-sgm out: nist-bleu-score @@ -1131,6 +1138,6 @@ analysis-precision [REPORTING] single report - in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model + in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis out: report default-name: evaluation/report diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index 706cbe07c..777147af8 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -2646,6 +2646,7 @@ sub define_evaluation_decode { my $report_segmentation = &backoff_and_get("EVALUATION:$set:report-segmentation"); my $analyze_search_graph = &backoff_and_get("EVALUATION:$set:analyze-search-graph"); my $report_precision_by_coverage = &backoff_and_get("EVALUATION:$set:report-precision-by-coverage"); + my $use_wade = &backoff_and_get("EVALUATION:$set:wade"); my $hierarchical = &get("TRAINING:hierarchical-rule-set"); my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules"); @@ -2666,6 +2667,9 @@ sub define_evaluation_decode { $settings .= " -t"; } } + if ($use_wade) { + $settings .= " -T $system_output.details"; + } $settings .= " -text-type \"test\""; my $addTags = &backoff_and_get("EVALUATION:$set:add-tags"); diff --git a/scripts/ems/support/run-wade.perl b/scripts/ems/support/run-wade.perl new file mode 100755 index 000000000..418ff8c97 --- /dev/null +++ b/scripts/ems/support/run-wade.perl @@ -0,0 +1,30 @@ +#!/usr/bin/perl + +use strict; +use File::Temp qw/ tempfile tempdir /; + +if (scalar(@ARGV) != 7) { + print STDERR "Usage: run-wade.perl wade-script filtered-dir input reference alignment output wade-output\n"; + exit 1; +} + +my ($wade_script, $filtered_dir, $input, $reference, $alignment, $output, $wade_output) = @ARGV; +my ($cfh, $cfile) = tempfile(); + +#unless ($phrase_table =~ /.*\.gz/){ $phrase_table .= ".gz";} + +print $cfh "phrase-table=$filtered_dir/phrase-table.0-0.1.1.gz\n"; +print $cfh "fr_test=$input\n"; +print $cfh "en_test=$reference\n"; +print $cfh "fren_align_test=$alignment\n"; +print $cfh "translation_details=$output.details\n"; + +print $cfh "candidatealign_out=$wade_output.candalign\n"; +print $cfh "annotatedalign_out=$wade_output.annotalign\n"; +print $cfh "transspans_out=$wade_output.transspans\n"; +print $cfh "wade_output=$wade_output\n"; +close $cfh; + +print $cfile . "\n"; + +system("python $wade_script $cfile");