Merge branch 'master' of github.com:moses-smt/mosesdecoder

moses/FF/Factory.cpp

@@ -109,7 +109,7 @@ template <class F> void FeatureFactory::DefaultSetup(F *feature)
     // if it's tuneable, ini file MUST have weights
     // even it it's not tuneable, people can still set the weights in the ini file
     static_data.SetWeights(feature, weights);
-  } else {
+  } else if (feature->GetNumScoreComponents() > 0) {
     std::vector<float> defaultWeights = feature->DefaultWeights();
     static_data.SetWeights(feature, defaultWeights);
   }

moses/FF/SoftMatchingFeature.cpp

@@ -11,17 +11,24 @@ namespace Moses
 SoftMatchingFeature::SoftMatchingFeature(const std::string &line)
   : StatelessFeatureFunction(0, line)
 {
-  std::cerr << "Initializing SoftMatchingFeature.." << std::endl;
-
-  for (size_t i = 0; i < m_args.size(); ++i) {
-    const std::vector<std::string> &args = m_args[i];
-    if (args[0] == "path") {
-      const std::string filePath = args[1];
-      Load(filePath);
-    }
-  } // for
+  ReadParameters();
 }
 
+void SoftMatchingFeature::SetParameter(const std::string& key, const std::string& value)
+{
+  std::cerr << "setting: " << this->GetScoreProducerDescription() << " - " << key << "\n";
+  if (key == "tuneable") {
+    m_tuneable = Scan<bool>(value);
+  } else if (key == "filterable") { //ignore
+  } else if (key == "path") {
+      const std::string filePath = value;
+      Load(filePath);
+  } else {
+    UTIL_THROW(util::Exception, "Unknown argument " << key << "=" << value);
+  }
+}
+
+
 bool SoftMatchingFeature::Load(const std::string& filePath)
 {
 
@@ -103,6 +110,5 @@ const std::string& SoftMatchingFeature::GetFeatureName(const Word& LHS, const Wo
   return m_soft_matching_cache.find(key)->second;
 }
 
-
 }
 

moses/FF/SoftMatchingFeature.h

@@ -48,6 +48,7 @@ public:
   }
 
   const std::string& GetFeatureName(const Word& LHS, const Word& RHS) const;
+  void SetParameter(const std::string& key, const std::string& value);
 
 private:
   std::map<Word, std::set<Word> > m_soft_matches; // map LHS of old rule to RHS of new rle

moses/FF/SyntaxConstraintFeature.h

@@ -75,7 +75,9 @@ class SyntaxConstraintFeature : public StatefulFeatureFunction
 {
 public:
   SyntaxConstraintFeature(const std::string &line)
-    :StatefulFeatureFunction(0, line) {}
+    :StatefulFeatureFunction(0, line) {
+      ReadParameters();
+    }
 
   virtual const FFState* EmptyHypothesisState(const InputType &input) const {
     return new TreeState(TreePointer());

moses/Parameter.h

@@ -115,6 +115,9 @@ public:
   }
 
   std::vector<float> GetWeights(const std::string &name);
+  std::map<std::string, std::vector<float> > GetAllWeights() const {
+      return m_weights;
+  }
   std::set<std::string> GetWeightNames() const;
 
   const PARAM_MAP &GetParams() const {

moses/StaticData.cpp

@@ -534,6 +534,7 @@ bool StaticData::LoadData(Parameter *parameter)
 
   if (!LoadDecodeGraphs()) return false;
 
+
   if (!CheckWeights()) {
     return false;
   }
@@ -555,6 +556,9 @@ bool StaticData::LoadData(Parameter *parameter)
     m_allWeights.PlusEquals(extraWeights);
   }
 
+  //Load sparse features from config (overrules weight file)
+  LoadSparseWeightsFromConfig();
+
   // alternate weight settings
   if (m_parameter->GetParam("alternate-weight-setting").size() > 0) {
     if (!LoadAlternateWeightSettings()) {
@@ -933,11 +937,13 @@ void StaticData::LoadFeatureFunctions()
 bool StaticData::CheckWeights() const
 {
   set<string> weightNames = m_parameter->GetWeightNames();
+  set<string> featureNames;
 
   const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
   for (size_t i = 0; i < ffs.size(); ++i) {
     const FeatureFunction &ff = *ffs[i];
     const string &descr = ff.GetScoreProducerDescription();
+    featureNames.insert(descr);
 
     set<string>::iterator iter = weightNames.find(descr);
     if (iter == weightNames.end()) {
@@ -947,6 +953,21 @@ bool StaticData::CheckWeights() const
     }
   }
 
+  //sparse features
+  if (!weightNames.empty()) {
+    set<string>::iterator iter;
+    for (iter = weightNames.begin(); iter != weightNames.end(); ) {
+      string fname = (*iter).substr(0, (*iter).find("_"));
+      cerr << fname << "\n";
+      if (featureNames.find(fname) != featureNames.end()) {
+        weightNames.erase(iter++);
+      }
+      else {
+        ++iter;
+      }
+    }
+  }
+
   if (!weightNames.empty()) {
     cerr << "The following weights have no feature function. Maybe incorrectly spelt weights: ";
     set<string>::iterator iter;
@@ -959,6 +980,29 @@ bool StaticData::CheckWeights() const
   return true;
 }
 
+
+void StaticData::LoadSparseWeightsFromConfig() {
+  set<string> featureNames;
+  const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+  for (size_t i = 0; i < ffs.size(); ++i) {
+    const FeatureFunction &ff = *ffs[i];
+    const string &descr = ff.GetScoreProducerDescription();
+    featureNames.insert(descr);
+  }
+
+  std::map<std::string, std::vector<float> > weights = m_parameter->GetAllWeights();
+  std::map<std::string, std::vector<float> >::iterator iter;
+  for (iter = weights.begin(); iter != weights.end(); ++iter) {
+    // this indicates that it is sparse feature
+    if (featureNames.find(iter->first) == featureNames.end()) {
+      UTIL_THROW_IF2(iter->second.size() != 1, "ERROR: only one weight per sparse feature allowed: " << iter->first);
+        m_allWeights.Assign(iter->first, iter->second[0]);
+    }
+  }
+
+}
+
+
 /**! Read in settings for alternative weights */
 bool StaticData::LoadAlternateWeightSettings()
 {

moses/StaticData.h

@@ -715,6 +715,7 @@ public:
 
   void LoadFeatureFunctions();
   bool CheckWeights() const;
+  void LoadSparseWeightsFromConfig();
   bool LoadWeightSettings();
   bool LoadAlternateWeightSettings();
 

scripts/training/mert-moses.pl

@@ -1422,7 +1422,7 @@ sub create_config {
 			# leave weights 'til last. We're changing it
 			while ($line = <$ini_fh>) {
 			  last if $line =~ /^\[/;
-			  if ($line =~ /^([^=\s]+)/) {
+			  if ($line =~ /^([^_=\s]+)/) {
 			    for( @{$featlist->{"untuneables"}} ){
 			      if ($1 eq $_ ) {# if weight is untuneable, copy it into new config
 			        push @keep_weights, $line;