Merge ../mosesdecoder into merge-cmd

mert/MiraWeightVector.cpp

@@ -139,7 +139,7 @@ ostream& operator<<(ostream& o, const MiraWeightVector& e)
   for(size_t i=0; i<e.m_weights.size(); i++) {
     if(abs(e.m_weights[i])>1e-8) {
       if(i>0) o << " ";
-      cerr << i << ":" << e.m_weights[i];
+      o << i << ":" << e.m_weights[i];
     }
   }
   return o;

mert/kbmira.cpp

@@ -72,6 +72,7 @@ int main(int argc, char** argv)
   float decay = 0.999; // Pseudo-corpus decay \gamma
   int n_iters = 60;    // Max epochs J
   bool streaming = false; // Stream all k-best lists?
+  bool streaming_out = false; // Stream output after each sentence?
   bool no_shuffle = false; // Don't shuffle, even for in memory version
   bool model_bg = false; // Use model for background corpus
   bool verbose = false; // Verbose updates
@@ -97,6 +98,7 @@ int main(int argc, char** argv)
   ("dense-init,d", po::value<string>(&denseInitFile), "Weight file for dense features. This should have 'name= value' on each line, or (legacy) should be the Moses mert 'init.opt' format.")
   ("sparse-init,s", po::value<string>(&sparseInitFile), "Weight file for sparse features")
   ("streaming", po::value(&streaming)->zero_tokens()->default_value(false), "Stream n-best lists to save memory, implies --no-shuffle")
+  ("streaming-out", po::value(&streaming_out)->zero_tokens()->default_value(false), "Stream weights to stdout after each sentence")
   ("no-shuffle", po::value(&no_shuffle)->zero_tokens()->default_value(false), "Don't shuffle hypotheses before each epoch")
   ("model-bg", po::value(&model_bg)->zero_tokens()->default_value(false), "Use model instead of hope for BLEU background")
   ("verbose", po::value(&verbose)->zero_tokens()->default_value(false), "Verbose updates")
@@ -235,7 +237,8 @@ int main(int argc, char** argv)
   }
 
   // Training loop
-  cerr << "Initial BLEU = " << decoder->Evaluate(wv.avg()) << endl;
+  if (!streaming_out)
+    cerr << "Initial BLEU = " << decoder->Evaluate(wv.avg()) << endl;
   ValType bestBleu = 0;
   for(int j=0; j<n_iters; j++) {
     // MIRA train for one epoch
@@ -283,6 +286,8 @@ int main(int argc, char** argv)
       }
       iNumExamples++;
       ++sentenceIndex;
+      if (streaming_out)
+        cout << wv << endl;
     }
     // Training Epoch summary
     cerr << iNumUpdates << "/" << iNumExamples << " updates"

moses/XmlOption.cpp

@@ -24,6 +24,8 @@
 #include <vector>
 #include <string>
 #include <iostream>
+#include <boost/foreach.hpp>
+#include <boost/unordered_map.hpp>
 #include "Util.h"
 #include "StaticData.h"
 #include "WordsRange.h"
@@ -348,6 +350,52 @@ bool ProcessAndStripXMLTags(string &line, vector<XmlOption*> &res, ReorderingCon
 #endif
         }
 
+        // weight-overwrite: update feature weights, unspecified weights remain unchanged
+        // IMPORTANT: translation models that cache phrases or apply table-limit during load
+        // based on initial weights need to be reset.  Sending an empty update will do this
+        // for PhraseDictionaryBitextSampling (Mmsapt) models:
+        // <update name="TranslationModelName" source=" " target=" " alignment=" " />
+        else if (tagName == "weight-overwrite") {
+            
+            // is a name->ff map stored anywhere so we don't have to build it every time?
+            const vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
+            boost::unordered_map<string, FeatureFunction*> map;
+            BOOST_FOREACH(FeatureFunction* const& ff, ffs) {
+                map[ff->GetScoreProducerDescription()] = ff;
+            }
+
+            // update each weight listed
+            ScoreComponentCollection allWeights = StaticData::Instance().GetAllWeights();
+            boost::unordered_map<string, FeatureFunction*>::iterator ffi;
+            string ffName("");
+            vector<float> ffWeights;
+            vector<string> toks = Tokenize(ParseXmlTagAttribute(tagContent,"weights"));
+            BOOST_FOREACH(string const& tok, toks) {
+                if (tok.substr(tok.size() - 1, 1) == "=") {
+                    // start new feature
+                    if (ffName != "") {
+                        // set previous feature weights
+                        if (ffi != map.end()) {
+                            allWeights.Assign(ffi->second, ffWeights);
+                        }
+                        ffWeights.clear();
+                    }
+                    ffName = tok.substr(0, tok.size() - 1);
+                    ffi = map.find(ffName);
+                    if (ffi == map.end()) {
+                        TRACE_ERR("ERROR: No FeatureFunction with name " << ffName << ", no weight update" << endl);
+                    }
+                } else {
+                    // weight for current feature
+                    ffWeights.push_back(Scan<float>(tok));
+                }
+            }
+            if (ffi != map.end()) {
+                allWeights.Assign(ffi->second, ffWeights);
+            }
+            StaticData::InstanceNonConst().SetAllWeights(allWeights);
+        }
+
         // default: opening tag that specifies translation options
         else {
           if (startPos > endPos) {

scripts/training/mert-moses.pl

@@ -396,16 +396,16 @@ if (!defined $mertargs) {
 }
 
 my $scconfig = undef;
-if ($mertargs =~ /\-\-scconfig\s+(.+?)(\s|$)/) {
+if ($mertargs =~ /\-\-scconfig(?:\s+|=)(.+?)(\s|$)/) {
   $scconfig = $1;
   $scconfig =~ s/\,/ /g;
-  $mertargs =~ s/\-\-scconfig\s+(.+?)(\s|$)//;
+  $mertargs =~ s/\-\-scconfig(?:\s+|=)(.+?)(\s|$)//;
 }
 
 my $sctype = "--sctype BLEU";
-if ($mertargs =~ /(\-\-sctype\s+.+?)(\s|$)/) {
+if ($mertargs =~ /(\-\-sctype(?:\s+|=).+?)(\s|$)/) {
   $sctype = $1;
-  $mertargs =~ s/(\-\-sctype\s+.+?)(\s|$)//;
+  $mertargs =~ s/(\-\-sctype(?:\s+|=)+.+?)(\s|$)//;
 }