Merge branch 'vw_integration' of github.com:moses-smt/mosesdecoder into vw_integration

moses/Jamfile

@@ -19,7 +19,7 @@ if $(with-oxlm) {
 
 local vw = ;
 if [ option.get "with-vw" ] {
-  classifier += ..//vw//vw ;
+  vw += ..//vw//vw ;
 }
 
 alias headers : ../util//kenutil $(vw) : : : $(max-factors) $(dlib) $(oxlm) ; 

moses/Util.h

@@ -331,6 +331,20 @@ std::string Join(const std::string& delimiter, const std::vector<T>& items)
   return outstr.str();
 }
 
+/*
+ * Convert any container to string
+ */
+template<typename It>
+std::string Join(const std::string &delim, It begin, It end)
+{
+  std::ostringstream outstr;
+  if (begin != end)
+    outstr << *begin++;
+  for ( ; begin != end; ++begin)
+    outstr << delim << *begin;
+  return outstr.str();
+}
+
 //! transform prob to natural log score
 inline float TransformScore(float prob)
 {

vw/Classifier.h

@@ -41,11 +41,13 @@ public:
 
   /**
    * Train using current example. Use loss to distinguish positive and negative training examples.
+   * Throws away current label-dependent features (so that features for another label/class can now be set).
    */
   virtual void Train(const StringPiece &label, float loss) = 0;
   
   /**
    * Predict the loss (inverse of score) of current example.
+   * Throws away current label-dependent features (so that features for another label/class can now be set).
    */
   virtual float Predict(const StringPiece &label) = 0;
 
@@ -76,7 +78,6 @@ public:
 
 protected:
   void AddFeature(const StringPiece &name, float value);
-  void FinishExample();
   void Finish();
 
   bool m_isFirstSource, m_isFirstTarget, m_isFirstExample;
@@ -106,7 +107,6 @@ public:
 
 protected:
   void AddFeature(const StringPiece &name, float value);
-  void FinishExample();
   void Finish();
 
   ::vw *m_VWInstance;
@@ -139,7 +139,7 @@ public:
   /**
   * Release a VWPredictor instance.
   */
-  void Release(VWPredictor * fc);
+  void Release(VWPredictor *vwpred);
 
   ~VWPredictorFactory();
 

vw/FeatureExtractor.cpp

@@ -1,9 +1,6 @@
 #include "FeatureExtractor.h"
 #include "Util.h"
 
-#include <algorithm>
-#include <set>
-
 using namespace std;
 using namespace Moses;
 

vw/Jamfile

@@ -7,8 +7,8 @@ local with-vw = [ option.get "with-vw" ] ;
 if $(with-vw) {
   lib vwlib : : <search>$(with-vw)/lib ;
   lib allreduce : : <search>$(with-vw)/lib ;
-  obj VWLibraryConsumer.o : VWLibraryConsumer.cpp headers : <include>$(with-vw)/library <include>$(with-vw)/vowpalwabbit ;
-  alias vw_objects : VWLibraryConsumer.o vwlib allreduce : : : <library>boost_program_options ;
-  lib vw : [ glob *.cpp : VWLibraryConsumer.cpp ] vw_objects headers ;
+  obj VWPredictor.o : VWPredictor.cpp VWPredictorFactory.cpp headers : <include>$(with-vw)/library <include>$(with-vw)/vowpalwabbit ;
+  alias vw_objects : VWPredictor.o vwlib allreduce : : : <library>boost_program_options ;
+  lib vw : [ glob *.cpp : VWPredictor.cpp VWPredictorFactory.cpp ] vw_objects headers ;
   echo "Linking with Vowpal Wabbit" ;
 }

vw/VWPredictor.cpp

@@ -1,10 +1,6 @@
 #include "Classifier.h"
 #include "vw.h"
-#include "Util.h"
 #include "ezexample.h"
-#include <stdexcept>
-#include <exception>
-#include <string>
 
 namespace Discriminative {
 
@@ -18,7 +14,7 @@ VWPredictor::VWPredictor(const string &modelFile, const string &vwOptions)
   m_isFirstSource = m_isFirstTarget = true;
 }
 
-VWPredictor::VWPredictor(vw * instance, int index)
+VWPredictor::VWPredictor(vw *instance, int index)
 {
   m_VWInstance = instance;
   m_sharedVwInstance = true;
@@ -29,17 +25,26 @@ VWPredictor::VWPredictor(vw * instance, int index)
 
 void VWPredictor::AddLabelIndependentFeature(const StringPiece &name, float value)
 {
+  // label-independent features are kept in a different feature namespace ('s' = source)
+
   if (m_isFirstSource) {
+    // the first feature of a new example => create the source namespace for
+    // label-independent features to live in
     m_isFirstSource = false;
     m_ex->clear_features(); // removes all namespaces along with features
     m_ex->addns('s');
   }
-  AddFeature(name, value);
+  AddFeature(name, value); // namespace 's' is set up, add the feature
 }
 
 void VWPredictor::AddLabelDependentFeature(const StringPiece &name, float value)
 {
+  // VW does not use the label directly, instead, we do a Cartesian product between source and target feature
+  // namespaces, where the source namespace ('s') contains label-independent features and the target
+  // namespace ('t') contains label-dependent features
+
   if (m_isFirstTarget) {
+    // the first target-side feature => create namespace 't'
     m_isFirstTarget = false;
     m_ex->addns('t');
   }

vw/VWPredictorFactory.cpp

@@ -1,10 +1,5 @@
 #include "Classifier.h"
 #include "vw.h"
-#include "Util.h"
-#include "ezexample.h"
-#include <stdexcept>
-#include <exception>
-#include <string>
 
 using namespace std;
 
@@ -63,7 +58,7 @@ VWPredictorFactory::~VWPredictorFactory()
   VW::finish(*m_VWInstance);
 }
 
-VWPredictor * VWPredictorFactory::Acquire()
+VWPredictor *VWPredictorFactory::Acquire()
 {
   boost::unique_lock<boost::mutex> lock(m_mutex);
   while (m_firstFree == EMPTY_LIST)
@@ -74,17 +69,17 @@ VWPredictor * VWPredictorFactory::Acquire()
   return m_predictors[free];
 }
 
-void VWPredictorFactory::Release(VWPredictor * fc)
+void VWPredictorFactory::Release(VWPredictor *vwpred)
 {
   // use scope block to handle the lock
   {
     boost::unique_lock<boost::mutex> lock(m_mutex);
-    int index = fc->m_index;
+    int index = vwpred->m_index;
 
     if (index < 0 || index >= (int)m_predictors.size())
       throw std::runtime_error("bad index at VWPredictorFactory::Release");
 
-    if (fc != m_predictors[index])
+    if (vwpred != m_predictors[index])
       throw std::runtime_error("mismatched pointer at VWPredictorFactory::Release");
 
     m_nextFree[index] = m_firstFree;

vw/VWTrainer.cpp

@@ -1,8 +1,5 @@
 #include "Util.h"
 #include "Classifier.h"
-#include <stdexcept>
-#include <exception>
-#include <string>
 #include <boost/iostreams/device/file.hpp>
 
 using namespace std;
@@ -68,7 +65,6 @@ float VWTrainer::Predict(const StringPiece &label)
 
 void VWTrainer::AddFeature(const StringPiece &name, float value)
 {
-  // TODO take advantage of StringPiece here somehow?
   m_outputBuffer.push_back(EscapeSpecialChars(name.as_string()) + ":" + SPrint(value));
 }