Work in progress.

moses/Jamfile

@@ -46,7 +46,6 @@ lib moses :
 [ glob 
   *.cpp
   TranslationModel/*.cpp
-  TranslationModel/UG/*.cpp
   TranslationModel/fuzzy-match/*.cpp
   TranslationModel/DynSAInclude/*.cpp
   TranslationModel/RuleTable/*.cpp
@@ -61,12 +60,14 @@ lib moses :
   *Test.cpp Mock*.cpp
   FF/Factory.cpp
 ]
-#headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT synlm ThreadPool rt 
-headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT synlm ThreadPool  
+headers FF_Factory.o LM//LM TranslationModel/CompactPT//CompactPT synlm ThreadPool
 ..//search ../util/double-conversion//double-conversion ..//z ../OnDiskPt//OnDiskPt 
-TranslationModel/UG/generic//generic TranslationModel/UG/mm//mm
-$(TOP)//boost_iostreams ;
+$(TOP)//boost_iostreams TranslationModel/UG/generic//generic TranslationModel/UG/mm//mm
+:
+<threading>single:<source>../util//rt
+;
 
+#generic//generic mm//mm
 
 alias headers-to-install : [ glob-tree *.h ] ;
 

moses/TranslationModel/UG/mm/ug_bitext.cc

@@ -346,7 +346,7 @@ namespace Moses
       
       // we add the sentences in separate threads (so it's faster)
       boost::thread thread1(snt_adder<TKN>(s1,*ret->V1,ret->myT1,ret->myI1));
-      thread1.join(); // for debugging
+      // thread1.join(); // for debugging
       boost::thread thread2(snt_adder<TKN>(s2,*ret->V2,ret->myT2,ret->myI2));
       BOOST_FOREACH(string const& a, aln)
 	{

moses/TranslationModel/UG/mm/ug_im_tsa.h

@@ -151,6 +151,7 @@ namespace ugdiss
 	filter2.set();
 	filter = &filter2;
       }
+    assert(filter);
     // In the first iteration over the corpus, we obtain word counts.
     // They allows us to 
     //    a. allocate the exact amount of memory we need

moses/TranslationModel/UG/mmsapt.cpp

@@ -62,8 +62,45 @@ namespace Moses
     m = param.find("ifactor");
     input_factor = m != param.end() ? atoi(m->second.c_str()) : 0;
     poolCounts = true;
+    m = param.find("extra");
+    if (m != param.end()) 
+      {
+	extra_data = m->second;
+	cerr << "have extra data" << endl;
+      }
   }
 
+  void
+  Mmsapt::
+  load_extra_data(string bname)
+  {
+    vector<string> text1,text2,symal;
+    string line;
+    filtering_istream in1,in2,ina; 
+    open_input_stream(bname+L1+".txt.gz",in1);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    while(getline(in1,line)) text1.push_back(line);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    open_input_stream(bname+L2+".txt.gz",in2);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    while(getline(in2,line)) text2.push_back(line);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    open_input_stream(bname+L1+"-"+L2+".symal.gz",ina);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    while(getline(ina,line)) 
+      {
+	cerr << line << endl;
+	symal.push_back(line);
+      }
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    // cerr << "Read " << btdyn->T1->size() << " sentence pairs" << endl;
+    lock_guard<mutex> guard(this->lock);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    btdyn = btdyn->add(text1,text2,symal);
+    cerr << __FILE__ << ":" << __LINE__ << endl;
+    cerr << "Loaded " << btdyn->T1->size() << " sentence pairs" << endl;
+  }
+  
   void
   Mmsapt::
   Load()
@@ -82,7 +119,7 @@ namespace Moses
 	num_feats  = calc_pfwd_dyn.init(num_feats,lbop_parameter);
 	num_feats  = calc_pbwd_dyn.init(num_feats,lbop_parameter);
       }
-    btdyn.reset(new imBitext<Token>(btfix.V1, btfix.V2));
+
     if (num_feats != this->m_numScoreComponents)
       {
 	ostringstream buf;
@@ -94,6 +131,11 @@ namespace Moses
     // cerr << "MMSAPT provides " << num_feats << " features at " 
     // << __FILE__ << ":" << __LINE__ << endl;
 
+    btdyn.reset(new imBitext<Token>(btfix.V1, btfix.V2));
+    cerr << "btdyn initialized at " << __FILE__ << ":" << __LINE__ << endl;
+    if (extra_data.size()) load_extra_data(extra_data);
+
+
     LexicalPhraseScorer2<Token>::table_t & COOC = calc_lex.scorer.COOC;
     typedef LexicalPhraseScorer2<Token>::table_t::Cell cell_t;
     wlex21.resize(COOC.numCols);

moses/TranslationModel/UG/mmsapt.h

@@ -52,7 +52,7 @@ namespace Moses
   private:
     mmbitext btfix; 
     sptr<imbitext> btdyn;
-    string bname;
+    string bname,extra_data;
     string L1;
     string L2;
     float  lbop_parameter;
@@ -118,6 +118,9 @@ namespace Moses
      TargetPhraseCollection* tpcoll
      ) const;
 
+    void
+    load_extra_data(string bname);
+
   public:
     // Mmsapt(string const& description, string const& line);
     Mmsapt(string const& line);