Bug fix. Added sanity check when adding data to dynamic suffix array.

moses/TranslationModel/UG/mm/ug_bitext.cc

@@ -338,6 +338,10 @@ namespace Moses
       typedef L2R_Token<SimpleWordId> TKN;
       assert(s1.size() == s2.size() && s1.size() == aln.size());
       
+#ifndef NDEBUG
+      size_t first_new_snt = this->T1 ? this->T1->size() : 0;
+#endif
+
       sptr<imBitext<TKN> > ret;
       {
 	lock_guard<mutex> guard(this->lock);
@@ -353,23 +357,52 @@ namespace Moses
 	  istringstream ibuf(a);
 	  ostringstream obuf;
 	  uint32_t row,col; char c;
-	  while (ibuf>>row>>c>>col)
+	  while (ibuf >> row >> c >> col)
 	    {
 	      assert(c == '-');
 	      binwrite(obuf,row);
 	      binwrite(obuf,col);
 	    }
-	  char const* x = obuf.str().c_str();
-	  vector<char> v(x,x+obuf.str().size());
+	  // important: DO NOT replace the two lines below this comment by 
+	  // char const* x = obuf.str().c_str(), as the memory x is pointing 
+	  // to is freed immediately upon return. There's probably a way to 
+	  // access the ostream's buffer directly. To be done.
+	  string foo = obuf.str(); 
+	  char const* x = foo.c_str();
+	  vector<char> v(x,x+foo.size());
 	  ret->myTx = append(ret->myTx, v);
 	}
+
       thread1.join();
       thread2.join();
+
       ret->Tx = ret->myTx;
       ret->T1 = ret->myT1;
       ret->T2 = ret->myT2;
       ret->I1 = ret->myI1;
       ret->I2 = ret->myI2;
+
+#ifndef NDEBUG
+      // sanity check
+      for (size_t i = first_new_snt; i < ret->T1->size(); ++i)
+	{
+	  size_t slen1  = ret->T1->sntLen(i);
+	  size_t slen2  = ret->T2->sntLen(i);
+	  char const* p = ret->Tx->sntStart(i);
+	  char const* q = ret->Tx->sntEnd(i);
+	  size_t k;
+	  while (p < q)
+	    {
+	      p = binread(p,k);
+	      assert(p);
+	      assert(p < q);
+	      assert(k < slen1);
+	      p = binread(p,k);
+	      assert(p);
+	      assert(k < slen2);
+	    }
+	}
+#endif
       return ret;
     }