Fix full left/full right definition to be zero length

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mtm_lm@4289 1f5c12ca-751b-0410-a591-d2e778427230

kenlm/lm/left.cc

@@ -3,6 +3,7 @@
 #include "left.hh"
 #include "../../moses/src/Phrase.h"
 #include "../../moses/src/ChartHypothesis.h"
+#include "../../moses/src/ChartManager.h"
 
 using namespace std;
 using namespace Moses;
@@ -19,61 +20,46 @@ ChartState::~ChartState()
 }
 
 int ChartState::Compare(const ChartState &other) const {
-	assert(prefix);
-	assert(suffix);
-	assert(prefix->GetSize() > 0);
-	assert(suffix->GetSize() > 0);
+  int comparePre = prefix->Compare(*other.prefix);
+	int compareSuf = suffix->Compare(*other.suffix);
+
+  if (hypo->GetCurrSourceRange().GetStartPos() == 0) comparePre = 0;
+  if (hypo->GetCurrSourceRange().GetEndPos() >= hypo->GetManager().GetSource().GetSize() - 1) compareSuf = 0;
 	
 	int lres = left.Compare(other.left);
-	if (lres) return lres;
+	int fres = (int)full - (int)other.full;
 	int rres = right.Compare(other.right);
-	if (rres) return rres;
-	int ret = (int)full - (int)other.full;
 
-	if (lres == 0 && rres != 0)
-	    recombCount[4]++;
-	else if (lres == 0 && rres == 0)
-	    recombCount[5]++;
-	else if (lres != 0 && rres == 0)
-	    recombCount[6]++;
-	else
-	{
-	    assert(lres);
-	    assert(rres);
-	    recombCount[7]++;
-	}
+  if (!comparePre) {
+    assert(!lres);
+    assert(!fres);
+  }
+  if (!compareSuf) assert(!rres);
 
-	if (ret == 0)
-	{
-		int comparePre = prefix->Compare(*other.prefix);
-		int compareSuf = suffix->Compare(*other.suffix);
+  if (lres) return lres;
+  if (fres) return fres;
+  if (rres) return rres;
 
-    if (comparePre)
-    {
-      if (compareSuf)
-        recombCount[1]++; // both. middle
-      else
-        recombCount[0]++; // only prefix. left
-    }
-    else if (compareSuf)
-      recombCount[2]++; // only suffix. right
+  if (comparePre)
+  {
+    if (compareSuf)
+      recombCount[1]++; // both. middle
     else
-      recombCount[3]++; // nothing. last
-    
-    /*
-		cerr << "recomb_stats " << comparePre << " " << compareSuf << endl
-				<< *prefix << " ||| " << *other.prefix << endl
-				<< *suffix << " ||| " << *other.suffix << endl;
-     */
-	}
-	
-	return ret;
+      recombCount[0]++; // only prefix. left
+  }
+  else if (compareSuf)
+    recombCount[2]++; // only suffix. right
+  else
+    recombCount[3]++; // nothing. last
+
+  return 0;
 }
 
-void ChartState::CreatePreAndSuffices(const ChartHypothesis &hypo) const 
+void ChartState::CreatePreAndSuffices(const ChartHypothesis &hyp) 
 {
-  prefix = &hypo.GetPrefix();
-  suffix = &hypo.GetSuffix();
+  prefix = &hyp.GetPrefix();
+  suffix = &hyp.GetSuffix();
+  hypo = &hyp;
 }
 
 }

kenlm/lm/left.hh

@@ -59,21 +59,21 @@ struct Left {
   bool operator==(const Left &other) const {
     return 
       (length == other.length) && 
-      pointers[length - 1] == other.pointers[length - 1];
+      (!length || pointers[length - 1] == other.pointers[length - 1]);
   }
 
   int Compare(const Left &other) const {
     if (length != other.length) {
       return (int)length - (int)other.length;
     }
+    if (!length) return 0;
     if (pointers[length - 1] > other.pointers[length - 1]) return 1;
     if (pointers[length - 1] < other.pointers[length - 1]) return -1;
     return 0;
   }
 
   bool operator<(const Left &other) const {
-    if (length != other.length) return length < other.length;
-    return pointers[length - 1] < other.pointers[length - 1];
+    return Compare(other) == -1;
   }
 
   void ZeroRemaining() {
@@ -86,7 +86,7 @@ struct Left {
 };
 
 inline size_t hash_value(const Left &left) {
-  return util::MurmurHashNative(&left.length, 1, left.pointers[left.length - 1]);
+  return util::MurmurHashNative(&left.length, 1, left.length ? left.pointers[left.length - 1] : 0);
 }
 
 class ChartState {
@@ -117,12 +117,13 @@ public:
   State right;
   bool full;
 
-	void CreatePreAndSuffices(const Moses::ChartHypothesis &hypo) const;
+	void CreatePreAndSuffices(const Moses::ChartHypothesis &hyp);
 	
   static std::vector<int> recombCount;
   
 protected:
-  mutable const Moses::Phrase *prefix, *suffix;  
+  const Moses::ChartHypothesis *hypo;
+  const Moses::Phrase *prefix, *suffix;  
 };
 
 inline size_t hash_value(const ChartState &state) {
@@ -197,21 +198,21 @@ template <class M> class RuleScore {
       ProcessRet(model_.ExtendLeft(out_.right.words, out_.right.words + out_.right.length, out_.right.backoff, in.left.pointers[0], 1, back, next_use));
       if (next_use != out_.right.length) {
         left_done_ = true;
-	if (!next_use) {
+        if (!next_use) {
           out_.right = in.right;
           return;
-	}
+        }
       }
       unsigned char extend_length = 2;
       for (const uint64_t *i = in.left.pointers + 1; i < in.left.pointers + in.left.length; ++i, ++extend_length) {
         ProcessRet(model_.ExtendLeft(out_.right.words, out_.right.words + next_use, back, *i, extend_length, back2, next_use));
-	if (next_use != out_.right.length) {
-	  left_done_ = true;
-	  if (!next_use) {
-	    out_.right = in.right;
-	    return;
-	  }
-	}
+        if (next_use != out_.right.length) {
+          left_done_ = true;
+          if (!next_use) {
+            out_.right = in.right;
+            return;
+          }
+        }
         std::swap(back, back2);
       }
 

moses/src/LanguageModelKen.cpp

@@ -163,8 +163,7 @@ FFState *LanguageModelKen<Model>::EvaluateChart(
 {
   LanguageModelChartStateKenLM *newState = new LanguageModelChartStateKenLM(hypo);
 
-  const lm::ngram::ChartState &chartState = newState->GetChartState();
-  chartState.CreatePreAndSuffices(hypo); 
+  newState->GetChartState().CreatePreAndSuffices(hypo); 
   
   lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState->GetChartState());
   const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();