moses_chart: rule lookup now produces a vector of stack pointers (and a

TargetPhraseCollection pointer) instead of a CYK+ dotted rule.

moses-chart-cmd/src/IOWrapper.cpp

@@ -227,14 +227,71 @@ void OutputInput(std::ostream& os, const ChartHypothesis* hypo)
 }
 */
 
-void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, long translationId)
+namespace {
+
+typedef std::vector<std::pair<Word, WordsRange> > ApplicationContext;
+
+// Given a hypothesis and sentence, reconstructs the 'application context' --
+// the source RHS symbols of the SCFG rule that was applied, plus their spans.
+void ReconstructApplicationContext(const ChartHypothesis &hypo,
+                                   const Sentence &sentence,
+                                   ApplicationContext &context)
 {
+  context.clear();
+  const std::vector<const ChartHypothesis*> &prevHypos = hypo.GetPrevHypos();
+  std::vector<const ChartHypothesis*>::const_iterator p = prevHypos.begin();
+  std::vector<const ChartHypothesis*>::const_iterator end = prevHypos.end();
+  const WordsRange &span = hypo.GetCurrSourceRange();
+  size_t i = span.GetStartPos();
+  while (i <= span.GetEndPos()) {
+    if (p == end || i < (*p)->GetCurrSourceRange().GetStartPos()) {
+      // Symbol is a terminal.
+      const Word &symbol = sentence.GetWord(i);
+      context.push_back(std::make_pair(symbol, WordsRange(i, i)));
+      ++i;
+    } else {
+      // Symbol is a non-terminal.
+      const Word &symbol = (*p)->GetTargetLHS();
+      const WordsRange &range = (*p)->GetCurrSourceRange();
+      context.push_back(std::make_pair(symbol, range));
+      i = range.GetEndPos()+1;
+      ++p;
+    }
+  }
+}
+
+// Emulates the old operator<<(ostream &, const DottedRule &) function.  The
+// output format is a bit odd (reverse order and double spacing between symbols)
+// but there are scripts and tools that expect the output of -T to look like
+// that.
+void WriteApplicationContext(std::ostream &out,
+                             const ApplicationContext &context)
+{
+  assert(!context.empty());
+  ApplicationContext::const_reverse_iterator p = context.rbegin();
+  while (true) {
+    out << p->second << "=" << p->first << " ";
+    if (++p == context.rend()) {
+      break;
+    }
+    out << " ";
+  }
+}
+
+}  // anonymous namespace
+
+void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
+{
+  static ApplicationContext applicationContext;
+
   // recursive
   if (hypo != NULL) {
+    ReconstructApplicationContext(*hypo, sentence, applicationContext);
     out << "Trans Opt " << translationId
         << " " << hypo->GetCurrSourceRange()
-        << ": " << hypo->GetTranslationOption().GetDottedRule()
-        << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
+        << ": ";
+    WriteApplicationContext(out, applicationContext);
+    out << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
         << "->" << hypo->GetCurrTargetPhrase()
         << " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown()
         << endl;
@@ -244,19 +301,20 @@ void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, lo
   std::vector<const ChartHypothesis*>::const_iterator iter;
   for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
     const ChartHypothesis *prevHypo = *iter;
-    OutputTranslationOptions(out, prevHypo, translationId);
+    OutputTranslationOptions(out, prevHypo, sentence, translationId);
   }
 }
 
 void IOWrapper::OutputDetailedTranslationReport(
   const ChartHypothesis *hypo,
+  const Sentence &sentence,
   long translationId)
 {
   if (hypo == NULL) {
     return;
   }
   std::ostringstream out;
-  OutputTranslationOptions(out, hypo, translationId);
+  OutputTranslationOptions(out, hypo, sentence, translationId);
   CHECK(m_detailOutputCollector);
   m_detailOutputCollector->Write(translationId, out.str());
 }

moses-chart-cmd/src/IOWrapper.h

@@ -82,7 +82,7 @@ public:
   void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
   void OutputBestHypo(const std::vector<const Moses::Factor*>&  mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors);
   void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartHypothesis *bestHypo, const Moses::TranslationSystem* system, long translationId);
-  void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, long translationId);
+  void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
   void Backtrack(const Moses::ChartHypothesis *hypo);
 
   void ResetTranslationId();

moses-chart-cmd/src/Main.cpp

@@ -98,7 +98,8 @@ public:
     }
 
     if (staticData.IsDetailedTranslationReportingEnabled()) {
-      m_ioWrapper.OutputDetailedTranslationReport(bestHypo, lineNumber);
+      const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
+      m_ioWrapper.OutputDetailedTranslationReport(bestHypo, sentence, lineNumber);
     }
 
     // n-best

moses/src/ChartCellLabel.h

@@ -19,20 +19,20 @@
 
 #pragma once
 
+#include "HypoList.h"
 #include "Word.h"
 #include "WordsRange.h"
 
 namespace Moses
 {
 
-class ChartHypothesisCollection;
 class Word;
 
 class ChartCellLabel
 {
  public:
   ChartCellLabel(const WordsRange &coverage, const Word &label,
-                 const ChartHypothesisCollection *stack=NULL)
+                 const HypoList *stack=NULL)
     : m_coverage(coverage)
     , m_label(label)
     , m_stack(stack)
@@ -40,7 +40,7 @@ class ChartCellLabel
 
   const WordsRange &GetCoverage() const { return m_coverage; }
   const Word &GetLabel() const { return m_label; }
-  const ChartHypothesisCollection *GetStack() const { return m_stack; }
+  const HypoList *GetStack() const { return m_stack; }
 
   bool operator<(const ChartCellLabel &other) const
   {
@@ -55,7 +55,7 @@ class ChartCellLabel
  private:
   const WordsRange &m_coverage;
   const Word &m_label;
-  const ChartHypothesisCollection *m_stack;
+  const HypoList *m_stack;
 };
 
 }

moses/src/ChartCellLabelSet.h

@@ -55,9 +55,10 @@ class ChartCellLabelSet
     m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w)));
   }
 
-  void AddConstituent(const Word &w, const ChartHypothesisCollection &stack)
+  void AddConstituent(const Word &w, const ChartHypothesisCollection &coll)
   {
-    m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w, &stack)));
+    const HypoList *stack = &(coll.GetSortedHypotheses());
+    m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w, stack)));
   }
 
   bool Empty() const { return m_map.empty(); }

moses/src/ChartHypothesis.cpp

@@ -1,4 +1,3 @@
-// $Id$
 // vim:tabstop=2
 /***********************************************************************
  Moses - factored phrase-based language decoder
@@ -45,7 +44,6 @@ ChartHypothesis::ChartHypothesis(const ChartTranslationOption &transOpt,
                                  const RuleCubeItem &item,
                                  ChartManager &manager)
   :m_targetPhrase(*(item.GetTranslationDimension().GetTargetPhrase()))
-  ,m_transOpt(transOpt)
   ,m_currSourceWordsRange(transOpt.GetSourceWordsRange())
   ,m_ffStates(manager.GetTranslationSystem()->GetStatefulFeatureFunctions().size())
   ,m_arcList(NULL)

moses/src/ChartHypothesis.h

@@ -1,4 +1,3 @@
-// $Id$
 // vim:tabstop=2
 /***********************************************************************
  Moses - factored phrase-based language decoder
@@ -48,7 +47,6 @@ protected:
 #endif
 
   const TargetPhrase &m_targetPhrase;
-  const ChartTranslationOption &m_transOpt;
 
   WordsRange					m_currSourceWordsRange;
 	std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
@@ -92,9 +90,6 @@ public:
 
   unsigned GetId() const { return m_id; }
 
-  const ChartTranslationOption &GetTranslationOption()const {
-    return m_transOpt;
-  }
   const TargetPhrase &GetCurrTargetPhrase()const {
     return m_targetPhrase;
   }

moses/src/ChartRuleLookupManagerCYKPlus.cpp

@@ -0,0 +1,69 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2012 University of Edinburgh
+ 
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+ 
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+ 
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#include "ChartRuleLookupManagerCYKPlus.h"
+
+#include "PhraseDictionarySCFG.h"
+#include "InputType.h"
+#include "ChartTranslationOptionList.h"
+#include "CellCollection.h"
+#include "DotChartInMemory.h"
+#include "StaticData.h"
+#include "NonTerminal.h"
+#include "ChartCellCollection.h"
+
+namespace Moses
+{
+
+void ChartRuleLookupManagerCYKPlus::AddCompletedRule(
+  const DottedRule &dottedRule,
+  const TargetPhraseCollection &tpc,
+  size_t ruleLimit,
+  bool adhereTableLimit,
+  ChartTranslationOptionList &outColl)
+{
+  // Determine the rule's rank.
+  size_t rank = 0;
+  const DottedRule *node = &dottedRule;
+  while (!node->IsRoot()) {
+    if (node->IsNonTerminal()) {
+      ++rank;
+    }
+    node = node->GetPrev();
+  }
+
+  // Fill m_stackVec with a stack pointer for each non-terminal.
+  m_stackVec.resize(rank);
+  node = &dottedRule;
+  while (rank > 0) {
+    if (node->IsNonTerminal()) {
+      // add the score of the best underlying hypothesis
+      const ChartCellLabel &cellLabel = node->GetChartCellLabel();
+      const HypoList *stack = cellLabel.GetStack();
+      assert(stack);
+      m_stackVec[--rank] = stack;
+    }
+    node = node->GetPrev();
+  }
+
+  // Add the (TargetPhraseCollection, StackVec) pair to the collection.
+  outColl.Add(tpc, m_stackVec, adhereTableLimit, ruleLimit);
+}
+
+}  // namespace Moses

moses/src/ChartRuleLookupManagerCYKPlus.h

@@ -0,0 +1,53 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2012 University of Edinburgh
+ 
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+ 
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+ 
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+***********************************************************************/
+
+#pragma once
+#ifndef moses_ChartRuleLookupManagerCYKPlus_h
+#define moses_ChartRuleLookupManagerCYKPlus_h
+
+#include "ChartRuleLookupManager.h"
+#include "StackVec.h"
+
+namespace Moses
+{
+
+class DottedRule;
+class TargetPhraseCollection;
+
+class ChartRuleLookupManagerCYKPlus : public ChartRuleLookupManager
+{
+ public:
+  ChartRuleLookupManagerCYKPlus(const InputType &sentence,
+                                const ChartCellCollection &cellColl)
+    : ChartRuleLookupManager(sentence, cellColl) {}
+
+ protected:
+  void AddCompletedRule(
+    const DottedRule &dottedRule,
+    const TargetPhraseCollection &tpc,
+    size_t ruleLimit,
+    bool adhereTableLimit,
+    ChartTranslationOptionList &outColl);
+
+  StackVec m_stackVec;
+};
+
+}  // namespace Moses
+
+#endif

moses/src/ChartRuleLookupManagerMemory.cpp

@@ -35,7 +35,7 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
   const InputType &src,
   const ChartCellCollection &cellColl,
   const PhraseDictionarySCFG &ruleTable)
-  : ChartRuleLookupManager(src, cellColl)
+  : ChartRuleLookupManagerCYKPlus(src, cellColl)
   , m_ruleTable(ruleTable)
 {
   CHECK(m_dottedRuleColls.size() == 0);
@@ -152,25 +152,24 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
   DottedRuleList &rules = dottedRuleCol.Get(relEndPos + 1);
 
   // look up target sides for the rules
-  size_t rulesLimit = StaticData::Instance().GetRuleLimit();
+  const size_t ruleLimit = StaticData::Instance().GetRuleLimit();
   DottedRuleList::const_iterator iterRule;
   for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) {
     const DottedRuleInMemory &dottedRule = **iterRule;
     const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode();
 
     // look up target sides
-    const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
+    const TargetPhraseCollection *tpc = node.GetTargetPhraseCollection();
 
     // add the fully expanded rule (with lexical target side)
-    if (targetPhraseCollection != NULL) {
-      outColl.Add(*targetPhraseCollection, dottedRule,
-                  GetCellCollection(), adhereTableLimit, rulesLimit);
+    if (tpc != NULL) {
+      AddCompletedRule(dottedRule, *tpc, ruleLimit, adhereTableLimit, outColl);
     }
   }
 
   dottedRuleCol.Clear(relEndPos+1);
 
-  outColl.CreateChartRules(rulesLimit);
+  outColl.CreateChartRules(ruleLimit);
 }
 
 // Given a partial rule application ending at startPos-1 and given the sets of

moses/src/ChartRuleLookupManagerMemory.h

@@ -27,11 +27,12 @@
 #include <boost/pool/object_pool.hpp>
 #endif
 
-#include "ChartRuleLookupManager.h"
+#include "ChartRuleLookupManagerCYKPlus.h"
 #include "DotChartInMemory.h"
 #include "NonTerminal.h"
 #include "PhraseDictionaryNodeSCFG.h"
 #include "PhraseDictionarySCFG.h"
+#include "StackVec.h"
 
 namespace Moses
 {
@@ -41,9 +42,9 @@ class DottedRuleColl;
 class WordsRange;
 
 // Implementation of ChartRuleLookupManager for in-memory rule tables.
-class ChartRuleLookupManagerMemory : public ChartRuleLookupManager
+class ChartRuleLookupManagerMemory : public ChartRuleLookupManagerCYKPlus
 {
-public:
+ public:
   ChartRuleLookupManagerMemory(const InputType &sentence,
                                const ChartCellCollection &cellColl,
                                const PhraseDictionarySCFG &ruleTable);
@@ -55,7 +56,7 @@ public:
     bool adhereTableLimit,
     ChartTranslationOptionList &outColl);
 
-private:
+ private:
   void ExtendPartialRuleApplication(
     const DottedRuleInMemory &prevDottedRule,
     size_t startPos,

moses/src/ChartRuleLookupManagerOnDisk.cpp

@@ -43,7 +43,7 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk(
   const std::vector<FactorType> &outputFactorsVec,
   const std::vector<float> &weight,
   const std::string &filePath)
-  : ChartRuleLookupManager(sentence, cellColl)
+  : ChartRuleLookupManagerCYKPlus(sentence, cellColl)
   , m_dictionary(dictionary)
   , m_dbWrapper(dbWrapper)
   , m_languageModels(languageModels)
@@ -260,8 +260,8 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
 
           CHECK(targetPhraseCollection);
           if (!targetPhraseCollection->IsEmpty()) {
-            outColl.Add(*targetPhraseCollection, prevDottedRule,
-                        GetCellCollection(), adhereTableLimit, rulesLimit);
+            AddCompletedRule(prevDottedRule, *targetPhraseCollection,
+                             rulesLimit, adhereTableLimit, outColl);
           }
 
         } // if (node)

moses/src/ChartRuleLookupManagerOnDisk.h

@@ -23,7 +23,7 @@
 
 #include "../../OnDiskPt/OnDiskWrapper.h"
 
-#include "ChartRuleLookupManager.h"
+#include "ChartRuleLookupManagerCYKPlus.h"
 #include "ChartTranslationOptionList.h"
 #include "DotChartOnDisk.h"
 #include "InputType.h"
@@ -33,9 +33,9 @@ namespace Moses
 {
 
 // Implementation of ChartRuleLookupManager for on-disk rule tables.
-class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManager
+class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManagerCYKPlus
 {
-public:
+ public:
   ChartRuleLookupManagerOnDisk(const InputType &sentence,
                                const ChartCellCollection &cellColl,
                                const PhraseDictionaryOnDisk &dictionary,
@@ -53,7 +53,7 @@ public:
                                       bool adhereTableLimit,
                                       ChartTranslationOptionList &outColl);
 
-private:
+ private:
   const PhraseDictionaryOnDisk &m_dictionary;
   OnDiskPt::OnDiskWrapper &m_dbWrapper;
   const LMList *m_languageModels;

moses/src/ChartTranslationOption.cpp

@@ -1,4 +1,3 @@
-// $Id$
 /***********************************************************************
  Moses - factored phrase-based language decoder
  Copyright (C) 2010 Hieu Hoang
@@ -20,33 +19,23 @@
 
 #include "ChartTranslationOption.h"
 
-#include "AlignmentInfo.h"
-#include "ChartCellCollection.h"
-#include "DotChart.h"
-
-#include <vector>
+#include "ChartHypothesis.h"
 
 namespace Moses
 {
 
-void ChartTranslationOption::CalcEstimateOfBestScore(
-    const ChartCellCollection &allChartCells)
+void ChartTranslationOption::CalcEstimateOfBestScore()
 {
   const TargetPhrase &targetPhrase = **(m_targetPhraseCollection.begin());
   m_estimateOfBestScore = targetPhrase.GetFutureScore();
 
-  const DottedRule *rule = &m_dottedRule;
-
-  // only deal with non-terminals
-  while (!rule->IsRoot()) {
-    if (rule->IsNonTerminal()) {
-      // add the score of the best underlying hypothesis
-      const ChartCellLabel &cellLabel = rule->GetChartCellLabel();
-      const ChartHypothesisCollection *hypoColl = cellLabel.GetStack();
-      CHECK(hypoColl);
-      m_estimateOfBestScore += hypoColl->GetBestScore();
-    }
-    rule = rule->GetPrev();
+  for (StackVec::const_iterator p = m_stackVec.begin(); p != m_stackVec.end();
+       ++p) {
+    const HypoList *stack = *p;
+    assert(stack);
+    assert(!stack->empty());
+    const ChartHypothesis &bestHypo = **(stack->begin());
+    m_estimateOfBestScore += bestHypo.GetTotalScore();
   }
 }
 

moses/src/ChartTranslationOption.h

@@ -1,4 +1,3 @@
-// $Id$
 /***********************************************************************
  Moses - factored phrase-based language decoder
  Copyright (C) 2010 Hieu Hoang
@@ -20,6 +19,7 @@
 
 #pragma once
 
+#include "StackVec.h"
 #include "TargetPhrase.h"
 #include "TargetPhraseCollection.h"
 #include "WordsRange.h"
@@ -30,29 +30,25 @@
 namespace Moses
 {
 
-class DottedRule;
-class ChartCellCollection;
-
 // Similar to a DottedRule, but contains a direct reference to a list
 // of translations and provdes an estimate of the best score.
 class ChartTranslationOption
 {
  public:
   ChartTranslationOption(const TargetPhraseCollection &targetPhraseColl,
-                         const DottedRule &dottedRule,
-                         const WordsRange &wordsRange,
-                         const ChartCellCollection &allChartCells)
-    : m_dottedRule(dottedRule)
-    , m_targetPhraseCollection(targetPhraseColl)
-    , m_wordsRange(wordsRange)
-    , m_estimateOfBestScore(0)
+                         const StackVec &stackVec,
+                         const WordsRange &wordsRange)
+      : m_stackVec(stackVec)
+      , m_targetPhraseCollection(targetPhraseColl)
+      , m_wordsRange(wordsRange)
+      , m_estimateOfBestScore(0)
   {
-    CalcEstimateOfBestScore(allChartCells);
+    CalcEstimateOfBestScore();
   }
 
   ~ChartTranslationOption() {}
 
-  const DottedRule &GetDottedRule() const { return m_dottedRule; }
+  const StackVec &GetStackVec() const { return m_stackVec; }
 
   const TargetPhraseCollection &GetTargetPhraseCollection() const { 
     return m_targetPhraseCollection;
@@ -71,9 +67,9 @@ class ChartTranslationOption
   // not implemented
   ChartTranslationOption &operator=(const ChartTranslationOption &);
 
-  void CalcEstimateOfBestScore(const ChartCellCollection &);
+  void CalcEstimateOfBestScore();
 
-  const DottedRule &m_dottedRule;
+  const StackVec m_stackVec;
   const TargetPhraseCollection &m_targetPhraseCollection;
   const WordsRange &m_wordsRange;
   float m_estimateOfBestScore;

moses/src/ChartTranslationOptionCollection.cpp

@@ -1,4 +1,3 @@
-// $Id$
 // vim:tabstop=2
 /***********************************************************************
  Moses - factored phrase-based language decoder
@@ -59,15 +58,6 @@ ChartTranslationOptionCollection::~ChartTranslationOptionCollection()
 {
   RemoveAllInColl(m_unksrcs);
   RemoveAllInColl(m_cacheTargetPhraseCollection);
-
-  std::list<std::vector<DottedRule*>* >::iterator iterOuter;
-  for (iterOuter = m_dottedRuleCache.begin(); iterOuter != m_dottedRuleCache.end(); ++iterOuter) {
-    std::vector<DottedRule*> &inner = **iterOuter;
-    RemoveAllInColl(inner);
-  }
-
-  RemoveAllInColl(m_dottedRuleCache);
-
 }
 
 void ChartTranslationOptionCollection::CreateTranslationOptionsForRange(
@@ -203,12 +193,6 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
 
   //TranslationOption *transOpt;
   if (! staticData.GetDropUnknown() || isDigit) {
-    // create dotted rules
-    std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
-    m_dottedRuleCache.push_back(dottedRuleList);
-    dottedRuleList->push_back(new DottedRule());
-    dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
-
     // loop
     const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
     UnknownLHSList::const_iterator iterLHS;
@@ -243,9 +227,8 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
 
       // chart rule
       ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
-          , *dottedRuleList->back()
-          , range
-          , m_hypoStackColl);
+          , m_emptyStackVec
+          , range);
       transOptColl.Add(chartRule);
     } // for (iterLHS
   } else {
@@ -271,17 +254,10 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
       targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
       targetPhrase->SetTargetLHS(targetLHS);
 
-      // words consumed
-      std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
-      m_dottedRuleCache.push_back(dottedRuleList);
-      dottedRuleList->push_back(new DottedRule());
-      dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
-
       // chart rule
       ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
-          , *dottedRuleList->back()
-          , range
-          , m_hypoStackColl);
+          , m_emptyStackVec
+          , range);
       transOptColl.Add(chartRule);
     }
   }

moses/src/ChartTranslationOptionCollection.h

@@ -1,4 +1,3 @@
-// $Id$
 // vim:tabstop=2
 /***********************************************************************
  Moses - factored phrase-based language decoder
@@ -26,6 +25,7 @@
 #include "DecodeGraph.h"
 #include "ChartTranslationOptionList.h"
 #include "ChartRuleLookupManager.h"
+#include "StackVec.h"
 
 namespace Moses
 {
@@ -49,7 +49,7 @@ protected:
   std::vector< std::vector< ChartTranslationOptionList > > m_collection; /*< contains translation options */
   std::vector<Phrase*> m_unksrcs;
   std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection;
-  std::list<std::vector<DottedRule*>* > m_dottedRuleCache;
+  StackVec m_emptyStackVec;
 
   // for adding 1 trans opt in unknown word proc
   void Add(ChartTranslationOption *transOpt, size_t pos);

moses/src/ChartTranslationOptionList.cpp

@@ -54,8 +54,7 @@ public:
 };
 
 void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseCollection
-                                     , const DottedRule &dottedRule
-                                     , const ChartCellCollection &chartCellColl
+                                     , const StackVec &stackVec
                                      , bool /* adhereTableLimit */
                                      , size_t ruleLimit)
 {
@@ -66,15 +65,14 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseC
   if (m_collection.size() < ruleLimit) {
     // not yet filled out quota. add everything
     ChartTranslationOption *option = new ChartTranslationOption(
-        targetPhraseCollection, dottedRule, m_range, chartCellColl);
+        targetPhraseCollection, stackVec, m_range);
     m_collection.push_back(option);
     float score = option->GetEstimateOfBestScore();
     m_scoreThreshold = (score < m_scoreThreshold) ? score : m_scoreThreshold;
   }
   else {
     // full but not bursting. add if better than worst score
-    ChartTranslationOption option(targetPhraseCollection, dottedRule,
-                                  m_range, chartCellColl);
+    ChartTranslationOption option(targetPhraseCollection, stackVec, m_range);
     float score = option.GetEstimateOfBestScore();
     if (score > m_scoreThreshold) {
       // dynamic allocation deferred until here on the assumption that most

moses/src/ChartTranslationOptionList.h

@@ -100,8 +100,7 @@ public:
   }
 
   void Add(const TargetPhraseCollection &targetPhraseCollection
-           , const DottedRule &dottedRule
-           , const ChartCellCollection &
+           , const StackVec &stackVec
            , bool ruleLimit
            , size_t tableLimit);
   void Add(ChartTranslationOption *transOpt);

moses/src/ChartTrellisNode.cpp

@@ -78,10 +78,6 @@ Phrase ChartTrellisNode::GetOutputPhrase() const
   // exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
   Phrase ret(ARRAY_SIZE_INCR);
 
-  const ChartTranslationOption &transOpt = m_hypo.GetTranslationOption();
-
-  VERBOSE(3, "Trans Opt:" << transOpt.GetDottedRule() << ": " << m_hypo.GetCurrTargetPhrase().GetTargetLHS() << "->" << m_hypo.GetCurrTargetPhrase() << std::endl);
-
   const Phrase &currTargetPhrase = m_hypo.GetCurrTargetPhrase();
   const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
     m_hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();

moses/src/DotChart.h

@@ -26,8 +26,6 @@ namespace Moses
 
 class DottedRule
 {
-  friend std::ostream& operator<<(std::ostream &, const DottedRule &);
-
  public:
   // used only to init dot stack.
   DottedRule()

moses/src/DotChartOnDisk.cpp

@@ -54,16 +54,4 @@ void DottedRuleStackOnDisk::SortSavedNodes()
   sort(m_savedNode.begin(), m_savedNode.end(), SavedNodesOderer());
 }
 
-std::ostream& operator<<(std::ostream &out, const DottedRuleCollOnDisk &coll)
-{
-  DottedRuleCollOnDisk::CollType::const_iterator iter;
-  for (iter = coll.begin(); iter != coll.end(); ++iter) {
-    const DottedRuleOnDisk &rule = **iter;
-    out << rule << endl;
-
-  }
-
-  return out;
-}
-
 };

moses/src/DotChartOnDisk.h

@@ -60,8 +60,6 @@ class DottedRuleOnDisk : public DottedRule
 
 class DottedRuleCollOnDisk
 {
-  friend std::ostream& operator<<(std::ostream&, const DottedRuleCollOnDisk&);
-
 protected:
   typedef std::vector<const DottedRuleOnDisk*> CollType;
   CollType m_coll;

moses/src/HypoList.h

@@ -1,7 +1,6 @@
-// $Id$
 /***********************************************************************
- Moses - factored phrase-based language decoder
- Copyright (C) 2010 Hieu Hoang
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2012 University of Edinburgh
 
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
@@ -18,20 +17,15 @@
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  ***********************************************************************/
 
-#include "DotChart.h"
+#pragma once
+
+#include <vector>
 
 namespace Moses
 {
 
-std::ostream &operator<<(std::ostream &out, const DottedRule &rule)
-{
-  if (!rule.IsRoot()) {
-    out << rule.GetWordsRange() << "=" << rule.GetSourceWord() << " ";
-    if (!rule.m_prev->IsRoot()) {
-      out << " " << *rule.m_prev;
-    }
-  }
-  return out;
-}
+class ChartHypothesis;
+
+typedef std::vector<const ChartHypothesis*> HypoList;
 
 }

moses/src/RuleCubeItem.cpp

@@ -21,7 +21,6 @@
 #include "ChartCellCollection.h"
 #include "ChartTranslationOption.h"
 #include "ChartTranslationOptionCollection.h"
-#include "DotChart.h"
 #include "RuleCubeItem.h"
 #include "RuleCubeQueue.h"
 #include "WordsRange.h"
@@ -39,12 +38,12 @@ std::size_t hash_value(const HypothesisDimension &dimension)
 }
 
 RuleCubeItem::RuleCubeItem(const ChartTranslationOption &transOpt,
-                           const ChartCellCollection &allChartCells)
+                           const ChartCellCollection &/*allChartCells*/)
   : m_translationDimension(0,
                            transOpt.GetTargetPhraseCollection().GetCollection())
   , m_hypothesis(0)
 {
-  CreateHypothesisDimensions(transOpt.GetDottedRule(), allChartCells);
+  CreateHypothesisDimensions(transOpt.GetStackVec());
 }
 
 // create the RuleCube from an existing one, differing only in one dimension
@@ -94,31 +93,19 @@ ChartHypothesis *RuleCubeItem::ReleaseHypothesis()
 
 // for each non-terminal, create a ordered list of matching hypothesis from the
 // chart
-void RuleCubeItem::CreateHypothesisDimensions(
-  const DottedRule &dottedRule,
-  const ChartCellCollection &allChartCells)
+void RuleCubeItem::CreateHypothesisDimensions(const StackVec &stackVec)
 {
-  CHECK(!dottedRule.IsRoot());
-
-  const DottedRule *prev = dottedRule.GetPrev();
-  if (!prev->IsRoot()) {
-    CreateHypothesisDimensions(*prev, allChartCells);
-  }
-
-  // only deal with non-terminals
-  if (dottedRule.IsNonTerminal()) {
-    // get a sorted list of the underlying hypotheses
-    const ChartCellLabel &cellLabel = dottedRule.GetChartCellLabel();
-    const ChartHypothesisCollection *hypoColl = cellLabel.GetStack();
-    CHECK(hypoColl);
-    const HypoList &hypoList = hypoColl->GetSortedHypotheses();
+  for (StackVec::const_iterator p = stackVec.begin(); p != stackVec.end();
+       ++p) {
+    const HypoList *stack = *p;
+    assert(stack);
 
     // there have to be hypothesis with the desired non-terminal
     // (otherwise the rule would not be considered)
-    CHECK(!hypoList.empty());
+    assert(!stack->empty());
 
     // create a list of hypotheses that match the non-terminal
-    HypothesisDimension dimension(0, hypoList);
+    HypothesisDimension dimension(0, *stack);
     // add them to the vector for such lists
     m_hypothesisDimensions.push_back(dimension);
   }

moses/src/RuleCubeItem.h

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include "StackVec.h"
+
 #include <vector>
 
 namespace Moses
@@ -28,7 +30,6 @@ class ChartCellCollection;
 class ChartHypothesis;
 class ChartManager;
 class ChartTranslationOption;
-class DottedRule;
 class TargetPhrase;
 
 typedef std::vector<const ChartHypothesis*> HypoList;
@@ -130,8 +131,7 @@ class RuleCubeItem
   RuleCubeItem(const RuleCubeItem &);  // Not implemented
   RuleCubeItem &operator=(const RuleCubeItem &);  // Not implemented
 
-  void CreateHypothesisDimensions(const DottedRule &,
-                                  const ChartCellCollection &);
+  void CreateHypothesisDimensions(const StackVec &);
 
   TranslationDimension m_translationDimension;
   std::vector<HypothesisDimension> m_hypothesisDimensions;

moses/src/StackVec.h

@@ -0,0 +1,31 @@
+/***********************************************************************
+ Moses - statistical machine translation system
+ Copyright (C) 2006-2012 University of Edinburgh
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+
+#pragma once
+
+#include "HypoList.h"
+
+#include <vector>
+
+namespace Moses
+{
+
+typedef std::vector<const HypoList *> StackVec;
+
+}