more logging for incremental chart search

moses-chart-cmd/IOWrapper.cpp

@@ -300,6 +300,38 @@ void IOWrapper::ReconstructApplicationContext(const ChartHypothesis &hypo,
   }
 }
 
+
+// Given a hypothesis and sentence, reconstructs the 'application context' --
+// the source RHS symbols of the SCFG rule that was applied, plus their spans.
+void IOWrapper::ReconstructApplicationContext(const search::Applied *applied,
+    const Sentence &sentence,
+    ApplicationContext &context)
+{
+  context.clear();
+  const WordsRange &span = applied->GetRange();
+  const search::Applied *child = applied->Children();
+  size_t i = span.GetStartPos();
+  size_t j = 0;
+
+  while (i <= span.GetEndPos()) {
+    if (j == applied->GetArity() || i < child->GetRange().GetStartPos()) {
+      // Symbol is a terminal.
+      const Word &symbol = sentence.GetWord(i);
+      context.push_back(std::make_pair(symbol, WordsRange(i, i)));
+      ++i;
+    } else {
+      // Symbol is a non-terminal.
+      const Word &symbol = static_cast<const TargetPhrase*>(child->GetNote().vp)->GetTargetLHS();
+      const WordsRange &range = child->GetRange();
+      context.push_back(std::make_pair(symbol, range));
+      i = range.GetEndPos()+1;
+      ++child;
+      ++j;
+    }
+  }
+}
+
+
 // Emulates the old operator<<(ostream &, const DottedRule &) function.  The
 // output format is a bit odd (reverse order and double spacing between symbols)
 // but there are scripts and tools that expect the output of -T to look like
@@ -330,6 +362,20 @@ void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &a
       << " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown();
 }
 
+void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
+{
+  ReconstructApplicationContext(applied, sentence, applicationContext);
+  const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
+  out << "Trans Opt " << translationId
+      << " " << applied->GetRange()
+      << ": ";
+  WriteApplicationContext(out, applicationContext);
+  out << ": " << phrase.GetTargetLHS()
+      << "->" << phrase
+      << " " << applied->GetScore(); // << hypo->GetScoreBreakdown() TODO: missing in incremental search hypothesis
+}
+
+
 void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
 {
   if (hypo != NULL) {
@@ -346,6 +392,21 @@ void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &
   }
 }
 
+
+void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
+{
+  if (applied != NULL) {
+    OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
+    out << std::endl;
+  }
+
+  // recursive
+  const search::Applied *child = applied->Children();
+  for (size_t i = 0; i < applied->GetArity(); i++) {
+      OutputTranslationOptions(out, applicationContext, child++, sentence, translationId);
+  }
+}
+
 void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
 {
 
@@ -375,6 +436,33 @@ void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, Applica
   }
 }
 
+void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
+{
+
+  if (applied != NULL) {
+    OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
+
+    const std::string key = "Tree";
+    std::string value;
+    bool hasProperty;
+    const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
+    currTarPhr.GetProperty(key, value, hasProperty);
+
+    out << " ||| ";
+    if (hasProperty)
+      out << " " << value;
+    else
+      out << " " << "noTreeInfo";
+    out << std::endl;
+  }
+
+  // recursive
+  const search::Applied *child = applied->Children();
+  for (size_t i = 0; i < applied->GetArity(); i++) {
+      OutputTreeFragmentsTranslationOptions(out, applicationContext, child++, sentence, translationId);
+  }
+}
+
 void IOWrapper::OutputDetailedTranslationReport(
   const ChartHypothesis *hypo,
   const Sentence &sentence,
@@ -392,6 +480,23 @@ void IOWrapper::OutputDetailedTranslationReport(
   m_detailOutputCollector->Write(translationId, out.str());
 }
 
+void IOWrapper::OutputDetailedTranslationReport(
+  const search::Applied *applied,
+  const Sentence &sentence,
+  long translationId)
+{
+  if (applied == NULL) {
+    return;
+  }
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTranslationOptions(out, applicationContext, applied, sentence, translationId);
+  UTIL_THROW_IF2(m_detailOutputCollector == NULL,
+                  "No ouput file for detailed reports specified");
+  m_detailOutputCollector->Write(translationId, out.str());
+}
+
 void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
   const ChartHypothesis *hypo,
   const Sentence &sentence,
@@ -424,6 +529,28 @@ void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
 
 }
 
+void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
+  const search::Applied *applied,
+  const Sentence &sentence,
+  long translationId)
+{
+  if (applied == NULL) {
+    return;
+  }
+  std::ostringstream out;
+  ApplicationContext applicationContext;
+
+  OutputTreeFragmentsTranslationOptions(out, applicationContext, applied, sentence, translationId);
+  UTIL_THROW_IF2(m_detailTreeFragmentsOutputCollector == NULL,
+                  "No output file for tree fragments specified");
+
+  //Tree of full sentence
+  //TODO: incremental search doesn't support stateful features
+
+  m_detailTreeFragmentsOutputCollector->Write(translationId, out.str());
+
+}
+
 //DIMw
 void IOWrapper::OutputDetailedAllTranslationReport(
   const ChartTrellisPathList &nBestList,

moses-chart-cmd/IOWrapper.h

@@ -93,11 +93,17 @@ protected:
   size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
   void OutputAlignment(std::vector< std::set<size_t> > &retAlignmentsS2T, const Moses::AlignmentInfo &ai);
   void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
+  void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
   void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
+  void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
   void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
+  void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
   void ReconstructApplicationContext(const Moses::ChartHypothesis &hypo,
                                      const Moses::Sentence &sentence,
                                      ApplicationContext &context);
+  void ReconstructApplicationContext(const search::Applied *applied,
+                                     const Moses::Sentence &sentence,
+                                     ApplicationContext &context);
   void WriteApplicationContext(std::ostream &out,
                                const ApplicationContext &context);
 
@@ -125,7 +131,9 @@ public:
   void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, long translationId);
   void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
   void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
+  void OutputDetailedTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
   void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
+  void OutputDetailedTreeFragmentsTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
   void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
   void Backtrack(const Moses::ChartHypothesis *hypo);
 

moses-chart-cmd/Main.cpp

@@ -102,6 +102,14 @@ public:
       const std::vector<search::Applied> &nbest = manager.ProcessSentence();
       if (!nbest.empty()) {
         m_ioWrapper.OutputBestHypo(nbest[0], translationId);
+        if (staticData.IsDetailedTranslationReportingEnabled()) {
+          const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
+          m_ioWrapper.OutputDetailedTranslationReport(&nbest[0], sentence, translationId);
+        }
+        if (staticData.IsDetailedTreeFragmentsTranslationReportingEnabled()) {
+          const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
+          m_ioWrapper.OutputDetailedTreeFragmentsTranslationReport(&nbest[0], sentence, translationId);
+        }
       } else {
         m_ioWrapper.OutputBestNone(translationId);
       }

moses/Incremental.cpp

@@ -152,12 +152,13 @@ template <class Model> void Fill<Model>::Add(const TargetPhraseCollection &targe
     search::Note note;
     note.vp = &phrase;
     edge.SetNote(note);
+    edge.SetRange(range);
 
     edges_.AddEdge(edge);
   }
 }
 
-template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &)
+template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &range)
 {
   std::vector<lm::WordIndex> words;
   UTIL_THROW_IF2(phrase.GetSize() > 1,
@@ -173,6 +174,7 @@ template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std:
   search::Note note;
   note.vp = &phrase;
   edge.SetNote(note);
+  edge.SetRange(range);
 
   edges_.AddEdge(edge);
 }

search/applied.hh

@@ -24,9 +24,10 @@ template <class Below> class GenericApplied : public Header {
         *child_out = Below(part->End());
     }
     
-    GenericApplied(void *location, Score score, Arity arity, Note note) : Header(location, arity) {
+    GenericApplied(void *location, Score score, Arity arity, Note note, Moses::WordsRange range) : Header(location, arity) {
       SetScore(score);
       SetNote(note);
+      SetRange(range);
     }
 
     explicit GenericApplied(History from) : Header(from) {}

search/edge_generator.cc

@@ -80,6 +80,7 @@ template <class Model> PartialEdge EdgeGenerator::Pop(Context<Model> &context) {
     alternate.SetScore(top.GetScore() + alternate_changed.Bound() - old_value.Bound());
 
     alternate.SetNote(top.GetNote());
+    alternate.SetRange(top.GetRange());
 
     PartialVertex *alternate_nt = alternate.NT();
     for (Arity i = 0; i < victim; ++i) alternate_nt[i] = top_nt[i];

search/header.hh

@@ -1,9 +1,10 @@
 #ifndef SEARCH_HEADER__
 #define SEARCH_HEADER__
 
-// Header consisting of Score, Arity, and Note
+// Header consisting of Score, Arity, Note and WordsRange
 
 #include "search/types.hh"
+#include "moses/WordsRange.h"
 
 #include <stdint.h>
 
@@ -38,6 +39,13 @@ class Header {
       *reinterpret_cast<Note*>(base_ + sizeof(Score) + sizeof(Arity)) = to;
     }
 
+    Moses::WordsRange GetRange() const {
+      return *reinterpret_cast<const Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note));
+    }
+    void SetRange(Moses::WordsRange to) {
+      *reinterpret_cast<Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note)) = to;
+    }
+
     uint8_t *Base() { return base_; }
     const uint8_t *Base() const { return base_; }
 
@@ -50,7 +58,7 @@ class Header {
       *reinterpret_cast<Arity*>(base_ + sizeof(Score)) = arity;
     }
 
-    static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note);
+    static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note) + sizeof(Moses::WordsRange);
 
     uint8_t *After() { return base_ + kHeaderSize; }
     const uint8_t *After() const { return base_ + kHeaderSize; }

search/nbest.cc

@@ -70,7 +70,7 @@ void NBestList::MoveTop(util::Pool &pool) {
     Score change = child->in_->Visit(pool, child->index_);
     if (change != -INFINITY) {
       assert(change < 0.001);
-      QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote());
+      QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote(), entry.GetRange());
       std::copy(children_begin, child, new_entry.Children());
       RevealedRef *update = new_entry.Children() + (child - children_begin);
       update->in_ = child->in_;