mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
more logging for incremental chart search
This commit is contained in:
parent
59098ac664
commit
395285d879
@ -300,6 +300,38 @@ void IOWrapper::ReconstructApplicationContext(const ChartHypothesis &hypo,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Given a hypothesis and sentence, reconstructs the 'application context' --
|
||||
// the source RHS symbols of the SCFG rule that was applied, plus their spans.
|
||||
void IOWrapper::ReconstructApplicationContext(const search::Applied *applied,
|
||||
const Sentence &sentence,
|
||||
ApplicationContext &context)
|
||||
{
|
||||
context.clear();
|
||||
const WordsRange &span = applied->GetRange();
|
||||
const search::Applied *child = applied->Children();
|
||||
size_t i = span.GetStartPos();
|
||||
size_t j = 0;
|
||||
|
||||
while (i <= span.GetEndPos()) {
|
||||
if (j == applied->GetArity() || i < child->GetRange().GetStartPos()) {
|
||||
// Symbol is a terminal.
|
||||
const Word &symbol = sentence.GetWord(i);
|
||||
context.push_back(std::make_pair(symbol, WordsRange(i, i)));
|
||||
++i;
|
||||
} else {
|
||||
// Symbol is a non-terminal.
|
||||
const Word &symbol = static_cast<const TargetPhrase*>(child->GetNote().vp)->GetTargetLHS();
|
||||
const WordsRange &range = child->GetRange();
|
||||
context.push_back(std::make_pair(symbol, range));
|
||||
i = range.GetEndPos()+1;
|
||||
++child;
|
||||
++j;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Emulates the old operator<<(ostream &, const DottedRule &) function. The
|
||||
// output format is a bit odd (reverse order and double spacing between symbols)
|
||||
// but there are scripts and tools that expect the output of -T to look like
|
||||
@ -330,6 +362,20 @@ void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &a
|
||||
<< " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown();
|
||||
}
|
||||
|
||||
void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||
{
|
||||
ReconstructApplicationContext(applied, sentence, applicationContext);
|
||||
const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
|
||||
out << "Trans Opt " << translationId
|
||||
<< " " << applied->GetRange()
|
||||
<< ": ";
|
||||
WriteApplicationContext(out, applicationContext);
|
||||
out << ": " << phrase.GetTargetLHS()
|
||||
<< "->" << phrase
|
||||
<< " " << applied->GetScore(); // << hypo->GetScoreBreakdown() TODO: missing in incremental search hypothesis
|
||||
}
|
||||
|
||||
|
||||
void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
||||
{
|
||||
if (hypo != NULL) {
|
||||
@ -346,6 +392,21 @@ void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||
{
|
||||
if (applied != NULL) {
|
||||
OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
// recursive
|
||||
const search::Applied *child = applied->Children();
|
||||
for (size_t i = 0; i < applied->GetArity(); i++) {
|
||||
OutputTranslationOptions(out, applicationContext, child++, sentence, translationId);
|
||||
}
|
||||
}
|
||||
|
||||
void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
||||
{
|
||||
|
||||
@ -375,6 +436,33 @@ void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, Applica
|
||||
}
|
||||
}
|
||||
|
||||
void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||
{
|
||||
|
||||
if (applied != NULL) {
|
||||
OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
|
||||
|
||||
const std::string key = "Tree";
|
||||
std::string value;
|
||||
bool hasProperty;
|
||||
const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
|
||||
currTarPhr.GetProperty(key, value, hasProperty);
|
||||
|
||||
out << " ||| ";
|
||||
if (hasProperty)
|
||||
out << " " << value;
|
||||
else
|
||||
out << " " << "noTreeInfo";
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
// recursive
|
||||
const search::Applied *child = applied->Children();
|
||||
for (size_t i = 0; i < applied->GetArity(); i++) {
|
||||
OutputTreeFragmentsTranslationOptions(out, applicationContext, child++, sentence, translationId);
|
||||
}
|
||||
}
|
||||
|
||||
void IOWrapper::OutputDetailedTranslationReport(
|
||||
const ChartHypothesis *hypo,
|
||||
const Sentence &sentence,
|
||||
@ -392,6 +480,23 @@ void IOWrapper::OutputDetailedTranslationReport(
|
||||
m_detailOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::OutputDetailedTranslationReport(
|
||||
const search::Applied *applied,
|
||||
const Sentence &sentence,
|
||||
long translationId)
|
||||
{
|
||||
if (applied == NULL) {
|
||||
return;
|
||||
}
|
||||
std::ostringstream out;
|
||||
ApplicationContext applicationContext;
|
||||
|
||||
OutputTranslationOptions(out, applicationContext, applied, sentence, translationId);
|
||||
UTIL_THROW_IF2(m_detailOutputCollector == NULL,
|
||||
"No ouput file for detailed reports specified");
|
||||
m_detailOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
||||
void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
||||
const ChartHypothesis *hypo,
|
||||
const Sentence &sentence,
|
||||
@ -424,6 +529,28 @@ void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
||||
|
||||
}
|
||||
|
||||
void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
||||
const search::Applied *applied,
|
||||
const Sentence &sentence,
|
||||
long translationId)
|
||||
{
|
||||
if (applied == NULL) {
|
||||
return;
|
||||
}
|
||||
std::ostringstream out;
|
||||
ApplicationContext applicationContext;
|
||||
|
||||
OutputTreeFragmentsTranslationOptions(out, applicationContext, applied, sentence, translationId);
|
||||
UTIL_THROW_IF2(m_detailTreeFragmentsOutputCollector == NULL,
|
||||
"No output file for tree fragments specified");
|
||||
|
||||
//Tree of full sentence
|
||||
//TODO: incremental search doesn't support stateful features
|
||||
|
||||
m_detailTreeFragmentsOutputCollector->Write(translationId, out.str());
|
||||
|
||||
}
|
||||
|
||||
//DIMw
|
||||
void IOWrapper::OutputDetailedAllTranslationReport(
|
||||
const ChartTrellisPathList &nBestList,
|
||||
|
@ -93,11 +93,17 @@ protected:
|
||||
size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
|
||||
void OutputAlignment(std::vector< std::set<size_t> > &retAlignmentsS2T, const Moses::AlignmentInfo &ai);
|
||||
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||
void ReconstructApplicationContext(const Moses::ChartHypothesis &hypo,
|
||||
const Moses::Sentence &sentence,
|
||||
ApplicationContext &context);
|
||||
void ReconstructApplicationContext(const search::Applied *applied,
|
||||
const Moses::Sentence &sentence,
|
||||
ApplicationContext &context);
|
||||
void WriteApplicationContext(std::ostream &out,
|
||||
const ApplicationContext &context);
|
||||
|
||||
@ -125,7 +131,9 @@ public:
|
||||
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, long translationId);
|
||||
void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
|
||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputDetailedTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputDetailedTreeFragmentsTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||
void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
|
||||
void Backtrack(const Moses::ChartHypothesis *hypo);
|
||||
|
||||
|
@ -102,6 +102,14 @@ public:
|
||||
const std::vector<search::Applied> &nbest = manager.ProcessSentence();
|
||||
if (!nbest.empty()) {
|
||||
m_ioWrapper.OutputBestHypo(nbest[0], translationId);
|
||||
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
|
||||
m_ioWrapper.OutputDetailedTranslationReport(&nbest[0], sentence, translationId);
|
||||
}
|
||||
if (staticData.IsDetailedTreeFragmentsTranslationReportingEnabled()) {
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
|
||||
m_ioWrapper.OutputDetailedTreeFragmentsTranslationReport(&nbest[0], sentence, translationId);
|
||||
}
|
||||
} else {
|
||||
m_ioWrapper.OutputBestNone(translationId);
|
||||
}
|
||||
|
@ -152,12 +152,13 @@ template <class Model> void Fill<Model>::Add(const TargetPhraseCollection &targe
|
||||
search::Note note;
|
||||
note.vp = &phrase;
|
||||
edge.SetNote(note);
|
||||
edge.SetRange(range);
|
||||
|
||||
edges_.AddEdge(edge);
|
||||
}
|
||||
}
|
||||
|
||||
template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &)
|
||||
template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &range)
|
||||
{
|
||||
std::vector<lm::WordIndex> words;
|
||||
UTIL_THROW_IF2(phrase.GetSize() > 1,
|
||||
@ -173,6 +174,7 @@ template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std:
|
||||
search::Note note;
|
||||
note.vp = &phrase;
|
||||
edge.SetNote(note);
|
||||
edge.SetRange(range);
|
||||
|
||||
edges_.AddEdge(edge);
|
||||
}
|
||||
|
@ -24,9 +24,10 @@ template <class Below> class GenericApplied : public Header {
|
||||
*child_out = Below(part->End());
|
||||
}
|
||||
|
||||
GenericApplied(void *location, Score score, Arity arity, Note note) : Header(location, arity) {
|
||||
GenericApplied(void *location, Score score, Arity arity, Note note, Moses::WordsRange range) : Header(location, arity) {
|
||||
SetScore(score);
|
||||
SetNote(note);
|
||||
SetRange(range);
|
||||
}
|
||||
|
||||
explicit GenericApplied(History from) : Header(from) {}
|
||||
|
@ -80,6 +80,7 @@ template <class Model> PartialEdge EdgeGenerator::Pop(Context<Model> &context) {
|
||||
alternate.SetScore(top.GetScore() + alternate_changed.Bound() - old_value.Bound());
|
||||
|
||||
alternate.SetNote(top.GetNote());
|
||||
alternate.SetRange(top.GetRange());
|
||||
|
||||
PartialVertex *alternate_nt = alternate.NT();
|
||||
for (Arity i = 0; i < victim; ++i) alternate_nt[i] = top_nt[i];
|
||||
|
@ -1,9 +1,10 @@
|
||||
#ifndef SEARCH_HEADER__
|
||||
#define SEARCH_HEADER__
|
||||
|
||||
// Header consisting of Score, Arity, and Note
|
||||
// Header consisting of Score, Arity, Note and WordsRange
|
||||
|
||||
#include "search/types.hh"
|
||||
#include "moses/WordsRange.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
@ -38,6 +39,13 @@ class Header {
|
||||
*reinterpret_cast<Note*>(base_ + sizeof(Score) + sizeof(Arity)) = to;
|
||||
}
|
||||
|
||||
Moses::WordsRange GetRange() const {
|
||||
return *reinterpret_cast<const Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note));
|
||||
}
|
||||
void SetRange(Moses::WordsRange to) {
|
||||
*reinterpret_cast<Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note)) = to;
|
||||
}
|
||||
|
||||
uint8_t *Base() { return base_; }
|
||||
const uint8_t *Base() const { return base_; }
|
||||
|
||||
@ -50,7 +58,7 @@ class Header {
|
||||
*reinterpret_cast<Arity*>(base_ + sizeof(Score)) = arity;
|
||||
}
|
||||
|
||||
static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note);
|
||||
static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note) + sizeof(Moses::WordsRange);
|
||||
|
||||
uint8_t *After() { return base_ + kHeaderSize; }
|
||||
const uint8_t *After() const { return base_ + kHeaderSize; }
|
||||
|
@ -70,7 +70,7 @@ void NBestList::MoveTop(util::Pool &pool) {
|
||||
Score change = child->in_->Visit(pool, child->index_);
|
||||
if (change != -INFINITY) {
|
||||
assert(change < 0.001);
|
||||
QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote());
|
||||
QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote(), entry.GetRange());
|
||||
std::copy(children_begin, child, new_entry.Children());
|
||||
RevealedRef *update = new_entry.Children() + (child - children_begin);
|
||||
update->in_ = child->in_;
|
||||
|
Loading…
Reference in New Issue
Block a user