mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
more logging for incremental chart search
This commit is contained in:
parent
59098ac664
commit
395285d879
@ -300,6 +300,38 @@ void IOWrapper::ReconstructApplicationContext(const ChartHypothesis &hypo,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Given a hypothesis and sentence, reconstructs the 'application context' --
|
||||||
|
// the source RHS symbols of the SCFG rule that was applied, plus their spans.
|
||||||
|
void IOWrapper::ReconstructApplicationContext(const search::Applied *applied,
|
||||||
|
const Sentence &sentence,
|
||||||
|
ApplicationContext &context)
|
||||||
|
{
|
||||||
|
context.clear();
|
||||||
|
const WordsRange &span = applied->GetRange();
|
||||||
|
const search::Applied *child = applied->Children();
|
||||||
|
size_t i = span.GetStartPos();
|
||||||
|
size_t j = 0;
|
||||||
|
|
||||||
|
while (i <= span.GetEndPos()) {
|
||||||
|
if (j == applied->GetArity() || i < child->GetRange().GetStartPos()) {
|
||||||
|
// Symbol is a terminal.
|
||||||
|
const Word &symbol = sentence.GetWord(i);
|
||||||
|
context.push_back(std::make_pair(symbol, WordsRange(i, i)));
|
||||||
|
++i;
|
||||||
|
} else {
|
||||||
|
// Symbol is a non-terminal.
|
||||||
|
const Word &symbol = static_cast<const TargetPhrase*>(child->GetNote().vp)->GetTargetLHS();
|
||||||
|
const WordsRange &range = child->GetRange();
|
||||||
|
context.push_back(std::make_pair(symbol, range));
|
||||||
|
i = range.GetEndPos()+1;
|
||||||
|
++child;
|
||||||
|
++j;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// Emulates the old operator<<(ostream &, const DottedRule &) function. The
|
// Emulates the old operator<<(ostream &, const DottedRule &) function. The
|
||||||
// output format is a bit odd (reverse order and double spacing between symbols)
|
// output format is a bit odd (reverse order and double spacing between symbols)
|
||||||
// but there are scripts and tools that expect the output of -T to look like
|
// but there are scripts and tools that expect the output of -T to look like
|
||||||
@ -330,6 +362,20 @@ void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &a
|
|||||||
<< " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown();
|
<< " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IOWrapper::OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||||
|
{
|
||||||
|
ReconstructApplicationContext(applied, sentence, applicationContext);
|
||||||
|
const TargetPhrase &phrase = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
|
||||||
|
out << "Trans Opt " << translationId
|
||||||
|
<< " " << applied->GetRange()
|
||||||
|
<< ": ";
|
||||||
|
WriteApplicationContext(out, applicationContext);
|
||||||
|
out << ": " << phrase.GetTargetLHS()
|
||||||
|
<< "->" << phrase
|
||||||
|
<< " " << applied->GetScore(); // << hypo->GetScoreBreakdown() TODO: missing in incremental search hypothesis
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
||||||
{
|
{
|
||||||
if (hypo != NULL) {
|
if (hypo != NULL) {
|
||||||
@ -346,6 +392,21 @@ void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void IOWrapper::OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||||
|
{
|
||||||
|
if (applied != NULL) {
|
||||||
|
OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
|
||||||
|
out << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
// recursive
|
||||||
|
const search::Applied *child = applied->Children();
|
||||||
|
for (size_t i = 0; i < applied->GetArity(); i++) {
|
||||||
|
OutputTranslationOptions(out, applicationContext, child++, sentence, translationId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -375,6 +436,33 @@ void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, Applica
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IOWrapper::OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Sentence &sentence, long translationId)
|
||||||
|
{
|
||||||
|
|
||||||
|
if (applied != NULL) {
|
||||||
|
OutputTranslationOption(out, applicationContext, applied, sentence, translationId);
|
||||||
|
|
||||||
|
const std::string key = "Tree";
|
||||||
|
std::string value;
|
||||||
|
bool hasProperty;
|
||||||
|
const TargetPhrase &currTarPhr = *static_cast<const TargetPhrase*>(applied->GetNote().vp);
|
||||||
|
currTarPhr.GetProperty(key, value, hasProperty);
|
||||||
|
|
||||||
|
out << " ||| ";
|
||||||
|
if (hasProperty)
|
||||||
|
out << " " << value;
|
||||||
|
else
|
||||||
|
out << " " << "noTreeInfo";
|
||||||
|
out << std::endl;
|
||||||
|
}
|
||||||
|
|
||||||
|
// recursive
|
||||||
|
const search::Applied *child = applied->Children();
|
||||||
|
for (size_t i = 0; i < applied->GetArity(); i++) {
|
||||||
|
OutputTreeFragmentsTranslationOptions(out, applicationContext, child++, sentence, translationId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void IOWrapper::OutputDetailedTranslationReport(
|
void IOWrapper::OutputDetailedTranslationReport(
|
||||||
const ChartHypothesis *hypo,
|
const ChartHypothesis *hypo,
|
||||||
const Sentence &sentence,
|
const Sentence &sentence,
|
||||||
@ -392,6 +480,23 @@ void IOWrapper::OutputDetailedTranslationReport(
|
|||||||
m_detailOutputCollector->Write(translationId, out.str());
|
m_detailOutputCollector->Write(translationId, out.str());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IOWrapper::OutputDetailedTranslationReport(
|
||||||
|
const search::Applied *applied,
|
||||||
|
const Sentence &sentence,
|
||||||
|
long translationId)
|
||||||
|
{
|
||||||
|
if (applied == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::ostringstream out;
|
||||||
|
ApplicationContext applicationContext;
|
||||||
|
|
||||||
|
OutputTranslationOptions(out, applicationContext, applied, sentence, translationId);
|
||||||
|
UTIL_THROW_IF2(m_detailOutputCollector == NULL,
|
||||||
|
"No ouput file for detailed reports specified");
|
||||||
|
m_detailOutputCollector->Write(translationId, out.str());
|
||||||
|
}
|
||||||
|
|
||||||
void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
||||||
const ChartHypothesis *hypo,
|
const ChartHypothesis *hypo,
|
||||||
const Sentence &sentence,
|
const Sentence &sentence,
|
||||||
@ -424,6 +529,28 @@ void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IOWrapper::OutputDetailedTreeFragmentsTranslationReport(
|
||||||
|
const search::Applied *applied,
|
||||||
|
const Sentence &sentence,
|
||||||
|
long translationId)
|
||||||
|
{
|
||||||
|
if (applied == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
std::ostringstream out;
|
||||||
|
ApplicationContext applicationContext;
|
||||||
|
|
||||||
|
OutputTreeFragmentsTranslationOptions(out, applicationContext, applied, sentence, translationId);
|
||||||
|
UTIL_THROW_IF2(m_detailTreeFragmentsOutputCollector == NULL,
|
||||||
|
"No output file for tree fragments specified");
|
||||||
|
|
||||||
|
//Tree of full sentence
|
||||||
|
//TODO: incremental search doesn't support stateful features
|
||||||
|
|
||||||
|
m_detailTreeFragmentsOutputCollector->Write(translationId, out.str());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
//DIMw
|
//DIMw
|
||||||
void IOWrapper::OutputDetailedAllTranslationReport(
|
void IOWrapper::OutputDetailedAllTranslationReport(
|
||||||
const ChartTrellisPathList &nBestList,
|
const ChartTrellisPathList &nBestList,
|
||||||
|
@ -93,11 +93,17 @@ protected:
|
|||||||
size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
|
size_t OutputAlignment(Alignments &retAlign, const Moses::ChartHypothesis *hypo, size_t startTarget);
|
||||||
void OutputAlignment(std::vector< std::set<size_t> > &retAlignmentsS2T, const Moses::AlignmentInfo &ai);
|
void OutputAlignment(std::vector< std::set<size_t> > &retAlignmentsS2T, const Moses::AlignmentInfo &ai);
|
||||||
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||||
|
void OutputTranslationOption(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||||
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||||
|
void OutputTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||||
void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||||
|
void OutputTreeFragmentsTranslationOptions(std::ostream &out, ApplicationContext &applicationContext, const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||||
void ReconstructApplicationContext(const Moses::ChartHypothesis &hypo,
|
void ReconstructApplicationContext(const Moses::ChartHypothesis &hypo,
|
||||||
const Moses::Sentence &sentence,
|
const Moses::Sentence &sentence,
|
||||||
ApplicationContext &context);
|
ApplicationContext &context);
|
||||||
|
void ReconstructApplicationContext(const search::Applied *applied,
|
||||||
|
const Moses::Sentence &sentence,
|
||||||
|
ApplicationContext &context);
|
||||||
void WriteApplicationContext(std::ostream &out,
|
void WriteApplicationContext(std::ostream &out,
|
||||||
const ApplicationContext &context);
|
const ApplicationContext &context);
|
||||||
|
|
||||||
@ -125,7 +131,9 @@ public:
|
|||||||
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, long translationId);
|
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, long translationId);
|
||||||
void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
|
void OutputNBestList(const std::vector<search::Applied> &nbest, long translationId);
|
||||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||||
|
void OutputDetailedTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||||
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
void OutputDetailedTreeFragmentsTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||||
|
void OutputDetailedTreeFragmentsTranslationReport(const search::Applied *applied, const Moses::Sentence &sentence, long translationId);
|
||||||
void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
|
void OutputDetailedAllTranslationReport(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartManager &manager, const Moses::Sentence &sentence, long translationId);
|
||||||
void Backtrack(const Moses::ChartHypothesis *hypo);
|
void Backtrack(const Moses::ChartHypothesis *hypo);
|
||||||
|
|
||||||
|
@ -102,6 +102,14 @@ public:
|
|||||||
const std::vector<search::Applied> &nbest = manager.ProcessSentence();
|
const std::vector<search::Applied> &nbest = manager.ProcessSentence();
|
||||||
if (!nbest.empty()) {
|
if (!nbest.empty()) {
|
||||||
m_ioWrapper.OutputBestHypo(nbest[0], translationId);
|
m_ioWrapper.OutputBestHypo(nbest[0], translationId);
|
||||||
|
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
||||||
|
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
|
||||||
|
m_ioWrapper.OutputDetailedTranslationReport(&nbest[0], sentence, translationId);
|
||||||
|
}
|
||||||
|
if (staticData.IsDetailedTreeFragmentsTranslationReportingEnabled()) {
|
||||||
|
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
|
||||||
|
m_ioWrapper.OutputDetailedTreeFragmentsTranslationReport(&nbest[0], sentence, translationId);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
m_ioWrapper.OutputBestNone(translationId);
|
m_ioWrapper.OutputBestNone(translationId);
|
||||||
}
|
}
|
||||||
|
@ -152,12 +152,13 @@ template <class Model> void Fill<Model>::Add(const TargetPhraseCollection &targe
|
|||||||
search::Note note;
|
search::Note note;
|
||||||
note.vp = &phrase;
|
note.vp = &phrase;
|
||||||
edge.SetNote(note);
|
edge.SetNote(note);
|
||||||
|
edge.SetRange(range);
|
||||||
|
|
||||||
edges_.AddEdge(edge);
|
edges_.AddEdge(edge);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &)
|
template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &, const WordsRange &range)
|
||||||
{
|
{
|
||||||
std::vector<lm::WordIndex> words;
|
std::vector<lm::WordIndex> words;
|
||||||
UTIL_THROW_IF2(phrase.GetSize() > 1,
|
UTIL_THROW_IF2(phrase.GetSize() > 1,
|
||||||
@ -173,6 +174,7 @@ template <class Model> void Fill<Model>::AddPhraseOOV(TargetPhrase &phrase, std:
|
|||||||
search::Note note;
|
search::Note note;
|
||||||
note.vp = &phrase;
|
note.vp = &phrase;
|
||||||
edge.SetNote(note);
|
edge.SetNote(note);
|
||||||
|
edge.SetRange(range);
|
||||||
|
|
||||||
edges_.AddEdge(edge);
|
edges_.AddEdge(edge);
|
||||||
}
|
}
|
||||||
|
@ -24,9 +24,10 @@ template <class Below> class GenericApplied : public Header {
|
|||||||
*child_out = Below(part->End());
|
*child_out = Below(part->End());
|
||||||
}
|
}
|
||||||
|
|
||||||
GenericApplied(void *location, Score score, Arity arity, Note note) : Header(location, arity) {
|
GenericApplied(void *location, Score score, Arity arity, Note note, Moses::WordsRange range) : Header(location, arity) {
|
||||||
SetScore(score);
|
SetScore(score);
|
||||||
SetNote(note);
|
SetNote(note);
|
||||||
|
SetRange(range);
|
||||||
}
|
}
|
||||||
|
|
||||||
explicit GenericApplied(History from) : Header(from) {}
|
explicit GenericApplied(History from) : Header(from) {}
|
||||||
|
@ -80,6 +80,7 @@ template <class Model> PartialEdge EdgeGenerator::Pop(Context<Model> &context) {
|
|||||||
alternate.SetScore(top.GetScore() + alternate_changed.Bound() - old_value.Bound());
|
alternate.SetScore(top.GetScore() + alternate_changed.Bound() - old_value.Bound());
|
||||||
|
|
||||||
alternate.SetNote(top.GetNote());
|
alternate.SetNote(top.GetNote());
|
||||||
|
alternate.SetRange(top.GetRange());
|
||||||
|
|
||||||
PartialVertex *alternate_nt = alternate.NT();
|
PartialVertex *alternate_nt = alternate.NT();
|
||||||
for (Arity i = 0; i < victim; ++i) alternate_nt[i] = top_nt[i];
|
for (Arity i = 0; i < victim; ++i) alternate_nt[i] = top_nt[i];
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
#ifndef SEARCH_HEADER__
|
#ifndef SEARCH_HEADER__
|
||||||
#define SEARCH_HEADER__
|
#define SEARCH_HEADER__
|
||||||
|
|
||||||
// Header consisting of Score, Arity, and Note
|
// Header consisting of Score, Arity, Note and WordsRange
|
||||||
|
|
||||||
#include "search/types.hh"
|
#include "search/types.hh"
|
||||||
|
#include "moses/WordsRange.h"
|
||||||
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
@ -38,6 +39,13 @@ class Header {
|
|||||||
*reinterpret_cast<Note*>(base_ + sizeof(Score) + sizeof(Arity)) = to;
|
*reinterpret_cast<Note*>(base_ + sizeof(Score) + sizeof(Arity)) = to;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Moses::WordsRange GetRange() const {
|
||||||
|
return *reinterpret_cast<const Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note));
|
||||||
|
}
|
||||||
|
void SetRange(Moses::WordsRange to) {
|
||||||
|
*reinterpret_cast<Moses::WordsRange*>(base_ + sizeof(Score) + sizeof(Arity) + sizeof(Note)) = to;
|
||||||
|
}
|
||||||
|
|
||||||
uint8_t *Base() { return base_; }
|
uint8_t *Base() { return base_; }
|
||||||
const uint8_t *Base() const { return base_; }
|
const uint8_t *Base() const { return base_; }
|
||||||
|
|
||||||
@ -50,7 +58,7 @@ class Header {
|
|||||||
*reinterpret_cast<Arity*>(base_ + sizeof(Score)) = arity;
|
*reinterpret_cast<Arity*>(base_ + sizeof(Score)) = arity;
|
||||||
}
|
}
|
||||||
|
|
||||||
static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note);
|
static const std::size_t kHeaderSize = sizeof(Score) + sizeof(Arity) + sizeof(Note) + sizeof(Moses::WordsRange);
|
||||||
|
|
||||||
uint8_t *After() { return base_ + kHeaderSize; }
|
uint8_t *After() { return base_ + kHeaderSize; }
|
||||||
const uint8_t *After() const { return base_ + kHeaderSize; }
|
const uint8_t *After() const { return base_ + kHeaderSize; }
|
||||||
|
@ -70,7 +70,7 @@ void NBestList::MoveTop(util::Pool &pool) {
|
|||||||
Score change = child->in_->Visit(pool, child->index_);
|
Score change = child->in_->Visit(pool, child->index_);
|
||||||
if (change != -INFINITY) {
|
if (change != -INFINITY) {
|
||||||
assert(change < 0.001);
|
assert(change < 0.001);
|
||||||
QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote());
|
QueueEntry new_entry(pool.Allocate(QueueEntry::Size(entry.GetArity())), basis + change, entry.GetArity(), entry.GetNote(), entry.GetRange());
|
||||||
std::copy(children_begin, child, new_entry.Children());
|
std::copy(children_begin, child, new_entry.Children());
|
||||||
RevealedRef *update = new_entry.Children() + (child - children_begin);
|
RevealedRef *update = new_entry.Children() + (child - children_begin);
|
||||||
update->in_ = child->in_;
|
update->in_ = child->in_;
|
||||||
|
Loading…
Reference in New Issue
Block a user