From 8e23a3f2c1a7b08d20f350e2a1dd4e41f2a0bb1b Mon Sep 17 00:00:00 2001 From: phikoehn Date: Sun, 13 Oct 2013 06:59:05 +0100 Subject: [PATCH] more enriched trace --- moses-cmd/IOWrapper.cpp | 15 ++++++++++----- moses-cmd/IOWrapper.h | 1 + moses-cmd/Main.cpp | 3 +++ moses/LM/Base.cpp | 5 +++++ moses/LM/Base.h | 1 + moses/LM/Ken.cpp | 30 ++++++++++++++++++++++++++++++ moses/LM/Ken.h | 1 + moses/Manager.cpp | 13 ++++++++++++- moses/Manager.h | 1 + 9 files changed, 64 insertions(+), 6 deletions(-) diff --git a/moses-cmd/IOWrapper.cpp b/moses-cmd/IOWrapper.cpp index ac85271d7..650b0c4aa 100644 --- a/moses-cmd/IOWrapper.cpp +++ b/moses-cmd/IOWrapper.cpp @@ -262,17 +262,23 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector< } } - // trace option "-t" / "-tt" + // trace ("report segmentation") option "-t" / "-tt" if (reportSegmentation > 0 && phrase.GetSize() > 0) { const WordsRange &sourceRange = edge.GetCurrSourceWordsRange(); const int sourceStart = sourceRange.GetStartPos(); const int sourceEnd = sourceRange.GetEndPos(); - out << "|" << sourceStart << "-" << sourceEnd; - // enriched "-tt" + out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt" if (reportSegmentation == 2) { - out << ",0, "; + out << ",wa="; const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm(); OutputAlignment(out, ai, 0, 0); + out << ",total="; + out << edge.GetTotalScore()-edge.GetPrevHypo()->GetTotalScore(); + out << ","; + ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown()); + scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown()); + out << scoreBreakdown; + } out << "| "; } @@ -443,7 +449,6 @@ void OutputNBest(std::ostream& out , char reportSegmentation) { const StaticData &staticData = StaticData::Instance(); - bool labeledOutput = staticData.IsLabeledNBestList(); bool reportAllFactors = staticData.GetReportAllFactorsNBest(); bool includeSegmentation = staticData.NBestIncludesSegmentation(); bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest(); diff --git a/moses-cmd/IOWrapper.h b/moses-cmd/IOWrapper.h index 7c6bdd119..901e89dc2 100644 --- a/moses-cmd/IOWrapper.h +++ b/moses-cmd/IOWrapper.h @@ -133,6 +133,7 @@ public: IOWrapper *GetIOWrapper(const Moses::StaticData &staticData); bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source); +void OutputLanguageModelOrder(std::ostream &out, const Moses::Hypothesis *hypo, Moses::Manager &manager); void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector &outputFactorOrder, char reportSegmentation, bool reportAllFactors); void OutputLatticeMBRNBest(std::ostream& out, const std::vector& solutions,long translationId); void OutputBestHypo(const std::vector& mbrBestHypo, long /*translationId*/, diff --git a/moses-cmd/Main.cpp b/moses-cmd/Main.cpp index 2d8a214d7..b3695a06f 100644 --- a/moses-cmd/Main.cpp +++ b/moses-cmd/Main.cpp @@ -291,6 +291,9 @@ public: out << m_source->GetTranslationId() << " "; } + if (staticData.GetReportSegmentation() == 2) { + manager.GetOutputLanguageModelOrder(out, bestHypo); + } OutputBestSurface( out, bestHypo, diff --git a/moses/LM/Base.cpp b/moses/LM/Base.cpp index 6c6cad1dd..b77d64fef 100644 --- a/moses/LM/Base.cpp +++ b/moses/LM/Base.cpp @@ -64,6 +64,11 @@ void LanguageModel::IncrementalCallback(Incremental::Manager &manager) const UTIL_THROW(util::Exception, "Incremental search is only supported by KenLM."); } +void LanguageModel::ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const +{ + // out << "ReportHistoryOrder not implemented"; +} + void LanguageModel::Evaluate(const Phrase &source , const TargetPhrase &targetPhrase , ScoreComponentCollection &scoreBreakdown diff --git a/moses/LM/Base.h b/moses/LM/Base.h index 186cfe101..5c9616e46 100644 --- a/moses/LM/Base.h +++ b/moses/LM/Base.h @@ -85,6 +85,7 @@ public: // KenLM only (others throw an exception): call incremental search with the model and mapping. virtual void IncrementalCallback(Incremental::Manager &manager) const; + virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const; virtual void Evaluate(const Phrase &source , const TargetPhrase &targetPhrase diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp index c49b9d2f7..62d28e3ca 100644 --- a/moses/LM/Ken.cpp +++ b/moses/LM/Ken.cpp @@ -356,6 +356,36 @@ template void LanguageModelKen::IncrementalCallback(Increme manager.LMCallback(*m_ngram, m_lmIdLookup); } +template void LanguageModelKen::ReportHistoryOrder(std::ostream &out, const Phrase &phrase) const +{ + out << "!lm-history=("; + if (!phrase.GetSize()) return; + + std::auto_ptr initial(new KenLMState()); + typename Model::State aux_state; + typename Model::State *state0 = &initial->state; + typename Model::State *state1 = &aux_state; + + float score = 0; + for (std::size_t position=0; positionScore(*state0, idx, *state1); + // out << phrase.GetWord(position) << " "; + // out << TransformLMScore(score) << " "; + if (position) { + out << ","; + } + if (idx == 0) { + out << "0"; + } + else { + out << (int)state1->Length(); + } + std::swap(state0, state1); + } + out << ")| "; +} + template bool LanguageModelKen::IsUseable(const FactorMask &mask) const { diff --git a/moses/LM/Ken.h b/moses/LM/Ken.h index 1c83df6ef..e5950f591 100644 --- a/moses/LM/Ken.h +++ b/moses/LM/Ken.h @@ -60,6 +60,7 @@ public: virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const; virtual void IncrementalCallback(Incremental::Manager &manager) const; + virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const; virtual bool IsUseable(const FactorMask &mask) const; diff --git a/moses/Manager.cpp b/moses/Manager.cpp index ae9feecb7..0618d2d91 100644 --- a/moses/Manager.cpp +++ b/moses/Manager.cpp @@ -532,6 +532,18 @@ void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo outputWordGraphStream << endl; } +void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ) { + Phrase translation; + hypo->GetOutputPhrase(translation); + const std::vector &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions(); + for (size_t i = 0; i < statefulFFs.size(); ++i) { + const StatefulFeatureFunction *ff = statefulFFs[i]; + if (const LanguageModel *lm = dynamic_cast(ff)) { + lm->ReportHistoryOrder(out, translation); + } + } +} + void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const { const StaticData &staticData = StaticData::Instance(); @@ -543,7 +555,6 @@ void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStre << "UTTERANCE=" << translationId << endl; size_t linkId = 0; - size_t stackNo = 1; std::vector < HypothesisStack* >::const_iterator iterStack; for (iterStack = ++hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) { const HypothesisStack &stack = **iterStack; diff --git a/moses/Manager.h b/moses/Manager.h index fd329c309..ccf57c527 100644 --- a/moses/Manager.h +++ b/moses/Manager.h @@ -143,6 +143,7 @@ public: void PrintAllDerivations(long translationId, std::ostream& outputStream) const; void printDivergentHypothesis(long translationId, const Hypothesis* hypo, const std::vector & remainingPhrases, float remainingScore , std::ostream& outputStream) const; void printThisHypothesis(long translationId, const Hypothesis* hypo, const std::vector & remainingPhrases, float remainingScore , std::ostream& outputStream) const; + void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ); void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const; int GetNextHypoId(); #ifdef HAVE_PROTOBUF