moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-25 12:52:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

more enriched trace

Browse Source
This commit is contained in:
phikoehn 2013-10-13 06:59:05 +01:00
parent 793686ea14
commit 8e23a3f2c1
9 changed files with 64 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
moses-cmd/IOWrapper.cpp
View File

@ -262,17 +262,23 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
}
}
// trace option "-t" / "-tt"
// trace ("report segmentation") option "-t" / "-tt"
if (reportSegmentation > 0 && phrase.GetSize() > 0) {
const WordsRange &sourceRange = edge.GetCurrSourceWordsRange();
const int sourceStart = sourceRange.GetStartPos();
const int sourceEnd = sourceRange.GetEndPos();
out << "|" << sourceStart << "-" << sourceEnd;
// enriched "-tt"
out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt"
if (reportSegmentation == 2) {
out << ",0, ";
out << ",wa=";
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
OutputAlignment(out, ai, 0, 0);
out << ",total=";
out << edge.GetTotalScore()-edge.GetPrevHypo()->GetTotalScore();
out << ",";
ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown());
scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown());
out << scoreBreakdown;
}
out << "| ";
}
@ -443,7 +449,6 @@ void OutputNBest(std::ostream& out
, char reportSegmentation)
{
const StaticData &staticData = StaticData::Instance();
bool labeledOutput = staticData.IsLabeledNBestList();
bool reportAllFactors = staticData.GetReportAllFactorsNBest();
bool includeSegmentation = staticData.NBestIncludesSegmentation();
bool includeWordAlignment = staticData.PrintAlignmentInfoInNbest();

1
moses-cmd/IOWrapper.h
View File

@ -133,6 +133,7 @@ public:
IOWrapper *GetIOWrapper(const Moses::StaticData &staticData);
bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source);
void OutputLanguageModelOrder(std::ostream &out, const Moses::Hypothesis *hypo, Moses::Manager &manager);
void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector<Moses::FactorType> &outputFactorOrder, char reportSegmentation, bool reportAllFactors);
void OutputLatticeMBRNBest(std::ostream& out, const std::vector<LatticeMBRSolution>& solutions,long translationId);
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,

3
moses-cmd/Main.cpp
View File

@ -291,6 +291,9 @@ public:
out << m_source->GetTranslationId() << " ";
}
if (staticData.GetReportSegmentation() == 2) {
manager.GetOutputLanguageModelOrder(out, bestHypo);
}
OutputBestSurface(
out,
bestHypo,

5
moses/LM/Base.cpp
View File

@ -64,6 +64,11 @@ void LanguageModel::IncrementalCallback(Incremental::Manager &manager) const
UTIL_THROW(util::Exception, "Incremental search is only supported by KenLM.");
}
void LanguageModel::ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const
{
// out << "ReportHistoryOrder not implemented";
}
void LanguageModel::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown

1
moses/LM/Base.h
View File

@ -85,6 +85,7 @@ public:
// KenLM only (others throw an exception): call incremental search with the model and mapping.
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const;
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase

30
moses/LM/Ken.cpp
View File

@ -356,6 +356,36 @@ template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Increme
manager.LMCallback(*m_ngram, m_lmIdLookup);
}
template <class Model> void LanguageModelKen<Model>::ReportHistoryOrder(std::ostream &out, const Phrase &phrase) const
{
out << "!lm-history=(";
if (!phrase.GetSize()) return;
std::auto_ptr<KenLMState> initial(new KenLMState());
typename Model::State aux_state;
typename Model::State *state0 = &initial->state;
typename Model::State *state1 = &aux_state;
float score = 0;
for (std::size_t position=0; position<phrase.GetSize(); position++) {
const lm::WordIndex idx = TranslateID(phrase.GetWord(position));
score += m_ngram->Score(*state0, idx, *state1);
// out << phrase.GetWord(position) << " ";
// out << TransformLMScore(score) << " ";
if (position) {
out << ",";
}
if (idx == 0) {
out << "0";
}
else {
out << (int)state1->Length();
}
std::swap(state0, state1);
}
out << ")| ";
}
template <class Model>
bool LanguageModelKen<Model>::IsUseable(const FactorMask &mask) const
{

1
moses/LM/Ken.h
View File

@ -60,6 +60,7 @@ public:
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const;
virtual bool IsUseable(const FactorMask &mask) const;

13
moses/Manager.cpp
View File

@ -532,6 +532,18 @@ void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo
outputWordGraphStream << endl;
}
void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ) {
Phrase translation;
hypo->GetOutputPhrase(translation);
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
if (const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff)) {
lm->ReportHistoryOrder(out, translation);
}
}
}
void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const
{
const StaticData &staticData = StaticData::Instance();
@ -543,7 +555,6 @@ void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStre
<< "UTTERANCE=" << translationId << endl;
size_t linkId = 0;
size_t stackNo = 1;
std::vector < HypothesisStack* >::const_iterator iterStack;
for (iterStack = ++hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
const HypothesisStack &stack = **iterStack;

1
moses/Manager.h
View File

@ -143,6 +143,7 @@ public:
void PrintAllDerivations(long translationId, std::ostream& outputStream) const;
void printDivergentHypothesis(long translationId, const Hypothesis* hypo, const std::vector <const TargetPhrase*> & remainingPhrases, float remainingScore , std::ostream& outputStream) const;
void printThisHypothesis(long translationId, const Hypothesis* hypo, const std::vector <const TargetPhrase* > & remainingPhrases, float remainingScore , std::ostream& outputStream) const;
void GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo );
void GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const;
int GetNextHypoId();
#ifdef HAVE_PROTOBUF