mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-16 15:00:33 +03:00
2017 lines
71 KiB
C++
2017 lines
71 KiB
C++
// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*-
|
|
// vim:tabstop=2
|
|
/***********************************************************************
|
|
Moses - factored phrase-based language decoder
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
modify it under the terms of the GNU Lesser General Public
|
|
License as published by the Free Software Foundation; either
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Lesser General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
License along with this library; if not, write to the Free Software
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
***********************************************************************/
|
|
#ifdef WIN32
|
|
#include <hash_set>
|
|
#else
|
|
// #include <ext/hash_set>
|
|
#endif
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <limits>
|
|
#include <map>
|
|
#include <set>
|
|
#include "Manager.h"
|
|
#include "TypeDef.h"
|
|
#include "Util.h"
|
|
#include "TargetPhrase.h"
|
|
#include "TrellisPath.h"
|
|
#include "TrellisPathCollection.h"
|
|
#include "TranslationOption.h"
|
|
#include "TranslationOptionCollection.h"
|
|
#include "Timer.h"
|
|
#include "moses/OutputCollector.h"
|
|
#include "moses/FF/DistortionScoreProducer.h"
|
|
#include "moses/LM/Base.h"
|
|
#include "moses/TranslationModel/PhraseDictionary.h"
|
|
#include "moses/TranslationAnalysis.h"
|
|
#include "moses/TranslationTask.h"
|
|
#include "moses/HypergraphOutput.h"
|
|
#include "moses/mbr.h"
|
|
#include "moses/LatticeMBR.h"
|
|
#include "moses/SearchNormal.h"
|
|
#include "moses/SearchCubePruning.h"
|
|
#include <boost/foreach.hpp>
|
|
|
|
#ifdef HAVE_PROTOBUF
|
|
#include "hypergraph.pb.h"
|
|
#include "rule.pb.h"
|
|
#endif
|
|
|
|
#include "util/exception.hh"
|
|
#include "util/random.hh"
|
|
#include "util/string_stream.hh"
|
|
|
|
using namespace std;
|
|
|
|
namespace Moses
|
|
{
|
|
|
|
Manager::Manager(ttasksptr const& ttask)
|
|
: BaseManager(ttask)
|
|
, interrupted_flag(0)
|
|
, m_hypoId(0)
|
|
{
|
|
boost::shared_ptr<InputType> source = ttask->GetSource();
|
|
m_transOptColl = source->CreateTranslationOptionCollection(ttask);
|
|
|
|
switch(options()->search.algo) {
|
|
case Normal:
|
|
m_search = new SearchNormal(*this, *m_transOptColl);
|
|
break;
|
|
case CubePruning:
|
|
m_search = new SearchCubePruning(*this, *m_transOptColl);
|
|
break;
|
|
default:
|
|
UTIL_THROW2("ERROR: search. Aborting\n");
|
|
}
|
|
|
|
StaticData::Instance().InitializeForInput(ttask);
|
|
}
|
|
|
|
Manager::~Manager()
|
|
{
|
|
delete m_transOptColl;
|
|
delete m_search;
|
|
StaticData::Instance().CleanUpAfterSentenceProcessing(m_ttask.lock());
|
|
}
|
|
|
|
const InputType&
|
|
Manager::GetSource() const
|
|
{
|
|
return m_source ;
|
|
}
|
|
|
|
/**
|
|
* Main decoder loop that translates a sentence by expanding
|
|
* hypotheses stack by stack, until the end of the sentence.
|
|
*/
|
|
void Manager::Decode()
|
|
{
|
|
|
|
//std::cerr << options().nbest.nbest_size << " "
|
|
// << options().nbest.enabled << " " << std::endl;
|
|
|
|
// initialize statistics
|
|
ResetSentenceStats(m_source);
|
|
IFVERBOSE(2) {
|
|
GetSentenceStats().StartTimeTotal();
|
|
}
|
|
|
|
// check if alternate weight setting is used
|
|
// this is not thread safe! it changes StaticData
|
|
if (StaticData::Instance().GetHasAlternateWeightSettings()) {
|
|
if (m_source.GetSpecifiesWeightSetting()) {
|
|
StaticData::Instance().SetWeightSetting(m_source.GetWeightSetting());
|
|
} else {
|
|
StaticData::Instance().SetWeightSetting("default");
|
|
}
|
|
}
|
|
|
|
// get translation options
|
|
IFVERBOSE(1) {
|
|
GetSentenceStats().StartTimeCollectOpts();
|
|
}
|
|
m_transOptColl->CreateTranslationOptions();
|
|
|
|
// some reporting on how long this took
|
|
IFVERBOSE(1) {
|
|
GetSentenceStats().StopTimeCollectOpts();
|
|
TRACE_ERR("Line "<< m_source.GetTranslationId()
|
|
<< ": Collecting options took "
|
|
<< GetSentenceStats().GetTimeCollectOpts() << " seconds at "
|
|
<< __FILE__ << " Line " << __LINE__ << endl);
|
|
}
|
|
|
|
// search for best translation with the specified algorithm
|
|
Timer searchTime;
|
|
searchTime.start();
|
|
m_search->Decode();
|
|
VERBOSE(1, "Line " << m_source.GetTranslationId()
|
|
<< ": Search took " << searchTime << " seconds" << endl);
|
|
IFVERBOSE(2) {
|
|
GetSentenceStats().StopTimeTotal();
|
|
TRACE_ERR(GetSentenceStats());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Print all derivations in search graph. Note: The number of derivations is exponential in the sentence length
|
|
*
|
|
*/
|
|
|
|
void Manager::PrintAllDerivations(long translationId, ostream& outputStream) const
|
|
{
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
|
|
|
|
if (sortedPureHypo.size() == 0)
|
|
return;
|
|
|
|
float remainingScore = 0;
|
|
vector<const TargetPhrase*> remainingPhrases;
|
|
|
|
// add all pure paths
|
|
vector<const Hypothesis*>::const_iterator iterBestHypo;
|
|
for (iterBestHypo = sortedPureHypo.begin()
|
|
; iterBestHypo != sortedPureHypo.end()
|
|
; ++iterBestHypo) {
|
|
printThisHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
|
|
printDivergentHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
|
|
}
|
|
}
|
|
|
|
const TranslationOptionCollection* Manager::getSntTranslationOptions()
|
|
{
|
|
return m_transOptColl;
|
|
}
|
|
|
|
void Manager::printDivergentHypothesis(long translationId, const Hypothesis* hypo, const vector <const TargetPhrase*> & remainingPhrases, float remainingScore , ostream& outputStream ) const
|
|
{
|
|
//Backtrack from the predecessor
|
|
if (hypo->GetId() > 0) {
|
|
vector <const TargetPhrase*> followingPhrases;
|
|
followingPhrases.push_back(& (hypo->GetCurrTargetPhrase()));
|
|
///((Phrase) hypo->GetPrevHypo()->GetTargetPhrase());
|
|
followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
|
|
printDivergentHypothesis(translationId, hypo->GetPrevHypo(), followingPhrases , remainingScore + hypo->GetScore() - hypo->GetPrevHypo()->GetScore(), outputStream);
|
|
}
|
|
|
|
//Process the arcs
|
|
const ArcList *pAL = hypo->GetArcList();
|
|
if (pAL) {
|
|
const ArcList &arcList = *pAL;
|
|
// every possible Arc to replace this edge
|
|
ArcList::const_iterator iterArc;
|
|
for (iterArc = arcList.begin() ; iterArc != arcList.end() ; ++iterArc) {
|
|
const Hypothesis *loserHypo = *iterArc;
|
|
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
|
|
float arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
vector <const TargetPhrase* > followingPhrases;
|
|
followingPhrases.push_back(&(loserHypo->GetCurrTargetPhrase()));
|
|
followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
|
|
printThisHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
|
|
printDivergentHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void
|
|
Manager::
|
|
printThisHypothesis(long translationId, const Hypothesis* hypo,
|
|
const vector <const TargetPhrase*> & remainingPhrases,
|
|
float remainingScore, ostream& outputStream) const
|
|
{
|
|
|
|
outputStream << translationId << " ||| ";
|
|
|
|
//Yield of this hypothesis
|
|
hypo->ToStream(outputStream);
|
|
for (size_t p = 0; p < remainingPhrases.size(); ++p) {
|
|
const TargetPhrase * phrase = remainingPhrases[p];
|
|
size_t size = phrase->GetSize();
|
|
for (size_t pos = 0 ; pos < size ; pos++) {
|
|
const Factor *factor = phrase->GetFactor(pos, 0);
|
|
outputStream << *factor;
|
|
outputStream << " ";
|
|
}
|
|
}
|
|
|
|
outputStream << "||| " << hypo->GetScore() + remainingScore;
|
|
outputStream << endl;
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
* After decoding, the hypotheses in the stacks and additional arcs
|
|
* form a search graph that can be mined for n-best lists.
|
|
* The heavy lifting is done in the TrellisPath and TrellisPathCollection
|
|
* this function controls this for one sentence.
|
|
*
|
|
* \param count the number of n-best translations to produce
|
|
* \param ret holds the n-best list that was calculated
|
|
*/
|
|
void Manager::CalcNBest(size_t count, TrellisPathList &ret, bool onlyDistinct) const
|
|
{
|
|
if (count <= 0)
|
|
return;
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
|
|
|
|
if (sortedPureHypo.size() == 0)
|
|
return;
|
|
|
|
TrellisPathCollection contenders;
|
|
|
|
set<Phrase> distinctHyps;
|
|
|
|
// add all pure paths
|
|
vector<const Hypothesis*>::const_iterator iterBestHypo;
|
|
for (iterBestHypo = sortedPureHypo.begin()
|
|
; iterBestHypo != sortedPureHypo.end()
|
|
; ++iterBestHypo) {
|
|
contenders.Add(new TrellisPath(*iterBestHypo));
|
|
}
|
|
|
|
// factor defines stopping point for distinct n-best list if too
|
|
// many candidates identical
|
|
size_t nBestFactor = options()->nbest.factor;
|
|
if (nBestFactor < 1) nBestFactor = 1000; // 0 = unlimited
|
|
|
|
// MAIN loop
|
|
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++) {
|
|
// get next best from list of contenders
|
|
TrellisPath *path = contenders.pop();
|
|
UTIL_THROW_IF2(path == NULL, "path is NULL");
|
|
// create deviations from current best
|
|
path->CreateDeviantPaths(contenders);
|
|
if(onlyDistinct) {
|
|
Phrase tgtPhrase = path->GetSurfacePhrase();
|
|
if (distinctHyps.insert(tgtPhrase).second) {
|
|
ret.Add(path);
|
|
} else {
|
|
delete path;
|
|
path = NULL;
|
|
}
|
|
} else {
|
|
ret.Add(path);
|
|
}
|
|
|
|
|
|
if(onlyDistinct) {
|
|
const size_t nBestFactor = options()->nbest.factor;
|
|
if (nBestFactor > 0)
|
|
contenders.Prune(count * nBestFactor);
|
|
} else {
|
|
contenders.Prune(count);
|
|
}
|
|
}
|
|
}
|
|
|
|
struct SGNReverseCompare {
|
|
bool operator() (const SearchGraphNode& s1, const SearchGraphNode& s2) const {
|
|
return s1.hypo->GetId() > s2.hypo->GetId();
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Implements lattice sampling, as in Chatterjee & Cancedda, emnlp 2010
|
|
**/
|
|
void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const
|
|
{
|
|
|
|
vector<SearchGraphNode> searchGraph;
|
|
GetSearchGraph(searchGraph);
|
|
|
|
//Calculation of the sigmas of each hypothesis and edge. In C&C notation this is
|
|
//the "log of the cumulative unnormalized probability of all the paths in the
|
|
// lattice for the hypothesis to a final node"
|
|
typedef pair<int, int> Edge;
|
|
map<const Hypothesis*, float> sigmas;
|
|
map<Edge, float> edgeScores;
|
|
map<const Hypothesis*, set<const Hypothesis*> > outgoingHyps;
|
|
map<int,const Hypothesis*> idToHyp;
|
|
map<int,float> fscores;
|
|
|
|
//Iterating through the hypos in reverse order of id gives a reverse
|
|
//topological order. We rely on the fact that hypo ids are given out
|
|
//sequentially, as the search proceeds.
|
|
//NB: Could just sort by stack.
|
|
sort(searchGraph.begin(), searchGraph.end(), SGNReverseCompare());
|
|
|
|
//first task is to fill in the outgoing hypos and edge scores.
|
|
for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
|
|
i != searchGraph.end(); ++i) {
|
|
const Hypothesis* hypo = i->hypo;
|
|
idToHyp[hypo->GetId()] = hypo;
|
|
fscores[hypo->GetId()] = i->fscore;
|
|
if (hypo->GetId()) {
|
|
//back to current
|
|
const Hypothesis* prevHypo = i->hypo->GetPrevHypo();
|
|
outgoingHyps[prevHypo].insert(hypo);
|
|
edgeScores[Edge(prevHypo->GetId(),hypo->GetId())] =
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
}
|
|
//forward from current
|
|
if (i->forward >= 0) {
|
|
map<int,const Hypothesis*>::const_iterator idToHypIter = idToHyp.find(i->forward);
|
|
UTIL_THROW_IF2(idToHypIter == idToHyp.end(),
|
|
"Couldn't find hypothesis " << i->forward);
|
|
const Hypothesis* nextHypo = idToHypIter->second;
|
|
outgoingHyps[hypo].insert(nextHypo);
|
|
map<int,float>::const_iterator fscoreIter = fscores.find(nextHypo->GetId());
|
|
UTIL_THROW_IF2(fscoreIter == fscores.end(),
|
|
"Couldn't find scores for hypothsis " << nextHypo->GetId());
|
|
edgeScores[Edge(hypo->GetId(),nextHypo->GetId())] =
|
|
i->fscore - fscoreIter->second;
|
|
}
|
|
}
|
|
|
|
|
|
//then run through again to calculate sigmas
|
|
for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
|
|
i != searchGraph.end(); ++i) {
|
|
|
|
if (i->forward == -1) {
|
|
sigmas[i->hypo] = 0;
|
|
} else {
|
|
map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
|
|
outgoingHyps.find(i->hypo);
|
|
|
|
UTIL_THROW_IF2(outIter == outgoingHyps.end(),
|
|
"Couldn't find hypothesis " << i->hypo->GetId());
|
|
float sigma = 0;
|
|
for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
|
|
j != outIter->second.end(); ++j) {
|
|
map<const Hypothesis*, float>::const_iterator succIter = sigmas.find(*j);
|
|
UTIL_THROW_IF2(succIter == sigmas.end(),
|
|
"Couldn't find hypothesis " << (*j)->GetId());
|
|
map<Edge,float>::const_iterator edgeScoreIter =
|
|
edgeScores.find(Edge(i->hypo->GetId(),(*j)->GetId()));
|
|
UTIL_THROW_IF2(edgeScoreIter == edgeScores.end(),
|
|
"Couldn't find edge for hypothesis " << (*j)->GetId());
|
|
float term = edgeScoreIter->second + succIter->second; // Add sigma(*j)
|
|
if (sigma == 0) {
|
|
sigma = term;
|
|
} else {
|
|
sigma = log_sum(sigma,term);
|
|
}
|
|
}
|
|
sigmas[i->hypo] = sigma;
|
|
}
|
|
}
|
|
|
|
//The actual sampling!
|
|
const Hypothesis* startHypo = searchGraph.back().hypo;
|
|
UTIL_THROW_IF2(startHypo->GetId() != 0, "Expecting the start hypothesis ");
|
|
for (size_t i = 0; i < count; ++i) {
|
|
vector<const Hypothesis*> path;
|
|
path.push_back(startHypo);
|
|
while(1) {
|
|
map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
|
|
outgoingHyps.find(path.back());
|
|
if (outIter == outgoingHyps.end() || !outIter->second.size()) {
|
|
//end of the path
|
|
break;
|
|
}
|
|
//score the possibles
|
|
vector<const Hypothesis*> candidates;
|
|
vector<float> candidateScores;
|
|
float scoreTotal = 0;
|
|
for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
|
|
j != outIter->second.end(); ++j) {
|
|
candidates.push_back(*j);
|
|
UTIL_THROW_IF2(sigmas.find(*j) == sigmas.end(),
|
|
"Hypothesis " << (*j)->GetId() << " not found");
|
|
Edge edge(path.back()->GetId(),(*j)->GetId());
|
|
UTIL_THROW_IF2(edgeScores.find(edge) == edgeScores.end(),
|
|
"Edge not found");
|
|
candidateScores.push_back(sigmas[*j] + edgeScores[edge]);
|
|
if (scoreTotal == 0) {
|
|
scoreTotal = candidateScores.back();
|
|
} else {
|
|
scoreTotal = log_sum(candidateScores.back(), scoreTotal);
|
|
}
|
|
}
|
|
|
|
//normalise
|
|
transform(candidateScores.begin(), candidateScores.end(), candidateScores.begin(), bind2nd(minus<float>(),scoreTotal));
|
|
//copy(candidateScores.begin(),candidateScores.end(),ostream_iterator<float>(cerr," "));
|
|
//cerr << endl;
|
|
|
|
//draw the sample
|
|
const float frandom = log(util::rand_incl(0.0f, 1.0f));
|
|
size_t position = 1;
|
|
float sum = candidateScores[0];
|
|
for (; position < candidateScores.size() && sum < frandom; ++position) {
|
|
sum = log_sum(sum,candidateScores[position]);
|
|
}
|
|
//cerr << "Random: " << frandom << " Chose " << position-1 << endl;
|
|
const Hypothesis* chosen = candidates[position-1];
|
|
path.push_back(chosen);
|
|
}
|
|
//cerr << "Path: " << endl;
|
|
//for (size_t j = 0; j < path.size(); ++j) {
|
|
// cerr << path[j]->GetId() << " " << path[j]->GetScoreBreakdown() << endl;
|
|
//}
|
|
//cerr << endl;
|
|
|
|
//Convert the hypos to TrellisPath
|
|
ret.Add(new TrellisPath(path));
|
|
//cerr << ret.at(ret.GetSize()-1).GetScoreBreakdown() << endl;
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void Manager::CalcDecoderStatistics() const
|
|
{
|
|
const Hypothesis *hypo = GetBestHypothesis();
|
|
if (hypo != NULL) {
|
|
GetSentenceStats().CalcFinalStats(*hypo);
|
|
IFVERBOSE(2) {
|
|
if (hypo != NULL) {
|
|
string buff;
|
|
string buff2;
|
|
TRACE_ERR( "Source and Target Units:"
|
|
<< hypo->GetInput());
|
|
buff2.insert(0,"] ");
|
|
buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
|
|
buff2.insert(0,":");
|
|
buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
|
|
buff2.insert(0,"[");
|
|
|
|
hypo = hypo->GetPrevHypo();
|
|
while (hypo != NULL) {
|
|
//dont print out the empty final hypo
|
|
buff.insert(0,buff2);
|
|
buff2.clear();
|
|
buff2.insert(0,"] ");
|
|
buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
|
|
buff2.insert(0,":");
|
|
buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
|
|
buff2.insert(0,"[");
|
|
hypo = hypo->GetPrevHypo();
|
|
}
|
|
TRACE_ERR( buff << endl);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Manager::OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId) const
|
|
{
|
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
|
|
|
|
outputWordGraphStream << "J=" << linkId++
|
|
<< "\tS=" << prevHypo->GetId()
|
|
<< "\tE=" << hypo->GetId()
|
|
<< "\ta=";
|
|
|
|
// phrase table scores
|
|
const std::vector<PhraseDictionary*> &phraseTables = PhraseDictionary::GetColl();
|
|
std::vector<PhraseDictionary*>::const_iterator iterPhraseTable;
|
|
for (iterPhraseTable = phraseTables.begin() ; iterPhraseTable != phraseTables.end() ; ++iterPhraseTable) {
|
|
const PhraseDictionary *phraseTable = *iterPhraseTable;
|
|
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(phraseTable);
|
|
|
|
outputWordGraphStream << scores[0];
|
|
vector<float>::const_iterator iterScore;
|
|
for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
|
|
outputWordGraphStream << ", " << *iterScore;
|
|
}
|
|
}
|
|
|
|
// language model scores
|
|
outputWordGraphStream << "\tl=";
|
|
|
|
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
|
for (size_t i = 0; i < statefulFFs.size(); ++i) {
|
|
const StatefulFeatureFunction *ff = statefulFFs[i];
|
|
const LanguageModel *lm = static_cast<const LanguageModel*>(ff);
|
|
|
|
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lm);
|
|
|
|
outputWordGraphStream << scores[0];
|
|
vector<float>::const_iterator iterScore;
|
|
for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
|
|
outputWordGraphStream << ", " << *iterScore;
|
|
}
|
|
}
|
|
|
|
// re-ordering
|
|
outputWordGraphStream << "\tr=";
|
|
|
|
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
|
|
std::vector<FeatureFunction*>::const_iterator iter;
|
|
for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
|
|
const FeatureFunction *ff = *iter;
|
|
|
|
const DistortionScoreProducer *model = dynamic_cast<const DistortionScoreProducer*>(ff);
|
|
if (model) {
|
|
outputWordGraphStream << hypo->GetScoreBreakdown().GetScoreForProducer(model);
|
|
}
|
|
}
|
|
|
|
// output both source and target phrases in the word graph
|
|
outputWordGraphStream << "\tw=" << hypo->GetSourcePhraseStringRep()
|
|
<< "|" << hypo->GetCurrTargetPhrase();
|
|
|
|
outputWordGraphStream << endl;
|
|
}
|
|
|
|
// VN put back of OutputPassthroughInformation
|
|
void Manager::OutputPassthroughInformation(std::ostream &out, const Hypothesis *hypo) const
|
|
{
|
|
const std::string passthrough = hypo->GetManager().GetSource().GetPassthroughInformation();
|
|
out << passthrough;
|
|
}
|
|
// end of put back
|
|
|
|
void Manager::GetOutputLanguageModelOrder( std::ostream &out, const Hypothesis *hypo ) const
|
|
{
|
|
Phrase translation;
|
|
hypo->GetOutputPhrase(translation);
|
|
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
|
for (size_t i = 0; i < statefulFFs.size(); ++i) {
|
|
const StatefulFeatureFunction *ff = statefulFFs[i];
|
|
if (const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff)) {
|
|
lm->ReportHistoryOrder(out, translation);
|
|
}
|
|
}
|
|
}
|
|
|
|
void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const
|
|
{
|
|
const StaticData &staticData = StaticData::Instance();
|
|
const PARAM_VEC *params;
|
|
|
|
string fileName;
|
|
bool outputNBest = false;
|
|
params = staticData.GetParameter().GetParam("output-word-graph");
|
|
if (params && params->size()) {
|
|
fileName = params->at(0);
|
|
|
|
if (params->size() == 2) {
|
|
outputNBest = Scan<bool>(params->at(1));
|
|
}
|
|
}
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
outputWordGraphStream << "VERSION=1.0" << endl
|
|
<< "UTTERANCE=" << translationId << endl;
|
|
|
|
size_t linkId = 0;
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
|
for (iterStack = ++hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
|
|
const HypothesisStack &stack = **iterStack;
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
OutputWordGraph(outputWordGraphStream, hypo, linkId);
|
|
|
|
if (outputNBest) {
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
OutputWordGraph(outputWordGraphStream, loserHypo, linkId);
|
|
}
|
|
}
|
|
} //if (outputNBest)
|
|
} //for (iterHypo
|
|
} // for (iterStack
|
|
}
|
|
|
|
void Manager::GetSearchGraph(vector<SearchGraphNode>& searchGraph) const
|
|
{
|
|
std::map < int, bool > connected;
|
|
std::map < int, int > forward;
|
|
std::map < int, double > forwardScore;
|
|
|
|
// *** find connected hypotheses ***
|
|
std::vector< const Hypothesis *> connectedList;
|
|
GetConnectedGraph(&connected, &connectedList);
|
|
|
|
// ** compute best forward path for each hypothesis *** //
|
|
|
|
// forward cost of hypotheses on final stack is 0
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
forwardScore[ hypo->GetId() ] = 0.0f;
|
|
forward[ hypo->GetId() ] = -1;
|
|
}
|
|
|
|
// compete for best forward score of previous hypothesis
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
|
for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
|
|
const HypothesisStack &stack = **iterStack;
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
// make a play for previous hypothesis
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
|
|
|| forwardScore.find( prevHypo->GetId() )->second < fscore) {
|
|
forwardScore[ prevHypo->GetId() ] = fscore;
|
|
forward[ prevHypo->GetId() ] = hypo->GetId();
|
|
}
|
|
// all arcs also make a play
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
// make a play
|
|
const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
|
|
|| forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
|
|
forwardScore[ loserPrevHypo->GetId() ] = fscore;
|
|
forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
|
|
}
|
|
} // end for arc list
|
|
} // end if arc list empty
|
|
} // end if hypo connected
|
|
} // end for hypo
|
|
} // end for stack
|
|
|
|
// *** output all connected hypotheses *** //
|
|
|
|
connected[ 0 ] = true;
|
|
for (iterStack = hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
|
|
const HypothesisStack &stack = **iterStack;
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
searchGraph.push_back(SearchGraphNode(hypo,NULL,forward[hypo->GetId()],
|
|
forwardScore[hypo->GetId()]));
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
searchGraph.push_back(SearchGraphNode(loserHypo,hypo,
|
|
forward[hypo->GetId()], forwardScore[hypo->GetId()]));
|
|
}
|
|
} // end if arcList empty
|
|
} // end if connected
|
|
} // end for iterHypo
|
|
} // end for iterStack
|
|
|
|
}
|
|
|
|
void Manager::OutputFeatureWeightsForSLF(std::ostream &outputSearchGraphStream) const
|
|
{
|
|
outputSearchGraphStream.setf(std::ios::fixed);
|
|
outputSearchGraphStream.precision(6);
|
|
|
|
const vector<const StatelessFeatureFunction*>& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
|
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
|
size_t featureIndex = 1;
|
|
for (size_t i = 0; i < sff.size(); ++i) {
|
|
featureIndex = OutputFeatureWeightsForSLF(featureIndex, sff[i], outputSearchGraphStream);
|
|
}
|
|
for (size_t i = 0; i < slf.size(); ++i) {
|
|
/*
|
|
if (slf[i]->GetScoreProducerWeightShortName() != "u" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "tm" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "I" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "g")
|
|
*/
|
|
{
|
|
featureIndex = OutputFeatureWeightsForSLF(featureIndex, slf[i], outputSearchGraphStream);
|
|
}
|
|
}
|
|
const vector<PhraseDictionary*>& pds = PhraseDictionary::GetColl();
|
|
for( size_t i=0; i<pds.size(); i++ ) {
|
|
featureIndex = OutputFeatureWeightsForSLF(featureIndex, pds[i], outputSearchGraphStream);
|
|
}
|
|
const vector<GenerationDictionary*>& gds = GenerationDictionary::GetColl();
|
|
for( size_t i=0; i<gds.size(); i++ ) {
|
|
featureIndex = OutputFeatureWeightsForSLF(featureIndex, gds[i], outputSearchGraphStream);
|
|
}
|
|
}
|
|
|
|
void Manager::OutputFeatureValuesForSLF(const Hypothesis* hypo, bool zeros, std::ostream &outputSearchGraphStream) const
|
|
{
|
|
outputSearchGraphStream.setf(std::ios::fixed);
|
|
outputSearchGraphStream.precision(6);
|
|
|
|
// outputSearchGraphStream << endl;
|
|
// outputSearchGraphStream << (*hypo) << endl;
|
|
// const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
|
|
// outputSearchGraphStream << scoreCollection << endl;
|
|
|
|
const vector<const StatelessFeatureFunction*>& slf =StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
|
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
|
|
size_t featureIndex = 1;
|
|
for (size_t i = 0; i < sff.size(); ++i) {
|
|
featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, sff[i], outputSearchGraphStream);
|
|
}
|
|
for (size_t i = 0; i < slf.size(); ++i) {
|
|
/*
|
|
if (slf[i]->GetScoreProducerWeightShortName() != "u" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "tm" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "I" &&
|
|
slf[i]->GetScoreProducerWeightShortName() != "g")
|
|
*/
|
|
{
|
|
featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, slf[i], outputSearchGraphStream);
|
|
}
|
|
}
|
|
const vector<PhraseDictionary*>& pds = PhraseDictionary::GetColl();
|
|
for( size_t i=0; i<pds.size(); i++ ) {
|
|
featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, pds[i], outputSearchGraphStream);
|
|
}
|
|
const vector<GenerationDictionary*>& gds = GenerationDictionary::GetColl();
|
|
for( size_t i=0; i<gds.size(); i++ ) {
|
|
featureIndex = OutputFeatureValuesForSLF(featureIndex, zeros, hypo, gds[i], outputSearchGraphStream);
|
|
}
|
|
|
|
}
|
|
|
|
void Manager::OutputFeatureValuesForHypergraph(const Hypothesis* hypo, std::ostream &outputSearchGraphStream) const
|
|
{
|
|
outputSearchGraphStream.setf(std::ios::fixed);
|
|
outputSearchGraphStream.precision(6);
|
|
ScoreComponentCollection scores = hypo->GetScoreBreakdown();
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
if (prevHypo) {
|
|
scores.MinusEquals(prevHypo->GetScoreBreakdown());
|
|
}
|
|
scores.Save(outputSearchGraphStream, false);
|
|
}
|
|
|
|
|
|
size_t Manager::OutputFeatureWeightsForSLF(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream) const
|
|
{
|
|
size_t numScoreComps = ff->GetNumScoreComponents();
|
|
if (numScoreComps != 0) {
|
|
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
|
|
for (size_t i = 0; i < numScoreComps; ++i) {
|
|
outputSearchGraphStream << "# " << ff->GetScoreProducerDescription()
|
|
<< " " << ff->GetScoreProducerDescription()
|
|
<< " " << (i+1) << " of " << numScoreComps << endl
|
|
<< "x" << (index+i) << "scale=" << values[i] << endl;
|
|
}
|
|
return index+numScoreComps;
|
|
} else {
|
|
cerr << "Sparse features are not supported when outputting HTK standard lattice format" << endl;
|
|
assert(false);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
size_t
|
|
Manager::
|
|
OutputFeatureValuesForSLF(size_t index, bool zeros, const Hypothesis* hypo,
|
|
const FeatureFunction* ff, std::ostream &out) const
|
|
{
|
|
const ScoreComponentCollection& scoreCollection = hypo->GetScoreBreakdown();
|
|
vector<float> featureValues = scoreCollection.GetScoresForProducer(ff);
|
|
size_t numScoreComps = featureValues.size();
|
|
for (size_t i = 0; i < numScoreComps; ++i) {
|
|
out << "x" << (index+i) << "=" << ((zeros) ? 0.0 : featureValues[i]) << " ";
|
|
}
|
|
return index + numScoreComps;
|
|
}
|
|
|
|
/**! Output search graph in hypergraph format of Kenneth Heafield's lazy hypergraph decoder */
|
|
void
|
|
Manager::
|
|
OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const
|
|
{
|
|
|
|
VERBOSE(2,"Getting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
|
|
|
vector<SearchGraphNode> searchGraph;
|
|
GetSearchGraph(searchGraph);
|
|
|
|
|
|
map<int,int> mosesIDToHypergraphID;
|
|
// map<int,int> hypergraphIDToMosesID;
|
|
set<int> terminalNodes;
|
|
multimap<int,int> hypergraphIDToArcs;
|
|
|
|
VERBOSE(2,"Gathering information about search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
|
|
|
long numNodes = 0;
|
|
long endNode = 0;
|
|
{
|
|
long hypergraphHypothesisID = 0;
|
|
for (size_t arcNumber = 0, size=searchGraph.size(); arcNumber < size; ++arcNumber) {
|
|
|
|
// Get an id number for the previous hypothesis
|
|
const Hypothesis *prevHypo = searchGraph[arcNumber].hypo->GetPrevHypo();
|
|
if (prevHypo!=NULL) {
|
|
int mosesPrevHypothesisID = prevHypo->GetId();
|
|
if (mosesIDToHypergraphID.count(mosesPrevHypothesisID) == 0) {
|
|
mosesIDToHypergraphID[mosesPrevHypothesisID] = hypergraphHypothesisID;
|
|
// hypergraphIDToMosesID[hypergraphHypothesisID] = mosesPrevHypothesisID;
|
|
hypergraphHypothesisID += 1;
|
|
}
|
|
}
|
|
|
|
// Get an id number for this hypothesis
|
|
int mosesHypothesisID;
|
|
if (searchGraph[arcNumber].recombinationHypo) {
|
|
mosesHypothesisID = searchGraph[arcNumber].recombinationHypo->GetId();
|
|
} else {
|
|
mosesHypothesisID = searchGraph[arcNumber].hypo->GetId();
|
|
}
|
|
|
|
if (mosesIDToHypergraphID.count(mosesHypothesisID) == 0) {
|
|
|
|
mosesIDToHypergraphID[mosesHypothesisID] = hypergraphHypothesisID;
|
|
// hypergraphIDToMosesID[hypergraphHypothesisID] = mosesHypothesisID;
|
|
|
|
bool terminalNode = (searchGraph[arcNumber].forward == -1);
|
|
if (terminalNode) {
|
|
// Final arc to end node, representing the end of the sentence </s>
|
|
terminalNodes.insert(hypergraphHypothesisID);
|
|
}
|
|
|
|
hypergraphHypothesisID += 1;
|
|
}
|
|
|
|
// Record that this arc ends at this node
|
|
hypergraphIDToArcs.insert(pair<int,int>(mosesIDToHypergraphID[mosesHypothesisID],arcNumber));
|
|
|
|
}
|
|
|
|
// Unique end node
|
|
endNode = hypergraphHypothesisID;
|
|
// mosesIDToHypergraphID[hypergraphHypothesisID] = hypergraphHypothesisID;
|
|
numNodes = endNode + 1;
|
|
|
|
}
|
|
|
|
|
|
long numArcs = searchGraph.size() + terminalNodes.size();
|
|
|
|
//Header
|
|
outputSearchGraphStream << "# target ||| features ||| source-covered" << endl;
|
|
|
|
// Print number of nodes and arcs
|
|
outputSearchGraphStream << numNodes << " " << numArcs << endl;
|
|
|
|
VERBOSE(2,"Search graph to output as hypergraph for sentence " << m_source.GetTranslationId()
|
|
<< " contains " << numArcs << " arcs and " << numNodes << " nodes" << std::endl)
|
|
|
|
VERBOSE(2,"Outputting search graph to output as hypergraph for sentence " << m_source.GetTranslationId() << std::endl)
|
|
|
|
|
|
for (int hypergraphHypothesisID=0; hypergraphHypothesisID < endNode; hypergraphHypothesisID+=1) {
|
|
if (hypergraphHypothesisID % 100000 == 0) {
|
|
VERBOSE(2,"Processed " << hypergraphHypothesisID << " of " << numNodes << " hypergraph nodes for sentence " << m_source.GetTranslationId() << std::endl);
|
|
}
|
|
// int mosesID = hypergraphIDToMosesID[hypergraphHypothesisID];
|
|
size_t count = hypergraphIDToArcs.count(hypergraphHypothesisID);
|
|
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has " << count << " incoming arcs" << std::endl)
|
|
if (count > 0) {
|
|
outputSearchGraphStream << "# node " << hypergraphHypothesisID << endl;
|
|
outputSearchGraphStream << count << "\n";
|
|
|
|
pair<multimap<int,int>::iterator, multimap<int,int>::iterator> range =
|
|
hypergraphIDToArcs.equal_range(hypergraphHypothesisID);
|
|
for (multimap<int,int>::iterator it=range.first; it!=range.second; ++it) {
|
|
int lineNumber = (*it).second;
|
|
const Hypothesis *thisHypo = searchGraph[lineNumber].hypo;
|
|
int mosesHypothesisID;// = thisHypo->GetId();
|
|
if (searchGraph[lineNumber].recombinationHypo) {
|
|
mosesHypothesisID = searchGraph[lineNumber].recombinationHypo->GetId();
|
|
} else {
|
|
mosesHypothesisID = searchGraph[lineNumber].hypo->GetId();
|
|
}
|
|
// int actualHypergraphHypothesisID = mosesIDToHypergraphID[mosesHypothesisID];
|
|
UTIL_THROW_IF2(
|
|
(hypergraphHypothesisID != mosesIDToHypergraphID[mosesHypothesisID]),
|
|
"Error while writing search lattice as hypergraph for sentence " << m_source.GetTranslationId() << ". " <<
|
|
"Moses node " << mosesHypothesisID << " was expected to have hypergraph id " << hypergraphHypothesisID <<
|
|
", but actually had hypergraph id " << mosesIDToHypergraphID[mosesHypothesisID] <<
|
|
". There are " << numNodes << " nodes in the search lattice."
|
|
);
|
|
|
|
const Hypothesis *prevHypo = thisHypo->GetPrevHypo();
|
|
if (prevHypo==NULL) {
|
|
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " start of sentence" << std::endl)
|
|
outputSearchGraphStream << "<s> ||| ||| 0\n";
|
|
} else {
|
|
int startNode = mosesIDToHypergraphID[prevHypo->GetId()];
|
|
// VERBOSE(2,"Hypergraph node " << hypergraphHypothesisID << " has parent node " << startNode << std::endl)
|
|
UTIL_THROW_IF2(
|
|
(startNode >= hypergraphHypothesisID),
|
|
"Error while writing search lattice as hypergraph for sentence" << m_source.GetTranslationId() << ". " <<
|
|
"The nodes must be output in topological order. The code attempted to violate this restriction."
|
|
);
|
|
|
|
const TargetPhrase &targetPhrase = thisHypo->GetCurrTargetPhrase();
|
|
int targetWordCount = targetPhrase.GetSize();
|
|
|
|
outputSearchGraphStream << "[" << startNode << "] ";
|
|
for (int targetWordIndex=0; targetWordIndex<targetWordCount; targetWordIndex+=1) {
|
|
outputSearchGraphStream << targetPhrase.GetWord(targetWordIndex)[0]->GetString() << " ";
|
|
}
|
|
outputSearchGraphStream << " ||| ";
|
|
OutputFeatureValuesForHypergraph(thisHypo, outputSearchGraphStream);
|
|
outputSearchGraphStream << " ||| " << thisHypo->GetWordsBitmap().GetNumWordsCovered();
|
|
outputSearchGraphStream << "\n";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Print node and arc(s) for end of sentence </s>
|
|
outputSearchGraphStream << "# node " << endNode << endl;
|
|
outputSearchGraphStream << terminalNodes.size() << "\n";
|
|
for (set<int>::iterator it=terminalNodes.begin(); it!=terminalNodes.end(); ++it) {
|
|
outputSearchGraphStream << "[" << (*it) << "] </s> ||| ||| " << GetSource().GetSize() << "\n";
|
|
}
|
|
|
|
}
|
|
|
|
|
|
/**! Output search graph in HTK standard lattice format (SLF) */
|
|
void Manager::OutputSearchGraphAsSLF(long translationId, std::ostream &outputSearchGraphStream) const
|
|
{
|
|
|
|
vector<SearchGraphNode> searchGraph;
|
|
GetSearchGraph(searchGraph);
|
|
|
|
long numArcs = 0;
|
|
long numNodes = 0;
|
|
|
|
map<int,int> nodes;
|
|
set<int> terminalNodes;
|
|
|
|
// Unique start node
|
|
nodes[0] = 0;
|
|
|
|
for (size_t arcNumber = 0; arcNumber < searchGraph.size(); ++arcNumber) {
|
|
|
|
int targetWordCount = searchGraph[arcNumber].hypo->GetCurrTargetPhrase().GetSize();
|
|
numArcs += targetWordCount;
|
|
|
|
int hypothesisID = searchGraph[arcNumber].hypo->GetId();
|
|
if (nodes.count(hypothesisID) == 0) {
|
|
|
|
numNodes += targetWordCount;
|
|
nodes[hypothesisID] = numNodes;
|
|
//numNodes += 1;
|
|
|
|
bool terminalNode = (searchGraph[arcNumber].forward == -1);
|
|
if (terminalNode) {
|
|
numArcs += 1;
|
|
}
|
|
}
|
|
|
|
}
|
|
numNodes += 1;
|
|
|
|
// Unique end node
|
|
nodes[numNodes] = numNodes;
|
|
|
|
outputSearchGraphStream << "UTTERANCE=Sentence_" << translationId << endl;
|
|
outputSearchGraphStream << "VERSION=1.1" << endl;
|
|
outputSearchGraphStream << "base=2.71828182845905" << endl;
|
|
outputSearchGraphStream << "NODES=" << (numNodes+1) << endl;
|
|
outputSearchGraphStream << "LINKS=" << numArcs << endl;
|
|
|
|
OutputFeatureWeightsForSLF(outputSearchGraphStream);
|
|
|
|
for (size_t arcNumber = 0, lineNumber = 0; lineNumber < searchGraph.size(); ++lineNumber) {
|
|
const Hypothesis *thisHypo = searchGraph[lineNumber].hypo;
|
|
const Hypothesis *prevHypo = thisHypo->GetPrevHypo();
|
|
if (prevHypo) {
|
|
|
|
int startNode = nodes[prevHypo->GetId()];
|
|
int endNode = nodes[thisHypo->GetId()];
|
|
bool terminalNode = (searchGraph[lineNumber].forward == -1);
|
|
const TargetPhrase &targetPhrase = thisHypo->GetCurrTargetPhrase();
|
|
int targetWordCount = targetPhrase.GetSize();
|
|
|
|
for (int targetWordIndex=0; targetWordIndex<targetWordCount; targetWordIndex+=1) {
|
|
int x = (targetWordCount-targetWordIndex);
|
|
|
|
outputSearchGraphStream << "J=" << arcNumber;
|
|
|
|
if (targetWordIndex==0) {
|
|
outputSearchGraphStream << " S=" << startNode;
|
|
} else {
|
|
outputSearchGraphStream << " S=" << endNode - x;
|
|
}
|
|
|
|
outputSearchGraphStream << " E=" << endNode - (x-1)
|
|
<< " W=" << targetPhrase.GetWord(targetWordIndex);
|
|
|
|
OutputFeatureValuesForSLF(thisHypo, (targetWordIndex>0), outputSearchGraphStream);
|
|
|
|
outputSearchGraphStream << endl;
|
|
|
|
arcNumber += 1;
|
|
}
|
|
|
|
if (terminalNode && terminalNodes.count(endNode) == 0) {
|
|
terminalNodes.insert(endNode);
|
|
outputSearchGraphStream << "J=" << arcNumber
|
|
<< " S=" << endNode
|
|
<< " E=" << numNodes
|
|
<< endl;
|
|
arcNumber += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
|
|
void
|
|
OutputSearchNode(AllOptions const& opts, long translationId,
|
|
std::ostream &out,
|
|
SearchGraphNode const& searchNode)
|
|
{
|
|
const vector<FactorType> &outputFactorOrder = opts.output.factor_order;
|
|
bool extendedFormat = opts.output.SearchGraphExtended.size();
|
|
out << translationId;
|
|
|
|
// special case: initial hypothesis
|
|
if ( searchNode.hypo->GetId() == 0 ) {
|
|
out << " hyp=0 stack=0";
|
|
if (extendedFormat) {
|
|
out << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore;
|
|
}
|
|
out << endl;
|
|
return;
|
|
}
|
|
|
|
const Hypothesis *prevHypo = searchNode.hypo->GetPrevHypo();
|
|
|
|
// output in traditional format
|
|
if (!extendedFormat) {
|
|
out << " hyp=" << searchNode.hypo->GetId()
|
|
<< " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
|
|
<< " back=" << prevHypo->GetId()
|
|
<< " score=" << searchNode.hypo->GetScore()
|
|
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
|
|
|
if (searchNode.recombinationHypo != NULL)
|
|
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
|
|
|
out << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
|
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
|
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos()
|
|
<< " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder)
|
|
<< endl;
|
|
return;
|
|
}
|
|
|
|
out << " hyp=" << searchNode.hypo->GetId();
|
|
out << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
|
|
<< " back=" << prevHypo->GetId()
|
|
<< " score=" << searchNode.hypo->GetScore()
|
|
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
|
|
|
if (searchNode.recombinationHypo != NULL)
|
|
out << " recombined=" << searchNode.recombinationHypo->GetId();
|
|
|
|
out << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
|
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
|
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos();
|
|
|
|
// Modified so that -osgx is a superset of -osg (GST Oct 2011)
|
|
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
|
|
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
|
|
out << " scores=\"" << scoreBreakdown << "\""
|
|
<< " out=\"" << searchNode.hypo->GetSourcePhraseStringRep()
|
|
<< "|" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << "\"" << endl;
|
|
}
|
|
|
|
void Manager::GetConnectedGraph(
|
|
std::map< int, bool >* pConnected,
|
|
std::vector< const Hypothesis* >* pConnectedList) const
|
|
{
|
|
std::map < int, bool >& connected = *pConnected;
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
|
|
|
// start with the ones in the final stack
|
|
const std::vector < HypothesisStack* > &hypoStackColl
|
|
= m_search->GetHypothesisStacks();
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
connected[ hypo->GetId() ] = true;
|
|
connectedList.push_back( hypo );
|
|
}
|
|
// move back from known connected hypotheses
|
|
for(size_t i=0; i<connectedList.size(); i++) {
|
|
const Hypothesis *hypo = connectedList[i];
|
|
|
|
// add back pointer
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
if (prevHypo && prevHypo->GetId() > 0 // don't add empty hypothesis
|
|
&& connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
|
|
connected[ prevHypo->GetId() ] = true;
|
|
connectedList.push_back( prevHypo );
|
|
}
|
|
|
|
// add arcs
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
if (connected.find( loserHypo->GetId() ) == connected.end()) { // don't add already added
|
|
connected[ loserHypo->GetId() ] = true;
|
|
connectedList.push_back( loserHypo );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Manager::GetWinnerConnectedGraph(
|
|
std::map< int, bool >* pConnected,
|
|
std::vector< const Hypothesis* >* pConnectedList) const
|
|
{
|
|
std::map < int, bool >& connected = *pConnected;
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
|
|
|
// start with the ones in the final stack
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
connected[ hypo->GetId() ] = true;
|
|
connectedList.push_back( hypo );
|
|
}
|
|
|
|
// move back from known connected hypotheses
|
|
for(size_t i=0; i<connectedList.size(); i++) {
|
|
const Hypothesis *hypo = connectedList[i];
|
|
|
|
// add back pointer
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
if (prevHypo->GetId() > 0 // don't add empty hypothesis
|
|
&& connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
|
|
connected[ prevHypo->GetId() ] = true;
|
|
connectedList.push_back( prevHypo );
|
|
}
|
|
|
|
// add arcs
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
if (connected.find( loserHypo->GetPrevHypo()->GetId() ) == connected.end() && loserHypo->GetPrevHypo()->GetId() > 0) { // don't add already added & don't add hyp 0
|
|
connected[ loserHypo->GetPrevHypo()->GetId() ] = true;
|
|
connectedList.push_back( loserHypo->GetPrevHypo() );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
#ifdef HAVE_PROTOBUF
|
|
|
|
void SerializeEdgeInfo(const Hypothesis* hypo, hgmert::Hypergraph_Edge* edge)
|
|
{
|
|
hgmert::Rule* rule = edge->mutable_rule();
|
|
hypo->GetCurrTargetPhrase().WriteToRulePB(rule);
|
|
const Hypothesis* prev = hypo->GetPrevHypo();
|
|
// if the feature values are empty, they default to 0
|
|
if (!prev) return;
|
|
// score breakdown is an aggregate (forward) quantity, but the exported
|
|
// graph object just wants the feature values on the edges
|
|
const ScoreComponentCollection& scores = hypo->GetScoreBreakdown();
|
|
const ScoreComponentCollection& pscores = prev->GetScoreBreakdown();
|
|
for (unsigned int i = 0; i < scores.size(); ++i)
|
|
edge->add_feature_values((scores[i] - pscores[i]) * -1.0);
|
|
}
|
|
|
|
hgmert::Hypergraph_Node* GetHGNode(
|
|
const Hypothesis* hypo,
|
|
std::map< int, int>* i2hgnode,
|
|
hgmert::Hypergraph* hg,
|
|
int* hgNodeIdx)
|
|
{
|
|
hgmert::Hypergraph_Node* hgnode;
|
|
std::map < int, int >::iterator idxi = i2hgnode->find(hypo->GetId());
|
|
if (idxi == i2hgnode->end()) {
|
|
*hgNodeIdx = ((*i2hgnode)[hypo->GetId()] = hg->nodes_size());
|
|
hgnode = hg->add_nodes();
|
|
} else {
|
|
*hgNodeIdx = idxi->second;
|
|
hgnode = hg->mutable_nodes(*hgNodeIdx);
|
|
}
|
|
return hgnode;
|
|
}
|
|
|
|
void Manager::SerializeSearchGraphPB(
|
|
long translationId,
|
|
std::ostream& outputStream) const
|
|
{
|
|
using namespace hgmert;
|
|
std::map < int, bool > connected;
|
|
std::map < int, int > i2hgnode;
|
|
std::vector< const Hypothesis *> connectedList;
|
|
GetConnectedGraph(&connected, &connectedList);
|
|
connected[ 0 ] = true;
|
|
Hypergraph hg;
|
|
hg.set_is_sorted(false);
|
|
int num_feats = (*m_search->GetHypothesisStacks().back()->begin())->GetScoreBreakdown().size();
|
|
hg.set_num_features(num_feats);
|
|
StaticData::Instance().GetScoreIndexManager().SerializeFeatureNamesToPB(&hg);
|
|
Hypergraph_Node* goal = hg.add_nodes(); // idx=0 goal node must have idx 0
|
|
Hypergraph_Node* source = hg.add_nodes(); // idx=1
|
|
i2hgnode[-1] = 1; // source node
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
for (std::vector < HypothesisStack* >::const_iterator iterStack = hypoStackColl.begin();
|
|
iterStack != hypoStackColl.end() ; ++iterStack) {
|
|
const HypothesisStack &stack = **iterStack;
|
|
HypothesisStack::const_iterator iterHypo;
|
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
bool is_goal = hypo->GetWordsBitmap().IsComplete();
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
int headNodeIdx;
|
|
Hypergraph_Node* headNode = GetHGNode(hypo, &i2hgnode, &hg, &headNodeIdx);
|
|
if (is_goal) {
|
|
Hypergraph_Edge* ge = hg.add_edges();
|
|
ge->set_head_node(0); // goal
|
|
ge->add_tail_nodes(headNodeIdx);
|
|
ge->mutable_rule()->add_trg_words("[X,1]");
|
|
}
|
|
Hypergraph_Edge* edge = hg.add_edges();
|
|
SerializeEdgeInfo(hypo, edge);
|
|
edge->set_head_node(headNodeIdx);
|
|
const Hypothesis* prev = hypo->GetPrevHypo();
|
|
int tailNodeIdx = 1; // source
|
|
if (prev)
|
|
tailNodeIdx = i2hgnode.find(prev->GetId())->second;
|
|
edge->add_tail_nodes(tailNodeIdx);
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
UTIL_THROW_IF2(!connected[loserHypo->GetId()],
|
|
"Hypothesis " << loserHypo->GetId() << " is not connected");
|
|
Hypergraph_Edge* edge = hg.add_edges();
|
|
SerializeEdgeInfo(loserHypo, edge);
|
|
edge->set_head_node(headNodeIdx);
|
|
tailNodeIdx = i2hgnode.find(loserHypo->GetPrevHypo()->GetId())->second;
|
|
edge->add_tail_nodes(tailNodeIdx);
|
|
}
|
|
} // end if arcList empty
|
|
} // end if connected
|
|
} // end for iterHypo
|
|
} // end for iterStack
|
|
hg.SerializeToOstream(&outputStream);
|
|
}
|
|
#endif
|
|
|
|
void
|
|
Manager::
|
|
OutputSearchGraph(long translationId, std::ostream &out) const
|
|
{
|
|
vector<SearchGraphNode> searchGraph;
|
|
GetSearchGraph(searchGraph);
|
|
for (size_t i = 0; i < searchGraph.size(); ++i) {
|
|
OutputSearchNode(*options(),translationId,out,searchGraph[i]);
|
|
}
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
GetForwardBackwardSearchGraph
|
|
( std::map< int, bool >* pConnected,
|
|
std::vector<Hypothesis const* >* pConnectedList,
|
|
std::map<Hypothesis const*, set<Hypothesis const*> >* pOutgoingHyps,
|
|
vector< float>* pFwdBwdScores) const
|
|
{
|
|
std::map < int, bool > &connected = *pConnected;
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
|
std::map < int, int > forward;
|
|
std::map < int, double > forwardScore;
|
|
|
|
std::map < const Hypothesis*, set <const Hypothesis*> > & outgoingHyps
|
|
= *pOutgoingHyps;
|
|
vector< float> & estimatedScores = *pFwdBwdScores;
|
|
|
|
// *** find connected hypotheses ***
|
|
GetWinnerConnectedGraph(&connected, &connectedList);
|
|
|
|
// ** compute best forward path for each hypothesis *** //
|
|
|
|
// forward cost of hypotheses on final stack is 0
|
|
const std::vector < HypothesisStack* > &hypoStackColl
|
|
= m_search->GetHypothesisStacks();
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
forwardScore[ hypo->GetId() ] = 0.0f;
|
|
forward[ hypo->GetId() ] = -1;
|
|
}
|
|
|
|
// compete for best forward score of previous hypothesis
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
|
for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
|
|
const HypothesisStack &stack = **iterStack;
|
|
HypothesisStack::const_iterator iterHypo;
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
const Hypothesis *hypo = *iterHypo;
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
// make a play for previous hypothesis
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
|
|
|| forwardScore.find( prevHypo->GetId() )->second < fscore) {
|
|
forwardScore[ prevHypo->GetId() ] = fscore;
|
|
forward[ prevHypo->GetId() ] = hypo->GetId();
|
|
}
|
|
//store outgoing info
|
|
outgoingHyps[prevHypo].insert(hypo);
|
|
|
|
// all arcs also make a play
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
if (arcList != NULL) {
|
|
ArcList::const_iterator iterArcList;
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
// make a play
|
|
const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
|
|
|| forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
|
|
forwardScore[ loserPrevHypo->GetId() ] = fscore;
|
|
forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
|
|
}
|
|
//store outgoing info
|
|
outgoingHyps[loserPrevHypo].insert(hypo);
|
|
|
|
|
|
} // end for arc list
|
|
} // end if arc list empty
|
|
} // end if hypo connected
|
|
} // end for hypo
|
|
} // end for stack
|
|
|
|
for (std::vector< const Hypothesis *>::iterator it = connectedList.begin(); it != connectedList.end(); ++it) {
|
|
float estimatedScore = (*it)->GetScore() + forwardScore[(*it)->GetId()];
|
|
estimatedScores.push_back(estimatedScore);
|
|
}
|
|
}
|
|
|
|
|
|
const Hypothesis *Manager::GetBestHypothesis() const
|
|
{
|
|
return m_search->GetBestHypothesis();
|
|
}
|
|
|
|
int Manager::GetNextHypoId()
|
|
{
|
|
GetSentenceStats().AddCreated(); // count created hypotheses
|
|
return m_hypoId++;
|
|
}
|
|
|
|
void Manager::ResetSentenceStats(const InputType& source)
|
|
{
|
|
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
|
|
}
|
|
SentenceStats& Manager::GetSentenceStats() const
|
|
{
|
|
return *m_sentenceStats;
|
|
|
|
}
|
|
|
|
void Manager::OutputBest(OutputCollector *collector) const
|
|
{
|
|
long translationId = m_source.GetTranslationId();
|
|
|
|
Timer additionalReportingTime;
|
|
|
|
// apply decision rule and output best translation(s)
|
|
if (collector) {
|
|
ostringstream out;
|
|
ostringstream debug;
|
|
FixPrecision(debug,PRECISION);
|
|
|
|
// all derivations - send them to debug stream
|
|
if (options()->output.PrintAllDerivations) {
|
|
additionalReportingTime.start();
|
|
PrintAllDerivations(translationId, debug);
|
|
additionalReportingTime.stop();
|
|
}
|
|
|
|
Timer decisionRuleTime;
|
|
decisionRuleTime.start();
|
|
|
|
// MAP decoding: best hypothesis
|
|
const Hypothesis* bestHypo = NULL;
|
|
if (!options()->mbr.enabled) {
|
|
bestHypo = GetBestHypothesis();
|
|
if (bestHypo) {
|
|
if (options()->output.ReportHypoScore) {
|
|
out << bestHypo->GetFutureScore() << ' ';
|
|
}
|
|
if (options()->output.RecoverPath) {
|
|
bestHypo->OutputInput(out);
|
|
out << "||| ";
|
|
}
|
|
|
|
if (options()->output.PrintID) {
|
|
out << translationId << " ";
|
|
}
|
|
|
|
// VN : I put back the code for OutputPassthroughInformation
|
|
if (options()->output.PrintPassThrough) {
|
|
OutputPassthroughInformation(out, bestHypo);
|
|
}
|
|
// end of add back
|
|
|
|
if (options()->output.ReportSegmentation == 2) {
|
|
GetOutputLanguageModelOrder(out, bestHypo);
|
|
}
|
|
OutputSurface(out,*bestHypo, true);
|
|
if (options()->output.PrintAlignmentInfo) {
|
|
out << "||| ";
|
|
bestHypo->OutputAlignment(out, true);
|
|
}
|
|
|
|
IFVERBOSE(1) {
|
|
debug << "BEST TRANSLATION: " << *bestHypo << endl;
|
|
}
|
|
} else {
|
|
VERBOSE(1, "NO BEST TRANSLATION" << endl);
|
|
}
|
|
|
|
out << endl;
|
|
} // if (!staticData.UseMBR())
|
|
|
|
// MBR decoding (n-best MBR, lattice MBR, consensus)
|
|
else {
|
|
// we first need the n-best translations
|
|
size_t nBestSize = options()->mbr.size;
|
|
if (nBestSize <= 0) {
|
|
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
|
|
exit(1);
|
|
}
|
|
TrellisPathList nBestList;
|
|
CalcNBest(nBestSize, nBestList, true);
|
|
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
|
|
IFVERBOSE(2) {
|
|
PrintUserTime("calculated n-best list for (L)MBR decoding");
|
|
}
|
|
|
|
// lattice MBR
|
|
if (options()->lmbr.enabled) {
|
|
if (options()->nbest.enabled) {
|
|
//lattice mbr nbest
|
|
vector<LatticeMBRSolution> solutions;
|
|
size_t n = min(nBestSize, options()->nbest.nbest_size);
|
|
getLatticeMBRNBest(*this,nBestList,solutions,n);
|
|
OutputLatticeMBRNBest(m_latticeNBestOut, solutions, translationId);
|
|
} else {
|
|
//Lattice MBR decoding
|
|
vector<Word> mbrBestHypo = doLatticeMBR(*this,nBestList);
|
|
OutputBestHypo(mbrBestHypo, out);
|
|
IFVERBOSE(2) {
|
|
PrintUserTime("finished Lattice MBR decoding");
|
|
}
|
|
}
|
|
}
|
|
|
|
// consensus decoding
|
|
else if (options()->search.consensus) {
|
|
const TrellisPath &conBestHypo = doConsensusDecoding(*this,nBestList);
|
|
OutputBestHypo(conBestHypo, out);
|
|
OutputAlignment(m_alignmentOut, conBestHypo);
|
|
IFVERBOSE(2) {
|
|
PrintUserTime("finished Consensus decoding");
|
|
}
|
|
}
|
|
|
|
// n-best MBR decoding
|
|
else {
|
|
const TrellisPath &mbrBestHypo = doMBR(nBestList, *options());
|
|
OutputBestHypo(mbrBestHypo, out);
|
|
OutputAlignment(m_alignmentOut, mbrBestHypo);
|
|
IFVERBOSE(2) {
|
|
PrintUserTime("finished MBR decoding");
|
|
}
|
|
}
|
|
}
|
|
|
|
// report best translation to output collector
|
|
collector->Write(translationId,out.str(),debug.str());
|
|
|
|
decisionRuleTime.stop();
|
|
VERBOSE(1, "Line " << translationId << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
|
|
} // if (m_ioWrapper.GetSingleBestOutputCollector())
|
|
|
|
}
|
|
|
|
void Manager::OutputNBest(OutputCollector *collector) const
|
|
{
|
|
if (collector == NULL) {
|
|
return;
|
|
}
|
|
|
|
if (options()->lmbr.enabled) {
|
|
if (options()->nbest.enabled) {
|
|
collector->Write(m_source.GetTranslationId(), m_latticeNBestOut.str());
|
|
}
|
|
} else {
|
|
TrellisPathList nBestList;
|
|
ostringstream out;
|
|
NBestOptions const& nbo = options()->nbest;
|
|
CalcNBest(nbo.nbest_size, nBestList, nbo.only_distinct);
|
|
OutputNBest(out, nBestList);
|
|
collector->Write(m_source.GetTranslationId(), out.str());
|
|
}
|
|
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputNBest(std::ostream& out, Moses::TrellisPathList const& nBestList) const
|
|
{
|
|
NBestOptions const& nbo = options()->nbest;
|
|
bool reportAllFactors = nbo.include_all_factors;
|
|
bool includeSegmentation = nbo.include_segmentation;
|
|
bool includeWordAlignment = nbo.include_alignment_info;
|
|
|
|
TrellisPathList::const_iterator iter;
|
|
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
|
|
const TrellisPath &path = **iter;
|
|
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
|
|
|
// print the surface factor of the translation
|
|
out << m_source.GetTranslationId() << " ||| ";
|
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
OutputSurface(out, edge);
|
|
}
|
|
out << " |||";
|
|
|
|
// print scores with feature names
|
|
bool with_labels = options()->nbest.include_feature_labels;
|
|
path.GetScoreBreakdown()->OutputAllFeatureScores(out, with_labels);
|
|
|
|
// total
|
|
out << " ||| " << path.GetFutureScore();
|
|
|
|
//phrase-to-phrase segmentation
|
|
if (includeSegmentation) {
|
|
out << " |||";
|
|
for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
const Range &sourceRange = edge.GetCurrSourceWordsRange();
|
|
Range targetRange = path.GetTargetWordsRange(edge);
|
|
out << " " << sourceRange.GetStartPos();
|
|
if (sourceRange.GetStartPos() < sourceRange.GetEndPos()) {
|
|
out << "-" << sourceRange.GetEndPos();
|
|
}
|
|
out<< "=" << targetRange.GetStartPos();
|
|
if (targetRange.GetStartPos() < targetRange.GetEndPos()) {
|
|
out<< "-" << targetRange.GetEndPos();
|
|
}
|
|
}
|
|
}
|
|
|
|
if (includeWordAlignment) {
|
|
out << " ||| ";
|
|
for (int currEdge = (int)edges.size() - 2 ; currEdge >= 0 ; currEdge--) {
|
|
const Hypothesis &edge = *edges[currEdge];
|
|
const Range &sourceRange = edge.GetCurrSourceWordsRange();
|
|
Range targetRange = path.GetTargetWordsRange(edge);
|
|
const int sourceOffset = sourceRange.GetStartPos();
|
|
const int targetOffset = targetRange.GetStartPos();
|
|
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
|
|
|
|
OutputAlignment(out, ai, sourceOffset, targetOffset);
|
|
|
|
}
|
|
}
|
|
|
|
if (options()->output.RecoverPath) {
|
|
out << " ||| ";
|
|
OutputInput(out, edges[0]);
|
|
}
|
|
|
|
out << endl;
|
|
}
|
|
|
|
out << std::flush;
|
|
}
|
|
|
|
//////////////////////////////////////////////////////////////////////////
|
|
/***
|
|
* print surface factor only for the given phrase
|
|
*/
|
|
void
|
|
Manager::
|
|
OutputSurface(std::ostream &out, Hypothesis const& edge, bool const recursive) const
|
|
{
|
|
if (recursive && edge.GetPrevHypo()) {
|
|
OutputSurface(out,*edge.GetPrevHypo(), true);
|
|
}
|
|
|
|
std::vector<FactorType> outputFactorOrder = options()->output.factor_order;
|
|
UTIL_THROW_IF2(outputFactorOrder.size() == 0,
|
|
"Must specific at least 1 output factor");
|
|
|
|
FactorType placeholderFactor = options()->input.placeholder_factor;
|
|
std::map<size_t, const Factor*> placeholders;
|
|
if (placeholderFactor != NOT_FOUND) {
|
|
// creates map of target position -> factor for placeholders
|
|
placeholders = GetPlaceholders(edge, placeholderFactor);
|
|
}
|
|
|
|
bool markUnknown = options()->unk.mark;
|
|
std::string const& fd = options()->output.factor_delimiter;
|
|
|
|
TargetPhrase const& phrase = edge.GetCurrTargetPhrase();
|
|
size_t size = phrase.GetSize();
|
|
for (size_t pos = 0 ; pos < size ; pos++) {
|
|
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
|
|
if (placeholders.size()) {
|
|
// do placeholders
|
|
std::map<size_t, const Factor*>::const_iterator iter = placeholders.find(pos);
|
|
if (iter != placeholders.end()) {
|
|
factor = iter->second;
|
|
}
|
|
}
|
|
|
|
UTIL_THROW_IF2(factor == NULL, "No factor 0 at position " << pos);
|
|
|
|
//preface surface form with UNK if marking unknowns
|
|
const Word &word = phrase.GetWord(pos);
|
|
if(markUnknown && word.IsOOV()) {
|
|
out << options()->unk.prefix;
|
|
}
|
|
|
|
out << *factor;
|
|
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
|
|
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
|
|
if (factor) out << fd << *factor;
|
|
//else out << fd << UNKNOWN_FACTOR;
|
|
}
|
|
|
|
if(markUnknown && word.IsOOV()) {
|
|
out << options()->unk.suffix;
|
|
}
|
|
|
|
out << " ";
|
|
|
|
}
|
|
|
|
// trace ("report segmentation") option "-t" / "-tt"
|
|
int reportSegmentation = options()->output.ReportSegmentation;
|
|
if (reportSegmentation > 0 && phrase.GetSize() > 0) {
|
|
const Range &sourceRange = edge.GetCurrSourceWordsRange();
|
|
const int sourceStart = sourceRange.GetStartPos();
|
|
const int sourceEnd = sourceRange.GetEndPos();
|
|
out << "|" << sourceStart << "-" << sourceEnd; // enriched "-tt"
|
|
if (reportSegmentation == 2) {
|
|
out << ",wa=";
|
|
const AlignmentInfo &ai = edge.GetCurrTargetPhrase().GetAlignTerm();
|
|
OutputAlignment(out, ai, 0, 0);
|
|
out << ",total=";
|
|
out << edge.GetScore() - edge.GetPrevHypo()->GetScore();
|
|
out << ",";
|
|
ScoreComponentCollection scoreBreakdown(edge.GetScoreBreakdown());
|
|
scoreBreakdown.MinusEquals(edge.GetPrevHypo()->GetScoreBreakdown());
|
|
bool with_labels = options()->nbest.include_feature_labels;
|
|
scoreBreakdown.OutputAllFeatureScores(out, with_labels);
|
|
}
|
|
out << "| ";
|
|
}
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputAlignment(ostream &out, const AlignmentInfo &ai,
|
|
size_t sourceOffset, size_t targetOffset) const
|
|
{
|
|
typedef std::vector< const std::pair<size_t,size_t>* > AlignVec;
|
|
AlignVec alignments = ai.GetSortedAlignments(options()->output.WA_SortOrder);
|
|
|
|
AlignVec::const_iterator it;
|
|
for (it = alignments.begin(); it != alignments.end(); ++it) {
|
|
const std::pair<size_t,size_t> &alignment = **it;
|
|
out << alignment.first + sourceOffset << "-"
|
|
<< alignment.second + targetOffset << " ";
|
|
}
|
|
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputInput(std::ostream& os, const Hypothesis* hypo) const
|
|
{
|
|
size_t len = hypo->GetInput().GetSize();
|
|
std::vector<const Phrase*> inp_phrases(len, 0);
|
|
OutputInput(inp_phrases, hypo);
|
|
for (size_t i=0; i<len; ++i)
|
|
if (inp_phrases[i]) os << *inp_phrases[i];
|
|
}
|
|
|
|
void Manager::OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo) const
|
|
{
|
|
if (hypo->GetPrevHypo()) {
|
|
OutputInput(map, hypo->GetPrevHypo());
|
|
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
|
|
}
|
|
}
|
|
|
|
std::map<size_t, const Factor*> Manager::GetPlaceholders(const Hypothesis &hypo, FactorType placeholderFactor) const
|
|
{
|
|
const InputPath &inputPath = hypo.GetTranslationOption().GetInputPath();
|
|
const Phrase &inputPhrase = inputPath.GetPhrase();
|
|
|
|
std::map<size_t, const Factor*> ret;
|
|
|
|
for (size_t sourcePos = 0; sourcePos < inputPhrase.GetSize(); ++sourcePos) {
|
|
const Factor *factor = inputPhrase.GetFactor(sourcePos, placeholderFactor);
|
|
if (factor) {
|
|
TargetPhrase const& tp = hypo.GetTranslationOption().GetTargetPhrase();
|
|
std::set<size_t> targetPos = tp.GetAlignTerm().GetAlignmentsForSource(sourcePos);
|
|
UTIL_THROW_IF2(targetPos.size() != 1,
|
|
"Placeholder should be aligned to 1, and only 1, word");
|
|
ret[*targetPos.begin()] = factor;
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
void Manager::OutputLatticeSamples(OutputCollector *collector) const
|
|
{
|
|
if (collector) {
|
|
TrellisPathList latticeSamples;
|
|
ostringstream out;
|
|
CalcLatticeSamples(options()->output.lattice_sample_size, latticeSamples);
|
|
OutputNBest(out,latticeSamples);
|
|
collector->Write(m_source.GetTranslationId(), out.str());
|
|
}
|
|
|
|
}
|
|
|
|
void Manager::OutputAlignment(OutputCollector *collector) const
|
|
{
|
|
if (collector == NULL) {
|
|
return;
|
|
}
|
|
|
|
if (!m_alignmentOut.str().empty()) {
|
|
collector->Write(m_source.GetTranslationId(), m_alignmentOut.str());
|
|
} else {
|
|
std::vector<const Hypothesis *> edges;
|
|
const Hypothesis *currentHypo = GetBestHypothesis();
|
|
while (currentHypo) {
|
|
edges.push_back(currentHypo);
|
|
currentHypo = currentHypo->GetPrevHypo();
|
|
}
|
|
ostringstream out;
|
|
size_t targetOffset = 0;
|
|
BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) {
|
|
const TargetPhrase &tp = e->GetCurrTargetPhrase();
|
|
size_t sourceOffset = e->GetCurrSourceWordsRange().GetStartPos();
|
|
OutputAlignment(out, tp.GetAlignTerm(), sourceOffset, targetOffset);
|
|
targetOffset += tp.GetSize();
|
|
}
|
|
out << std::endl; // Used by --alignment-output-file so requires endl
|
|
collector->Write(m_source.GetTranslationId(), out.str());
|
|
|
|
}
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputDetailedTranslationReport(OutputCollector *collector) const
|
|
{
|
|
if (collector) {
|
|
ostringstream out;
|
|
FixPrecision(out,PRECISION);
|
|
TranslationAnalysis::PrintTranslationAnalysis(out, GetBestHypothesis());
|
|
collector->Write(m_source.GetTranslationId(),out.str());
|
|
}
|
|
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputUnknowns(OutputCollector *collector) const
|
|
{
|
|
if (collector) {
|
|
long translationId = m_source.GetTranslationId();
|
|
const vector<const Phrase*>& unknowns = m_transOptColl->GetUnknownSources();
|
|
ostringstream out;
|
|
for (size_t i = 0; i < unknowns.size(); ++i) {
|
|
out << *(unknowns[i]);
|
|
}
|
|
out << endl;
|
|
collector->Write(translationId, out.str());
|
|
}
|
|
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputWordGraph(OutputCollector *collector) const
|
|
{
|
|
if (collector) {
|
|
long translationId = m_source.GetTranslationId();
|
|
ostringstream out;
|
|
FixPrecision(out,PRECISION);
|
|
GetWordGraph(translationId, out);
|
|
collector->Write(translationId, out.str());
|
|
}
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputSearchGraph(OutputCollector *collector) const
|
|
{
|
|
if (collector) {
|
|
long translationId = m_source.GetTranslationId();
|
|
ostringstream out;
|
|
FixPrecision(out,PRECISION);
|
|
OutputSearchGraph(translationId, out);
|
|
collector->Write(translationId, out.str());
|
|
|
|
#ifdef HAVE_PROTOBUF
|
|
const StaticData &staticData = StaticData::Instance();
|
|
if (staticData.GetOutputSearchGraphPB()) {
|
|
ostringstream sfn;
|
|
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << translationId << ".pb" << ends;
|
|
string fn = sfn.str();
|
|
VERBOSE(2, "Writing search graph to " << fn << endl);
|
|
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
|
|
SerializeSearchGraphPB(translationId, output);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
}
|
|
|
|
void Manager::OutputSearchGraphSLF() const
|
|
{
|
|
// const StaticData &staticData = StaticData::Instance();
|
|
long translationId = m_source.GetTranslationId();
|
|
|
|
// Output search graph in HTK standard lattice format (SLF)
|
|
std::string const& slf = options()->output.SearchGraphSLF;
|
|
if (slf.size()) {
|
|
util::StringStream fileName;
|
|
fileName << slf << "/" << translationId << ".slf";
|
|
ofstream *file = new ofstream;
|
|
file->open(fileName.str().c_str());
|
|
if (file->is_open() && file->good()) {
|
|
ostringstream out;
|
|
FixPrecision(out,PRECISION);
|
|
OutputSearchGraphAsSLF(translationId, out);
|
|
*file << out.str();
|
|
file -> flush();
|
|
} else {
|
|
TRACE_ERR("Cannot output HTK standard lattice for line " << translationId << " because the output file is not open or not ready for writing" << endl);
|
|
}
|
|
delete file;
|
|
}
|
|
|
|
}
|
|
|
|
void Manager::OutputLatticeMBRNBest(std::ostream& out, const vector<LatticeMBRSolution>& solutions,long translationId) const
|
|
{
|
|
for (vector<LatticeMBRSolution>::const_iterator si = solutions.begin(); si != solutions.end(); ++si) {
|
|
out << translationId;
|
|
out << " |||";
|
|
const vector<Word> mbrHypo = si->GetWords();
|
|
for (size_t i = 0 ; i < mbrHypo.size() ; i++) {
|
|
const Factor *factor = mbrHypo[i].GetFactor(options()->output.factor_order[0]);
|
|
if (i>0) out << " " << *factor;
|
|
else out << *factor;
|
|
}
|
|
out << " |||";
|
|
out << " map: " << si->GetMapScore();
|
|
out << " w: " << mbrHypo.size();
|
|
const vector<float>& ngramScores = si->GetNgramScores();
|
|
for (size_t i = 0; i < ngramScores.size(); ++i) {
|
|
out << " " << ngramScores[i];
|
|
}
|
|
out << " ||| " << si->GetScore();
|
|
|
|
out << endl;
|
|
}
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputBestHypo(const std::vector<Word>& mbrBestHypo, ostream& out) const
|
|
{
|
|
FactorType f = options()->output.factor_order[0];
|
|
for (size_t i = 0 ; i < mbrBestHypo.size() ; i++) {
|
|
const Factor *factor = mbrBestHypo[i].GetFactor(f);
|
|
UTIL_THROW_IF2(factor == NULL, "No factor " << f << " at position " << i);
|
|
if (i) out << " ";
|
|
out << *factor;
|
|
}
|
|
out << endl;
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputBestHypo(const Moses::TrellisPath &path, std::ostream &out) const
|
|
{
|
|
std::vector<const Hypothesis *> const& edges = path.GetEdges();
|
|
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
|
Hypothesis const& edge = *edges[currEdge];
|
|
OutputSurface(out, edge);
|
|
}
|
|
out << endl;
|
|
}
|
|
|
|
void
|
|
Manager::
|
|
OutputAlignment(std::ostringstream &out, const TrellisPath &path) const
|
|
{
|
|
WordAlignmentSort waso = options()->output.WA_SortOrder;
|
|
BOOST_REVERSE_FOREACH(Hypothesis const* e, path.GetEdges())
|
|
e->OutputAlignment(out, false);
|
|
// Hypothesis::OutputAlignment(out, path.GetEdges(), waso);
|
|
// Used by --alignment-output-file so requires endl
|
|
out << std::endl;
|
|
}
|
|
|
|
} // namespace
|