2008-06-11 14:52:57 +04:00
|
|
|
// $Id$
|
|
|
|
// vim:tabstop=2
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
|
|
|
#ifdef WIN32
|
|
|
|
#include <hash_set>
|
|
|
|
#else
|
|
|
|
#include <ext/hash_set>
|
|
|
|
#endif
|
|
|
|
|
2011-10-04 19:46:24 +04:00
|
|
|
#include <algorithm>
|
2008-06-11 14:52:57 +04:00
|
|
|
#include <limits>
|
|
|
|
#include <cmath>
|
|
|
|
#include "Manager.h"
|
|
|
|
#include "TypeDef.h"
|
|
|
|
#include "Util.h"
|
|
|
|
#include "TargetPhrase.h"
|
|
|
|
#include "TrellisPath.h"
|
|
|
|
#include "TrellisPathCollection.h"
|
|
|
|
#include "TranslationOption.h"
|
2010-01-28 15:12:57 +03:00
|
|
|
#include "LexicalReordering.h"
|
2008-06-11 14:52:57 +04:00
|
|
|
#include "LMList.h"
|
|
|
|
#include "TranslationOptionCollection.h"
|
|
|
|
#include "DummyScoreProducers.h"
|
2008-09-24 20:48:23 +04:00
|
|
|
#ifdef HAVE_PROTOBUF
|
|
|
|
#include "hypergraph.pb.h"
|
|
|
|
#include "rule.pb.h"
|
|
|
|
#endif
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm, const TranslationSystem* system)
|
2010-08-10 17:12:00 +04:00
|
|
|
:m_system(system)
|
2011-02-24 16:14:42 +03:00
|
|
|
,m_transOptColl(source.CreateTranslationOptionCollection(system))
|
|
|
|
,m_search(Search::CreateSearch(*this, source, searchAlgorithm, *m_transOptColl))
|
|
|
|
,m_start(clock())
|
|
|
|
,interrupted_flag(0)
|
|
|
|
,m_hypoId(0)
|
|
|
|
,m_source(source)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
m_system->InitializeBeforeSentenceProcessing(source);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
Manager::~Manager()
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
delete m_transOptColl;
|
|
|
|
delete m_search;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2012-07-19 19:41:41 +04:00
|
|
|
m_system->CleanUpAfterSentenceProcessing(m_source);
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
clock_t end = clock();
|
|
|
|
float et = (end - m_start);
|
|
|
|
et /= (float)CLOCKS_PER_SEC;
|
|
|
|
VERBOSE(1, "Translation took " << et << " seconds" << endl);
|
|
|
|
VERBOSE(1, "Finished translating" << endl);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Main decoder loop that translates a sentence by expanding
|
|
|
|
* hypotheses stack by stack, until the end of the sentence.
|
|
|
|
*/
|
|
|
|
void Manager::ProcessSentence()
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
// reset statistics
|
|
|
|
ResetSentenceStats(m_source);
|
2008-12-13 15:08:55 +03:00
|
|
|
|
|
|
|
// collect translation options for this sentence
|
2011-02-24 16:14:42 +03:00
|
|
|
m_system->InitializeBeforeSentenceProcessing(m_source);
|
|
|
|
m_transOptColl->CreateTranslationOptions();
|
2008-12-13 15:08:55 +03:00
|
|
|
|
|
|
|
// some reporting on how long this took
|
|
|
|
clock_t gotOptions = clock();
|
|
|
|
float et = (gotOptions - m_start);
|
2011-02-24 16:14:42 +03:00
|
|
|
IFVERBOSE(2) {
|
|
|
|
GetSentenceStats().AddTimeCollectOpts( gotOptions - m_start );
|
|
|
|
}
|
2008-12-13 15:08:55 +03:00
|
|
|
et /= (float)CLOCKS_PER_SEC;
|
|
|
|
VERBOSE(1, "Collecting options took " << et << " seconds" << endl);
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// search for best translation with the specified algorithm
|
|
|
|
m_search->ProcessSentence();
|
|
|
|
VERBOSE(1, "Search took " << ((clock()-m_start)/(float)CLOCKS_PER_SEC) << " seconds" << endl);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-03-07 10:57:48 +03:00
|
|
|
/**
|
|
|
|
* Print all derivations in search graph. Note: The number of derivations is exponential in the sentence length
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2010-05-19 20:42:18 +04:00
|
|
|
void Manager::PrintAllDerivations(long translationId, ostream& outputStream) const
|
2010-03-07 10:57:48 +03:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
2010-03-07 10:57:48 +03:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
|
2010-03-07 10:57:48 +03:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
if (sortedPureHypo.size() == 0)
|
|
|
|
return;
|
2010-03-07 10:57:48 +03:00
|
|
|
|
2010-03-08 18:28:40 +03:00
|
|
|
float remainingScore = 0;
|
|
|
|
vector<const TargetPhrase*> remainingPhrases;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
// add all pure paths
|
|
|
|
vector<const Hypothesis*>::const_iterator iterBestHypo;
|
|
|
|
for (iterBestHypo = sortedPureHypo.begin()
|
|
|
|
; iterBestHypo != sortedPureHypo.end()
|
|
|
|
; ++iterBestHypo) {
|
|
|
|
printThisHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
|
2010-05-19 20:42:18 +04:00
|
|
|
printDivergentHypothesis(translationId, *iterBestHypo, remainingPhrases, remainingScore, outputStream);
|
2010-03-08 18:28:40 +03:00
|
|
|
}
|
|
|
|
}
|
2010-03-07 10:57:48 +03:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
const TranslationOptionCollection* Manager::getSntTranslationOptions()
|
|
|
|
{
|
|
|
|
return m_transOptColl;
|
|
|
|
}
|
2010-03-08 18:28:40 +03:00
|
|
|
|
2010-05-19 20:42:18 +04:00
|
|
|
void Manager::printDivergentHypothesis(long translationId, const Hypothesis* hypo, const vector <const TargetPhrase*> & remainingPhrases, float remainingScore , ostream& outputStream ) const
|
2010-03-08 18:28:40 +03:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
//Backtrack from the predecessor
|
|
|
|
if (hypo->GetId() > 0) {
|
|
|
|
vector <const TargetPhrase*> followingPhrases;
|
|
|
|
followingPhrases.push_back(& (hypo->GetCurrTargetPhrase()));
|
|
|
|
///((Phrase) hypo->GetPrevHypo()->GetTargetPhrase());
|
|
|
|
followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
|
|
|
|
printDivergentHypothesis(translationId, hypo->GetPrevHypo(), followingPhrases , remainingScore + hypo->GetScore() - hypo->GetPrevHypo()->GetScore(), outputStream);
|
|
|
|
}
|
|
|
|
|
|
|
|
//Process the arcs
|
|
|
|
const ArcList *pAL = hypo->GetArcList();
|
|
|
|
if (pAL) {
|
|
|
|
const ArcList &arcList = *pAL;
|
|
|
|
// every possible Arc to replace this edge
|
|
|
|
ArcList::const_iterator iterArc;
|
|
|
|
for (iterArc = arcList.begin() ; iterArc != arcList.end() ; ++iterArc) {
|
|
|
|
const Hypothesis *loserHypo = *iterArc;
|
|
|
|
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
|
|
|
|
float arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
|
|
vector <const TargetPhrase* > followingPhrases;
|
|
|
|
followingPhrases.push_back(&(loserHypo->GetCurrTargetPhrase()));
|
|
|
|
followingPhrases.insert(followingPhrases.end()--, remainingPhrases.begin(), remainingPhrases.end());
|
|
|
|
printThisHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
|
|
|
|
printDivergentHypothesis(translationId, loserPrevHypo, followingPhrases, remainingScore + arcScore, outputStream);
|
|
|
|
}
|
|
|
|
}
|
2010-03-08 18:28:40 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2010-05-19 20:42:18 +04:00
|
|
|
void Manager::printThisHypothesis(long translationId, const Hypothesis* hypo, const vector <const TargetPhrase*> & remainingPhrases, float remainingScore, ostream& outputStream) const
|
2010-03-08 18:28:40 +03:00
|
|
|
{
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
outputStream << translationId << " ||| ";
|
|
|
|
|
2010-03-08 18:28:40 +03:00
|
|
|
//Yield of this hypothesis
|
2011-02-24 16:14:42 +03:00
|
|
|
hypo->ToStream(outputStream);
|
2010-03-08 18:28:40 +03:00
|
|
|
for (size_t p = 0; p < remainingPhrases.size(); ++p) {
|
|
|
|
const TargetPhrase * phrase = remainingPhrases[p];
|
|
|
|
size_t size = phrase->GetSize();
|
2011-02-24 16:14:42 +03:00
|
|
|
for (size_t pos = 0 ; pos < size ; pos++) {
|
|
|
|
const Factor *factor = phrase->GetFactor(pos, 0);
|
|
|
|
outputStream << *factor;
|
|
|
|
outputStream << " ";
|
2010-03-07 10:57:48 +03:00
|
|
|
}
|
2010-03-08 18:28:40 +03:00
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-05-19 20:42:18 +04:00
|
|
|
outputStream << "||| " << hypo->GetScore() + remainingScore;
|
|
|
|
outputStream << endl;
|
2010-03-07 10:57:48 +03:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-03-08 18:28:40 +03:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* After decoding, the hypotheses in the stacks and additional arcs
|
|
|
|
* form a search graph that can be mined for n-best lists.
|
|
|
|
* The heavy lifting is done in the TrellisPath and TrellisPathCollection
|
|
|
|
* this function controls this for one sentence.
|
|
|
|
*
|
|
|
|
* \param count the number of n-best translations to produce
|
|
|
|
* \param ret holds the n-best list that was calculated
|
|
|
|
*/
|
|
|
|
void Manager::CalcNBest(size_t count, TrellisPathList &ret,bool onlyDistinct) const
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
if (count <= 0)
|
|
|
|
return;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
vector<const Hypothesis*> sortedPureHypo = hypoStackColl.back()->GetSortedList();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
if (sortedPureHypo.size() == 0)
|
|
|
|
return;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
TrellisPathCollection contenders;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
set<Phrase> distinctHyps;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// add all pure paths
|
|
|
|
vector<const Hypothesis*>::const_iterator iterBestHypo;
|
|
|
|
for (iterBestHypo = sortedPureHypo.begin()
|
|
|
|
; iterBestHypo != sortedPureHypo.end()
|
|
|
|
; ++iterBestHypo) {
|
|
|
|
contenders.Add(new TrellisPath(*iterBestHypo));
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
// factor defines stopping point for distinct n-best list if too many candidates identical
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
2008-06-11 14:52:57 +04:00
|
|
|
if (nBestFactor < 1) nBestFactor = 1000; // 0 = unlimited
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// MAIN loop
|
|
|
|
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++) {
|
|
|
|
// get next best from list of contenders
|
|
|
|
TrellisPath *path = contenders.pop();
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(path);
|
2011-02-24 16:14:42 +03:00
|
|
|
// create deviations from current best
|
|
|
|
path->CreateDeviantPaths(contenders);
|
|
|
|
if(onlyDistinct) {
|
|
|
|
Phrase tgtPhrase = path->GetSurfacePhrase();
|
|
|
|
if (distinctHyps.insert(tgtPhrase).second) {
|
2008-06-11 14:52:57 +04:00
|
|
|
ret.Add(path);
|
2010-03-31 17:16:39 +04:00
|
|
|
} else {
|
|
|
|
delete path;
|
|
|
|
path = NULL;
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
} else {
|
|
|
|
ret.Add(path);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
|
|
|
|
if(onlyDistinct) {
|
|
|
|
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
|
|
|
if (nBestFactor > 0)
|
|
|
|
contenders.Prune(count * nBestFactor);
|
|
|
|
} else {
|
|
|
|
contenders.Prune(count);
|
|
|
|
}
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2011-10-04 19:46:24 +04:00
|
|
|
struct SGNReverseCompare {
|
|
|
|
bool operator() (const SearchGraphNode& s1, const SearchGraphNode& s2) const {
|
|
|
|
return s1.hypo->GetId() > s2.hypo->GetId();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Implements lattice sampling, as in Chatterjee & Cancedda, emnlp 2010
|
|
|
|
**/
|
|
|
|
void Manager::CalcLatticeSamples(size_t count, TrellisPathList &ret) const {
|
|
|
|
|
|
|
|
vector<SearchGraphNode> searchGraph;
|
|
|
|
GetSearchGraph(searchGraph);
|
|
|
|
|
|
|
|
//Calculation of the sigmas of each hypothesis and edge. In C&C notation this is
|
|
|
|
//the "log of the cumulative unnormalized probability of all the paths in the
|
|
|
|
// lattice for the hypothesis to a final node"
|
|
|
|
typedef pair<int, int> Edge;
|
|
|
|
map<const Hypothesis*, float> sigmas;
|
|
|
|
map<Edge, float> edgeScores;
|
|
|
|
map<const Hypothesis*, set<const Hypothesis*> > outgoingHyps;
|
|
|
|
map<int,const Hypothesis*> idToHyp;
|
|
|
|
map<int,float> fscores;
|
|
|
|
|
|
|
|
//Iterating through the hypos in reverse order of id gives a reverse
|
|
|
|
//topological order. We rely on the fact that hypo ids are given out
|
|
|
|
//sequentially, as the search proceeds.
|
|
|
|
//NB: Could just sort by stack.
|
|
|
|
sort(searchGraph.begin(), searchGraph.end(), SGNReverseCompare());
|
|
|
|
|
|
|
|
//first task is to fill in the outgoing hypos and edge scores.
|
|
|
|
for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
|
|
|
|
i != searchGraph.end(); ++i) {
|
|
|
|
const Hypothesis* hypo = i->hypo;
|
|
|
|
idToHyp[hypo->GetId()] = hypo;
|
|
|
|
fscores[hypo->GetId()] = i->fscore;
|
|
|
|
if (hypo->GetId()) {
|
|
|
|
//back to current
|
|
|
|
const Hypothesis* prevHypo = i->hypo->GetPrevHypo();
|
|
|
|
outgoingHyps[prevHypo].insert(hypo);
|
|
|
|
edgeScores[Edge(prevHypo->GetId(),hypo->GetId())] =
|
|
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
|
|
}
|
|
|
|
//forward from current
|
|
|
|
if (i->forward >= 0) {
|
|
|
|
map<int,const Hypothesis*>::const_iterator idToHypIter = idToHyp.find(i->forward);
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(idToHypIter != idToHyp.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
const Hypothesis* nextHypo = idToHypIter->second;
|
|
|
|
outgoingHyps[hypo].insert(nextHypo);
|
|
|
|
map<int,float>::const_iterator fscoreIter = fscores.find(nextHypo->GetId());
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(fscoreIter != fscores.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
edgeScores[Edge(hypo->GetId(),nextHypo->GetId())] =
|
|
|
|
i->fscore - fscoreIter->second;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
//then run through again to calculate sigmas
|
|
|
|
for (vector<SearchGraphNode>::const_iterator i = searchGraph.begin();
|
|
|
|
i != searchGraph.end(); ++i) {
|
|
|
|
|
|
|
|
if (i->forward == -1) {
|
|
|
|
sigmas[i->hypo] = 0;
|
|
|
|
} else {
|
|
|
|
map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
|
|
|
|
outgoingHyps.find(i->hypo);
|
|
|
|
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(outIter != outgoingHyps.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
float sigma = 0;
|
|
|
|
for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
|
|
|
|
j != outIter->second.end(); ++j) {
|
|
|
|
map<const Hypothesis*, float>::const_iterator succIter = sigmas.find(*j);
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(succIter != sigmas.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
map<Edge,float>::const_iterator edgeScoreIter =
|
|
|
|
edgeScores.find(Edge(i->hypo->GetId(),(*j)->GetId()));
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(edgeScoreIter != edgeScores.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
float term = edgeScoreIter->second + succIter->second; // Add sigma(*j)
|
|
|
|
if (sigma == 0) {
|
|
|
|
sigma = term;
|
|
|
|
} else {
|
|
|
|
sigma = log_sum(sigma,term);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
sigmas[i->hypo] = sigma;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//The actual sampling!
|
|
|
|
const Hypothesis* startHypo = searchGraph.back().hypo;
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(startHypo->GetId() == 0);
|
2011-10-04 19:46:24 +04:00
|
|
|
for (size_t i = 0; i < count; ++i) {
|
|
|
|
vector<const Hypothesis*> path;
|
|
|
|
path.push_back(startHypo);
|
|
|
|
while(1) {
|
|
|
|
map<const Hypothesis*, set<const Hypothesis*> >::const_iterator outIter =
|
|
|
|
outgoingHyps.find(path.back());
|
|
|
|
if (outIter == outgoingHyps.end() || !outIter->second.size()) {
|
|
|
|
//end of the path
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
//score the possibles
|
|
|
|
vector<const Hypothesis*> candidates;
|
|
|
|
vector<float> candidateScores;
|
|
|
|
float scoreTotal = 0;
|
|
|
|
for (set<const Hypothesis*>::const_iterator j = outIter->second.begin();
|
|
|
|
j != outIter->second.end(); ++j) {
|
|
|
|
candidates.push_back(*j);
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(sigmas.find(*j) != sigmas.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
Edge edge(path.back()->GetId(),(*j)->GetId());
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(edgeScores.find(edge) != edgeScores.end());
|
2011-10-04 19:46:24 +04:00
|
|
|
candidateScores.push_back(sigmas[*j] + edgeScores[edge]);
|
|
|
|
if (scoreTotal == 0) {
|
|
|
|
scoreTotal = candidateScores.back();
|
|
|
|
} else {
|
|
|
|
scoreTotal = log_sum(candidateScores.back(), scoreTotal);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//normalise
|
|
|
|
transform(candidateScores.begin(), candidateScores.end(), candidateScores.begin(), bind2nd(minus<float>(),scoreTotal));
|
|
|
|
//copy(candidateScores.begin(),candidateScores.end(),ostream_iterator<float>(cerr," "));
|
|
|
|
//cerr << endl;
|
|
|
|
|
|
|
|
//draw the sample
|
|
|
|
float random = log((float)rand()/RAND_MAX);
|
|
|
|
size_t position = 1;
|
|
|
|
float sum = candidateScores[0];
|
|
|
|
for (; position < candidateScores.size() && sum < random; ++position) {
|
|
|
|
sum = log_sum(sum,candidateScores[position]);
|
|
|
|
}
|
|
|
|
//cerr << "Random: " << random << " Chose " << position-1 << endl;
|
|
|
|
const Hypothesis* chosen = candidates[position-1];
|
|
|
|
path.push_back(chosen);
|
|
|
|
}
|
|
|
|
//cerr << "Path: " << endl;
|
|
|
|
//for (size_t j = 0; j < path.size(); ++j) {
|
|
|
|
// cerr << path[j]->GetId() << " " << path[j]->GetScoreBreakdown() << endl;
|
|
|
|
//}
|
|
|
|
//cerr << endl;
|
|
|
|
|
|
|
|
//Convert the hypos to TrellisPath
|
|
|
|
ret.Add(new TrellisPath(path));
|
|
|
|
//cerr << ret.at(ret.GetSize()-1).GetScoreBreakdown() << endl;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void Manager::CalcDecoderStatistics() const
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
|
|
|
const Hypothesis *hypo = GetBestHypothesis();
|
2011-02-24 16:14:42 +03:00
|
|
|
if (hypo != NULL) {
|
|
|
|
GetSentenceStats().CalcFinalStats(*hypo);
|
2008-06-11 14:52:57 +04:00
|
|
|
IFVERBOSE(2) {
|
2011-02-24 16:14:42 +03:00
|
|
|
if (hypo != NULL) {
|
|
|
|
string buff;
|
|
|
|
string buff2;
|
|
|
|
TRACE_ERR( "Source and Target Units:"
|
|
|
|
<< hypo->GetInput());
|
|
|
|
buff2.insert(0,"] ");
|
|
|
|
buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
|
|
|
|
buff2.insert(0,":");
|
|
|
|
buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
|
|
|
|
buff2.insert(0,"[");
|
|
|
|
|
|
|
|
hypo = hypo->GetPrevHypo();
|
|
|
|
while (hypo != NULL) {
|
|
|
|
//dont print out the empty final hypo
|
|
|
|
buff.insert(0,buff2);
|
|
|
|
buff2.clear();
|
|
|
|
buff2.insert(0,"] ");
|
|
|
|
buff2.insert(0,(hypo->GetCurrTargetPhrase()).ToString());
|
|
|
|
buff2.insert(0,":");
|
|
|
|
buff2.insert(0,(hypo->GetCurrSourceWordsRange()).ToString());
|
|
|
|
buff2.insert(0,"[");
|
|
|
|
hypo = hypo->GetPrevHypo();
|
|
|
|
}
|
|
|
|
TRACE_ERR( buff << endl);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-08-10 17:12:00 +04:00
|
|
|
void OutputWordGraph(std::ostream &outputWordGraphStream, const Hypothesis *hypo, size_t &linkId, const TranslationSystem* system)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
|
|
|
|
|
|
|
|
|
|
outputWordGraphStream << "J=" << linkId++
|
|
|
|
<< "\tS=" << prevHypo->GetId()
|
|
|
|
<< "\tE=" << hypo->GetId()
|
|
|
|
<< "\ta=";
|
|
|
|
|
|
|
|
// phrase table scores
|
|
|
|
const std::vector<PhraseDictionaryFeature*> &phraseTables = system->GetPhraseDictionaries();
|
|
|
|
std::vector<PhraseDictionaryFeature*>::const_iterator iterPhraseTable;
|
|
|
|
for (iterPhraseTable = phraseTables.begin() ; iterPhraseTable != phraseTables.end() ; ++iterPhraseTable) {
|
|
|
|
const PhraseDictionaryFeature *phraseTable = *iterPhraseTable;
|
|
|
|
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(phraseTable);
|
|
|
|
|
|
|
|
outputWordGraphStream << scores[0];
|
|
|
|
vector<float>::const_iterator iterScore;
|
|
|
|
for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
|
|
|
|
outputWordGraphStream << ", " << *iterScore;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// language model scores
|
|
|
|
outputWordGraphStream << "\tl=";
|
|
|
|
const LMList &lmList = system->GetLanguageModels();
|
|
|
|
LMList::const_iterator iterLM;
|
|
|
|
for (iterLM = lmList.begin() ; iterLM != lmList.end() ; ++iterLM) {
|
|
|
|
LanguageModel *lm = *iterLM;
|
|
|
|
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lm);
|
|
|
|
|
|
|
|
outputWordGraphStream << scores[0];
|
|
|
|
vector<float>::const_iterator iterScore;
|
|
|
|
for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
|
|
|
|
outputWordGraphStream << ", " << *iterScore;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// re-ordering
|
|
|
|
outputWordGraphStream << "\tr=";
|
|
|
|
|
|
|
|
outputWordGraphStream << hypo->GetScoreBreakdown().GetScoreForProducer(system->GetDistortionProducer());
|
|
|
|
|
|
|
|
// lexicalised re-ordering
|
|
|
|
const std::vector<LexicalReordering*> &lexOrderings = system->GetReorderModels();
|
|
|
|
std::vector<LexicalReordering*>::const_iterator iterLexOrdering;
|
|
|
|
for (iterLexOrdering = lexOrderings.begin() ; iterLexOrdering != lexOrderings.end() ; ++iterLexOrdering) {
|
|
|
|
LexicalReordering *lexicalReordering = *iterLexOrdering;
|
|
|
|
vector<float> scores = hypo->GetScoreBreakdown().GetScoresForProducer(lexicalReordering);
|
|
|
|
|
|
|
|
outputWordGraphStream << scores[0];
|
|
|
|
vector<float>::const_iterator iterScore;
|
|
|
|
for (iterScore = ++scores.begin() ; iterScore != scores.end() ; ++iterScore) {
|
|
|
|
outputWordGraphStream << ", " << *iterScore;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// words !!
|
2011-05-16 17:41:15 +04:00
|
|
|
// outputWordGraphStream << "\tw=" << hypo->GetCurrTargetPhrase();
|
|
|
|
|
|
|
|
// output both source and target phrases in the word graph
|
|
|
|
outputWordGraphStream << "\tw=" << hypo->GetSourcePhraseStringRep() << "|" << hypo->GetCurrTargetPhrase();
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
outputWordGraphStream << endl;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void Manager::GetWordGraph(long translationId, std::ostream &outputWordGraphStream) const
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
string fileName = staticData.GetParam("output-word-graph")[0];
|
|
|
|
bool outputNBest = Scan<bool>(staticData.GetParam("output-word-graph")[1]);
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
|
|
|
|
outputWordGraphStream << "VERSION=1.0" << endl
|
|
|
|
<< "UTTERANCE=" << translationId << endl;
|
|
|
|
|
|
|
|
size_t linkId = 0;
|
|
|
|
size_t stackNo = 1;
|
|
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
|
|
|
for (iterStack = ++hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
|
|
|
|
cerr << endl << stackNo++ << endl;
|
|
|
|
const HypothesisStack &stack = **iterStack;
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
|
|
|
const Hypothesis *hypo = *iterHypo;
|
|
|
|
OutputWordGraph(outputWordGraphStream, hypo, linkId, m_system);
|
|
|
|
|
|
|
|
if (outputNBest) {
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
|
|
if (arcList != NULL) {
|
|
|
|
ArcList::const_iterator iterArcList;
|
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
|
|
OutputWordGraph(outputWordGraphStream, loserHypo, linkId,m_system);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} //if (outputNBest)
|
|
|
|
} //for (iterHypo
|
|
|
|
} // for (iterStack
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
void Manager::GetSearchGraph(vector<SearchGraphNode>& searchGraph) const
|
|
|
|
{
|
2010-05-05 20:53:41 +04:00
|
|
|
std::map < int, bool > connected;
|
|
|
|
std::map < int, int > forward;
|
|
|
|
std::map < int, double > forwardScore;
|
|
|
|
|
|
|
|
// *** find connected hypotheses ***
|
|
|
|
std::vector< const Hypothesis *> connectedList;
|
|
|
|
GetConnectedGraph(&connected, &connectedList);
|
|
|
|
|
|
|
|
// ** compute best forward path for each hypothesis *** //
|
|
|
|
|
|
|
|
// forward cost of hypotheses on final stack is 0
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
2010-05-05 20:53:41 +04:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
|
|
|
forwardScore[ hypo->GetId() ] = 0.0f;
|
|
|
|
forward[ hypo->GetId() ] = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
// compete for best forward score of previous hypothesis
|
|
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
|
2010-05-05 20:53:41 +04:00
|
|
|
const HypothesisStack &stack = **iterStack;
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
2010-05-05 20:53:41 +04:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
|
|
// make a play for previous hypothesis
|
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
|
|
if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
|
|
|
|
|| forwardScore.find( prevHypo->GetId() )->second < fscore) {
|
|
|
|
forwardScore[ prevHypo->GetId() ] = fscore;
|
|
|
|
forward[ prevHypo->GetId() ] = hypo->GetId();
|
|
|
|
}
|
|
|
|
// all arcs also make a play
|
2010-05-05 20:53:41 +04:00
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
2011-02-24 16:14:42 +03:00
|
|
|
if (arcList != NULL) {
|
|
|
|
ArcList::const_iterator iterArcList;
|
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
|
|
// make a play
|
|
|
|
const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
|
|
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
|
|
loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
|
|
if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
|
|
|
|
|| forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
|
|
|
|
forwardScore[ loserPrevHypo->GetId() ] = fscore;
|
|
|
|
forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
|
|
|
|
}
|
|
|
|
} // end for arc list
|
|
|
|
} // end if arc list empty
|
2010-05-05 20:53:41 +04:00
|
|
|
} // end if hypo connected
|
|
|
|
} // end for hypo
|
|
|
|
} // end for stack
|
|
|
|
|
|
|
|
// *** output all connected hypotheses *** //
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-05-05 20:53:41 +04:00
|
|
|
connected[ 0 ] = true;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterStack = hypoStackColl.begin() ; iterStack != hypoStackColl.end() ; ++iterStack) {
|
2010-05-05 20:53:41 +04:00
|
|
|
const HypothesisStack &stack = **iterStack;
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
2010-05-05 20:53:41 +04:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
2010-05-05 20:53:41 +04:00
|
|
|
searchGraph.push_back(SearchGraphNode(hypo,NULL,forward[hypo->GetId()],
|
2011-02-24 16:14:42 +03:00
|
|
|
forwardScore[hypo->GetId()]));
|
|
|
|
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
|
|
if (arcList != NULL) {
|
|
|
|
ArcList::const_iterator iterArcList;
|
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
|
|
searchGraph.push_back(SearchGraphNode(loserHypo,hypo,
|
|
|
|
forward[hypo->GetId()], forwardScore[hypo->GetId()]));
|
|
|
|
}
|
|
|
|
} // end if arcList empty
|
2010-05-05 20:53:41 +04:00
|
|
|
} // end if connected
|
|
|
|
} // end for iterHypo
|
2011-02-24 16:14:42 +03:00
|
|
|
} // end for iterStack
|
2010-05-05 20:53:41 +04:00
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
void OutputSearchNode(long translationId, std::ostream &outputSearchGraphStream,
|
2011-02-24 16:14:42 +03:00
|
|
|
const SearchGraphNode& searchNode)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
const vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
|
|
|
bool extendedFormat = StaticData::Instance().GetOutputSearchGraphExtended();
|
|
|
|
outputSearchGraphStream << translationId;
|
|
|
|
|
|
|
|
// special case: initial hypothesis
|
|
|
|
if ( searchNode.hypo->GetId() == 0 ) {
|
|
|
|
outputSearchGraphStream << " hyp=0 stack=0";
|
|
|
|
if (!extendedFormat) {
|
|
|
|
outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore;
|
|
|
|
}
|
|
|
|
outputSearchGraphStream << endl;
|
|
|
|
return;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2010-05-05 20:53:41 +04:00
|
|
|
const Hypothesis *prevHypo = searchNode.hypo->GetPrevHypo();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// output in traditional format
|
|
|
|
if (!extendedFormat) {
|
|
|
|
outputSearchGraphStream << " hyp=" << searchNode.hypo->GetId()
|
|
|
|
<< " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
|
|
|
|
<< " back=" << prevHypo->GetId()
|
|
|
|
<< " score=" << searchNode.hypo->GetScore()
|
|
|
|
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
|
|
|
|
|
|
|
if (searchNode.recombinationHypo != NULL)
|
|
|
|
outputSearchGraphStream << " recombined=" << searchNode.recombinationHypo->GetId();
|
|
|
|
|
|
|
|
outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
|
|
|
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
|
|
|
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos()
|
|
|
|
<< " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder)
|
|
|
|
<< endl;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// output in extended format
|
|
|
|
if (searchNode.recombinationHypo != NULL)
|
|
|
|
outputSearchGraphStream << " hyp=" << searchNode.recombinationHypo->GetId();
|
|
|
|
else
|
|
|
|
outputSearchGraphStream << " hyp=" << searchNode.hypo->GetId();
|
|
|
|
|
2011-10-27 15:26:08 +04:00
|
|
|
outputSearchGraphStream << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered()
|
|
|
|
<< " back=" << prevHypo->GetId()
|
|
|
|
<< " score=" << searchNode.hypo->GetScore()
|
|
|
|
<< " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore());
|
|
|
|
|
|
|
|
if (searchNode.recombinationHypo != NULL)
|
|
|
|
outputSearchGraphStream << " recombined=" << searchNode.recombinationHypo->GetId();
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2011-10-27 15:26:08 +04:00
|
|
|
outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore
|
|
|
|
<< " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos()
|
|
|
|
<< "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos();
|
|
|
|
|
|
|
|
// Modified so that -osgx is a superset of -osg (GST Oct 2011)
|
2011-02-24 16:14:42 +03:00
|
|
|
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
|
|
|
|
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
|
2011-10-27 15:26:08 +04:00
|
|
|
outputSearchGraphStream << " scores=[ ";
|
2011-02-24 16:14:42 +03:00
|
|
|
StaticData::Instance().GetScoreIndexManager().PrintLabeledScores( outputSearchGraphStream, scoreBreakdown );
|
|
|
|
outputSearchGraphStream << " ]";
|
2011-10-27 15:26:08 +04:00
|
|
|
|
|
|
|
|
2011-05-16 17:41:15 +04:00
|
|
|
outputSearchGraphStream << " out=" << searchNode.hypo->GetSourcePhraseStringRep() << "|" <<
|
|
|
|
searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl;
|
|
|
|
// outputSearchGraphStream << " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2008-09-23 23:39:56 +04:00
|
|
|
void Manager::GetConnectedGraph(
|
2011-02-24 16:14:42 +03:00
|
|
|
std::map< int, bool >* pConnected,
|
|
|
|
std::vector< const Hypothesis* >* pConnectedList) const
|
|
|
|
{
|
2008-09-23 23:39:56 +04:00
|
|
|
std::map < int, bool >& connected = *pConnected;
|
|
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
// start with the ones in the final stack
|
2008-09-23 23:39:56 +04:00
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
2008-06-11 14:52:57 +04:00
|
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
2008-06-11 14:52:57 +04:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
|
|
|
connected[ hypo->GetId() ] = true;
|
|
|
|
connectedList.push_back( hypo );
|
|
|
|
}
|
|
|
|
// move back from known connected hypotheses
|
|
|
|
for(size_t i=0; i<connectedList.size(); i++) {
|
|
|
|
const Hypothesis *hypo = connectedList[i];
|
|
|
|
|
|
|
|
// add back pointer
|
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
2011-05-16 17:41:15 +04:00
|
|
|
if (prevHypo && prevHypo->GetId() > 0 // don't add empty hypothesis
|
2011-02-24 16:14:42 +03:00
|
|
|
&& connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
|
2008-06-11 14:52:57 +04:00
|
|
|
connected[ prevHypo->GetId() ] = true;
|
|
|
|
connectedList.push_back( prevHypo );
|
|
|
|
}
|
|
|
|
|
|
|
|
// add arcs
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
2011-02-24 16:14:42 +03:00
|
|
|
if (arcList != NULL) {
|
2008-06-11 14:52:57 +04:00
|
|
|
ArcList::const_iterator iterArcList;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
|
|
if (connected.find( loserHypo->GetId() ) == connected.end()) { // don't add already added
|
|
|
|
connected[ loserHypo->GetId() ] = true;
|
|
|
|
connectedList.push_back( loserHypo );
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2008-09-23 23:39:56 +04:00
|
|
|
}
|
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
void Manager::GetWinnerConnectedGraph(
|
2011-02-24 16:14:42 +03:00
|
|
|
std::map< int, bool >* pConnected,
|
|
|
|
std::vector< const Hypothesis* >* pConnectedList) const
|
|
|
|
{
|
2010-02-03 13:23:32 +03:00
|
|
|
std::map < int, bool >& connected = *pConnected;
|
|
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
// start with the ones in the final stack
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
2010-02-03 13:23:32 +03:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
|
|
|
connected[ hypo->GetId() ] = true;
|
|
|
|
connectedList.push_back( hypo );
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
// move back from known connected hypotheses
|
|
|
|
for(size_t i=0; i<connectedList.size(); i++) {
|
|
|
|
const Hypothesis *hypo = connectedList[i];
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
// add back pointer
|
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
|
|
if (prevHypo->GetId() > 0 // don't add empty hypothesis
|
2011-02-24 16:14:42 +03:00
|
|
|
&& connected.find( prevHypo->GetId() ) == connected.end()) { // don't add already added
|
2010-02-03 13:23:32 +03:00
|
|
|
connected[ prevHypo->GetId() ] = true;
|
|
|
|
connectedList.push_back( prevHypo );
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
// add arcs
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
2011-02-24 16:14:42 +03:00
|
|
|
if (arcList != NULL) {
|
2010-02-03 13:23:32 +03:00
|
|
|
ArcList::const_iterator iterArcList;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
2010-02-03 13:23:32 +03:00
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
2011-02-24 16:14:42 +03:00
|
|
|
if (connected.find( loserHypo->GetPrevHypo()->GetId() ) == connected.end() && loserHypo->GetPrevHypo()->GetId() > 0) { // don't add already added & don't add hyp 0
|
2010-02-03 13:23:32 +03:00
|
|
|
connected[ loserHypo->GetPrevHypo()->GetId() ] = true;
|
|
|
|
connectedList.push_back( loserHypo->GetPrevHypo() );
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
|
2008-09-24 20:48:23 +04:00
|
|
|
#ifdef HAVE_PROTOBUF
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
void SerializeEdgeInfo(const Hypothesis* hypo, hgmert::Hypergraph_Edge* edge)
|
|
|
|
{
|
|
|
|
hgmert::Rule* rule = edge->mutable_rule();
|
|
|
|
hypo->GetCurrTargetPhrase().WriteToRulePB(rule);
|
|
|
|
const Hypothesis* prev = hypo->GetPrevHypo();
|
|
|
|
// if the feature values are empty, they default to 0
|
|
|
|
if (!prev) return;
|
|
|
|
// score breakdown is an aggregate (forward) quantity, but the exported
|
|
|
|
// graph object just wants the feature values on the edges
|
|
|
|
const ScoreComponentCollection& scores = hypo->GetScoreBreakdown();
|
|
|
|
const ScoreComponentCollection& pscores = prev->GetScoreBreakdown();
|
|
|
|
for (unsigned int i = 0; i < scores.size(); ++i)
|
|
|
|
edge->add_feature_values((scores[i] - pscores[i]) * -1.0);
|
2008-09-24 20:48:23 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
hgmert::Hypergraph_Node* GetHGNode(
|
2011-02-24 16:14:42 +03:00
|
|
|
const Hypothesis* hypo,
|
|
|
|
std::map< int, int>* i2hgnode,
|
|
|
|
hgmert::Hypergraph* hg,
|
|
|
|
int* hgNodeIdx)
|
|
|
|
{
|
|
|
|
hgmert::Hypergraph_Node* hgnode;
|
2008-09-24 20:48:23 +04:00
|
|
|
std::map < int, int >::iterator idxi = i2hgnode->find(hypo->GetId());
|
2011-02-24 16:14:42 +03:00
|
|
|
if (idxi == i2hgnode->end()) {
|
|
|
|
*hgNodeIdx = ((*i2hgnode)[hypo->GetId()] = hg->nodes_size());
|
|
|
|
hgnode = hg->add_nodes();
|
|
|
|
} else {
|
|
|
|
*hgNodeIdx = idxi->second;
|
|
|
|
hgnode = hg->mutable_nodes(*hgNodeIdx);
|
|
|
|
}
|
|
|
|
return hgnode;
|
2008-09-24 20:48:23 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void Manager::SerializeSearchGraphPB(
|
2011-02-24 16:14:42 +03:00
|
|
|
long translationId,
|
|
|
|
std::ostream& outputStream) const
|
|
|
|
{
|
|
|
|
using namespace hgmert;
|
2008-09-24 20:48:23 +04:00
|
|
|
std::map < int, bool > connected;
|
|
|
|
std::map < int, int > i2hgnode;
|
|
|
|
std::vector< const Hypothesis *> connectedList;
|
2011-02-24 16:14:42 +03:00
|
|
|
GetConnectedGraph(&connected, &connectedList);
|
2008-09-24 20:48:23 +04:00
|
|
|
connected[ 0 ] = true;
|
|
|
|
Hypergraph hg;
|
2011-02-24 16:14:42 +03:00
|
|
|
hg.set_is_sorted(false);
|
|
|
|
int num_feats = (*m_search->GetHypothesisStacks().back()->begin())->GetScoreBreakdown().size();
|
|
|
|
hg.set_num_features(num_feats);
|
|
|
|
StaticData::Instance().GetScoreIndexManager().SerializeFeatureNamesToPB(&hg);
|
|
|
|
Hypergraph_Node* goal = hg.add_nodes(); // idx=0 goal node must have idx 0
|
|
|
|
Hypergraph_Node* source = hg.add_nodes(); // idx=1
|
|
|
|
i2hgnode[-1] = 1; // source node
|
2008-09-24 20:48:23 +04:00
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
|
|
for (std::vector < HypothesisStack* >::const_iterator iterStack = hypoStackColl.begin();
|
2011-02-24 16:14:42 +03:00
|
|
|
iterStack != hypoStackColl.end() ; ++iterStack) {
|
2008-09-24 20:48:23 +04:00
|
|
|
const HypothesisStack &stack = **iterStack;
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
2008-09-24 20:48:23 +04:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
bool is_goal = hypo->GetWordsBitmap().IsComplete();
|
|
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
|
|
int headNodeIdx;
|
|
|
|
Hypergraph_Node* headNode = GetHGNode(hypo, &i2hgnode, &hg, &headNodeIdx);
|
|
|
|
if (is_goal) {
|
|
|
|
Hypergraph_Edge* ge = hg.add_edges();
|
|
|
|
ge->set_head_node(0); // goal
|
|
|
|
ge->add_tail_nodes(headNodeIdx);
|
|
|
|
ge->mutable_rule()->add_trg_words("[X,1]");
|
|
|
|
}
|
|
|
|
Hypergraph_Edge* edge = hg.add_edges();
|
|
|
|
SerializeEdgeInfo(hypo, edge);
|
|
|
|
edge->set_head_node(headNodeIdx);
|
|
|
|
const Hypothesis* prev = hypo->GetPrevHypo();
|
|
|
|
int tailNodeIdx = 1; // source
|
|
|
|
if (prev)
|
|
|
|
tailNodeIdx = i2hgnode.find(prev->GetId())->second;
|
|
|
|
edge->add_tail_nodes(tailNodeIdx);
|
2008-09-24 20:48:23 +04:00
|
|
|
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
2011-02-24 16:14:42 +03:00
|
|
|
if (arcList != NULL) {
|
2008-09-24 20:48:23 +04:00
|
|
|
ArcList::const_iterator iterArcList;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
2008-09-24 20:48:23 +04:00
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(connected[loserHypo->GetId()]);
|
2011-02-24 16:14:42 +03:00
|
|
|
Hypergraph_Edge* edge = hg.add_edges();
|
|
|
|
SerializeEdgeInfo(loserHypo, edge);
|
|
|
|
edge->set_head_node(headNodeIdx);
|
|
|
|
tailNodeIdx = i2hgnode.find(loserHypo->GetPrevHypo()->GetId())->second;
|
|
|
|
edge->add_tail_nodes(tailNodeIdx);
|
2008-09-24 20:48:23 +04:00
|
|
|
}
|
|
|
|
} // end if arcList empty
|
|
|
|
} // end if connected
|
|
|
|
} // end for iterHypo
|
|
|
|
} // end for iterStack
|
2011-02-24 16:14:42 +03:00
|
|
|
hg.SerializeToOstream(&outputStream);
|
2008-09-24 20:48:23 +04:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-05-05 20:53:41 +04:00
|
|
|
void Manager::OutputSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
|
2008-09-23 23:39:56 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
vector<SearchGraphNode> searchGraph;
|
|
|
|
GetSearchGraph(searchGraph);
|
|
|
|
for (size_t i = 0; i < searchGraph.size(); ++i) {
|
|
|
|
OutputSearchNode(translationId,outputSearchGraphStream,searchGraph[i]);
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
void Manager::GetForwardBackwardSearchGraph(std::map< int, bool >* pConnected,
|
|
|
|
std::vector< const Hypothesis* >* pConnectedList, std::map < const Hypothesis*, set< const Hypothesis* > >* pOutgoingHyps, vector< float>* pFwdBwdScores) const
|
|
|
|
{
|
|
|
|
std::map < int, bool > &connected = *pConnected;
|
|
|
|
std::vector< const Hypothesis *>& connectedList = *pConnectedList;
|
|
|
|
std::map < int, int > forward;
|
|
|
|
std::map < int, double > forwardScore;
|
|
|
|
|
|
|
|
std::map < const Hypothesis*, set <const Hypothesis*> > & outgoingHyps = *pOutgoingHyps;
|
|
|
|
vector< float> & estimatedScores = *pFwdBwdScores;
|
|
|
|
|
|
|
|
// *** find connected hypotheses ***
|
|
|
|
GetWinnerConnectedGraph(&connected, &connectedList);
|
|
|
|
|
|
|
|
// ** compute best forward path for each hypothesis *** //
|
|
|
|
|
|
|
|
// forward cost of hypotheses on final stack is 0
|
|
|
|
const std::vector < HypothesisStack* > &hypoStackColl = m_search->GetHypothesisStacks();
|
|
|
|
const HypothesisStack &finalStack = *hypoStackColl.back();
|
|
|
|
HypothesisStack::const_iterator iterHypo;
|
|
|
|
for (iterHypo = finalStack.begin() ; iterHypo != finalStack.end() ; ++iterHypo) {
|
|
|
|
const Hypothesis *hypo = *iterHypo;
|
|
|
|
forwardScore[ hypo->GetId() ] = 0.0f;
|
|
|
|
forward[ hypo->GetId() ] = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
// compete for best forward score of previous hypothesis
|
|
|
|
std::vector < HypothesisStack* >::const_iterator iterStack;
|
|
|
|
for (iterStack = --hypoStackColl.end() ; iterStack != hypoStackColl.begin() ; --iterStack) {
|
|
|
|
const HypothesisStack &stack = **iterStack;
|
2010-02-03 13:23:32 +03:00
|
|
|
HypothesisStack::const_iterator iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
for (iterHypo = stack.begin() ; iterHypo != stack.end() ; ++iterHypo) {
|
2010-02-03 13:23:32 +03:00
|
|
|
const Hypothesis *hypo = *iterHypo;
|
2011-02-24 16:14:42 +03:00
|
|
|
if (connected.find( hypo->GetId() ) != connected.end()) {
|
|
|
|
// make a play for previous hypothesis
|
|
|
|
const Hypothesis *prevHypo = hypo->GetPrevHypo();
|
|
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
|
|
hypo->GetScore() - prevHypo->GetScore();
|
|
|
|
if (forwardScore.find( prevHypo->GetId() ) == forwardScore.end()
|
|
|
|
|| forwardScore.find( prevHypo->GetId() )->second < fscore) {
|
|
|
|
forwardScore[ prevHypo->GetId() ] = fscore;
|
|
|
|
forward[ prevHypo->GetId() ] = hypo->GetId();
|
|
|
|
}
|
|
|
|
//store outgoing info
|
|
|
|
outgoingHyps[prevHypo].insert(hypo);
|
|
|
|
|
|
|
|
// all arcs also make a play
|
|
|
|
const ArcList *arcList = hypo->GetArcList();
|
|
|
|
if (arcList != NULL) {
|
|
|
|
ArcList::const_iterator iterArcList;
|
|
|
|
for (iterArcList = arcList->begin() ; iterArcList != arcList->end() ; ++iterArcList) {
|
|
|
|
const Hypothesis *loserHypo = *iterArcList;
|
|
|
|
// make a play
|
|
|
|
const Hypothesis *loserPrevHypo = loserHypo->GetPrevHypo();
|
|
|
|
double fscore = forwardScore[ hypo->GetId() ] +
|
|
|
|
loserHypo->GetScore() - loserPrevHypo->GetScore();
|
|
|
|
if (forwardScore.find( loserPrevHypo->GetId() ) == forwardScore.end()
|
|
|
|
|| forwardScore.find( loserPrevHypo->GetId() )->second < fscore) {
|
|
|
|
forwardScore[ loserPrevHypo->GetId() ] = fscore;
|
|
|
|
forward[ loserPrevHypo->GetId() ] = loserHypo->GetId();
|
|
|
|
}
|
|
|
|
//store outgoing info
|
|
|
|
outgoingHyps[loserPrevHypo].insert(hypo);
|
|
|
|
|
|
|
|
|
|
|
|
} // end for arc list
|
|
|
|
} // end if arc list empty
|
|
|
|
} // end if hypo connected
|
|
|
|
} // end for hypo
|
|
|
|
} // end for stack
|
|
|
|
|
|
|
|
for (std::vector< const Hypothesis *>::iterator it = connectedList.begin(); it != connectedList.end(); ++it) {
|
|
|
|
float estimatedScore = (*it)->GetScore() + forwardScore[(*it)->GetId()];
|
|
|
|
estimatedScores.push_back(estimatedScore);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
const Hypothesis *Manager::GetBestHypothesis() const
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
return m_search->GetBestHypothesis();
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2008-10-09 03:51:26 +04:00
|
|
|
|
2010-05-14 13:33:03 +04:00
|
|
|
int Manager::GetNextHypoId()
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
return m_hypoId++;
|
2010-05-14 13:33:03 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void Manager::ResetSentenceStats(const InputType& source)
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));
|
2010-05-14 13:33:03 +04:00
|
|
|
}
|
|
|
|
SentenceStats& Manager::GetSentenceStats() const
|
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
return *m_sentenceStats;
|
2010-05-14 13:33:03 +04:00
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
}
|
|
|
|
|
2010-05-14 13:33:03 +04:00
|
|
|
}
|