2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
#include "Timer.h"
|
|
|
|
#include "SearchNormal.h"
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
2008-06-11 14:52:57 +04:00
|
|
|
#undef DEBUGLATTICE
|
|
|
|
#ifdef DEBUGLATTICE
|
|
|
|
static bool debug2 = false;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
SearchNormal::SearchNormal(const InputType &source, const TranslationOptionCollection &transOptColl)
|
2008-06-17 21:32:19 +04:00
|
|
|
:m_source(source)
|
|
|
|
,m_hypoStackColl(source.GetSize() + 1)
|
|
|
|
,m_initialTargetPhrase(Output)
|
|
|
|
,m_start(clock())
|
|
|
|
,interrupted_flag(0)
|
|
|
|
,m_transOptColl(transOptColl)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
|
|
|
VERBOSE(1, "Translating: " << m_source << endl);
|
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
|
2008-12-13 15:08:55 +03:00
|
|
|
// only if constraint decoding (having to match a specified output)
|
2008-06-19 03:14:09 +04:00
|
|
|
long sentenceID = source.GetTranslationId();
|
|
|
|
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
|
|
|
|
|
2008-12-13 15:08:55 +03:00
|
|
|
// initialize the stacks: create data structure and set limits
|
2008-06-11 14:52:57 +04:00
|
|
|
std::vector < HypothesisStackNormal >::iterator iterStack;
|
|
|
|
for (size_t ind = 0 ; ind < m_hypoStackColl.size() ; ++ind)
|
|
|
|
{
|
|
|
|
HypothesisStackNormal *sourceHypoColl = new HypothesisStackNormal();
|
2008-12-13 15:08:55 +03:00
|
|
|
sourceHypoColl->SetMaxHypoStackSize(staticData.GetMaxHypoStackSize(),staticData.GetMinHypoStackDiversity());
|
2008-06-11 14:52:57 +04:00
|
|
|
sourceHypoColl->SetBeamWidth(staticData.GetBeamWidth());
|
|
|
|
|
|
|
|
m_hypoStackColl[ind] = sourceHypoColl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
SearchNormal::~SearchNormal()
|
|
|
|
{
|
|
|
|
RemoveAllInColl(m_hypoStackColl);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Main decoder loop that translates a sentence by expanding
|
|
|
|
* hypotheses stack by stack, until the end of the sentence.
|
|
|
|
*/
|
|
|
|
void SearchNormal::ProcessSentence()
|
|
|
|
{
|
|
|
|
const StaticData &staticData = StaticData::Instance();
|
2008-12-14 16:23:33 +03:00
|
|
|
SentenceStats &stats = staticData.GetSentenceStats();
|
|
|
|
clock_t t=0; // used to track time for steps
|
2008-06-17 21:32:19 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// initial seed hypothesis: nothing translated, no words produced
|
|
|
|
Hypothesis *hypo = Hypothesis::Create(m_source, m_initialTargetPhrase);
|
|
|
|
m_hypoStackColl[0]->AddPrune(hypo);
|
2008-06-17 21:32:19 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// go through each stack
|
|
|
|
std::vector < HypothesisStack* >::iterator iterStack;
|
|
|
|
for (iterStack = m_hypoStackColl.begin() ; iterStack != m_hypoStackColl.end() ; ++iterStack)
|
|
|
|
{
|
2008-12-13 15:08:55 +03:00
|
|
|
// check if decoding ran out of time
|
2008-06-11 14:52:57 +04:00
|
|
|
double _elapsed_time = GetUserTime();
|
|
|
|
if (_elapsed_time > staticData.GetTimeoutThreshold()){
|
2008-06-17 21:32:19 +04:00
|
|
|
VERBOSE(1,"Decoding is out of time (" << _elapsed_time << "," << staticData.GetTimeoutThreshold() << ")" << std::endl);
|
2008-06-11 14:52:57 +04:00
|
|
|
interrupted_flag = 1;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
HypothesisStackNormal &sourceHypoColl = *static_cast<HypothesisStackNormal*>(*iterStack);
|
|
|
|
|
|
|
|
// the stack is pruned before processing (lazy pruning):
|
|
|
|
VERBOSE(3,"processing hypothesis from next stack");
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { t = clock(); }
|
2008-06-11 14:52:57 +04:00
|
|
|
sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
|
|
|
|
VERBOSE(3,std::endl);
|
|
|
|
sourceHypoColl.CleanupArcList();
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { stats.AddTimeStack( clock()-t ); }
|
2008-12-13 15:08:55 +03:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// go through each hypothesis on the stack and try to expand it
|
|
|
|
HypothesisStackNormal::const_iterator iterHypo;
|
|
|
|
for (iterHypo = sourceHypoColl.begin() ; iterHypo != sourceHypoColl.end() ; ++iterHypo)
|
2008-06-17 21:32:19 +04:00
|
|
|
{
|
|
|
|
Hypothesis &hypothesis = **iterHypo;
|
|
|
|
ProcessOneHypothesis(hypothesis); // expand the hypothesis
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
// some logging
|
|
|
|
IFVERBOSE(2) { OutputHypoStackSize(); }
|
|
|
|
|
2008-12-13 15:08:55 +03:00
|
|
|
// this stack is fully expanded;
|
2008-06-11 14:52:57 +04:00
|
|
|
actual_hypoStack = &sourceHypoColl;
|
|
|
|
}
|
|
|
|
|
|
|
|
// some more logging
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { staticData.GetSentenceStats().SetTimeTotal( clock()-m_start ); }
|
2008-06-11 14:52:57 +04:00
|
|
|
VERBOSE(2, staticData.GetSentenceStats());
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Find all translation options to expand one hypothesis, trigger expansion
|
|
|
|
* this is mostly a check for overlap with already covered words, and for
|
|
|
|
* violation of reordering limits.
|
|
|
|
* \param hypothesis hypothesis to be expanded upon
|
|
|
|
*/
|
|
|
|
void SearchNormal::ProcessOneHypothesis(const Hypothesis &hypothesis)
|
|
|
|
{
|
|
|
|
// since we check for reordering limits, its good to have that limit handy
|
|
|
|
int maxDistortion = StaticData::Instance().GetMaxDistortion();
|
|
|
|
bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;
|
|
|
|
|
|
|
|
// no limit of reordering: only check for overlap
|
|
|
|
if (maxDistortion < 0)
|
|
|
|
{
|
|
|
|
const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
|
|
|
|
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos()
|
2008-06-17 21:32:19 +04:00
|
|
|
, sourceSize = m_source.GetSize();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
|
|
|
|
{
|
|
|
|
size_t maxSize = sourceSize - startPos;
|
|
|
|
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
|
|
|
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
2008-06-17 21:32:19 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
|
|
|
|
{
|
2008-12-15 15:52:38 +03:00
|
|
|
if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)) || !m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) )
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2008-08-26 21:04:43 +04:00
|
|
|
//TODO: does this method include incompatible WordLattice hypotheses?
|
2008-06-11 14:52:57 +04:00
|
|
|
ExpandAllHypotheses(hypothesis
|
2008-06-17 21:32:19 +04:00
|
|
|
, m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos)));
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return; // done with special case (no reordering limit)
|
|
|
|
}
|
|
|
|
|
|
|
|
// if there are reordering limits, make sure it is not violated
|
|
|
|
// the coverage bitmap is handy here (and the position of the first gap)
|
|
|
|
const WordsBitmap hypoBitmap = hypothesis.GetWordsBitmap();
|
|
|
|
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos()
|
2008-06-17 21:32:19 +04:00
|
|
|
, sourceSize = m_source.GetSize();
|
|
|
|
|
2008-12-15 15:52:38 +03:00
|
|
|
// MAIN LOOP. go through each possible range
|
2008-06-11 14:52:57 +04:00
|
|
|
for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
|
|
|
|
{
|
2008-06-17 21:32:19 +04:00
|
|
|
size_t maxSize = sourceSize - startPos;
|
|
|
|
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-08-26 21:04:43 +04:00
|
|
|
const int INTEREST = 29;
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
2008-06-17 21:32:19 +04:00
|
|
|
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
|
|
|
size_t closestLeft = hypoBitmap.GetEdgeToTheLeftOf(startPos);
|
2008-06-11 14:52:57 +04:00
|
|
|
if (isWordLattice) {
|
|
|
|
// first question: is there a path from the closest translated word to the left
|
|
|
|
// of the hypothesized extension to the start of the hypothesized extension?
|
2008-08-26 21:04:43 +04:00
|
|
|
// long version: is there anything to our left? is it farther left than where we're starting anyway? can we get to it?
|
|
|
|
//closestLeft is exclusive: a value of 3 means 2 is covered, our arc is currently ENDING at 3 and can start at 3 implicitly
|
2008-06-17 21:32:19 +04:00
|
|
|
//if (closestLeft != startPos && closestLeft != 0 && ((startPos - closestLeft) != 1 && !m_source.CanIGetFromAToB(closestLeft+1, startPos+1))) {
|
2008-08-26 21:04:43 +04:00
|
|
|
//if (closestLeft != startPos && closestLeft != 0 && ((startPos - closestLeft) != 0 && !m_source.CanIGetFromAToB(closestLeft, startPos))) {
|
|
|
|
if (closestLeft != 0 && closestLeft != startPos && !m_source.CanIGetFromAToB(closestLeft, startPos)) {
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-06-17 21:32:19 +04:00
|
|
|
if (startPos == INTEREST) {
|
2008-08-26 21:04:43 +04:00
|
|
|
std::cerr << "Die0: hyp = " << hypothesis <<"\n";
|
|
|
|
std::cerr << "Die0: CanIGetFromAToB(" << (closestLeft) << "," << (startPos) << ") = " << m_source.CanIGetFromAToB(closestLeft,startPos) << "\n";
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
#endif
|
2008-06-17 21:32:19 +04:00
|
|
|
continue;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
//if (startPos == INTEREST) { std::cerr << "INTEREST: " << hypothesis << "\n"; }
|
|
|
|
|
|
|
|
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
|
|
|
|
{
|
2008-12-15 15:52:38 +03:00
|
|
|
// check if passes specified reordering constraints
|
|
|
|
// (set with -monotone-at-punctuation or xml)
|
|
|
|
if (!m_source.GetReorderingConstraint().Check( hypoBitmap, startPos, endPos ) )
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
// check for overlap
|
2008-06-17 21:32:19 +04:00
|
|
|
WordsRange extRange(startPos, endPos);
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
|
|
|
//if (startPos == INTEREST) { std::cerr << " (" << hypoFirstGapPos << ")-> wr: " << extRange << "\n"; }
|
2008-08-26 21:04:43 +04:00
|
|
|
bool interest = ((startPos == endPos) && (startPos == INTEREST));// || (startPos ==0 && endPos == 1);
|
2008-06-17 21:32:19 +04:00
|
|
|
bool debug = interest;
|
|
|
|
//(startPos > (INTEREST-8) && hypoFirstGapPos > 0 && startPos <= INTEREST && endPos >=INTEREST && endPos < (INTEREST+25) && hypoFirstGapPos == INTEREST);
|
2008-08-26 21:04:43 +04:00
|
|
|
//debug2 = debug;
|
|
|
|
//if (debug) { std::cerr << (startPos==INTEREST? "LOOK-->" : "") << "XP: " << hypothesis << "\next: " << extRange << "\n"; }
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
|
|
|
if (hypoBitmap.Overlap(extRange) ||
|
2008-08-26 21:04:43 +04:00
|
|
|
(isWordLattice && !m_source.IsCoveragePossible(extRange)
|
|
|
|
//|| !m_source.IsExtensionPossible(hypothesis.GetCurrSourceWordsRange(), extRange))
|
|
|
|
//isExtensionPossible is redundant. We just need to know 1. is it consistant? 2. can our end touch the next thing to the right.
|
2008-06-17 21:32:19 +04:00
|
|
|
)
|
2008-06-11 14:52:57 +04:00
|
|
|
)
|
2008-06-17 21:32:19 +04:00
|
|
|
{
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-08-26 21:04:43 +04:00
|
|
|
if (interest) {
|
|
|
|
//std::cerr << "Die1: hyp: " << hypothesis << "\n";
|
|
|
|
//std::cerr << "Die1: extRange: " << extRange << "\n";
|
|
|
|
//std::cerr << "Die1: currSrcRange: " << hypothesis.GetCurrSourceWordsRange() << "\n";
|
|
|
|
//std::cerr << "Die1: Overlap=" << hypoBitmap.Overlap(extRange) << " IsCoveragePossible=" << m_source.IsCoveragePossible(extRange) << " IsExtensionPossible=" << m_source.IsExtensionPossible(hypothesis.GetCurrSourceWordsRange(), extRange)<< "\n";
|
2008-06-17 21:32:19 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
2008-06-17 21:32:19 +04:00
|
|
|
continue;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
bool leftMostEdge = (hypoFirstGapPos == startPos);
|
2008-06-17 21:32:19 +04:00
|
|
|
size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(endPos);
|
|
|
|
// ask second question here:
|
2008-08-26 21:04:43 +04:00
|
|
|
// we already know we can get to our starting point from the closest thing to the left. We now ask the follow up:
|
|
|
|
// can we get from our end to the closest thing on the right?
|
|
|
|
// long version: is anything to our right? is it farther right than our (inclusive) end? can our end reach it?
|
|
|
|
// closest right definition:
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
if (isWordLattice) {
|
2008-08-26 21:04:43 +04:00
|
|
|
//if (!leftMostEdge && closestRight != endPos && closestRight != sourceSize && !m_source.CanIGetFromAToB(endPos, closestRight + 1)) {
|
|
|
|
if (closestRight != endPos && ((closestRight + 1) < sourceSize) && !m_source.CanIGetFromAToB(endPos, closestRight + 1)) {
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-08-26 21:04:43 +04:00
|
|
|
if (interest) {
|
|
|
|
std::cerr << "Die2: hyp = " << hypothesis <<"\n";
|
|
|
|
std::cerr << "Die2: extRange = " << extRange <<"\n";
|
|
|
|
std::cerr << "Die2: CloseRight = " << closestRight <<"\n";
|
|
|
|
std::cerr << "Die2: CanIGetFromAToB(" << (endPos) << "," << (closestRight+1) << ") = " << m_source.CanIGetFromAToB(endPos,closestRight + 1) << "\n";
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
2008-06-17 21:32:19 +04:00
|
|
|
continue;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
2008-06-17 21:32:19 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// any length extension is okay if starting at left-most edge
|
|
|
|
if (leftMostEdge)
|
|
|
|
{
|
|
|
|
#ifdef DEBUGLATTICE
|
2008-06-17 21:32:19 +04:00
|
|
|
size_t vl = StaticData::Instance().GetVerboseLevel();
|
|
|
|
if (debug2) {
|
2008-08-26 21:04:43 +04:00
|
|
|
//std::cerr << "Ext!\n";
|
2008-06-17 21:32:19 +04:00
|
|
|
StaticData::Instance().SetVerboseLevel(4);
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
|
|
|
ExpandAllHypotheses(hypothesis
|
2008-06-17 21:32:19 +04:00
|
|
|
,m_transOptColl.GetTranslationOptionList(extRange));
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-06-17 21:32:19 +04:00
|
|
|
StaticData::Instance().SetVerboseLevel(vl);
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
|
|
|
}
|
|
|
|
// starting somewhere other than left-most edge, use caution
|
|
|
|
else
|
|
|
|
{
|
|
|
|
// the basic idea is this: we would like to translate a phrase starting
|
|
|
|
// from a position further right than the left-most open gap. The
|
|
|
|
// distortion penalty for the following phrase will be computed relative
|
|
|
|
// to the ending position of the current extension, so we ask now what
|
|
|
|
// its maximum value will be (which will always be the value of the
|
2008-08-26 21:04:43 +04:00
|
|
|
// hypothesis starting at the left-most edge). If this value is less than
|
2008-06-11 14:52:57 +04:00
|
|
|
// the distortion limit, we don't allow this extension to be made.
|
|
|
|
WordsRange bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
|
|
|
|
int required_distortion =
|
|
|
|
m_source.ComputeDistortionDistance(extRange, bestNextExtension);
|
|
|
|
|
2008-11-24 20:31:26 +03:00
|
|
|
if (required_distortion > maxDistortion) {
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-08-26 21:04:43 +04:00
|
|
|
if (interest) {
|
|
|
|
std::cerr << "Distortion violation\n";
|
|
|
|
std::cerr << (startPos==INTEREST? "VLOOK-->" : "") << "XP: " << hypothesis << "\next: " << extRange << "\n";
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
2008-11-24 20:31:26 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-12-15 15:52:38 +03:00
|
|
|
// everything is fine, we're good to go
|
2008-11-24 20:31:26 +03:00
|
|
|
ExpandAllHypotheses(hypothesis
|
|
|
|
,m_transOptColl.GetTranslationOptionList(extRange));
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
}
|
2008-06-17 21:32:19 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
/**
|
|
|
|
* Expand a hypothesis given a list of translation options
|
|
|
|
* \param hypothesis hypothesis to be expanded upon
|
|
|
|
* \param transOptList list of translation options to be applied
|
|
|
|
*/
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
void SearchNormal::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2008-06-17 21:32:19 +04:00
|
|
|
TranslationOptionList::const_iterator iter;
|
|
|
|
for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
|
|
|
|
{
|
|
|
|
ExpandHypothesis(hypothesis, **iter);
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
/**
|
|
|
|
* Expand one hypothesis with a translation option.
|
|
|
|
* this involves initial creation, scoring and adding it to the proper stack
|
|
|
|
* \param hypothesis hypothesis to be expanded upon
|
|
|
|
* \param transOpt translation option (phrase translation)
|
|
|
|
* that is applied to create the new hypothesis
|
|
|
|
*/
|
|
|
|
void SearchNormal::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
|
|
|
|
{
|
2008-12-14 16:23:33 +03:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
SentenceStats &stats = staticData.GetSentenceStats();
|
|
|
|
clock_t t=0; // used to track time for steps
|
2008-12-13 15:08:55 +03:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
#ifdef DEBUGLATTICE
|
2008-06-17 21:32:19 +04:00
|
|
|
if (debug2) { std::cerr << "::EXT: " << transOpt << "\n"; }
|
2008-06-11 14:52:57 +04:00
|
|
|
#endif
|
2008-12-14 16:23:33 +03:00
|
|
|
Hypothesis *newHypo;
|
2008-06-20 02:40:51 +04:00
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
// expand hypothesis further if transOpt was linked
|
2008-12-14 16:23:33 +03:00
|
|
|
// this code will get nuked soon, it has many problems
|
|
|
|
// for (std::vector<TranslationOption*>::const_iterator iterLinked = transOpt.GetLinkedTransOpts().begin();
|
|
|
|
// iterLinked != transOpt.GetLinkedTransOpts().end(); iterLinked++) {
|
|
|
|
// t = clock();
|
|
|
|
// newHypo = hypothesis.CreateNext(transOpt, m_constraint);
|
|
|
|
// StaticData::Instance().GetSentenceStats().AddTimeBuildHyp( clock()-t );
|
|
|
|
// if (newHypo==NULL) return;
|
|
|
|
// const WordsBitmap hypoBitmap = newHypo->GetWordsBitmap();
|
|
|
|
// if (hypoBitmap.Overlap((**iterLinked).GetSourceWordsRange())) {
|
2008-06-17 21:32:19 +04:00
|
|
|
// don't want to add a hypothesis that has some but not all of a linked TO set, so return
|
2008-12-14 16:23:33 +03:00
|
|
|
// FREEHYPO( newHypo );
|
|
|
|
// return;
|
|
|
|
// }
|
|
|
|
// else
|
|
|
|
// {
|
|
|
|
// newHypo->CalcScore(m_transOptColl.GetFutureScore());
|
|
|
|
// newHypo = newHypo->CreateNext(**iterLinked, m_constraint); // TODO FIXME This is absolutely broken - don't pass null here
|
|
|
|
// }
|
|
|
|
// }
|
|
|
|
|
|
|
|
if (! staticData.UseEarlyDiscarding())
|
|
|
|
{
|
|
|
|
// simple build, no questions asked
|
|
|
|
IFVERBOSE(2) { t = clock(); }
|
|
|
|
newHypo = hypothesis.CreateNext(transOpt, m_constraint);
|
|
|
|
IFVERBOSE(2) { stats.AddTimeBuildHyp( clock()-t ); }
|
|
|
|
if (newHypo==NULL) return;
|
|
|
|
newHypo->CalcScore(m_transOptColl.GetFutureScore());
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2008-12-14 16:23:33 +03:00
|
|
|
else
|
2008-12-13 15:08:55 +03:00
|
|
|
// early discarding: check if hypothesis is too bad to build
|
2008-12-14 16:23:33 +03:00
|
|
|
// this idea is explained in (Moore&Quirk, MT Summit 2007)
|
|
|
|
{
|
2008-12-17 15:32:52 +03:00
|
|
|
// what is the worst possible score?
|
|
|
|
|
2008-12-13 15:08:55 +03:00
|
|
|
size_t wordsTranslated = hypothesis.GetWordsBitmap().GetNumWordsCovered() + transOpt.GetSize();
|
2008-12-14 16:23:33 +03:00
|
|
|
float allowedScore = m_hypoStackColl[wordsTranslated]->GetWorstScore() + staticData.GetEarlyDiscardingThreshold();
|
|
|
|
|
2008-12-17 15:32:52 +03:00
|
|
|
// expected score is based on score of current hypothesis
|
|
|
|
float expectedScore = hypothesis.GetScore();
|
|
|
|
// add new future cost estimate
|
|
|
|
WordsBitmap *newSourceCompleted = new WordsBitmap ( hypothesis.GetWordsBitmap() );
|
|
|
|
newSourceCompleted->SetValue(transOpt.GetSourceWordsRange().GetStartPos(), transOpt.GetSourceWordsRange().GetEndPos(), true);
|
|
|
|
expectedScore += m_transOptColl.GetFutureScore().CalcFutureScore( *newSourceCompleted );
|
|
|
|
delete(newSourceCompleted);
|
|
|
|
// add expected score of translation option
|
|
|
|
expectedScore += transOpt.GetFutureScore();
|
2008-12-14 16:23:33 +03:00
|
|
|
// check if transOpt cost push it already below limit
|
|
|
|
if (expectedScore < allowedScore)
|
|
|
|
{
|
|
|
|
IFVERBOSE(2) { stats.AddNotBuilt(); }
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// build the hypothesis without scoring
|
|
|
|
IFVERBOSE(2) { t = clock(); }
|
|
|
|
newHypo = hypothesis.CreateNext(transOpt, m_constraint);
|
|
|
|
if (newHypo==NULL) return;
|
|
|
|
IFVERBOSE(2) { stats.AddTimeBuildHyp( clock()-t ); }
|
|
|
|
|
|
|
|
// compute expected score (all but correct LM)
|
|
|
|
expectedScore = newHypo->CalcExpectedScore( m_transOptColl.GetFutureScore() );
|
|
|
|
// ... and check if that is below the limit
|
2008-12-13 15:08:55 +03:00
|
|
|
if (expectedScore < allowedScore)
|
|
|
|
{
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { stats.AddEarlyDiscarded(); }
|
2008-12-13 15:08:55 +03:00
|
|
|
FREEHYPO( newHypo );
|
|
|
|
return;
|
|
|
|
}
|
2008-12-14 16:23:33 +03:00
|
|
|
|
|
|
|
// ok, all is good, compute remaining scores
|
2008-12-13 15:08:55 +03:00
|
|
|
newHypo->CalcRemainingScore();
|
2008-12-17 15:32:52 +03:00
|
|
|
|
2008-12-13 15:08:55 +03:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
// logging for the curious
|
|
|
|
IFVERBOSE(3) {
|
2008-12-13 15:08:55 +03:00
|
|
|
newHypo->PrintHypothesis();
|
2008-06-17 21:32:19 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
// add to hypothesis stack
|
|
|
|
size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { t = clock(); }
|
2008-06-17 21:32:19 +04:00
|
|
|
m_hypoStackColl[wordsTranslated]->AddPrune(newHypo);
|
2008-12-14 16:23:33 +03:00
|
|
|
IFVERBOSE(2) { stats.AddTimeStack( clock()-t ); }
|
2008-06-17 21:32:19 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
const std::vector < HypothesisStack* >& SearchNormal::GetHypothesisStacks() const
|
|
|
|
{
|
|
|
|
return m_hypoStackColl;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2008-06-17 21:32:19 +04:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Find best hypothesis on the last stack.
|
|
|
|
* This is the end point of the best translation, which can be traced back from here
|
|
|
|
*/
|
|
|
|
const Hypothesis *SearchNormal::GetBestHypothesis() const
|
|
|
|
{
|
|
|
|
// const HypothesisStackNormal &hypoColl = m_hypoStackColl.back();
|
|
|
|
if (interrupted_flag == 0){
|
|
|
|
const HypothesisStackNormal &hypoColl = *static_cast<HypothesisStackNormal*>(m_hypoStackColl.back());
|
|
|
|
return hypoColl.GetBestHypothesis();
|
|
|
|
}
|
|
|
|
else{
|
|
|
|
const HypothesisStackNormal &hypoColl = *actual_hypoStack;
|
|
|
|
return hypoColl.GetBestHypothesis();
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2008-06-17 21:32:19 +04:00
|
|
|
/**
|
|
|
|
* Logging of hypothesis stack sizes
|
|
|
|
*/
|
|
|
|
void SearchNormal::OutputHypoStackSize()
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2008-06-17 21:32:19 +04:00
|
|
|
std::vector < HypothesisStack* >::const_iterator iterStack = m_hypoStackColl.begin();
|
|
|
|
TRACE_ERR( "Stack sizes: " << (int)(*iterStack)->size());
|
|
|
|
for (++iterStack; iterStack != m_hypoStackColl.end() ; ++iterStack)
|
|
|
|
{
|
|
|
|
TRACE_ERR( ", " << (int)(*iterStack)->size());
|
|
|
|
}
|
|
|
|
TRACE_ERR( endl);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2008-10-09 03:51:26 +04:00
|
|
|
|
|
|
|
}
|
|
|
|
|