moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

documentation of chart decoder

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3921 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
phkoehn 2011-03-09 17:55:27 +00:00
parent ad140593b0
commit aead480bea
4 changed files with 118 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
moses-chart/src/ChartCell.cpp
View File

@ -73,16 +73,20 @@ void ChartCell::PruneToSize()
}
}
// decoding at span level: fill chart cell with hypotheses
// (implementation of cube pruning)
void ChartCell::ProcessSentence(const ChartTranslationOptionList &transOptList
, const ChartCellCollection &allChartCells)
{
const StaticData &staticData = StaticData::Instance();
// priority queue for applicable rules with selected hypotheses
Cube cube;
// add all trans opt into queue. using only 1st child node.
ChartTranslationOptionList::const_iterator iterList;
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList) {
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList)
{
const ChartTranslationOption &transOpt = **iterList;
QueueEntry *queueEntry = new QueueEntry(transOpt, allChartCells);
cube.Add(queueEntry);
@ -90,19 +94,17 @@ void ChartCell::ProcessSentence(const ChartTranslationOptionList &transOptList
// pluck things out of queue and add to hypo collection
const size_t popLimit = staticData.GetCubePruningPopLimit();
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops) {
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops)
{
QueueEntry *queueEntry = cube.Pop();
queueEntry->GetTranslationOption().GetTotalScore();
// create hypothesis from QueueEntry
Hypothesis *hypo = new Hypothesis(*queueEntry, m_manager);
assert(hypo);
hypo->CalcScore();
AddHypothesis(hypo);
// Expand queue entry
// add neighbors to the queue
queueEntry->CreateDeviants(cube);
}
}

112
moses-chart/src/ChartHypothesis.cpp
View File

@ -43,6 +43,7 @@ unsigned int Hypothesis::s_HypothesesCreated = 0;
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
#endif
/** Create a hypothesis from a rule */
Hypothesis::Hypothesis(const QueueEntry &queueEntry, Manager &manager)
:m_transOpt(queueEntry.GetTranslationOption())
,m_wordsConsumedTargetOrder(queueEntry.GetTranslationOption().GetWordsConsumedTargetOrder())
@ -56,21 +57,27 @@ Hypothesis::Hypothesis(const QueueEntry &queueEntry, Manager &manager)
assert(GetCurrTargetPhrase().GetSize() == m_wordsConsumedTargetOrder.size());
//TRACE_ERR(m_targetPhrase << endl);
// underlying hypotheses for sub-spans
m_numTargetTerminals = GetCurrTargetPhrase().GetNumTerminals();
const std::vector<ChildEntry> &childEntries = queueEntry.GetChildEntries();
// ... are stored
assert(m_prevHypos.empty());
m_prevHypos.reserve(childEntries.size());
vector<ChildEntry>::const_iterator iter;
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter)
{
const ChildEntry &childEntry = *iter;
const Hypothesis *prevHypo = childEntry.GetHypothesis();
// keep count of words (= length of generated string)
m_numTargetTerminals += prevHypo->GetNumTargetTerminals();
m_prevHypos.push_back(prevHypo);
}
// compute the relevant context for language model scoring (prefix and suffix strings)
size_t maxNGram = manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder();
CalcPrefix(m_contextPrefix, maxNGram - 1);
CalcSuffix(m_contextSuffix, maxNGram - 1);
@ -78,6 +85,7 @@ Hypothesis::Hypothesis(const QueueEntry &queueEntry, Manager &manager)
Hypothesis::~Hypothesis()
{
// delete hypotheses that are not in the chart (recombined away)
if (m_arcList) {
ArcList::iterator iter;
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
@ -90,6 +98,9 @@ Hypothesis::~Hypothesis()
}
}
/** Create full output phrase that is contained in the hypothesis (and its children)
* \param outPhrase full output phrase
*/
void Hypothesis::CreateOutputPhrase(Phrase &outPhrase) const
{
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
@ -99,12 +110,14 @@ void Hypothesis::CreateOutputPhrase(Phrase &outPhrase) const
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->CreateOutputPhrase(outPhrase);
} else {
}
else {
outPhrase.AddWord(word);
}
}
}
/** Return full output phrase */
Phrase Hypothesis::GetOutputPhrase() const
{
Phrase outPhrase(Output, ARRAY_SIZE_INCR);
@ -112,20 +125,29 @@ Phrase Hypothesis::GetOutputPhrase() const
return outPhrase;
}
/** Construct the prefix string of up to specified size
* \param ret prefix string
* \param size maximum size (typically max lm context window)
*/
size_t Hypothesis::CalcPrefix(Phrase &ret, size_t size) const
{
// loop over the rule that is being applied
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
// for non-terminals, retrieve it from underlying hypothesis
if (word.IsNonTerminal()) {
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcPrefix(ret, size);
} else {
}
// for words, add word
else {
ret.AddWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
// finish when maximum length reached
if (size==0)
break;
}
@ -133,12 +155,18 @@ size_t Hypothesis::CalcPrefix(Phrase &ret, size_t size) const
return size;
}
/** Construct the suffix phrase of up to specified size
* will always be called after the construction of prefix phrase
* \param ret suffix phrase
* \param size maximum size of suffix
*/
size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
{
assert(m_contextPrefix.GetSize() <= m_numTargetTerminals);
// special handling for small hypotheses
// does the prefix match the entire hypothesis string? -> just copy prefix
if (m_contextPrefix.GetSize() == m_numTargetTerminals) {
// small hypo. the prefix will contains the whole hypo
size_t maxCount = min(m_contextPrefix.GetSize(), size)
, pos = m_contextPrefix.GetSize() - 1;
@ -150,7 +178,9 @@ size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
size -= maxCount;
return size;
} else {
}
// construct suffix analogous to prefix
else {
for (int pos = (int) GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
@ -158,7 +188,8 @@ size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcSuffix(ret, size);
} else {
}
else {
ret.PrependWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
@ -171,6 +202,11 @@ size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
}
}
/** Check recombinability due to language model context
* if two hypotheses match in their edge phrases, they are indistinguishable
* in terms of future search, so the weaker one can be dropped.
* \param other other hypothesis for comparison
*/
int Hypothesis::LMContextCompare(const Hypothesis &other) const
{
// prefix
@ -212,47 +248,66 @@ void Hypothesis::CalcScore()
m_totalScore = m_scoreBreakdown.GetWeightedScore();
}
// compute languane model score for the hypothesis
void Hypothesis::CalcLMScore()
{
// get the language models involved
const LMList& lmList = m_manager.GetTranslationSystem()->GetLanguageModels();
assert(m_lmNGram.GetWeightedScore() == 0);
// start scores with 0
m_scoreBreakdown.ZeroAllLM(lmList);
Phrase outPhrase(Output, ARRAY_SIZE_INCR); // = GetOutputPhrase();
bool calcNow = false, firstPhrase = true;
// initialize output string (empty)
Phrase outPhrase(Output, ARRAY_SIZE_INCR);
bool firstPhrase = true; // flag for first word
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos) {
// loop over rule
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos)
{
// consult rule for either word or non-terminal
const Word &targetWord = GetCurrTargetPhrase().GetWord(targetPhrasePos);
if (!targetWord.IsNonTerminal()) {
// just a word, add to phrase for lm scoring
// just a word, add it to phrase for lm scoring
if (!targetWord.IsNonTerminal())
{
outPhrase.AddWord(targetWord);
} else {
}
// non-terminal, add phrase from underlying hypothesis
else
{
// look up underlying hypothesis
size_t nonTermInd = m_wordsConsumedTargetOrder[targetPhrasePos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size_t numTargetTerminals = prevHypo->GetNumTargetTerminals();
if (numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
// large hypo (for trigram lm, another hypo equal or over 2 words). just take the prefix & suffix
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
// calc & add overlapping lm scores
// prefix
outPhrase.Append(prevHypo->GetPrefix());
calcNow = true;
} else {
// small hypo (for trigram lm, 1-word hypos).
// check if we are dealing with a large sub-phrase
// (large sub-phrases have internal language model scores)
if (numTargetTerminals < lmList.GetMaxNGramOrder() - 1)
{
// small sub-phrase (for trigram lm, 1 or 2-word hypos).
// add target phrase to temp phrase and continue, but don't score yet
outPhrase.Append(prevHypo->GetPrefix());
}
else {
// large sub-phrase
// we add the internal scores
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
if (calcNow) {
// we will still have to score the prefix
outPhrase.Append(prevHypo->GetPrefix());
// preliminary language model scoring
// special case: rule starts with non-terminal
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
// get from other prev hypo. faster
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
m_lmNGram.Assign(prevHypo->m_lmNGram);
} else {
// calc
}
// general case
else {
// score everything we got so far
lmList.CalcAllLMScores(outPhrase
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
@ -263,8 +318,7 @@ void Hypothesis::CalcLMScore()
outPhrase.Append(prevHypo->GetSuffix());
firstPhrase = false;
calcNow = false;
}
} // if long sub-phrase
} // if (!targetWord.IsNonTerminal())
} // for (size_t targetPhrasePos
@ -272,7 +326,9 @@ void Hypothesis::CalcLMScore()
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
// add score estimate for prefix
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmPrefix);
// add real score for words with full history
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmNGram);
/*

5
moses-chart/src/ChartHypothesisCollection.cpp
View File

@ -63,6 +63,8 @@ bool HypothesisCollection::AddHypothesis(Hypothesis *hypo, Manager &manager)
// over threshold, try to add to collection
std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
// does it have the same state as an existing hypothesis?
if (addRet.second) {
// nothing found. add to collection
return true;
@ -99,7 +101,8 @@ bool HypothesisCollection::AddHypothesis(Hypothesis *hypo, Manager &manager)
VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
if (m_nBestIsEnabled) {
hypoExisting->AddArc(hypo);
} else {
}
else {
Hypothesis::Delete(hypo);
}
return false;

28
moses-chart/src/QueueEntry.cpp
View File

@ -39,6 +39,7 @@ using namespace Moses;
namespace MosesChart
{
// create a cube for a rule
QueueEntry::QueueEntry(const Moses::ChartTranslationOption &transOpt
, const ChartCellCollection &allChartCells)
:m_transOpt(transOpt)
@ -48,28 +49,37 @@ QueueEntry::QueueEntry(const Moses::ChartTranslationOption &transOpt
CalcScore();
}
// for each non-terminal, create a ordered list of matching hypothesis from the chart
void QueueEntry::CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells)
{
// recursvile do the 1st first
// recurse through the linked list of source side non-terminals and terminals
const WordConsumed *prevWordsConsumed = wordsConsumed->GetPrevWordsConsumed();
if (prevWordsConsumed)
CreateChildEntry(prevWordsConsumed, allChartCells);
if (wordsConsumed->IsNonTerminal()) {
// non-term
const WordsRange &childRange = wordsConsumed->GetWordsRange();
const ChartCell &childCell = allChartCells.Get(childRange);
const Word &headWord = wordsConsumed->GetSourceWord();
// only deal with non-terminals
if (wordsConsumed->IsNonTerminal())
{
// get the essential information about the non-terminal
const WordsRange &childRange = wordsConsumed->GetWordsRange(); // span covered by child
const ChartCell &childCell = allChartCells.Get(childRange); // list of all hypos for that span
const Word &headWord = wordsConsumed->GetSourceWord(); // target (sic!) non-terminal label
// there have to be hypothesis with the desired non-terminal
// (otherwise the rule would not be considered)
assert(!childCell.GetSortedHypotheses(headWord).empty());
// ??? why are we looking it up again?
const Moses::Word &nonTerm = wordsConsumed->GetSourceWord();
assert(nonTerm.IsNonTerminal());
// create a list of hypotheses that match the non-terminal
ChildEntry childEntry(0, childCell.GetSortedHypotheses(nonTerm), nonTerm);
// add them to the vector for such lists
m_childEntries.push_back(childEntry);
}
}
// create the QueueEntry from an existing one, differing only in one child hypothesis
QueueEntry::QueueEntry(const QueueEntry &copy, size_t childEntryIncr)
:m_transOpt(copy.m_transOpt)
,m_childEntries(copy.m_childEntries)
@ -84,8 +94,11 @@ QueueEntry::~QueueEntry()
//Moses::RemoveAllInColl(m_childEntries);
}
// create new QueueEntry for neighboring principle rules
// (duplicate detection is handled in Cube)
void QueueEntry::CreateDeviants(Cube &cube) const
{
// loop over all child hypotheses
for (size_t ind = 0; ind < m_childEntries.size(); ind++) {
const ChildEntry &childEntry = m_childEntries[ind];
@ -96,6 +109,8 @@ void QueueEntry::CreateDeviants(Cube &cube) const
}
}
// compute an estimated cost of the principle rule
// (consisting of rule translation scores plus child hypotheses scores)
void QueueEntry::CalcScore()
{
m_combinedScore = m_transOpt.GetTotalScore();
@ -105,7 +120,6 @@ void QueueEntry::CalcScore()
const Hypothesis *hypo = childEntry.GetHypothesis();
m_combinedScore += hypo->GetTotalScore();
}
}
bool QueueEntry::operator<(const QueueEntry &compare) const