mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
moses_chart: rule lookup now produces a vector of stack pointers (and a
TargetPhraseCollection pointer) instead of a CYK+ dotted rule.
This commit is contained in:
parent
5254e7917b
commit
72baeab59e
@ -227,14 +227,71 @@ void OutputInput(std::ostream& os, const ChartHypothesis* hypo)
|
||||
}
|
||||
*/
|
||||
|
||||
void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, long translationId)
|
||||
namespace {
|
||||
|
||||
typedef std::vector<std::pair<Word, WordsRange> > ApplicationContext;
|
||||
|
||||
// Given a hypothesis and sentence, reconstructs the 'application context' --
|
||||
// the source RHS symbols of the SCFG rule that was applied, plus their spans.
|
||||
void ReconstructApplicationContext(const ChartHypothesis &hypo,
|
||||
const Sentence &sentence,
|
||||
ApplicationContext &context)
|
||||
{
|
||||
context.clear();
|
||||
const std::vector<const ChartHypothesis*> &prevHypos = hypo.GetPrevHypos();
|
||||
std::vector<const ChartHypothesis*>::const_iterator p = prevHypos.begin();
|
||||
std::vector<const ChartHypothesis*>::const_iterator end = prevHypos.end();
|
||||
const WordsRange &span = hypo.GetCurrSourceRange();
|
||||
size_t i = span.GetStartPos();
|
||||
while (i <= span.GetEndPos()) {
|
||||
if (p == end || i < (*p)->GetCurrSourceRange().GetStartPos()) {
|
||||
// Symbol is a terminal.
|
||||
const Word &symbol = sentence.GetWord(i);
|
||||
context.push_back(std::make_pair(symbol, WordsRange(i, i)));
|
||||
++i;
|
||||
} else {
|
||||
// Symbol is a non-terminal.
|
||||
const Word &symbol = (*p)->GetTargetLHS();
|
||||
const WordsRange &range = (*p)->GetCurrSourceRange();
|
||||
context.push_back(std::make_pair(symbol, range));
|
||||
i = range.GetEndPos()+1;
|
||||
++p;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emulates the old operator<<(ostream &, const DottedRule &) function. The
|
||||
// output format is a bit odd (reverse order and double spacing between symbols)
|
||||
// but there are scripts and tools that expect the output of -T to look like
|
||||
// that.
|
||||
void WriteApplicationContext(std::ostream &out,
|
||||
const ApplicationContext &context)
|
||||
{
|
||||
assert(!context.empty());
|
||||
ApplicationContext::const_reverse_iterator p = context.rbegin();
|
||||
while (true) {
|
||||
out << p->second << "=" << p->first << " ";
|
||||
if (++p == context.rend()) {
|
||||
break;
|
||||
}
|
||||
out << " ";
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, const Sentence &sentence, long translationId)
|
||||
{
|
||||
static ApplicationContext applicationContext;
|
||||
|
||||
// recursive
|
||||
if (hypo != NULL) {
|
||||
ReconstructApplicationContext(*hypo, sentence, applicationContext);
|
||||
out << "Trans Opt " << translationId
|
||||
<< " " << hypo->GetCurrSourceRange()
|
||||
<< ": " << hypo->GetTranslationOption().GetDottedRule()
|
||||
<< ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
|
||||
<< ": ";
|
||||
WriteApplicationContext(out, applicationContext);
|
||||
out << ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
|
||||
<< "->" << hypo->GetCurrTargetPhrase()
|
||||
<< " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown()
|
||||
<< endl;
|
||||
@ -244,19 +301,20 @@ void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, lo
|
||||
std::vector<const ChartHypothesis*>::const_iterator iter;
|
||||
for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
|
||||
const ChartHypothesis *prevHypo = *iter;
|
||||
OutputTranslationOptions(out, prevHypo, translationId);
|
||||
OutputTranslationOptions(out, prevHypo, sentence, translationId);
|
||||
}
|
||||
}
|
||||
|
||||
void IOWrapper::OutputDetailedTranslationReport(
|
||||
const ChartHypothesis *hypo,
|
||||
const Sentence &sentence,
|
||||
long translationId)
|
||||
{
|
||||
if (hypo == NULL) {
|
||||
return;
|
||||
}
|
||||
std::ostringstream out;
|
||||
OutputTranslationOptions(out, hypo, translationId);
|
||||
OutputTranslationOptions(out, hypo, sentence, translationId);
|
||||
CHECK(m_detailOutputCollector);
|
||||
m_detailOutputCollector->Write(translationId, out.str());
|
||||
}
|
||||
|
@ -82,7 +82,7 @@ public:
|
||||
void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
|
||||
void OutputBestHypo(const std::vector<const Moses::Factor*>& mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors);
|
||||
void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartHypothesis *bestHypo, const Moses::TranslationSystem* system, long translationId);
|
||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, long translationId);
|
||||
void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId);
|
||||
void Backtrack(const Moses::ChartHypothesis *hypo);
|
||||
|
||||
void ResetTranslationId();
|
||||
|
@ -98,7 +98,8 @@ public:
|
||||
}
|
||||
|
||||
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
||||
m_ioWrapper.OutputDetailedTranslationReport(bestHypo, lineNumber);
|
||||
const Sentence &sentence = dynamic_cast<const Sentence &>(*m_source);
|
||||
m_ioWrapper.OutputDetailedTranslationReport(bestHypo, sentence, lineNumber);
|
||||
}
|
||||
|
||||
// n-best
|
||||
|
@ -19,20 +19,20 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "HypoList.h"
|
||||
#include "Word.h"
|
||||
#include "WordsRange.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ChartHypothesisCollection;
|
||||
class Word;
|
||||
|
||||
class ChartCellLabel
|
||||
{
|
||||
public:
|
||||
ChartCellLabel(const WordsRange &coverage, const Word &label,
|
||||
const ChartHypothesisCollection *stack=NULL)
|
||||
const HypoList *stack=NULL)
|
||||
: m_coverage(coverage)
|
||||
, m_label(label)
|
||||
, m_stack(stack)
|
||||
@ -40,7 +40,7 @@ class ChartCellLabel
|
||||
|
||||
const WordsRange &GetCoverage() const { return m_coverage; }
|
||||
const Word &GetLabel() const { return m_label; }
|
||||
const ChartHypothesisCollection *GetStack() const { return m_stack; }
|
||||
const HypoList *GetStack() const { return m_stack; }
|
||||
|
||||
bool operator<(const ChartCellLabel &other) const
|
||||
{
|
||||
@ -55,7 +55,7 @@ class ChartCellLabel
|
||||
private:
|
||||
const WordsRange &m_coverage;
|
||||
const Word &m_label;
|
||||
const ChartHypothesisCollection *m_stack;
|
||||
const HypoList *m_stack;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -55,9 +55,10 @@ class ChartCellLabelSet
|
||||
m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w)));
|
||||
}
|
||||
|
||||
void AddConstituent(const Word &w, const ChartHypothesisCollection &stack)
|
||||
void AddConstituent(const Word &w, const ChartHypothesisCollection &coll)
|
||||
{
|
||||
m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w, &stack)));
|
||||
const HypoList *stack = &(coll.GetSortedHypotheses());
|
||||
m_map.insert(std::make_pair(w, ChartCellLabel(m_coverage, w, stack)));
|
||||
}
|
||||
|
||||
bool Empty() const { return m_map.empty(); }
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
@ -45,7 +44,6 @@ ChartHypothesis::ChartHypothesis(const ChartTranslationOption &transOpt,
|
||||
const RuleCubeItem &item,
|
||||
ChartManager &manager)
|
||||
:m_targetPhrase(*(item.GetTranslationDimension().GetTargetPhrase()))
|
||||
,m_transOpt(transOpt)
|
||||
,m_currSourceWordsRange(transOpt.GetSourceWordsRange())
|
||||
,m_ffStates(manager.GetTranslationSystem()->GetStatefulFeatureFunctions().size())
|
||||
,m_arcList(NULL)
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
@ -48,7 +47,6 @@ protected:
|
||||
#endif
|
||||
|
||||
const TargetPhrase &m_targetPhrase;
|
||||
const ChartTranslationOption &m_transOpt;
|
||||
|
||||
WordsRange m_currSourceWordsRange;
|
||||
std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
|
||||
@ -92,9 +90,6 @@ public:
|
||||
|
||||
unsigned GetId() const { return m_id; }
|
||||
|
||||
const ChartTranslationOption &GetTranslationOption()const {
|
||||
return m_transOpt;
|
||||
}
|
||||
const TargetPhrase &GetCurrTargetPhrase()const {
|
||||
return m_targetPhrase;
|
||||
}
|
||||
|
69
moses/src/ChartRuleLookupManagerCYKPlus.cpp
Normal file
69
moses/src/ChartRuleLookupManagerCYKPlus.cpp
Normal file
@ -0,0 +1,69 @@
|
||||
/***********************************************************************
|
||||
Moses - statistical machine translation system
|
||||
Copyright (C) 2006-2012 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "ChartRuleLookupManagerCYKPlus.h"
|
||||
|
||||
#include "PhraseDictionarySCFG.h"
|
||||
#include "InputType.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "CellCollection.h"
|
||||
#include "DotChartInMemory.h"
|
||||
#include "StaticData.h"
|
||||
#include "NonTerminal.h"
|
||||
#include "ChartCellCollection.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
void ChartRuleLookupManagerCYKPlus::AddCompletedRule(
|
||||
const DottedRule &dottedRule,
|
||||
const TargetPhraseCollection &tpc,
|
||||
size_t ruleLimit,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl)
|
||||
{
|
||||
// Determine the rule's rank.
|
||||
size_t rank = 0;
|
||||
const DottedRule *node = &dottedRule;
|
||||
while (!node->IsRoot()) {
|
||||
if (node->IsNonTerminal()) {
|
||||
++rank;
|
||||
}
|
||||
node = node->GetPrev();
|
||||
}
|
||||
|
||||
// Fill m_stackVec with a stack pointer for each non-terminal.
|
||||
m_stackVec.resize(rank);
|
||||
node = &dottedRule;
|
||||
while (rank > 0) {
|
||||
if (node->IsNonTerminal()) {
|
||||
// add the score of the best underlying hypothesis
|
||||
const ChartCellLabel &cellLabel = node->GetChartCellLabel();
|
||||
const HypoList *stack = cellLabel.GetStack();
|
||||
assert(stack);
|
||||
m_stackVec[--rank] = stack;
|
||||
}
|
||||
node = node->GetPrev();
|
||||
}
|
||||
|
||||
// Add the (TargetPhraseCollection, StackVec) pair to the collection.
|
||||
outColl.Add(tpc, m_stackVec, adhereTableLimit, ruleLimit);
|
||||
}
|
||||
|
||||
} // namespace Moses
|
53
moses/src/ChartRuleLookupManagerCYKPlus.h
Normal file
53
moses/src/ChartRuleLookupManagerCYKPlus.h
Normal file
@ -0,0 +1,53 @@
|
||||
/***********************************************************************
|
||||
Moses - statistical machine translation system
|
||||
Copyright (C) 2006-2012 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
#ifndef moses_ChartRuleLookupManagerCYKPlus_h
|
||||
#define moses_ChartRuleLookupManagerCYKPlus_h
|
||||
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "StackVec.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class DottedRule;
|
||||
class TargetPhraseCollection;
|
||||
|
||||
class ChartRuleLookupManagerCYKPlus : public ChartRuleLookupManager
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManagerCYKPlus(const InputType &sentence,
|
||||
const ChartCellCollection &cellColl)
|
||||
: ChartRuleLookupManager(sentence, cellColl) {}
|
||||
|
||||
protected:
|
||||
void AddCompletedRule(
|
||||
const DottedRule &dottedRule,
|
||||
const TargetPhraseCollection &tpc,
|
||||
size_t ruleLimit,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
StackVec m_stackVec;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
#endif
|
@ -35,7 +35,7 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
|
||||
const InputType &src,
|
||||
const ChartCellCollection &cellColl,
|
||||
const PhraseDictionarySCFG &ruleTable)
|
||||
: ChartRuleLookupManager(src, cellColl)
|
||||
: ChartRuleLookupManagerCYKPlus(src, cellColl)
|
||||
, m_ruleTable(ruleTable)
|
||||
{
|
||||
CHECK(m_dottedRuleColls.size() == 0);
|
||||
@ -152,25 +152,24 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
|
||||
DottedRuleList &rules = dottedRuleCol.Get(relEndPos + 1);
|
||||
|
||||
// look up target sides for the rules
|
||||
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
|
||||
const size_t ruleLimit = StaticData::Instance().GetRuleLimit();
|
||||
DottedRuleList::const_iterator iterRule;
|
||||
for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) {
|
||||
const DottedRuleInMemory &dottedRule = **iterRule;
|
||||
const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode();
|
||||
|
||||
// look up target sides
|
||||
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
|
||||
const TargetPhraseCollection *tpc = node.GetTargetPhraseCollection();
|
||||
|
||||
// add the fully expanded rule (with lexical target side)
|
||||
if (targetPhraseCollection != NULL) {
|
||||
outColl.Add(*targetPhraseCollection, dottedRule,
|
||||
GetCellCollection(), adhereTableLimit, rulesLimit);
|
||||
if (tpc != NULL) {
|
||||
AddCompletedRule(dottedRule, *tpc, ruleLimit, adhereTableLimit, outColl);
|
||||
}
|
||||
}
|
||||
|
||||
dottedRuleCol.Clear(relEndPos+1);
|
||||
|
||||
outColl.CreateChartRules(rulesLimit);
|
||||
outColl.CreateChartRules(ruleLimit);
|
||||
}
|
||||
|
||||
// Given a partial rule application ending at startPos-1 and given the sets of
|
||||
|
@ -27,11 +27,12 @@
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#endif
|
||||
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "ChartRuleLookupManagerCYKPlus.h"
|
||||
#include "DotChartInMemory.h"
|
||||
#include "NonTerminal.h"
|
||||
#include "PhraseDictionaryNodeSCFG.h"
|
||||
#include "PhraseDictionarySCFG.h"
|
||||
#include "StackVec.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -41,9 +42,9 @@ class DottedRuleColl;
|
||||
class WordsRange;
|
||||
|
||||
// Implementation of ChartRuleLookupManager for in-memory rule tables.
|
||||
class ChartRuleLookupManagerMemory : public ChartRuleLookupManager
|
||||
class ChartRuleLookupManagerMemory : public ChartRuleLookupManagerCYKPlus
|
||||
{
|
||||
public:
|
||||
public:
|
||||
ChartRuleLookupManagerMemory(const InputType &sentence,
|
||||
const ChartCellCollection &cellColl,
|
||||
const PhraseDictionarySCFG &ruleTable);
|
||||
@ -55,7 +56,7 @@ public:
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
private:
|
||||
private:
|
||||
void ExtendPartialRuleApplication(
|
||||
const DottedRuleInMemory &prevDottedRule,
|
||||
size_t startPos,
|
||||
|
@ -43,7 +43,7 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk(
|
||||
const std::vector<FactorType> &outputFactorsVec,
|
||||
const std::vector<float> &weight,
|
||||
const std::string &filePath)
|
||||
: ChartRuleLookupManager(sentence, cellColl)
|
||||
: ChartRuleLookupManagerCYKPlus(sentence, cellColl)
|
||||
, m_dictionary(dictionary)
|
||||
, m_dbWrapper(dbWrapper)
|
||||
, m_languageModels(languageModels)
|
||||
@ -260,8 +260,8 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
|
||||
|
||||
CHECK(targetPhraseCollection);
|
||||
if (!targetPhraseCollection->IsEmpty()) {
|
||||
outColl.Add(*targetPhraseCollection, prevDottedRule,
|
||||
GetCellCollection(), adhereTableLimit, rulesLimit);
|
||||
AddCompletedRule(prevDottedRule, *targetPhraseCollection,
|
||||
rulesLimit, adhereTableLimit, outColl);
|
||||
}
|
||||
|
||||
} // if (node)
|
||||
|
@ -23,7 +23,7 @@
|
||||
|
||||
#include "../../OnDiskPt/OnDiskWrapper.h"
|
||||
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "ChartRuleLookupManagerCYKPlus.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "DotChartOnDisk.h"
|
||||
#include "InputType.h"
|
||||
@ -33,9 +33,9 @@ namespace Moses
|
||||
{
|
||||
|
||||
// Implementation of ChartRuleLookupManager for on-disk rule tables.
|
||||
class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManager
|
||||
class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManagerCYKPlus
|
||||
{
|
||||
public:
|
||||
public:
|
||||
ChartRuleLookupManagerOnDisk(const InputType &sentence,
|
||||
const ChartCellCollection &cellColl,
|
||||
const PhraseDictionaryOnDisk &dictionary,
|
||||
@ -53,7 +53,7 @@ public:
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
private:
|
||||
private:
|
||||
const PhraseDictionaryOnDisk &m_dictionary;
|
||||
OnDiskPt::OnDiskWrapper &m_dbWrapper;
|
||||
const LMList *m_languageModels;
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
@ -20,33 +19,23 @@
|
||||
|
||||
#include "ChartTranslationOption.h"
|
||||
|
||||
#include "AlignmentInfo.h"
|
||||
#include "ChartCellCollection.h"
|
||||
#include "DotChart.h"
|
||||
|
||||
#include <vector>
|
||||
#include "ChartHypothesis.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
void ChartTranslationOption::CalcEstimateOfBestScore(
|
||||
const ChartCellCollection &allChartCells)
|
||||
void ChartTranslationOption::CalcEstimateOfBestScore()
|
||||
{
|
||||
const TargetPhrase &targetPhrase = **(m_targetPhraseCollection.begin());
|
||||
m_estimateOfBestScore = targetPhrase.GetFutureScore();
|
||||
|
||||
const DottedRule *rule = &m_dottedRule;
|
||||
|
||||
// only deal with non-terminals
|
||||
while (!rule->IsRoot()) {
|
||||
if (rule->IsNonTerminal()) {
|
||||
// add the score of the best underlying hypothesis
|
||||
const ChartCellLabel &cellLabel = rule->GetChartCellLabel();
|
||||
const ChartHypothesisCollection *hypoColl = cellLabel.GetStack();
|
||||
CHECK(hypoColl);
|
||||
m_estimateOfBestScore += hypoColl->GetBestScore();
|
||||
}
|
||||
rule = rule->GetPrev();
|
||||
for (StackVec::const_iterator p = m_stackVec.begin(); p != m_stackVec.end();
|
||||
++p) {
|
||||
const HypoList *stack = *p;
|
||||
assert(stack);
|
||||
assert(!stack->empty());
|
||||
const ChartHypothesis &bestHypo = **(stack->begin());
|
||||
m_estimateOfBestScore += bestHypo.GetTotalScore();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
@ -20,6 +19,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "StackVec.h"
|
||||
#include "TargetPhrase.h"
|
||||
#include "TargetPhraseCollection.h"
|
||||
#include "WordsRange.h"
|
||||
@ -30,29 +30,25 @@
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class DottedRule;
|
||||
class ChartCellCollection;
|
||||
|
||||
// Similar to a DottedRule, but contains a direct reference to a list
|
||||
// of translations and provdes an estimate of the best score.
|
||||
class ChartTranslationOption
|
||||
{
|
||||
public:
|
||||
ChartTranslationOption(const TargetPhraseCollection &targetPhraseColl,
|
||||
const DottedRule &dottedRule,
|
||||
const WordsRange &wordsRange,
|
||||
const ChartCellCollection &allChartCells)
|
||||
: m_dottedRule(dottedRule)
|
||||
, m_targetPhraseCollection(targetPhraseColl)
|
||||
, m_wordsRange(wordsRange)
|
||||
, m_estimateOfBestScore(0)
|
||||
const StackVec &stackVec,
|
||||
const WordsRange &wordsRange)
|
||||
: m_stackVec(stackVec)
|
||||
, m_targetPhraseCollection(targetPhraseColl)
|
||||
, m_wordsRange(wordsRange)
|
||||
, m_estimateOfBestScore(0)
|
||||
{
|
||||
CalcEstimateOfBestScore(allChartCells);
|
||||
CalcEstimateOfBestScore();
|
||||
}
|
||||
|
||||
~ChartTranslationOption() {}
|
||||
|
||||
const DottedRule &GetDottedRule() const { return m_dottedRule; }
|
||||
const StackVec &GetStackVec() const { return m_stackVec; }
|
||||
|
||||
const TargetPhraseCollection &GetTargetPhraseCollection() const {
|
||||
return m_targetPhraseCollection;
|
||||
@ -71,9 +67,9 @@ class ChartTranslationOption
|
||||
// not implemented
|
||||
ChartTranslationOption &operator=(const ChartTranslationOption &);
|
||||
|
||||
void CalcEstimateOfBestScore(const ChartCellCollection &);
|
||||
void CalcEstimateOfBestScore();
|
||||
|
||||
const DottedRule &m_dottedRule;
|
||||
const StackVec m_stackVec;
|
||||
const TargetPhraseCollection &m_targetPhraseCollection;
|
||||
const WordsRange &m_wordsRange;
|
||||
float m_estimateOfBestScore;
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
@ -59,15 +58,6 @@ ChartTranslationOptionCollection::~ChartTranslationOptionCollection()
|
||||
{
|
||||
RemoveAllInColl(m_unksrcs);
|
||||
RemoveAllInColl(m_cacheTargetPhraseCollection);
|
||||
|
||||
std::list<std::vector<DottedRule*>* >::iterator iterOuter;
|
||||
for (iterOuter = m_dottedRuleCache.begin(); iterOuter != m_dottedRuleCache.end(); ++iterOuter) {
|
||||
std::vector<DottedRule*> &inner = **iterOuter;
|
||||
RemoveAllInColl(inner);
|
||||
}
|
||||
|
||||
RemoveAllInColl(m_dottedRuleCache);
|
||||
|
||||
}
|
||||
|
||||
void ChartTranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
@ -203,12 +193,6 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
|
||||
|
||||
//TranslationOption *transOpt;
|
||||
if (! staticData.GetDropUnknown() || isDigit) {
|
||||
// create dotted rules
|
||||
std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
|
||||
m_dottedRuleCache.push_back(dottedRuleList);
|
||||
dottedRuleList->push_back(new DottedRule());
|
||||
dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
|
||||
|
||||
// loop
|
||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
||||
UnknownLHSList::const_iterator iterLHS;
|
||||
@ -243,9 +227,8 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
|
||||
|
||||
// chart rule
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
|
||||
, *dottedRuleList->back()
|
||||
, range
|
||||
, m_hypoStackColl);
|
||||
, m_emptyStackVec
|
||||
, range);
|
||||
transOptColl.Add(chartRule);
|
||||
} // for (iterLHS
|
||||
} else {
|
||||
@ -271,17 +254,10 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
|
||||
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
|
||||
// words consumed
|
||||
std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
|
||||
m_dottedRuleCache.push_back(dottedRuleList);
|
||||
dottedRuleList->push_back(new DottedRule());
|
||||
dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
|
||||
|
||||
// chart rule
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
|
||||
, *dottedRuleList->back()
|
||||
, range
|
||||
, m_hypoStackColl);
|
||||
, m_emptyStackVec
|
||||
, range);
|
||||
transOptColl.Add(chartRule);
|
||||
}
|
||||
}
|
||||
|
@ -1,4 +1,3 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
@ -26,6 +25,7 @@
|
||||
#include "DecodeGraph.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "StackVec.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -49,7 +49,7 @@ protected:
|
||||
std::vector< std::vector< ChartTranslationOptionList > > m_collection; /*< contains translation options */
|
||||
std::vector<Phrase*> m_unksrcs;
|
||||
std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection;
|
||||
std::list<std::vector<DottedRule*>* > m_dottedRuleCache;
|
||||
StackVec m_emptyStackVec;
|
||||
|
||||
// for adding 1 trans opt in unknown word proc
|
||||
void Add(ChartTranslationOption *transOpt, size_t pos);
|
||||
|
@ -54,8 +54,7 @@ public:
|
||||
};
|
||||
|
||||
void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseCollection
|
||||
, const DottedRule &dottedRule
|
||||
, const ChartCellCollection &chartCellColl
|
||||
, const StackVec &stackVec
|
||||
, bool /* adhereTableLimit */
|
||||
, size_t ruleLimit)
|
||||
{
|
||||
@ -66,15 +65,14 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseC
|
||||
if (m_collection.size() < ruleLimit) {
|
||||
// not yet filled out quota. add everything
|
||||
ChartTranslationOption *option = new ChartTranslationOption(
|
||||
targetPhraseCollection, dottedRule, m_range, chartCellColl);
|
||||
targetPhraseCollection, stackVec, m_range);
|
||||
m_collection.push_back(option);
|
||||
float score = option->GetEstimateOfBestScore();
|
||||
m_scoreThreshold = (score < m_scoreThreshold) ? score : m_scoreThreshold;
|
||||
}
|
||||
else {
|
||||
// full but not bursting. add if better than worst score
|
||||
ChartTranslationOption option(targetPhraseCollection, dottedRule,
|
||||
m_range, chartCellColl);
|
||||
ChartTranslationOption option(targetPhraseCollection, stackVec, m_range);
|
||||
float score = option.GetEstimateOfBestScore();
|
||||
if (score > m_scoreThreshold) {
|
||||
// dynamic allocation deferred until here on the assumption that most
|
||||
|
@ -100,8 +100,7 @@ public:
|
||||
}
|
||||
|
||||
void Add(const TargetPhraseCollection &targetPhraseCollection
|
||||
, const DottedRule &dottedRule
|
||||
, const ChartCellCollection &
|
||||
, const StackVec &stackVec
|
||||
, bool ruleLimit
|
||||
, size_t tableLimit);
|
||||
void Add(ChartTranslationOption *transOpt);
|
||||
|
@ -78,10 +78,6 @@ Phrase ChartTrellisNode::GetOutputPhrase() const
|
||||
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
|
||||
Phrase ret(ARRAY_SIZE_INCR);
|
||||
|
||||
const ChartTranslationOption &transOpt = m_hypo.GetTranslationOption();
|
||||
|
||||
VERBOSE(3, "Trans Opt:" << transOpt.GetDottedRule() << ": " << m_hypo.GetCurrTargetPhrase().GetTargetLHS() << "->" << m_hypo.GetCurrTargetPhrase() << std::endl);
|
||||
|
||||
const Phrase &currTargetPhrase = m_hypo.GetCurrTargetPhrase();
|
||||
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =
|
||||
m_hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap();
|
||||
|
@ -26,8 +26,6 @@ namespace Moses
|
||||
|
||||
class DottedRule
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const DottedRule &);
|
||||
|
||||
public:
|
||||
// used only to init dot stack.
|
||||
DottedRule()
|
||||
|
@ -54,16 +54,4 @@ void DottedRuleStackOnDisk::SortSavedNodes()
|
||||
sort(m_savedNode.begin(), m_savedNode.end(), SavedNodesOderer());
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const DottedRuleCollOnDisk &coll)
|
||||
{
|
||||
DottedRuleCollOnDisk::CollType::const_iterator iter;
|
||||
for (iter = coll.begin(); iter != coll.end(); ++iter) {
|
||||
const DottedRuleOnDisk &rule = **iter;
|
||||
out << rule << endl;
|
||||
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -60,8 +60,6 @@ class DottedRuleOnDisk : public DottedRule
|
||||
|
||||
class DottedRuleCollOnDisk
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const DottedRuleCollOnDisk&);
|
||||
|
||||
protected:
|
||||
typedef std::vector<const DottedRuleOnDisk*> CollType;
|
||||
CollType m_coll;
|
||||
|
@ -1,7 +1,6 @@
|
||||
// $Id$
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
Moses - statistical machine translation system
|
||||
Copyright (C) 2006-2012 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
@ -18,20 +17,15 @@
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "DotChart.h"
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
std::ostream &operator<<(std::ostream &out, const DottedRule &rule)
|
||||
{
|
||||
if (!rule.IsRoot()) {
|
||||
out << rule.GetWordsRange() << "=" << rule.GetSourceWord() << " ";
|
||||
if (!rule.m_prev->IsRoot()) {
|
||||
out << " " << *rule.m_prev;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
class ChartHypothesis;
|
||||
|
||||
typedef std::vector<const ChartHypothesis*> HypoList;
|
||||
|
||||
}
|
@ -21,7 +21,6 @@
|
||||
#include "ChartCellCollection.h"
|
||||
#include "ChartTranslationOption.h"
|
||||
#include "ChartTranslationOptionCollection.h"
|
||||
#include "DotChart.h"
|
||||
#include "RuleCubeItem.h"
|
||||
#include "RuleCubeQueue.h"
|
||||
#include "WordsRange.h"
|
||||
@ -39,12 +38,12 @@ std::size_t hash_value(const HypothesisDimension &dimension)
|
||||
}
|
||||
|
||||
RuleCubeItem::RuleCubeItem(const ChartTranslationOption &transOpt,
|
||||
const ChartCellCollection &allChartCells)
|
||||
const ChartCellCollection &/*allChartCells*/)
|
||||
: m_translationDimension(0,
|
||||
transOpt.GetTargetPhraseCollection().GetCollection())
|
||||
, m_hypothesis(0)
|
||||
{
|
||||
CreateHypothesisDimensions(transOpt.GetDottedRule(), allChartCells);
|
||||
CreateHypothesisDimensions(transOpt.GetStackVec());
|
||||
}
|
||||
|
||||
// create the RuleCube from an existing one, differing only in one dimension
|
||||
@ -94,31 +93,19 @@ ChartHypothesis *RuleCubeItem::ReleaseHypothesis()
|
||||
|
||||
// for each non-terminal, create a ordered list of matching hypothesis from the
|
||||
// chart
|
||||
void RuleCubeItem::CreateHypothesisDimensions(
|
||||
const DottedRule &dottedRule,
|
||||
const ChartCellCollection &allChartCells)
|
||||
void RuleCubeItem::CreateHypothesisDimensions(const StackVec &stackVec)
|
||||
{
|
||||
CHECK(!dottedRule.IsRoot());
|
||||
|
||||
const DottedRule *prev = dottedRule.GetPrev();
|
||||
if (!prev->IsRoot()) {
|
||||
CreateHypothesisDimensions(*prev, allChartCells);
|
||||
}
|
||||
|
||||
// only deal with non-terminals
|
||||
if (dottedRule.IsNonTerminal()) {
|
||||
// get a sorted list of the underlying hypotheses
|
||||
const ChartCellLabel &cellLabel = dottedRule.GetChartCellLabel();
|
||||
const ChartHypothesisCollection *hypoColl = cellLabel.GetStack();
|
||||
CHECK(hypoColl);
|
||||
const HypoList &hypoList = hypoColl->GetSortedHypotheses();
|
||||
for (StackVec::const_iterator p = stackVec.begin(); p != stackVec.end();
|
||||
++p) {
|
||||
const HypoList *stack = *p;
|
||||
assert(stack);
|
||||
|
||||
// there have to be hypothesis with the desired non-terminal
|
||||
// (otherwise the rule would not be considered)
|
||||
CHECK(!hypoList.empty());
|
||||
assert(!stack->empty());
|
||||
|
||||
// create a list of hypotheses that match the non-terminal
|
||||
HypothesisDimension dimension(0, hypoList);
|
||||
HypothesisDimension dimension(0, *stack);
|
||||
// add them to the vector for such lists
|
||||
m_hypothesisDimensions.push_back(dimension);
|
||||
}
|
||||
|
@ -19,6 +19,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "StackVec.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace Moses
|
||||
@ -28,7 +30,6 @@ class ChartCellCollection;
|
||||
class ChartHypothesis;
|
||||
class ChartManager;
|
||||
class ChartTranslationOption;
|
||||
class DottedRule;
|
||||
class TargetPhrase;
|
||||
|
||||
typedef std::vector<const ChartHypothesis*> HypoList;
|
||||
@ -130,8 +131,7 @@ class RuleCubeItem
|
||||
RuleCubeItem(const RuleCubeItem &); // Not implemented
|
||||
RuleCubeItem &operator=(const RuleCubeItem &); // Not implemented
|
||||
|
||||
void CreateHypothesisDimensions(const DottedRule &,
|
||||
const ChartCellCollection &);
|
||||
void CreateHypothesisDimensions(const StackVec &);
|
||||
|
||||
TranslationDimension m_translationDimension;
|
||||
std::vector<HypothesisDimension> m_hypothesisDimensions;
|
||||
|
31
moses/src/StackVec.h
Normal file
31
moses/src/StackVec.h
Normal file
@ -0,0 +1,31 @@
|
||||
/***********************************************************************
|
||||
Moses - statistical machine translation system
|
||||
Copyright (C) 2006-2012 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "HypoList.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
typedef std::vector<const HypoList *> StackVec;
|
||||
|
||||
}
|
Loading…
Reference in New Issue
Block a user