moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

moses_chart: merge DottedRule and CoveredChartSpan classes. This saves

Browse Source 
some memory for models that require a lot of lookup state (generally
grammars with lots of target categories).

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4078 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
pjwilliams 2011-07-18 21:44:27 +00:00
parent 1190b75528
commit beba4b475f
24 changed files with 299 additions and 419 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
moses-chart-cmd/src/IOWrapper.cpp
View File

@ -45,7 +45,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "ChartTrellisPath.h"
#include "ChartTranslationOption.h"
#include "ChartHypothesis.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
using namespace std;
@ -229,7 +229,7 @@ void OutputTranslationOptions(std::ostream &out, const ChartHypothesis *hypo, lo
if (hypo != NULL) {
out << "Trans Opt " << translationId
<< " " << hypo->GetCurrSourceRange()
<< ": " << hypo->GetTranslationOption().GetLastCoveredChartSpan()
<< ": " << hypo->GetTranslationOption().GetDottedRule()
<< ": " << hypo->GetCurrTargetPhrase().GetTargetLHS()
<< "->" << hypo->GetCurrTargetPhrase()
<< " " << hypo->GetTotalScore() << hypo->GetScoreBreakdown()

1
moses/src/ChartCell.h
View File

@ -30,7 +30,6 @@
#include "NonTerminal.h"
#include "ChartHypothesis.h"
#include "ChartHypothesisCollection.h"
#include "CoveredChartSpan.h"
#include "RuleCube.h"
#include "ChartCellLabelSet.h"

79
moses/src/ChartRuleLookupManagerMemory.cpp
View File

@ -23,7 +23,7 @@
#include "InputType.h"
#include "ChartTranslationOptionList.h"
#include "CellCollection.h"
#include "DotChart.h"
#include "DotChartInMemory.h"
#include "StaticData.h"
#include "NonTerminal.h"
#include "ChartCellCollection.h"
@ -46,10 +46,10 @@ ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
for (size_t ind = 0; ind < m_dottedRuleColls.size(); ++ind) {
#ifdef USE_BOOST_POOL
DottedRule *initDottedRule = m_dottedRulePool.malloc();
new (initDottedRule) DottedRule(rootNode);
DottedRuleInMemory *initDottedRule = m_dottedRulePool.malloc();
new (initDottedRule) DottedRuleInMemory(rootNode);
#else
DottedRule *initDottedRule = new DottedRule(rootNode);
DottedRuleInMemory *initDottedRule = new DottedRuleInMemory(rootNode);
#endif
DottedRuleColl *dottedRuleColl = new DottedRuleColl(sourceSize - ind + 1);
@ -85,15 +85,11 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
// through calls to ExtendPartialRuleApplication())
for (size_t ind = 0; ind < expandableDottedRuleList.size(); ++ind) {
// rule we are about to extend
const DottedRule &prevDottedRule = *expandableDottedRuleList[ind];
// note where it was found in the prefix tree of the rule dictionary
const PhraseDictionaryNodeSCFG &prevNode = prevDottedRule.GetLastNode();
// look up end position of the span it covers
const CoveredChartSpan *prevCoveredChartSpan = prevDottedRule.GetLastCoveredChartSpan();
const DottedRuleInMemory &prevDottedRule = *expandableDottedRuleList[ind];
// we will now try to extend it, starting after where it ended
// (note: prevCoveredChartSpan == NULL matches for the dummy rule
// at root of the prefix tree)
size_t startPos = (prevCoveredChartSpan == NULL) ? range.GetStartPos() : prevCoveredChartSpan->GetWordsRange().GetEndPos() + 1;
size_t startPos = prevDottedRule.IsRoot()
? range.GetStartPos()
: prevDottedRule.GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
// (if only one more word position needs to be covered)
@ -102,20 +98,19 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
// look up in rule dictionary, if the current rule can be extended
// with the source word in the last position
const Word &sourceWord = sourceWordLabel.GetLabel();
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceWord);
const PhraseDictionaryNodeSCFG *node = prevDottedRule.GetLastNode().GetChild(sourceWord);
// if we found a new rule -> create it and add it to the list
if (node != NULL) {
// create the rule
#ifdef USE_BOOST_POOL
CoveredChartSpan *newCoveredChartSpan = m_coveredChartSpanPool.malloc();
new (newCoveredChartSpan) CoveredChartSpan(sourceWordLabel, prevCoveredChartSpan);
DottedRule *dottedRule = m_dottedRulePool.malloc();
new (dottedRule) DottedRule(*node, newCoveredChartSpan);
DottedRuleInMemory *dottedRule = m_dottedRulePool.malloc();
new (dottedRule) DottedRuleInMemory(*node, sourceWordLabel,
prevDottedRule);
#else
CoveredChartSpan *newCoveredChartSpan = new CoveredChartSpan(sourceWordLabel, prevCoveredChartSpan);
DottedRule *dottedRule = new DottedRule(*node,
newCoveredChartSpan);
DottedRuleInMemory *dottedRule = new DottedRuleInMemory(*node,
sourceWordLabel,
prevDottedRule);
#endif
dottedRuleCol.Add(relEndPos+1, dottedRule);
}
@ -149,8 +144,8 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
}
ExtendPartialRuleApplication(prevNode, prevCoveredChartSpan, startPos,
endPos, stackInd, dottedRuleCol);
ExtendPartialRuleApplication(prevDottedRule, startPos, endPos, stackInd,
dottedRuleCol);
}
// list of rules that that cover the entire span
@ -160,17 +155,15 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
DottedRuleList::const_iterator iterRule;
for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) {
const DottedRule &dottedRule = **iterRule;
const DottedRuleInMemory &dottedRule = **iterRule;
const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode();
const CoveredChartSpan *coveredChartSpan = dottedRule.GetLastCoveredChartSpan();
assert(coveredChartSpan);
// look up target sides
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
// add the fully expanded rule (with lexical target side)
if (targetPhraseCollection != NULL) {
outColl.Add(*targetPhraseCollection, *coveredChartSpan,
outColl.Add(*targetPhraseCollection, dottedRule,
GetCellCollection(), adhereTableLimit, rulesLimit);
}
}
@ -185,8 +178,7 @@ void ChartRuleLookupManagerMemory::GetChartRuleCollection(
// determines the full or partial rule applications that can be produced through
// extending the current rule application by a single non-terminal.
void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
const PhraseDictionaryNodeSCFG & node,
const CoveredChartSpan *prevCoveredChartSpan,
const DottedRuleInMemory &prevDottedRule,
size_t startPos,
size_t endPos,
size_t stackInd,
@ -200,6 +192,9 @@ void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
const ChartCellLabelSet &targetNonTerms =
GetCellCollection().Get(WordsRange(startPos, endPos)).GetTargetLabelSet();
// note where it was found in the prefix tree of the rule dictionary
const PhraseDictionaryNodeSCFG &node = prevDottedRule.GetLastNode();
const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap =
node.GetNonTerminalMap();
@ -243,13 +238,11 @@ void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
// create new rule
#ifdef USE_BOOST_POOL
CoveredChartSpan *wc = m_coveredChartSpanPool.malloc();
new (wc) CoveredChartSpan(cellLabel, prevCoveredChartSpan);
DottedRule *rule = m_dottedRulePool.malloc();
new (rule) DottedRule(*child, wc);
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
new (rule) DottedRuleInMemory(*child, cellLabel, prevDottedRule);
#else
CoveredChartSpan * wc = new CoveredChartSpan(cellLabel, prevCoveredChartSpan);
DottedRule * rule = new DottedRule(*child, wc);
DottedRuleInMemory *rule = new DottedRuleInMemory(*child, cellLabel,
prevDottedRule);
#endif
dottedRuleColl.Add(stackInd, rule);
}
@ -263,27 +256,25 @@ void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
nonTermMap.end();
for (p = nonTermMap.begin(); p != end; ++p) {
// does it match possible source and target non-terminals?
const PhraseDictionaryNodeSCFG::NonTerminalMapKey & key = p->first;
const Word & sourceNonTerm = key.first;
const PhraseDictionaryNodeSCFG::NonTerminalMapKey &key = p->first;
const Word &sourceNonTerm = key.first;
if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end()) {
continue;
}
const Word & targetNonTerm = key.second;
const Word &targetNonTerm = key.second;
const ChartCellLabel *cellLabel = targetNonTerms.Find(targetNonTerm);
if (!cellLabel) {
continue;
}
// create new rule
const PhraseDictionaryNodeSCFG & child = p->second;
const PhraseDictionaryNodeSCFG &child = p->second;
#ifdef USE_BOOST_POOL
CoveredChartSpan *wc = m_coveredChartSpanPool.malloc();
new (wc) CoveredChartSpan(*cellLabel, prevCoveredChartSpan);
DottedRule *rule = m_dottedRulePool.malloc();
new (rule) DottedRule(child, wc);
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
new (rule) DottedRuleInMemory(child, *cellLabel, prevDottedRule);
#else
CoveredChartSpan * wc = new CoveredChartSpan(*cellLabel, prevCoveredChartSpan);
DottedRule * rule = new DottedRule(child, wc);
DottedRuleInMemory *rule = new DottedRuleInMemory(child, *cellLabel,
prevDottedRule);
#endif
dottedRuleColl.Add(stackInd, rule);
}

15
moses/src/ChartRuleLookupManagerMemory.h
View File

@ -31,11 +31,10 @@
#endif
#include "ChartRuleLookupManager.h"
#include "DotChart.h"
#include "DotChartInMemory.h"
#include "NonTerminal.h"
#include "PhraseDictionaryNodeSCFG.h"
#include "PhraseDictionarySCFG.h"
#include "CoveredChartSpan.h"
namespace Moses
{
@ -61,8 +60,7 @@ public:
private:
void ExtendPartialRuleApplication(
const PhraseDictionaryNodeSCFG &node,
const CoveredChartSpan *prevCoveredChartSpan,
const DottedRuleInMemory &prevDottedRule,
size_t startPos,
size_t endPos,
size_t stackInd,
@ -71,11 +69,10 @@ private:
std::vector<DottedRuleColl*> m_dottedRuleColls;
const PhraseDictionarySCFG &m_ruleTable;
#ifdef USE_BOOST_POOL
// Use object pools to allocate the DottedRule and CoveredChartSpan objects
// for this sentence. We allocate a lot of them and this has been seen to
// significantly improve performance, especially for multithreaded decoding.
boost::object_pool<DottedRule> m_dottedRulePool;
boost::object_pool<CoveredChartSpan> m_coveredChartSpanPool;
// Use an object pool to allocate the dotted rules for this sentence. We
// allocate a lot of them and this has been seen to significantly improve
// performance, especially for multithreaded decoding.
boost::object_pool<DottedRuleInMemory> m_dottedRulePool;
#endif
};

14
moses/src/ChartRuleLookupManagerOnDisk.cpp
View File

@ -106,8 +106,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
const DottedRuleOnDisk &prevDottedRule = savedNode.GetDottedRule();
const OnDiskPt::PhraseNode &prevNode = prevDottedRule.GetLastNode();
const CoveredChartSpan *prevCoveredChartSpan = prevDottedRule.GetLastCoveredChartSpan();
size_t startPos = (prevCoveredChartSpan == NULL) ? range.GetStartPos() : prevCoveredChartSpan->GetWordsRange().GetEndPos() + 1;
size_t startPos = prevDottedRule.IsRoot() ? range.GetStartPos() : prevDottedRule.GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
if (startPos == absEndPos) {
@ -119,8 +118,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
// TODO figure out why source word is needed from node, not from sentence
// prob to do with factors or non-term
//const Word &sourceWord = node->GetSourceWord();
CoveredChartSpan *newCoveredChartSpan = new CoveredChartSpan(sourceWordLabel, prevCoveredChartSpan);
DottedRuleOnDisk *dottedRule = new DottedRuleOnDisk(*node, newCoveredChartSpan);
DottedRuleOnDisk *dottedRule = new DottedRuleOnDisk(*node, sourceWordLabel, prevDottedRule);
expandableDottedRuleList.Add(relEndPos+1, dottedRule);
// cache for cleanup
@ -199,8 +197,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
// found matching entry
//const Word &sourceWord = node->GetSourceWord();
CoveredChartSpan *newCoveredChartSpan = new CoveredChartSpan(cellLabel, prevCoveredChartSpan);
DottedRuleOnDisk *dottedRule = new DottedRuleOnDisk(*node, newCoveredChartSpan);
DottedRuleOnDisk *dottedRule = new DottedRuleOnDisk(*node, cellLabel, prevDottedRule);
expandableDottedRuleList.Add(stackInd, dottedRule);
m_sourcePhraseNode.push_back(node);
@ -223,9 +220,6 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
continue;
prevDottedRule.Done(true);
const CoveredChartSpan *coveredChartSpan = prevDottedRule.GetLastCoveredChartSpan();
assert(coveredChartSpan);
const OnDiskPt::PhraseNode &prevNode = prevDottedRule.GetLastNode();
//get node for each source LHS
@ -266,7 +260,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
assert(targetPhraseCollection);
if (!targetPhraseCollection->IsEmpty()) {
outColl.Add(*targetPhraseCollection, *coveredChartSpan,
outColl.Add(*targetPhraseCollection, prevDottedRule,
GetCellCollection(), adhereTableLimit, rulesLimit);
}

46
moses/src/ChartTranslationOption.cpp
View File

@ -22,7 +22,7 @@
#include "AlignmentInfo.h"
#include "ChartCellCollection.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
#include <vector>
@ -30,34 +30,32 @@ namespace Moses
{
void ChartTranslationOption::CalcEstimateOfBestScore(
const CoveredChartSpan *coveredChartSpan,
const ChartCellCollection &allChartCells)
const ChartCellCollection &allChartCells)
{
// recurse through the linked list of source side non-terminals and terminals
const CoveredChartSpan *prevCoveredChartSpan =
coveredChartSpan->GetPrevCoveredChartSpan();
if (prevCoveredChartSpan)
{
CalcEstimateOfBestScore(prevCoveredChartSpan, allChartCells);
}
const TargetPhrase &targetPhrase = **(m_targetPhraseCollection.begin());
m_estimateOfBestScore = targetPhrase.GetFutureScore();
const DottedRule *rule = &m_dottedRule;
// only deal with non-terminals
if (coveredChartSpan->IsNonTerminal())
{
// get the essential information about the non-terminal
const WordsRange &childRange = coveredChartSpan->GetWordsRange();
const ChartCell &childCell = allChartCells.Get(childRange);
const Word &nonTerm = coveredChartSpan->GetSourceWord();
while (!rule->IsRoot()) {
if (rule->IsNonTerminal()) {
// get the essential information about the non-terminal
const WordsRange &childRange = rule->GetWordsRange();
const ChartCell &childCell = allChartCells.Get(childRange);
const Word &nonTerm = rule->GetSourceWord();
// there have to be hypotheses with the desired non-terminal
// (otherwise the rule would not be considered)
assert(!childCell.GetSortedHypotheses(nonTerm).empty());
// there have to be hypotheses with the desired non-terminal
// (otherwise the rule would not be considered)
assert(!childCell.GetSortedHypotheses(nonTerm).empty());
// create a list of hypotheses that match the non-terminal
const std::vector<const ChartHypothesis *> &stack =
childCell.GetSortedHypotheses(nonTerm);
const ChartHypothesis *hypo = stack[0];
m_estimateOfBestScore += hypo->GetTotalScore();
// create a list of hypotheses that match the non-terminal
const std::vector<const ChartHypothesis *> &stack =
childCell.GetSortedHypotheses(nonTerm);
const ChartHypothesis *hypo = stack[0];
m_estimateOfBestScore += hypo->GetTotalScore();
}
rule = rule->GetPrev();
}
}

19
moses/src/ChartTranslationOption.h
View File

@ -30,7 +30,7 @@
namespace Moses
{
class CoveredChartSpan;
class DottedRule;
class ChartCellCollection;
// Similar to a DottedRule, but contains a direct reference to a list
@ -39,24 +39,20 @@ class ChartTranslationOption
{
public:
ChartTranslationOption(const TargetPhraseCollection &targetPhraseColl,
const CoveredChartSpan &lastCoveredChartSpan,
const DottedRule &dottedRule,
const WordsRange &wordsRange,
const ChartCellCollection &allChartCells)
: m_lastCoveredChartSpan(lastCoveredChartSpan)
: m_dottedRule(dottedRule)
, m_targetPhraseCollection(targetPhraseColl)
, m_wordsRange(wordsRange)
, m_estimateOfBestScore(0)
{
const TargetPhrase &targetPhrase = **(m_targetPhraseCollection.begin());
m_estimateOfBestScore = targetPhrase.GetFutureScore();
CalcEstimateOfBestScore(&m_lastCoveredChartSpan, allChartCells);
CalcEstimateOfBestScore(allChartCells);
}
~ChartTranslationOption() {}
const CoveredChartSpan &GetLastCoveredChartSpan() const {
return m_lastCoveredChartSpan;
}
const DottedRule &GetDottedRule() const { return m_dottedRule; }
const TargetPhraseCollection &GetTargetPhraseCollection() const {
return m_targetPhraseCollection;
@ -75,10 +71,9 @@ class ChartTranslationOption
// not implemented
ChartTranslationOption &operator=(const ChartTranslationOption &);
void CalcEstimateOfBestScore(const CoveredChartSpan *,
const ChartCellCollection &);
void CalcEstimateOfBestScore(const ChartCellCollection &);
const CoveredChartSpan &m_lastCoveredChartSpan;
const DottedRule &m_dottedRule;
const TargetPhraseCollection &m_targetPhraseCollection;
const WordsRange &m_wordsRange;
float m_estimateOfBestScore;

34
moses/src/ChartTranslationOptionCollection.cpp
View File

@ -26,7 +26,7 @@
#include "StaticData.h"
#include "DecodeStep.h"
#include "DummyScoreProducers.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
#include "Util.h"
using namespace std;
@ -60,13 +60,13 @@ ChartTranslationOptionCollection::~ChartTranslationOptionCollection()
RemoveAllInColl(m_unksrcs);
RemoveAllInColl(m_cacheTargetPhraseCollection);
std::list<std::vector<CoveredChartSpan*>* >::iterator iterOuter;
for (iterOuter = m_coveredChartSpanCache.begin(); iterOuter != m_coveredChartSpanCache.end(); ++iterOuter) {
std::vector<CoveredChartSpan*> &inner = **iterOuter;
std::list<std::vector<DottedRule*>* >::iterator iterOuter;
for (iterOuter = m_dottedRuleCache.begin(); iterOuter != m_dottedRuleCache.end(); ++iterOuter) {
std::vector<DottedRule*> &inner = **iterOuter;
RemoveAllInColl(inner);
}
RemoveAllInColl(m_coveredChartSpanCache);
RemoveAllInColl(m_dottedRuleCache);
}
@ -203,13 +203,11 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
//TranslationOption *transOpt;
if (! staticData.GetDropUnknown() || isDigit) {
// words consumed
std::vector<CoveredChartSpan*> *coveredChartSpanList = new std::vector<CoveredChartSpan*>();
m_coveredChartSpanCache.push_back(coveredChartSpanList);
CoveredChartSpan *wc = new CoveredChartSpan(sourceWordLabel, NULL);
coveredChartSpanList->push_back(wc);
assert(coveredChartSpanList->size());
// create dotted rules
std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
m_dottedRuleCache.push_back(dottedRuleList);
dottedRuleList->push_back(new DottedRule());
dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
@ -245,7 +243,7 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
// chart rule
ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
, *coveredChartSpanList->back()
, *dottedRuleList->back()
, range
, m_hypoStackColl);
transOptColl.Add(chartRule);
@ -274,14 +272,14 @@ void ChartTranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceW
targetPhrase->SetTargetLHS(targetLHS);
// words consumed
std::vector<CoveredChartSpan*> *coveredChartSpanList = new std::vector<CoveredChartSpan*>;
m_coveredChartSpanCache.push_back(coveredChartSpanList);
coveredChartSpanList->push_back(new CoveredChartSpan(sourceWordLabel, NULL));
std::vector<DottedRule*> *dottedRuleList = new std::vector<DottedRule*>();
m_dottedRuleCache.push_back(dottedRuleList);
dottedRuleList->push_back(new DottedRule());
dottedRuleList->push_back(new DottedRule(sourceWordLabel, *(dottedRuleList->back())));
// chart rule
assert(coveredChartSpanList->size());
ChartTranslationOption *chartRule = new ChartTranslationOption(*tpc
, *coveredChartSpanList->back()
, *dottedRuleList->back()
, range
, m_hypoStackColl);
transOptColl.Add(chartRule);

4
moses/src/ChartTranslationOptionCollection.h
View File

@ -32,7 +32,7 @@ namespace Moses
class DecodeGraph;
class Word;
class ChartTranslationOption;
class CoveredChartSpan;
class DottedRule;
class WordPenaltyProducer;
class ChartCellCollection;
@ -49,7 +49,7 @@ protected:
std::vector< std::vector< ChartTranslationOptionList > > m_collection; /*< contains translation options */
std::vector<Phrase*> m_unksrcs;
std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection;
std::list<std::vector<CoveredChartSpan*>* > m_coveredChartSpanCache;
std::list<std::vector<DottedRule*>* > m_dottedRuleCache;
// for adding 1 trans opt in unknown word proc
void Add(ChartTranslationOption *transOpt, size_t pos);

6
moses/src/ChartTranslationOptionList.cpp
View File

@ -54,7 +54,7 @@ public:
};
void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseCollection
, const CoveredChartSpan &coveredChartSpan
, const DottedRule &dottedRule
, const ChartCellCollection &chartCellColl
, bool /* adhereTableLimit */
, size_t ruleLimit)
@ -66,14 +66,14 @@ void ChartTranslationOptionList::Add(const TargetPhraseCollection &targetPhraseC
if (m_collection.size() < ruleLimit) {
// not yet filled out quota. add everything
ChartTranslationOption *option = new ChartTranslationOption(
targetPhraseCollection, coveredChartSpan, m_range, chartCellColl);
targetPhraseCollection, dottedRule, m_range, chartCellColl);
m_collection.push_back(option);
float score = option->GetEstimateOfBestScore();
m_scoreThreshold = (score < m_scoreThreshold) ? score : m_scoreThreshold;
}
else {
// full but not bursting. add if better than worst score
ChartTranslationOption option(targetPhraseCollection, coveredChartSpan,
ChartTranslationOption option(targetPhraseCollection, dottedRule,
m_range, chartCellColl);
float score = option.GetEstimateOfBestScore();
if (score > m_scoreThreshold) {

2
moses/src/ChartTranslationOptionList.h
View File

@ -100,7 +100,7 @@ public:
}
void Add(const TargetPhraseCollection &targetPhraseCollection
, const CoveredChartSpan &coveredChartSpan
, const DottedRule &dottedRule
, const ChartCellCollection &
, bool ruleLimit
, size_t tableLimit);

4
moses/src/ChartTrellisNode.cpp
View File

@ -21,7 +21,7 @@
#include "ChartTrellisNode.h"
#include "ChartHypothesis.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
#include "ScoreComponentCollection.h"
#include "StaticData.h"
@ -104,7 +104,7 @@ Phrase ChartTrellisNode::GetOutputPhrase() const
const ChartTranslationOption &transOpt = m_hypo->GetTranslationOption();
VERBOSE(3, "Trans Opt:" << transOpt.GetLastCoveredChartSpan() << ": " << m_hypo->GetCurrTargetPhrase().GetTargetLHS() << "->" << m_hypo->GetCurrTargetPhrase() << std::endl);
VERBOSE(3, "Trans Opt:" << transOpt.GetDottedRule() << ": " << m_hypo->GetCurrTargetPhrase().GetTargetLHS() << "->" << m_hypo->GetCurrTargetPhrase() << std::endl);
const Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
const AlignmentInfo::NonTermIndexMap &nonTermIndexMap =

71
moses/src/CoveredChartSpan.h
View File

@ -1,71 +0,0 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#include <iostream>
#include "ChartCellLabel.h"
#include "WordsRange.h"
#include "Word.h"
namespace Moses
{
class CoveredChartSpan
{
friend std::ostream& operator<<(std::ostream&, const CoveredChartSpan&);
protected:
const ChartCellLabel &m_cellLabel;
const CoveredChartSpan *m_prevCoveredChartSpan;
public:
CoveredChartSpan(); // not implmented
CoveredChartSpan(const ChartCellLabel &cellLabel,
const CoveredChartSpan *prevCoveredChartSpan)
:m_cellLabel(cellLabel)
,m_prevCoveredChartSpan(prevCoveredChartSpan)
{}
const WordsRange &GetWordsRange() const {
return m_cellLabel.GetCoverage();
}
const Word &GetSourceWord() const {
return m_cellLabel.GetLabel();
}
bool IsNonTerminal() const {
return m_cellLabel.GetLabel().IsNonTerminal();
}
const CoveredChartSpan *GetPrevCoveredChartSpan() const {
return m_prevCoveredChartSpan;
}
//! transitive comparison used for adding objects into FactorCollection
inline bool operator<(const CoveredChartSpan &compare) const {
if (IsNonTerminal() < compare.IsNonTerminal())
return true;
else if (IsNonTerminal() == compare.IsNonTerminal())
return m_cellLabel.GetCoverage() < compare.m_cellLabel.GetCoverage();
return false;
}
};
}; // namespace

41
moses/src/DotChart.cpp
View File

@ -17,44 +17,21 @@
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "DotChart.h"
#include "Util.h"
#include <algorithm>
using namespace std;
namespace Moses
{
DottedRuleColl::~DottedRuleColl()
std::ostream &operator<<(std::ostream &out, const DottedRule &rule)
{
#ifdef USE_BOOST_POOL
// Do nothing. DottedRule objects are stored in object pools owned by
// the sentence-specific ChartRuleLookupManagers.
#else
std::for_each(m_coll.begin(), m_coll.end(),
RemoveAllInColl<CollType::value_type>);
#endif
}
std::ostream& operator<<(std::ostream &out, const DottedRule& /* rule */)
{
//const PhraseDictionaryNode &node = rule.GetLastNode();
//out << node;
return out;
}
std::ostream& operator<<(std::ostream &out, const DottedRuleList &coll)
{
DottedRuleList::const_iterator iter;
for (iter = coll.begin(); iter != coll.end(); ++iter) {
const DottedRule &rule = **iter;
out << rule << endl;
if (!rule.IsRoot()) {
out << rule.GetWordsRange() << "=" << rule.GetSourceWord() << " ";
if (!rule.m_prev->IsRoot()) {
out << " " << *rule.m_prev;
}
}
return out;
}
};
}

111
moses/src/DotChart.h
View File

@ -23,110 +23,35 @@
#include "config.h"
#endif
#include <vector>
#include <cassert>
#include "PhraseDictionaryNodeSCFG.h"
#include "ChartTranslationOption.h"
#include "CoveredChartSpan.h"
#include "ChartCellLabel.h"
namespace Moses
{
class DottedRule
{
friend std::ostream& operator<<(std::ostream&, const DottedRule&);
friend std::ostream& operator<<(std::ostream &, const DottedRule &);
protected:
const PhraseDictionaryNodeSCFG &m_lastNode;
const CoveredChartSpan *m_coveredChartSpan; // usually contains something, unless its the init processed rule
public:
public:
// used only to init dot stack.
explicit DottedRule(const PhraseDictionaryNodeSCFG &lastNode)
:m_lastNode(lastNode)
,m_coveredChartSpan(NULL)
{}
DottedRule(const PhraseDictionaryNodeSCFG &lastNode, const CoveredChartSpan *coveredChartSpan)
:m_lastNode(lastNode)
,m_coveredChartSpan(coveredChartSpan)
{}
~DottedRule() {
#ifdef USE_BOOST_POOL
// Do nothing. CoveredChartSpan objects are stored in object pools owned by
// the sentence-specific ChartRuleLookupManagers.
#else
delete m_coveredChartSpan;
#endif
}
const PhraseDictionaryNodeSCFG &GetLastNode() const {
return m_lastNode;
}
const CoveredChartSpan *GetLastCoveredChartSpan() const {
return m_coveredChartSpan;
}
};
DottedRule()
: m_cellLabel(NULL)
, m_prev(NULL) {}
typedef std::vector<const DottedRule*> DottedRuleList;
DottedRule(const ChartCellLabel &ccl, const DottedRule &prev)
: m_cellLabel(&ccl)
, m_prev(&prev) {}
// Collection of all DottedRules that share a common start point,
// grouped by end point. Additionally, maintains a list of all
// DottedRules that could be expanded further, i.e. for which the
// corresponding PhraseDictionaryNodeSCFG is not a leaf.
class DottedRuleColl
{
protected:
typedef std::vector<DottedRuleList> CollType;
CollType m_coll;
DottedRuleList m_expandableDottedRuleList;
public:
typedef CollType::iterator iterator;
typedef CollType::const_iterator const_iterator;
const_iterator begin() const {
return m_coll.begin();
}
const_iterator end() const {
return m_coll.end();
}
iterator begin() {
return m_coll.begin();
}
iterator end() {
return m_coll.end();
}
DottedRuleColl(size_t size)
: m_coll(size)
{}
~DottedRuleColl();
const DottedRuleList &Get(size_t pos) const {
return m_coll[pos];
}
DottedRuleList &Get(size_t pos) {
return m_coll[pos];
}
void Add(size_t pos, const DottedRule *dottedRule) {
assert(dottedRule);
m_coll[pos].push_back(dottedRule);
if (!dottedRule->GetLastNode().IsLeaf()) {
m_expandableDottedRuleList.push_back(dottedRule);
}
}
void Clear(size_t pos) {
#ifdef USE_BOOST_POOL
m_coll[pos].clear();
#endif
}
const DottedRuleList &GetExpandableDottedRuleList() const {
return m_expandableDottedRuleList;
}
const WordsRange &GetWordsRange() const { return m_cellLabel->GetCoverage(); }
const Word &GetSourceWord() const { return m_cellLabel->GetLabel(); }
bool IsNonTerminal() const { return m_cellLabel->GetLabel().IsNonTerminal(); }
const DottedRule *GetPrev() const { return m_prev; }
bool IsRoot() const { return m_prev == NULL; }
private:
const ChartCellLabel *m_cellLabel; // usually contains something, unless
// it's the init processed rule
const DottedRule *m_prev;
};
}

38
moses/src/CoveredChartSpan.cpp → moses/src/DotChartInMemory.cpp
View File

@ -1,38 +1,40 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
Moses - statistical machine translation system
Copyright (C) 2006-2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
***********************************************************************/
#include "CoveredChartSpan.h"
#include "DotChartInMemory.h"
#include "Util.h"
#include <algorithm>
namespace Moses
{
std::ostream& operator<<(std::ostream &out, const CoveredChartSpan &coveredChartSpan)
DottedRuleColl::~DottedRuleColl()
{
out << coveredChartSpan.GetWordsRange()
<< "=" << coveredChartSpan.GetSourceWord() << " ";
if (coveredChartSpan.m_prevCoveredChartSpan)
out << " " << *coveredChartSpan.m_prevCoveredChartSpan;
return out;
#ifdef USE_BOOST_POOL
// Do nothing. DottedRule objects are stored in object pools owned by
// the sentence-specific ChartRuleLookupManagers.
#else
std::for_each(m_coll.begin(), m_coll.end(),
RemoveAllInColl<CollType::value_type>);
#endif
}
} // namespace
}

118
moses/src/DotChartInMemory.h Normal file
View File

@ -0,0 +1,118 @@
/***********************************************************************
Moses - statistical machine translation system
Copyright (C) 2006-2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#if HAVE_CONFIG_H
#include "config.h"
#endif
#include "DotChart.h"
#include "PhraseDictionaryNodeSCFG.h"
#include <cassert>
#include <vector>
namespace Moses
{
class DottedRuleInMemory : public DottedRule
{
public:
// used only to init dot stack.
explicit DottedRuleInMemory(const PhraseDictionaryNodeSCFG &node)
: DottedRule()
, m_node(node) {}
DottedRuleInMemory(const PhraseDictionaryNodeSCFG &node,
const ChartCellLabel &cellLabel,
const DottedRuleInMemory &prev)
: DottedRule(cellLabel, prev)
, m_node(node) {}
const PhraseDictionaryNodeSCFG &GetLastNode() const { return m_node; }
private:
const PhraseDictionaryNodeSCFG &m_node;
};
typedef std::vector<const DottedRuleInMemory*> DottedRuleList;
// Collection of all in-memory DottedRules that share a common start point,
// grouped by end point. Additionally, maintains a list of all
// DottedRules that could be expanded further, i.e. for which the
// corresponding PhraseDictionaryNodeSCFG is not a leaf.
class DottedRuleColl
{
protected:
typedef std::vector<DottedRuleList> CollType;
CollType m_coll;
DottedRuleList m_expandableDottedRuleList;
public:
typedef CollType::iterator iterator;
typedef CollType::const_iterator const_iterator;
const_iterator begin() const {
return m_coll.begin();
}
const_iterator end() const {
return m_coll.end();
}
iterator begin() {
return m_coll.begin();
}
iterator end() {
return m_coll.end();
}
DottedRuleColl(size_t size)
: m_coll(size)
{}
~DottedRuleColl();
const DottedRuleList &Get(size_t pos) const {
return m_coll[pos];
}
DottedRuleList &Get(size_t pos) {
return m_coll[pos];
}
void Add(size_t pos, const DottedRuleInMemory *dottedRule) {
assert(dottedRule);
m_coll[pos].push_back(dottedRule);
if (!dottedRule->GetLastNode().IsLeaf()) {
m_expandableDottedRuleList.push_back(dottedRule);
}
}
void Clear(size_t pos) {
#ifdef USE_BOOST_POOL
m_coll[pos].clear();
#endif
}
const DottedRuleList &GetExpandableDottedRuleList() const {
return m_expandableDottedRuleList;
}
};
}

8
moses/src/DotChartOnDisk.cpp
View File

@ -54,14 +54,6 @@ void DottedRuleStackOnDisk::SortSavedNodes()
sort(m_savedNode.begin(), m_savedNode.end(), SavedNodesOderer());
}
std::ostream& operator<<(std::ostream &out, const DottedRuleOnDisk & /* rule */)
{
//const MosesBerkeleyPt::SourcePhraseNode &node = rule.GetLastNode();
//out << node;
return out;
}
std::ostream& operator<<(std::ostream &out, const DottedRuleCollOnDisk &coll)
{
DottedRuleCollOnDisk::CollType::const_iterator iter;

71
moses/src/DotChartOnDisk.h
View File

@ -21,8 +21,8 @@
#include <vector>
#include <cassert>
#include "ChartTranslationOption.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
namespace OnDiskPt
{
@ -32,63 +32,30 @@ class PhraseNode;
namespace Moses
{
class DottedRuleOnDisk
class DottedRuleOnDisk : public DottedRule
{
friend std::ostream& operator<<(std::ostream&, const DottedRuleOnDisk&);
protected:
const OnDiskPt::PhraseNode &m_lastNode;
const CoveredChartSpan *m_coveredChartSpan; // usually contains something, unless its the init processed rule
mutable bool m_done;
public:
public:
// used only to init dot stack.
explicit DottedRuleOnDisk(const OnDiskPt::PhraseNode &lastNode)
:m_lastNode(lastNode)
,m_coveredChartSpan(NULL)
,m_done(false)
{}
DottedRuleOnDisk(const OnDiskPt::PhraseNode &lastNode, const CoveredChartSpan *coveredChartSpan)
:m_lastNode(lastNode)
,m_coveredChartSpan(coveredChartSpan)
,m_done(false)
{}
~DottedRuleOnDisk() {
delete m_coveredChartSpan;
}
const OnDiskPt::PhraseNode &GetLastNode() const {
return m_lastNode;
}
const CoveredChartSpan *GetLastCoveredChartSpan() const {
return m_coveredChartSpan;
}
: DottedRule()
, m_lastNode(lastNode)
, m_done(false) {}
bool IsCurrNonTerminal() const {
assert(m_coveredChartSpan);
return m_coveredChartSpan->IsNonTerminal();
}
DottedRuleOnDisk(const OnDiskPt::PhraseNode &lastNode,
const ChartCellLabel &cellLabel,
const DottedRuleOnDisk &prev)
: DottedRule(cellLabel, prev)
, m_lastNode(lastNode)
, m_done(false) {}
bool Done() const {
return m_done;
}
void Done(bool value) const {
m_done = value;
}
const OnDiskPt::PhraseNode &GetLastNode() const { return m_lastNode; }
/*
inline int Compare(const DottedRule &compare) const
{
if (m_lastNode < compare.m_lastNode)
return -1;
if (m_lastNode > compare.m_lastNode)
return 1;
bool Done() const { return m_done; }
void Done(bool value) const { m_done = value; }
return m_coveredChartSpan < compare.m_coveredChartSpan;
}
inline bool operator<(const DottedRule &compare) const
{
return Compare(compare) < 0;
}
*/
private:
const OnDiskPt::PhraseNode &m_lastNode;
mutable bool m_done;
};
class DottedRuleCollOnDisk

4
moses/src/Makefile.am
View File

@ -25,7 +25,6 @@ libmoses_la_HEADERS = \
ChartTrellisPathCollection.h \
ChartTrellisPathList.h \
ConfusionNet.h \
CoveredChartSpan.h \
DecodeFeature.h \
DecodeGraph.h \
DecodeStep.h \
@ -33,6 +32,7 @@ libmoses_la_HEADERS = \
DecodeStepTranslation.h \
Dictionary.h \
DotChart.h \
DotChartInMemory.h \
DotChartOnDisk.h \
DummyScoreProducers.h \
DynSAInclude/file.h \
@ -186,7 +186,6 @@ libmoses_la_SOURCES = \
ChartTrellisPathCollection.cpp \
ChartTrellisPathList.cpp \
ConfusionNet.cpp \
CoveredChartSpan.cpp \
DecodeFeature.cpp \
DecodeGraph.cpp \
DecodeStep.cpp \
@ -194,6 +193,7 @@ libmoses_la_SOURCES = \
DecodeStepTranslation.cpp \
Dictionary.cpp \
DotChart.cpp \
DotChartInMemory.cpp \
DotChartOnDisk.cpp \
DummyScoreProducers.cpp \
DynSAInclude/file.cpp \

1
moses/src/PhraseDictionarySCFG.h
View File

@ -25,7 +25,6 @@
#include "PhraseDictionary.h"
#include "PhraseDictionaryNodeSCFG.h"
#include "InputType.h"
#include "CoveredChartSpan.h"
#include "NonTerminal.h"
namespace Moses

1
moses/src/RuleCube.cpp
View File

@ -23,7 +23,6 @@
#include "ChartCellCollection.h"
#include "ChartTranslationOption.h"
#include "ChartTranslationOptionCollection.h"
#include "CoveredChartSpan.h"
#include "RuleCube.h"
#include "RuleCubeQueue.h"
#include "StaticData.h"

22
moses/src/RuleCubeItem.cpp
View File

@ -21,7 +21,7 @@
#include "ChartCellCollection.h"
#include "ChartTranslationOption.h"
#include "ChartTranslationOptionCollection.h"
#include "CoveredChartSpan.h"
#include "DotChart.h"
#include "RuleCubeItem.h"
#include "RuleCubeQueue.h"
#include "WordsRange.h"
@ -48,8 +48,7 @@ RuleCubeItem::RuleCubeItem(const ChartTranslationOption &transOpt,
transOpt.GetTargetPhraseCollection().GetCollection())
, m_hypothesis(0)
{
const CoveredChartSpan *lastCCS = &transOpt.GetLastCoveredChartSpan();
CreateHypothesisDimensions(lastCCS, allChartCells);
CreateHypothesisDimensions(transOpt.GetDottedRule(), allChartCells);
}
// create the RuleCube from an existing one, differing only in one dimension
@ -100,24 +99,25 @@ ChartHypothesis *RuleCubeItem::ReleaseHypothesis()
// for each non-terminal, create a ordered list of matching hypothesis from the
// chart
void RuleCubeItem::CreateHypothesisDimensions(
const CoveredChartSpan *coveredChartSpan,
const DottedRule &dottedRule,
const ChartCellCollection &allChartCells)
{
// recurse through the linked list of source side non-terminals and terminals
const CoveredChartSpan *prev = coveredChartSpan->GetPrevCoveredChartSpan();
if (prev) {
CreateHypothesisDimensions(prev, allChartCells);
assert(!dottedRule.IsRoot());
const DottedRule *prev = dottedRule.GetPrev();
if (!prev->IsRoot()) {
CreateHypothesisDimensions(*prev, allChartCells);
}
// only deal with non-terminals
if (coveredChartSpan->IsNonTerminal()) {
if (dottedRule.IsNonTerminal()) {
// get the essential information about the non-terminal:
// span covered by child
const WordsRange &childRange = coveredChartSpan->GetWordsRange();
const WordsRange &childRange = dottedRule.GetWordsRange();
// list of all hypos for that span
const ChartCell &childCell = allChartCells.Get(childRange);
// target (sic!) non-terminal label
const Word &nonTerm = coveredChartSpan->GetSourceWord();
const Word &nonTerm = dottedRule.GetSourceWord();
// there have to be hypothesis with the desired non-terminal
// (otherwise the rule would not be considered)

4
moses/src/RuleCubeItem.h
View File

@ -32,7 +32,7 @@ class ChartCellCollection;
class ChartHypothesis;
class ChartManager;
class ChartTranslationOption;
class CoveredChartSpan;
class DottedRule;
class TargetPhrase;
typedef std::vector<const ChartHypothesis*> HypoList;
@ -136,7 +136,7 @@ class RuleCubeItem
RuleCubeItem(const RuleCubeItem &); // Not implemented
RuleCubeItem &operator=(const RuleCubeItem &); // Not implemented
void CreateHypothesisDimensions(const CoveredChartSpan *,
void CreateHypothesisDimensions(const DottedRule &,
const ChartCellCollection &);
TranslationDimension m_translationDimension;