moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

run beautify.perl. Consistent formatting for .h & .cpp files

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3896 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972 2011-02-24 12:36:50 +00:00
parent 5f0eacce4b
commit 508d89eda8
24 changed files with 1654 additions and 1758 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
moses-chart/src/ChartCell.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -36,196 +36,185 @@ using namespace Moses;
namespace Moses
{
extern bool g_debug;
extern bool g_debug;
}
namespace MosesChart
{
ChartCell::ChartCell(size_t startPos, size_t endPos, Manager &manager)
:m_coverage(startPos, endPos)
,m_manager(manager)
:m_coverage(startPos, endPos)
,m_manager(manager)
{
const StaticData &staticData = StaticData::Instance();
m_nBestIsEnabled = staticData.IsNBestEnabled();
const StaticData &staticData = StaticData::Instance();
m_nBestIsEnabled = staticData.IsNBestEnabled();
}
const HypoList &ChartCell::GetSortedHypotheses(const Moses::Word &headWord) const
{
std::map<Moses::Word, HypothesisCollection>::const_iterator
iter = m_hypoColl.find(headWord);
assert(iter != m_hypoColl.end());
return iter->second.GetSortedHypotheses();
std::map<Moses::Word, HypothesisCollection>::const_iterator
iter = m_hypoColl.find(headWord);
assert(iter != m_hypoColl.end());
return iter->second.GetSortedHypotheses();
}
bool ChartCell::AddHypothesis(Hypothesis *hypo)
{
const Word &targetLHS = hypo->GetTargetLHS();
return m_hypoColl[targetLHS].AddHypothesis(hypo, m_manager);
const Word &targetLHS = hypo->GetTargetLHS();
return m_hypoColl[targetLHS].AddHypothesis(hypo, m_manager);
}
void ChartCell::PruneToSize()
{
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
HypothesisCollection &coll = iter->second;
coll.PruneToSize(m_manager);
}
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
HypothesisCollection &coll = iter->second;
coll.PruneToSize(m_manager);
}
}
void ChartCell::ProcessSentence(const ChartTranslationOptionList &transOptList
, const ChartCellCollection &allChartCells)
, const ChartCellCollection &allChartCells)
{
const StaticData &staticData = StaticData::Instance();
const StaticData &staticData = StaticData::Instance();
Cube cube;
Cube cube;
// add all trans opt into queue. using only 1st child node.
ChartTranslationOptionList::const_iterator iterList;
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList)
{
const ChartTranslationOption &transOpt = **iterList;
QueueEntry *queueEntry = new QueueEntry(transOpt, allChartCells);
cube.Add(queueEntry);
}
// pluck things out of queue and add to hypo collection
const size_t popLimit = staticData.GetCubePruningPopLimit();
// add all trans opt into queue. using only 1st child node.
ChartTranslationOptionList::const_iterator iterList;
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList) {
const ChartTranslationOption &transOpt = **iterList;
QueueEntry *queueEntry = new QueueEntry(transOpt, allChartCells);
cube.Add(queueEntry);
}
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops)
{
QueueEntry *queueEntry = cube.Pop();
queueEntry->GetTranslationOption().GetTotalScore();
Hypothesis *hypo = new Hypothesis(*queueEntry, m_manager);
assert(hypo);
hypo->CalcScore();
AddHypothesis(hypo);
// pluck things out of queue and add to hypo collection
const size_t popLimit = staticData.GetCubePruningPopLimit();
// Expand queue entry
queueEntry->CreateDeviants(cube);
}
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops) {
QueueEntry *queueEntry = cube.Pop();
queueEntry->GetTranslationOption().GetTotalScore();
Hypothesis *hypo = new Hypothesis(*queueEntry, m_manager);
assert(hypo);
hypo->CalcScore();
AddHypothesis(hypo);
// Expand queue entry
queueEntry->CreateDeviants(cube);
}
}
void ChartCell::SortHypotheses()
{
// sort each mini cells & fill up target lhs list
assert(m_headWords.empty());
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
m_headWords.insert(iter->first);
// sort each mini cells & fill up target lhs list
assert(m_headWords.empty());
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
m_headWords.insert(iter->first);
HypothesisCollection &coll = iter->second;
coll.SortHypotheses();
}
HypothesisCollection &coll = iter->second;
coll.SortHypotheses();
}
}
const Hypothesis *ChartCell::GetBestHypothesis() const
{
const Hypothesis *ret = NULL;
float bestScore = -std::numeric_limits<float>::infinity();
const Hypothesis *ret = NULL;
float bestScore = -std::numeric_limits<float>::infinity();
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
const HypoList &sortedList = iter->second.GetSortedHypotheses();
assert(sortedList.size() > 0);
const Hypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore)
{
bestScore = hypo->GetTotalScore();
ret = hypo;
};
}
return ret;
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
const HypoList &sortedList = iter->second.GetSortedHypotheses();
assert(sortedList.size() > 0);
const Hypothesis *hypo = sortedList[0];
if (hypo->GetTotalScore() > bestScore) {
bestScore = hypo->GetTotalScore();
ret = hypo;
};
}
return ret;
}
bool ChartCell::HeadwordExists(const Moses::Word &headWord) const
{
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
iter = m_hypoColl.find(headWord);
return (iter != m_hypoColl.end());
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
iter = m_hypoColl.find(headWord);
return (iter != m_hypoColl.end());
}
void ChartCell::CleanupArcList()
{
// only necessary if n-best calculations are enabled
if (!m_nBestIsEnabled) return;
// only necessary if n-best calculations are enabled
if (!m_nBestIsEnabled) return;
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
HypothesisCollection &coll = iter->second;
coll.CleanupArcList();
}
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
HypothesisCollection &coll = iter->second;
coll.CleanupArcList();
}
}
void ChartCell::OutputSizes(std::ostream &out) const
{
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
const Moses::Word &targetLHS = iter->first;
const HypothesisCollection &coll = iter->second;
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
const Moses::Word &targetLHS = iter->first;
const HypothesisCollection &coll = iter->second;
out << targetLHS << "=" << coll.GetSize() << " ";
}
out << targetLHS << "=" << coll.GetSize() << " ";
}
}
size_t ChartCell::GetSize() const
{
size_t ret = 0;
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
const HypothesisCollection &coll = iter->second;
ret += coll.GetSize();
}
return ret;
size_t ret = 0;
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
const HypothesisCollection &coll = iter->second;
ret += coll.GetSize();
}
return ret;
}
void ChartCell::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
{
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside)
{
const HypothesisCollection &coll = iterOutside->second;
coll.GetSearchGraph(translationId, outputSearchGraphStream);
}
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside) {
const HypothesisCollection &coll = iterOutside->second;
coll.GetSearchGraph(translationId, outputSearchGraphStream);
}
}
std::ostream& operator<<(std::ostream &out, const ChartCell &cell)
{
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside)
{
const Moses::Word &targetLHS = iterOutside->first;
cerr << targetLHS << ":" << endl;
const HypothesisCollection &coll = iterOutside->second;
cerr << coll;
}
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside) {
const Moses::Word &targetLHS = iterOutside->first;
cerr << targetLHS << ":" << endl;
/*
ChartCell::HCType::const_iterator iter;
for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
{
const Hypothesis &hypo = **iter;
out << hypo << endl;
}
*/
return out;
const HypothesisCollection &coll = iterOutside->second;
cerr << coll;
}
/*
ChartCell::HCType::const_iterator iter;
for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
{
const Hypothesis &hypo = **iter;
out << hypo << endl;
}
*/
return out;
}
} // namespace

64
moses-chart/src/ChartCell.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -34,7 +34,7 @@
namespace Moses
{
class ChartTranslationOptionList;
class ChartTranslationOptionList;
}
namespace MosesChart
@ -48,48 +48,48 @@ class Manager;
class ChartCell
{
friend std::ostream& operator<<(std::ostream&, const ChartCell&);
friend std::ostream& operator<<(std::ostream&, const ChartCell&);
public:
protected:
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
Moses::NonTerminalSet m_headWords;
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
Moses::NonTerminalSet m_headWords;
Moses::WordsRange m_coverage;
Moses::WordsRange m_coverage;
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
Manager &m_manager;
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
Manager &m_manager;
public:
ChartCell(size_t startPos, size_t endPos, Manager &manager);
ChartCell(size_t startPos, size_t endPos, Manager &manager);
void ProcessSentence(const Moses::ChartTranslationOptionList &transOptList
,const ChartCellCollection &allChartCells);
void ProcessSentence(const Moses::ChartTranslationOptionList &transOptList
,const ChartCellCollection &allChartCells);
const HypoList &GetSortedHypotheses(const Moses::Word &headWord) const;
bool AddHypothesis(Hypothesis *hypo);
const HypoList &GetSortedHypotheses(const Moses::Word &headWord) const;
bool AddHypothesis(Hypothesis *hypo);
void SortHypotheses();
void PruneToSize();
void SortHypotheses();
void PruneToSize();
const Hypothesis *GetBestHypothesis() const;
const Hypothesis *GetBestHypothesis() const;
bool HeadwordExists(const Moses::Word &headWord) const;
const Moses::NonTerminalSet &GetHeadwords() const
{ return m_headWords; }
bool HeadwordExists(const Moses::Word &headWord) const;
const Moses::NonTerminalSet &GetHeadwords() const {
return m_headWords;
}
void CleanupArcList();
void CleanupArcList();
void OutputSizes(std::ostream &out) const;
size_t GetSize() const;
//! transitive comparison used for adding objects into set
inline bool operator<(const ChartCell &compare) const
{
return m_coverage < compare.m_coverage;
}
void OutputSizes(std::ostream &out) const;
size_t GetSize() const;
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
//! transitive comparison used for adding objects into set
inline bool operator<(const ChartCell &compare) const {
return m_coverage < compare.m_coverage;
}
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
};

43
moses-chart/src/ChartCellCollection.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -26,32 +26,29 @@
namespace MosesChart
{
ChartCellCollection::ChartCellCollection(const Moses::InputType &input, Manager &manager)
:m_hypoStackColl(input.GetSize())
:m_hypoStackColl(input.GetSize())
{
size_t size = input.GetSize();
for (size_t startPos = 0; startPos < size; ++startPos)
{
InnerCollType &inner = m_hypoStackColl[startPos];
inner.resize(size - startPos);
size_t size = input.GetSize();
for (size_t startPos = 0; startPos < size; ++startPos) {
InnerCollType &inner = m_hypoStackColl[startPos];
inner.resize(size - startPos);
size_t ind = 0;
for (size_t endPos = startPos ; endPos < size; ++endPos)
{
ChartCell *cell = new ChartCell(startPos, endPos, manager);
inner[ind] = cell;
++ind;
}
}
size_t ind = 0;
for (size_t endPos = startPos ; endPos < size; ++endPos) {
ChartCell *cell = new ChartCell(startPos, endPos, manager);
inner[ind] = cell;
++ind;
}
}
}
ChartCellCollection::~ChartCellCollection()
{
OuterCollType::iterator iter;
for (iter = m_hypoStackColl.begin(); iter != m_hypoStackColl.end(); ++iter)
{
InnerCollType &inner = *iter;
Moses::RemoveAllInColl(inner);
}
OuterCollType::iterator iter;
for (iter = m_hypoStackColl.begin(); iter != m_hypoStackColl.end(); ++iter) {
InnerCollType &inner = *iter;
Moses::RemoveAllInColl(inner);
}
}
} // namespace

43
moses-chart/src/ChartCellCollection.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -32,34 +32,31 @@ class InputType;
namespace MosesChart
{
class Manager;
class ChartCellCollection : public Moses::CellCollection
{
public:
typedef std::vector<ChartCell*> InnerCollType;
typedef std::vector<InnerCollType> OuterCollType;
typedef std::vector<ChartCell*> InnerCollType;
typedef std::vector<InnerCollType> OuterCollType;
protected:
OuterCollType m_hypoStackColl;
OuterCollType m_hypoStackColl;
public:
ChartCellCollection(const Moses::InputType &input, Manager &manager);
~ChartCellCollection();
ChartCellCollection(const Moses::InputType &input, Manager &manager);
~ChartCellCollection();
ChartCell &Get(const Moses::WordsRange &coverage)
{
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
}
const ChartCell &Get(const Moses::WordsRange &coverage) const
{
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
}
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const
{
const ChartCell &cell = Get(coverage);
return cell.GetHeadwords();
}
ChartCell &Get(const Moses::WordsRange &coverage) {
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
}
const ChartCell &Get(const Moses::WordsRange &coverage) const {
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
}
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const {
const ChartCell &cell = Get(coverage);
return cell.GetHeadwords();
}
};
}

548
moses-chart/src/ChartHypothesis.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -40,364 +40,333 @@ namespace MosesChart
unsigned int Hypothesis::s_HypothesesCreated = 0;
#ifdef USE_HYPO_POOL
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
#endif
Hypothesis::Hypothesis(const QueueEntry &queueEntry, Manager &manager)
:m_transOpt(queueEntry.GetTranslationOption())
,m_wordsConsumedTargetOrder(queueEntry.GetTranslationOption().GetWordsConsumedTargetOrder())
,m_id(++s_HypothesesCreated)
,m_currSourceWordsRange(queueEntry.GetTranslationOption().GetSourceWordsRange())
,m_contextPrefix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
,m_contextSuffix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
,m_arcList(NULL)
,m_manager(manager)
:m_transOpt(queueEntry.GetTranslationOption())
,m_wordsConsumedTargetOrder(queueEntry.GetTranslationOption().GetWordsConsumedTargetOrder())
,m_id(++s_HypothesesCreated)
,m_currSourceWordsRange(queueEntry.GetTranslationOption().GetSourceWordsRange())
,m_contextPrefix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
,m_contextSuffix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
,m_arcList(NULL)
,m_manager(manager)
{
assert(GetCurrTargetPhrase().GetSize() == m_wordsConsumedTargetOrder.size());
//TRACE_ERR(m_targetPhrase << endl);
assert(GetCurrTargetPhrase().GetSize() == m_wordsConsumedTargetOrder.size());
//TRACE_ERR(m_targetPhrase << endl);
m_numTargetTerminals = GetCurrTargetPhrase().GetNumTerminals();
m_numTargetTerminals = GetCurrTargetPhrase().GetNumTerminals();
const std::vector<ChildEntry> &childEntries = queueEntry.GetChildEntries();
assert(m_prevHypos.empty());
m_prevHypos.reserve(childEntries.size());
vector<ChildEntry>::const_iterator iter;
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter)
{
const ChildEntry &childEntry = *iter;
const Hypothesis *prevHypo = childEntry.GetHypothesis();
const std::vector<ChildEntry> &childEntries = queueEntry.GetChildEntries();
assert(m_prevHypos.empty());
m_prevHypos.reserve(childEntries.size());
vector<ChildEntry>::const_iterator iter;
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
const ChildEntry &childEntry = *iter;
const Hypothesis *prevHypo = childEntry.GetHypothesis();
m_numTargetTerminals += prevHypo->GetNumTargetTerminals();
m_numTargetTerminals += prevHypo->GetNumTargetTerminals();
m_prevHypos.push_back(prevHypo);
}
m_prevHypos.push_back(prevHypo);
}
size_t maxNGram = manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder();
CalcPrefix(m_contextPrefix, maxNGram - 1);
CalcSuffix(m_contextSuffix, maxNGram - 1);
size_t maxNGram = manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder();
CalcPrefix(m_contextPrefix, maxNGram - 1);
CalcSuffix(m_contextSuffix, maxNGram - 1);
}
Hypothesis::~Hypothesis()
{
if (m_arcList)
{
ArcList::iterator iter;
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter)
{
Hypothesis *hypo = *iter;
Delete(hypo);
}
m_arcList->clear();
if (m_arcList) {
ArcList::iterator iter;
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
Hypothesis *hypo = *iter;
Delete(hypo);
}
m_arcList->clear();
delete m_arcList;
}
delete m_arcList;
}
}
void Hypothesis::CreateOutputPhrase(Phrase &outPhrase) const
{
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
{
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (word.IsNonTerminal())
{ // non-term. fill out with prev hypo
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->CreateOutputPhrase(outPhrase);
}
else
{
outPhrase.AddWord(word);
}
}
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (word.IsNonTerminal()) {
// non-term. fill out with prev hypo
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->CreateOutputPhrase(outPhrase);
} else {
outPhrase.AddWord(word);
}
}
}
Phrase Hypothesis::GetOutputPhrase() const
{
Phrase outPhrase(Output);
CreateOutputPhrase(outPhrase);
return outPhrase;
Phrase outPhrase(Output);
CreateOutputPhrase(outPhrase);
return outPhrase;
}
size_t Hypothesis::CalcPrefix(Phrase &ret, size_t size) const
{
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
{
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (word.IsNonTerminal())
{
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcPrefix(ret, size);
}
else
{
ret.AddWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (size==0)
break;
}
if (word.IsNonTerminal()) {
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcPrefix(ret, size);
} else {
ret.AddWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
return size;
if (size==0)
break;
}
return size;
}
size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
{
assert(m_contextPrefix.GetSize() <= m_numTargetTerminals);
assert(m_contextPrefix.GetSize() <= m_numTargetTerminals);
if (m_contextPrefix.GetSize() == m_numTargetTerminals)
{ // small hypo. the prefix will contains the whole hypo
size_t maxCount = min(m_contextPrefix.GetSize(), size)
, pos = m_contextPrefix.GetSize() - 1;
if (m_contextPrefix.GetSize() == m_numTargetTerminals) {
// small hypo. the prefix will contains the whole hypo
size_t maxCount = min(m_contextPrefix.GetSize(), size)
, pos = m_contextPrefix.GetSize() - 1;
for (size_t ind = 0; ind < maxCount; ++ind)
{
const Word &word = m_contextPrefix.GetWord(pos);
ret.PrependWord(word);
--pos;
}
for (size_t ind = 0; ind < maxCount; ++ind) {
const Word &word = m_contextPrefix.GetWord(pos);
ret.PrependWord(word);
--pos;
}
size -= maxCount;
return size;
}
else
{
for (int pos = (int) GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos)
{
const Word &word = GetCurrTargetPhrase().GetWord(pos);
size -= maxCount;
return size;
} else {
for (int pos = (int) GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos) {
const Word &word = GetCurrTargetPhrase().GetWord(pos);
if (word.IsNonTerminal())
{
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcSuffix(ret, size);
}
else
{
ret.PrependWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
if (word.IsNonTerminal()) {
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size = prevHypo->CalcSuffix(ret, size);
} else {
ret.PrependWord(GetCurrTargetPhrase().GetWord(pos));
size--;
}
if (size==0)
break;
}
if (size==0)
break;
}
return size;
}
return size;
}
}
int Hypothesis::LMContextCompare(const Hypothesis &other) const
{
// prefix
if (m_currSourceWordsRange.GetStartPos() > 0)
{
int ret = GetPrefix().Compare(other.GetPrefix());
if (ret != 0)
return ret;
}
// prefix
if (m_currSourceWordsRange.GetStartPos() > 0) {
int ret = GetPrefix().Compare(other.GetPrefix());
if (ret != 0)
return ret;
}
// suffix
size_t inputSize = m_manager.GetSource().GetSize();
if (m_currSourceWordsRange.GetEndPos() < inputSize - 1)
{
int ret = GetSuffix().Compare(other.GetSuffix());
if (ret != 0)
return ret;
}
// suffix
size_t inputSize = m_manager.GetSource().GetSize();
if (m_currSourceWordsRange.GetEndPos() < inputSize - 1) {
int ret = GetSuffix().Compare(other.GetSuffix());
if (ret != 0)
return ret;
}
// they're the same
return 0;
// they're the same
return 0;
}
void Hypothesis::CalcScore()
{
// total scores from prev hypos
std::vector<const Hypothesis*>::iterator iter;
for (iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter)
{
const Hypothesis &prevHypo = **iter;
const ScoreComponentCollection &scoreBreakdown = prevHypo.GetScoreBreakdown();
// total scores from prev hypos
std::vector<const Hypothesis*>::iterator iter;
for (iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
const Hypothesis &prevHypo = **iter;
const ScoreComponentCollection &scoreBreakdown = prevHypo.GetScoreBreakdown();
m_scoreBreakdown.PlusEquals(scoreBreakdown);
}
m_scoreBreakdown.PlusEquals(scoreBreakdown);
}
// translation models & word penalty
const ScoreComponentCollection &scoreBreakdown = GetCurrTargetPhrase().GetScoreBreakdown();
m_scoreBreakdown.PlusEquals(scoreBreakdown);
// translation models & word penalty
const ScoreComponentCollection &scoreBreakdown = GetCurrTargetPhrase().GetScoreBreakdown();
m_scoreBreakdown.PlusEquals(scoreBreakdown);
CalcLMScore();
CalcLMScore();
m_totalScore = m_scoreBreakdown.GetWeightedScore();
m_totalScore = m_scoreBreakdown.GetWeightedScore();
}
void Hypothesis::CalcLMScore()
{
const LMList& lmList = m_manager.GetTranslationSystem()->GetLanguageModels();
assert(m_lmNGram.GetWeightedScore() == 0);
assert(m_lmNGram.GetWeightedScore() == 0);
m_scoreBreakdown.ZeroAllLM(lmList);
m_scoreBreakdown.ZeroAllLM(lmList);
Phrase outPhrase(Output); // = GetOutputPhrase();
bool calcNow = false, firstPhrase = true;
Phrase outPhrase(Output); // = GetOutputPhrase();
bool calcNow = false, firstPhrase = true;
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos)
{
const Word &targetWord = GetCurrTargetPhrase().GetWord(targetPhrasePos);
if (!targetWord.IsNonTerminal())
{ // just a word, add to phrase for lm scoring
outPhrase.AddWord(targetWord);
}
else
{
size_t nonTermInd = m_wordsConsumedTargetOrder[targetPhrasePos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size_t numTargetTerminals = prevHypo->GetNumTargetTerminals();
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos) {
const Word &targetWord = GetCurrTargetPhrase().GetWord(targetPhrasePos);
if (!targetWord.IsNonTerminal()) {
// just a word, add to phrase for lm scoring
outPhrase.AddWord(targetWord);
} else {
size_t nonTermInd = m_wordsConsumedTargetOrder[targetPhrasePos];
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
size_t numTargetTerminals = prevHypo->GetNumTargetTerminals();
if (numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
{ // large hypo (for trigram lm, another hypo equal or over 2 words). just take the prefix & suffix
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
if (numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
// large hypo (for trigram lm, another hypo equal or over 2 words). just take the prefix & suffix
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
// calc & add overlapping lm scores
// prefix
outPhrase.Append(prevHypo->GetPrefix());
calcNow = true;
}
else
{ // small hypo (for trigram lm, 1-word hypos).
// add target phrase to temp phrase and continue, but don't score yet
outPhrase.Append(prevHypo->GetPrefix());
}
// calc & add overlapping lm scores
// prefix
outPhrase.Append(prevHypo->GetPrefix());
calcNow = true;
} else {
// small hypo (for trigram lm, 1-word hypos).
// add target phrase to temp phrase and continue, but don't score yet
outPhrase.Append(prevHypo->GetPrefix());
}
if (calcNow)
{
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
{ // get from other prev hypo. faster
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
m_lmNGram.Assign(prevHypo->m_lmNGram);
}
else
{ // calc
lmList.CalcAllLMScores(outPhrase
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
}
if (calcNow) {
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
// get from other prev hypo. faster
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
m_lmNGram.Assign(prevHypo->m_lmNGram);
} else {
// calc
lmList.CalcAllLMScores(outPhrase
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
}
// create new phrase from suffix. score later when appended with next words
outPhrase.Clear();
outPhrase.Append(prevHypo->GetSuffix());
// create new phrase from suffix. score later when appended with next words
outPhrase.Clear();
outPhrase.Append(prevHypo->GetSuffix());
firstPhrase = false;
calcNow = false;
}
} // if (!targetWord.IsNonTerminal())
} // for (size_t targetPhrasePos
firstPhrase = false;
calcNow = false;
}
} // if (!targetWord.IsNonTerminal())
} // for (size_t targetPhrasePos
lmList.CalcAllLMScores(outPhrase
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
lmList.CalcAllLMScores(outPhrase
, m_lmNGram
, (firstPhrase) ? &m_lmPrefix : NULL);
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmPrefix);
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmNGram);
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmPrefix);
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmNGram);
/*
// lazy way. keep for comparison
Phrase outPhrase = GetOutputPhrase();
// cerr << outPhrase << " ";
/*
// lazy way. keep for comparison
Phrase outPhrase = GetOutputPhrase();
// cerr << outPhrase << " ";
float retFullScore, retNGramScore;
StaticData::Instance().GetAllLM().CalcScore(outPhrase
, retFullScore
, retNGramScore
, m_scoreBreakdown
, &m_lmNGram
, false);
*/
float retFullScore, retNGramScore;
StaticData::Instance().GetAllLM().CalcScore(outPhrase
, retFullScore
, retNGramScore
, m_scoreBreakdown
, &m_lmNGram
, false);
*/
}
void Hypothesis::AddArc(Hypothesis *loserHypo)
{
if (!m_arcList) {
if (loserHypo->m_arcList) // we don't have an arcList, but loser does
{
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
loserHypo->m_arcList = 0; // prevent a double deletion
}
else
{ this->m_arcList = new ArcList(); }
} else {
if (loserHypo->m_arcList) { // both have an arc list: merge. delete loser
size_t my_size = m_arcList->size();
size_t add_size = loserHypo->m_arcList->size();
this->m_arcList->resize(my_size + add_size, 0);
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(Hypothesis *));
delete loserHypo->m_arcList;
loserHypo->m_arcList = 0;
} else { // loserHypo doesn't have any arcs
// DO NOTHING
}
}
m_arcList->push_back(loserHypo);
if (!m_arcList) {
if (loserHypo->m_arcList) { // we don't have an arcList, but loser does
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
loserHypo->m_arcList = 0; // prevent a double deletion
} else {
this->m_arcList = new ArcList();
}
} else {
if (loserHypo->m_arcList) { // both have an arc list: merge. delete loser
size_t my_size = m_arcList->size();
size_t add_size = loserHypo->m_arcList->size();
this->m_arcList->resize(my_size + add_size, 0);
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(Hypothesis *));
delete loserHypo->m_arcList;
loserHypo->m_arcList = 0;
} else { // loserHypo doesn't have any arcs
// DO NOTHING
}
}
m_arcList->push_back(loserHypo);
}
// sorting helper
struct CompareChartHypothesisTotalScore
{
bool operator()(const Hypothesis* hypo1, const Hypothesis* hypo2) const
{
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
}
struct CompareChartHypothesisTotalScore {
bool operator()(const Hypothesis* hypo1, const Hypothesis* hypo2) const {
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
}
};
void Hypothesis::CleanupArcList()
{
// point this hypo's main hypo to itself
m_winningHypo = this;
// point this hypo's main hypo to itself
m_winningHypo = this;
if (!m_arcList) return;
if (!m_arcList) return;
/* keep only number of arcs we need to create all n-best paths.
* However, may not be enough if only unique candidates are needed,
* so we'll keep all of arc list if nedd distinct n-best list
*/
const StaticData &staticData = StaticData::Instance();
size_t nBestSize = staticData.GetNBestSize();
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph();
/* keep only number of arcs we need to create all n-best paths.
* However, may not be enough if only unique candidates are needed,
* so we'll keep all of arc list if nedd distinct n-best list
*/
const StaticData &staticData = StaticData::Instance();
size_t nBestSize = staticData.GetNBestSize();
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph();
if (!distinctNBest && m_arcList->size() > nBestSize)
{ // prune arc list only if there too many arcs
nth_element(m_arcList->begin()
, m_arcList->begin() + nBestSize - 1
, m_arcList->end()
, CompareChartHypothesisTotalScore());
// delete bad ones
ArcList::iterator iter;
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter)
{
Hypothesis *arc = *iter;
Hypothesis::Delete(arc);
}
m_arcList->erase(m_arcList->begin() + nBestSize
, m_arcList->end());
}
if (!distinctNBest && m_arcList->size() > nBestSize) {
// prune arc list only if there too many arcs
nth_element(m_arcList->begin()
, m_arcList->begin() + nBestSize - 1
, m_arcList->end()
, CompareChartHypothesisTotalScore());
// delete bad ones
ArcList::iterator iter;
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) {
Hypothesis *arc = *iter;
Hypothesis::Delete(arc);
}
m_arcList->erase(m_arcList->begin() + nBestSize
, m_arcList->end());
}
// set all arc's main hypo variable to this hypo
ArcList::iterator iter = m_arcList->begin();
for (; iter != m_arcList->end() ; ++iter) {
Hypothesis *arc = *iter;
arc->SetWinningHypo(this);
}
// set all arc's main hypo variable to this hypo
ArcList::iterator iter = m_arcList->begin();
for (; iter != m_arcList->end() ; ++iter)
{
Hypothesis *arc = *iter;
arc->SetWinningHypo(this);
}
//cerr << m_arcList->size() << " ";
}
void Hypothesis::SetWinningHypo(const Hypothesis *hypo)
{
m_winningHypo = hypo;
// never gonna use to recombine. clear prefix & suffix phrases to save mem
m_contextPrefix.Clear();
m_contextSuffix.Clear();
@ -408,29 +377,28 @@ TO_STRING_BODY(Hypothesis)
// friend
ostream& operator<<(ostream& out, const Hypothesis& hypo)
{
//Phrase outPhrase(Output);
//hypo.CreateOutputPhrase(outPhrase);
//Phrase outPhrase(Output);
//hypo.CreateOutputPhrase(outPhrase);
// words bitmap
out << " " << hypo.GetId()
<< " " << hypo.GetCurrTargetPhrase()
//<< " " << outPhrase
<< " " << hypo.GetCurrSourceRange();
//<< " " << hypo.m_currSourceWordsRange
HypoList::const_iterator iter;
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter)
{
const Hypothesis &prevHypo = **iter;
out << " " << prevHypo.GetId();
}
// words bitmap
out << " " << hypo.GetId()
<< " " << hypo.GetCurrTargetPhrase()
//<< " " << outPhrase
<< " " << hypo.GetCurrSourceRange();
//<< " " << hypo.m_currSourceWordsRange
out << " [total=" << hypo.GetTotalScore() << "]";
out << " " << hypo.GetScoreBreakdown();
HypoList::const_iterator iter;
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter) {
const Hypothesis &prevHypo = **iter;
out << " " << prevHypo.GetId();
}
//out << endl;
out << " [total=" << hypo.GetTotalScore() << "]";
out << " " << hypo.GetScoreBreakdown();
return out;
//out << endl;
return out;
}
}

180
moses-chart/src/ChartHypothesis.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -34,127 +34,133 @@ namespace MosesChart
class QueueEntry;
class Hypothesis;
class Manager;
typedef std::vector<Hypothesis*> ArcList;
class Hypothesis
{
friend std::ostream& operator<<(std::ostream&, const Hypothesis&);
friend std::ostream& operator<<(std::ostream&, const Hypothesis&);
protected:
#ifdef USE_HYPO_POOL
static ObjectPool<Hypothesis> s_objectPool;
static ObjectPool<Hypothesis> s_objectPool;
#endif
static unsigned int s_HypothesesCreated;
static unsigned int s_HypothesesCreated;
int m_id; /**< numeric ID of this hypothesis, used for logging */
const Moses::ChartTranslationOption &m_transOpt;
int m_id; /**< numeric ID of this hypothesis, used for logging */
const Moses::ChartTranslationOption &m_transOpt;
Moses::Phrase m_contextPrefix, m_contextSuffix;
const std::vector<size_t> &m_wordsConsumedTargetOrder; // same size as target phrase ?
Moses::WordsRange m_currSourceWordsRange;
Moses::ScoreComponentCollection m_scoreBreakdown /*! detailed score break-down by components (for instance language model, word penalty, etc) */
,m_lmNGram
,m_lmPrefix;
float m_totalScore;
size_t m_numTargetTerminals;
Moses::Phrase m_contextPrefix, m_contextSuffix;
const std::vector<size_t> &m_wordsConsumedTargetOrder; // same size as target phrase ?
Moses::WordsRange m_currSourceWordsRange;
Moses::ScoreComponentCollection m_scoreBreakdown /*! detailed score break-down by components (for instance language model, word penalty, etc) */
,m_lmNGram
,m_lmPrefix;
float m_totalScore;
size_t m_numTargetTerminals;
ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
const Hypothesis *m_winningHypo;
ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
const Hypothesis *m_winningHypo;
std::vector<const Hypothesis*> m_prevHypos;
std::vector<const Hypothesis*> m_prevHypos;
Manager& m_manager;
size_t CalcPrefix(Moses::Phrase &ret, size_t size) const;
size_t CalcSuffix(Moses::Phrase &ret, size_t size) const;
Manager& m_manager;
void CalcLMScore();
size_t CalcPrefix(Moses::Phrase &ret, size_t size) const;
size_t CalcSuffix(Moses::Phrase &ret, size_t size) const;
Hypothesis(); // not implemented
Hypothesis(const Hypothesis &copy); // not implemented
void CalcLMScore();
Hypothesis(); // not implemented
Hypothesis(const Hypothesis &copy); // not implemented
public:
static void ResetHypoCount()
{ s_HypothesesCreated = 0; }
static unsigned int GetHypoCount()
{ return s_HypothesesCreated; }
static void ResetHypoCount() {
s_HypothesesCreated = 0;
}
static unsigned int GetHypoCount() {
return s_HypothesesCreated;
}
#ifdef USE_HYPO_POOL
void *operator new(size_t num_bytes)
{
void *ptr = s_objectPool.getPtr();
return ptr;
}
void *operator new(size_t num_bytes) {
void *ptr = s_objectPool.getPtr();
return ptr;
}
static void Delete(Hypothesis *hypo)
{
s_objectPool.freeObject(hypo);
}
static void Delete(Hypothesis *hypo) {
s_objectPool.freeObject(hypo);
}
#else
static void Delete(Hypothesis *hypo)
{
delete hypo;
}
static void Delete(Hypothesis *hypo) {
delete hypo;
}
#endif
explicit Hypothesis(const QueueEntry &queueEntry, Manager &manager);
~Hypothesis();
explicit Hypothesis(const QueueEntry &queueEntry, Manager &manager);
~Hypothesis();
int GetId()const
{ return m_id;}
const Moses::ChartTranslationOption &GetTranslationOption()const
{ return m_transOpt; }
const Moses::TargetPhrase &GetCurrTargetPhrase()const
{ return m_transOpt.GetTargetPhrase(); }
const Moses::WordsRange &GetCurrSourceRange()const
{ return m_currSourceWordsRange; }
inline const ArcList* GetArcList() const
{
return m_arcList;
}
int GetId()const {
return m_id;
}
const Moses::ChartTranslationOption &GetTranslationOption()const {
return m_transOpt;
}
const Moses::TargetPhrase &GetCurrTargetPhrase()const {
return m_transOpt.GetTargetPhrase();
}
const Moses::WordsRange &GetCurrSourceRange()const {
return m_currSourceWordsRange;
}
inline const ArcList* GetArcList() const {
return m_arcList;
}
void CreateOutputPhrase(Moses::Phrase &outPhrase) const;
Moses::Phrase GetOutputPhrase() const;
void CreateOutputPhrase(Moses::Phrase &outPhrase) const;
Moses::Phrase GetOutputPhrase() const;
int LMContextCompare(const Hypothesis &other) const;
int LMContextCompare(const Hypothesis &other) const;
const Moses::Phrase &GetPrefix() const
{ return m_contextPrefix; }
const Moses::Phrase &GetSuffix() const
{ return m_contextSuffix; }
const Moses::Phrase &GetPrefix() const {
return m_contextPrefix;
}
const Moses::Phrase &GetSuffix() const {
return m_contextSuffix;
}
void CalcScore();
void CalcScore();
void AddArc(Hypothesis *loserHypo);
void CleanupArcList();
void SetWinningHypo(const Hypothesis *hypo);
void AddArc(Hypothesis *loserHypo);
void CleanupArcList();
void SetWinningHypo(const Hypothesis *hypo);
const Moses::ScoreComponentCollection &GetScoreBreakdown() const
{ return m_scoreBreakdown; }
float GetTotalScore() const
{ return m_totalScore; }
const Moses::ScoreComponentCollection &GetScoreBreakdown() const {
return m_scoreBreakdown;
}
float GetTotalScore() const {
return m_totalScore;
}
const std::vector<const Hypothesis*> &GetPrevHypos() const
{ return m_prevHypos; }
const std::vector<const Hypothesis*> &GetPrevHypos() const {
return m_prevHypos;
}
size_t GetWordsConsumedTargetOrder(size_t pos) const
{
assert(pos < m_wordsConsumedTargetOrder.size());
return m_wordsConsumedTargetOrder[pos];
}
size_t GetWordsConsumedTargetOrder(size_t pos) const {
assert(pos < m_wordsConsumedTargetOrder.size());
return m_wordsConsumedTargetOrder[pos];
}
const Moses::Word &GetTargetLHS() const
{ return GetCurrTargetPhrase().GetTargetLHS(); }
const Moses::Word &GetTargetLHS() const {
return GetCurrTargetPhrase().GetTargetLHS();
}
size_t GetNumTargetTerminals() const
{
return m_numTargetTerminals;
}
size_t GetNumTargetTerminals() const {
return m_numTargetTerminals;
}
TO_STRING();
TO_STRING();
}; // class Hypothesis

451
moses-chart/src/ChartHypothesisCollection.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -33,276 +33,253 @@ namespace MosesChart
HypothesisCollection::HypothesisCollection()
{
const StaticData &staticData = StaticData::Instance();
const StaticData &staticData = StaticData::Instance();
m_beamWidth = staticData.GetBeamWidth();
m_maxHypoStackSize = staticData.GetMaxHypoStackSize();
m_nBestIsEnabled = staticData.IsNBestEnabled();
m_bestScore = -std::numeric_limits<float>::infinity();
m_beamWidth = staticData.GetBeamWidth();
m_maxHypoStackSize = staticData.GetMaxHypoStackSize();
m_nBestIsEnabled = staticData.IsNBestEnabled();
m_bestScore = -std::numeric_limits<float>::infinity();
}
HypothesisCollection::~HypothesisCollection()
{
HCType::iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
{
Hypothesis *hypo = *iter;
Hypothesis::Delete(hypo);
}
//Moses::RemoveAllInColl(m_hypos);
HCType::iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
Hypothesis *hypo = *iter;
Hypothesis::Delete(hypo);
}
//Moses::RemoveAllInColl(m_hypos);
}
bool HypothesisCollection::AddHypothesis(Hypothesis *hypo, Manager &manager)
{
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth)
{ // really bad score. don't bother adding hypo into collection
manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, too bad for stack" << std::endl);
Hypothesis::Delete(hypo);
return false;
}
// over threshold, try to add to collection
std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
if (addRet.second)
{ // nothing found. add to collection
return true;
}
// equiv hypo exists, recombine with other hypo
HCType::iterator &iterExisting = addRet.first;
Hypothesis *hypoExisting = *iterExisting;
assert(iterExisting != m_hypos.end());
//StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
// found existing hypo with same target ending.
// keep the best 1
if (hypo->GetTotalScore() > hypoExisting->GetTotalScore())
{ // incoming hypo is better than the one we have
VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
if (m_nBestIsEnabled) {
hypo->AddArc(hypoExisting);
Detach(iterExisting);
} else {
Remove(iterExisting);
}
bool added = Add(hypo, manager).second;
if (!added)
{
iterExisting = m_hypos.find(hypo);
TRACE_ERR("Offending hypo = " << **iterExisting << endl);
abort();
}
return false;
}
else
{ // already storing the best hypo. discard current hypo
VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
if (m_nBestIsEnabled) {
hypoExisting->AddArc(hypo);
} else {
Hypothesis::Delete(hypo);
}
return false;
}
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth) {
// really bad score. don't bother adding hypo into collection
manager.GetSentenceStats().AddDiscarded();
VERBOSE(3,"discarded, too bad for stack" << std::endl);
Hypothesis::Delete(hypo);
return false;
}
// over threshold, try to add to collection
std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
if (addRet.second) {
// nothing found. add to collection
return true;
}
// equiv hypo exists, recombine with other hypo
HCType::iterator &iterExisting = addRet.first;
Hypothesis *hypoExisting = *iterExisting;
assert(iterExisting != m_hypos.end());
//StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
// found existing hypo with same target ending.
// keep the best 1
if (hypo->GetTotalScore() > hypoExisting->GetTotalScore()) {
// incoming hypo is better than the one we have
VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
if (m_nBestIsEnabled) {
hypo->AddArc(hypoExisting);
Detach(iterExisting);
} else {
Remove(iterExisting);
}
bool added = Add(hypo, manager).second;
if (!added) {
iterExisting = m_hypos.find(hypo);
TRACE_ERR("Offending hypo = " << **iterExisting << endl);
abort();
}
return false;
} else {
// already storing the best hypo. discard current hypo
VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
if (m_nBestIsEnabled) {
hypoExisting->AddArc(hypo);
} else {
Hypothesis::Delete(hypo);
}
return false;
}
}
pair<HypothesisCollection::HCType::iterator, bool> HypothesisCollection::Add(Hypothesis *hypo, Manager &manager)
{
std::pair<HCType::iterator, bool> ret = m_hypos.insert(hypo);
if (ret.second)
{ // equiv hypo doesn't exists
VERBOSE(3,"added hyp to stack");
// Update best score, if this hypothesis is new best
if (hypo->GetTotalScore() > m_bestScore)
{
VERBOSE(3,", best on stack");
m_bestScore = hypo->GetTotalScore();
}
// Prune only if stack is twice as big as needed (lazy pruning)
VERBOSE(3,", now size " << m_hypos.size());
if (m_hypos.size() > 2*m_maxHypoStackSize-1)
{
PruneToSize(manager);
}
else {
VERBOSE(3,std::endl);
}
}
return ret;
std::pair<HCType::iterator, bool> ret = m_hypos.insert(hypo);
if (ret.second) {
// equiv hypo doesn't exists
VERBOSE(3,"added hyp to stack");
// Update best score, if this hypothesis is new best
if (hypo->GetTotalScore() > m_bestScore) {
VERBOSE(3,", best on stack");
m_bestScore = hypo->GetTotalScore();
}
// Prune only if stack is twice as big as needed (lazy pruning)
VERBOSE(3,", now size " << m_hypos.size());
if (m_hypos.size() > 2*m_maxHypoStackSize-1) {
PruneToSize(manager);
} else {
VERBOSE(3,std::endl);
}
}
return ret;
}
/** Remove hypothesis pointed to by iterator but don't delete the object. */
void HypothesisCollection::Detach(const HCType::iterator &iter)
{
m_hypos.erase(iter);
m_hypos.erase(iter);
}
void HypothesisCollection::Remove(const HCType::iterator &iter)
{
Hypothesis *h = *iter;
/*
stringstream strme("");
strme << h->GetOutputPhrase();
string toFind = "the goal of gene scientists is ";
size_t pos = toFind.find(strme.str());
if (pos == 0)
{
cerr << pos << " " << strme.str() << *h << endl;
cerr << *this << endl;
}
*/
Detach(iter);
Hypothesis::Delete(h);
Hypothesis *h = *iter;
/*
stringstream strme("");
strme << h->GetOutputPhrase();
string toFind = "the goal of gene scientists is ";
size_t pos = toFind.find(strme.str());
if (pos == 0)
{
cerr << pos << " " << strme.str() << *h << endl;
cerr << *this << endl;
}
*/
Detach(iter);
Hypothesis::Delete(h);
}
void HypothesisCollection::PruneToSize(Manager &manager)
{
if (GetSize() > m_maxHypoStackSize) // ok, if not over the limit
{
priority_queue<float> bestScores;
// push all scores to a heap
// (but never push scores below m_bestScore+m_beamWidth)
HCType::iterator iter = m_hypos.begin();
float score = 0;
while (iter != m_hypos.end())
{
Hypothesis *hypo = *iter;
score = hypo->GetTotalScore();
if (score > m_bestScore+m_beamWidth)
{
bestScores.push(score);
}
++iter;
}
// pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
// ensure to never pop beyond heap size
size_t minNewSizeHeapSize = m_maxHypoStackSize > bestScores.size() ? bestScores.size() : m_maxHypoStackSize;
for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
bestScores.pop();
// and remember the threshold
float scoreThreshold = bestScores.top();
// delete all hypos under score threshold
iter = m_hypos.begin();
while (iter != m_hypos.end())
{
Hypothesis *hypo = *iter;
float score = hypo->GetTotalScore();
if (score < scoreThreshold)
{
HCType::iterator iterRemove = iter++;
Remove(iterRemove);
manager.GetSentenceStats().AddPruning();
}
else
{
++iter;
}
}
VERBOSE(3,", pruned to size " << m_hypos.size() << endl);
IFVERBOSE(3)
{
TRACE_ERR("stack now contains: ");
for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++)
{
Hypothesis *hypo = *iter;
TRACE_ERR( hypo->GetId() << " (" << hypo->GetTotalScore() << ") ");
}
TRACE_ERR( endl);
}
// desperation pruning
if (m_hypos.size() > m_maxHypoStackSize * 2)
{
std::vector<Hypothesis*> hyposOrdered;
// sort hypos
std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(hyposOrdered, hyposOrdered.end()));
std::sort(hyposOrdered.begin(), hyposOrdered.end(), ChartHypothesisScoreOrderer());
//keep only |size|. delete the rest
std::vector<Hypothesis*>::iterator iter;
for (iter = hyposOrdered.begin() + (m_maxHypoStackSize * 2); iter != hyposOrdered.end(); ++iter)
{
Hypothesis *hypo = *iter;
HCType::iterator iterFindHypo = m_hypos.find(hypo);
assert(iterFindHypo != m_hypos.end());
Remove(iterFindHypo);
}
}
}
if (GetSize() > m_maxHypoStackSize) { // ok, if not over the limit
priority_queue<float> bestScores;
// push all scores to a heap
// (but never push scores below m_bestScore+m_beamWidth)
HCType::iterator iter = m_hypos.begin();
float score = 0;
while (iter != m_hypos.end()) {
Hypothesis *hypo = *iter;
score = hypo->GetTotalScore();
if (score > m_bestScore+m_beamWidth) {
bestScores.push(score);
}
++iter;
}
// pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
// ensure to never pop beyond heap size
size_t minNewSizeHeapSize = m_maxHypoStackSize > bestScores.size() ? bestScores.size() : m_maxHypoStackSize;
for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
bestScores.pop();
// and remember the threshold
float scoreThreshold = bestScores.top();
// delete all hypos under score threshold
iter = m_hypos.begin();
while (iter != m_hypos.end()) {
Hypothesis *hypo = *iter;
float score = hypo->GetTotalScore();
if (score < scoreThreshold) {
HCType::iterator iterRemove = iter++;
Remove(iterRemove);
manager.GetSentenceStats().AddPruning();
} else {
++iter;
}
}
VERBOSE(3,", pruned to size " << m_hypos.size() << endl);
IFVERBOSE(3) {
TRACE_ERR("stack now contains: ");
for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++) {
Hypothesis *hypo = *iter;
TRACE_ERR( hypo->GetId() << " (" << hypo->GetTotalScore() << ") ");
}
TRACE_ERR( endl);
}
// desperation pruning
if (m_hypos.size() > m_maxHypoStackSize * 2) {
std::vector<Hypothesis*> hyposOrdered;
// sort hypos
std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(hyposOrdered, hyposOrdered.end()));
std::sort(hyposOrdered.begin(), hyposOrdered.end(), ChartHypothesisScoreOrderer());
//keep only |size|. delete the rest
std::vector<Hypothesis*>::iterator iter;
for (iter = hyposOrdered.begin() + (m_maxHypoStackSize * 2); iter != hyposOrdered.end(); ++iter) {
Hypothesis *hypo = *iter;
HCType::iterator iterFindHypo = m_hypos.find(hypo);
assert(iterFindHypo != m_hypos.end());
Remove(iterFindHypo);
}
}
}
}
void HypothesisCollection::SortHypotheses()
{
assert(m_hyposOrdered.empty());
if (!m_hypos.empty())
{
// done everything for this cell.
// sort
// put into vec
m_hyposOrdered.reserve(m_hypos.size());
std::copy(m_hypos.begin(), m_hypos.end(), back_inserter(m_hyposOrdered));
std::sort(m_hyposOrdered.begin(), m_hyposOrdered.end(), ChartHypothesisScoreOrderer());
}
assert(m_hyposOrdered.empty());
if (!m_hypos.empty()) {
// done everything for this cell.
// sort
// put into vec
m_hyposOrdered.reserve(m_hypos.size());
std::copy(m_hypos.begin(), m_hypos.end(), back_inserter(m_hyposOrdered));
std::sort(m_hyposOrdered.begin(), m_hyposOrdered.end(), ChartHypothesisScoreOrderer());
}
}
void HypothesisCollection::CleanupArcList()
{
HCType::iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
{
Hypothesis *mainHypo = *iter;
mainHypo->CleanupArcList();
}
{
HCType::iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
Hypothesis *mainHypo = *iter;
mainHypo->CleanupArcList();
}
}
void HypothesisCollection::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
{
HCType::const_iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
{
Hypothesis &mainHypo = **iter;
outputSearchGraphStream << translationId << " " << mainHypo << endl;
const ArcList *arcList = mainHypo.GetArcList();
if (arcList)
{
ArcList::const_iterator iterArc;
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc)
{
const Hypothesis &arc = **iterArc;
outputSearchGraphStream << translationId << " " << arc << endl;
}
}
}
}
HCType::const_iterator iter;
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
Hypothesis &mainHypo = **iter;
outputSearchGraphStream << translationId << " " << mainHypo << endl;
const ArcList *arcList = mainHypo.GetArcList();
if (arcList) {
ArcList::const_iterator iterArc;
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
const Hypothesis &arc = **iterArc;
outputSearchGraphStream << translationId << " " << arc << endl;
}
}
}
}
std::ostream& operator<<(std::ostream &out, const HypothesisCollection &coll)
{
HypoList::const_iterator iterInside;
for (iterInside = coll.m_hyposOrdered.begin(); iterInside != coll.m_hyposOrdered.end(); ++iterInside)
{
const Hypothesis &hypo = **iterInside;
out << hypo << endl;
}
return out;
{
HypoList::const_iterator iterInside;
for (iterInside = coll.m_hyposOrdered.begin(); iterInside != coll.m_hyposOrdered.end(); ++iterInside) {
const Hypothesis &hypo = **iterInside;
out << hypo << endl;
}
return out;
}
} // namespace

158
moses-chart/src/ChartHypothesisCollection.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -27,104 +27,108 @@
namespace MosesChart
{
// order by descending score
class ChartHypothesisScoreOrderer
{
public:
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
{
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
}
};
{
public:
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
}
};
class HypothesisRecombinationOrderer
{
public:
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
{
// assert in same cell
const Moses::WordsRange &rangeA = hypoA->GetCurrSourceRange()
, &rangeB = hypoB->GetCurrSourceRange();
assert(rangeA == rangeB);
/*
int ret = Moses::Word::Compare(hypoA->GetTargetLHS(), hypoB->GetTargetLHS());
if (ret != 0)
return (ret < 0);
*/
// shouldn't be mixing hypos with different lhs
assert(hypoA->GetTargetLHS() == hypoB->GetTargetLHS());
int ret = hypoA->LMContextCompare(*hypoB);
if (ret != 0)
return (ret < 0);
return false;
}
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
// assert in same cell
const Moses::WordsRange &rangeA = hypoA->GetCurrSourceRange()
, &rangeB = hypoB->GetCurrSourceRange();
assert(rangeA == rangeB);
/*
int ret = Moses::Word::Compare(hypoA->GetTargetLHS(), hypoB->GetTargetLHS());
if (ret != 0)
return (ret < 0);
*/
// shouldn't be mixing hypos with different lhs
assert(hypoA->GetTargetLHS() == hypoB->GetTargetLHS());
int ret = hypoA->LMContextCompare(*hypoB);
if (ret != 0)
return (ret < 0);
return false;
}
};
// order by descending score
class HypothesisScoreOrderer
{
public:
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
{
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
}
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
}
};
// 1 of these for each target LHS in each cell
class HypothesisCollection
{
friend std::ostream& operator<<(std::ostream&, const HypothesisCollection&);
friend std::ostream& operator<<(std::ostream&, const HypothesisCollection&);
protected:
typedef std::set<Hypothesis*, HypothesisRecombinationOrderer> HCType;
HCType m_hypos;
HypoList m_hyposOrdered;
float m_bestScore; /**< score of the best hypothesis in collection */
float m_beamWidth; /**< minimum score due to threashold pruning */
size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
/** add hypothesis to stack. Prune if necessary.
* Returns false if equiv hypo exists in collection, otherwise returns true
*/
std::pair<HCType::iterator, bool> Add(Hypothesis *hypo, Manager &manager);
typedef std::set<Hypothesis*, HypothesisRecombinationOrderer> HCType;
HCType m_hypos;
HypoList m_hyposOrdered;
float m_bestScore; /**< score of the best hypothesis in collection */
float m_beamWidth; /**< minimum score due to threashold pruning */
size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
/** add hypothesis to stack. Prune if necessary.
* Returns false if equiv hypo exists in collection, otherwise returns true
*/
std::pair<HCType::iterator, bool> Add(Hypothesis *hypo, Manager &manager);
public:
typedef HCType::iterator iterator;
typedef HCType::const_iterator const_iterator;
//! iterators
const_iterator begin() const { return m_hypos.begin(); }
const_iterator end() const { return m_hypos.end(); }
HypothesisCollection();
~HypothesisCollection();
bool AddHypothesis(Hypothesis *hypo, Manager &manager);
typedef HCType::iterator iterator;
typedef HCType::const_iterator const_iterator;
//! iterators
const_iterator begin() const {
return m_hypos.begin();
}
const_iterator end() const {
return m_hypos.end();
}
//! remove hypothesis pointed to by iterator but don't delete the object
void Detach(const HCType::iterator &iter);
/** destroy Hypothesis pointed to by iterator (object pool version) */
void Remove(const HCType::iterator &iter);
void PruneToSize(Manager &manager);
HypothesisCollection();
~HypothesisCollection();
bool AddHypothesis(Hypothesis *hypo, Manager &manager);
size_t GetSize() const
{ return m_hypos.size(); }
size_t GetHypo() const
{ return m_hypos.size(); }
//! remove hypothesis pointed to by iterator but don't delete the object
void Detach(const HCType::iterator &iter);
/** destroy Hypothesis pointed to by iterator (object pool version) */
void Remove(const HCType::iterator &iter);
void SortHypotheses();
void CleanupArcList();
void PruneToSize(Manager &manager);
const HypoList &GetSortedHypotheses() const
{ return m_hyposOrdered; }
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
size_t GetSize() const {
return m_hypos.size();
}
size_t GetHypo() const {
return m_hypos.size();
}
void SortHypotheses();
void CleanupArcList();
const HypoList &GetSortedHypotheses() const {
return m_hyposOrdered;
}
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
};

270
moses-chart/src/ChartManager.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -34,25 +34,24 @@ using namespace Moses;
namespace Moses
{
extern bool g_debug;
extern bool g_debug;
}
namespace MosesChart
{
Manager::Manager(InputType const& source, const TranslationSystem* system)
:m_source(source)
,m_hypoStackColl(source, *this)
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
,m_system(system)
,m_start(clock())
:m_source(source)
,m_hypoStackColl(source, *this)
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
,m_system(system)
,m_start(clock())
{
m_system->InitializeBeforeSentenceProcessing(source);
m_system->InitializeBeforeSentenceProcessing(source);
const std::vector<PhraseDictionaryFeature*> &dictionaries = m_system->GetPhraseDictionaries();
m_ruleLookupManagers.reserve(dictionaries.size());
for (std::vector<PhraseDictionaryFeature*>::const_iterator p = dictionaries.begin();
p != dictionaries.end(); ++p)
{
p != dictionaries.end(); ++p) {
PhraseDictionaryFeature *pdf = *p;
const PhraseDictionary *dict = pdf->GetDictionary();
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
@ -62,150 +61,139 @@ Manager::Manager(InputType const& source, const TranslationSystem* system)
Manager::~Manager()
{
m_system->CleanUpAfterSentenceProcessing();
m_system->CleanUpAfterSentenceProcessing();
RemoveAllInColl(m_ruleLookupManagers);
clock_t end = clock();
float et = (end - m_start);
et /= (float)CLOCKS_PER_SEC;
VERBOSE(1, "Translation took " << et << " seconds" << endl);
clock_t end = clock();
float et = (end - m_start);
et /= (float)CLOCKS_PER_SEC;
VERBOSE(1, "Translation took " << et << " seconds" << endl);
}
void Manager::ProcessSentence()
{
VERBOSE(1,"Translating: " << m_source << endl);
VERBOSE(1,"Translating: " << m_source << endl);
ResetSentenceStats(m_source);
ResetSentenceStats(m_source);
VERBOSE(2,"Decoding: " << endl);
//Hypothesis::ResetHypoCount();
VERBOSE(2,"Decoding: " << endl);
//Hypothesis::ResetHypoCount();
// MAIN LOOP
size_t size = m_source.GetSize();
for (size_t width = 1; width <= size; ++width)
{
for (size_t startPos = 0; startPos <= size-width; ++startPos)
{
size_t endPos = startPos + width - 1;
WordsRange range(startPos, endPos);
//TRACE_ERR(" " << range << "=");
// create trans opt
m_transOptColl.CreateTranslationOptionsForRange(startPos, endPos);
//if (g_debug)
// cerr << m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
// decode
ChartCell &cell = m_hypoStackColl.Get(range);
// MAIN LOOP
size_t size = m_source.GetSize();
for (size_t width = 1; width <= size; ++width) {
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
size_t endPos = startPos + width - 1;
WordsRange range(startPos, endPos);
//TRACE_ERR(" " << range << "=");
cell.ProcessSentence(m_transOptColl.GetTranslationOptionList(range)
,m_hypoStackColl);
cell.PruneToSize();
cell.CleanupArcList();
cell.SortHypotheses();
//cerr << cell.GetSize();
//cerr << cell << endl;
//cell.OutputSizes(cerr);
}
}
// create trans opt
m_transOptColl.CreateTranslationOptionsForRange(startPos, endPos);
//if (g_debug)
// cerr << m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
IFVERBOSE(1) {
cerr << "Num of hypo = " << Hypothesis::GetHypoCount() << " --- cells:" << endl;
for (size_t startPos = 0; startPos < size; ++startPos)
{
cerr.width(3);
cerr << startPos << " ";
}
cerr << endl;
for (size_t width = 1; width <= size; width++)
{
for( size_t space = 0; space < width-1; space++ )
{
cerr << " ";
}
for (size_t startPos = 0; startPos <= size-width; ++startPos)
{
WordsRange range(startPos, startPos+width-1);
cerr.width(3);
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
}
cerr << endl;
}
}
// decode
ChartCell &cell = m_hypoStackColl.Get(range);
cell.ProcessSentence(m_transOptColl.GetTranslationOptionList(range)
,m_hypoStackColl);
cell.PruneToSize();
cell.CleanupArcList();
cell.SortHypotheses();
//cerr << cell.GetSize();
//cerr << cell << endl;
//cell.OutputSizes(cerr);
}
}
IFVERBOSE(1) {
cerr << "Num of hypo = " << Hypothesis::GetHypoCount() << " --- cells:" << endl;
for (size_t startPos = 0; startPos < size; ++startPos) {
cerr.width(3);
cerr << startPos << " ";
}
cerr << endl;
for (size_t width = 1; width <= size; width++) {
for( size_t space = 0; space < width-1; space++ ) {
cerr << " ";
}
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
WordsRange range(startPos, startPos+width-1);
cerr.width(3);
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
}
cerr << endl;
}
}
}
const Hypothesis *Manager::GetBestHypothesis() const
{
size_t size = m_source.GetSize();
size_t size = m_source.GetSize();
if (size == 0) // empty source
return NULL;
else
{
WordsRange range(0, size-1);
const ChartCell &lastCell = m_hypoStackColl.Get(range);
return lastCell.GetBestHypothesis();
}
if (size == 0) // empty source
return NULL;
else {
WordsRange range(0, size-1);
const ChartCell &lastCell = m_hypoStackColl.Get(range);
return lastCell.GetBestHypothesis();
}
}
void Manager::CalcNBest(size_t count, TrellisPathList &ret,bool onlyDistinct) const
{
size_t size = m_source.GetSize();
if (count == 0 || size == 0)
return;
TrellisPathCollection contenders;
set<Phrase> distinctHyps;
size_t size = m_source.GetSize();
if (count == 0 || size == 0)
return;
// add all pure paths
WordsRange range(0, size-1);
const ChartCell &lastCell = m_hypoStackColl.Get(range);
const Hypothesis *hypo = lastCell.GetBestHypothesis();
if (hypo == NULL)
{ // no hypothesis
return;
}
TrellisPathCollection contenders;
set<Phrase> distinctHyps;
MosesChart::TrellisPath *purePath = new TrellisPath(hypo);
contenders.Add(purePath);
// add all pure paths
WordsRange range(0, size-1);
const ChartCell &lastCell = m_hypoStackColl.Get(range);
const Hypothesis *hypo = lastCell.GetBestHypothesis();
// factor defines stopping point for distinct n-best list if too many candidates identical
size_t nBestFactor = StaticData::Instance().GetNBestFactor();
if (hypo == NULL) {
// no hypothesis
return;
}
MosesChart::TrellisPath *purePath = new TrellisPath(hypo);
contenders.Add(purePath);
// factor defines stopping point for distinct n-best list if too many candidates identical
size_t nBestFactor = StaticData::Instance().GetNBestFactor();
if (nBestFactor < 1) nBestFactor = 1000; // 0 = unlimited
// MAIN loop
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++)
{
// get next best from list of contenders
TrellisPath *path = contenders.pop();
assert(path);
// MAIN loop
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++) {
// get next best from list of contenders
TrellisPath *path = contenders.pop();
assert(path);
// create deviations from current best
path->CreateDeviantPaths(contenders);
// create deviations from current best
path->CreateDeviantPaths(contenders);
if(onlyDistinct)
{
Phrase tgtPhrase = path->GetOutputPhrase();
if (distinctHyps.insert(tgtPhrase).second)
if(onlyDistinct) {
Phrase tgtPhrase = path->GetOutputPhrase();
if (distinctHyps.insert(tgtPhrase).second)
ret.Add(path);
else
delete path;
else
delete path;
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
if (nBestFactor > 0)
contenders.Prune(count * nBestFactor);
}
else
{
ret.Add(path);
contenders.Prune(count);
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
if (nBestFactor > 0)
contenders.Prune(count * nBestFactor);
} else {
ret.Add(path);
contenders.Prune(count);
}
}
}
}
void Manager::CalcDecoderStatistics() const
@ -214,22 +202,20 @@ void Manager::CalcDecoderStatistics() const
void Manager::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
{
size_t size = m_source.GetSize();
for (size_t width = 1; width <= size; ++width)
{
for (size_t startPos = 0; startPos <= size-width; ++startPos)
{
size_t endPos = startPos + width - 1;
WordsRange range(startPos, endPos);
TRACE_ERR(" " << range << "=");
const ChartCell &cell = m_hypoStackColl.Get(range);
cell.GetSearchGraph(translationId, outputSearchGraphStream);
}
}
size_t size = m_source.GetSize();
for (size_t width = 1; width <= size; ++width) {
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
size_t endPos = startPos + width - 1;
WordsRange range(startPos, endPos);
TRACE_ERR(" " << range << "=");
const ChartCell &cell = m_hypoStackColl.Get(range);
cell.GetSearchGraph(translationId, outputSearchGraphStream);
}
}
}
} // namespace

52
moses-chart/src/ChartManager.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -41,39 +41,41 @@ class TrellisPathList;
class Manager
{
protected:
Moses::InputType const& m_source; /**< source sentence to be translated */
ChartCellCollection m_hypoStackColl;
TranslationOptionCollection m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
Moses::InputType const& m_source; /**< source sentence to be translated */
ChartCellCollection m_hypoStackColl;
TranslationOptionCollection m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
std::auto_ptr<Moses::SentenceStats> m_sentenceStats;
const Moses::TranslationSystem* m_system;
clock_t m_start; /**< starting time, used for logging */
clock_t m_start; /**< starting time, used for logging */
std::vector<Moses::ChartRuleLookupManager*> m_ruleLookupManagers;
public:
Manager(Moses::InputType const& source, const Moses::TranslationSystem* system);
~Manager();
void ProcessSentence();
const Hypothesis *GetBestHypothesis() const;
void CalcNBest(size_t count, MosesChart::TrellisPathList &ret,bool onlyDistinct=0) const;
~Manager();
void ProcessSentence();
const Hypothesis *GetBestHypothesis() const;
void CalcNBest(size_t count, MosesChart::TrellisPathList &ret,bool onlyDistinct=0) const;
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
const Moses::InputType& GetSource() const {return m_source;}
const Moses::TranslationSystem* GetTranslationSystem() const {return m_system;}
Moses::SentenceStats& GetSentenceStats() const
{
const Moses::InputType& GetSource() const {
return m_source;
}
const Moses::TranslationSystem* GetTranslationSystem() const {
return m_system;
}
Moses::SentenceStats& GetSentenceStats() const {
return *m_sentenceStats;
}
/***
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
*/
void CalcDecoderStatistics() const;
void ResetSentenceStats(const Moses::InputType& source)
{
/***
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
*/
void CalcDecoderStatistics() const;
void ResetSentenceStats(const Moses::InputType& source) {
m_sentenceStats = std::auto_ptr<Moses::SentenceStats>(new Moses::SentenceStats(source));
}
};
}

362
moses-chart/src/ChartTranslationOptionCollection.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -36,266 +36,252 @@ namespace MosesChart
{
TranslationOptionCollection::TranslationOptionCollection(InputType const& source
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
:m_source(source)
,m_system(system)
,m_decodeGraphList(system->GetDecodeGraphs())
,m_hypoStackColl(hypoStackColl)
,m_collection(source.GetSize())
,m_ruleLookupManagers(ruleLookupManagers)
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
:m_source(source)
,m_system(system)
,m_decodeGraphList(system->GetDecodeGraphs())
,m_hypoStackColl(hypoStackColl)
,m_collection(source.GetSize())
,m_ruleLookupManagers(ruleLookupManagers)
{
// create 2-d vector
size_t size = source.GetSize();
for (size_t startPos = 0 ; startPos < size ; ++startPos)
{
m_collection[startPos].reserve(size-startPos);
for (size_t endPos = startPos ; endPos < size ; ++endPos)
{
m_collection[startPos].push_back( ChartTranslationOptionList(WordsRange(startPos, endPos)) );
}
}
// create 2-d vector
size_t size = source.GetSize();
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
m_collection[startPos].reserve(size-startPos);
for (size_t endPos = startPos ; endPos < size ; ++endPos) {
m_collection[startPos].push_back( ChartTranslationOptionList(WordsRange(startPos, endPos)) );
}
}
}
TranslationOptionCollection::~TranslationOptionCollection()
{
RemoveAllInColl(m_unksrcs);
RemoveAllInColl(m_cacheTargetPhrase);
RemoveAllInColl(m_unksrcs);
RemoveAllInColl(m_cacheTargetPhrase);
std::list<std::vector<Moses::WordConsumed*>* >::iterator iterOuter;
for (iterOuter = m_cachedWordsConsumed.begin(); iterOuter != m_cachedWordsConsumed.end(); ++iterOuter)
{
std::vector<Moses::WordConsumed*> &inner = **iterOuter;
RemoveAllInColl(inner);
}
std::list<std::vector<Moses::WordConsumed*>* >::iterator iterOuter;
for (iterOuter = m_cachedWordsConsumed.begin(); iterOuter != m_cachedWordsConsumed.end(); ++iterOuter) {
std::vector<Moses::WordConsumed*> &inner = **iterOuter;
RemoveAllInColl(inner);
}
RemoveAllInColl(m_cachedWordsConsumed);
RemoveAllInColl(m_cachedWordsConsumed);
}
void TranslationOptionCollection::CreateTranslationOptionsForRange(
size_t startPos
, size_t endPos)
size_t startPos
, size_t endPos)
{
ChartTranslationOptionList &chartRuleColl = GetTranslationOptionList(startPos, endPos);
const WordsRange &wordsRange = chartRuleColl.GetSourceRange();
assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
{
const DecodeGraph &decodeGraph = **iterDecodeGraph;
assert(decodeGraph.GetSize() == 1);
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
const DecodeGraph &decodeGraph = **iterDecodeGraph;
assert(decodeGraph.GetSize() == 1);
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
size_t maxSpan = decodeGraph.GetMaxChartSpan();
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan)
{
size_t maxSpan = decodeGraph.GetMaxChartSpan();
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan) {
ruleLookupManager.GetChartRuleCollection(wordsRange, true, chartRuleColl);
}
}
}
}
ProcessUnknownWord(startPos, endPos);
ProcessUnknownWord(startPos, endPos);
Prune(startPos, endPos);
Prune(startPos, endPos);
Sort(startPos, endPos);
Sort(startPos, endPos);
}
//! Force a creation of a translation option where there are none for a particular source position.
void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t endPos)
{
if (startPos != endPos)
{ // only for 1 word phrases
return;
}
if (startPos != endPos) {
// only for 1 word phrases
return;
}
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
const WordsRange &wordsRange = fullList.GetSourceRange();
// try to translation for coverage with no trans by expanding table limit
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
// try to translation for coverage with no trans by expanding table limit
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
{
const DecodeGraph &decodeGraph = **iterDecodeGraph;
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
const DecodeGraph &decodeGraph = **iterDecodeGraph;
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
size_t numTransOpt = fullList.GetSize();
if (numTransOpt == 0)
{
size_t numTransOpt = fullList.GetSize();
if (numTransOpt == 0) {
ruleLookupManager.GetChartRuleCollection(wordsRange, false, fullList);
}
}
}
}
assert(iterRuleLookupManagers == m_ruleLookupManagers.end());
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
// create unknown words for 1 word coverage where we don't have any trans options
if (fullList.GetSize() == 0 || alwaysCreateDirectTranslationOption)
ProcessUnknownWord(startPos);
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
// create unknown words for 1 word coverage where we don't have any trans options
if (fullList.GetSize() == 0 || alwaysCreateDirectTranslationOption)
ProcessUnknownWord(startPos);
}
ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos)
{
size_t sizeVec = m_collection[startPos].size();
assert(endPos-startPos < sizeVec);
return m_collection[startPos][endPos - startPos];
size_t sizeVec = m_collection[startPos].size();
assert(endPos-startPos < sizeVec);
return m_collection[startPos][endPos - startPos];
}
const ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos) const
{
size_t sizeVec = m_collection[startPos].size();
assert(endPos-startPos < sizeVec);
return m_collection[startPos][endPos - startPos];
size_t sizeVec = m_collection[startPos].size();
assert(endPos-startPos < sizeVec);
return m_collection[startPos][endPos - startPos];
}
std::ostream& operator<<(std::ostream &out, const TranslationOptionCollection &coll)
{
std::vector< std::vector< ChartTranslationOptionList > >::const_iterator iterOuter;
for (iterOuter = coll.m_collection.begin(); iterOuter != coll.m_collection.end(); ++iterOuter)
{
const std::vector< ChartTranslationOptionList > &vecInner = *iterOuter;
std::vector< ChartTranslationOptionList >::const_iterator iterInner;
for (iterOuter = coll.m_collection.begin(); iterOuter != coll.m_collection.end(); ++iterOuter) {
const std::vector< ChartTranslationOptionList > &vecInner = *iterOuter;
std::vector< ChartTranslationOptionList >::const_iterator iterInner;
for (iterInner = vecInner.begin(); iterInner != vecInner.end(); ++iterInner)
{
const ChartTranslationOptionList &list = *iterInner;
out << list.GetSourceRange() << " = " << list.GetSize() << std::endl;
}
for (iterInner = vecInner.begin(); iterInner != vecInner.end(); ++iterInner) {
const ChartTranslationOptionList &list = *iterInner;
out << list.GetSourceRange() << " = " << list.GetSize() << std::endl;
}
}
return out;
return out;
}
// taken from TranslationOptionCollectionText.
void TranslationOptionCollection::ProcessUnknownWord(size_t sourcePos)
{
const Word &sourceWord = m_source.GetWord(sourcePos);
ProcessOneUnknownWord(sourceWord,sourcePos);
const Word &sourceWord = m_source.GetWord(sourcePos);
ProcessOneUnknownWord(sourceWord,sourcePos);
}
//! special handling of ONE unknown words.
void TranslationOptionCollection::ProcessOneUnknownWord(const Moses::Word &sourceWord, size_t sourcePos, size_t length)
{
// unknown word, add as trans opt
const StaticData &staticData = StaticData::Instance();
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = m_system->GetUnknownWordPenaltyProducer();
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
// unknown word, add as trans opt
const StaticData &staticData = StaticData::Instance();
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = m_system->GetUnknownWordPenaltyProducer();
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
Moses::ChartTranslationOptionList &transOptColl = GetTranslationOptionList(sourcePos, sourcePos);
const WordsRange &range = transOptColl.GetSourceRange();
size_t isDigit = 0;
if (staticData.GetDropUnknown())
{
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
const string &s = f->GetString();
isDigit = s.find_first_of("0123456789");
if (isDigit == string::npos)
isDigit = 0;
else
isDigit = 1;
// modify the starting bitmap
}
Phrase* m_unksrc = new Phrase(Input);
m_unksrc->AddWord() = sourceWord;
m_unksrcs.push_back(m_unksrc);
Moses::ChartTranslationOptionList &transOptColl = GetTranslationOptionList(sourcePos, sourcePos);
const WordsRange &range = transOptColl.GetSourceRange();
TranslationOption *transOpt;
if (! staticData.GetDropUnknown() || isDigit)
{
// words consumed
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>();
m_cachedWordsConsumed.push_back(wordsConsumed);
WordConsumed *wc = new WordConsumed(sourcePos, sourcePos, sourceWord, NULL);
wordsConsumed->push_back(wc);
assert(wordsConsumed->size());
size_t isDigit = 0;
if (staticData.GetDropUnknown()) {
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
const string &s = f->GetString();
isDigit = s.find_first_of("0123456789");
if (isDigit == string::npos)
isDigit = 0;
else
isDigit = 1;
// modify the starting bitmap
}
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS)
{
const string &targetLHSStr = iterLHS->first;
float prob = iterLHS->second;
// lhs
const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
Word targetLHS(true);
Phrase* m_unksrc = new Phrase(Input);
m_unksrc->AddWord() = sourceWord;
m_unksrcs.push_back(m_unksrc);
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
assert(targetLHS.GetFactor(0) != NULL);
TranslationOption *transOpt;
if (! staticData.GetDropUnknown() || isDigit) {
// words consumed
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>();
m_cachedWordsConsumed.push_back(wordsConsumed);
// add to dictionary
TargetPhrase *targetPhrase = new TargetPhrase(Output);
WordConsumed *wc = new WordConsumed(sourcePos, sourcePos, sourceWord, NULL);
wordsConsumed->push_back(wc);
assert(wordsConsumed->size());
m_cacheTargetPhrase.push_back(targetPhrase);
Word &targetWord = targetPhrase->AddWord();
targetWord.CreateUnknownWord(sourceWord);
// scores
vector<float> unknownScore(1, FloorScore(TransformScore(prob)));
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
const string &targetLHSStr = iterLHS->first;
float prob = iterLHS->second;
//targetPhrase->SetScore();
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore);
targetPhrase->SetSourcePhrase(m_unksrc);
targetPhrase->SetTargetLHS(targetLHS);
// chart rule
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
, *wordsConsumed->back()
, range);
chartRule->CreateNonTermIndex();
transOptColl.Add(chartRule);
} // for (iterLHS
}
else
{ // drop source word. create blank trans opt
vector<float> unknownScore(1, FloorScore(-numeric_limits<float>::infinity()));
// lhs
const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
Word targetLHS(true);
TargetPhrase *targetPhrase = new TargetPhrase(Output);
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS)
{
const string &targetLHSStr = iterLHS->first;
float prob = iterLHS->second;
Word targetLHS(true);
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
assert(targetLHS.GetFactor(0) != NULL);
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
assert(targetLHS.GetFactor(0) != NULL);
m_cacheTargetPhrase.push_back(targetPhrase);
targetPhrase->SetSourcePhrase(m_unksrc);
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
targetPhrase->SetTargetLHS(targetLHS);
// add to dictionary
TargetPhrase *targetPhrase = new TargetPhrase(Output);
// words consumed
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>;
m_cachedWordsConsumed.push_back(wordsConsumed);
wordsConsumed->push_back(new WordConsumed(sourcePos, sourcePos, sourceWord, NULL));
m_cacheTargetPhrase.push_back(targetPhrase);
Word &targetWord = targetPhrase->AddWord();
targetWord.CreateUnknownWord(sourceWord);
// chart rule
assert(wordsConsumed->size());
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
, *wordsConsumed->back()
, range);
chartRule->CreateNonTermIndex();
transOptColl.Add(chartRule);
}
}
// scores
vector<float> unknownScore(1, FloorScore(TransformScore(prob)));
//targetPhrase->SetScore();
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore);
targetPhrase->SetSourcePhrase(m_unksrc);
targetPhrase->SetTargetLHS(targetLHS);
// chart rule
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
, *wordsConsumed->back()
, range);
chartRule->CreateNonTermIndex();
transOptColl.Add(chartRule);
} // for (iterLHS
} else {
// drop source word. create blank trans opt
vector<float> unknownScore(1, FloorScore(-numeric_limits<float>::infinity()));
TargetPhrase *targetPhrase = new TargetPhrase(Output);
// loop
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
UnknownLHSList::const_iterator iterLHS;
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
const string &targetLHSStr = iterLHS->first;
float prob = iterLHS->second;
Word targetLHS(true);
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
assert(targetLHS.GetFactor(0) != NULL);
m_cacheTargetPhrase.push_back(targetPhrase);
targetPhrase->SetSourcePhrase(m_unksrc);
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
targetPhrase->SetTargetLHS(targetLHS);
// words consumed
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>;
m_cachedWordsConsumed.push_back(wordsConsumed);
wordsConsumed->push_back(new WordConsumed(sourcePos, sourcePos, sourceWord, NULL));
// chart rule
assert(wordsConsumed->size());
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
, *wordsConsumed->back()
, range);
chartRule->CreateNonTermIndex();
transOptColl.Add(chartRule);
}
}
}
void TranslationOptionCollection::Add(ChartTranslationOption *transOpt, size_t pos)
{
ChartTranslationOptionList &transOptList = GetTranslationOptionList(pos, pos);
transOptList.Add(transOpt);
ChartTranslationOptionList &transOptList = GetTranslationOptionList(pos, pos);
transOptList.Add(transOpt);
}
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
@ -307,8 +293,8 @@ void TranslationOptionCollection::Prune(size_t startPos, size_t endPos)
//! sort all trans opt in each list for cube pruning */
void TranslationOptionCollection::Sort(size_t startPos, size_t endPos)
{
ChartTranslationOptionList &list = GetTranslationOptionList(startPos, endPos);
list.Sort();
ChartTranslationOptionList &list = GetTranslationOptionList(startPos, endPos);
list.Sort();
}

85
moses-chart/src/ChartTranslationOptionCollection.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -29,11 +29,11 @@
namespace Moses
{
class DecodeGraph;
class Word;
class ChartTranslationOption;
class WordConsumed;
class WordPenaltyProducer;
class DecodeGraph;
class Word;
class ChartTranslationOption;
class WordConsumed;
class WordPenaltyProducer;
};
namespace MosesChart
@ -42,53 +42,52 @@ class ChartCellCollection;
class TranslationOptionCollection
{
friend std::ostream& operator<<(std::ostream&, const TranslationOptionCollection&);
friend std::ostream& operator<<(std::ostream&, const TranslationOptionCollection&);
protected:
const Moses::InputType &m_source;
const Moses::TranslationSystem* m_system;
const Moses::InputType &m_source;
const Moses::TranslationSystem* m_system;
std::vector <Moses::DecodeGraph*> m_decodeGraphList;
const ChartCellCollection &m_hypoStackColl;
const ChartCellCollection &m_hypoStackColl;
const std::vector<Moses::ChartRuleLookupManager*> &m_ruleLookupManagers;
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
std::vector<Moses::Phrase*> m_unksrcs;
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
// for adding 1 trans opt in unknown word proc
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
std::vector<Moses::Phrase*> m_unksrcs;
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos);
const Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos) const;
// for adding 1 trans opt in unknown word proc
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
void ProcessUnknownWord(size_t startPos, size_t endPos);
Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos);
const Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos) const;
// taken from TranslationOptionCollectionText.
void ProcessUnknownWord(size_t sourcePos);
void ProcessUnknownWord(size_t startPos, size_t endPos);
//! special handling of ONE unknown words.
virtual void ProcessOneUnknownWord(const Moses::Word &sourceWord
, size_t sourcePos, size_t length = 1);
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
void Prune(size_t startPos, size_t endPos);
// taken from TranslationOptionCollectionText.
void ProcessUnknownWord(size_t sourcePos);
//! sort all trans opt in each list for cube pruning */
void Sort(size_t startPos, size_t endPos);
//! special handling of ONE unknown words.
virtual void ProcessOneUnknownWord(const Moses::Word &sourceWord
, size_t sourcePos, size_t length = 1);
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
void Prune(size_t startPos, size_t endPos);
//! sort all trans opt in each list for cube pruning */
void Sort(size_t startPos, size_t endPos);
public:
TranslationOptionCollection(Moses::InputType const& source
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
virtual ~TranslationOptionCollection();
void CreateTranslationOptionsForRange(size_t startPos
, size_t endPos);
TranslationOptionCollection(Moses::InputType const& source
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
virtual ~TranslationOptionCollection();
void CreateTranslationOptionsForRange(size_t startPos
, size_t endPos);
const Moses::ChartTranslationOptionList &GetTranslationOptionList(const Moses::WordsRange &range) const
{
return GetTranslationOptionList(range.GetStartPos(), range.GetEndPos());
}
const Moses::ChartTranslationOptionList &GetTranslationOptionList(const Moses::WordsRange &range) const {
return GetTranslationOptionList(range.GetStartPos(), range.GetEndPos());
}
};

156
moses-chart/src/ChartTrellisNode.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -29,112 +29,104 @@ namespace MosesChart
{
TrellisNode::TrellisNode(const Hypothesis *hypo)
:m_hypo(hypo)
:m_hypo(hypo)
{
const std::vector<const Hypothesis*> &prevHypos = hypo->GetPrevHypos();
const std::vector<const Hypothesis*> &prevHypos = hypo->GetPrevHypos();
m_edge.reserve(prevHypos.size());
for (size_t ind = 0; ind < prevHypos.size(); ++ind)
{
const Hypothesis *prevHypo = prevHypos[ind];
TrellisNode *child = new TrellisNode(prevHypo);
m_edge.push_back(child);
}
m_edge.reserve(prevHypos.size());
for (size_t ind = 0; ind < prevHypos.size(); ++ind) {
const Hypothesis *prevHypo = prevHypos[ind];
TrellisNode *child = new TrellisNode(prevHypo);
m_edge.push_back(child);
}
assert(m_hypo);
assert(m_hypo);
}
TrellisNode::TrellisNode(const TrellisNode &origNode
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange
, const TrellisNode *&nodeChanged)
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange
, const TrellisNode *&nodeChanged)
{
if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis())
{ // this node should be replaced
m_hypo = &replacementHypo;
nodeChanged = this;
if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis()) {
// this node should be replaced
m_hypo = &replacementHypo;
nodeChanged = this;
// scores
assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node
// scores
assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node
scoreChange = replacementHypo.GetScoreBreakdown();
scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());
float deltaScore = scoreChange.GetWeightedScore();
assert(deltaScore <= 0.0005);
scoreChange = replacementHypo.GetScoreBreakdown();
scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());
// follow prev hypos back to beginning
const std::vector<const Hypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
vector<const Hypothesis*>::const_iterator iter;
assert(m_edge.empty());
m_edge.reserve(prevHypos.size());
for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter)
{
const Hypothesis *prevHypo = *iter;
TrellisNode *prevNode = new TrellisNode(prevHypo);
m_edge.push_back(prevNode);
}
float deltaScore = scoreChange.GetWeightedScore();
assert(deltaScore <= 0.0005);
}
else
{ // not the node we're looking for. Copy as-is and continue finding node
m_hypo = &origNode.GetHypothesis();
NodeChildren::const_iterator iter;
assert(m_edge.empty());
m_edge.reserve(origNode.m_edge.size());
for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter)
{
const TrellisNode &prevNode = **iter;
TrellisNode *newPrevNode = new TrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
m_edge.push_back(newPrevNode);
}
}
// follow prev hypos back to beginning
const std::vector<const Hypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
vector<const Hypothesis*>::const_iterator iter;
assert(m_edge.empty());
m_edge.reserve(prevHypos.size());
for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
const Hypothesis *prevHypo = *iter;
TrellisNode *prevNode = new TrellisNode(prevHypo);
m_edge.push_back(prevNode);
}
assert(m_hypo);
} else {
// not the node we're looking for. Copy as-is and continue finding node
m_hypo = &origNode.GetHypothesis();
NodeChildren::const_iterator iter;
assert(m_edge.empty());
m_edge.reserve(origNode.m_edge.size());
for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter) {
const TrellisNode &prevNode = **iter;
TrellisNode *newPrevNode = new TrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
m_edge.push_back(newPrevNode);
}
}
assert(m_hypo);
}
TrellisNode::~TrellisNode()
{
Moses::RemoveAllInColl(m_edge);
Moses::RemoveAllInColl(m_edge);
}
Moses::Phrase TrellisNode::GetOutputPhrase() const
{
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
Moses::Phrase ret(Moses::Output);
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
Moses::Phrase ret(Moses::Output);
const Moses::Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos)
{
const Moses::Word &word = currTargetPhrase.GetWord(pos);
if (word.IsNonTerminal())
{ // non-term. fill out with prev hypo
size_t nonTermInd = m_hypo->GetWordsConsumedTargetOrder(pos);
const TrellisNode &childNode = GetChild(nonTermInd);
Moses::Phrase childPhrase = childNode.GetOutputPhrase();
ret.Append(childPhrase);
}
else
{
ret.AddWord(word);
}
}
const Moses::Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos) {
const Moses::Word &word = currTargetPhrase.GetWord(pos);
if (word.IsNonTerminal()) {
// non-term. fill out with prev hypo
size_t nonTermInd = m_hypo->GetWordsConsumedTargetOrder(pos);
const TrellisNode &childNode = GetChild(nonTermInd);
Moses::Phrase childPhrase = childNode.GetOutputPhrase();
ret.Append(childPhrase);
} else {
ret.AddWord(word);
}
}
return ret;
return ret;
}
std::ostream& operator<<(std::ostream &out, const TrellisNode &node)
{
out << "* " << node.GetHypothesis() << endl;
TrellisNode::NodeChildren::const_iterator iter;
for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter)
{
out << **iter;
}
out << "* " << node.GetHypothesis() << endl;
return out;
TrellisNode::NodeChildren::const_iterator iter;
for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter) {
out << **iter;
}
return out;
}
}

51
moses-chart/src/ChartTrellisNode.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -26,7 +26,7 @@
namespace Moses
{
class ScoreComponentCollection;
class ScoreComponentCollection;
}
namespace MosesChart
@ -36,35 +36,36 @@ class Hypothesis;
class TrellisNode
{
friend std::ostream& operator<<(std::ostream&, const TrellisNode&);
friend std::ostream& operator<<(std::ostream&, const TrellisNode&);
public:
typedef std::vector<TrellisNode*> NodeChildren;
typedef std::vector<TrellisNode*> NodeChildren;
protected:
const Hypothesis *m_hypo;
NodeChildren m_edge;
const Hypothesis *m_hypo;
NodeChildren m_edge;
public:
TrellisNode(const Hypothesis *hypo);
TrellisNode(const TrellisNode &origNode
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange
, const TrellisNode *&nodeChanged);
~TrellisNode();
TrellisNode(const Hypothesis *hypo);
TrellisNode(const TrellisNode &origNode
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange
, const TrellisNode *&nodeChanged);
~TrellisNode();
const Hypothesis &GetHypothesis() const
{
return *m_hypo;
}
const NodeChildren &GetChildren() const
{ return m_edge; }
const Hypothesis &GetHypothesis() const {
return *m_hypo;
}
const TrellisNode &GetChild(size_t ind) const
{ return *m_edge[ind]; }
const NodeChildren &GetChildren() const {
return m_edge;
}
Moses::Phrase GetOutputPhrase() const;
const TrellisNode &GetChild(size_t ind) const {
return *m_edge[ind];
}
Moses::Phrase GetOutputPhrase() const;
};
}

124
moses-chart/src/ChartTrellisPath.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -31,103 +31,97 @@ namespace MosesChart
{
TrellisPath::TrellisPath(const Hypothesis *hypo)
:m_finalNode(new TrellisNode(hypo))
,m_scoreBreakdown(hypo->GetScoreBreakdown())
,m_totalScore(hypo->GetTotalScore())
,m_prevNodeChanged(NULL)
,m_prevPath(NULL)
:m_finalNode(new TrellisNode(hypo))
,m_scoreBreakdown(hypo->GetScoreBreakdown())
,m_totalScore(hypo->GetTotalScore())
,m_prevNodeChanged(NULL)
,m_prevPath(NULL)
{
}
TrellisPath::TrellisPath(const TrellisPath &origPath
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange)
:m_scoreBreakdown(origPath.GetScoreBreakdown())
,m_prevPath(&origPath)
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange)
:m_scoreBreakdown(origPath.GetScoreBreakdown())
,m_prevPath(&origPath)
{
m_finalNode = new TrellisNode(origPath.GetFinalNode()
, soughtNode
, replacementHypo
, scoreChange
, m_prevNodeChanged);
m_finalNode = new TrellisNode(origPath.GetFinalNode()
, soughtNode
, replacementHypo
, scoreChange
, m_prevNodeChanged);
m_scoreBreakdown.PlusEquals(scoreChange);
m_scoreBreakdown.PlusEquals(scoreChange);
m_totalScore = m_scoreBreakdown.GetWeightedScore();
m_totalScore = m_scoreBreakdown.GetWeightedScore();
}
TrellisPath::~TrellisPath()
{
delete m_finalNode;
delete m_finalNode;
}
Moses::Phrase TrellisPath::GetOutputPhrase() const
{
Moses::Phrase ret = GetFinalNode().GetOutputPhrase();
return ret;
Moses::Phrase ret = GetFinalNode().GetOutputPhrase();
return ret;
}
void TrellisPath::CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const
{
// copy this path but replace startHypo with its arc
const ArcList *arcList = soughtNode.GetHypothesis().GetArcList();
// copy this path but replace startHypo with its arc
const ArcList *arcList = soughtNode.GetHypothesis().GetArcList();
if (arcList)
{
ArcList::const_iterator iterArcList;
for (iterArcList = arcList->begin(); iterArcList != arcList->end(); ++iterArcList)
{
Moses::ScoreComponentCollection scoreChange;
if (arcList) {
ArcList::const_iterator iterArcList;
for (iterArcList = arcList->begin(); iterArcList != arcList->end(); ++iterArcList) {
Moses::ScoreComponentCollection scoreChange;
const Hypothesis &replacementHypo = **iterArcList;
TrellisPath *newPath = new TrellisPath(*this, soughtNode, replacementHypo, scoreChange);
pathColl.Add(newPath);
}
}
const Hypothesis &replacementHypo = **iterArcList;
TrellisPath *newPath = new TrellisPath(*this, soughtNode, replacementHypo, scoreChange);
pathColl.Add(newPath);
}
}
// recusively create deviant paths for child nodes
const TrellisNode::NodeChildren &children = soughtNode.GetChildren();
// recusively create deviant paths for child nodes
const TrellisNode::NodeChildren &children = soughtNode.GetChildren();
TrellisNode::NodeChildren::const_iterator iter;
for (iter = children.begin(); iter != children.end(); ++iter)
{
const TrellisNode &child = **iter;
CreateDeviantPaths(pathColl, child);
}
TrellisNode::NodeChildren::const_iterator iter;
for (iter = children.begin(); iter != children.end(); ++iter) {
const TrellisNode &child = **iter;
CreateDeviantPaths(pathColl, child);
}
}
void TrellisPath::CreateDeviantPaths(TrellisPathCollection &pathColl) const
{
if (m_prevNodeChanged == NULL)
{ // initial enumeration from a pure hypo
CreateDeviantPaths(pathColl, GetFinalNode());
}
else
{ // don't change m_prevNodeChanged, just it's children
const TrellisNode::NodeChildren &children = m_prevNodeChanged->GetChildren();
if (m_prevNodeChanged == NULL) {
// initial enumeration from a pure hypo
CreateDeviantPaths(pathColl, GetFinalNode());
} else {
// don't change m_prevNodeChanged, just it's children
const TrellisNode::NodeChildren &children = m_prevNodeChanged->GetChildren();
TrellisNode::NodeChildren::const_iterator iter;
for (iter = children.begin(); iter != children.end(); ++iter)
{
const TrellisNode &child = **iter;
CreateDeviantPaths(pathColl, child);
}
}
TrellisNode::NodeChildren::const_iterator iter;
for (iter = children.begin(); iter != children.end(); ++iter) {
const TrellisNode &child = **iter;
CreateDeviantPaths(pathColl, child);
}
}
}
std::ostream& operator<<(std::ostream &out, const TrellisPath &path)
{
out << &path << " " << path.m_prevPath << " " << path.GetOutputPhrase() << endl;
out << &path << " " << path.m_prevPath << " " << path.GetOutputPhrase() << endl;
if (path.m_prevNodeChanged)
{
out << "changed " << path.m_prevNodeChanged->GetHypothesis() << endl;
}
if (path.m_prevNodeChanged) {
out << "changed " << path.m_prevNodeChanged->GetHypothesis() << endl;
}
out << path.GetFinalNode() << endl;
out << path.GetFinalNode() << endl;
return out;
return out;
}
};

65
moses-chart/src/ChartTrellisPath.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -32,48 +32,47 @@ class TrellisPathCollection;
class TrellisPath
{
friend std::ostream& operator<<(std::ostream&, const TrellisPath&);
friend std::ostream& operator<<(std::ostream&, const TrellisPath&);
protected:
// recursively point backwards
TrellisNode *m_finalNode;
const TrellisNode *m_prevNodeChanged;
const TrellisPath *m_prevPath;
// recursively point backwards
TrellisNode *m_finalNode;
const TrellisNode *m_prevNodeChanged;
const TrellisPath *m_prevPath;
Moses::ScoreComponentCollection m_scoreBreakdown;
float m_totalScore;
Moses::ScoreComponentCollection m_scoreBreakdown;
float m_totalScore;
// deviate by 1 hypo
TrellisPath(const TrellisPath &origPath
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange);
// deviate by 1 hypo
TrellisPath(const TrellisPath &origPath
, const TrellisNode &soughtNode
, const Hypothesis &replacementHypo
, Moses::ScoreComponentCollection &scoreChange);
void CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const;
void CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const;
const TrellisNode &GetFinalNode() const
{
assert (m_finalNode);
return *m_finalNode;
}
const TrellisNode &GetFinalNode() const {
assert (m_finalNode);
return *m_finalNode;
}
public:
TrellisPath(const Hypothesis *hypo);
~TrellisPath();
TrellisPath(const Hypothesis *hypo);
~TrellisPath();
//! get score for this path throught trellis
inline float GetTotalScore() const
{ return m_totalScore; }
//! get score for this path throught trellis
inline float GetTotalScore() const {
return m_totalScore;
}
Moses::Phrase GetOutputPhrase() const;
Moses::Phrase GetOutputPhrase() const;
/** returns detailed component scores */
inline const Moses::ScoreComponentCollection &GetScoreBreakdown() const
{
return m_scoreBreakdown;
}
/** returns detailed component scores */
inline const Moses::ScoreComponentCollection &GetScoreBreakdown() const {
return m_scoreBreakdown;
}
void CreateDeviantPaths(TrellisPathCollection &pathColl) const;
void CreateDeviantPaths(TrellisPathCollection &pathColl) const;
};

51
moses-chart/src/ChartTrellisPathCollection.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -27,39 +27,38 @@ namespace MosesChart
TrellisPathCollection::~TrellisPathCollection()
{
// clean up
Moses::RemoveAllInColl(m_collection);
// clean up
Moses::RemoveAllInColl(m_collection);
}
void TrellisPathCollection::Add(TrellisPath *path)
{
m_collection.insert(path);
m_collection.insert(path);
}
void TrellisPathCollection::Prune(size_t newSize)
{
size_t currSize = m_collection.size();
size_t currSize = m_collection.size();
if (currSize <= newSize)
return; // don't need to prune
if (currSize <= newSize)
return; // don't need to prune
CollectionType::reverse_iterator iterRev;
for (iterRev = m_collection.rbegin() ; iterRev != m_collection.rend() ; ++iterRev)
{
TrellisPath *trellisPath = *iterRev;
delete trellisPath;
currSize--;
if (currSize == newSize)
break;
}
// delete path in m_collection
CollectionType::iterator iter = m_collection.begin();
for (size_t i = 0 ; i < newSize ; ++i)
iter++;
m_collection.erase(iter, m_collection.end());
CollectionType::reverse_iterator iterRev;
for (iterRev = m_collection.rbegin() ; iterRev != m_collection.rend() ; ++iterRev) {
TrellisPath *trellisPath = *iterRev;
delete trellisPath;
currSize--;
if (currSize == newSize)
break;
}
// delete path in m_collection
CollectionType::iterator iter = m_collection.begin();
for (size_t i = 0 ; i < newSize ; ++i)
iter++;
m_collection.erase(iter, m_collection.end());
}
}

44
moses-chart/src/ChartTrellisPathCollection.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -28,37 +28,35 @@ namespace MosesChart
class TrellisPath;
struct CompareTrellisPathCollection
{
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const
{
return (pathA->GetTotalScore() > pathB->GetTotalScore());
}
struct CompareTrellisPathCollection {
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const {
return (pathA->GetTotalScore() > pathB->GetTotalScore());
}
};
class TrellisPathCollection
{
protected:
typedef std::multiset<TrellisPath*, CompareTrellisPathCollection> CollectionType;
CollectionType m_collection;
typedef std::multiset<TrellisPath*, CompareTrellisPathCollection> CollectionType;
CollectionType m_collection;
public:
~TrellisPathCollection();
~TrellisPathCollection();
size_t GetSize() const
{ return m_collection.size(); }
size_t GetSize() const {
return m_collection.size();
}
void Add(TrellisPath *path);
void Prune(size_t newSize);
void Add(TrellisPath *path);
void Prune(size_t newSize);
TrellisPath *pop()
{
TrellisPath *top = *m_collection.begin();
TrellisPath *pop() {
TrellisPath *top = *m_collection.begin();
// Detach
m_collection.erase(m_collection.begin());
return top;
}
// Detach
m_collection.erase(m_collection.begin());
return top;
}
};

10
moses-chart/src/ChartTrellisPathList.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -29,8 +29,8 @@ namespace MosesChart
{
TrellisPathList::~TrellisPathList()
{
// clean up
Moses::RemoveAllInColl(m_collection);
// clean up
Moses::RemoveAllInColl(m_collection);
}
} // namespace

54
moses-chart/src/ChartTrellisPathList.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -30,30 +30,40 @@ class TrellisPath;
class TrellisPathList
{
protected:
typedef std::vector<const TrellisPath*> CollType;
CollType m_collection;
typedef std::vector<const TrellisPath*> CollType;
CollType m_collection;
public:
// iters
typedef CollType::iterator iterator;
typedef CollType::const_iterator const_iterator;
iterator begin() { return m_collection.begin(); }
iterator end() { return m_collection.end(); }
const_iterator begin() const { return m_collection.begin(); }
const_iterator end() const { return m_collection.end(); }
void clear() { m_collection.clear(); }
// iters
typedef CollType::iterator iterator;
typedef CollType::const_iterator const_iterator;
virtual ~TrellisPathList();
iterator begin() {
return m_collection.begin();
}
iterator end() {
return m_collection.end();
}
const_iterator begin() const {
return m_collection.begin();
}
const_iterator end() const {
return m_collection.end();
}
void clear() {
m_collection.clear();
}
size_t GetSize() const
{ return m_collection.size(); }
virtual ~TrellisPathList();
//! add a new entry into collection
void Add(TrellisPath *trellisPath)
{
m_collection.push_back(trellisPath);
}
size_t GetSize() const {
return m_collection.size();
}
//! add a new entry into collection
void Add(TrellisPath *trellisPath) {
m_collection.push_back(trellisPath);
}
};
} // namespace

47
moses-chart/src/Cube.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -29,35 +29,34 @@ namespace MosesChart
{
Cube::~Cube()
{
Moses::RemoveAllInColl(m_uniqueEntry);
Moses::RemoveAllInColl(m_uniqueEntry);
}
bool Cube::Add(QueueEntry *queueEntry)
{
pair<UniqueCubeEntry::iterator, bool> inserted = m_uniqueEntry.insert(queueEntry);
if (inserted.second)
{ // inserted
m_sortedByScore.push(queueEntry);
}
else
{ // already there
//cerr << "already there\n";
delete queueEntry;
}
//assert(m_uniqueEntry.size() == m_sortedByScore.size());
return inserted.second;
pair<UniqueCubeEntry::iterator, bool> inserted = m_uniqueEntry.insert(queueEntry);
if (inserted.second) {
// inserted
m_sortedByScore.push(queueEntry);
} else {
// already there
//cerr << "already there\n";
delete queueEntry;
}
//assert(m_uniqueEntry.size() == m_sortedByScore.size());
return inserted.second;
}
QueueEntry *Cube::Pop()
{
QueueEntry *entry = m_sortedByScore.top();
m_sortedByScore.pop();
QueueEntry *entry = m_sortedByScore.top();
m_sortedByScore.pop();
return entry;
}
return entry;
}
}

75
moses-chart/src/Cube.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -36,68 +36,65 @@ namespace MosesChart
class QueueEntryUniqueHasher
{
public:
size_t operator()(const QueueEntry * p) const
{
size_t seed = 0;
boost::hash_combine(seed, &(p->GetTranslationOption()));
boost::hash_combine(seed, p->GetChildEntries());
return seed;
}
size_t operator()(const QueueEntry * p) const {
size_t seed = 0;
boost::hash_combine(seed, &(p->GetTranslationOption()));
boost::hash_combine(seed, p->GetChildEntries());
return seed;
}
};
class QueueEntryUniqueEqualityPred
{
public:
bool operator()(const QueueEntry * p, const QueueEntry * q) const
{
return ((&(p->GetTranslationOption()) == &(q->GetTranslationOption()))
&& (p->GetChildEntries() == q->GetChildEntries()));
}
bool operator()(const QueueEntry * p, const QueueEntry * q) const {
return ((&(p->GetTranslationOption()) == &(q->GetTranslationOption()))
&& (p->GetChildEntries() == q->GetChildEntries()));
}
};
#endif
class QueueEntryUniqueOrderer
{
public:
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const
{
return (*entryA) < (*entryB);
}
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const {
return (*entryA) < (*entryB);
}
};
class QueueEntryScoreOrderer
{
public:
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const
{
return (entryA->GetCombinedScore() < entryB->GetCombinedScore());
}
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const {
return (entryA->GetCombinedScore() < entryB->GetCombinedScore());
}
};
class Cube
{
protected:
protected:
#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
typedef boost::unordered_set<QueueEntry*,
QueueEntryUniqueHasher,
QueueEntryUniqueEqualityPred> UniqueCubeEntry;
typedef boost::unordered_set<QueueEntry*,
QueueEntryUniqueHasher,
QueueEntryUniqueEqualityPred> UniqueCubeEntry;
#else
typedef std::set<QueueEntry*, QueueEntryUniqueOrderer> UniqueCubeEntry;
typedef std::set<QueueEntry*, QueueEntryUniqueOrderer> UniqueCubeEntry;
#endif
UniqueCubeEntry m_uniqueEntry;
typedef std::priority_queue<QueueEntry*, std::vector<QueueEntry*>, QueueEntryScoreOrderer> SortedByScore;
SortedByScore m_sortedByScore;
UniqueCubeEntry m_uniqueEntry;
typedef std::priority_queue<QueueEntry*, std::vector<QueueEntry*>, QueueEntryScoreOrderer> SortedByScore;
SortedByScore m_sortedByScore;
public:
~Cube();
bool IsEmpty() const
{ return m_sortedByScore.empty(); }
~Cube();
bool IsEmpty() const {
return m_sortedByScore.empty();
}
QueueEntry *Pop();
bool Add(QueueEntry *queueEntry);
QueueEntry *Pop();
bool Add(QueueEntry *queueEntry);
};

120
moses-chart/src/QueueEntry.cpp
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -40,109 +40,105 @@ namespace MosesChart
{
QueueEntry::QueueEntry(const Moses::ChartTranslationOption &transOpt
, const ChartCellCollection &allChartCells)
:m_transOpt(transOpt)
, const ChartCellCollection &allChartCells)
:m_transOpt(transOpt)
{
const WordConsumed *wordsConsumed = &transOpt.GetLastWordConsumed();
CreateChildEntry(wordsConsumed, allChartCells);
CalcScore();
const WordConsumed *wordsConsumed = &transOpt.GetLastWordConsumed();
CreateChildEntry(wordsConsumed, allChartCells);
CalcScore();
}
void QueueEntry::CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells)
{
// recursvile do the 1st first
const WordConsumed *prevWordsConsumed = wordsConsumed->GetPrevWordsConsumed();
if (prevWordsConsumed)
CreateChildEntry(prevWordsConsumed, allChartCells);
// recursvile do the 1st first
const WordConsumed *prevWordsConsumed = wordsConsumed->GetPrevWordsConsumed();
if (prevWordsConsumed)
CreateChildEntry(prevWordsConsumed, allChartCells);
if (wordsConsumed->IsNonTerminal())
{ // non-term
const WordsRange &childRange = wordsConsumed->GetWordsRange();
const ChartCell &childCell = allChartCells.Get(childRange);
const Word &headWord = wordsConsumed->GetSourceWord();
if (wordsConsumed->IsNonTerminal()) {
// non-term
const WordsRange &childRange = wordsConsumed->GetWordsRange();
const ChartCell &childCell = allChartCells.Get(childRange);
const Word &headWord = wordsConsumed->GetSourceWord();
assert(!childCell.GetSortedHypotheses(headWord).empty());
assert(!childCell.GetSortedHypotheses(headWord).empty());
const Moses::Word &nonTerm = wordsConsumed->GetSourceWord();
assert(nonTerm.IsNonTerminal());
ChildEntry childEntry(0, childCell.GetSortedHypotheses(nonTerm), nonTerm);
m_childEntries.push_back(childEntry);
}
const Moses::Word &nonTerm = wordsConsumed->GetSourceWord();
assert(nonTerm.IsNonTerminal());
ChildEntry childEntry(0, childCell.GetSortedHypotheses(nonTerm), nonTerm);
m_childEntries.push_back(childEntry);
}
}
QueueEntry::QueueEntry(const QueueEntry &copy, size_t childEntryIncr)
:m_transOpt(copy.m_transOpt)
,m_childEntries(copy.m_childEntries)
:m_transOpt(copy.m_transOpt)
,m_childEntries(copy.m_childEntries)
{
ChildEntry &childEntry = m_childEntries[childEntryIncr];
childEntry.IncrementPos();
CalcScore();
ChildEntry &childEntry = m_childEntries[childEntryIncr];
childEntry.IncrementPos();
CalcScore();
}
QueueEntry::~QueueEntry()
{
//Moses::RemoveAllInColl(m_childEntries);
//Moses::RemoveAllInColl(m_childEntries);
}
void QueueEntry::CreateDeviants(Cube &cube) const
{
for (size_t ind = 0; ind < m_childEntries.size(); ind++)
{
const ChildEntry &childEntry = m_childEntries[ind];
for (size_t ind = 0; ind < m_childEntries.size(); ind++) {
const ChildEntry &childEntry = m_childEntries[ind];
if (childEntry.HasMoreHypo())
{
QueueEntry *newEntry = new QueueEntry(*this, ind);
cube.Add(newEntry);
}
}
if (childEntry.HasMoreHypo()) {
QueueEntry *newEntry = new QueueEntry(*this, ind);
cube.Add(newEntry);
}
}
}
void QueueEntry::CalcScore()
{
m_combinedScore = m_transOpt.GetTotalScore();
for (size_t ind = 0; ind < m_childEntries.size(); ind++)
{
const ChildEntry &childEntry = m_childEntries[ind];
m_combinedScore = m_transOpt.GetTotalScore();
for (size_t ind = 0; ind < m_childEntries.size(); ind++) {
const ChildEntry &childEntry = m_childEntries[ind];
const Hypothesis *hypo = childEntry.GetHypothesis();
m_combinedScore += hypo->GetTotalScore();
}
const Hypothesis *hypo = childEntry.GetHypothesis();
m_combinedScore += hypo->GetTotalScore();
}
}
bool QueueEntry::operator<(const QueueEntry &compare) const
{
if (&m_transOpt != &compare.m_transOpt)
return &m_transOpt < &compare.m_transOpt;
bool ret = m_childEntries < compare.m_childEntries;
return ret;
{
if (&m_transOpt != &compare.m_transOpt)
return &m_transOpt < &compare.m_transOpt;
bool ret = m_childEntries < compare.m_childEntries;
return ret;
}
#ifdef HAVE_BOOST
std::size_t hash_value(const ChildEntry & entry)
{
boost::hash<const Hypothesis*> hasher;
return hasher(entry.GetHypothesis());
boost::hash<const Hypothesis*> hasher;
return hasher(entry.GetHypothesis());
}
#endif
std::ostream& operator<<(std::ostream &out, const ChildEntry &entry)
{
out << *entry.GetHypothesis();
return out;
out << *entry.GetHypothesis();
return out;
}
std::ostream& operator<<(std::ostream &out, const QueueEntry &entry)
{
out << entry.GetTranslationOption() << endl;
std::vector<ChildEntry>::const_iterator iter;
for (iter = entry.GetChildEntries().begin(); iter != entry.GetChildEntries().end(); ++iter)
{
out << *iter << endl;
}
return out;
out << entry.GetTranslationOption() << endl;
std::vector<ChildEntry>::const_iterator iter;
for (iter = entry.GetChildEntries().begin(); iter != entry.GetChildEntries().end(); ++iter) {
out << *iter << endl;
}
return out;
}
}

114
moses-chart/src/QueueEntry.h
View File

@ -3,17 +3,17 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
@ -32,9 +32,9 @@
namespace Moses
{
class WordConsumed;
class ChartTranslationOption;
extern bool g_debug;
class WordConsumed;
class ChartTranslationOption;
extern bool g_debug;
};
namespace MosesChart
@ -49,81 +49,81 @@ class Cube;
typedef std::vector<const Hypothesis*> HypoList;
// wrapper around list of hypothese for a particular non-term of a trans opt
// wrapper around list of hypothese for a particular non-term of a trans opt
class ChildEntry
{
friend std::ostream& operator<<(std::ostream&, const ChildEntry&);
friend std::ostream& operator<<(std::ostream&, const ChildEntry&);
protected:
size_t m_pos;
const HypoList *m_orderedHypos;
size_t m_pos;
const HypoList *m_orderedHypos;
public:
ChildEntry(size_t pos, const HypoList &orderedHypos, const Moses::Word & /*headWord*/ )
:m_pos(pos)
,m_orderedHypos(&orderedHypos)
//,m_headWord(headWord)
{}
:m_pos(pos)
,m_orderedHypos(&orderedHypos)
//,m_headWord(headWord)
{}
size_t IncrementPos()
{ return m_pos++; }
bool HasMoreHypo() const
{
return m_pos + 1 < m_orderedHypos->size();
}
const Hypothesis *GetHypothesis() const
{
return (*m_orderedHypos)[m_pos];
}
//const Moses::Word &GetHeadWord() const
//{ return m_headWord; }
size_t IncrementPos() {
return m_pos++;
}
//! transitive comparison used for adding objects into FactorCollection
bool operator<(const ChildEntry &compare) const
{
return GetHypothesis() < compare.GetHypothesis();
}
bool HasMoreHypo() const {
return m_pos + 1 < m_orderedHypos->size();
}
bool operator==(const ChildEntry & compare) const
{
return GetHypothesis() == compare.GetHypothesis();
}
const Hypothesis *GetHypothesis() const {
return (*m_orderedHypos)[m_pos];
}
//const Moses::Word &GetHeadWord() const
//{ return m_headWord; }
//! transitive comparison used for adding objects into FactorCollection
bool operator<(const ChildEntry &compare) const {
return GetHypothesis() < compare.GetHypothesis();
}
bool operator==(const ChildEntry & compare) const {
return GetHypothesis() == compare.GetHypothesis();
}
};
// entry in the cub of 1 trans opt and all the hypotheses that goes with each non term.
class QueueEntry
{
friend std::ostream& operator<<(std::ostream&, const QueueEntry&);
friend std::ostream& operator<<(std::ostream&, const QueueEntry&);
protected:
const Moses::ChartTranslationOption &m_transOpt;
std::vector<ChildEntry> m_childEntries;
const Moses::ChartTranslationOption &m_transOpt;
std::vector<ChildEntry> m_childEntries;
float m_combinedScore;
float m_combinedScore;
QueueEntry(const QueueEntry &copy, size_t childEntryIncr);
void CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells);
QueueEntry(const QueueEntry &copy, size_t childEntryIncr);
void CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells);
void CalcScore();
void CalcScore();
public:
QueueEntry(const Moses::ChartTranslationOption &transOpt
, const ChartCellCollection &allChartCells);
~QueueEntry();
QueueEntry(const Moses::ChartTranslationOption &transOpt
, const ChartCellCollection &allChartCells);
~QueueEntry();
const Moses::ChartTranslationOption &GetTranslationOption() const
{ return m_transOpt; }
const std::vector<ChildEntry> &GetChildEntries() const
{ return m_childEntries; }
float GetCombinedScore() const
{ return m_combinedScore; }
const Moses::ChartTranslationOption &GetTranslationOption() const {
return m_transOpt;
}
const std::vector<ChildEntry> &GetChildEntries() const {
return m_childEntries;
}
float GetCombinedScore() const {
return m_combinedScore;
}
void CreateDeviants(Cube &) const;
void CreateDeviants(Cube &) const;
bool operator<(const QueueEntry &compare) const;
bool operator<(const QueueEntry &compare) const;
};
#ifdef HAVE_BOOST