mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
run beautify.perl. Consistent formatting for .h & .cpp files
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3896 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
5f0eacce4b
commit
508d89eda8
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -36,196 +36,185 @@ using namespace Moses;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
extern bool g_debug;
|
||||
extern bool g_debug;
|
||||
}
|
||||
|
||||
namespace MosesChart
|
||||
{
|
||||
|
||||
ChartCell::ChartCell(size_t startPos, size_t endPos, Manager &manager)
|
||||
:m_coverage(startPos, endPos)
|
||||
,m_manager(manager)
|
||||
:m_coverage(startPos, endPos)
|
||||
,m_manager(manager)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
m_nBestIsEnabled = staticData.IsNBestEnabled();
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
m_nBestIsEnabled = staticData.IsNBestEnabled();
|
||||
}
|
||||
|
||||
const HypoList &ChartCell::GetSortedHypotheses(const Moses::Word &headWord) const
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator
|
||||
iter = m_hypoColl.find(headWord);
|
||||
assert(iter != m_hypoColl.end());
|
||||
return iter->second.GetSortedHypotheses();
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator
|
||||
iter = m_hypoColl.find(headWord);
|
||||
assert(iter != m_hypoColl.end());
|
||||
return iter->second.GetSortedHypotheses();
|
||||
}
|
||||
|
||||
bool ChartCell::AddHypothesis(Hypothesis *hypo)
|
||||
{
|
||||
const Word &targetLHS = hypo->GetTargetLHS();
|
||||
return m_hypoColl[targetLHS].AddHypothesis(hypo, m_manager);
|
||||
const Word &targetLHS = hypo->GetTargetLHS();
|
||||
return m_hypoColl[targetLHS].AddHypothesis(hypo, m_manager);
|
||||
}
|
||||
|
||||
void ChartCell::PruneToSize()
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.PruneToSize(m_manager);
|
||||
}
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.PruneToSize(m_manager);
|
||||
}
|
||||
}
|
||||
|
||||
void ChartCell::ProcessSentence(const ChartTranslationOptionList &transOptList
|
||||
, const ChartCellCollection &allChartCells)
|
||||
, const ChartCellCollection &allChartCells)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
Cube cube;
|
||||
Cube cube;
|
||||
|
||||
// add all trans opt into queue. using only 1st child node.
|
||||
ChartTranslationOptionList::const_iterator iterList;
|
||||
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList)
|
||||
{
|
||||
const ChartTranslationOption &transOpt = **iterList;
|
||||
QueueEntry *queueEntry = new QueueEntry(transOpt, allChartCells);
|
||||
cube.Add(queueEntry);
|
||||
}
|
||||
|
||||
// pluck things out of queue and add to hypo collection
|
||||
const size_t popLimit = staticData.GetCubePruningPopLimit();
|
||||
// add all trans opt into queue. using only 1st child node.
|
||||
ChartTranslationOptionList::const_iterator iterList;
|
||||
for (iterList = transOptList.begin(); iterList != transOptList.end(); ++iterList) {
|
||||
const ChartTranslationOption &transOpt = **iterList;
|
||||
QueueEntry *queueEntry = new QueueEntry(transOpt, allChartCells);
|
||||
cube.Add(queueEntry);
|
||||
}
|
||||
|
||||
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops)
|
||||
{
|
||||
QueueEntry *queueEntry = cube.Pop();
|
||||
|
||||
queueEntry->GetTranslationOption().GetTotalScore();
|
||||
Hypothesis *hypo = new Hypothesis(*queueEntry, m_manager);
|
||||
assert(hypo);
|
||||
|
||||
hypo->CalcScore();
|
||||
|
||||
AddHypothesis(hypo);
|
||||
// pluck things out of queue and add to hypo collection
|
||||
const size_t popLimit = staticData.GetCubePruningPopLimit();
|
||||
|
||||
// Expand queue entry
|
||||
queueEntry->CreateDeviants(cube);
|
||||
}
|
||||
for (size_t numPops = 0; numPops < popLimit && !cube.IsEmpty(); ++numPops) {
|
||||
QueueEntry *queueEntry = cube.Pop();
|
||||
|
||||
queueEntry->GetTranslationOption().GetTotalScore();
|
||||
Hypothesis *hypo = new Hypothesis(*queueEntry, m_manager);
|
||||
assert(hypo);
|
||||
|
||||
hypo->CalcScore();
|
||||
|
||||
AddHypothesis(hypo);
|
||||
|
||||
// Expand queue entry
|
||||
queueEntry->CreateDeviants(cube);
|
||||
}
|
||||
}
|
||||
|
||||
void ChartCell::SortHypotheses()
|
||||
{
|
||||
// sort each mini cells & fill up target lhs list
|
||||
assert(m_headWords.empty());
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
m_headWords.insert(iter->first);
|
||||
// sort each mini cells & fill up target lhs list
|
||||
assert(m_headWords.empty());
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
m_headWords.insert(iter->first);
|
||||
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.SortHypotheses();
|
||||
}
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.SortHypotheses();
|
||||
}
|
||||
}
|
||||
|
||||
const Hypothesis *ChartCell::GetBestHypothesis() const
|
||||
{
|
||||
const Hypothesis *ret = NULL;
|
||||
float bestScore = -std::numeric_limits<float>::infinity();
|
||||
const Hypothesis *ret = NULL;
|
||||
float bestScore = -std::numeric_limits<float>::infinity();
|
||||
|
||||
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
const HypoList &sortedList = iter->second.GetSortedHypotheses();
|
||||
assert(sortedList.size() > 0);
|
||||
|
||||
const Hypothesis *hypo = sortedList[0];
|
||||
if (hypo->GetTotalScore() > bestScore)
|
||||
{
|
||||
bestScore = hypo->GetTotalScore();
|
||||
ret = hypo;
|
||||
};
|
||||
}
|
||||
|
||||
return ret;
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
const HypoList &sortedList = iter->second.GetSortedHypotheses();
|
||||
assert(sortedList.size() > 0);
|
||||
|
||||
const Hypothesis *hypo = sortedList[0];
|
||||
if (hypo->GetTotalScore() > bestScore) {
|
||||
bestScore = hypo->GetTotalScore();
|
||||
ret = hypo;
|
||||
};
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ChartCell::HeadwordExists(const Moses::Word &headWord) const
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
iter = m_hypoColl.find(headWord);
|
||||
return (iter != m_hypoColl.end());
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
iter = m_hypoColl.find(headWord);
|
||||
return (iter != m_hypoColl.end());
|
||||
}
|
||||
|
||||
void ChartCell::CleanupArcList()
|
||||
{
|
||||
// only necessary if n-best calculations are enabled
|
||||
if (!m_nBestIsEnabled) return;
|
||||
// only necessary if n-best calculations are enabled
|
||||
if (!m_nBestIsEnabled) return;
|
||||
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.CleanupArcList();
|
||||
}
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.CleanupArcList();
|
||||
}
|
||||
}
|
||||
|
||||
void ChartCell::OutputSizes(std::ostream &out) const
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
const Moses::Word &targetLHS = iter->first;
|
||||
const HypothesisCollection &coll = iter->second;
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
const Moses::Word &targetLHS = iter->first;
|
||||
const HypothesisCollection &coll = iter->second;
|
||||
|
||||
out << targetLHS << "=" << coll.GetSize() << " ";
|
||||
}
|
||||
out << targetLHS << "=" << coll.GetSize() << " ";
|
||||
}
|
||||
}
|
||||
|
||||
size_t ChartCell::GetSize() const
|
||||
{
|
||||
size_t ret = 0;
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
const HypothesisCollection &coll = iter->second;
|
||||
|
||||
ret += coll.GetSize();
|
||||
}
|
||||
|
||||
return ret;
|
||||
size_t ret = 0;
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter) {
|
||||
const HypothesisCollection &coll = iter->second;
|
||||
|
||||
ret += coll.GetSize();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void ChartCell::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
|
||||
for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside)
|
||||
{
|
||||
const HypothesisCollection &coll = iterOutside->second;
|
||||
coll.GetSearchGraph(translationId, outputSearchGraphStream);
|
||||
}
|
||||
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
|
||||
for (iterOutside = m_hypoColl.begin(); iterOutside != m_hypoColl.end(); ++iterOutside) {
|
||||
const HypothesisCollection &coll = iterOutside->second;
|
||||
coll.GetSearchGraph(translationId, outputSearchGraphStream);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const ChartCell &cell)
|
||||
{
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
|
||||
for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside)
|
||||
{
|
||||
const Moses::Word &targetLHS = iterOutside->first;
|
||||
cerr << targetLHS << ":" << endl;
|
||||
|
||||
const HypothesisCollection &coll = iterOutside->second;
|
||||
cerr << coll;
|
||||
}
|
||||
std::map<Moses::Word, HypothesisCollection>::const_iterator iterOutside;
|
||||
for (iterOutside = cell.m_hypoColl.begin(); iterOutside != cell.m_hypoColl.end(); ++iterOutside) {
|
||||
const Moses::Word &targetLHS = iterOutside->first;
|
||||
cerr << targetLHS << ":" << endl;
|
||||
|
||||
/*
|
||||
ChartCell::HCType::const_iterator iter;
|
||||
for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
|
||||
{
|
||||
const Hypothesis &hypo = **iter;
|
||||
out << hypo << endl;
|
||||
}
|
||||
*/
|
||||
|
||||
return out;
|
||||
const HypothesisCollection &coll = iterOutside->second;
|
||||
cerr << coll;
|
||||
}
|
||||
|
||||
/*
|
||||
ChartCell::HCType::const_iterator iter;
|
||||
for (iter = cell.m_hypos.begin(); iter != cell.m_hypos.end(); ++iter)
|
||||
{
|
||||
const Hypothesis &hypo = **iter;
|
||||
out << hypo << endl;
|
||||
}
|
||||
*/
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -34,7 +34,7 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ChartTranslationOptionList;
|
||||
class ChartTranslationOptionList;
|
||||
}
|
||||
|
||||
namespace MosesChart
|
||||
@ -48,48 +48,48 @@ class Manager;
|
||||
|
||||
class ChartCell
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const ChartCell&);
|
||||
friend std::ostream& operator<<(std::ostream&, const ChartCell&);
|
||||
public:
|
||||
|
||||
protected:
|
||||
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
|
||||
Moses::NonTerminalSet m_headWords;
|
||||
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
|
||||
Moses::NonTerminalSet m_headWords;
|
||||
|
||||
Moses::WordsRange m_coverage;
|
||||
Moses::WordsRange m_coverage;
|
||||
|
||||
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
|
||||
Manager &m_manager;
|
||||
|
||||
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
|
||||
Manager &m_manager;
|
||||
|
||||
public:
|
||||
ChartCell(size_t startPos, size_t endPos, Manager &manager);
|
||||
ChartCell(size_t startPos, size_t endPos, Manager &manager);
|
||||
|
||||
void ProcessSentence(const Moses::ChartTranslationOptionList &transOptList
|
||||
,const ChartCellCollection &allChartCells);
|
||||
void ProcessSentence(const Moses::ChartTranslationOptionList &transOptList
|
||||
,const ChartCellCollection &allChartCells);
|
||||
|
||||
const HypoList &GetSortedHypotheses(const Moses::Word &headWord) const;
|
||||
bool AddHypothesis(Hypothesis *hypo);
|
||||
const HypoList &GetSortedHypotheses(const Moses::Word &headWord) const;
|
||||
bool AddHypothesis(Hypothesis *hypo);
|
||||
|
||||
void SortHypotheses();
|
||||
void PruneToSize();
|
||||
void SortHypotheses();
|
||||
void PruneToSize();
|
||||
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
|
||||
bool HeadwordExists(const Moses::Word &headWord) const;
|
||||
const Moses::NonTerminalSet &GetHeadwords() const
|
||||
{ return m_headWords; }
|
||||
bool HeadwordExists(const Moses::Word &headWord) const;
|
||||
const Moses::NonTerminalSet &GetHeadwords() const {
|
||||
return m_headWords;
|
||||
}
|
||||
|
||||
void CleanupArcList();
|
||||
void CleanupArcList();
|
||||
|
||||
void OutputSizes(std::ostream &out) const;
|
||||
size_t GetSize() const;
|
||||
|
||||
//! transitive comparison used for adding objects into set
|
||||
inline bool operator<(const ChartCell &compare) const
|
||||
{
|
||||
return m_coverage < compare.m_coverage;
|
||||
}
|
||||
void OutputSizes(std::ostream &out) const;
|
||||
size_t GetSize() const;
|
||||
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
//! transitive comparison used for adding objects into set
|
||||
inline bool operator<(const ChartCell &compare) const {
|
||||
return m_coverage < compare.m_coverage;
|
||||
}
|
||||
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -26,32 +26,29 @@
|
||||
namespace MosesChart
|
||||
{
|
||||
ChartCellCollection::ChartCellCollection(const Moses::InputType &input, Manager &manager)
|
||||
:m_hypoStackColl(input.GetSize())
|
||||
:m_hypoStackColl(input.GetSize())
|
||||
{
|
||||
size_t size = input.GetSize();
|
||||
for (size_t startPos = 0; startPos < size; ++startPos)
|
||||
{
|
||||
InnerCollType &inner = m_hypoStackColl[startPos];
|
||||
inner.resize(size - startPos);
|
||||
size_t size = input.GetSize();
|
||||
for (size_t startPos = 0; startPos < size; ++startPos) {
|
||||
InnerCollType &inner = m_hypoStackColl[startPos];
|
||||
inner.resize(size - startPos);
|
||||
|
||||
size_t ind = 0;
|
||||
for (size_t endPos = startPos ; endPos < size; ++endPos)
|
||||
{
|
||||
ChartCell *cell = new ChartCell(startPos, endPos, manager);
|
||||
inner[ind] = cell;
|
||||
++ind;
|
||||
}
|
||||
}
|
||||
size_t ind = 0;
|
||||
for (size_t endPos = startPos ; endPos < size; ++endPos) {
|
||||
ChartCell *cell = new ChartCell(startPos, endPos, manager);
|
||||
inner[ind] = cell;
|
||||
++ind;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ChartCellCollection::~ChartCellCollection()
|
||||
{
|
||||
OuterCollType::iterator iter;
|
||||
for (iter = m_hypoStackColl.begin(); iter != m_hypoStackColl.end(); ++iter)
|
||||
{
|
||||
InnerCollType &inner = *iter;
|
||||
Moses::RemoveAllInColl(inner);
|
||||
}
|
||||
OuterCollType::iterator iter;
|
||||
for (iter = m_hypoStackColl.begin(); iter != m_hypoStackColl.end(); ++iter) {
|
||||
InnerCollType &inner = *iter;
|
||||
Moses::RemoveAllInColl(inner);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -32,34 +32,31 @@ class InputType;
|
||||
namespace MosesChart
|
||||
{
|
||||
class Manager;
|
||||
|
||||
|
||||
class ChartCellCollection : public Moses::CellCollection
|
||||
{
|
||||
public:
|
||||
typedef std::vector<ChartCell*> InnerCollType;
|
||||
typedef std::vector<InnerCollType> OuterCollType;
|
||||
typedef std::vector<ChartCell*> InnerCollType;
|
||||
typedef std::vector<InnerCollType> OuterCollType;
|
||||
|
||||
protected:
|
||||
OuterCollType m_hypoStackColl;
|
||||
OuterCollType m_hypoStackColl;
|
||||
|
||||
public:
|
||||
ChartCellCollection(const Moses::InputType &input, Manager &manager);
|
||||
~ChartCellCollection();
|
||||
ChartCellCollection(const Moses::InputType &input, Manager &manager);
|
||||
~ChartCellCollection();
|
||||
|
||||
ChartCell &Get(const Moses::WordsRange &coverage)
|
||||
{
|
||||
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
const ChartCell &Get(const Moses::WordsRange &coverage) const
|
||||
{
|
||||
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
|
||||
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const
|
||||
{
|
||||
const ChartCell &cell = Get(coverage);
|
||||
return cell.GetHeadwords();
|
||||
}
|
||||
ChartCell &Get(const Moses::WordsRange &coverage) {
|
||||
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
const ChartCell &Get(const Moses::WordsRange &coverage) const {
|
||||
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
|
||||
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const {
|
||||
const ChartCell &cell = Get(coverage);
|
||||
return cell.GetHeadwords();
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -40,364 +40,333 @@ namespace MosesChart
|
||||
unsigned int Hypothesis::s_HypothesesCreated = 0;
|
||||
|
||||
#ifdef USE_HYPO_POOL
|
||||
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
|
||||
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
|
||||
#endif
|
||||
|
||||
Hypothesis::Hypothesis(const QueueEntry &queueEntry, Manager &manager)
|
||||
:m_transOpt(queueEntry.GetTranslationOption())
|
||||
,m_wordsConsumedTargetOrder(queueEntry.GetTranslationOption().GetWordsConsumedTargetOrder())
|
||||
,m_id(++s_HypothesesCreated)
|
||||
,m_currSourceWordsRange(queueEntry.GetTranslationOption().GetSourceWordsRange())
|
||||
,m_contextPrefix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
|
||||
,m_contextSuffix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
|
||||
,m_arcList(NULL)
|
||||
,m_manager(manager)
|
||||
:m_transOpt(queueEntry.GetTranslationOption())
|
||||
,m_wordsConsumedTargetOrder(queueEntry.GetTranslationOption().GetWordsConsumedTargetOrder())
|
||||
,m_id(++s_HypothesesCreated)
|
||||
,m_currSourceWordsRange(queueEntry.GetTranslationOption().GetSourceWordsRange())
|
||||
,m_contextPrefix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
|
||||
,m_contextSuffix(Output, manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder())
|
||||
,m_arcList(NULL)
|
||||
,m_manager(manager)
|
||||
{
|
||||
assert(GetCurrTargetPhrase().GetSize() == m_wordsConsumedTargetOrder.size());
|
||||
//TRACE_ERR(m_targetPhrase << endl);
|
||||
assert(GetCurrTargetPhrase().GetSize() == m_wordsConsumedTargetOrder.size());
|
||||
//TRACE_ERR(m_targetPhrase << endl);
|
||||
|
||||
m_numTargetTerminals = GetCurrTargetPhrase().GetNumTerminals();
|
||||
m_numTargetTerminals = GetCurrTargetPhrase().GetNumTerminals();
|
||||
|
||||
const std::vector<ChildEntry> &childEntries = queueEntry.GetChildEntries();
|
||||
assert(m_prevHypos.empty());
|
||||
m_prevHypos.reserve(childEntries.size());
|
||||
vector<ChildEntry>::const_iterator iter;
|
||||
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter)
|
||||
{
|
||||
const ChildEntry &childEntry = *iter;
|
||||
const Hypothesis *prevHypo = childEntry.GetHypothesis();
|
||||
const std::vector<ChildEntry> &childEntries = queueEntry.GetChildEntries();
|
||||
assert(m_prevHypos.empty());
|
||||
m_prevHypos.reserve(childEntries.size());
|
||||
vector<ChildEntry>::const_iterator iter;
|
||||
for (iter = childEntries.begin(); iter != childEntries.end(); ++iter) {
|
||||
const ChildEntry &childEntry = *iter;
|
||||
const Hypothesis *prevHypo = childEntry.GetHypothesis();
|
||||
|
||||
m_numTargetTerminals += prevHypo->GetNumTargetTerminals();
|
||||
m_numTargetTerminals += prevHypo->GetNumTargetTerminals();
|
||||
|
||||
m_prevHypos.push_back(prevHypo);
|
||||
}
|
||||
m_prevHypos.push_back(prevHypo);
|
||||
}
|
||||
|
||||
size_t maxNGram = manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder();
|
||||
CalcPrefix(m_contextPrefix, maxNGram - 1);
|
||||
CalcSuffix(m_contextSuffix, maxNGram - 1);
|
||||
size_t maxNGram = manager.GetTranslationSystem()->GetLanguageModels().GetMaxNGramOrder();
|
||||
CalcPrefix(m_contextPrefix, maxNGram - 1);
|
||||
CalcSuffix(m_contextSuffix, maxNGram - 1);
|
||||
}
|
||||
|
||||
Hypothesis::~Hypothesis()
|
||||
{
|
||||
if (m_arcList)
|
||||
{
|
||||
ArcList::iterator iter;
|
||||
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter)
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
Delete(hypo);
|
||||
}
|
||||
m_arcList->clear();
|
||||
if (m_arcList) {
|
||||
ArcList::iterator iter;
|
||||
for (iter = m_arcList->begin() ; iter != m_arcList->end() ; ++iter) {
|
||||
Hypothesis *hypo = *iter;
|
||||
Delete(hypo);
|
||||
}
|
||||
m_arcList->clear();
|
||||
|
||||
delete m_arcList;
|
||||
}
|
||||
delete m_arcList;
|
||||
}
|
||||
}
|
||||
|
||||
void Hypothesis::CreateOutputPhrase(Phrase &outPhrase) const
|
||||
{
|
||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
|
||||
{
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
if (word.IsNonTerminal())
|
||||
{ // non-term. fill out with prev hypo
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
prevHypo->CreateOutputPhrase(outPhrase);
|
||||
}
|
||||
else
|
||||
{
|
||||
outPhrase.AddWord(word);
|
||||
}
|
||||
}
|
||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
if (word.IsNonTerminal()) {
|
||||
// non-term. fill out with prev hypo
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
prevHypo->CreateOutputPhrase(outPhrase);
|
||||
} else {
|
||||
outPhrase.AddWord(word);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Phrase Hypothesis::GetOutputPhrase() const
|
||||
{
|
||||
Phrase outPhrase(Output);
|
||||
CreateOutputPhrase(outPhrase);
|
||||
return outPhrase;
|
||||
Phrase outPhrase(Output);
|
||||
CreateOutputPhrase(outPhrase);
|
||||
return outPhrase;
|
||||
}
|
||||
|
||||
size_t Hypothesis::CalcPrefix(Phrase &ret, size_t size) const
|
||||
{
|
||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos)
|
||||
{
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
|
||||
if (word.IsNonTerminal())
|
||||
{
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size = prevHypo->CalcPrefix(ret, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
ret.AddWord(GetCurrTargetPhrase().GetWord(pos));
|
||||
size--;
|
||||
}
|
||||
for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) {
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
|
||||
if (size==0)
|
||||
break;
|
||||
}
|
||||
if (word.IsNonTerminal()) {
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size = prevHypo->CalcPrefix(ret, size);
|
||||
} else {
|
||||
ret.AddWord(GetCurrTargetPhrase().GetWord(pos));
|
||||
size--;
|
||||
}
|
||||
|
||||
return size;
|
||||
if (size==0)
|
||||
break;
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
size_t Hypothesis::CalcSuffix(Phrase &ret, size_t size) const
|
||||
{
|
||||
assert(m_contextPrefix.GetSize() <= m_numTargetTerminals);
|
||||
assert(m_contextPrefix.GetSize() <= m_numTargetTerminals);
|
||||
|
||||
if (m_contextPrefix.GetSize() == m_numTargetTerminals)
|
||||
{ // small hypo. the prefix will contains the whole hypo
|
||||
size_t maxCount = min(m_contextPrefix.GetSize(), size)
|
||||
, pos = m_contextPrefix.GetSize() - 1;
|
||||
if (m_contextPrefix.GetSize() == m_numTargetTerminals) {
|
||||
// small hypo. the prefix will contains the whole hypo
|
||||
size_t maxCount = min(m_contextPrefix.GetSize(), size)
|
||||
, pos = m_contextPrefix.GetSize() - 1;
|
||||
|
||||
for (size_t ind = 0; ind < maxCount; ++ind)
|
||||
{
|
||||
const Word &word = m_contextPrefix.GetWord(pos);
|
||||
ret.PrependWord(word);
|
||||
--pos;
|
||||
}
|
||||
for (size_t ind = 0; ind < maxCount; ++ind) {
|
||||
const Word &word = m_contextPrefix.GetWord(pos);
|
||||
ret.PrependWord(word);
|
||||
--pos;
|
||||
}
|
||||
|
||||
size -= maxCount;
|
||||
return size;
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int pos = (int) GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos)
|
||||
{
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
size -= maxCount;
|
||||
return size;
|
||||
} else {
|
||||
for (int pos = (int) GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos) {
|
||||
const Word &word = GetCurrTargetPhrase().GetWord(pos);
|
||||
|
||||
if (word.IsNonTerminal())
|
||||
{
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size = prevHypo->CalcSuffix(ret, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
ret.PrependWord(GetCurrTargetPhrase().GetWord(pos));
|
||||
size--;
|
||||
}
|
||||
if (word.IsNonTerminal()) {
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[pos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size = prevHypo->CalcSuffix(ret, size);
|
||||
} else {
|
||||
ret.PrependWord(GetCurrTargetPhrase().GetWord(pos));
|
||||
size--;
|
||||
}
|
||||
|
||||
if (size==0)
|
||||
break;
|
||||
}
|
||||
if (size==0)
|
||||
break;
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
}
|
||||
|
||||
int Hypothesis::LMContextCompare(const Hypothesis &other) const
|
||||
{
|
||||
// prefix
|
||||
if (m_currSourceWordsRange.GetStartPos() > 0)
|
||||
{
|
||||
int ret = GetPrefix().Compare(other.GetPrefix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
// prefix
|
||||
if (m_currSourceWordsRange.GetStartPos() > 0) {
|
||||
int ret = GetPrefix().Compare(other.GetPrefix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
// suffix
|
||||
size_t inputSize = m_manager.GetSource().GetSize();
|
||||
if (m_currSourceWordsRange.GetEndPos() < inputSize - 1)
|
||||
{
|
||||
int ret = GetSuffix().Compare(other.GetSuffix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
// suffix
|
||||
size_t inputSize = m_manager.GetSource().GetSize();
|
||||
if (m_currSourceWordsRange.GetEndPos() < inputSize - 1) {
|
||||
int ret = GetSuffix().Compare(other.GetSuffix());
|
||||
if (ret != 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
// they're the same
|
||||
return 0;
|
||||
// they're the same
|
||||
return 0;
|
||||
}
|
||||
|
||||
void Hypothesis::CalcScore()
|
||||
{
|
||||
// total scores from prev hypos
|
||||
std::vector<const Hypothesis*>::iterator iter;
|
||||
for (iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter)
|
||||
{
|
||||
const Hypothesis &prevHypo = **iter;
|
||||
const ScoreComponentCollection &scoreBreakdown = prevHypo.GetScoreBreakdown();
|
||||
// total scores from prev hypos
|
||||
std::vector<const Hypothesis*>::iterator iter;
|
||||
for (iter = m_prevHypos.begin(); iter != m_prevHypos.end(); ++iter) {
|
||||
const Hypothesis &prevHypo = **iter;
|
||||
const ScoreComponentCollection &scoreBreakdown = prevHypo.GetScoreBreakdown();
|
||||
|
||||
m_scoreBreakdown.PlusEquals(scoreBreakdown);
|
||||
}
|
||||
m_scoreBreakdown.PlusEquals(scoreBreakdown);
|
||||
}
|
||||
|
||||
// translation models & word penalty
|
||||
const ScoreComponentCollection &scoreBreakdown = GetCurrTargetPhrase().GetScoreBreakdown();
|
||||
m_scoreBreakdown.PlusEquals(scoreBreakdown);
|
||||
// translation models & word penalty
|
||||
const ScoreComponentCollection &scoreBreakdown = GetCurrTargetPhrase().GetScoreBreakdown();
|
||||
m_scoreBreakdown.PlusEquals(scoreBreakdown);
|
||||
|
||||
CalcLMScore();
|
||||
CalcLMScore();
|
||||
|
||||
m_totalScore = m_scoreBreakdown.GetWeightedScore();
|
||||
m_totalScore = m_scoreBreakdown.GetWeightedScore();
|
||||
}
|
||||
|
||||
void Hypothesis::CalcLMScore()
|
||||
{
|
||||
const LMList& lmList = m_manager.GetTranslationSystem()->GetLanguageModels();
|
||||
assert(m_lmNGram.GetWeightedScore() == 0);
|
||||
assert(m_lmNGram.GetWeightedScore() == 0);
|
||||
|
||||
m_scoreBreakdown.ZeroAllLM(lmList);
|
||||
m_scoreBreakdown.ZeroAllLM(lmList);
|
||||
|
||||
Phrase outPhrase(Output); // = GetOutputPhrase();
|
||||
bool calcNow = false, firstPhrase = true;
|
||||
Phrase outPhrase(Output); // = GetOutputPhrase();
|
||||
bool calcNow = false, firstPhrase = true;
|
||||
|
||||
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos)
|
||||
{
|
||||
const Word &targetWord = GetCurrTargetPhrase().GetWord(targetPhrasePos);
|
||||
if (!targetWord.IsNonTerminal())
|
||||
{ // just a word, add to phrase for lm scoring
|
||||
outPhrase.AddWord(targetWord);
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[targetPhrasePos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size_t numTargetTerminals = prevHypo->GetNumTargetTerminals();
|
||||
for (size_t targetPhrasePos = 0; targetPhrasePos < GetCurrTargetPhrase().GetSize(); ++targetPhrasePos) {
|
||||
const Word &targetWord = GetCurrTargetPhrase().GetWord(targetPhrasePos);
|
||||
if (!targetWord.IsNonTerminal()) {
|
||||
// just a word, add to phrase for lm scoring
|
||||
outPhrase.AddWord(targetWord);
|
||||
} else {
|
||||
size_t nonTermInd = m_wordsConsumedTargetOrder[targetPhrasePos];
|
||||
const Hypothesis *prevHypo = m_prevHypos[nonTermInd];
|
||||
size_t numTargetTerminals = prevHypo->GetNumTargetTerminals();
|
||||
|
||||
if (numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
|
||||
{ // large hypo (for trigram lm, another hypo equal or over 2 words). just take the prefix & suffix
|
||||
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
|
||||
if (numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
|
||||
// large hypo (for trigram lm, another hypo equal or over 2 words). just take the prefix & suffix
|
||||
m_lmNGram.PlusEqualsAllLM(lmList, prevHypo->m_lmNGram);
|
||||
|
||||
// calc & add overlapping lm scores
|
||||
// prefix
|
||||
outPhrase.Append(prevHypo->GetPrefix());
|
||||
calcNow = true;
|
||||
}
|
||||
else
|
||||
{ // small hypo (for trigram lm, 1-word hypos).
|
||||
// add target phrase to temp phrase and continue, but don't score yet
|
||||
outPhrase.Append(prevHypo->GetPrefix());
|
||||
}
|
||||
// calc & add overlapping lm scores
|
||||
// prefix
|
||||
outPhrase.Append(prevHypo->GetPrefix());
|
||||
calcNow = true;
|
||||
} else {
|
||||
// small hypo (for trigram lm, 1-word hypos).
|
||||
// add target phrase to temp phrase and continue, but don't score yet
|
||||
outPhrase.Append(prevHypo->GetPrefix());
|
||||
}
|
||||
|
||||
if (calcNow)
|
||||
{
|
||||
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
|
||||
{ // get from other prev hypo. faster
|
||||
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
|
||||
m_lmNGram.Assign(prevHypo->m_lmNGram);
|
||||
}
|
||||
else
|
||||
{ // calc
|
||||
lmList.CalcAllLMScores(outPhrase
|
||||
, m_lmNGram
|
||||
, (firstPhrase) ? &m_lmPrefix : NULL);
|
||||
}
|
||||
if (calcNow) {
|
||||
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1) {
|
||||
// get from other prev hypo. faster
|
||||
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
|
||||
m_lmNGram.Assign(prevHypo->m_lmNGram);
|
||||
} else {
|
||||
// calc
|
||||
lmList.CalcAllLMScores(outPhrase
|
||||
, m_lmNGram
|
||||
, (firstPhrase) ? &m_lmPrefix : NULL);
|
||||
}
|
||||
|
||||
// create new phrase from suffix. score later when appended with next words
|
||||
outPhrase.Clear();
|
||||
outPhrase.Append(prevHypo->GetSuffix());
|
||||
// create new phrase from suffix. score later when appended with next words
|
||||
outPhrase.Clear();
|
||||
outPhrase.Append(prevHypo->GetSuffix());
|
||||
|
||||
firstPhrase = false;
|
||||
calcNow = false;
|
||||
}
|
||||
} // if (!targetWord.IsNonTerminal())
|
||||
} // for (size_t targetPhrasePos
|
||||
firstPhrase = false;
|
||||
calcNow = false;
|
||||
}
|
||||
} // if (!targetWord.IsNonTerminal())
|
||||
} // for (size_t targetPhrasePos
|
||||
|
||||
lmList.CalcAllLMScores(outPhrase
|
||||
, m_lmNGram
|
||||
, (firstPhrase) ? &m_lmPrefix : NULL);
|
||||
lmList.CalcAllLMScores(outPhrase
|
||||
, m_lmNGram
|
||||
, (firstPhrase) ? &m_lmPrefix : NULL);
|
||||
|
||||
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmPrefix);
|
||||
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmNGram);
|
||||
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmPrefix);
|
||||
m_scoreBreakdown.PlusEqualsAllLM(lmList, m_lmNGram);
|
||||
|
||||
/*
|
||||
// lazy way. keep for comparison
|
||||
Phrase outPhrase = GetOutputPhrase();
|
||||
// cerr << outPhrase << " ";
|
||||
/*
|
||||
// lazy way. keep for comparison
|
||||
Phrase outPhrase = GetOutputPhrase();
|
||||
// cerr << outPhrase << " ";
|
||||
|
||||
float retFullScore, retNGramScore;
|
||||
StaticData::Instance().GetAllLM().CalcScore(outPhrase
|
||||
, retFullScore
|
||||
, retNGramScore
|
||||
, m_scoreBreakdown
|
||||
, &m_lmNGram
|
||||
, false);
|
||||
*/
|
||||
float retFullScore, retNGramScore;
|
||||
StaticData::Instance().GetAllLM().CalcScore(outPhrase
|
||||
, retFullScore
|
||||
, retNGramScore
|
||||
, m_scoreBreakdown
|
||||
, &m_lmNGram
|
||||
, false);
|
||||
*/
|
||||
}
|
||||
|
||||
void Hypothesis::AddArc(Hypothesis *loserHypo)
|
||||
{
|
||||
if (!m_arcList) {
|
||||
if (loserHypo->m_arcList) // we don't have an arcList, but loser does
|
||||
{
|
||||
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
|
||||
loserHypo->m_arcList = 0; // prevent a double deletion
|
||||
}
|
||||
else
|
||||
{ this->m_arcList = new ArcList(); }
|
||||
} else {
|
||||
if (loserHypo->m_arcList) { // both have an arc list: merge. delete loser
|
||||
size_t my_size = m_arcList->size();
|
||||
size_t add_size = loserHypo->m_arcList->size();
|
||||
this->m_arcList->resize(my_size + add_size, 0);
|
||||
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(Hypothesis *));
|
||||
delete loserHypo->m_arcList;
|
||||
loserHypo->m_arcList = 0;
|
||||
} else { // loserHypo doesn't have any arcs
|
||||
// DO NOTHING
|
||||
}
|
||||
}
|
||||
m_arcList->push_back(loserHypo);
|
||||
if (!m_arcList) {
|
||||
if (loserHypo->m_arcList) { // we don't have an arcList, but loser does
|
||||
this->m_arcList = loserHypo->m_arcList; // take ownership, we'll delete
|
||||
loserHypo->m_arcList = 0; // prevent a double deletion
|
||||
} else {
|
||||
this->m_arcList = new ArcList();
|
||||
}
|
||||
} else {
|
||||
if (loserHypo->m_arcList) { // both have an arc list: merge. delete loser
|
||||
size_t my_size = m_arcList->size();
|
||||
size_t add_size = loserHypo->m_arcList->size();
|
||||
this->m_arcList->resize(my_size + add_size, 0);
|
||||
std::memcpy(&(*m_arcList)[0] + my_size, &(*loserHypo->m_arcList)[0], add_size * sizeof(Hypothesis *));
|
||||
delete loserHypo->m_arcList;
|
||||
loserHypo->m_arcList = 0;
|
||||
} else { // loserHypo doesn't have any arcs
|
||||
// DO NOTHING
|
||||
}
|
||||
}
|
||||
m_arcList->push_back(loserHypo);
|
||||
}
|
||||
|
||||
// sorting helper
|
||||
struct CompareChartHypothesisTotalScore
|
||||
{
|
||||
bool operator()(const Hypothesis* hypo1, const Hypothesis* hypo2) const
|
||||
{
|
||||
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
|
||||
}
|
||||
struct CompareChartHypothesisTotalScore {
|
||||
bool operator()(const Hypothesis* hypo1, const Hypothesis* hypo2) const {
|
||||
return hypo1->GetTotalScore() > hypo2->GetTotalScore();
|
||||
}
|
||||
};
|
||||
|
||||
void Hypothesis::CleanupArcList()
|
||||
{
|
||||
// point this hypo's main hypo to itself
|
||||
m_winningHypo = this;
|
||||
// point this hypo's main hypo to itself
|
||||
m_winningHypo = this;
|
||||
|
||||
if (!m_arcList) return;
|
||||
if (!m_arcList) return;
|
||||
|
||||
/* keep only number of arcs we need to create all n-best paths.
|
||||
* However, may not be enough if only unique candidates are needed,
|
||||
* so we'll keep all of arc list if nedd distinct n-best list
|
||||
*/
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
size_t nBestSize = staticData.GetNBestSize();
|
||||
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph();
|
||||
/* keep only number of arcs we need to create all n-best paths.
|
||||
* However, may not be enough if only unique candidates are needed,
|
||||
* so we'll keep all of arc list if nedd distinct n-best list
|
||||
*/
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
size_t nBestSize = staticData.GetNBestSize();
|
||||
bool distinctNBest = staticData.GetDistinctNBest() || staticData.UseMBR() || staticData.GetOutputSearchGraph();
|
||||
|
||||
if (!distinctNBest && m_arcList->size() > nBestSize)
|
||||
{ // prune arc list only if there too many arcs
|
||||
nth_element(m_arcList->begin()
|
||||
, m_arcList->begin() + nBestSize - 1
|
||||
, m_arcList->end()
|
||||
, CompareChartHypothesisTotalScore());
|
||||
|
||||
// delete bad ones
|
||||
ArcList::iterator iter;
|
||||
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter)
|
||||
{
|
||||
Hypothesis *arc = *iter;
|
||||
Hypothesis::Delete(arc);
|
||||
}
|
||||
m_arcList->erase(m_arcList->begin() + nBestSize
|
||||
, m_arcList->end());
|
||||
}
|
||||
if (!distinctNBest && m_arcList->size() > nBestSize) {
|
||||
// prune arc list only if there too many arcs
|
||||
nth_element(m_arcList->begin()
|
||||
, m_arcList->begin() + nBestSize - 1
|
||||
, m_arcList->end()
|
||||
, CompareChartHypothesisTotalScore());
|
||||
|
||||
// delete bad ones
|
||||
ArcList::iterator iter;
|
||||
for (iter = m_arcList->begin() + nBestSize ; iter != m_arcList->end() ; ++iter) {
|
||||
Hypothesis *arc = *iter;
|
||||
Hypothesis::Delete(arc);
|
||||
}
|
||||
m_arcList->erase(m_arcList->begin() + nBestSize
|
||||
, m_arcList->end());
|
||||
}
|
||||
|
||||
// set all arc's main hypo variable to this hypo
|
||||
ArcList::iterator iter = m_arcList->begin();
|
||||
for (; iter != m_arcList->end() ; ++iter) {
|
||||
Hypothesis *arc = *iter;
|
||||
arc->SetWinningHypo(this);
|
||||
}
|
||||
|
||||
// set all arc's main hypo variable to this hypo
|
||||
ArcList::iterator iter = m_arcList->begin();
|
||||
for (; iter != m_arcList->end() ; ++iter)
|
||||
{
|
||||
Hypothesis *arc = *iter;
|
||||
arc->SetWinningHypo(this);
|
||||
}
|
||||
|
||||
//cerr << m_arcList->size() << " ";
|
||||
}
|
||||
|
||||
void Hypothesis::SetWinningHypo(const Hypothesis *hypo)
|
||||
{
|
||||
m_winningHypo = hypo;
|
||||
|
||||
|
||||
// never gonna use to recombine. clear prefix & suffix phrases to save mem
|
||||
m_contextPrefix.Clear();
|
||||
m_contextSuffix.Clear();
|
||||
@ -408,29 +377,28 @@ TO_STRING_BODY(Hypothesis)
|
||||
// friend
|
||||
ostream& operator<<(ostream& out, const Hypothesis& hypo)
|
||||
{
|
||||
//Phrase outPhrase(Output);
|
||||
//hypo.CreateOutputPhrase(outPhrase);
|
||||
//Phrase outPhrase(Output);
|
||||
//hypo.CreateOutputPhrase(outPhrase);
|
||||
|
||||
// words bitmap
|
||||
out << " " << hypo.GetId()
|
||||
<< " " << hypo.GetCurrTargetPhrase()
|
||||
//<< " " << outPhrase
|
||||
<< " " << hypo.GetCurrSourceRange();
|
||||
//<< " " << hypo.m_currSourceWordsRange
|
||||
|
||||
HypoList::const_iterator iter;
|
||||
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter)
|
||||
{
|
||||
const Hypothesis &prevHypo = **iter;
|
||||
out << " " << prevHypo.GetId();
|
||||
}
|
||||
// words bitmap
|
||||
out << " " << hypo.GetId()
|
||||
<< " " << hypo.GetCurrTargetPhrase()
|
||||
//<< " " << outPhrase
|
||||
<< " " << hypo.GetCurrSourceRange();
|
||||
//<< " " << hypo.m_currSourceWordsRange
|
||||
|
||||
out << " [total=" << hypo.GetTotalScore() << "]";
|
||||
out << " " << hypo.GetScoreBreakdown();
|
||||
HypoList::const_iterator iter;
|
||||
for (iter = hypo.GetPrevHypos().begin(); iter != hypo.GetPrevHypos().end(); ++iter) {
|
||||
const Hypothesis &prevHypo = **iter;
|
||||
out << " " << prevHypo.GetId();
|
||||
}
|
||||
|
||||
//out << endl;
|
||||
out << " [total=" << hypo.GetTotalScore() << "]";
|
||||
out << " " << hypo.GetScoreBreakdown();
|
||||
|
||||
return out;
|
||||
//out << endl;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -34,127 +34,133 @@ namespace MosesChart
|
||||
class QueueEntry;
|
||||
class Hypothesis;
|
||||
class Manager;
|
||||
|
||||
|
||||
typedef std::vector<Hypothesis*> ArcList;
|
||||
|
||||
class Hypothesis
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const Hypothesis&);
|
||||
friend std::ostream& operator<<(std::ostream&, const Hypothesis&);
|
||||
|
||||
protected:
|
||||
|
||||
#ifdef USE_HYPO_POOL
|
||||
static ObjectPool<Hypothesis> s_objectPool;
|
||||
static ObjectPool<Hypothesis> s_objectPool;
|
||||
#endif
|
||||
|
||||
static unsigned int s_HypothesesCreated;
|
||||
static unsigned int s_HypothesesCreated;
|
||||
|
||||
int m_id; /**< numeric ID of this hypothesis, used for logging */
|
||||
const Moses::ChartTranslationOption &m_transOpt;
|
||||
int m_id; /**< numeric ID of this hypothesis, used for logging */
|
||||
const Moses::ChartTranslationOption &m_transOpt;
|
||||
|
||||
Moses::Phrase m_contextPrefix, m_contextSuffix;
|
||||
const std::vector<size_t> &m_wordsConsumedTargetOrder; // same size as target phrase ?
|
||||
Moses::WordsRange m_currSourceWordsRange;
|
||||
Moses::ScoreComponentCollection m_scoreBreakdown /*! detailed score break-down by components (for instance language model, word penalty, etc) */
|
||||
,m_lmNGram
|
||||
,m_lmPrefix;
|
||||
float m_totalScore;
|
||||
size_t m_numTargetTerminals;
|
||||
Moses::Phrase m_contextPrefix, m_contextSuffix;
|
||||
const std::vector<size_t> &m_wordsConsumedTargetOrder; // same size as target phrase ?
|
||||
Moses::WordsRange m_currSourceWordsRange;
|
||||
Moses::ScoreComponentCollection m_scoreBreakdown /*! detailed score break-down by components (for instance language model, word penalty, etc) */
|
||||
,m_lmNGram
|
||||
,m_lmPrefix;
|
||||
float m_totalScore;
|
||||
size_t m_numTargetTerminals;
|
||||
|
||||
ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
|
||||
const Hypothesis *m_winningHypo;
|
||||
ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
|
||||
const Hypothesis *m_winningHypo;
|
||||
|
||||
std::vector<const Hypothesis*> m_prevHypos;
|
||||
std::vector<const Hypothesis*> m_prevHypos;
|
||||
|
||||
Manager& m_manager;
|
||||
|
||||
size_t CalcPrefix(Moses::Phrase &ret, size_t size) const;
|
||||
size_t CalcSuffix(Moses::Phrase &ret, size_t size) const;
|
||||
Manager& m_manager;
|
||||
|
||||
void CalcLMScore();
|
||||
size_t CalcPrefix(Moses::Phrase &ret, size_t size) const;
|
||||
size_t CalcSuffix(Moses::Phrase &ret, size_t size) const;
|
||||
|
||||
Hypothesis(); // not implemented
|
||||
Hypothesis(const Hypothesis ©); // not implemented
|
||||
void CalcLMScore();
|
||||
|
||||
Hypothesis(); // not implemented
|
||||
Hypothesis(const Hypothesis ©); // not implemented
|
||||
|
||||
public:
|
||||
static void ResetHypoCount()
|
||||
{ s_HypothesesCreated = 0; }
|
||||
static unsigned int GetHypoCount()
|
||||
{ return s_HypothesesCreated; }
|
||||
static void ResetHypoCount() {
|
||||
s_HypothesesCreated = 0;
|
||||
}
|
||||
static unsigned int GetHypoCount() {
|
||||
return s_HypothesesCreated;
|
||||
}
|
||||
|
||||
#ifdef USE_HYPO_POOL
|
||||
void *operator new(size_t num_bytes)
|
||||
{
|
||||
void *ptr = s_objectPool.getPtr();
|
||||
return ptr;
|
||||
}
|
||||
void *operator new(size_t num_bytes) {
|
||||
void *ptr = s_objectPool.getPtr();
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static void Delete(Hypothesis *hypo)
|
||||
{
|
||||
s_objectPool.freeObject(hypo);
|
||||
}
|
||||
static void Delete(Hypothesis *hypo) {
|
||||
s_objectPool.freeObject(hypo);
|
||||
}
|
||||
#else
|
||||
static void Delete(Hypothesis *hypo)
|
||||
{
|
||||
delete hypo;
|
||||
}
|
||||
static void Delete(Hypothesis *hypo) {
|
||||
delete hypo;
|
||||
}
|
||||
#endif
|
||||
|
||||
explicit Hypothesis(const QueueEntry &queueEntry, Manager &manager);
|
||||
~Hypothesis();
|
||||
explicit Hypothesis(const QueueEntry &queueEntry, Manager &manager);
|
||||
~Hypothesis();
|
||||
|
||||
int GetId()const
|
||||
{ return m_id;}
|
||||
const Moses::ChartTranslationOption &GetTranslationOption()const
|
||||
{ return m_transOpt; }
|
||||
const Moses::TargetPhrase &GetCurrTargetPhrase()const
|
||||
{ return m_transOpt.GetTargetPhrase(); }
|
||||
const Moses::WordsRange &GetCurrSourceRange()const
|
||||
{ return m_currSourceWordsRange; }
|
||||
inline const ArcList* GetArcList() const
|
||||
{
|
||||
return m_arcList;
|
||||
}
|
||||
int GetId()const {
|
||||
return m_id;
|
||||
}
|
||||
const Moses::ChartTranslationOption &GetTranslationOption()const {
|
||||
return m_transOpt;
|
||||
}
|
||||
const Moses::TargetPhrase &GetCurrTargetPhrase()const {
|
||||
return m_transOpt.GetTargetPhrase();
|
||||
}
|
||||
const Moses::WordsRange &GetCurrSourceRange()const {
|
||||
return m_currSourceWordsRange;
|
||||
}
|
||||
inline const ArcList* GetArcList() const {
|
||||
return m_arcList;
|
||||
}
|
||||
|
||||
void CreateOutputPhrase(Moses::Phrase &outPhrase) const;
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
void CreateOutputPhrase(Moses::Phrase &outPhrase) const;
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
|
||||
int LMContextCompare(const Hypothesis &other) const;
|
||||
int LMContextCompare(const Hypothesis &other) const;
|
||||
|
||||
const Moses::Phrase &GetPrefix() const
|
||||
{ return m_contextPrefix; }
|
||||
const Moses::Phrase &GetSuffix() const
|
||||
{ return m_contextSuffix; }
|
||||
const Moses::Phrase &GetPrefix() const {
|
||||
return m_contextPrefix;
|
||||
}
|
||||
const Moses::Phrase &GetSuffix() const {
|
||||
return m_contextSuffix;
|
||||
}
|
||||
|
||||
void CalcScore();
|
||||
void CalcScore();
|
||||
|
||||
void AddArc(Hypothesis *loserHypo);
|
||||
void CleanupArcList();
|
||||
void SetWinningHypo(const Hypothesis *hypo);
|
||||
void AddArc(Hypothesis *loserHypo);
|
||||
void CleanupArcList();
|
||||
void SetWinningHypo(const Hypothesis *hypo);
|
||||
|
||||
const Moses::ScoreComponentCollection &GetScoreBreakdown() const
|
||||
{ return m_scoreBreakdown; }
|
||||
float GetTotalScore() const
|
||||
{ return m_totalScore; }
|
||||
const Moses::ScoreComponentCollection &GetScoreBreakdown() const {
|
||||
return m_scoreBreakdown;
|
||||
}
|
||||
float GetTotalScore() const {
|
||||
return m_totalScore;
|
||||
}
|
||||
|
||||
const std::vector<const Hypothesis*> &GetPrevHypos() const
|
||||
{ return m_prevHypos; }
|
||||
const std::vector<const Hypothesis*> &GetPrevHypos() const {
|
||||
return m_prevHypos;
|
||||
}
|
||||
|
||||
size_t GetWordsConsumedTargetOrder(size_t pos) const
|
||||
{
|
||||
assert(pos < m_wordsConsumedTargetOrder.size());
|
||||
return m_wordsConsumedTargetOrder[pos];
|
||||
}
|
||||
size_t GetWordsConsumedTargetOrder(size_t pos) const {
|
||||
assert(pos < m_wordsConsumedTargetOrder.size());
|
||||
return m_wordsConsumedTargetOrder[pos];
|
||||
}
|
||||
|
||||
const Moses::Word &GetTargetLHS() const
|
||||
{ return GetCurrTargetPhrase().GetTargetLHS(); }
|
||||
const Moses::Word &GetTargetLHS() const {
|
||||
return GetCurrTargetPhrase().GetTargetLHS();
|
||||
}
|
||||
|
||||
size_t GetNumTargetTerminals() const
|
||||
{
|
||||
return m_numTargetTerminals;
|
||||
}
|
||||
size_t GetNumTargetTerminals() const {
|
||||
return m_numTargetTerminals;
|
||||
}
|
||||
|
||||
TO_STRING();
|
||||
TO_STRING();
|
||||
|
||||
}; // class Hypothesis
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -33,276 +33,253 @@ namespace MosesChart
|
||||
|
||||
HypothesisCollection::HypothesisCollection()
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
m_beamWidth = staticData.GetBeamWidth();
|
||||
m_maxHypoStackSize = staticData.GetMaxHypoStackSize();
|
||||
m_nBestIsEnabled = staticData.IsNBestEnabled();
|
||||
m_bestScore = -std::numeric_limits<float>::infinity();
|
||||
m_beamWidth = staticData.GetBeamWidth();
|
||||
m_maxHypoStackSize = staticData.GetMaxHypoStackSize();
|
||||
m_nBestIsEnabled = staticData.IsNBestEnabled();
|
||||
m_bestScore = -std::numeric_limits<float>::infinity();
|
||||
}
|
||||
|
||||
HypothesisCollection::~HypothesisCollection()
|
||||
{
|
||||
HCType::iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
Hypothesis::Delete(hypo);
|
||||
}
|
||||
//Moses::RemoveAllInColl(m_hypos);
|
||||
HCType::iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
|
||||
Hypothesis *hypo = *iter;
|
||||
Hypothesis::Delete(hypo);
|
||||
}
|
||||
//Moses::RemoveAllInColl(m_hypos);
|
||||
}
|
||||
|
||||
|
||||
bool HypothesisCollection::AddHypothesis(Hypothesis *hypo, Manager &manager)
|
||||
{
|
||||
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth)
|
||||
{ // really bad score. don't bother adding hypo into collection
|
||||
manager.GetSentenceStats().AddDiscarded();
|
||||
VERBOSE(3,"discarded, too bad for stack" << std::endl);
|
||||
Hypothesis::Delete(hypo);
|
||||
return false;
|
||||
}
|
||||
|
||||
// over threshold, try to add to collection
|
||||
std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
|
||||
if (addRet.second)
|
||||
{ // nothing found. add to collection
|
||||
return true;
|
||||
}
|
||||
|
||||
// equiv hypo exists, recombine with other hypo
|
||||
HCType::iterator &iterExisting = addRet.first;
|
||||
Hypothesis *hypoExisting = *iterExisting;
|
||||
assert(iterExisting != m_hypos.end());
|
||||
|
||||
//StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
|
||||
|
||||
// found existing hypo with same target ending.
|
||||
// keep the best 1
|
||||
if (hypo->GetTotalScore() > hypoExisting->GetTotalScore())
|
||||
{ // incoming hypo is better than the one we have
|
||||
VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
|
||||
if (m_nBestIsEnabled) {
|
||||
hypo->AddArc(hypoExisting);
|
||||
Detach(iterExisting);
|
||||
} else {
|
||||
Remove(iterExisting);
|
||||
}
|
||||
|
||||
bool added = Add(hypo, manager).second;
|
||||
if (!added)
|
||||
{
|
||||
iterExisting = m_hypos.find(hypo);
|
||||
TRACE_ERR("Offending hypo = " << **iterExisting << endl);
|
||||
abort();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{ // already storing the best hypo. discard current hypo
|
||||
VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
|
||||
if (m_nBestIsEnabled) {
|
||||
hypoExisting->AddArc(hypo);
|
||||
} else {
|
||||
Hypothesis::Delete(hypo);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (hypo->GetTotalScore() < m_bestScore + m_beamWidth) {
|
||||
// really bad score. don't bother adding hypo into collection
|
||||
manager.GetSentenceStats().AddDiscarded();
|
||||
VERBOSE(3,"discarded, too bad for stack" << std::endl);
|
||||
Hypothesis::Delete(hypo);
|
||||
return false;
|
||||
}
|
||||
|
||||
// over threshold, try to add to collection
|
||||
std::pair<HCType::iterator, bool> addRet = Add(hypo, manager);
|
||||
if (addRet.second) {
|
||||
// nothing found. add to collection
|
||||
return true;
|
||||
}
|
||||
|
||||
// equiv hypo exists, recombine with other hypo
|
||||
HCType::iterator &iterExisting = addRet.first;
|
||||
Hypothesis *hypoExisting = *iterExisting;
|
||||
assert(iterExisting != m_hypos.end());
|
||||
|
||||
//StaticData::Instance().GetSentenceStats().AddRecombination(*hypo, **iterExisting);
|
||||
|
||||
// found existing hypo with same target ending.
|
||||
// keep the best 1
|
||||
if (hypo->GetTotalScore() > hypoExisting->GetTotalScore()) {
|
||||
// incoming hypo is better than the one we have
|
||||
VERBOSE(3,"better than matching hyp " << hypoExisting->GetId() << ", recombining, ");
|
||||
if (m_nBestIsEnabled) {
|
||||
hypo->AddArc(hypoExisting);
|
||||
Detach(iterExisting);
|
||||
} else {
|
||||
Remove(iterExisting);
|
||||
}
|
||||
|
||||
bool added = Add(hypo, manager).second;
|
||||
if (!added) {
|
||||
iterExisting = m_hypos.find(hypo);
|
||||
TRACE_ERR("Offending hypo = " << **iterExisting << endl);
|
||||
abort();
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
// already storing the best hypo. discard current hypo
|
||||
VERBOSE(3,"worse than matching hyp " << hypoExisting->GetId() << ", recombining" << std::endl)
|
||||
if (m_nBestIsEnabled) {
|
||||
hypoExisting->AddArc(hypo);
|
||||
} else {
|
||||
Hypothesis::Delete(hypo);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
pair<HypothesisCollection::HCType::iterator, bool> HypothesisCollection::Add(Hypothesis *hypo, Manager &manager)
|
||||
{
|
||||
std::pair<HCType::iterator, bool> ret = m_hypos.insert(hypo);
|
||||
if (ret.second)
|
||||
{ // equiv hypo doesn't exists
|
||||
VERBOSE(3,"added hyp to stack");
|
||||
|
||||
// Update best score, if this hypothesis is new best
|
||||
if (hypo->GetTotalScore() > m_bestScore)
|
||||
{
|
||||
VERBOSE(3,", best on stack");
|
||||
m_bestScore = hypo->GetTotalScore();
|
||||
}
|
||||
|
||||
// Prune only if stack is twice as big as needed (lazy pruning)
|
||||
VERBOSE(3,", now size " << m_hypos.size());
|
||||
if (m_hypos.size() > 2*m_maxHypoStackSize-1)
|
||||
{
|
||||
PruneToSize(manager);
|
||||
}
|
||||
else {
|
||||
VERBOSE(3,std::endl);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
std::pair<HCType::iterator, bool> ret = m_hypos.insert(hypo);
|
||||
if (ret.second) {
|
||||
// equiv hypo doesn't exists
|
||||
VERBOSE(3,"added hyp to stack");
|
||||
|
||||
// Update best score, if this hypothesis is new best
|
||||
if (hypo->GetTotalScore() > m_bestScore) {
|
||||
VERBOSE(3,", best on stack");
|
||||
m_bestScore = hypo->GetTotalScore();
|
||||
}
|
||||
|
||||
// Prune only if stack is twice as big as needed (lazy pruning)
|
||||
VERBOSE(3,", now size " << m_hypos.size());
|
||||
if (m_hypos.size() > 2*m_maxHypoStackSize-1) {
|
||||
PruneToSize(manager);
|
||||
} else {
|
||||
VERBOSE(3,std::endl);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** Remove hypothesis pointed to by iterator but don't delete the object. */
|
||||
void HypothesisCollection::Detach(const HCType::iterator &iter)
|
||||
{
|
||||
m_hypos.erase(iter);
|
||||
m_hypos.erase(iter);
|
||||
}
|
||||
|
||||
void HypothesisCollection::Remove(const HCType::iterator &iter)
|
||||
{
|
||||
Hypothesis *h = *iter;
|
||||
|
||||
/*
|
||||
stringstream strme("");
|
||||
strme << h->GetOutputPhrase();
|
||||
string toFind = "the goal of gene scientists is ";
|
||||
size_t pos = toFind.find(strme.str());
|
||||
|
||||
if (pos == 0)
|
||||
{
|
||||
cerr << pos << " " << strme.str() << *h << endl;
|
||||
cerr << *this << endl;
|
||||
}
|
||||
*/
|
||||
|
||||
Detach(iter);
|
||||
Hypothesis::Delete(h);
|
||||
Hypothesis *h = *iter;
|
||||
|
||||
/*
|
||||
stringstream strme("");
|
||||
strme << h->GetOutputPhrase();
|
||||
string toFind = "the goal of gene scientists is ";
|
||||
size_t pos = toFind.find(strme.str());
|
||||
|
||||
if (pos == 0)
|
||||
{
|
||||
cerr << pos << " " << strme.str() << *h << endl;
|
||||
cerr << *this << endl;
|
||||
}
|
||||
*/
|
||||
|
||||
Detach(iter);
|
||||
Hypothesis::Delete(h);
|
||||
}
|
||||
|
||||
void HypothesisCollection::PruneToSize(Manager &manager)
|
||||
{
|
||||
if (GetSize() > m_maxHypoStackSize) // ok, if not over the limit
|
||||
{
|
||||
priority_queue<float> bestScores;
|
||||
|
||||
// push all scores to a heap
|
||||
// (but never push scores below m_bestScore+m_beamWidth)
|
||||
HCType::iterator iter = m_hypos.begin();
|
||||
float score = 0;
|
||||
while (iter != m_hypos.end())
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
score = hypo->GetTotalScore();
|
||||
if (score > m_bestScore+m_beamWidth)
|
||||
{
|
||||
bestScores.push(score);
|
||||
}
|
||||
++iter;
|
||||
}
|
||||
|
||||
// pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
|
||||
// ensure to never pop beyond heap size
|
||||
size_t minNewSizeHeapSize = m_maxHypoStackSize > bestScores.size() ? bestScores.size() : m_maxHypoStackSize;
|
||||
for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
|
||||
bestScores.pop();
|
||||
|
||||
// and remember the threshold
|
||||
float scoreThreshold = bestScores.top();
|
||||
|
||||
// delete all hypos under score threshold
|
||||
iter = m_hypos.begin();
|
||||
while (iter != m_hypos.end())
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
float score = hypo->GetTotalScore();
|
||||
if (score < scoreThreshold)
|
||||
{
|
||||
HCType::iterator iterRemove = iter++;
|
||||
Remove(iterRemove);
|
||||
manager.GetSentenceStats().AddPruning();
|
||||
}
|
||||
else
|
||||
{
|
||||
++iter;
|
||||
}
|
||||
}
|
||||
VERBOSE(3,", pruned to size " << m_hypos.size() << endl);
|
||||
|
||||
IFVERBOSE(3)
|
||||
{
|
||||
TRACE_ERR("stack now contains: ");
|
||||
for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++)
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
TRACE_ERR( hypo->GetId() << " (" << hypo->GetTotalScore() << ") ");
|
||||
}
|
||||
TRACE_ERR( endl);
|
||||
}
|
||||
|
||||
// desperation pruning
|
||||
if (m_hypos.size() > m_maxHypoStackSize * 2)
|
||||
{
|
||||
std::vector<Hypothesis*> hyposOrdered;
|
||||
|
||||
// sort hypos
|
||||
std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(hyposOrdered, hyposOrdered.end()));
|
||||
std::sort(hyposOrdered.begin(), hyposOrdered.end(), ChartHypothesisScoreOrderer());
|
||||
|
||||
//keep only |size|. delete the rest
|
||||
std::vector<Hypothesis*>::iterator iter;
|
||||
for (iter = hyposOrdered.begin() + (m_maxHypoStackSize * 2); iter != hyposOrdered.end(); ++iter)
|
||||
{
|
||||
Hypothesis *hypo = *iter;
|
||||
HCType::iterator iterFindHypo = m_hypos.find(hypo);
|
||||
assert(iterFindHypo != m_hypos.end());
|
||||
Remove(iterFindHypo);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (GetSize() > m_maxHypoStackSize) { // ok, if not over the limit
|
||||
priority_queue<float> bestScores;
|
||||
|
||||
// push all scores to a heap
|
||||
// (but never push scores below m_bestScore+m_beamWidth)
|
||||
HCType::iterator iter = m_hypos.begin();
|
||||
float score = 0;
|
||||
while (iter != m_hypos.end()) {
|
||||
Hypothesis *hypo = *iter;
|
||||
score = hypo->GetTotalScore();
|
||||
if (score > m_bestScore+m_beamWidth) {
|
||||
bestScores.push(score);
|
||||
}
|
||||
++iter;
|
||||
}
|
||||
|
||||
// pop the top newSize scores (and ignore them, these are the scores of hyps that will remain)
|
||||
// ensure to never pop beyond heap size
|
||||
size_t minNewSizeHeapSize = m_maxHypoStackSize > bestScores.size() ? bestScores.size() : m_maxHypoStackSize;
|
||||
for (size_t i = 1 ; i < minNewSizeHeapSize ; i++)
|
||||
bestScores.pop();
|
||||
|
||||
// and remember the threshold
|
||||
float scoreThreshold = bestScores.top();
|
||||
|
||||
// delete all hypos under score threshold
|
||||
iter = m_hypos.begin();
|
||||
while (iter != m_hypos.end()) {
|
||||
Hypothesis *hypo = *iter;
|
||||
float score = hypo->GetTotalScore();
|
||||
if (score < scoreThreshold) {
|
||||
HCType::iterator iterRemove = iter++;
|
||||
Remove(iterRemove);
|
||||
manager.GetSentenceStats().AddPruning();
|
||||
} else {
|
||||
++iter;
|
||||
}
|
||||
}
|
||||
VERBOSE(3,", pruned to size " << m_hypos.size() << endl);
|
||||
|
||||
IFVERBOSE(3) {
|
||||
TRACE_ERR("stack now contains: ");
|
||||
for(iter = m_hypos.begin(); iter != m_hypos.end(); iter++) {
|
||||
Hypothesis *hypo = *iter;
|
||||
TRACE_ERR( hypo->GetId() << " (" << hypo->GetTotalScore() << ") ");
|
||||
}
|
||||
TRACE_ERR( endl);
|
||||
}
|
||||
|
||||
// desperation pruning
|
||||
if (m_hypos.size() > m_maxHypoStackSize * 2) {
|
||||
std::vector<Hypothesis*> hyposOrdered;
|
||||
|
||||
// sort hypos
|
||||
std::copy(m_hypos.begin(), m_hypos.end(), std::inserter(hyposOrdered, hyposOrdered.end()));
|
||||
std::sort(hyposOrdered.begin(), hyposOrdered.end(), ChartHypothesisScoreOrderer());
|
||||
|
||||
//keep only |size|. delete the rest
|
||||
std::vector<Hypothesis*>::iterator iter;
|
||||
for (iter = hyposOrdered.begin() + (m_maxHypoStackSize * 2); iter != hyposOrdered.end(); ++iter) {
|
||||
Hypothesis *hypo = *iter;
|
||||
HCType::iterator iterFindHypo = m_hypos.find(hypo);
|
||||
assert(iterFindHypo != m_hypos.end());
|
||||
Remove(iterFindHypo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void HypothesisCollection::SortHypotheses()
|
||||
{
|
||||
assert(m_hyposOrdered.empty());
|
||||
if (!m_hypos.empty())
|
||||
{
|
||||
// done everything for this cell.
|
||||
// sort
|
||||
// put into vec
|
||||
m_hyposOrdered.reserve(m_hypos.size());
|
||||
std::copy(m_hypos.begin(), m_hypos.end(), back_inserter(m_hyposOrdered));
|
||||
std::sort(m_hyposOrdered.begin(), m_hyposOrdered.end(), ChartHypothesisScoreOrderer());
|
||||
}
|
||||
assert(m_hyposOrdered.empty());
|
||||
if (!m_hypos.empty()) {
|
||||
// done everything for this cell.
|
||||
// sort
|
||||
// put into vec
|
||||
m_hyposOrdered.reserve(m_hypos.size());
|
||||
std::copy(m_hypos.begin(), m_hypos.end(), back_inserter(m_hyposOrdered));
|
||||
std::sort(m_hyposOrdered.begin(), m_hyposOrdered.end(), ChartHypothesisScoreOrderer());
|
||||
}
|
||||
}
|
||||
|
||||
void HypothesisCollection::CleanupArcList()
|
||||
{
|
||||
HCType::iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
|
||||
{
|
||||
Hypothesis *mainHypo = *iter;
|
||||
mainHypo->CleanupArcList();
|
||||
}
|
||||
{
|
||||
HCType::iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
|
||||
Hypothesis *mainHypo = *iter;
|
||||
mainHypo->CleanupArcList();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void HypothesisCollection::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
|
||||
{
|
||||
HCType::const_iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter)
|
||||
{
|
||||
Hypothesis &mainHypo = **iter;
|
||||
outputSearchGraphStream << translationId << " " << mainHypo << endl;
|
||||
|
||||
const ArcList *arcList = mainHypo.GetArcList();
|
||||
if (arcList)
|
||||
{
|
||||
ArcList::const_iterator iterArc;
|
||||
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc)
|
||||
{
|
||||
const Hypothesis &arc = **iterArc;
|
||||
outputSearchGraphStream << translationId << " " << arc << endl;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
HCType::const_iterator iter;
|
||||
for (iter = m_hypos.begin() ; iter != m_hypos.end() ; ++iter) {
|
||||
Hypothesis &mainHypo = **iter;
|
||||
outputSearchGraphStream << translationId << " " << mainHypo << endl;
|
||||
|
||||
const ArcList *arcList = mainHypo.GetArcList();
|
||||
if (arcList) {
|
||||
ArcList::const_iterator iterArc;
|
||||
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
|
||||
const Hypothesis &arc = **iterArc;
|
||||
outputSearchGraphStream << translationId << " " << arc << endl;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const HypothesisCollection &coll)
|
||||
{
|
||||
HypoList::const_iterator iterInside;
|
||||
for (iterInside = coll.m_hyposOrdered.begin(); iterInside != coll.m_hyposOrdered.end(); ++iterInside)
|
||||
{
|
||||
const Hypothesis &hypo = **iterInside;
|
||||
out << hypo << endl;
|
||||
}
|
||||
|
||||
return out;
|
||||
{
|
||||
HypoList::const_iterator iterInside;
|
||||
for (iterInside = coll.m_hyposOrdered.begin(); iterInside != coll.m_hyposOrdered.end(); ++iterInside) {
|
||||
const Hypothesis &hypo = **iterInside;
|
||||
out << hypo << endl;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // namespace
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -27,104 +27,108 @@
|
||||
|
||||
namespace MosesChart
|
||||
{
|
||||
|
||||
|
||||
// order by descending score
|
||||
class ChartHypothesisScoreOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
|
||||
{
|
||||
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
public:
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
|
||||
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
|
||||
}
|
||||
};
|
||||
|
||||
class HypothesisRecombinationOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
|
||||
{
|
||||
// assert in same cell
|
||||
const Moses::WordsRange &rangeA = hypoA->GetCurrSourceRange()
|
||||
, &rangeB = hypoB->GetCurrSourceRange();
|
||||
assert(rangeA == rangeB);
|
||||
|
||||
/*
|
||||
int ret = Moses::Word::Compare(hypoA->GetTargetLHS(), hypoB->GetTargetLHS());
|
||||
if (ret != 0)
|
||||
return (ret < 0);
|
||||
*/
|
||||
|
||||
// shouldn't be mixing hypos with different lhs
|
||||
assert(hypoA->GetTargetLHS() == hypoB->GetTargetLHS());
|
||||
|
||||
int ret = hypoA->LMContextCompare(*hypoB);
|
||||
if (ret != 0)
|
||||
return (ret < 0);
|
||||
|
||||
return false;
|
||||
}
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
|
||||
// assert in same cell
|
||||
const Moses::WordsRange &rangeA = hypoA->GetCurrSourceRange()
|
||||
, &rangeB = hypoB->GetCurrSourceRange();
|
||||
assert(rangeA == rangeB);
|
||||
|
||||
/*
|
||||
int ret = Moses::Word::Compare(hypoA->GetTargetLHS(), hypoB->GetTargetLHS());
|
||||
if (ret != 0)
|
||||
return (ret < 0);
|
||||
*/
|
||||
|
||||
// shouldn't be mixing hypos with different lhs
|
||||
assert(hypoA->GetTargetLHS() == hypoB->GetTargetLHS());
|
||||
|
||||
int ret = hypoA->LMContextCompare(*hypoB);
|
||||
if (ret != 0)
|
||||
return (ret < 0);
|
||||
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// order by descending score
|
||||
class HypothesisScoreOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const
|
||||
{
|
||||
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
|
||||
}
|
||||
bool operator()(const Hypothesis* hypoA, const Hypothesis* hypoB) const {
|
||||
return hypoA->GetTotalScore() > hypoB->GetTotalScore();
|
||||
}
|
||||
};
|
||||
|
||||
// 1 of these for each target LHS in each cell
|
||||
class HypothesisCollection
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const HypothesisCollection&);
|
||||
friend std::ostream& operator<<(std::ostream&, const HypothesisCollection&);
|
||||
|
||||
protected:
|
||||
typedef std::set<Hypothesis*, HypothesisRecombinationOrderer> HCType;
|
||||
HCType m_hypos;
|
||||
HypoList m_hyposOrdered;
|
||||
|
||||
float m_bestScore; /**< score of the best hypothesis in collection */
|
||||
float m_beamWidth; /**< minimum score due to threashold pruning */
|
||||
size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
|
||||
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
|
||||
|
||||
/** add hypothesis to stack. Prune if necessary.
|
||||
* Returns false if equiv hypo exists in collection, otherwise returns true
|
||||
*/
|
||||
std::pair<HCType::iterator, bool> Add(Hypothesis *hypo, Manager &manager);
|
||||
typedef std::set<Hypothesis*, HypothesisRecombinationOrderer> HCType;
|
||||
HCType m_hypos;
|
||||
HypoList m_hyposOrdered;
|
||||
|
||||
float m_bestScore; /**< score of the best hypothesis in collection */
|
||||
float m_beamWidth; /**< minimum score due to threashold pruning */
|
||||
size_t m_maxHypoStackSize; /**< maximum number of hypothesis allowed in this stack */
|
||||
bool m_nBestIsEnabled; /**< flag to determine whether to keep track of old arcs */
|
||||
|
||||
/** add hypothesis to stack. Prune if necessary.
|
||||
* Returns false if equiv hypo exists in collection, otherwise returns true
|
||||
*/
|
||||
std::pair<HCType::iterator, bool> Add(Hypothesis *hypo, Manager &manager);
|
||||
|
||||
public:
|
||||
typedef HCType::iterator iterator;
|
||||
typedef HCType::const_iterator const_iterator;
|
||||
//! iterators
|
||||
const_iterator begin() const { return m_hypos.begin(); }
|
||||
const_iterator end() const { return m_hypos.end(); }
|
||||
|
||||
HypothesisCollection();
|
||||
~HypothesisCollection();
|
||||
bool AddHypothesis(Hypothesis *hypo, Manager &manager);
|
||||
typedef HCType::iterator iterator;
|
||||
typedef HCType::const_iterator const_iterator;
|
||||
//! iterators
|
||||
const_iterator begin() const {
|
||||
return m_hypos.begin();
|
||||
}
|
||||
const_iterator end() const {
|
||||
return m_hypos.end();
|
||||
}
|
||||
|
||||
//! remove hypothesis pointed to by iterator but don't delete the object
|
||||
void Detach(const HCType::iterator &iter);
|
||||
/** destroy Hypothesis pointed to by iterator (object pool version) */
|
||||
void Remove(const HCType::iterator &iter);
|
||||
|
||||
void PruneToSize(Manager &manager);
|
||||
HypothesisCollection();
|
||||
~HypothesisCollection();
|
||||
bool AddHypothesis(Hypothesis *hypo, Manager &manager);
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_hypos.size(); }
|
||||
size_t GetHypo() const
|
||||
{ return m_hypos.size(); }
|
||||
//! remove hypothesis pointed to by iterator but don't delete the object
|
||||
void Detach(const HCType::iterator &iter);
|
||||
/** destroy Hypothesis pointed to by iterator (object pool version) */
|
||||
void Remove(const HCType::iterator &iter);
|
||||
|
||||
void SortHypotheses();
|
||||
void CleanupArcList();
|
||||
void PruneToSize(Manager &manager);
|
||||
|
||||
const HypoList &GetSortedHypotheses() const
|
||||
{ return m_hyposOrdered; }
|
||||
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
size_t GetSize() const {
|
||||
return m_hypos.size();
|
||||
}
|
||||
size_t GetHypo() const {
|
||||
return m_hypos.size();
|
||||
}
|
||||
|
||||
void SortHypotheses();
|
||||
void CleanupArcList();
|
||||
|
||||
const HypoList &GetSortedHypotheses() const {
|
||||
return m_hyposOrdered;
|
||||
}
|
||||
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -34,25 +34,24 @@ using namespace Moses;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
extern bool g_debug;
|
||||
extern bool g_debug;
|
||||
}
|
||||
|
||||
namespace MosesChart
|
||||
{
|
||||
|
||||
Manager::Manager(InputType const& source, const TranslationSystem* system)
|
||||
:m_source(source)
|
||||
,m_hypoStackColl(source, *this)
|
||||
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
|
||||
,m_system(system)
|
||||
,m_start(clock())
|
||||
:m_source(source)
|
||||
,m_hypoStackColl(source, *this)
|
||||
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
|
||||
,m_system(system)
|
||||
,m_start(clock())
|
||||
{
|
||||
m_system->InitializeBeforeSentenceProcessing(source);
|
||||
m_system->InitializeBeforeSentenceProcessing(source);
|
||||
const std::vector<PhraseDictionaryFeature*> &dictionaries = m_system->GetPhraseDictionaries();
|
||||
m_ruleLookupManagers.reserve(dictionaries.size());
|
||||
for (std::vector<PhraseDictionaryFeature*>::const_iterator p = dictionaries.begin();
|
||||
p != dictionaries.end(); ++p)
|
||||
{
|
||||
p != dictionaries.end(); ++p) {
|
||||
PhraseDictionaryFeature *pdf = *p;
|
||||
const PhraseDictionary *dict = pdf->GetDictionary();
|
||||
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
|
||||
@ -62,150 +61,139 @@ Manager::Manager(InputType const& source, const TranslationSystem* system)
|
||||
|
||||
Manager::~Manager()
|
||||
{
|
||||
m_system->CleanUpAfterSentenceProcessing();
|
||||
m_system->CleanUpAfterSentenceProcessing();
|
||||
|
||||
RemoveAllInColl(m_ruleLookupManagers);
|
||||
|
||||
clock_t end = clock();
|
||||
float et = (end - m_start);
|
||||
et /= (float)CLOCKS_PER_SEC;
|
||||
VERBOSE(1, "Translation took " << et << " seconds" << endl);
|
||||
|
||||
|
||||
clock_t end = clock();
|
||||
float et = (end - m_start);
|
||||
et /= (float)CLOCKS_PER_SEC;
|
||||
VERBOSE(1, "Translation took " << et << " seconds" << endl);
|
||||
|
||||
}
|
||||
|
||||
void Manager::ProcessSentence()
|
||||
{
|
||||
VERBOSE(1,"Translating: " << m_source << endl);
|
||||
VERBOSE(1,"Translating: " << m_source << endl);
|
||||
|
||||
ResetSentenceStats(m_source);
|
||||
ResetSentenceStats(m_source);
|
||||
|
||||
VERBOSE(2,"Decoding: " << endl);
|
||||
//Hypothesis::ResetHypoCount();
|
||||
VERBOSE(2,"Decoding: " << endl);
|
||||
//Hypothesis::ResetHypoCount();
|
||||
|
||||
// MAIN LOOP
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t width = 1; width <= size; ++width)
|
||||
{
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos)
|
||||
{
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
//TRACE_ERR(" " << range << "=");
|
||||
|
||||
// create trans opt
|
||||
m_transOptColl.CreateTranslationOptionsForRange(startPos, endPos);
|
||||
//if (g_debug)
|
||||
// cerr << m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
|
||||
|
||||
// decode
|
||||
ChartCell &cell = m_hypoStackColl.Get(range);
|
||||
// MAIN LOOP
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t width = 1; width <= size; ++width) {
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
//TRACE_ERR(" " << range << "=");
|
||||
|
||||
cell.ProcessSentence(m_transOptColl.GetTranslationOptionList(range)
|
||||
,m_hypoStackColl);
|
||||
cell.PruneToSize();
|
||||
cell.CleanupArcList();
|
||||
cell.SortHypotheses();
|
||||
|
||||
//cerr << cell.GetSize();
|
||||
//cerr << cell << endl;
|
||||
//cell.OutputSizes(cerr);
|
||||
}
|
||||
}
|
||||
// create trans opt
|
||||
m_transOptColl.CreateTranslationOptionsForRange(startPos, endPos);
|
||||
//if (g_debug)
|
||||
// cerr << m_transOptColl.GetTranslationOptionList(WordsRange(startPos, endPos));
|
||||
|
||||
IFVERBOSE(1) {
|
||||
cerr << "Num of hypo = " << Hypothesis::GetHypoCount() << " --- cells:" << endl;
|
||||
|
||||
for (size_t startPos = 0; startPos < size; ++startPos)
|
||||
{
|
||||
cerr.width(3);
|
||||
cerr << startPos << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
for (size_t width = 1; width <= size; width++)
|
||||
{
|
||||
for( size_t space = 0; space < width-1; space++ )
|
||||
{
|
||||
cerr << " ";
|
||||
}
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos)
|
||||
{
|
||||
WordsRange range(startPos, startPos+width-1);
|
||||
cerr.width(3);
|
||||
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
}
|
||||
}
|
||||
// decode
|
||||
ChartCell &cell = m_hypoStackColl.Get(range);
|
||||
|
||||
cell.ProcessSentence(m_transOptColl.GetTranslationOptionList(range)
|
||||
,m_hypoStackColl);
|
||||
cell.PruneToSize();
|
||||
cell.CleanupArcList();
|
||||
cell.SortHypotheses();
|
||||
|
||||
//cerr << cell.GetSize();
|
||||
//cerr << cell << endl;
|
||||
//cell.OutputSizes(cerr);
|
||||
}
|
||||
}
|
||||
|
||||
IFVERBOSE(1) {
|
||||
cerr << "Num of hypo = " << Hypothesis::GetHypoCount() << " --- cells:" << endl;
|
||||
|
||||
for (size_t startPos = 0; startPos < size; ++startPos) {
|
||||
cerr.width(3);
|
||||
cerr << startPos << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
for (size_t width = 1; width <= size; width++) {
|
||||
for( size_t space = 0; space < width-1; space++ ) {
|
||||
cerr << " ";
|
||||
}
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
||||
WordsRange range(startPos, startPos+width-1);
|
||||
cerr.width(3);
|
||||
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Hypothesis *Manager::GetBestHypothesis() const
|
||||
{
|
||||
size_t size = m_source.GetSize();
|
||||
size_t size = m_source.GetSize();
|
||||
|
||||
if (size == 0) // empty source
|
||||
return NULL;
|
||||
else
|
||||
{
|
||||
WordsRange range(0, size-1);
|
||||
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
||||
return lastCell.GetBestHypothesis();
|
||||
}
|
||||
if (size == 0) // empty source
|
||||
return NULL;
|
||||
else {
|
||||
WordsRange range(0, size-1);
|
||||
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
||||
return lastCell.GetBestHypothesis();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::CalcNBest(size_t count, TrellisPathList &ret,bool onlyDistinct) const
|
||||
{
|
||||
size_t size = m_source.GetSize();
|
||||
if (count == 0 || size == 0)
|
||||
return;
|
||||
|
||||
TrellisPathCollection contenders;
|
||||
set<Phrase> distinctHyps;
|
||||
size_t size = m_source.GetSize();
|
||||
if (count == 0 || size == 0)
|
||||
return;
|
||||
|
||||
// add all pure paths
|
||||
WordsRange range(0, size-1);
|
||||
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
||||
const Hypothesis *hypo = lastCell.GetBestHypothesis();
|
||||
|
||||
if (hypo == NULL)
|
||||
{ // no hypothesis
|
||||
return;
|
||||
}
|
||||
TrellisPathCollection contenders;
|
||||
set<Phrase> distinctHyps;
|
||||
|
||||
MosesChart::TrellisPath *purePath = new TrellisPath(hypo);
|
||||
contenders.Add(purePath);
|
||||
// add all pure paths
|
||||
WordsRange range(0, size-1);
|
||||
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
||||
const Hypothesis *hypo = lastCell.GetBestHypothesis();
|
||||
|
||||
// factor defines stopping point for distinct n-best list if too many candidates identical
|
||||
size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
||||
if (hypo == NULL) {
|
||||
// no hypothesis
|
||||
return;
|
||||
}
|
||||
|
||||
MosesChart::TrellisPath *purePath = new TrellisPath(hypo);
|
||||
contenders.Add(purePath);
|
||||
|
||||
// factor defines stopping point for distinct n-best list if too many candidates identical
|
||||
size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
||||
if (nBestFactor < 1) nBestFactor = 1000; // 0 = unlimited
|
||||
|
||||
// MAIN loop
|
||||
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++)
|
||||
{
|
||||
// get next best from list of contenders
|
||||
TrellisPath *path = contenders.pop();
|
||||
assert(path);
|
||||
// MAIN loop
|
||||
for (size_t iteration = 0 ; (onlyDistinct ? distinctHyps.size() : ret.GetSize()) < count && contenders.GetSize() > 0 && (iteration < count * nBestFactor) ; iteration++) {
|
||||
// get next best from list of contenders
|
||||
TrellisPath *path = contenders.pop();
|
||||
assert(path);
|
||||
|
||||
// create deviations from current best
|
||||
path->CreateDeviantPaths(contenders);
|
||||
// create deviations from current best
|
||||
path->CreateDeviantPaths(contenders);
|
||||
|
||||
if(onlyDistinct)
|
||||
{
|
||||
Phrase tgtPhrase = path->GetOutputPhrase();
|
||||
if (distinctHyps.insert(tgtPhrase).second)
|
||||
if(onlyDistinct) {
|
||||
Phrase tgtPhrase = path->GetOutputPhrase();
|
||||
if (distinctHyps.insert(tgtPhrase).second)
|
||||
ret.Add(path);
|
||||
else
|
||||
delete path;
|
||||
else
|
||||
delete path;
|
||||
|
||||
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
||||
if (nBestFactor > 0)
|
||||
contenders.Prune(count * nBestFactor);
|
||||
}
|
||||
else
|
||||
{
|
||||
ret.Add(path);
|
||||
contenders.Prune(count);
|
||||
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
||||
if (nBestFactor > 0)
|
||||
contenders.Prune(count * nBestFactor);
|
||||
} else {
|
||||
ret.Add(path);
|
||||
contenders.Prune(count);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::CalcDecoderStatistics() const
|
||||
@ -214,22 +202,20 @@ void Manager::CalcDecoderStatistics() const
|
||||
|
||||
void Manager::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
|
||||
{
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t width = 1; width <= size; ++width)
|
||||
{
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos)
|
||||
{
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
TRACE_ERR(" " << range << "=");
|
||||
|
||||
|
||||
const ChartCell &cell = m_hypoStackColl.Get(range);
|
||||
cell.GetSearchGraph(translationId, outputSearchGraphStream);
|
||||
}
|
||||
}
|
||||
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t width = 1; width <= size; ++width) {
|
||||
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
||||
size_t endPos = startPos + width - 1;
|
||||
WordsRange range(startPos, endPos);
|
||||
TRACE_ERR(" " << range << "=");
|
||||
|
||||
|
||||
const ChartCell &cell = m_hypoStackColl.Get(range);
|
||||
cell.GetSearchGraph(translationId, outputSearchGraphStream);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -41,39 +41,41 @@ class TrellisPathList;
|
||||
class Manager
|
||||
{
|
||||
protected:
|
||||
Moses::InputType const& m_source; /**< source sentence to be translated */
|
||||
ChartCellCollection m_hypoStackColl;
|
||||
TranslationOptionCollection m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
|
||||
Moses::InputType const& m_source; /**< source sentence to be translated */
|
||||
ChartCellCollection m_hypoStackColl;
|
||||
TranslationOptionCollection m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
|
||||
std::auto_ptr<Moses::SentenceStats> m_sentenceStats;
|
||||
const Moses::TranslationSystem* m_system;
|
||||
clock_t m_start; /**< starting time, used for logging */
|
||||
clock_t m_start; /**< starting time, used for logging */
|
||||
std::vector<Moses::ChartRuleLookupManager*> m_ruleLookupManagers;
|
||||
|
||||
|
||||
public:
|
||||
Manager(Moses::InputType const& source, const Moses::TranslationSystem* system);
|
||||
~Manager();
|
||||
void ProcessSentence();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
void CalcNBest(size_t count, MosesChart::TrellisPathList &ret,bool onlyDistinct=0) const;
|
||||
~Manager();
|
||||
void ProcessSentence();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
void CalcNBest(size_t count, MosesChart::TrellisPathList &ret,bool onlyDistinct=0) const;
|
||||
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
void GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const;
|
||||
|
||||
const Moses::InputType& GetSource() const {return m_source;}
|
||||
const Moses::TranslationSystem* GetTranslationSystem() const {return m_system;}
|
||||
|
||||
Moses::SentenceStats& GetSentenceStats() const
|
||||
{
|
||||
const Moses::InputType& GetSource() const {
|
||||
return m_source;
|
||||
}
|
||||
const Moses::TranslationSystem* GetTranslationSystem() const {
|
||||
return m_system;
|
||||
}
|
||||
|
||||
Moses::SentenceStats& GetSentenceStats() const {
|
||||
return *m_sentenceStats;
|
||||
}
|
||||
/***
|
||||
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
|
||||
*/
|
||||
void CalcDecoderStatistics() const;
|
||||
void ResetSentenceStats(const Moses::InputType& source)
|
||||
{
|
||||
/***
|
||||
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
|
||||
*/
|
||||
void CalcDecoderStatistics() const;
|
||||
void ResetSentenceStats(const Moses::InputType& source) {
|
||||
m_sentenceStats = std::auto_ptr<Moses::SentenceStats>(new Moses::SentenceStats(source));
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -36,266 +36,252 @@ namespace MosesChart
|
||||
{
|
||||
|
||||
TranslationOptionCollection::TranslationOptionCollection(InputType const& source
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
|
||||
:m_source(source)
|
||||
,m_system(system)
|
||||
,m_decodeGraphList(system->GetDecodeGraphs())
|
||||
,m_hypoStackColl(hypoStackColl)
|
||||
,m_collection(source.GetSize())
|
||||
,m_ruleLookupManagers(ruleLookupManagers)
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
|
||||
:m_source(source)
|
||||
,m_system(system)
|
||||
,m_decodeGraphList(system->GetDecodeGraphs())
|
||||
,m_hypoStackColl(hypoStackColl)
|
||||
,m_collection(source.GetSize())
|
||||
,m_ruleLookupManagers(ruleLookupManagers)
|
||||
{
|
||||
// create 2-d vector
|
||||
size_t size = source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos)
|
||||
{
|
||||
m_collection[startPos].reserve(size-startPos);
|
||||
for (size_t endPos = startPos ; endPos < size ; ++endPos)
|
||||
{
|
||||
m_collection[startPos].push_back( ChartTranslationOptionList(WordsRange(startPos, endPos)) );
|
||||
}
|
||||
}
|
||||
// create 2-d vector
|
||||
size_t size = source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
|
||||
m_collection[startPos].reserve(size-startPos);
|
||||
for (size_t endPos = startPos ; endPos < size ; ++endPos) {
|
||||
m_collection[startPos].push_back( ChartTranslationOptionList(WordsRange(startPos, endPos)) );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TranslationOptionCollection::~TranslationOptionCollection()
|
||||
{
|
||||
RemoveAllInColl(m_unksrcs);
|
||||
RemoveAllInColl(m_cacheTargetPhrase);
|
||||
RemoveAllInColl(m_unksrcs);
|
||||
RemoveAllInColl(m_cacheTargetPhrase);
|
||||
|
||||
std::list<std::vector<Moses::WordConsumed*>* >::iterator iterOuter;
|
||||
for (iterOuter = m_cachedWordsConsumed.begin(); iterOuter != m_cachedWordsConsumed.end(); ++iterOuter)
|
||||
{
|
||||
std::vector<Moses::WordConsumed*> &inner = **iterOuter;
|
||||
RemoveAllInColl(inner);
|
||||
}
|
||||
std::list<std::vector<Moses::WordConsumed*>* >::iterator iterOuter;
|
||||
for (iterOuter = m_cachedWordsConsumed.begin(); iterOuter != m_cachedWordsConsumed.end(); ++iterOuter) {
|
||||
std::vector<Moses::WordConsumed*> &inner = **iterOuter;
|
||||
RemoveAllInColl(inner);
|
||||
}
|
||||
|
||||
RemoveAllInColl(m_cachedWordsConsumed);
|
||||
RemoveAllInColl(m_cachedWordsConsumed);
|
||||
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
size_t startPos
|
||||
, size_t endPos)
|
||||
size_t startPos
|
||||
, size_t endPos)
|
||||
{
|
||||
ChartTranslationOptionList &chartRuleColl = GetTranslationOptionList(startPos, endPos);
|
||||
const WordsRange &wordsRange = chartRuleColl.GetSourceRange();
|
||||
|
||||
assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
|
||||
{
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
assert(decodeGraph.GetSize() == 1);
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
assert(decodeGraph.GetSize() == 1);
|
||||
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
|
||||
size_t maxSpan = decodeGraph.GetMaxChartSpan();
|
||||
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan)
|
||||
{
|
||||
size_t maxSpan = decodeGraph.GetMaxChartSpan();
|
||||
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan) {
|
||||
ruleLookupManager.GetChartRuleCollection(wordsRange, true, chartRuleColl);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ProcessUnknownWord(startPos, endPos);
|
||||
ProcessUnknownWord(startPos, endPos);
|
||||
|
||||
Prune(startPos, endPos);
|
||||
Prune(startPos, endPos);
|
||||
|
||||
Sort(startPos, endPos);
|
||||
|
||||
Sort(startPos, endPos);
|
||||
|
||||
}
|
||||
|
||||
//! Force a creation of a translation option where there are none for a particular source position.
|
||||
void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t endPos)
|
||||
{
|
||||
if (startPos != endPos)
|
||||
{ // only for 1 word phrases
|
||||
return;
|
||||
}
|
||||
if (startPos != endPos) {
|
||||
// only for 1 word phrases
|
||||
return;
|
||||
}
|
||||
|
||||
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
|
||||
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
|
||||
const WordsRange &wordsRange = fullList.GetSourceRange();
|
||||
|
||||
// try to translation for coverage with no trans by expanding table limit
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
// try to translation for coverage with no trans by expanding table limit
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
|
||||
{
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
|
||||
size_t numTransOpt = fullList.GetSize();
|
||||
if (numTransOpt == 0)
|
||||
{
|
||||
size_t numTransOpt = fullList.GetSize();
|
||||
if (numTransOpt == 0) {
|
||||
ruleLookupManager.GetChartRuleCollection(wordsRange, false, fullList);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(iterRuleLookupManagers == m_ruleLookupManagers.end());
|
||||
|
||||
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
// create unknown words for 1 word coverage where we don't have any trans options
|
||||
if (fullList.GetSize() == 0 || alwaysCreateDirectTranslationOption)
|
||||
ProcessUnknownWord(startPos);
|
||||
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
// create unknown words for 1 word coverage where we don't have any trans options
|
||||
if (fullList.GetSize() == 0 || alwaysCreateDirectTranslationOption)
|
||||
ProcessUnknownWord(startPos);
|
||||
}
|
||||
|
||||
|
||||
ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos)
|
||||
{
|
||||
size_t sizeVec = m_collection[startPos].size();
|
||||
assert(endPos-startPos < sizeVec);
|
||||
return m_collection[startPos][endPos - startPos];
|
||||
size_t sizeVec = m_collection[startPos].size();
|
||||
assert(endPos-startPos < sizeVec);
|
||||
return m_collection[startPos][endPos - startPos];
|
||||
}
|
||||
const ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos) const
|
||||
{
|
||||
size_t sizeVec = m_collection[startPos].size();
|
||||
assert(endPos-startPos < sizeVec);
|
||||
return m_collection[startPos][endPos - startPos];
|
||||
size_t sizeVec = m_collection[startPos].size();
|
||||
assert(endPos-startPos < sizeVec);
|
||||
return m_collection[startPos][endPos - startPos];
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TranslationOptionCollection &coll)
|
||||
{
|
||||
std::vector< std::vector< ChartTranslationOptionList > >::const_iterator iterOuter;
|
||||
for (iterOuter = coll.m_collection.begin(); iterOuter != coll.m_collection.end(); ++iterOuter)
|
||||
{
|
||||
const std::vector< ChartTranslationOptionList > &vecInner = *iterOuter;
|
||||
std::vector< ChartTranslationOptionList >::const_iterator iterInner;
|
||||
for (iterOuter = coll.m_collection.begin(); iterOuter != coll.m_collection.end(); ++iterOuter) {
|
||||
const std::vector< ChartTranslationOptionList > &vecInner = *iterOuter;
|
||||
std::vector< ChartTranslationOptionList >::const_iterator iterInner;
|
||||
|
||||
for (iterInner = vecInner.begin(); iterInner != vecInner.end(); ++iterInner)
|
||||
{
|
||||
const ChartTranslationOptionList &list = *iterInner;
|
||||
out << list.GetSourceRange() << " = " << list.GetSize() << std::endl;
|
||||
}
|
||||
for (iterInner = vecInner.begin(); iterInner != vecInner.end(); ++iterInner) {
|
||||
const ChartTranslationOptionList &list = *iterInner;
|
||||
out << list.GetSourceRange() << " = " << list.GetSize() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return out;
|
||||
return out;
|
||||
}
|
||||
|
||||
// taken from TranslationOptionCollectionText.
|
||||
void TranslationOptionCollection::ProcessUnknownWord(size_t sourcePos)
|
||||
{
|
||||
const Word &sourceWord = m_source.GetWord(sourcePos);
|
||||
ProcessOneUnknownWord(sourceWord,sourcePos);
|
||||
const Word &sourceWord = m_source.GetWord(sourcePos);
|
||||
ProcessOneUnknownWord(sourceWord,sourcePos);
|
||||
}
|
||||
|
||||
//! special handling of ONE unknown words.
|
||||
void TranslationOptionCollection::ProcessOneUnknownWord(const Moses::Word &sourceWord, size_t sourcePos, size_t length)
|
||||
{
|
||||
// unknown word, add as trans opt
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = m_system->GetUnknownWordPenaltyProducer();
|
||||
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
||||
// unknown word, add as trans opt
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = m_system->GetUnknownWordPenaltyProducer();
|
||||
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
||||
|
||||
Moses::ChartTranslationOptionList &transOptColl = GetTranslationOptionList(sourcePos, sourcePos);
|
||||
const WordsRange &range = transOptColl.GetSourceRange();
|
||||
|
||||
size_t isDigit = 0;
|
||||
if (staticData.GetDropUnknown())
|
||||
{
|
||||
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
|
||||
const string &s = f->GetString();
|
||||
isDigit = s.find_first_of("0123456789");
|
||||
if (isDigit == string::npos)
|
||||
isDigit = 0;
|
||||
else
|
||||
isDigit = 1;
|
||||
// modify the starting bitmap
|
||||
}
|
||||
|
||||
Phrase* m_unksrc = new Phrase(Input);
|
||||
m_unksrc->AddWord() = sourceWord;
|
||||
m_unksrcs.push_back(m_unksrc);
|
||||
Moses::ChartTranslationOptionList &transOptColl = GetTranslationOptionList(sourcePos, sourcePos);
|
||||
const WordsRange &range = transOptColl.GetSourceRange();
|
||||
|
||||
TranslationOption *transOpt;
|
||||
if (! staticData.GetDropUnknown() || isDigit)
|
||||
{
|
||||
// words consumed
|
||||
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>();
|
||||
m_cachedWordsConsumed.push_back(wordsConsumed);
|
||||
|
||||
WordConsumed *wc = new WordConsumed(sourcePos, sourcePos, sourceWord, NULL);
|
||||
wordsConsumed->push_back(wc);
|
||||
assert(wordsConsumed->size());
|
||||
size_t isDigit = 0;
|
||||
if (staticData.GetDropUnknown()) {
|
||||
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
|
||||
const string &s = f->GetString();
|
||||
isDigit = s.find_first_of("0123456789");
|
||||
if (isDigit == string::npos)
|
||||
isDigit = 0;
|
||||
else
|
||||
isDigit = 1;
|
||||
// modify the starting bitmap
|
||||
}
|
||||
|
||||
// loop
|
||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
||||
UnknownLHSList::const_iterator iterLHS;
|
||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS)
|
||||
{
|
||||
const string &targetLHSStr = iterLHS->first;
|
||||
float prob = iterLHS->second;
|
||||
|
||||
// lhs
|
||||
const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
||||
Word targetLHS(true);
|
||||
Phrase* m_unksrc = new Phrase(Input);
|
||||
m_unksrc->AddWord() = sourceWord;
|
||||
m_unksrcs.push_back(m_unksrc);
|
||||
|
||||
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
||||
assert(targetLHS.GetFactor(0) != NULL);
|
||||
TranslationOption *transOpt;
|
||||
if (! staticData.GetDropUnknown() || isDigit) {
|
||||
// words consumed
|
||||
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>();
|
||||
m_cachedWordsConsumed.push_back(wordsConsumed);
|
||||
|
||||
// add to dictionary
|
||||
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
||||
WordConsumed *wc = new WordConsumed(sourcePos, sourcePos, sourceWord, NULL);
|
||||
wordsConsumed->push_back(wc);
|
||||
assert(wordsConsumed->size());
|
||||
|
||||
m_cacheTargetPhrase.push_back(targetPhrase);
|
||||
Word &targetWord = targetPhrase->AddWord();
|
||||
targetWord.CreateUnknownWord(sourceWord);
|
||||
|
||||
// scores
|
||||
vector<float> unknownScore(1, FloorScore(TransformScore(prob)));
|
||||
// loop
|
||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
||||
UnknownLHSList::const_iterator iterLHS;
|
||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
||||
const string &targetLHSStr = iterLHS->first;
|
||||
float prob = iterLHS->second;
|
||||
|
||||
//targetPhrase->SetScore();
|
||||
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore);
|
||||
targetPhrase->SetSourcePhrase(m_unksrc);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
|
||||
// chart rule
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
|
||||
, *wordsConsumed->back()
|
||||
, range);
|
||||
chartRule->CreateNonTermIndex();
|
||||
transOptColl.Add(chartRule);
|
||||
} // for (iterLHS
|
||||
}
|
||||
else
|
||||
{ // drop source word. create blank trans opt
|
||||
vector<float> unknownScore(1, FloorScore(-numeric_limits<float>::infinity()));
|
||||
// lhs
|
||||
const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
||||
Word targetLHS(true);
|
||||
|
||||
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
||||
// loop
|
||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
||||
UnknownLHSList::const_iterator iterLHS;
|
||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS)
|
||||
{
|
||||
const string &targetLHSStr = iterLHS->first;
|
||||
float prob = iterLHS->second;
|
||||
|
||||
Word targetLHS(true);
|
||||
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
||||
assert(targetLHS.GetFactor(0) != NULL);
|
||||
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
||||
assert(targetLHS.GetFactor(0) != NULL);
|
||||
|
||||
m_cacheTargetPhrase.push_back(targetPhrase);
|
||||
targetPhrase->SetSourcePhrase(m_unksrc);
|
||||
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
// add to dictionary
|
||||
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
||||
|
||||
// words consumed
|
||||
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>;
|
||||
m_cachedWordsConsumed.push_back(wordsConsumed);
|
||||
wordsConsumed->push_back(new WordConsumed(sourcePos, sourcePos, sourceWord, NULL));
|
||||
m_cacheTargetPhrase.push_back(targetPhrase);
|
||||
Word &targetWord = targetPhrase->AddWord();
|
||||
targetWord.CreateUnknownWord(sourceWord);
|
||||
|
||||
// chart rule
|
||||
assert(wordsConsumed->size());
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
|
||||
, *wordsConsumed->back()
|
||||
, range);
|
||||
chartRule->CreateNonTermIndex();
|
||||
transOptColl.Add(chartRule);
|
||||
}
|
||||
}
|
||||
// scores
|
||||
vector<float> unknownScore(1, FloorScore(TransformScore(prob)));
|
||||
|
||||
//targetPhrase->SetScore();
|
||||
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore);
|
||||
targetPhrase->SetSourcePhrase(m_unksrc);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
|
||||
// chart rule
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
|
||||
, *wordsConsumed->back()
|
||||
, range);
|
||||
chartRule->CreateNonTermIndex();
|
||||
transOptColl.Add(chartRule);
|
||||
} // for (iterLHS
|
||||
} else {
|
||||
// drop source word. create blank trans opt
|
||||
vector<float> unknownScore(1, FloorScore(-numeric_limits<float>::infinity()));
|
||||
|
||||
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
||||
// loop
|
||||
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
||||
UnknownLHSList::const_iterator iterLHS;
|
||||
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
||||
const string &targetLHSStr = iterLHS->first;
|
||||
float prob = iterLHS->second;
|
||||
|
||||
Word targetLHS(true);
|
||||
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
||||
assert(targetLHS.GetFactor(0) != NULL);
|
||||
|
||||
m_cacheTargetPhrase.push_back(targetPhrase);
|
||||
targetPhrase->SetSourcePhrase(m_unksrc);
|
||||
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
|
||||
// words consumed
|
||||
std::vector<WordConsumed*> *wordsConsumed = new std::vector<WordConsumed*>;
|
||||
m_cachedWordsConsumed.push_back(wordsConsumed);
|
||||
wordsConsumed->push_back(new WordConsumed(sourcePos, sourcePos, sourceWord, NULL));
|
||||
|
||||
// chart rule
|
||||
assert(wordsConsumed->size());
|
||||
ChartTranslationOption *chartRule = new ChartTranslationOption(*targetPhrase
|
||||
, *wordsConsumed->back()
|
||||
, range);
|
||||
chartRule->CreateNonTermIndex();
|
||||
transOptColl.Add(chartRule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::Add(ChartTranslationOption *transOpt, size_t pos)
|
||||
{
|
||||
ChartTranslationOptionList &transOptList = GetTranslationOptionList(pos, pos);
|
||||
transOptList.Add(transOpt);
|
||||
ChartTranslationOptionList &transOptList = GetTranslationOptionList(pos, pos);
|
||||
transOptList.Add(transOpt);
|
||||
}
|
||||
|
||||
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
|
||||
@ -307,8 +293,8 @@ void TranslationOptionCollection::Prune(size_t startPos, size_t endPos)
|
||||
//! sort all trans opt in each list for cube pruning */
|
||||
void TranslationOptionCollection::Sort(size_t startPos, size_t endPos)
|
||||
{
|
||||
ChartTranslationOptionList &list = GetTranslationOptionList(startPos, endPos);
|
||||
list.Sort();
|
||||
ChartTranslationOptionList &list = GetTranslationOptionList(startPos, endPos);
|
||||
list.Sort();
|
||||
}
|
||||
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -29,11 +29,11 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class DecodeGraph;
|
||||
class Word;
|
||||
class ChartTranslationOption;
|
||||
class WordConsumed;
|
||||
class WordPenaltyProducer;
|
||||
class DecodeGraph;
|
||||
class Word;
|
||||
class ChartTranslationOption;
|
||||
class WordConsumed;
|
||||
class WordPenaltyProducer;
|
||||
};
|
||||
|
||||
namespace MosesChart
|
||||
@ -42,53 +42,52 @@ class ChartCellCollection;
|
||||
|
||||
class TranslationOptionCollection
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const TranslationOptionCollection&);
|
||||
friend std::ostream& operator<<(std::ostream&, const TranslationOptionCollection&);
|
||||
protected:
|
||||
const Moses::InputType &m_source;
|
||||
const Moses::TranslationSystem* m_system;
|
||||
const Moses::InputType &m_source;
|
||||
const Moses::TranslationSystem* m_system;
|
||||
std::vector <Moses::DecodeGraph*> m_decodeGraphList;
|
||||
const ChartCellCollection &m_hypoStackColl;
|
||||
const ChartCellCollection &m_hypoStackColl;
|
||||
const std::vector<Moses::ChartRuleLookupManager*> &m_ruleLookupManagers;
|
||||
|
||||
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
|
||||
std::vector<Moses::Phrase*> m_unksrcs;
|
||||
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
|
||||
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
|
||||
|
||||
// for adding 1 trans opt in unknown word proc
|
||||
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
|
||||
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
|
||||
std::vector<Moses::Phrase*> m_unksrcs;
|
||||
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
|
||||
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
|
||||
|
||||
Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos);
|
||||
const Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos) const;
|
||||
// for adding 1 trans opt in unknown word proc
|
||||
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
|
||||
|
||||
void ProcessUnknownWord(size_t startPos, size_t endPos);
|
||||
Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos);
|
||||
const Moses::ChartTranslationOptionList &GetTranslationOptionList(size_t startPos, size_t endPos) const;
|
||||
|
||||
// taken from TranslationOptionCollectionText.
|
||||
void ProcessUnknownWord(size_t sourcePos);
|
||||
void ProcessUnknownWord(size_t startPos, size_t endPos);
|
||||
|
||||
//! special handling of ONE unknown words.
|
||||
virtual void ProcessOneUnknownWord(const Moses::Word &sourceWord
|
||||
, size_t sourcePos, size_t length = 1);
|
||||
|
||||
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
|
||||
void Prune(size_t startPos, size_t endPos);
|
||||
// taken from TranslationOptionCollectionText.
|
||||
void ProcessUnknownWord(size_t sourcePos);
|
||||
|
||||
//! sort all trans opt in each list for cube pruning */
|
||||
void Sort(size_t startPos, size_t endPos);
|
||||
//! special handling of ONE unknown words.
|
||||
virtual void ProcessOneUnknownWord(const Moses::Word &sourceWord
|
||||
, size_t sourcePos, size_t length = 1);
|
||||
|
||||
//! pruning: only keep the top n (m_maxNoTransOptPerCoverage) elements */
|
||||
void Prune(size_t startPos, size_t endPos);
|
||||
|
||||
//! sort all trans opt in each list for cube pruning */
|
||||
void Sort(size_t startPos, size_t endPos);
|
||||
|
||||
public:
|
||||
TranslationOptionCollection(Moses::InputType const& source
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
|
||||
virtual ~TranslationOptionCollection();
|
||||
void CreateTranslationOptionsForRange(size_t startPos
|
||||
, size_t endPos);
|
||||
TranslationOptionCollection(Moses::InputType const& source
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
|
||||
virtual ~TranslationOptionCollection();
|
||||
void CreateTranslationOptionsForRange(size_t startPos
|
||||
, size_t endPos);
|
||||
|
||||
const Moses::ChartTranslationOptionList &GetTranslationOptionList(const Moses::WordsRange &range) const
|
||||
{
|
||||
return GetTranslationOptionList(range.GetStartPos(), range.GetEndPos());
|
||||
}
|
||||
const Moses::ChartTranslationOptionList &GetTranslationOptionList(const Moses::WordsRange &range) const {
|
||||
return GetTranslationOptionList(range.GetStartPos(), range.GetEndPos());
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -29,112 +29,104 @@ namespace MosesChart
|
||||
{
|
||||
|
||||
TrellisNode::TrellisNode(const Hypothesis *hypo)
|
||||
:m_hypo(hypo)
|
||||
:m_hypo(hypo)
|
||||
{
|
||||
const std::vector<const Hypothesis*> &prevHypos = hypo->GetPrevHypos();
|
||||
const std::vector<const Hypothesis*> &prevHypos = hypo->GetPrevHypos();
|
||||
|
||||
m_edge.reserve(prevHypos.size());
|
||||
for (size_t ind = 0; ind < prevHypos.size(); ++ind)
|
||||
{
|
||||
const Hypothesis *prevHypo = prevHypos[ind];
|
||||
TrellisNode *child = new TrellisNode(prevHypo);
|
||||
m_edge.push_back(child);
|
||||
}
|
||||
m_edge.reserve(prevHypos.size());
|
||||
for (size_t ind = 0; ind < prevHypos.size(); ++ind) {
|
||||
const Hypothesis *prevHypo = prevHypos[ind];
|
||||
TrellisNode *child = new TrellisNode(prevHypo);
|
||||
m_edge.push_back(child);
|
||||
}
|
||||
|
||||
assert(m_hypo);
|
||||
assert(m_hypo);
|
||||
}
|
||||
|
||||
TrellisNode::TrellisNode(const TrellisNode &origNode
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange
|
||||
, const TrellisNode *&nodeChanged)
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange
|
||||
, const TrellisNode *&nodeChanged)
|
||||
{
|
||||
if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis())
|
||||
{ // this node should be replaced
|
||||
m_hypo = &replacementHypo;
|
||||
nodeChanged = this;
|
||||
if (&origNode.GetHypothesis() == &soughtNode.GetHypothesis()) {
|
||||
// this node should be replaced
|
||||
m_hypo = &replacementHypo;
|
||||
nodeChanged = this;
|
||||
|
||||
// scores
|
||||
assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node
|
||||
// scores
|
||||
assert(scoreChange.GetWeightedScore() == 0); // should only be changing 1 node
|
||||
|
||||
scoreChange = replacementHypo.GetScoreBreakdown();
|
||||
scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());
|
||||
|
||||
float deltaScore = scoreChange.GetWeightedScore();
|
||||
assert(deltaScore <= 0.0005);
|
||||
scoreChange = replacementHypo.GetScoreBreakdown();
|
||||
scoreChange.MinusEquals(origNode.GetHypothesis().GetScoreBreakdown());
|
||||
|
||||
// follow prev hypos back to beginning
|
||||
const std::vector<const Hypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
|
||||
vector<const Hypothesis*>::const_iterator iter;
|
||||
assert(m_edge.empty());
|
||||
m_edge.reserve(prevHypos.size());
|
||||
for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter)
|
||||
{
|
||||
const Hypothesis *prevHypo = *iter;
|
||||
TrellisNode *prevNode = new TrellisNode(prevHypo);
|
||||
m_edge.push_back(prevNode);
|
||||
}
|
||||
float deltaScore = scoreChange.GetWeightedScore();
|
||||
assert(deltaScore <= 0.0005);
|
||||
|
||||
}
|
||||
else
|
||||
{ // not the node we're looking for. Copy as-is and continue finding node
|
||||
m_hypo = &origNode.GetHypothesis();
|
||||
NodeChildren::const_iterator iter;
|
||||
assert(m_edge.empty());
|
||||
m_edge.reserve(origNode.m_edge.size());
|
||||
for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter)
|
||||
{
|
||||
const TrellisNode &prevNode = **iter;
|
||||
TrellisNode *newPrevNode = new TrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
|
||||
m_edge.push_back(newPrevNode);
|
||||
}
|
||||
}
|
||||
// follow prev hypos back to beginning
|
||||
const std::vector<const Hypothesis*> &prevHypos = replacementHypo.GetPrevHypos();
|
||||
vector<const Hypothesis*>::const_iterator iter;
|
||||
assert(m_edge.empty());
|
||||
m_edge.reserve(prevHypos.size());
|
||||
for (iter = prevHypos.begin(); iter != prevHypos.end(); ++iter) {
|
||||
const Hypothesis *prevHypo = *iter;
|
||||
TrellisNode *prevNode = new TrellisNode(prevHypo);
|
||||
m_edge.push_back(prevNode);
|
||||
}
|
||||
|
||||
assert(m_hypo);
|
||||
} else {
|
||||
// not the node we're looking for. Copy as-is and continue finding node
|
||||
m_hypo = &origNode.GetHypothesis();
|
||||
NodeChildren::const_iterator iter;
|
||||
assert(m_edge.empty());
|
||||
m_edge.reserve(origNode.m_edge.size());
|
||||
for (iter = origNode.m_edge.begin(); iter != origNode.m_edge.end(); ++iter) {
|
||||
const TrellisNode &prevNode = **iter;
|
||||
TrellisNode *newPrevNode = new TrellisNode(prevNode, soughtNode, replacementHypo, scoreChange, nodeChanged);
|
||||
m_edge.push_back(newPrevNode);
|
||||
}
|
||||
}
|
||||
|
||||
assert(m_hypo);
|
||||
}
|
||||
|
||||
TrellisNode::~TrellisNode()
|
||||
{
|
||||
Moses::RemoveAllInColl(m_edge);
|
||||
Moses::RemoveAllInColl(m_edge);
|
||||
}
|
||||
|
||||
Moses::Phrase TrellisNode::GetOutputPhrase() const
|
||||
{
|
||||
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
|
||||
Moses::Phrase ret(Moses::Output);
|
||||
// exactly like same fn in hypothesis, but use trellis nodes instead of prevHypos pointer
|
||||
Moses::Phrase ret(Moses::Output);
|
||||
|
||||
const Moses::Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
|
||||
for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos)
|
||||
{
|
||||
const Moses::Word &word = currTargetPhrase.GetWord(pos);
|
||||
if (word.IsNonTerminal())
|
||||
{ // non-term. fill out with prev hypo
|
||||
size_t nonTermInd = m_hypo->GetWordsConsumedTargetOrder(pos);
|
||||
const TrellisNode &childNode = GetChild(nonTermInd);
|
||||
Moses::Phrase childPhrase = childNode.GetOutputPhrase();
|
||||
ret.Append(childPhrase);
|
||||
}
|
||||
else
|
||||
{
|
||||
ret.AddWord(word);
|
||||
}
|
||||
}
|
||||
const Moses::Phrase &currTargetPhrase = m_hypo->GetCurrTargetPhrase();
|
||||
for (size_t pos = 0; pos < currTargetPhrase.GetSize(); ++pos) {
|
||||
const Moses::Word &word = currTargetPhrase.GetWord(pos);
|
||||
if (word.IsNonTerminal()) {
|
||||
// non-term. fill out with prev hypo
|
||||
size_t nonTermInd = m_hypo->GetWordsConsumedTargetOrder(pos);
|
||||
const TrellisNode &childNode = GetChild(nonTermInd);
|
||||
Moses::Phrase childPhrase = childNode.GetOutputPhrase();
|
||||
ret.Append(childPhrase);
|
||||
} else {
|
||||
ret.AddWord(word);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TrellisNode &node)
|
||||
{
|
||||
out << "* " << node.GetHypothesis() << endl;
|
||||
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter)
|
||||
{
|
||||
out << **iter;
|
||||
}
|
||||
out << "* " << node.GetHypothesis() << endl;
|
||||
|
||||
return out;
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = node.GetChildren().begin(); iter != node.GetChildren().end(); ++iter) {
|
||||
out << **iter;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -26,7 +26,7 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ScoreComponentCollection;
|
||||
class ScoreComponentCollection;
|
||||
}
|
||||
|
||||
namespace MosesChart
|
||||
@ -36,35 +36,36 @@ class Hypothesis;
|
||||
|
||||
class TrellisNode
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const TrellisNode&);
|
||||
friend std::ostream& operator<<(std::ostream&, const TrellisNode&);
|
||||
public:
|
||||
typedef std::vector<TrellisNode*> NodeChildren;
|
||||
typedef std::vector<TrellisNode*> NodeChildren;
|
||||
|
||||
protected:
|
||||
const Hypothesis *m_hypo;
|
||||
NodeChildren m_edge;
|
||||
const Hypothesis *m_hypo;
|
||||
NodeChildren m_edge;
|
||||
|
||||
public:
|
||||
TrellisNode(const Hypothesis *hypo);
|
||||
TrellisNode(const TrellisNode &origNode
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange
|
||||
, const TrellisNode *&nodeChanged);
|
||||
~TrellisNode();
|
||||
TrellisNode(const Hypothesis *hypo);
|
||||
TrellisNode(const TrellisNode &origNode
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange
|
||||
, const TrellisNode *&nodeChanged);
|
||||
~TrellisNode();
|
||||
|
||||
const Hypothesis &GetHypothesis() const
|
||||
{
|
||||
return *m_hypo;
|
||||
}
|
||||
|
||||
const NodeChildren &GetChildren() const
|
||||
{ return m_edge; }
|
||||
const Hypothesis &GetHypothesis() const {
|
||||
return *m_hypo;
|
||||
}
|
||||
|
||||
const TrellisNode &GetChild(size_t ind) const
|
||||
{ return *m_edge[ind]; }
|
||||
const NodeChildren &GetChildren() const {
|
||||
return m_edge;
|
||||
}
|
||||
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
const TrellisNode &GetChild(size_t ind) const {
|
||||
return *m_edge[ind];
|
||||
}
|
||||
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -31,103 +31,97 @@ namespace MosesChart
|
||||
{
|
||||
|
||||
TrellisPath::TrellisPath(const Hypothesis *hypo)
|
||||
:m_finalNode(new TrellisNode(hypo))
|
||||
,m_scoreBreakdown(hypo->GetScoreBreakdown())
|
||||
,m_totalScore(hypo->GetTotalScore())
|
||||
,m_prevNodeChanged(NULL)
|
||||
,m_prevPath(NULL)
|
||||
:m_finalNode(new TrellisNode(hypo))
|
||||
,m_scoreBreakdown(hypo->GetScoreBreakdown())
|
||||
,m_totalScore(hypo->GetTotalScore())
|
||||
,m_prevNodeChanged(NULL)
|
||||
,m_prevPath(NULL)
|
||||
{
|
||||
}
|
||||
|
||||
TrellisPath::TrellisPath(const TrellisPath &origPath
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange)
|
||||
:m_scoreBreakdown(origPath.GetScoreBreakdown())
|
||||
,m_prevPath(&origPath)
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange)
|
||||
:m_scoreBreakdown(origPath.GetScoreBreakdown())
|
||||
,m_prevPath(&origPath)
|
||||
{
|
||||
m_finalNode = new TrellisNode(origPath.GetFinalNode()
|
||||
, soughtNode
|
||||
, replacementHypo
|
||||
, scoreChange
|
||||
, m_prevNodeChanged);
|
||||
m_finalNode = new TrellisNode(origPath.GetFinalNode()
|
||||
, soughtNode
|
||||
, replacementHypo
|
||||
, scoreChange
|
||||
, m_prevNodeChanged);
|
||||
|
||||
m_scoreBreakdown.PlusEquals(scoreChange);
|
||||
m_scoreBreakdown.PlusEquals(scoreChange);
|
||||
|
||||
m_totalScore = m_scoreBreakdown.GetWeightedScore();
|
||||
m_totalScore = m_scoreBreakdown.GetWeightedScore();
|
||||
}
|
||||
|
||||
TrellisPath::~TrellisPath()
|
||||
{
|
||||
delete m_finalNode;
|
||||
delete m_finalNode;
|
||||
}
|
||||
|
||||
Moses::Phrase TrellisPath::GetOutputPhrase() const
|
||||
{
|
||||
Moses::Phrase ret = GetFinalNode().GetOutputPhrase();
|
||||
return ret;
|
||||
Moses::Phrase ret = GetFinalNode().GetOutputPhrase();
|
||||
return ret;
|
||||
}
|
||||
|
||||
void TrellisPath::CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const
|
||||
{
|
||||
// copy this path but replace startHypo with its arc
|
||||
const ArcList *arcList = soughtNode.GetHypothesis().GetArcList();
|
||||
// copy this path but replace startHypo with its arc
|
||||
const ArcList *arcList = soughtNode.GetHypothesis().GetArcList();
|
||||
|
||||
if (arcList)
|
||||
{
|
||||
ArcList::const_iterator iterArcList;
|
||||
for (iterArcList = arcList->begin(); iterArcList != arcList->end(); ++iterArcList)
|
||||
{
|
||||
Moses::ScoreComponentCollection scoreChange;
|
||||
if (arcList) {
|
||||
ArcList::const_iterator iterArcList;
|
||||
for (iterArcList = arcList->begin(); iterArcList != arcList->end(); ++iterArcList) {
|
||||
Moses::ScoreComponentCollection scoreChange;
|
||||
|
||||
const Hypothesis &replacementHypo = **iterArcList;
|
||||
TrellisPath *newPath = new TrellisPath(*this, soughtNode, replacementHypo, scoreChange);
|
||||
pathColl.Add(newPath);
|
||||
}
|
||||
}
|
||||
const Hypothesis &replacementHypo = **iterArcList;
|
||||
TrellisPath *newPath = new TrellisPath(*this, soughtNode, replacementHypo, scoreChange);
|
||||
pathColl.Add(newPath);
|
||||
}
|
||||
}
|
||||
|
||||
// recusively create deviant paths for child nodes
|
||||
const TrellisNode::NodeChildren &children = soughtNode.GetChildren();
|
||||
// recusively create deviant paths for child nodes
|
||||
const TrellisNode::NodeChildren &children = soughtNode.GetChildren();
|
||||
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = children.begin(); iter != children.end(); ++iter)
|
||||
{
|
||||
const TrellisNode &child = **iter;
|
||||
CreateDeviantPaths(pathColl, child);
|
||||
}
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = children.begin(); iter != children.end(); ++iter) {
|
||||
const TrellisNode &child = **iter;
|
||||
CreateDeviantPaths(pathColl, child);
|
||||
}
|
||||
}
|
||||
|
||||
void TrellisPath::CreateDeviantPaths(TrellisPathCollection &pathColl) const
|
||||
{
|
||||
if (m_prevNodeChanged == NULL)
|
||||
{ // initial enumeration from a pure hypo
|
||||
CreateDeviantPaths(pathColl, GetFinalNode());
|
||||
}
|
||||
else
|
||||
{ // don't change m_prevNodeChanged, just it's children
|
||||
const TrellisNode::NodeChildren &children = m_prevNodeChanged->GetChildren();
|
||||
if (m_prevNodeChanged == NULL) {
|
||||
// initial enumeration from a pure hypo
|
||||
CreateDeviantPaths(pathColl, GetFinalNode());
|
||||
} else {
|
||||
// don't change m_prevNodeChanged, just it's children
|
||||
const TrellisNode::NodeChildren &children = m_prevNodeChanged->GetChildren();
|
||||
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = children.begin(); iter != children.end(); ++iter)
|
||||
{
|
||||
const TrellisNode &child = **iter;
|
||||
CreateDeviantPaths(pathColl, child);
|
||||
}
|
||||
}
|
||||
TrellisNode::NodeChildren::const_iterator iter;
|
||||
for (iter = children.begin(); iter != children.end(); ++iter) {
|
||||
const TrellisNode &child = **iter;
|
||||
CreateDeviantPaths(pathColl, child);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TrellisPath &path)
|
||||
{
|
||||
out << &path << " " << path.m_prevPath << " " << path.GetOutputPhrase() << endl;
|
||||
out << &path << " " << path.m_prevPath << " " << path.GetOutputPhrase() << endl;
|
||||
|
||||
if (path.m_prevNodeChanged)
|
||||
{
|
||||
out << "changed " << path.m_prevNodeChanged->GetHypothesis() << endl;
|
||||
}
|
||||
if (path.m_prevNodeChanged) {
|
||||
out << "changed " << path.m_prevNodeChanged->GetHypothesis() << endl;
|
||||
}
|
||||
|
||||
out << path.GetFinalNode() << endl;
|
||||
out << path.GetFinalNode() << endl;
|
||||
|
||||
return out;
|
||||
return out;
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -32,48 +32,47 @@ class TrellisPathCollection;
|
||||
|
||||
class TrellisPath
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const TrellisPath&);
|
||||
friend std::ostream& operator<<(std::ostream&, const TrellisPath&);
|
||||
|
||||
protected:
|
||||
// recursively point backwards
|
||||
TrellisNode *m_finalNode;
|
||||
const TrellisNode *m_prevNodeChanged;
|
||||
const TrellisPath *m_prevPath;
|
||||
// recursively point backwards
|
||||
TrellisNode *m_finalNode;
|
||||
const TrellisNode *m_prevNodeChanged;
|
||||
const TrellisPath *m_prevPath;
|
||||
|
||||
Moses::ScoreComponentCollection m_scoreBreakdown;
|
||||
float m_totalScore;
|
||||
Moses::ScoreComponentCollection m_scoreBreakdown;
|
||||
float m_totalScore;
|
||||
|
||||
// deviate by 1 hypo
|
||||
TrellisPath(const TrellisPath &origPath
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange);
|
||||
// deviate by 1 hypo
|
||||
TrellisPath(const TrellisPath &origPath
|
||||
, const TrellisNode &soughtNode
|
||||
, const Hypothesis &replacementHypo
|
||||
, Moses::ScoreComponentCollection &scoreChange);
|
||||
|
||||
void CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const;
|
||||
void CreateDeviantPaths(TrellisPathCollection &pathColl, const TrellisNode &soughtNode) const;
|
||||
|
||||
const TrellisNode &GetFinalNode() const
|
||||
{
|
||||
assert (m_finalNode);
|
||||
return *m_finalNode;
|
||||
}
|
||||
const TrellisNode &GetFinalNode() const {
|
||||
assert (m_finalNode);
|
||||
return *m_finalNode;
|
||||
}
|
||||
|
||||
public:
|
||||
TrellisPath(const Hypothesis *hypo);
|
||||
~TrellisPath();
|
||||
TrellisPath(const Hypothesis *hypo);
|
||||
~TrellisPath();
|
||||
|
||||
//! get score for this path throught trellis
|
||||
inline float GetTotalScore() const
|
||||
{ return m_totalScore; }
|
||||
//! get score for this path throught trellis
|
||||
inline float GetTotalScore() const {
|
||||
return m_totalScore;
|
||||
}
|
||||
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
Moses::Phrase GetOutputPhrase() const;
|
||||
|
||||
/** returns detailed component scores */
|
||||
inline const Moses::ScoreComponentCollection &GetScoreBreakdown() const
|
||||
{
|
||||
return m_scoreBreakdown;
|
||||
}
|
||||
/** returns detailed component scores */
|
||||
inline const Moses::ScoreComponentCollection &GetScoreBreakdown() const {
|
||||
return m_scoreBreakdown;
|
||||
}
|
||||
|
||||
void CreateDeviantPaths(TrellisPathCollection &pathColl) const;
|
||||
void CreateDeviantPaths(TrellisPathCollection &pathColl) const;
|
||||
};
|
||||
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -27,39 +27,38 @@ namespace MosesChart
|
||||
|
||||
TrellisPathCollection::~TrellisPathCollection()
|
||||
{
|
||||
// clean up
|
||||
Moses::RemoveAllInColl(m_collection);
|
||||
// clean up
|
||||
Moses::RemoveAllInColl(m_collection);
|
||||
}
|
||||
|
||||
void TrellisPathCollection::Add(TrellisPath *path)
|
||||
{
|
||||
m_collection.insert(path);
|
||||
m_collection.insert(path);
|
||||
}
|
||||
|
||||
void TrellisPathCollection::Prune(size_t newSize)
|
||||
{
|
||||
size_t currSize = m_collection.size();
|
||||
size_t currSize = m_collection.size();
|
||||
|
||||
if (currSize <= newSize)
|
||||
return; // don't need to prune
|
||||
if (currSize <= newSize)
|
||||
return; // don't need to prune
|
||||
|
||||
CollectionType::reverse_iterator iterRev;
|
||||
for (iterRev = m_collection.rbegin() ; iterRev != m_collection.rend() ; ++iterRev)
|
||||
{
|
||||
TrellisPath *trellisPath = *iterRev;
|
||||
delete trellisPath;
|
||||
|
||||
currSize--;
|
||||
if (currSize == newSize)
|
||||
break;
|
||||
}
|
||||
|
||||
// delete path in m_collection
|
||||
CollectionType::iterator iter = m_collection.begin();
|
||||
for (size_t i = 0 ; i < newSize ; ++i)
|
||||
iter++;
|
||||
|
||||
m_collection.erase(iter, m_collection.end());
|
||||
CollectionType::reverse_iterator iterRev;
|
||||
for (iterRev = m_collection.rbegin() ; iterRev != m_collection.rend() ; ++iterRev) {
|
||||
TrellisPath *trellisPath = *iterRev;
|
||||
delete trellisPath;
|
||||
|
||||
currSize--;
|
||||
if (currSize == newSize)
|
||||
break;
|
||||
}
|
||||
|
||||
// delete path in m_collection
|
||||
CollectionType::iterator iter = m_collection.begin();
|
||||
for (size_t i = 0 ; i < newSize ; ++i)
|
||||
iter++;
|
||||
|
||||
m_collection.erase(iter, m_collection.end());
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -28,37 +28,35 @@ namespace MosesChart
|
||||
|
||||
class TrellisPath;
|
||||
|
||||
struct CompareTrellisPathCollection
|
||||
{
|
||||
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const
|
||||
{
|
||||
return (pathA->GetTotalScore() > pathB->GetTotalScore());
|
||||
}
|
||||
struct CompareTrellisPathCollection {
|
||||
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const {
|
||||
return (pathA->GetTotalScore() > pathB->GetTotalScore());
|
||||
}
|
||||
};
|
||||
|
||||
class TrellisPathCollection
|
||||
{
|
||||
protected:
|
||||
typedef std::multiset<TrellisPath*, CompareTrellisPathCollection> CollectionType;
|
||||
CollectionType m_collection;
|
||||
typedef std::multiset<TrellisPath*, CompareTrellisPathCollection> CollectionType;
|
||||
CollectionType m_collection;
|
||||
|
||||
public:
|
||||
~TrellisPathCollection();
|
||||
~TrellisPathCollection();
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_collection.size(); }
|
||||
size_t GetSize() const {
|
||||
return m_collection.size();
|
||||
}
|
||||
|
||||
void Add(TrellisPath *path);
|
||||
void Prune(size_t newSize);
|
||||
void Add(TrellisPath *path);
|
||||
void Prune(size_t newSize);
|
||||
|
||||
TrellisPath *pop()
|
||||
{
|
||||
TrellisPath *top = *m_collection.begin();
|
||||
TrellisPath *pop() {
|
||||
TrellisPath *top = *m_collection.begin();
|
||||
|
||||
// Detach
|
||||
m_collection.erase(m_collection.begin());
|
||||
return top;
|
||||
}
|
||||
// Detach
|
||||
m_collection.erase(m_collection.begin());
|
||||
return top;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -29,8 +29,8 @@ namespace MosesChart
|
||||
{
|
||||
TrellisPathList::~TrellisPathList()
|
||||
{
|
||||
// clean up
|
||||
Moses::RemoveAllInColl(m_collection);
|
||||
// clean up
|
||||
Moses::RemoveAllInColl(m_collection);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -30,30 +30,40 @@ class TrellisPath;
|
||||
class TrellisPathList
|
||||
{
|
||||
protected:
|
||||
typedef std::vector<const TrellisPath*> CollType;
|
||||
CollType m_collection;
|
||||
typedef std::vector<const TrellisPath*> CollType;
|
||||
CollType m_collection;
|
||||
|
||||
public:
|
||||
// iters
|
||||
typedef CollType::iterator iterator;
|
||||
typedef CollType::const_iterator const_iterator;
|
||||
|
||||
iterator begin() { return m_collection.begin(); }
|
||||
iterator end() { return m_collection.end(); }
|
||||
const_iterator begin() const { return m_collection.begin(); }
|
||||
const_iterator end() const { return m_collection.end(); }
|
||||
void clear() { m_collection.clear(); }
|
||||
// iters
|
||||
typedef CollType::iterator iterator;
|
||||
typedef CollType::const_iterator const_iterator;
|
||||
|
||||
virtual ~TrellisPathList();
|
||||
iterator begin() {
|
||||
return m_collection.begin();
|
||||
}
|
||||
iterator end() {
|
||||
return m_collection.end();
|
||||
}
|
||||
const_iterator begin() const {
|
||||
return m_collection.begin();
|
||||
}
|
||||
const_iterator end() const {
|
||||
return m_collection.end();
|
||||
}
|
||||
void clear() {
|
||||
m_collection.clear();
|
||||
}
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_collection.size(); }
|
||||
virtual ~TrellisPathList();
|
||||
|
||||
//! add a new entry into collection
|
||||
void Add(TrellisPath *trellisPath)
|
||||
{
|
||||
m_collection.push_back(trellisPath);
|
||||
}
|
||||
size_t GetSize() const {
|
||||
return m_collection.size();
|
||||
}
|
||||
|
||||
//! add a new entry into collection
|
||||
void Add(TrellisPath *trellisPath) {
|
||||
m_collection.push_back(trellisPath);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -29,35 +29,34 @@ namespace MosesChart
|
||||
{
|
||||
Cube::~Cube()
|
||||
{
|
||||
Moses::RemoveAllInColl(m_uniqueEntry);
|
||||
Moses::RemoveAllInColl(m_uniqueEntry);
|
||||
}
|
||||
|
||||
bool Cube::Add(QueueEntry *queueEntry)
|
||||
{
|
||||
pair<UniqueCubeEntry::iterator, bool> inserted = m_uniqueEntry.insert(queueEntry);
|
||||
|
||||
if (inserted.second)
|
||||
{ // inserted
|
||||
m_sortedByScore.push(queueEntry);
|
||||
}
|
||||
else
|
||||
{ // already there
|
||||
//cerr << "already there\n";
|
||||
delete queueEntry;
|
||||
}
|
||||
|
||||
//assert(m_uniqueEntry.size() == m_sortedByScore.size());
|
||||
|
||||
return inserted.second;
|
||||
pair<UniqueCubeEntry::iterator, bool> inserted = m_uniqueEntry.insert(queueEntry);
|
||||
|
||||
if (inserted.second) {
|
||||
// inserted
|
||||
m_sortedByScore.push(queueEntry);
|
||||
} else {
|
||||
// already there
|
||||
//cerr << "already there\n";
|
||||
delete queueEntry;
|
||||
}
|
||||
|
||||
//assert(m_uniqueEntry.size() == m_sortedByScore.size());
|
||||
|
||||
return inserted.second;
|
||||
}
|
||||
|
||||
QueueEntry *Cube::Pop()
|
||||
{
|
||||
QueueEntry *entry = m_sortedByScore.top();
|
||||
m_sortedByScore.pop();
|
||||
QueueEntry *entry = m_sortedByScore.top();
|
||||
m_sortedByScore.pop();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -36,68 +36,65 @@ namespace MosesChart
|
||||
class QueueEntryUniqueHasher
|
||||
{
|
||||
public:
|
||||
size_t operator()(const QueueEntry * p) const
|
||||
{
|
||||
size_t seed = 0;
|
||||
boost::hash_combine(seed, &(p->GetTranslationOption()));
|
||||
boost::hash_combine(seed, p->GetChildEntries());
|
||||
return seed;
|
||||
}
|
||||
size_t operator()(const QueueEntry * p) const {
|
||||
size_t seed = 0;
|
||||
boost::hash_combine(seed, &(p->GetTranslationOption()));
|
||||
boost::hash_combine(seed, p->GetChildEntries());
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
class QueueEntryUniqueEqualityPred
|
||||
{
|
||||
public:
|
||||
bool operator()(const QueueEntry * p, const QueueEntry * q) const
|
||||
{
|
||||
return ((&(p->GetTranslationOption()) == &(q->GetTranslationOption()))
|
||||
&& (p->GetChildEntries() == q->GetChildEntries()));
|
||||
}
|
||||
bool operator()(const QueueEntry * p, const QueueEntry * q) const {
|
||||
return ((&(p->GetTranslationOption()) == &(q->GetTranslationOption()))
|
||||
&& (p->GetChildEntries() == q->GetChildEntries()));
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
class QueueEntryUniqueOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const
|
||||
{
|
||||
return (*entryA) < (*entryB);
|
||||
}
|
||||
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const {
|
||||
return (*entryA) < (*entryB);
|
||||
}
|
||||
};
|
||||
|
||||
class QueueEntryScoreOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const
|
||||
{
|
||||
return (entryA->GetCombinedScore() < entryB->GetCombinedScore());
|
||||
}
|
||||
bool operator()(const QueueEntry* entryA, const QueueEntry* entryB) const {
|
||||
return (entryA->GetCombinedScore() < entryB->GetCombinedScore());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
class Cube
|
||||
{
|
||||
protected:
|
||||
protected:
|
||||
#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
|
||||
typedef boost::unordered_set<QueueEntry*,
|
||||
QueueEntryUniqueHasher,
|
||||
QueueEntryUniqueEqualityPred> UniqueCubeEntry;
|
||||
typedef boost::unordered_set<QueueEntry*,
|
||||
QueueEntryUniqueHasher,
|
||||
QueueEntryUniqueEqualityPred> UniqueCubeEntry;
|
||||
#else
|
||||
typedef std::set<QueueEntry*, QueueEntryUniqueOrderer> UniqueCubeEntry;
|
||||
typedef std::set<QueueEntry*, QueueEntryUniqueOrderer> UniqueCubeEntry;
|
||||
#endif
|
||||
UniqueCubeEntry m_uniqueEntry;
|
||||
|
||||
typedef std::priority_queue<QueueEntry*, std::vector<QueueEntry*>, QueueEntryScoreOrderer> SortedByScore;
|
||||
SortedByScore m_sortedByScore;
|
||||
|
||||
UniqueCubeEntry m_uniqueEntry;
|
||||
|
||||
typedef std::priority_queue<QueueEntry*, std::vector<QueueEntry*>, QueueEntryScoreOrderer> SortedByScore;
|
||||
SortedByScore m_sortedByScore;
|
||||
|
||||
|
||||
public:
|
||||
~Cube();
|
||||
bool IsEmpty() const
|
||||
{ return m_sortedByScore.empty(); }
|
||||
~Cube();
|
||||
bool IsEmpty() const {
|
||||
return m_sortedByScore.empty();
|
||||
}
|
||||
|
||||
QueueEntry *Pop();
|
||||
bool Add(QueueEntry *queueEntry);
|
||||
QueueEntry *Pop();
|
||||
bool Add(QueueEntry *queueEntry);
|
||||
};
|
||||
|
||||
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -40,109 +40,105 @@ namespace MosesChart
|
||||
{
|
||||
|
||||
QueueEntry::QueueEntry(const Moses::ChartTranslationOption &transOpt
|
||||
, const ChartCellCollection &allChartCells)
|
||||
:m_transOpt(transOpt)
|
||||
, const ChartCellCollection &allChartCells)
|
||||
:m_transOpt(transOpt)
|
||||
{
|
||||
const WordConsumed *wordsConsumed = &transOpt.GetLastWordConsumed();
|
||||
CreateChildEntry(wordsConsumed, allChartCells);
|
||||
CalcScore();
|
||||
const WordConsumed *wordsConsumed = &transOpt.GetLastWordConsumed();
|
||||
CreateChildEntry(wordsConsumed, allChartCells);
|
||||
CalcScore();
|
||||
}
|
||||
|
||||
void QueueEntry::CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells)
|
||||
{
|
||||
// recursvile do the 1st first
|
||||
const WordConsumed *prevWordsConsumed = wordsConsumed->GetPrevWordsConsumed();
|
||||
if (prevWordsConsumed)
|
||||
CreateChildEntry(prevWordsConsumed, allChartCells);
|
||||
// recursvile do the 1st first
|
||||
const WordConsumed *prevWordsConsumed = wordsConsumed->GetPrevWordsConsumed();
|
||||
if (prevWordsConsumed)
|
||||
CreateChildEntry(prevWordsConsumed, allChartCells);
|
||||
|
||||
if (wordsConsumed->IsNonTerminal())
|
||||
{ // non-term
|
||||
const WordsRange &childRange = wordsConsumed->GetWordsRange();
|
||||
const ChartCell &childCell = allChartCells.Get(childRange);
|
||||
const Word &headWord = wordsConsumed->GetSourceWord();
|
||||
if (wordsConsumed->IsNonTerminal()) {
|
||||
// non-term
|
||||
const WordsRange &childRange = wordsConsumed->GetWordsRange();
|
||||
const ChartCell &childCell = allChartCells.Get(childRange);
|
||||
const Word &headWord = wordsConsumed->GetSourceWord();
|
||||
|
||||
assert(!childCell.GetSortedHypotheses(headWord).empty());
|
||||
assert(!childCell.GetSortedHypotheses(headWord).empty());
|
||||
|
||||
const Moses::Word &nonTerm = wordsConsumed->GetSourceWord();
|
||||
assert(nonTerm.IsNonTerminal());
|
||||
ChildEntry childEntry(0, childCell.GetSortedHypotheses(nonTerm), nonTerm);
|
||||
m_childEntries.push_back(childEntry);
|
||||
}
|
||||
const Moses::Word &nonTerm = wordsConsumed->GetSourceWord();
|
||||
assert(nonTerm.IsNonTerminal());
|
||||
ChildEntry childEntry(0, childCell.GetSortedHypotheses(nonTerm), nonTerm);
|
||||
m_childEntries.push_back(childEntry);
|
||||
}
|
||||
}
|
||||
|
||||
QueueEntry::QueueEntry(const QueueEntry ©, size_t childEntryIncr)
|
||||
:m_transOpt(copy.m_transOpt)
|
||||
,m_childEntries(copy.m_childEntries)
|
||||
:m_transOpt(copy.m_transOpt)
|
||||
,m_childEntries(copy.m_childEntries)
|
||||
{
|
||||
ChildEntry &childEntry = m_childEntries[childEntryIncr];
|
||||
childEntry.IncrementPos();
|
||||
CalcScore();
|
||||
ChildEntry &childEntry = m_childEntries[childEntryIncr];
|
||||
childEntry.IncrementPos();
|
||||
CalcScore();
|
||||
}
|
||||
|
||||
QueueEntry::~QueueEntry()
|
||||
{
|
||||
//Moses::RemoveAllInColl(m_childEntries);
|
||||
//Moses::RemoveAllInColl(m_childEntries);
|
||||
}
|
||||
|
||||
void QueueEntry::CreateDeviants(Cube &cube) const
|
||||
{
|
||||
for (size_t ind = 0; ind < m_childEntries.size(); ind++)
|
||||
{
|
||||
const ChildEntry &childEntry = m_childEntries[ind];
|
||||
for (size_t ind = 0; ind < m_childEntries.size(); ind++) {
|
||||
const ChildEntry &childEntry = m_childEntries[ind];
|
||||
|
||||
if (childEntry.HasMoreHypo())
|
||||
{
|
||||
QueueEntry *newEntry = new QueueEntry(*this, ind);
|
||||
cube.Add(newEntry);
|
||||
}
|
||||
}
|
||||
if (childEntry.HasMoreHypo()) {
|
||||
QueueEntry *newEntry = new QueueEntry(*this, ind);
|
||||
cube.Add(newEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void QueueEntry::CalcScore()
|
||||
{
|
||||
m_combinedScore = m_transOpt.GetTotalScore();
|
||||
for (size_t ind = 0; ind < m_childEntries.size(); ind++)
|
||||
{
|
||||
const ChildEntry &childEntry = m_childEntries[ind];
|
||||
m_combinedScore = m_transOpt.GetTotalScore();
|
||||
for (size_t ind = 0; ind < m_childEntries.size(); ind++) {
|
||||
const ChildEntry &childEntry = m_childEntries[ind];
|
||||
|
||||
const Hypothesis *hypo = childEntry.GetHypothesis();
|
||||
m_combinedScore += hypo->GetTotalScore();
|
||||
}
|
||||
const Hypothesis *hypo = childEntry.GetHypothesis();
|
||||
m_combinedScore += hypo->GetTotalScore();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
bool QueueEntry::operator<(const QueueEntry &compare) const
|
||||
{
|
||||
if (&m_transOpt != &compare.m_transOpt)
|
||||
return &m_transOpt < &compare.m_transOpt;
|
||||
|
||||
bool ret = m_childEntries < compare.m_childEntries;
|
||||
return ret;
|
||||
{
|
||||
if (&m_transOpt != &compare.m_transOpt)
|
||||
return &m_transOpt < &compare.m_transOpt;
|
||||
|
||||
bool ret = m_childEntries < compare.m_childEntries;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef HAVE_BOOST
|
||||
std::size_t hash_value(const ChildEntry & entry)
|
||||
{
|
||||
boost::hash<const Hypothesis*> hasher;
|
||||
return hasher(entry.GetHypothesis());
|
||||
boost::hash<const Hypothesis*> hasher;
|
||||
return hasher(entry.GetHypothesis());
|
||||
}
|
||||
|
||||
#endif
|
||||
std::ostream& operator<<(std::ostream &out, const ChildEntry &entry)
|
||||
{
|
||||
out << *entry.GetHypothesis();
|
||||
return out;
|
||||
out << *entry.GetHypothesis();
|
||||
return out;
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const QueueEntry &entry)
|
||||
{
|
||||
out << entry.GetTranslationOption() << endl;
|
||||
std::vector<ChildEntry>::const_iterator iter;
|
||||
for (iter = entry.GetChildEntries().begin(); iter != entry.GetChildEntries().end(); ++iter)
|
||||
{
|
||||
out << *iter << endl;
|
||||
}
|
||||
return out;
|
||||
out << entry.GetTranslationOption() << endl;
|
||||
std::vector<ChildEntry>::const_iterator iter;
|
||||
for (iter = entry.GetChildEntries().begin(); iter != entry.GetChildEntries().end(); ++iter) {
|
||||
out << *iter << endl;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,17 +3,17 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
@ -32,9 +32,9 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class WordConsumed;
|
||||
class ChartTranslationOption;
|
||||
extern bool g_debug;
|
||||
class WordConsumed;
|
||||
class ChartTranslationOption;
|
||||
extern bool g_debug;
|
||||
};
|
||||
|
||||
namespace MosesChart
|
||||
@ -49,81 +49,81 @@ class Cube;
|
||||
|
||||
typedef std::vector<const Hypothesis*> HypoList;
|
||||
|
||||
// wrapper around list of hypothese for a particular non-term of a trans opt
|
||||
// wrapper around list of hypothese for a particular non-term of a trans opt
|
||||
class ChildEntry
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const ChildEntry&);
|
||||
friend std::ostream& operator<<(std::ostream&, const ChildEntry&);
|
||||
|
||||
protected:
|
||||
size_t m_pos;
|
||||
const HypoList *m_orderedHypos;
|
||||
size_t m_pos;
|
||||
const HypoList *m_orderedHypos;
|
||||
|
||||
public:
|
||||
ChildEntry(size_t pos, const HypoList &orderedHypos, const Moses::Word & /*headWord*/ )
|
||||
:m_pos(pos)
|
||||
,m_orderedHypos(&orderedHypos)
|
||||
//,m_headWord(headWord)
|
||||
{}
|
||||
:m_pos(pos)
|
||||
,m_orderedHypos(&orderedHypos)
|
||||
//,m_headWord(headWord)
|
||||
{}
|
||||
|
||||
size_t IncrementPos()
|
||||
{ return m_pos++; }
|
||||
|
||||
bool HasMoreHypo() const
|
||||
{
|
||||
return m_pos + 1 < m_orderedHypos->size();
|
||||
}
|
||||
|
||||
const Hypothesis *GetHypothesis() const
|
||||
{
|
||||
return (*m_orderedHypos)[m_pos];
|
||||
}
|
||||
|
||||
//const Moses::Word &GetHeadWord() const
|
||||
//{ return m_headWord; }
|
||||
size_t IncrementPos() {
|
||||
return m_pos++;
|
||||
}
|
||||
|
||||
//! transitive comparison used for adding objects into FactorCollection
|
||||
bool operator<(const ChildEntry &compare) const
|
||||
{
|
||||
return GetHypothesis() < compare.GetHypothesis();
|
||||
}
|
||||
bool HasMoreHypo() const {
|
||||
return m_pos + 1 < m_orderedHypos->size();
|
||||
}
|
||||
|
||||
bool operator==(const ChildEntry & compare) const
|
||||
{
|
||||
return GetHypothesis() == compare.GetHypothesis();
|
||||
}
|
||||
const Hypothesis *GetHypothesis() const {
|
||||
return (*m_orderedHypos)[m_pos];
|
||||
}
|
||||
|
||||
//const Moses::Word &GetHeadWord() const
|
||||
//{ return m_headWord; }
|
||||
|
||||
//! transitive comparison used for adding objects into FactorCollection
|
||||
bool operator<(const ChildEntry &compare) const {
|
||||
return GetHypothesis() < compare.GetHypothesis();
|
||||
}
|
||||
|
||||
bool operator==(const ChildEntry & compare) const {
|
||||
return GetHypothesis() == compare.GetHypothesis();
|
||||
}
|
||||
};
|
||||
|
||||
// entry in the cub of 1 trans opt and all the hypotheses that goes with each non term.
|
||||
class QueueEntry
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const QueueEntry&);
|
||||
friend std::ostream& operator<<(std::ostream&, const QueueEntry&);
|
||||
protected:
|
||||
const Moses::ChartTranslationOption &m_transOpt;
|
||||
std::vector<ChildEntry> m_childEntries;
|
||||
const Moses::ChartTranslationOption &m_transOpt;
|
||||
std::vector<ChildEntry> m_childEntries;
|
||||
|
||||
float m_combinedScore;
|
||||
float m_combinedScore;
|
||||
|
||||
QueueEntry(const QueueEntry ©, size_t childEntryIncr);
|
||||
void CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells);
|
||||
QueueEntry(const QueueEntry ©, size_t childEntryIncr);
|
||||
void CreateChildEntry(const Moses::WordConsumed *wordsConsumed, const ChartCellCollection &allChartCells);
|
||||
|
||||
void CalcScore();
|
||||
void CalcScore();
|
||||
|
||||
public:
|
||||
QueueEntry(const Moses::ChartTranslationOption &transOpt
|
||||
, const ChartCellCollection &allChartCells);
|
||||
~QueueEntry();
|
||||
QueueEntry(const Moses::ChartTranslationOption &transOpt
|
||||
, const ChartCellCollection &allChartCells);
|
||||
~QueueEntry();
|
||||
|
||||
const Moses::ChartTranslationOption &GetTranslationOption() const
|
||||
{ return m_transOpt; }
|
||||
const std::vector<ChildEntry> &GetChildEntries() const
|
||||
{ return m_childEntries; }
|
||||
float GetCombinedScore() const
|
||||
{ return m_combinedScore; }
|
||||
const Moses::ChartTranslationOption &GetTranslationOption() const {
|
||||
return m_transOpt;
|
||||
}
|
||||
const std::vector<ChildEntry> &GetChildEntries() const {
|
||||
return m_childEntries;
|
||||
}
|
||||
float GetCombinedScore() const {
|
||||
return m_combinedScore;
|
||||
}
|
||||
|
||||
void CreateDeviants(Cube &) const;
|
||||
void CreateDeviants(Cube &) const;
|
||||
|
||||
bool operator<(const QueueEntry &compare) const;
|
||||
|
||||
bool operator<(const QueueEntry &compare) const;
|
||||
|
||||
};
|
||||
|
||||
#ifdef HAVE_BOOST
|
||||
|
Loading…
Reference in New Issue
Block a user