2010-07-18 03:23:09 +04:00
|
|
|
// $Id$
|
2010-04-12 14:15:49 +04:00
|
|
|
// vim:tabstop=2
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2010 Hieu Hoang
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
2010-04-08 21:16:10 +04:00
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include "ChartManager.h"
|
|
|
|
#include "ChartCell.h"
|
|
|
|
#include "ChartHypothesis.h"
|
2014-04-21 13:19:00 +04:00
|
|
|
#include "ChartKBestExtractor.h"
|
2012-09-27 01:49:33 +04:00
|
|
|
#include "ChartTranslationOptions.h"
|
2014-08-08 00:20:10 +04:00
|
|
|
#include "HypergraphOutput.h"
|
2011-03-11 16:08:43 +03:00
|
|
|
#include "StaticData.h"
|
|
|
|
#include "DecodeStep.h"
|
2012-03-28 07:29:24 +04:00
|
|
|
#include "TreeInput.h"
|
2013-05-25 02:57:06 +04:00
|
|
|
#include "moses/FF/WordPenaltyProducer.h"
|
2010-04-08 21:16:10 +04:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
using namespace Moses;
|
|
|
|
|
|
|
|
namespace Moses
|
|
|
|
{
|
2013-12-11 23:04:42 +04:00
|
|
|
extern bool g_mosesDebug;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
/* constructor. Initialize everything prior to decoding a particular sentence.
|
|
|
|
* \param source the sentence to be decoded
|
|
|
|
* \param system which particular set of models to use.
|
|
|
|
*/
|
2014-10-08 22:23:26 +04:00
|
|
|
ChartManager::ChartManager(InputType const& source)
|
2011-02-24 15:36:50 +03:00
|
|
|
:m_source(source)
|
|
|
|
,m_hypoStackColl(source, *this)
|
|
|
|
,m_start(clock())
|
2011-10-06 14:31:09 +04:00
|
|
|
,m_hypothesisId(0)
|
2013-05-11 17:13:26 +04:00
|
|
|
,m_parser(source, m_hypoStackColl)
|
2013-08-09 00:29:22 +04:00
|
|
|
,m_translationOptionList(StaticData::Instance().GetRuleLimit(), source)
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2011-03-11 16:08:43 +03:00
|
|
|
ChartManager::~ChartManager()
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
clock_t end = clock();
|
|
|
|
float et = (end - m_start);
|
|
|
|
et /= (float)CLOCKS_PER_SEC;
|
|
|
|
VERBOSE(1, "Translation took " << et << " seconds" << endl);
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
//! decode the sentence. This contains the main laps. Basically, the CKY++ algorithm
|
2011-03-11 16:08:43 +03:00
|
|
|
void ChartManager::ProcessSentence()
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
VERBOSE(1,"Translating: " << m_source << endl);
|
|
|
|
|
|
|
|
ResetSentenceStats(m_source);
|
|
|
|
|
|
|
|
VERBOSE(2,"Decoding: " << endl);
|
2011-03-11 16:08:43 +03:00
|
|
|
//ChartHypothesis::ResetHypoCount();
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2012-03-28 07:29:24 +04:00
|
|
|
AddXmlChartOptions();
|
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
// MAIN LOOP
|
|
|
|
size_t size = m_source.GetSize();
|
2014-03-21 14:53:15 +04:00
|
|
|
for (int startPos = size-1; startPos >= 0; --startPos) {
|
|
|
|
for (size_t width = 1; width <= size-startPos; ++width) {
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t endPos = startPos + width - 1;
|
|
|
|
WordsRange range(startPos, endPos);
|
|
|
|
|
|
|
|
// create trans opt
|
2012-10-11 16:26:47 +04:00
|
|
|
m_translationOptionList.Clear();
|
2012-10-11 15:29:02 +04:00
|
|
|
m_parser.Create(range, m_translationOptionList);
|
2012-10-11 16:26:47 +04:00
|
|
|
m_translationOptionList.ApplyThreshold();
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2013-09-13 11:27:48 +04:00
|
|
|
const InputPath &inputPath = m_parser.GetInputPath(range);
|
2014-08-08 21:11:30 +04:00
|
|
|
m_translationOptionList.EvaluateWithSourceContext(m_source, inputPath);
|
2013-09-13 11:27:48 +04:00
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
// decode
|
|
|
|
ChartCell &cell = m_hypoStackColl.Get(range);
|
2012-09-25 20:08:38 +04:00
|
|
|
cell.ProcessSentence(m_translationOptionList, m_hypoStackColl);
|
2013-12-12 00:03:30 +04:00
|
|
|
|
2012-09-25 20:08:38 +04:00
|
|
|
m_translationOptionList.Clear();
|
2011-02-24 15:36:50 +03:00
|
|
|
cell.PruneToSize();
|
|
|
|
cell.CleanupArcList();
|
|
|
|
cell.SortHypotheses();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
IFVERBOSE(1) {
|
|
|
|
|
|
|
|
for (size_t startPos = 0; startPos < size; ++startPos) {
|
|
|
|
cerr.width(3);
|
|
|
|
cerr << startPos << " ";
|
|
|
|
}
|
|
|
|
cerr << endl;
|
|
|
|
for (size_t width = 1; width <= size; width++) {
|
|
|
|
for( size_t space = 0; space < width-1; space++ ) {
|
|
|
|
cerr << " ";
|
|
|
|
}
|
|
|
|
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
|
|
|
WordsRange range(startPos, startPos+width-1);
|
|
|
|
cerr.width(3);
|
|
|
|
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
|
|
|
|
}
|
|
|
|
cerr << endl;
|
|
|
|
}
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
/** add specific translation options and hypotheses according to the XML override translation scheme.
|
|
|
|
* Doesn't seem to do anything about walls and zones.
|
|
|
|
* @todo check walls & zones. Check that the implementation doesn't leak, xml options sometimes does if you're not careful
|
|
|
|
*/
|
2013-05-29 21:16:15 +04:00
|
|
|
void ChartManager::AddXmlChartOptions()
|
|
|
|
{
|
2014-06-05 04:39:44 +04:00
|
|
|
// const StaticData &staticData = StaticData::Instance();
|
2013-08-31 04:37:15 +04:00
|
|
|
|
2012-09-25 20:34:43 +04:00
|
|
|
const std::vector <ChartTranslationOptions*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
|
2013-05-29 21:16:15 +04:00
|
|
|
IFVERBOSE(2) {
|
|
|
|
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
|
|
|
|
}
|
2012-03-28 07:29:24 +04:00
|
|
|
if (xmlChartOptionsList.size() == 0) return;
|
|
|
|
|
2012-09-25 20:34:43 +04:00
|
|
|
for(std::vector<ChartTranslationOptions*>::const_iterator i = xmlChartOptionsList.begin();
|
2012-03-28 07:29:24 +04:00
|
|
|
i != xmlChartOptionsList.end(); ++i) {
|
2012-09-25 20:34:43 +04:00
|
|
|
ChartTranslationOptions* opt = *i;
|
2012-03-28 07:29:24 +04:00
|
|
|
|
|
|
|
const WordsRange &range = opt->GetSourceWordsRange();
|
2013-07-24 18:55:17 +04:00
|
|
|
|
2012-03-28 07:29:24 +04:00
|
|
|
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
|
|
|
|
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
|
2014-08-08 19:41:16 +04:00
|
|
|
hypo->EvaluateWhenApplied();
|
2013-07-24 18:55:17 +04:00
|
|
|
|
|
|
|
|
2012-03-28 07:29:24 +04:00
|
|
|
ChartCell &cell = m_hypoStackColl.Get(range);
|
|
|
|
cell.AddHypothesis(hypo);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
//! get best complete translation from the top chart cell.
|
2011-03-11 16:08:43 +03:00
|
|
|
const ChartHypothesis *ChartManager::GetBestHypothesis() const
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t size = m_source.GetSize();
|
|
|
|
|
|
|
|
if (size == 0) // empty source
|
|
|
|
return NULL;
|
|
|
|
else {
|
|
|
|
WordsRange range(0, size-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
|
|
|
return lastCell.GetBestHypothesis();
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2014-04-21 13:19:00 +04:00
|
|
|
/** Calculate the n-best paths through the output hypergraph.
|
|
|
|
* Return the list of paths with the variable ret
|
|
|
|
* \param n how may paths to return
|
|
|
|
* \param ret return argument
|
|
|
|
* \param onlyDistinct whether to check for distinct output sentence or not (default - don't check, just return top n-paths)
|
|
|
|
*/
|
|
|
|
void ChartManager::CalcNBest(
|
|
|
|
std::size_t n,
|
|
|
|
std::vector<boost::shared_ptr<ChartKBestExtractor::Derivation> > &nBestList,
|
|
|
|
bool onlyDistinct) const
|
|
|
|
{
|
|
|
|
nBestList.clear();
|
|
|
|
if (n == 0 || m_source.GetSize() == 0) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the list of top-level hypotheses, sorted by score.
|
|
|
|
WordsRange range(0, m_source.GetSize()-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
|
|
|
boost::scoped_ptr<const std::vector<const ChartHypothesis*> > topLevelHypos(
|
|
|
|
lastCell.GetAllSortedHypotheses());
|
|
|
|
if (!topLevelHypos) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ChartKBestExtractor extractor;
|
|
|
|
|
|
|
|
if (!onlyDistinct) {
|
|
|
|
// Return the n-best list as is, including duplicate translations.
|
|
|
|
extractor.Extract(*topLevelHypos, n, nBestList);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Determine how many derivations to extract. If the n-best list is
|
|
|
|
// restricted to distinct translations then this limit should be bigger
|
|
|
|
// than n. The n-best factor determines how much bigger the limit should be,
|
|
|
|
// with 0 being 'unlimited.' This actually sets a large-ish limit in case
|
|
|
|
// too many translations are identical.
|
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
const std::size_t nBestFactor = staticData.GetNBestFactor();
|
|
|
|
std::size_t numDerivations = (nBestFactor == 0) ? n*1000 : n*nBestFactor;
|
|
|
|
|
|
|
|
// Extract the derivations.
|
|
|
|
ChartKBestExtractor::KBestVec bigList;
|
|
|
|
bigList.reserve(numDerivations);
|
|
|
|
extractor.Extract(*topLevelHypos, numDerivations, bigList);
|
|
|
|
|
|
|
|
// Copy derivations into nBestList, skipping ones with repeated translations.
|
|
|
|
std::set<Phrase> distinct;
|
|
|
|
for (ChartKBestExtractor::KBestVec::const_iterator p = bigList.begin();
|
2014-04-21 17:46:22 +04:00
|
|
|
nBestList.size() < n && p != bigList.end(); ++p) {
|
2014-04-21 13:19:00 +04:00
|
|
|
boost::shared_ptr<ChartKBestExtractor::Derivation> derivation = *p;
|
|
|
|
Phrase translation = ChartKBestExtractor::GetOutputPhrase(*derivation);
|
|
|
|
if (distinct.insert(translation).second) {
|
|
|
|
nBestList.push_back(derivation);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-08-08 00:20:10 +04:00
|
|
|
void ChartManager::WriteSearchGraph(const ChartSearchGraphWriter& writer) const
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2014-08-08 00:20:10 +04:00
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t size = m_source.GetSize();
|
2011-06-27 04:38:43 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
// which hypotheses are reachable?
|
|
|
|
std::map<unsigned,bool> reachable;
|
|
|
|
WordsRange fullRange(0, size-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(fullRange);
|
2011-06-27 04:38:43 +04:00
|
|
|
const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
|
|
|
|
|
|
|
|
if (hypo == NULL) {
|
|
|
|
// no hypothesis
|
|
|
|
return;
|
|
|
|
}
|
2014-08-08 00:20:10 +04:00
|
|
|
size_t winners = 0;
|
|
|
|
size_t losers = 0;
|
|
|
|
|
|
|
|
FindReachableHypotheses( hypo, reachable, &winners, &losers);
|
|
|
|
writer.WriteHeader(winners, losers);
|
2011-06-27 04:38:43 +04:00
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
for (size_t width = 1; width <= size; ++width) {
|
|
|
|
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
|
|
|
size_t endPos = startPos + width - 1;
|
|
|
|
WordsRange range(startPos, endPos);
|
|
|
|
TRACE_ERR(" " << range << "=");
|
|
|
|
|
|
|
|
const ChartCell &cell = m_hypoStackColl.Get(range);
|
2014-08-08 00:20:10 +04:00
|
|
|
cell.WriteSearchGraph(writer, reachable);
|
2011-02-24 15:36:50 +03:00
|
|
|
}
|
|
|
|
}
|
2011-06-27 04:38:43 +04:00
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2014-08-08 00:20:10 +04:00
|
|
|
void ChartManager::FindReachableHypotheses(
|
|
|
|
const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable, size_t* winners, size_t* losers) const
|
2011-06-27 04:38:43 +04:00
|
|
|
{
|
2013-05-29 21:16:15 +04:00
|
|
|
// do not recurse, if already visited
|
|
|
|
if (reachable.find(hypo->GetId()) != reachable.end()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// recurse
|
|
|
|
reachable[ hypo->GetId() ] = true;
|
2014-08-08 00:20:10 +04:00
|
|
|
if (hypo->GetWinningHypothesis() == hypo) {
|
|
|
|
(*winners)++;
|
|
|
|
} else {
|
|
|
|
(*losers)++;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
const std::vector<const ChartHypothesis*> &previous = hypo->GetPrevHypos();
|
|
|
|
for(std::vector<const ChartHypothesis*>::const_iterator i = previous.begin(); i != previous.end(); ++i) {
|
2014-08-08 00:20:10 +04:00
|
|
|
FindReachableHypotheses( *i, reachable, winners, losers );
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// also loop over recombined hypotheses (arcs)
|
|
|
|
const ChartArcList *arcList = hypo->GetArcList();
|
|
|
|
if (arcList) {
|
|
|
|
ChartArcList::const_iterator iterArc;
|
|
|
|
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
|
|
|
|
const ChartHypothesis &arc = **iterArc;
|
2014-08-08 00:20:10 +04:00
|
|
|
FindReachableHypotheses( &arc, reachable, winners, losers );
|
2013-05-29 21:16:15 +04:00
|
|
|
}
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2014-08-06 19:37:09 +04:00
|
|
|
void ChartManager::OutputSearchGraphAsHypergraph(std::ostream &outputSearchGraphStream) const {
|
2014-08-08 00:20:10 +04:00
|
|
|
ChartSearchGraphWriterHypergraph writer(&outputSearchGraphStream);
|
|
|
|
WriteSearchGraph(writer);
|
|
|
|
}
|
2014-08-06 19:37:09 +04:00
|
|
|
|
2014-08-08 00:20:10 +04:00
|
|
|
void ChartManager::OutputSearchGraphMoses(std::ostream &outputSearchGraphStream) const {
|
2014-10-08 22:23:26 +04:00
|
|
|
ChartSearchGraphWriterMoses writer(&outputSearchGraphStream, m_source.GetTranslationId());
|
2014-08-08 00:20:10 +04:00
|
|
|
WriteSearchGraph(writer);
|
2014-08-06 19:37:09 +04:00
|
|
|
}
|
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
} // namespace Moses
|