2010-07-18 03:23:09 +04:00
|
|
|
// $Id$
|
2010-04-12 14:15:49 +04:00
|
|
|
// vim:tabstop=2
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2010 Hieu Hoang
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2010-04-12 14:15:49 +04:00
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
2010-04-08 21:16:10 +04:00
|
|
|
|
|
|
|
#include <stdio.h>
|
|
|
|
#include "ChartManager.h"
|
|
|
|
#include "ChartCell.h"
|
|
|
|
#include "ChartHypothesis.h"
|
2011-11-08 15:28:02 +04:00
|
|
|
#include "ChartTrellisDetourQueue.h"
|
|
|
|
#include "ChartTrellisNode.h"
|
2010-04-08 21:16:10 +04:00
|
|
|
#include "ChartTrellisPath.h"
|
|
|
|
#include "ChartTrellisPathList.h"
|
2011-03-11 16:08:43 +03:00
|
|
|
#include "StaticData.h"
|
|
|
|
#include "DecodeStep.h"
|
2012-03-28 07:29:24 +04:00
|
|
|
#include "TreeInput.h"
|
2012-09-25 20:08:38 +04:00
|
|
|
#include "DummyScoreProducers.h"
|
2010-04-08 21:16:10 +04:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
using namespace Moses;
|
|
|
|
|
|
|
|
namespace Moses
|
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
extern bool g_debug;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
/* constructor. Initialize everything prior to decoding a particular sentence.
|
|
|
|
* \param source the sentence to be decoded
|
|
|
|
* \param system which particular set of models to use.
|
|
|
|
*/
|
2011-03-11 16:08:43 +03:00
|
|
|
ChartManager::ChartManager(InputType const& source, const TranslationSystem* system)
|
2011-02-24 15:36:50 +03:00
|
|
|
:m_source(source)
|
|
|
|
,m_hypoStackColl(source, *this)
|
|
|
|
,m_system(system)
|
|
|
|
,m_start(clock())
|
2011-10-06 14:31:09 +04:00
|
|
|
,m_hypothesisId(0)
|
2012-09-25 20:08:38 +04:00
|
|
|
,m_translationOptionList(StaticData::Instance().GetRuleLimit())
|
|
|
|
,m_decodeGraphList(system->GetDecodeGraphs())
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
m_system->InitializeBeforeSentenceProcessing(source);
|
2011-01-24 22:14:19 +03:00
|
|
|
const std::vector<PhraseDictionaryFeature*> &dictionaries = m_system->GetPhraseDictionaries();
|
|
|
|
m_ruleLookupManagers.reserve(dictionaries.size());
|
|
|
|
for (std::vector<PhraseDictionaryFeature*>::const_iterator p = dictionaries.begin();
|
2011-02-24 15:36:50 +03:00
|
|
|
p != dictionaries.end(); ++p) {
|
2011-01-24 22:14:19 +03:00
|
|
|
PhraseDictionaryFeature *pdf = *p;
|
|
|
|
const PhraseDictionary *dict = pdf->GetDictionary();
|
|
|
|
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
|
|
|
|
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(source, m_hypoStackColl));
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2011-03-11 16:08:43 +03:00
|
|
|
ChartManager::~ChartManager()
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2012-07-19 19:41:41 +04:00
|
|
|
m_system->CleanUpAfterSentenceProcessing(m_source);
|
2011-01-24 22:14:19 +03:00
|
|
|
|
|
|
|
RemoveAllInColl(m_ruleLookupManagers);
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2012-09-25 20:08:38 +04:00
|
|
|
RemoveAllInColl(m_unksrcs);
|
|
|
|
RemoveAllInColl(m_cacheTargetPhraseCollection);
|
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
clock_t end = clock();
|
|
|
|
float et = (end - m_start);
|
|
|
|
et /= (float)CLOCKS_PER_SEC;
|
|
|
|
VERBOSE(1, "Translation took " << et << " seconds" << endl);
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
//! decode the sentence. This contains the main laps. Basically, the CKY++ algorithm
|
2011-03-11 16:08:43 +03:00
|
|
|
void ChartManager::ProcessSentence()
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
VERBOSE(1,"Translating: " << m_source << endl);
|
|
|
|
|
|
|
|
ResetSentenceStats(m_source);
|
|
|
|
|
|
|
|
VERBOSE(2,"Decoding: " << endl);
|
2011-03-11 16:08:43 +03:00
|
|
|
//ChartHypothesis::ResetHypoCount();
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2012-03-28 07:29:24 +04:00
|
|
|
AddXmlChartOptions();
|
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
// MAIN LOOP
|
|
|
|
size_t size = m_source.GetSize();
|
|
|
|
for (size_t width = 1; width <= size; ++width) {
|
|
|
|
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
|
|
|
size_t endPos = startPos + width - 1;
|
|
|
|
WordsRange range(startPos, endPos);
|
|
|
|
|
|
|
|
// create trans opt
|
2012-09-25 20:08:38 +04:00
|
|
|
CreateTranslationOptionsForRange(range);
|
2011-02-24 15:36:50 +03:00
|
|
|
|
|
|
|
// decode
|
|
|
|
ChartCell &cell = m_hypoStackColl.Get(range);
|
|
|
|
|
2012-09-25 20:08:38 +04:00
|
|
|
cell.ProcessSentence(m_translationOptionList, m_hypoStackColl);
|
|
|
|
m_translationOptionList.Clear();
|
2011-02-24 15:36:50 +03:00
|
|
|
cell.PruneToSize();
|
|
|
|
cell.CleanupArcList();
|
|
|
|
cell.SortHypotheses();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
IFVERBOSE(1) {
|
|
|
|
|
|
|
|
for (size_t startPos = 0; startPos < size; ++startPos) {
|
|
|
|
cerr.width(3);
|
|
|
|
cerr << startPos << " ";
|
|
|
|
}
|
|
|
|
cerr << endl;
|
|
|
|
for (size_t width = 1; width <= size; width++) {
|
|
|
|
for( size_t space = 0; space < width-1; space++ ) {
|
|
|
|
cerr << " ";
|
|
|
|
}
|
|
|
|
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
|
|
|
WordsRange range(startPos, startPos+width-1);
|
|
|
|
cerr.width(3);
|
|
|
|
cerr << m_hypoStackColl.Get(range).GetSize() << " ";
|
|
|
|
}
|
|
|
|
cerr << endl;
|
|
|
|
}
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
/** add specific translation options and hypotheses according to the XML override translation scheme.
|
|
|
|
* Doesn't seem to do anything about walls and zones.
|
|
|
|
* @todo check walls & zones. Check that the implementation doesn't leak, xml options sometimes does if you're not careful
|
|
|
|
*/
|
2012-03-28 07:29:24 +04:00
|
|
|
void ChartManager::AddXmlChartOptions() {
|
2012-05-18 14:02:51 +04:00
|
|
|
const std::vector <ChartTranslationOption*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
|
2012-03-28 07:29:24 +04:00
|
|
|
IFVERBOSE(2) { cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl; }
|
|
|
|
if (xmlChartOptionsList.size() == 0) return;
|
|
|
|
|
|
|
|
for(std::vector<ChartTranslationOption*>::const_iterator i = xmlChartOptionsList.begin();
|
|
|
|
i != xmlChartOptionsList.end(); ++i) {
|
|
|
|
ChartTranslationOption* opt = *i;
|
|
|
|
|
|
|
|
Moses::Scores wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
|
|
|
opt->GetTargetPhraseCollection().GetCollection()[0]->SetScore((ScoreProducer*)m_system->GetWordPenaltyProducer(), wordPenaltyScore);
|
|
|
|
|
|
|
|
const WordsRange &range = opt->GetSourceWordsRange();
|
|
|
|
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
|
|
|
|
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
|
|
|
|
hypo->CalcScore();
|
|
|
|
ChartCell &cell = m_hypoStackColl.Get(range);
|
|
|
|
cell.AddHypothesis(hypo);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
//! get best complete translation from the top chart cell.
|
2011-03-11 16:08:43 +03:00
|
|
|
const ChartHypothesis *ChartManager::GetBestHypothesis() const
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t size = m_source.GetSize();
|
|
|
|
|
|
|
|
if (size == 0) // empty source
|
|
|
|
return NULL;
|
|
|
|
else {
|
|
|
|
WordsRange range(0, size-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
|
|
|
return lastCell.GetBestHypothesis();
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2012-07-03 21:11:53 +04:00
|
|
|
/** Calculate the n-best paths through the output hypergraph.
|
|
|
|
* Return the list of paths with the variable ret
|
|
|
|
* \param count how may paths to return
|
|
|
|
* \param ret return argument
|
|
|
|
* \param onlyDistinct whether to check for distinct output sentence or not (default - don't check, just return top n-paths)
|
|
|
|
*/
|
2011-03-11 16:08:43 +03:00
|
|
|
void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret,bool onlyDistinct) const
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t size = m_source.GetSize();
|
|
|
|
if (count == 0 || size == 0)
|
|
|
|
return;
|
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
// Build a ChartTrellisPath for the 1-best path, if any.
|
2011-02-24 15:36:50 +03:00
|
|
|
WordsRange range(0, size-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(range);
|
2011-03-11 16:08:43 +03:00
|
|
|
const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
|
2011-02-24 15:36:50 +03:00
|
|
|
if (hypo == NULL) {
|
|
|
|
// no hypothesis
|
|
|
|
return;
|
|
|
|
}
|
2011-11-08 15:28:02 +04:00
|
|
|
boost::shared_ptr<ChartTrellisPath> basePath(new ChartTrellisPath(*hypo));
|
|
|
|
|
|
|
|
// Add it to the n-best list.
|
|
|
|
ret.Add(basePath);
|
|
|
|
if (count == 1) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Record the output phrase if distinct translations are required.
|
|
|
|
set<Phrase> distinctHyps;
|
|
|
|
if (onlyDistinct) {
|
|
|
|
distinctHyps.insert(basePath->GetOutputPhrase());
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set a limit on the number of detours to pop. If the n-best list is
|
|
|
|
// restricted to distinct translations then this limit should be bigger
|
|
|
|
// than n. The n-best factor determines how much bigger the limit should be.
|
|
|
|
const size_t nBestFactor = StaticData::Instance().GetNBestFactor();
|
|
|
|
size_t popLimit;
|
|
|
|
if (!onlyDistinct) {
|
|
|
|
popLimit = count-1;
|
|
|
|
} else if (nBestFactor == 0) {
|
|
|
|
// 0 = 'unlimited.' This actually sets a large-ish limit in case too many
|
|
|
|
// translations are identical.
|
|
|
|
popLimit = count * 1000;
|
|
|
|
} else {
|
|
|
|
popLimit = count * nBestFactor;
|
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
// Create an empty priority queue of detour objects. It is bounded to
|
|
|
|
// contain no more than popLimit items.
|
|
|
|
ChartTrellisDetourQueue contenders(popLimit);
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
// Create a ChartTrellisDetour for each single-point deviation from basePath
|
|
|
|
// and add them to the queue.
|
|
|
|
CreateDeviantPaths(basePath, contenders);
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
// MAIN loop
|
2011-11-08 15:28:02 +04:00
|
|
|
for (size_t i = 0;
|
|
|
|
ret.GetSize() < count && !contenders.Empty() && i < popLimit;
|
|
|
|
++i) {
|
|
|
|
// Get the best detour from the queue.
|
|
|
|
std::auto_ptr<const ChartTrellisDetour> detour(contenders.Pop());
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(detour.get());
|
2011-11-08 15:28:02 +04:00
|
|
|
|
|
|
|
// Create a full base path from the chosen detour.
|
|
|
|
basePath.reset(new ChartTrellisPath(*detour));
|
|
|
|
|
|
|
|
// Generate new detours from this base path and add them to the queue of
|
|
|
|
// contenders. The new detours deviate from the base path by a single
|
|
|
|
// replacement along the previous detour sub-path.
|
2011-11-18 16:07:41 +04:00
|
|
|
CHECK(basePath->GetDeviationPoint());
|
2011-11-08 15:28:02 +04:00
|
|
|
CreateDeviantPaths(basePath, *(basePath->GetDeviationPoint()), contenders);
|
|
|
|
|
|
|
|
// If the n-best list is allowed to contain duplicate translations (at the
|
|
|
|
// surface level) then add the new path unconditionally, otherwise check
|
|
|
|
// whether the translation has seen before.
|
|
|
|
if (!onlyDistinct) {
|
|
|
|
ret.Add(basePath);
|
2011-02-24 15:36:50 +03:00
|
|
|
} else {
|
2011-11-08 15:28:02 +04:00
|
|
|
Phrase tgtPhrase = basePath->GetOutputPhrase();
|
|
|
|
if (distinctHyps.insert(tgtPhrase).second) {
|
|
|
|
ret.Add(basePath);
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2011-03-11 16:08:43 +03:00
|
|
|
void ChartManager::GetSearchGraph(long translationId, std::ostream &outputSearchGraphStream) const
|
2010-04-08 21:16:10 +04:00
|
|
|
{
|
2011-02-24 15:36:50 +03:00
|
|
|
size_t size = m_source.GetSize();
|
2011-06-27 04:38:43 +04:00
|
|
|
|
|
|
|
// which hypotheses are reachable?
|
2011-10-06 14:31:09 +04:00
|
|
|
std::map<unsigned,bool> reachable;
|
2011-06-27 04:38:43 +04:00
|
|
|
WordsRange fullRange(0, size-1);
|
|
|
|
const ChartCell &lastCell = m_hypoStackColl.Get(fullRange);
|
|
|
|
const ChartHypothesis *hypo = lastCell.GetBestHypothesis();
|
|
|
|
|
|
|
|
if (hypo == NULL) {
|
|
|
|
// no hypothesis
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
FindReachableHypotheses( hypo, reachable);
|
|
|
|
|
2011-02-24 15:36:50 +03:00
|
|
|
for (size_t width = 1; width <= size; ++width) {
|
|
|
|
for (size_t startPos = 0; startPos <= size-width; ++startPos) {
|
|
|
|
size_t endPos = startPos + width - 1;
|
|
|
|
WordsRange range(startPos, endPos);
|
|
|
|
TRACE_ERR(" " << range << "=");
|
|
|
|
|
|
|
|
const ChartCell &cell = m_hypoStackColl.Get(range);
|
2011-06-27 04:38:43 +04:00
|
|
|
cell.GetSearchGraph(translationId, outputSearchGraphStream, reachable);
|
2011-02-24 15:36:50 +03:00
|
|
|
}
|
|
|
|
}
|
2011-06-27 04:38:43 +04:00
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2011-10-06 14:31:09 +04:00
|
|
|
void ChartManager::FindReachableHypotheses( const ChartHypothesis *hypo, std::map<unsigned,bool> &reachable ) const
|
2011-06-27 04:38:43 +04:00
|
|
|
{
|
|
|
|
// do not recurse, if already visited
|
|
|
|
if (reachable.find(hypo->GetId()) != reachable.end())
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// recurse
|
|
|
|
reachable[ hypo->GetId() ] = true;
|
|
|
|
const std::vector<const ChartHypothesis*> &previous = hypo->GetPrevHypos();
|
|
|
|
for(std::vector<const ChartHypothesis*>::const_iterator i = previous.begin(); i != previous.end(); ++i)
|
|
|
|
{
|
|
|
|
FindReachableHypotheses( *i, reachable );
|
|
|
|
}
|
|
|
|
|
|
|
|
// also loop over recombined hypotheses (arcs)
|
|
|
|
const ChartArcList *arcList = hypo->GetArcList();
|
|
|
|
if (arcList) {
|
|
|
|
ChartArcList::const_iterator iterArc;
|
|
|
|
for (iterArc = arcList->begin(); iterArc != arcList->end(); ++iterArc) {
|
|
|
|
const ChartHypothesis &arc = **iterArc;
|
|
|
|
FindReachableHypotheses( &arc, reachable );
|
|
|
|
}
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
2011-02-24 15:36:50 +03:00
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
void ChartManager::CreateDeviantPaths(
|
|
|
|
boost::shared_ptr<const ChartTrellisPath> basePath,
|
|
|
|
ChartTrellisDetourQueue &q)
|
|
|
|
{
|
|
|
|
CreateDeviantPaths(basePath, basePath->GetFinalNode(), q);
|
|
|
|
}
|
|
|
|
|
|
|
|
void ChartManager::CreateDeviantPaths(
|
|
|
|
boost::shared_ptr<const ChartTrellisPath> basePath,
|
|
|
|
const ChartTrellisNode &substitutedNode,
|
|
|
|
ChartTrellisDetourQueue &queue)
|
|
|
|
{
|
|
|
|
const ChartArcList *arcList = substitutedNode.GetHypothesis().GetArcList();
|
|
|
|
if (arcList) {
|
|
|
|
for (ChartArcList::const_iterator iter = arcList->begin();
|
|
|
|
iter != arcList->end(); ++iter) {
|
|
|
|
const ChartHypothesis &replacement = **iter;
|
|
|
|
queue.Push(new ChartTrellisDetour(basePath, substitutedNode,
|
|
|
|
replacement));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// recusively create deviant paths for child nodes
|
|
|
|
const ChartTrellisNode::NodeChildren &children = substitutedNode.GetChildren();
|
|
|
|
ChartTrellisNode::NodeChildren::const_iterator iter;
|
|
|
|
for (iter = children.begin(); iter != children.end(); ++iter) {
|
|
|
|
const ChartTrellisNode &child = **iter;
|
|
|
|
CreateDeviantPaths(basePath, child, queue);
|
|
|
|
}
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2012-09-25 20:08:38 +04:00
|
|
|
void ChartManager::CreateTranslationOptionsForRange(const WordsRange &wordsRange)
|
|
|
|
{
|
|
|
|
assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
|
|
|
|
|
|
|
|
m_translationOptionList.Clear();
|
|
|
|
|
|
|
|
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
|
|
|
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
|
|
|
|
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers) {
|
|
|
|
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
|
|
|
assert(decodeGraph.GetSize() == 1);
|
|
|
|
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
|
|
|
|
size_t maxSpan = decodeGraph.GetMaxChartSpan();
|
|
|
|
if (maxSpan == 0 || wordsRange.GetNumWordsCovered() <= maxSpan) {
|
|
|
|
ruleLookupManager.GetChartRuleCollection(wordsRange, m_translationOptionList);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (wordsRange.GetNumWordsCovered() == 1 && wordsRange.GetStartPos() != 0 && wordsRange.GetStartPos() != m_source.GetSize()-1) {
|
|
|
|
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
|
|
|
if (m_translationOptionList.GetSize() == 0 || alwaysCreateDirectTranslationOption) {
|
|
|
|
// create unknown words for 1 word coverage where we don't have any trans options
|
|
|
|
const Word &sourceWord = m_source.GetWord(wordsRange.GetStartPos());
|
|
|
|
ProcessOneUnknownWord(sourceWord, wordsRange);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
m_translationOptionList.ApplyThreshold();
|
|
|
|
}
|
|
|
|
|
|
|
|
//! special handling of ONE unknown words.
|
|
|
|
void ChartManager::ProcessOneUnknownWord(const Word &sourceWord, const WordsRange &range)
|
|
|
|
{
|
|
|
|
// unknown word, add as trans opt
|
|
|
|
const StaticData &staticData = StaticData::Instance();
|
|
|
|
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = m_system->GetUnknownWordPenaltyProducer();
|
|
|
|
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
|
|
|
|
|
|
|
const ChartCell &chartCell = m_hypoStackColl.Get(range);
|
|
|
|
const ChartCellLabel &sourceWordLabel = chartCell.GetSourceWordLabel();
|
|
|
|
|
|
|
|
size_t isDigit = 0;
|
|
|
|
if (staticData.GetDropUnknown()) {
|
|
|
|
const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface
|
|
|
|
const string &s = f->GetString();
|
|
|
|
isDigit = s.find_first_of("0123456789");
|
|
|
|
if (isDigit == string::npos)
|
|
|
|
isDigit = 0;
|
|
|
|
else
|
|
|
|
isDigit = 1;
|
|
|
|
// modify the starting bitmap
|
|
|
|
}
|
|
|
|
|
|
|
|
Phrase* m_unksrc = new Phrase(1);
|
|
|
|
m_unksrc->AddWord() = sourceWord;
|
|
|
|
m_unksrcs.push_back(m_unksrc);
|
|
|
|
|
|
|
|
//TranslationOption *transOpt;
|
|
|
|
if (! staticData.GetDropUnknown() || isDigit) {
|
|
|
|
// loop
|
|
|
|
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
|
|
|
UnknownLHSList::const_iterator iterLHS;
|
|
|
|
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
|
|
|
const string &targetLHSStr = iterLHS->first;
|
|
|
|
float prob = iterLHS->second;
|
|
|
|
|
|
|
|
// lhs
|
|
|
|
//const Word &sourceLHS = staticData.GetInputDefaultNonTerminal();
|
|
|
|
Word targetLHS(true);
|
|
|
|
|
|
|
|
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
|
|
|
CHECK(targetLHS.GetFactor(0) != NULL);
|
|
|
|
|
|
|
|
// add to dictionary
|
|
|
|
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
|
|
|
TargetPhraseCollection *tpc = new TargetPhraseCollection;
|
|
|
|
tpc->Add(targetPhrase);
|
|
|
|
|
|
|
|
m_cacheTargetPhraseCollection.push_back(tpc);
|
|
|
|
Word &targetWord = targetPhrase->AddWord();
|
|
|
|
targetWord.CreateUnknownWord(sourceWord);
|
|
|
|
|
|
|
|
// scores
|
|
|
|
vector<float> unknownScore(1, FloorScore(TransformScore(prob)));
|
|
|
|
|
|
|
|
//targetPhrase->SetScore();
|
|
|
|
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
|
|
|
targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore);
|
|
|
|
targetPhrase->SetSourcePhrase(m_unksrc);
|
|
|
|
targetPhrase->SetTargetLHS(targetLHS);
|
|
|
|
|
|
|
|
// chart rule
|
|
|
|
m_translationOptionList.Add(*tpc, m_emptyStackVec, range);
|
|
|
|
} // for (iterLHS
|
|
|
|
} else {
|
|
|
|
// drop source word. create blank trans opt
|
|
|
|
vector<float> unknownScore(1, FloorScore(-numeric_limits<float>::infinity()));
|
|
|
|
|
|
|
|
TargetPhrase *targetPhrase = new TargetPhrase(Output);
|
|
|
|
TargetPhraseCollection *tpc = new TargetPhraseCollection;
|
|
|
|
tpc->Add(targetPhrase);
|
|
|
|
// loop
|
|
|
|
const UnknownLHSList &lhsList = staticData.GetUnknownLHS();
|
|
|
|
UnknownLHSList::const_iterator iterLHS;
|
|
|
|
for (iterLHS = lhsList.begin(); iterLHS != lhsList.end(); ++iterLHS) {
|
|
|
|
const string &targetLHSStr = iterLHS->first;
|
|
|
|
//float prob = iterLHS->second;
|
|
|
|
|
|
|
|
Word targetLHS(true);
|
|
|
|
targetLHS.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetLHSStr, true);
|
|
|
|
CHECK(targetLHS.GetFactor(0) != NULL);
|
|
|
|
|
|
|
|
m_cacheTargetPhraseCollection.push_back(tpc);
|
|
|
|
targetPhrase->SetSourcePhrase(m_unksrc);
|
|
|
|
targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore);
|
|
|
|
targetPhrase->SetTargetLHS(targetLHS);
|
|
|
|
|
|
|
|
// chart rule
|
|
|
|
m_translationOptionList.Add(*tpc, m_emptyStackVec, range);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-11-08 15:28:02 +04:00
|
|
|
} // namespace Moses
|