mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 05:14:36 +03:00
Add PhraseDictionaryTMExtract and ChartRuleLookupManagerMemoryPerSentence to deal with per per sentence grammar. eg. Adam's suffix array and Phi Koehn's TM extraction
This commit is contained in:
Hieu Hoang
parent
d988ca6ed4
commit
3f9c995e18
@ -11,6 +11,8 @@
|
||||
1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h */; };
|
||||
1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */; };
|
||||
1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E1D823F15AC29BB00FE42E9 /* FileHandler.h */; };
|
||||
1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */; };
|
||||
1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */; };
|
||||
1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */; };
|
||||
1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E879EA615A346F90051F346 /* SearchNormalBatch.h */; };
|
||||
1EAC363514CDC79300DF97C3 /* Loader.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EAC362C14CDC79300DF97C3 /* Loader.h */; };
|
||||
@ -307,6 +309,8 @@
|
||||
1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryTMExtract.h; path = ../../moses/src/RuleTable/PhraseDictionaryTMExtract.h; sourceTree = "<group>"; };
|
||||
1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FileHandler.cpp; sourceTree = "<group>"; };
|
||||
1E1D823F15AC29BB00FE42E9 /* FileHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FileHandler.h; sourceTree = "<group>"; };
|
||||
1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartRuleLookupManagerMemoryPerSentence.cpp; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp; sourceTree = "<group>"; };
|
||||
1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartRuleLookupManagerMemoryPerSentence.h; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h; sourceTree = "<group>"; };
|
||||
1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SearchNormalBatch.cpp; path = ../../moses/src/SearchNormalBatch.cpp; sourceTree = "<group>"; };
|
||||
1E879EA615A346F90051F346 /* SearchNormalBatch.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SearchNormalBatch.h; path = ../../moses/src/SearchNormalBatch.h; sourceTree = "<group>"; };
|
||||
1EAC362C14CDC79300DF97C3 /* Loader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Loader.h; path = ../../moses/src/RuleTable/Loader.h; sourceTree = "<group>"; };
|
||||
@ -939,6 +943,8 @@
|
||||
1EDA803414D19EB8003D2191 /* CYKPlusParser */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */,
|
||||
1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */,
|
||||
1EDA806214D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.cpp */,
|
||||
1EDA806314D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.h */,
|
||||
1EDA806414D19F12003D2191 /* ChartRuleLookupManagerMemory.cpp */,
|
||||
@ -1150,6 +1156,7 @@
|
||||
1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */,
|
||||
1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */,
|
||||
1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h in Headers */,
|
||||
1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
@ -1336,6 +1343,7 @@
|
||||
1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */,
|
||||
1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */,
|
||||
1E0BA41815B70E5F00AC70E1 /* PhraseDictionaryTMExtract.cpp in Sources */,
|
||||
1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
|
||||
@ -0,0 +1,281 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "ChartRuleLookupManagerMemoryPerSentence.h"
|
||||
|
||||
#include "RuleTable/PhraseDictionaryTMExtract.h"
|
||||
#include "InputType.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "CellCollection.h"
|
||||
#include "DotChartInMemory.h"
|
||||
#include "StaticData.h"
|
||||
#include "NonTerminal.h"
|
||||
#include "ChartCellCollection.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence(
|
||||
const InputType &src,
|
||||
const ChartCellCollection &cellColl,
|
||||
const PhraseDictionaryTMExtract &ruleTable)
|
||||
: ChartRuleLookupManagerCYKPlus(src, cellColl)
|
||||
, m_ruleTable(ruleTable)
|
||||
{
|
||||
CHECK(m_dottedRuleColls.size() == 0);
|
||||
size_t sourceSize = src.GetSize();
|
||||
m_dottedRuleColls.resize(sourceSize);
|
||||
|
||||
const PhraseDictionaryNodeSCFG &rootNode = m_ruleTable.GetRootNode(src);
|
||||
|
||||
for (size_t ind = 0; ind < m_dottedRuleColls.size(); ++ind) {
|
||||
#ifdef USE_BOOST_POOL
|
||||
DottedRuleInMemory *initDottedRule = m_dottedRulePool.malloc();
|
||||
new (initDottedRule) DottedRuleInMemory(rootNode);
|
||||
#else
|
||||
DottedRuleInMemory *initDottedRule = new DottedRuleInMemory(rootNode);
|
||||
#endif
|
||||
|
||||
DottedRuleColl *dottedRuleColl = new DottedRuleColl(sourceSize - ind + 1);
|
||||
dottedRuleColl->Add(0, initDottedRule); // init rule. stores the top node in tree
|
||||
|
||||
m_dottedRuleColls[ind] = dottedRuleColl;
|
||||
}
|
||||
}
|
||||
|
||||
ChartRuleLookupManagerMemoryPerSentence::~ChartRuleLookupManagerMemoryPerSentence()
|
||||
{
|
||||
RemoveAllInColl(m_dottedRuleColls);
|
||||
}
|
||||
|
||||
void ChartRuleLookupManagerMemoryPerSentence::GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
ChartTranslationOptionList &outColl)
|
||||
{
|
||||
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
|
||||
size_t absEndPos = range.GetEndPos();
|
||||
|
||||
// MAIN LOOP. create list of nodes of target phrases
|
||||
|
||||
// get list of all rules that apply to spans at same starting position
|
||||
DottedRuleColl &dottedRuleCol = *m_dottedRuleColls[range.GetStartPos()];
|
||||
const DottedRuleList &expandableDottedRuleList = dottedRuleCol.GetExpandableDottedRuleList();
|
||||
|
||||
const ChartCellLabel &sourceWordLabel = GetCellCollection().Get(WordsRange(absEndPos, absEndPos)).GetSourceWordLabel();
|
||||
|
||||
// loop through the rules
|
||||
// (note that expandableDottedRuleList can be expanded as the loop runs
|
||||
// through calls to ExtendPartialRuleApplication())
|
||||
for (size_t ind = 0; ind < expandableDottedRuleList.size(); ++ind) {
|
||||
// rule we are about to extend
|
||||
const DottedRuleInMemory &prevDottedRule = *expandableDottedRuleList[ind];
|
||||
// we will now try to extend it, starting after where it ended
|
||||
size_t startPos = prevDottedRule.IsRoot()
|
||||
? range.GetStartPos()
|
||||
: prevDottedRule.GetWordsRange().GetEndPos() + 1;
|
||||
|
||||
// search for terminal symbol
|
||||
// (if only one more word position needs to be covered)
|
||||
if (startPos == absEndPos) {
|
||||
|
||||
// look up in rule dictionary, if the current rule can be extended
|
||||
// with the source word in the last position
|
||||
const Word &sourceWord = sourceWordLabel.GetLabel();
|
||||
const PhraseDictionaryNodeSCFG *node = prevDottedRule.GetLastNode().GetChild(sourceWord);
|
||||
|
||||
// if we found a new rule -> create it and add it to the list
|
||||
if (node != NULL) {
|
||||
// create the rule
|
||||
#ifdef USE_BOOST_POOL
|
||||
DottedRuleInMemory *dottedRule = m_dottedRulePool.malloc();
|
||||
new (dottedRule) DottedRuleInMemory(*node, sourceWordLabel,
|
||||
prevDottedRule);
|
||||
#else
|
||||
DottedRuleInMemory *dottedRule = new DottedRuleInMemory(*node,
|
||||
sourceWordLabel,
|
||||
prevDottedRule);
|
||||
#endif
|
||||
dottedRuleCol.Add(relEndPos+1, dottedRule);
|
||||
}
|
||||
}
|
||||
|
||||
// search for non-terminals
|
||||
size_t endPos, stackInd;
|
||||
|
||||
// span is already complete covered? nothing can be done
|
||||
if (startPos > absEndPos)
|
||||
continue;
|
||||
|
||||
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos()) {
|
||||
// We're at the root of the prefix tree so won't try to cover the full
|
||||
// span (i.e. we don't allow non-lexical unary rules). However, we need
|
||||
// to match non-unary rules that begin with a non-terminal child, so we
|
||||
// do that in two steps: during this iteration we search for non-terminals
|
||||
// that cover all but the last source word in the span (there won't
|
||||
// already be running nodes for these because that would have required a
|
||||
// non-lexical unary rule match for an earlier span). Any matches will
|
||||
// result in running nodes being appended to the list and on subsequent
|
||||
// iterations (for this same span), we'll extend them to cover the final
|
||||
// word.
|
||||
endPos = absEndPos - 1;
|
||||
stackInd = relEndPos;
|
||||
}
|
||||
else
|
||||
{
|
||||
endPos = absEndPos;
|
||||
stackInd = relEndPos + 1;
|
||||
}
|
||||
|
||||
|
||||
ExtendPartialRuleApplication(prevDottedRule, startPos, endPos, stackInd,
|
||||
dottedRuleCol);
|
||||
}
|
||||
|
||||
// list of rules that that cover the entire span
|
||||
DottedRuleList &rules = dottedRuleCol.Get(relEndPos + 1);
|
||||
|
||||
// look up target sides for the rules
|
||||
DottedRuleList::const_iterator iterRule;
|
||||
for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) {
|
||||
const DottedRuleInMemory &dottedRule = **iterRule;
|
||||
const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode();
|
||||
|
||||
// look up target sides
|
||||
const TargetPhraseCollection *tpc = node.GetTargetPhraseCollection();
|
||||
|
||||
// add the fully expanded rule (with lexical target side)
|
||||
if (tpc != NULL) {
|
||||
AddCompletedRule(dottedRule, *tpc, range, outColl);
|
||||
}
|
||||
}
|
||||
|
||||
dottedRuleCol.Clear(relEndPos+1);
|
||||
|
||||
outColl.ShrinkToLimit();
|
||||
}
|
||||
|
||||
// Given a partial rule application ending at startPos-1 and given the sets of
|
||||
// source and target non-terminals covering the span [startPos, endPos],
|
||||
// determines the full or partial rule applications that can be produced through
|
||||
// extending the current rule application by a single non-terminal.
|
||||
void ChartRuleLookupManagerMemoryPerSentence::ExtendPartialRuleApplication(
|
||||
const DottedRuleInMemory &prevDottedRule,
|
||||
size_t startPos,
|
||||
size_t endPos,
|
||||
size_t stackInd,
|
||||
DottedRuleColl & dottedRuleColl)
|
||||
{
|
||||
// source non-terminal labels for the remainder
|
||||
const NonTerminalSet &sourceNonTerms =
|
||||
GetSentence().GetLabelSet(startPos, endPos);
|
||||
|
||||
// target non-terminal labels for the remainder
|
||||
const ChartCellLabelSet &targetNonTerms =
|
||||
GetCellCollection().Get(WordsRange(startPos, endPos)).GetTargetLabelSet();
|
||||
|
||||
// note where it was found in the prefix tree of the rule dictionary
|
||||
const PhraseDictionaryNodeSCFG &node = prevDottedRule.GetLastNode();
|
||||
|
||||
const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap =
|
||||
node.GetNonTerminalMap();
|
||||
|
||||
const size_t numChildren = nonTermMap.size();
|
||||
if (numChildren == 0) {
|
||||
return;
|
||||
}
|
||||
const size_t numSourceNonTerms = sourceNonTerms.size();
|
||||
const size_t numTargetNonTerms = targetNonTerms.GetSize();
|
||||
const size_t numCombinations = numSourceNonTerms * numTargetNonTerms;
|
||||
|
||||
// We can search by either:
|
||||
// 1. Enumerating all possible source-target NT pairs that are valid for
|
||||
// the span and then searching for matching children in the node,
|
||||
// or
|
||||
// 2. Iterating over all the NT children in the node, searching
|
||||
// for each source and target NT in the span's sets.
|
||||
// We'll do whichever minimises the number of lookups:
|
||||
if (numCombinations <= numChildren*2) {
|
||||
|
||||
// loop over possible source non-terminal labels (as found in input tree)
|
||||
NonTerminalSet::const_iterator p = sourceNonTerms.begin();
|
||||
NonTerminalSet::const_iterator sEnd = sourceNonTerms.end();
|
||||
for (; p != sEnd; ++p) {
|
||||
const Word & sourceNonTerm = *p;
|
||||
|
||||
// loop over possible target non-terminal labels (as found in chart)
|
||||
ChartCellLabelSet::const_iterator q = targetNonTerms.begin();
|
||||
ChartCellLabelSet::const_iterator tEnd = targetNonTerms.end();
|
||||
for (; q != tEnd; ++q) {
|
||||
const ChartCellLabel &cellLabel = q->second;
|
||||
|
||||
// try to match both source and target non-terminal
|
||||
const PhraseDictionaryNodeSCFG * child =
|
||||
node.GetChild(sourceNonTerm, cellLabel.GetLabel());
|
||||
|
||||
// nothing found? then we are done
|
||||
if (child == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// create new rule
|
||||
#ifdef USE_BOOST_POOL
|
||||
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
|
||||
new (rule) DottedRuleInMemory(*child, cellLabel, prevDottedRule);
|
||||
#else
|
||||
DottedRuleInMemory *rule = new DottedRuleInMemory(*child, cellLabel,
|
||||
prevDottedRule);
|
||||
#endif
|
||||
dottedRuleColl.Add(stackInd, rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// loop over possible expansions of the rule
|
||||
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator p;
|
||||
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator end =
|
||||
nonTermMap.end();
|
||||
for (p = nonTermMap.begin(); p != end; ++p) {
|
||||
// does it match possible source and target non-terminals?
|
||||
const PhraseDictionaryNodeSCFG::NonTerminalMapKey &key = p->first;
|
||||
const Word &sourceNonTerm = key.first;
|
||||
if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end()) {
|
||||
continue;
|
||||
}
|
||||
const Word &targetNonTerm = key.second;
|
||||
const ChartCellLabel *cellLabel = targetNonTerms.Find(targetNonTerm);
|
||||
if (!cellLabel) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// create new rule
|
||||
const PhraseDictionaryNodeSCFG &child = p->second;
|
||||
#ifdef USE_BOOST_POOL
|
||||
DottedRuleInMemory *rule = m_dottedRulePool.malloc();
|
||||
new (rule) DottedRuleInMemory(child, *cellLabel, prevDottedRule);
|
||||
#else
|
||||
DottedRuleInMemory *rule = new DottedRuleInMemory(child, *cellLabel,
|
||||
prevDottedRule);
|
||||
#endif
|
||||
dottedRuleColl.Add(stackInd, rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Moses
|
||||
@ -0,0 +1,78 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
#ifndef moses_ChartRuleLookupManagerMemory_h
|
||||
#define moses_ChartRuleLookupManagerMemory_h
|
||||
|
||||
#include <vector>
|
||||
|
||||
#ifdef USE_BOOST_POOL
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#endif
|
||||
|
||||
#include "ChartRuleLookupManagerCYKPlus.h"
|
||||
#include "DotChartInMemory.h"
|
||||
#include "NonTerminal.h"
|
||||
#include "RuleTable/PhraseDictionaryNodeSCFG.h"
|
||||
#include "RuleTable/PhraseDictionarySCFG.h"
|
||||
#include "StackVec.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ChartTranslationOptionList;
|
||||
class DottedRuleColl;
|
||||
class WordsRange;
|
||||
|
||||
//! Implementation of ChartRuleLookupManager for in-memory rule tables.
|
||||
class ChartRuleLookupManagerMemoryPerSentence : public ChartRuleLookupManagerCYKPlus
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManagerMemoryPerSentence(const InputType &sentence,
|
||||
const ChartCellCollection &cellColl,
|
||||
const PhraseDictionaryTMExtract &ruleTable);
|
||||
|
||||
~ChartRuleLookupManagerMemoryPerSentence();
|
||||
|
||||
virtual void GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
private:
|
||||
void ExtendPartialRuleApplication(
|
||||
const DottedRuleInMemory &prevDottedRule,
|
||||
size_t startPos,
|
||||
size_t endPos,
|
||||
size_t stackInd,
|
||||
DottedRuleColl &dottedRuleColl);
|
||||
|
||||
std::vector<DottedRuleColl*> m_dottedRuleColls;
|
||||
const PhraseDictionaryTMExtract &m_ruleTable;
|
||||
#ifdef USE_BOOST_POOL
|
||||
// Use an object pool to allocate the dotted rules for this sentence. We
|
||||
// allocate a lot of them and this has been seen to significantly improve
|
||||
// performance, especially for multithreaded decoding.
|
||||
boost::object_pool<DottedRuleInMemory> m_dottedRulePool;
|
||||
#endif
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
#endif
|
||||
@ -192,8 +192,9 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst
|
||||
#endif
|
||||
} else if (m_implementation == TMExtract) {
|
||||
|
||||
RuleTableTrie *dict = new PhraseDictionaryTMExtract(m_numScoreComponent, this);
|
||||
PhraseDictionaryTMExtract *dict = new PhraseDictionaryTMExtract(m_numScoreComponent, this);
|
||||
|
||||
/*
|
||||
bool ret = dict->Load(GetInput()
|
||||
, GetOutput()
|
||||
, m_filePath
|
||||
@ -202,6 +203,7 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst
|
||||
, system->GetLanguageModels()
|
||||
, system->GetWordPenaltyProducer());
|
||||
assert(ret);
|
||||
*/
|
||||
return dict;
|
||||
|
||||
} else {
|
||||
|
||||
@ -33,7 +33,7 @@
|
||||
#include "StaticData.h"
|
||||
#include "WordsRange.h"
|
||||
#include "UserMessage.h"
|
||||
#include "CYKPlusParser/ChartRuleLookupManagerMemory.h"
|
||||
#include "CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -41,7 +41,7 @@ namespace Moses
|
||||
{
|
||||
PhraseDictionaryTMExtract::PhraseDictionaryTMExtract(size_t numScoreComponents,
|
||||
PhraseDictionaryFeature* feature)
|
||||
: PhraseDictionarySCFG(numScoreComponents, feature)
|
||||
: PhraseDictionary(numScoreComponents, feature)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
CHECK(staticData.ThreadCount() == 1);
|
||||
@ -99,7 +99,7 @@ namespace Moses
|
||||
const InputType &sentence,
|
||||
const ChartCellCollection &cellCollection)
|
||||
{
|
||||
return new ChartRuleLookupManagerMemory(sentence, cellCollection, *this);
|
||||
return new ChartRuleLookupManagerMemoryPerSentence(sentence, cellCollection, *this);
|
||||
}
|
||||
|
||||
void PhraseDictionaryTMExtract::SortAndPrune()
|
||||
@ -128,7 +128,7 @@ namespace Moses
|
||||
|
||||
}
|
||||
|
||||
void PhraseDictionaryTMExtract::CleanUp()
|
||||
void PhraseDictionaryTMExtract::CleanUp(const InputType &source)
|
||||
{
|
||||
m_collection.Clear();
|
||||
}
|
||||
|
||||
@ -32,7 +32,7 @@ namespace Moses
|
||||
/** Implementation of a SCFG rule table in a trie. Looking up a rule of
|
||||
* length n symbols requires n look-ups to find the TargetPhraseCollection.
|
||||
*/
|
||||
class PhraseDictionaryTMExtract : public PhraseDictionarySCFG
|
||||
class PhraseDictionaryTMExtract : public PhraseDictionary
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryTMExtract&);
|
||||
friend class RuleTableLoader;
|
||||
@ -41,13 +41,18 @@ namespace Moses
|
||||
PhraseDictionaryTMExtract(size_t numScoreComponents,
|
||||
PhraseDictionaryFeature* feature);
|
||||
|
||||
const PhraseDictionaryNodeSCFG &GetRootNode() const { return m_collection; }
|
||||
const PhraseDictionaryNodeSCFG &GetRootNode(const InputType &source) const { return m_collection; }
|
||||
|
||||
ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const ChartCellCollection &);
|
||||
void InitializeForInput(InputType const& source);
|
||||
void CleanUp();
|
||||
void CleanUp(const InputType& source);
|
||||
|
||||
virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const
|
||||
{}
|
||||
virtual DecodeType GetDecodeType() const
|
||||
{}
|
||||
|
||||
TO_STRING();
|
||||
|
||||
@ -61,7 +66,7 @@ namespace Moses
|
||||
|
||||
void SortAndPrune();
|
||||
|
||||
//PhraseDictionaryNodeSCFG m_collection;
|
||||
PhraseDictionaryNodeSCFG m_collection;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
Loading…
Reference in New Issue
Block a user