From 3f9c995e180599545d8ad75891eff0e87ea6ab9c Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 19 Jul 2012 17:56:46 +0100 Subject: [PATCH] Add PhraseDictionaryTMExtract and ChartRuleLookupManagerMemoryPerSentence to deal with per per sentence grammar. eg. Adam's suffix array and Phi Koehn's TM extraction --- .../moses.xcodeproj/project.pbxproj | 8 + ...hartRuleLookupManagerMemoryPerSentence.cpp | 281 ++++++++++++++++++ .../ChartRuleLookupManagerMemoryPerSentence.h | 78 +++++ moses/src/PhraseDictionary.cpp | 4 +- .../RuleTable/PhraseDictionaryTMExtract.cpp | 8 +- .../src/RuleTable/PhraseDictionaryTMExtract.h | 13 +- 6 files changed, 383 insertions(+), 9 deletions(-) create mode 100644 moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp create mode 100644 moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h diff --git a/contrib/other-builds/moses.xcodeproj/project.pbxproj b/contrib/other-builds/moses.xcodeproj/project.pbxproj index db30c5793..64952ae6d 100644 --- a/contrib/other-builds/moses.xcodeproj/project.pbxproj +++ b/contrib/other-builds/moses.xcodeproj/project.pbxproj @@ -11,6 +11,8 @@ 1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h */; }; 1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */; }; 1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E1D823F15AC29BB00FE42E9 /* FileHandler.h */; }; + 1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */; }; + 1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */; }; 1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */; }; 1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E879EA615A346F90051F346 /* SearchNormalBatch.h */; }; 1EAC363514CDC79300DF97C3 /* Loader.h in Headers */ = {isa = PBXBuildFile; fileRef = 1EAC362C14CDC79300DF97C3 /* Loader.h */; }; @@ -307,6 +309,8 @@ 1E0BA41715B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PhraseDictionaryTMExtract.h; path = ../../moses/src/RuleTable/PhraseDictionaryTMExtract.h; sourceTree = ""; }; 1E1D823E15AC29BB00FE42E9 /* FileHandler.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = FileHandler.cpp; sourceTree = ""; }; 1E1D823F15AC29BB00FE42E9 /* FileHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FileHandler.h; sourceTree = ""; }; + 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ChartRuleLookupManagerMemoryPerSentence.cpp; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp; sourceTree = ""; }; + 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ChartRuleLookupManagerMemoryPerSentence.h; path = ../../moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h; sourceTree = ""; }; 1E879EA515A346F90051F346 /* SearchNormalBatch.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = SearchNormalBatch.cpp; path = ../../moses/src/SearchNormalBatch.cpp; sourceTree = ""; }; 1E879EA615A346F90051F346 /* SearchNormalBatch.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SearchNormalBatch.h; path = ../../moses/src/SearchNormalBatch.h; sourceTree = ""; }; 1EAC362C14CDC79300DF97C3 /* Loader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Loader.h; path = ../../moses/src/RuleTable/Loader.h; sourceTree = ""; }; @@ -939,6 +943,8 @@ 1EDA803414D19EB8003D2191 /* CYKPlusParser */ = { isa = PBXGroup; children = ( + 1E619E9F15B8713600C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp */, + 1E619EA015B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h */, 1EDA806214D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.cpp */, 1EDA806314D19F12003D2191 /* ChartRuleLookupManagerCYKPlus.h */, 1EDA806414D19F12003D2191 /* ChartRuleLookupManagerMemory.cpp */, @@ -1150,6 +1156,7 @@ 1E879EA815A346F90051F346 /* SearchNormalBatch.h in Headers */, 1E1D824115AC29BB00FE42E9 /* FileHandler.h in Headers */, 1E0BA41915B70E5F00AC70E1 /* PhraseDictionaryTMExtract.h in Headers */, + 1E619EA215B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.h in Headers */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1336,6 +1343,7 @@ 1E879EA715A346F90051F346 /* SearchNormalBatch.cpp in Sources */, 1E1D824015AC29BB00FE42E9 /* FileHandler.cpp in Sources */, 1E0BA41815B70E5F00AC70E1 /* PhraseDictionaryTMExtract.cpp in Sources */, + 1E619EA115B8713700C2D7A7 /* ChartRuleLookupManagerMemoryPerSentence.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp b/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp new file mode 100644 index 000000000..098ce8bf5 --- /dev/null +++ b/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.cpp @@ -0,0 +1,281 @@ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2011 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include "ChartRuleLookupManagerMemoryPerSentence.h" + +#include "RuleTable/PhraseDictionaryTMExtract.h" +#include "InputType.h" +#include "ChartTranslationOptionList.h" +#include "CellCollection.h" +#include "DotChartInMemory.h" +#include "StaticData.h" +#include "NonTerminal.h" +#include "ChartCellCollection.h" + +namespace Moses +{ + +ChartRuleLookupManagerMemoryPerSentence::ChartRuleLookupManagerMemoryPerSentence( + const InputType &src, + const ChartCellCollection &cellColl, + const PhraseDictionaryTMExtract &ruleTable) + : ChartRuleLookupManagerCYKPlus(src, cellColl) + , m_ruleTable(ruleTable) +{ + CHECK(m_dottedRuleColls.size() == 0); + size_t sourceSize = src.GetSize(); + m_dottedRuleColls.resize(sourceSize); + + const PhraseDictionaryNodeSCFG &rootNode = m_ruleTable.GetRootNode(src); + + for (size_t ind = 0; ind < m_dottedRuleColls.size(); ++ind) { +#ifdef USE_BOOST_POOL + DottedRuleInMemory *initDottedRule = m_dottedRulePool.malloc(); + new (initDottedRule) DottedRuleInMemory(rootNode); +#else + DottedRuleInMemory *initDottedRule = new DottedRuleInMemory(rootNode); +#endif + + DottedRuleColl *dottedRuleColl = new DottedRuleColl(sourceSize - ind + 1); + dottedRuleColl->Add(0, initDottedRule); // init rule. stores the top node in tree + + m_dottedRuleColls[ind] = dottedRuleColl; + } +} + +ChartRuleLookupManagerMemoryPerSentence::~ChartRuleLookupManagerMemoryPerSentence() +{ + RemoveAllInColl(m_dottedRuleColls); +} + +void ChartRuleLookupManagerMemoryPerSentence::GetChartRuleCollection( + const WordsRange &range, + ChartTranslationOptionList &outColl) +{ + size_t relEndPos = range.GetEndPos() - range.GetStartPos(); + size_t absEndPos = range.GetEndPos(); + + // MAIN LOOP. create list of nodes of target phrases + + // get list of all rules that apply to spans at same starting position + DottedRuleColl &dottedRuleCol = *m_dottedRuleColls[range.GetStartPos()]; + const DottedRuleList &expandableDottedRuleList = dottedRuleCol.GetExpandableDottedRuleList(); + + const ChartCellLabel &sourceWordLabel = GetCellCollection().Get(WordsRange(absEndPos, absEndPos)).GetSourceWordLabel(); + + // loop through the rules + // (note that expandableDottedRuleList can be expanded as the loop runs + // through calls to ExtendPartialRuleApplication()) + for (size_t ind = 0; ind < expandableDottedRuleList.size(); ++ind) { + // rule we are about to extend + const DottedRuleInMemory &prevDottedRule = *expandableDottedRuleList[ind]; + // we will now try to extend it, starting after where it ended + size_t startPos = prevDottedRule.IsRoot() + ? range.GetStartPos() + : prevDottedRule.GetWordsRange().GetEndPos() + 1; + + // search for terminal symbol + // (if only one more word position needs to be covered) + if (startPos == absEndPos) { + + // look up in rule dictionary, if the current rule can be extended + // with the source word in the last position + const Word &sourceWord = sourceWordLabel.GetLabel(); + const PhraseDictionaryNodeSCFG *node = prevDottedRule.GetLastNode().GetChild(sourceWord); + + // if we found a new rule -> create it and add it to the list + if (node != NULL) { + // create the rule +#ifdef USE_BOOST_POOL + DottedRuleInMemory *dottedRule = m_dottedRulePool.malloc(); + new (dottedRule) DottedRuleInMemory(*node, sourceWordLabel, + prevDottedRule); +#else + DottedRuleInMemory *dottedRule = new DottedRuleInMemory(*node, + sourceWordLabel, + prevDottedRule); +#endif + dottedRuleCol.Add(relEndPos+1, dottedRule); + } + } + + // search for non-terminals + size_t endPos, stackInd; + + // span is already complete covered? nothing can be done + if (startPos > absEndPos) + continue; + + else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos()) { + // We're at the root of the prefix tree so won't try to cover the full + // span (i.e. we don't allow non-lexical unary rules). However, we need + // to match non-unary rules that begin with a non-terminal child, so we + // do that in two steps: during this iteration we search for non-terminals + // that cover all but the last source word in the span (there won't + // already be running nodes for these because that would have required a + // non-lexical unary rule match for an earlier span). Any matches will + // result in running nodes being appended to the list and on subsequent + // iterations (for this same span), we'll extend them to cover the final + // word. + endPos = absEndPos - 1; + stackInd = relEndPos; + } + else + { + endPos = absEndPos; + stackInd = relEndPos + 1; + } + + + ExtendPartialRuleApplication(prevDottedRule, startPos, endPos, stackInd, + dottedRuleCol); + } + + // list of rules that that cover the entire span + DottedRuleList &rules = dottedRuleCol.Get(relEndPos + 1); + + // look up target sides for the rules + DottedRuleList::const_iterator iterRule; + for (iterRule = rules.begin(); iterRule != rules.end(); ++iterRule) { + const DottedRuleInMemory &dottedRule = **iterRule; + const PhraseDictionaryNodeSCFG &node = dottedRule.GetLastNode(); + + // look up target sides + const TargetPhraseCollection *tpc = node.GetTargetPhraseCollection(); + + // add the fully expanded rule (with lexical target side) + if (tpc != NULL) { + AddCompletedRule(dottedRule, *tpc, range, outColl); + } + } + + dottedRuleCol.Clear(relEndPos+1); + + outColl.ShrinkToLimit(); +} + +// Given a partial rule application ending at startPos-1 and given the sets of +// source and target non-terminals covering the span [startPos, endPos], +// determines the full or partial rule applications that can be produced through +// extending the current rule application by a single non-terminal. +void ChartRuleLookupManagerMemoryPerSentence::ExtendPartialRuleApplication( + const DottedRuleInMemory &prevDottedRule, + size_t startPos, + size_t endPos, + size_t stackInd, + DottedRuleColl & dottedRuleColl) +{ + // source non-terminal labels for the remainder + const NonTerminalSet &sourceNonTerms = + GetSentence().GetLabelSet(startPos, endPos); + + // target non-terminal labels for the remainder + const ChartCellLabelSet &targetNonTerms = + GetCellCollection().Get(WordsRange(startPos, endPos)).GetTargetLabelSet(); + + // note where it was found in the prefix tree of the rule dictionary + const PhraseDictionaryNodeSCFG &node = prevDottedRule.GetLastNode(); + + const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap = + node.GetNonTerminalMap(); + + const size_t numChildren = nonTermMap.size(); + if (numChildren == 0) { + return; + } + const size_t numSourceNonTerms = sourceNonTerms.size(); + const size_t numTargetNonTerms = targetNonTerms.GetSize(); + const size_t numCombinations = numSourceNonTerms * numTargetNonTerms; + + // We can search by either: + // 1. Enumerating all possible source-target NT pairs that are valid for + // the span and then searching for matching children in the node, + // or + // 2. Iterating over all the NT children in the node, searching + // for each source and target NT in the span's sets. + // We'll do whichever minimises the number of lookups: + if (numCombinations <= numChildren*2) { + + // loop over possible source non-terminal labels (as found in input tree) + NonTerminalSet::const_iterator p = sourceNonTerms.begin(); + NonTerminalSet::const_iterator sEnd = sourceNonTerms.end(); + for (; p != sEnd; ++p) { + const Word & sourceNonTerm = *p; + + // loop over possible target non-terminal labels (as found in chart) + ChartCellLabelSet::const_iterator q = targetNonTerms.begin(); + ChartCellLabelSet::const_iterator tEnd = targetNonTerms.end(); + for (; q != tEnd; ++q) { + const ChartCellLabel &cellLabel = q->second; + + // try to match both source and target non-terminal + const PhraseDictionaryNodeSCFG * child = + node.GetChild(sourceNonTerm, cellLabel.GetLabel()); + + // nothing found? then we are done + if (child == NULL) { + continue; + } + + // create new rule +#ifdef USE_BOOST_POOL + DottedRuleInMemory *rule = m_dottedRulePool.malloc(); + new (rule) DottedRuleInMemory(*child, cellLabel, prevDottedRule); +#else + DottedRuleInMemory *rule = new DottedRuleInMemory(*child, cellLabel, + prevDottedRule); +#endif + dottedRuleColl.Add(stackInd, rule); + } + } + } + else + { + // loop over possible expansions of the rule + PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator p; + PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator end = + nonTermMap.end(); + for (p = nonTermMap.begin(); p != end; ++p) { + // does it match possible source and target non-terminals? + const PhraseDictionaryNodeSCFG::NonTerminalMapKey &key = p->first; + const Word &sourceNonTerm = key.first; + if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end()) { + continue; + } + const Word &targetNonTerm = key.second; + const ChartCellLabel *cellLabel = targetNonTerms.Find(targetNonTerm); + if (!cellLabel) { + continue; + } + + // create new rule + const PhraseDictionaryNodeSCFG &child = p->second; +#ifdef USE_BOOST_POOL + DottedRuleInMemory *rule = m_dottedRulePool.malloc(); + new (rule) DottedRuleInMemory(child, *cellLabel, prevDottedRule); +#else + DottedRuleInMemory *rule = new DottedRuleInMemory(child, *cellLabel, + prevDottedRule); +#endif + dottedRuleColl.Add(stackInd, rule); + } + } +} + +} // namespace Moses diff --git a/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h b/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h new file mode 100644 index 000000000..3ee0b6eca --- /dev/null +++ b/moses/src/CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h @@ -0,0 +1,78 @@ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2011 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#pragma once +#ifndef moses_ChartRuleLookupManagerMemory_h +#define moses_ChartRuleLookupManagerMemory_h + +#include + +#ifdef USE_BOOST_POOL +#include +#endif + +#include "ChartRuleLookupManagerCYKPlus.h" +#include "DotChartInMemory.h" +#include "NonTerminal.h" +#include "RuleTable/PhraseDictionaryNodeSCFG.h" +#include "RuleTable/PhraseDictionarySCFG.h" +#include "StackVec.h" + +namespace Moses +{ + +class ChartTranslationOptionList; +class DottedRuleColl; +class WordsRange; + +//! Implementation of ChartRuleLookupManager for in-memory rule tables. +class ChartRuleLookupManagerMemoryPerSentence : public ChartRuleLookupManagerCYKPlus +{ + public: + ChartRuleLookupManagerMemoryPerSentence(const InputType &sentence, + const ChartCellCollection &cellColl, + const PhraseDictionaryTMExtract &ruleTable); + + ~ChartRuleLookupManagerMemoryPerSentence(); + + virtual void GetChartRuleCollection( + const WordsRange &range, + ChartTranslationOptionList &outColl); + + private: + void ExtendPartialRuleApplication( + const DottedRuleInMemory &prevDottedRule, + size_t startPos, + size_t endPos, + size_t stackInd, + DottedRuleColl &dottedRuleColl); + + std::vector m_dottedRuleColls; + const PhraseDictionaryTMExtract &m_ruleTable; +#ifdef USE_BOOST_POOL + // Use an object pool to allocate the dotted rules for this sentence. We + // allocate a lot of them and this has been seen to significantly improve + // performance, especially for multithreaded decoding. + boost::object_pool m_dottedRulePool; +#endif +}; + +} // namespace Moses + +#endif diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp index 2932fa3cf..5d35b5f72 100644 --- a/moses/src/PhraseDictionary.cpp +++ b/moses/src/PhraseDictionary.cpp @@ -192,8 +192,9 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst #endif } else if (m_implementation == TMExtract) { - RuleTableTrie *dict = new PhraseDictionaryTMExtract(m_numScoreComponent, this); + PhraseDictionaryTMExtract *dict = new PhraseDictionaryTMExtract(m_numScoreComponent, this); + /* bool ret = dict->Load(GetInput() , GetOutput() , m_filePath @@ -202,6 +203,7 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst , system->GetLanguageModels() , system->GetWordPenaltyProducer()); assert(ret); + */ return dict; } else { diff --git a/moses/src/RuleTable/PhraseDictionaryTMExtract.cpp b/moses/src/RuleTable/PhraseDictionaryTMExtract.cpp index 7ff522073..94297d3d8 100644 --- a/moses/src/RuleTable/PhraseDictionaryTMExtract.cpp +++ b/moses/src/RuleTable/PhraseDictionaryTMExtract.cpp @@ -33,7 +33,7 @@ #include "StaticData.h" #include "WordsRange.h" #include "UserMessage.h" -#include "CYKPlusParser/ChartRuleLookupManagerMemory.h" +#include "CYKPlusParser/ChartRuleLookupManagerMemoryPerSentence.h" using namespace std; @@ -41,7 +41,7 @@ namespace Moses { PhraseDictionaryTMExtract::PhraseDictionaryTMExtract(size_t numScoreComponents, PhraseDictionaryFeature* feature) - : PhraseDictionarySCFG(numScoreComponents, feature) + : PhraseDictionary(numScoreComponents, feature) { const StaticData &staticData = StaticData::Instance(); CHECK(staticData.ThreadCount() == 1); @@ -99,7 +99,7 @@ namespace Moses const InputType &sentence, const ChartCellCollection &cellCollection) { - return new ChartRuleLookupManagerMemory(sentence, cellCollection, *this); + return new ChartRuleLookupManagerMemoryPerSentence(sentence, cellCollection, *this); } void PhraseDictionaryTMExtract::SortAndPrune() @@ -128,7 +128,7 @@ namespace Moses } - void PhraseDictionaryTMExtract::CleanUp() + void PhraseDictionaryTMExtract::CleanUp(const InputType &source) { m_collection.Clear(); } diff --git a/moses/src/RuleTable/PhraseDictionaryTMExtract.h b/moses/src/RuleTable/PhraseDictionaryTMExtract.h index 8f73fedcd..eceedab3e 100644 --- a/moses/src/RuleTable/PhraseDictionaryTMExtract.h +++ b/moses/src/RuleTable/PhraseDictionaryTMExtract.h @@ -32,7 +32,7 @@ namespace Moses /** Implementation of a SCFG rule table in a trie. Looking up a rule of * length n symbols requires n look-ups to find the TargetPhraseCollection. */ - class PhraseDictionaryTMExtract : public PhraseDictionarySCFG + class PhraseDictionaryTMExtract : public PhraseDictionary { friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryTMExtract&); friend class RuleTableLoader; @@ -41,13 +41,18 @@ namespace Moses PhraseDictionaryTMExtract(size_t numScoreComponents, PhraseDictionaryFeature* feature); - const PhraseDictionaryNodeSCFG &GetRootNode() const { return m_collection; } + const PhraseDictionaryNodeSCFG &GetRootNode(const InputType &source) const { return m_collection; } ChartRuleLookupManager *CreateRuleLookupManager( const InputType &, const ChartCellCollection &); void InitializeForInput(InputType const& source); - void CleanUp(); + void CleanUp(const InputType& source); + + virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const + {} + virtual DecodeType GetDecodeType() const + {} TO_STRING(); @@ -61,7 +66,7 @@ namespace Moses void SortAndPrune(); - //PhraseDictionaryNodeSCFG m_collection; + PhraseDictionaryNodeSCFG m_collection; }; } // namespace Moses