mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-27 03:49:57 +03:00
Safer way to cleanup LM caches after a given number of processed sentences.
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3197 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
476528c05f
commit
cf468273bc
@ -145,6 +145,7 @@ int main(int argc, char* argv[])
|
||||
//cerr << *source << endl;
|
||||
|
||||
MosesChart::Manager manager(*source);
|
||||
StaticData::Instance().InitializeBeforeSentenceProcessing(*source);
|
||||
manager.ProcessSentence();
|
||||
|
||||
assert(!staticData.UseMBR());
|
||||
@ -201,7 +202,8 @@ int main(int argc, char* argv[])
|
||||
|
||||
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
|
||||
|
||||
manager.CalcDecoderStatistics();
|
||||
manager.CalcDecoderStatistics();
|
||||
StaticData::Instance().CleanUpAfterSentenceProcessing(lineCount);
|
||||
} // while(ReadInput
|
||||
|
||||
delete ioWrapper;
|
||||
|
@ -44,13 +44,13 @@ Manager::Manager(InputType const& source)
|
||||
,m_hypoStackColl(source, *this)
|
||||
,m_transOptColl(source, StaticData::Instance().GetDecodeStepVL(source), m_hypoStackColl)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
staticData.InitializeBeforeSentenceProcessing(source);
|
||||
// const StaticData &staticData = StaticData::Instance();
|
||||
// staticData.InitializeBeforeSentenceProcessing(source);
|
||||
}
|
||||
|
||||
Manager::~Manager()
|
||||
{
|
||||
StaticData::Instance().CleanUpAfterSentenceProcessing();
|
||||
// StaticData::Instance().CleanUpAfterSentenceProcessing();
|
||||
}
|
||||
|
||||
void Manager::ProcessSentence()
|
||||
|
@ -93,8 +93,7 @@ int main(int argc, char* argv[])
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
//const StaticData &staticData = StaticData::Instance();
|
||||
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
if (!StaticData::LoadDataStatic(¶meter))
|
||||
return EXIT_FAILURE;
|
||||
|
||||
@ -131,6 +130,7 @@ int main(int argc, char* argv[])
|
||||
VERBOSE(2,"\nTRANSLATING(" << lineCount << "): " << *source);
|
||||
|
||||
Manager manager(*source, staticData.GetSearchAlgorithm());
|
||||
StaticData::Instance().InitializeBeforeSentenceProcessing(*source);
|
||||
manager.ProcessSentence();
|
||||
|
||||
|
||||
@ -230,8 +230,7 @@ int main(int argc, char* argv[])
|
||||
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
|
||||
|
||||
manager.CalcDecoderStatistics();
|
||||
|
||||
staticData.SetNumberOfSentences(lineCount);
|
||||
StaticData::Instance().CleanUpAfterSentenceProcessing(lineCount);
|
||||
}
|
||||
|
||||
delete ioWrapper;
|
||||
|
@ -57,8 +57,8 @@ Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
|
||||
,m_start(clock())
|
||||
,interrupted_flag(0)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
staticData.InitializeBeforeSentenceProcessing(source);
|
||||
// const StaticData &staticData = StaticData::Instance();
|
||||
// staticData.InitializeBeforeSentenceProcessing(source);
|
||||
}
|
||||
|
||||
Manager::~Manager()
|
||||
@ -66,7 +66,7 @@ Manager::~Manager()
|
||||
delete m_transOptColl;
|
||||
delete m_search;
|
||||
|
||||
StaticData::Instance().CleanUpAfterSentenceProcessing();
|
||||
// StaticData::Instance().CleanUpAfterSentenceProcessing();
|
||||
|
||||
clock_t end = clock();
|
||||
float et = (end - m_start);
|
||||
|
@ -1039,7 +1039,8 @@ vector<DecodeGraph*> StaticData::GetDecodeStepVL(const InputType& source) const
|
||||
}
|
||||
|
||||
|
||||
bool StaticData::LMCacheCleanup() const{
|
||||
bool StaticData::LMCacheCleanup(size_t m_sentences_done) const{
|
||||
if (m_sentences_done==-1) return 1;
|
||||
if (m_lmcache_cleanup_threshold)
|
||||
if (m_sentences_done % m_lmcache_cleanup_threshold == 0)
|
||||
return 1;
|
||||
@ -1047,7 +1048,7 @@ bool StaticData::LMCacheCleanup() const{
|
||||
}
|
||||
|
||||
#include "PhraseDictionary.h"
|
||||
void StaticData::CleanUpAfterSentenceProcessing() const
|
||||
void StaticData::CleanUpAfterSentenceProcessing(size_t m_sentences_done) const
|
||||
{
|
||||
|
||||
for(size_t i=0;i<m_phraseDictionary.size();++i)
|
||||
@ -1061,7 +1062,7 @@ void StaticData::CleanUpAfterSentenceProcessing() const
|
||||
for(size_t i=0;i<m_generationDictionary.size();++i)
|
||||
m_generationDictionary[i]->CleanUp();
|
||||
|
||||
if (LMCacheCleanup()){
|
||||
if (LMCacheCleanup(m_sentences_done)){
|
||||
//something LMs could do after each sentence
|
||||
LMList::const_iterator iterLM;
|
||||
for (iterLM = m_languageModel.begin() ; iterLM != m_languageModel.end() ; ++iterLM)
|
||||
|
@ -164,7 +164,6 @@ protected:
|
||||
float m_lmbrPRatio; //! decaying factor for ngram thetas - see Tromble et al 08 for more details
|
||||
float m_lmbrMapWeight; //! Weight given to the map solution. See Kumar et al 09 for details
|
||||
|
||||
size_t m_sentences_done; //! number of translations already done
|
||||
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
|
||||
|
||||
bool m_timeout; //! use timeout
|
||||
@ -477,7 +476,7 @@ public:
|
||||
SearchAlgorithm GetSearchAlgorithm() const {return m_searchAlgorithm;}
|
||||
size_t GetNumInputScores() const {return m_numInputScores;}
|
||||
void InitializeBeforeSentenceProcessing(InputType const&) const;
|
||||
void CleanUpAfterSentenceProcessing() const;
|
||||
void CleanUpAfterSentenceProcessing(size_t m_sentences_done=-1) const;
|
||||
|
||||
const std::vector<float>& GetAllWeights() const
|
||||
{
|
||||
@ -528,8 +527,7 @@ public:
|
||||
bool UseTimeout() const { return m_timeout; }
|
||||
size_t GetTimeoutThreshold() const { return m_timeout_threshold; }
|
||||
|
||||
void SetNumberOfSentences(size_t v) { m_sentences_done=v; }
|
||||
bool LMCacheCleanup() const;
|
||||
bool LMCacheCleanup(size_t m_sentences_done) const;
|
||||
|
||||
bool GetOutputSearchGraph() const { return m_outputSearchGraph; }
|
||||
bool GetOutputSearchGraphExtended() const { return m_outputSearchGraphExtended; }
|
||||
|
Loading…
Reference in New Issue
Block a user