moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-27 03:49:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

Safer way to cleanup LM caches after a given number of processed sentences.

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3197 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
nicolabertoldi 2010-04-24 00:06:49 +00:00
parent 476528c05f
commit cf468273bc
6 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
moses-chart-cmd/src/Main.cpp
View File

@ -145,6 +145,7 @@ int main(int argc, char* argv[])
//cerr << *source << endl;
MosesChart::Manager manager(*source);
StaticData::Instance().InitializeBeforeSentenceProcessing(*source);
manager.ProcessSentence();
assert(!staticData.UseMBR());
@ -201,7 +202,8 @@ int main(int argc, char* argv[])
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
manager.CalcDecoderStatistics();
manager.CalcDecoderStatistics();
StaticData::Instance().CleanUpAfterSentenceProcessing(lineCount);
} // while(ReadInput
delete ioWrapper;

6
moses-chart/src/ChartManager.cpp
View File

@ -44,13 +44,13 @@ Manager::Manager(InputType const& source)
,m_hypoStackColl(source, *this)
,m_transOptColl(source, StaticData::Instance().GetDecodeStepVL(source), m_hypoStackColl)
{
const StaticData &staticData = StaticData::Instance();
staticData.InitializeBeforeSentenceProcessing(source);
// const StaticData &staticData = StaticData::Instance();
// staticData.InitializeBeforeSentenceProcessing(source);
}
Manager::~Manager()
{
StaticData::Instance().CleanUpAfterSentenceProcessing();
// StaticData::Instance().CleanUpAfterSentenceProcessing();
}
void Manager::ProcessSentence()

7
moses-cmd/src/Main.cpp
View File

@ -93,8 +93,7 @@ int main(int argc, char* argv[])
return EXIT_FAILURE;
}
//const StaticData &staticData = StaticData::Instance();
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
const StaticData &staticData = StaticData::Instance();
if (!StaticData::LoadDataStatic(&parameter))
return EXIT_FAILURE;
@ -131,6 +130,7 @@ int main(int argc, char* argv[])
VERBOSE(2,"\nTRANSLATING(" << lineCount << "): " << *source);
Manager manager(*source, staticData.GetSearchAlgorithm());
StaticData::Instance().InitializeBeforeSentenceProcessing(*source);
manager.ProcessSentence();
@ -230,8 +230,7 @@ int main(int argc, char* argv[])
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
manager.CalcDecoderStatistics();
staticData.SetNumberOfSentences(lineCount);
StaticData::Instance().CleanUpAfterSentenceProcessing(lineCount);
}
delete ioWrapper;

6
moses/src/Manager.cpp
View File

@ -57,8 +57,8 @@ Manager::Manager(InputType const& source, SearchAlgorithm searchAlgorithm)
,m_start(clock())
,interrupted_flag(0)
{
const StaticData &staticData = StaticData::Instance();
staticData.InitializeBeforeSentenceProcessing(source);
// const StaticData &staticData = StaticData::Instance();
// staticData.InitializeBeforeSentenceProcessing(source);
}
Manager::~Manager()
@ -66,7 +66,7 @@ Manager::~Manager()
delete m_transOptColl;
delete m_search;
StaticData::Instance().CleanUpAfterSentenceProcessing();
// StaticData::Instance().CleanUpAfterSentenceProcessing();
clock_t end = clock();
float et = (end - m_start);

7
moses/src/StaticData.cpp
View File

@ -1039,7 +1039,8 @@ vector<DecodeGraph*> StaticData::GetDecodeStepVL(const InputType& source) const
}
bool StaticData::LMCacheCleanup() const{
bool StaticData::LMCacheCleanup(size_t m_sentences_done) const{
if (m_sentences_done==-1) return 1;
if (m_lmcache_cleanup_threshold)
if (m_sentences_done % m_lmcache_cleanup_threshold == 0)
return 1;
@ -1047,7 +1048,7 @@ bool StaticData::LMCacheCleanup() const{
}
#include "PhraseDictionary.h"
void StaticData::CleanUpAfterSentenceProcessing() const
void StaticData::CleanUpAfterSentenceProcessing(size_t m_sentences_done) const
{
for(size_t i=0;i<m_phraseDictionary.size();++i)
@ -1061,7 +1062,7 @@ void StaticData::CleanUpAfterSentenceProcessing() const
for(size_t i=0;i<m_generationDictionary.size();++i)
m_generationDictionary[i]->CleanUp();
if (LMCacheCleanup()){
if (LMCacheCleanup(m_sentences_done)){
//something LMs could do after each sentence
LMList::const_iterator iterLM;
for (iterLM = m_languageModel.begin() ; iterLM != m_languageModel.end() ; ++iterLM)

6
moses/src/StaticData.h
View File

@ -164,7 +164,6 @@ protected:
float m_lmbrPRatio; //! decaying factor for ngram thetas - see Tromble et al 08 for more details
float m_lmbrMapWeight; //! Weight given to the map solution. See Kumar et al 09 for details
size_t m_sentences_done; //! number of translations already done
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
bool m_timeout; //! use timeout
@ -477,7 +476,7 @@ public:
SearchAlgorithm GetSearchAlgorithm() const {return m_searchAlgorithm;}
size_t GetNumInputScores() const {return m_numInputScores;}
void InitializeBeforeSentenceProcessing(InputType const&) const;
void CleanUpAfterSentenceProcessing() const;
void CleanUpAfterSentenceProcessing(size_t m_sentences_done=-1) const;
const std::vector<float>& GetAllWeights() const
{
@ -528,8 +527,7 @@ public:
bool UseTimeout() const { return m_timeout; }
size_t GetTimeoutThreshold() const { return m_timeout_threshold; }
void SetNumberOfSentences(size_t v) { m_sentences_done=v; }
bool LMCacheCleanup() const;
bool LMCacheCleanup(size_t m_sentences_done) const;
bool GetOutputSearchGraph() const { return m_outputSearchGraph; }
bool GetOutputSearchGraphExtended() const { return m_outputSearchGraphExtended; }