moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-02 17:09:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Revert "put lm caching code in LM"

Browse Source 
This reverts commit 8dcdd4c1b7.
This commit is contained in:
Hieu Hoang 2016-02-18 10:46:19 -05:00
parent fb80a2fc22
commit 4c6af3bb29
4 changed files with 50 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
contrib/other-builds/moses2/LM/KENLM.cpp
View File

@ -102,14 +102,13 @@ void KENLM::Load(System &system)
void KENLM::InitializeForInput(const Manager &mgr) const
{
CacheColl &cache = GetCache();
cache.clear();
mgr.lmCache = &cache;
}
// clean up temporary memory, called after processing each sentence
void KENLM::CleanUpAfterSentenceProcessing(const Manager &mgr) const
{
}
void KENLM::SetParameter(const std::string& key, const std::string& value)
@ -303,35 +302,22 @@ lm::WordIndex *KENLM::LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) co
float KENLM::ScoreAndCache(const Manager &mgr, const lm::ngram::State &in_state, const lm::WordIndex new_word, lm::ngram::State &out_state) const
{
//cerr << "score=";
float score;
//cerr << "score=";
float score;
if (mgr.FindLMCache(in_state, new_word, score, out_state)) {
// found in cache. score & set set in the call
//cerr << "in cache ";
}
else {
//cerr << "not cache ";
score = m_ngram->Score(in_state, new_word, out_state);
mgr.AddLMCache(in_state, new_word, score, out_state);
}
CacheColl &m_lmCache = *((CacheColl*)mgr.lmCache);
LMCacheKey key(in_state, new_word);
CacheColl::iterator iter;
iter = m_lmCache.find(key);
if (iter == m_lmCache.end()) {
score = m_ngram->Score(in_state, new_word, out_state);
//score = m_ngram->Score(in_state, new_word, out_state);
LMCacheValue &val = m_lmCache[key];
val.first = score;
val.second = out_state;
}
else {
const LMCacheValue &val = iter->second;
score = val.first;
out_state = val.second;
}
//score = m_ngram->Score(in_state, new_word, out_state);
//cerr << score << " " << (int) out_state.length << endl;
return score;
}
KENLM::CacheColl &KENLM::GetCache() const
{
return GetThreadSpecificObj(m_cache);
//cerr << score << " " << (int) out_state.length << endl;
return score;
}
}

12
contrib/other-builds/moses2/LM/KENLM.h
View File

@ -9,7 +9,6 @@
#define FF_LM_KENLM_H_
#include <boost/shared_ptr.hpp>
#include <boost/unordered_map.hpp>
#include "../FF/StatefulFeatureFunction.h"
#include "lm/model.hh"
#include "../legacy/Factor.h"
@ -78,12 +77,6 @@ protected:
typedef lm::ngram::ProbingModel Model;
boost::shared_ptr<Model> m_ngram;
std::vector<lm::WordIndex> m_lmIdLookup;
typedef std::pair<lm::ngram::State, lm::WordIndex> LMCacheKey;
typedef std::pair<float, lm::ngram::State> LMCacheValue;
typedef boost::unordered_map<LMCacheKey, LMCacheValue> CacheColl;
mutable boost::thread_specific_ptr<CacheColl> m_cache;
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const;
@ -95,10 +88,9 @@ protected:
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const;
std::vector<lm::WordIndex> m_lmIdLookup;
float ScoreAndCache(const Manager &mgr, const lm::ngram::State &in_state, const lm::WordIndex new_word, lm::ngram::State &out_state) const;
CacheColl &GetCache() const;
};
}

24
contrib/other-builds/moses2/Search/Manager.cpp
View File

@ -226,5 +226,29 @@ void Manager::OutputBest() const
//cerr << endl;
}
void Manager::AddLMCache(const lm::ngram::State &in_state, const lm::WordIndex new_word, float score, const lm::ngram::State &out_state) const
{
LMCacheKey key(in_state, new_word);
LMCacheValue &val = m_lmCache[key];
val.first = score;
val.second = out_state;
}
bool Manager::FindLMCache(const lm::ngram::State &in_state, const lm::WordIndex new_word, float &score, lm::ngram::State &out_state) const
{
LMCacheKey key(in_state, new_word);
boost::unordered_map<LMCacheKey, LMCacheValue>::iterator iter;
iter = m_lmCache.find(key);
if (iter == m_lmCache.end()) {
return false;
}
else {
const LMCacheValue &val = iter->second;
score = val.first;
out_state = val.second;
return true;
}
}
}

9
contrib/other-builds/moses2/Search/Manager.h
View File

@ -69,7 +69,10 @@ public:
void OutputBest() const;
mutable void *lmCache;
//void AddLMCache(const lm::ngram::State &in_state, const lm::WordIndex new_word) const;
bool FindLMCache(const lm::ngram::State &in_state, const lm::WordIndex new_word, float &score, lm::ngram::State &out_state) const;
void AddLMCache(const lm::ngram::State &in_state, const lm::WordIndex new_word, float score, const lm::ngram::State &out_state) const;
protected:
mutable MemPool *m_pool, *m_systemPool;
mutable Recycler<Hypothesis*> *m_hypoRecycle;
@ -84,6 +87,10 @@ protected:
Search *m_search;
typedef std::pair<lm::ngram::State, lm::WordIndex> LMCacheKey;
typedef std::pair<float, lm::ngram::State> LMCacheValue;
mutable boost::unordered_map<LMCacheKey, LMCacheValue> m_lmCache;
// must be run in same thread as Decode()
void Init();