moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-25 12:52:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

move caching code to PhraseDictionary. Available to all phrase tables

Browse Source
This commit is contained in:
Hieu Hoang 2013-08-15 20:50:22 +01:00
parent 841ce108a5
commit aa8b8cdc07
4 changed files with 56 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
moses/TranslationModel/PhraseDictionary.cpp
View File

@ -35,10 +35,48 @@ namespace Moses
PhraseDictionary::PhraseDictionary(const std::string &description, const std::string &line)
:DecodeFeature(description, line)
,m_tableLimit(20) // default
,m_useCache(666)
{
}
const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollection(const Phrase& src) const
{
const TargetPhraseCollection *ret;
if (m_useCache) {
size_t hash = hash_value(src);
std::map<size_t, const TargetPhraseCollection*>::const_iterator iter;
{ // scope of read lock
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
iter = m_cache.find(hash);
}
if (iter == m_cache.end()) {
ret = GetTargetPhraseCollectionNonCache(src);
if (ret) {
ret = new TargetPhraseCollection(*ret);
}
#ifdef WITH_THREADS
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
m_cache[hash] = ret;
}
else {
ret = iter->second;
}
}
else {
ret = GetTargetPhraseCollectionNonCache(src);
}
return ret;
}
const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionNonCache(const Phrase& src) const
{
UTIL_THROW(util::Exception, "Legacy method not implemented");
}
@ -54,7 +92,9 @@ GetTargetPhraseCollectionLegacy(InputType const& src,WordsRange const& range) co
void PhraseDictionary::SetParameter(const std::string& key, const std::string& value)
{
if (key == "path") {
if (key == "use-cache") {
m_useCache = Scan<int>(value);
} else if (key == "path") {
m_filePath = value;
} else if (key == "table-limit") {
m_tableLimit = Scan<size_t>(value);

12
moses/TranslationModel/PhraseDictionary.h
View File

@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifdef WITH_THREADS
#include <boost/thread/tss.hpp>
#include <boost/thread/shared_mutex.hpp>
#endif
#include "moses/Phrase.h"
@ -112,6 +113,17 @@ protected:
// MUST be called at the start of Load()
void SetFeaturesToApply();
// cache
int m_useCache; // 666=not yet set, otherwise act like a bool
mutable std::map<size_t, const TargetPhraseCollection*> m_cache;
#ifdef WITH_THREADS
//reader-writer lock
mutable boost::shared_mutex m_accessLock;
#endif
virtual const TargetPhraseCollection *GetTargetPhraseCollectionNonCache(const Phrase& src) const;
};
}

45
moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
View File

@ -28,7 +28,6 @@ namespace Moses
PhraseDictionaryTreeAdaptor::
PhraseDictionaryTreeAdaptor(const std::string &line)
: PhraseDictionary("PhraseDictionaryBinary", line)
, m_useCache(true)
{
ReadParameters();
}
@ -47,15 +46,6 @@ void PhraseDictionaryTreeAdaptor::Load()
SetFeaturesToApply();
}
void PhraseDictionaryTreeAdaptor::SetParameter(const std::string& key, const std::string& value)
{
if (key == "use-cache") {
m_useCache = Scan<bool>(value);
} else {
PhraseDictionary::SetParameter(key, value);
}
}
void PhraseDictionaryTreeAdaptor::InitializeForInput(InputType const& source)
{
const StaticData &staticData = StaticData::Instance();
@ -88,40 +78,9 @@ void PhraseDictionaryTreeAdaptor::CleanUpAfterSentenceProcessing(InputType const
}
TargetPhraseCollection const*
PhraseDictionaryTreeAdaptor::GetTargetPhraseCollection(Phrase const &src) const
PhraseDictionaryTreeAdaptor::GetTargetPhraseCollectionNonCache(Phrase const &src) const
{
const TargetPhraseCollection *ret;
if (m_useCache) {
size_t hash = hash_value(src);
std::map<size_t, const TargetPhraseCollection*>::const_iterator iter;
{ // scope of read lock
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
iter = m_cache.find(hash);
}
if (iter == m_cache.end()) {
ret = GetImplementation().GetTargetPhraseCollection(src);
if (ret) {
ret = new TargetPhraseCollection(*ret);
}
#ifdef WITH_THREADS
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
m_cache[hash] = ret;
}
else {
ret = iter->second;
}
}
else {
ret = GetImplementation().GetTargetPhraseCollection(src);
}
const TargetPhraseCollection *ret = GetImplementation().GetTargetPhraseCollection(src);
return ret;
}

13
moses/TranslationModel/PhraseDictionaryTreeAdaptor.h
View File

@ -11,7 +11,6 @@
#ifdef WITH_THREADS
#include <boost/thread/tss.hpp>
#include <boost/thread/shared_mutex.hpp>
#else
#include <boost/scoped_ptr.hpp>
#endif
@ -45,21 +44,11 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionary
PDTAimp& GetImplementation();
const PDTAimp& GetImplementation() const;
// cache
bool m_useCache;
mutable std::map<size_t, const TargetPhraseCollection*> m_cache;
#ifdef WITH_THREADS
//reader-writer lock
mutable boost::shared_mutex m_accessLock;
#endif
public:
PhraseDictionaryTreeAdaptor(const std::string &line);
virtual ~PhraseDictionaryTreeAdaptor();
void Load();
void SetParameter(const std::string& key, const std::string& value);
// enable/disable caching
// you enable caching if you request the target candidates for a source phrase multiple times
// if you do caching somewhere else, disable it
@ -71,7 +60,7 @@ public:
// get translation candidates for a given source phrase
// returns null pointer if nothing found
TargetPhraseCollection const* GetTargetPhraseCollection(Phrase const &src) const;
TargetPhraseCollection const* GetTargetPhraseCollectionNonCache(Phrase const &src) const;
void InitializeForInput(InputType const& source);
void CleanUpAfterSentenceProcessing(InputType const& source);