moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 13:23:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

make fuzzy match pt threadable by marking it as non-threadable! TODO - do it properly

Browse Source
This commit is contained in:
Hieu Hoang 2012-11-23 18:52:06 +00:00
parent cb95548871
commit 7b36e11276
5 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
moses/FactorCollection.cpp
View File

@ -43,7 +43,7 @@ const Factor *FactorCollection::AddFactor(const StringPiece &factorString)
FactorFriend to_ins; FactorFriend to_ins;
to_ins.in.m_string.assign(factorString.data(), factorString.size()); to_ins.in.m_string.assign(factorString.data(), factorString.size());
#endif // BOOST_VERSION #endif // BOOST_VERSION
{ { // read=lock scope
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock); boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#if BOOST_VERSION >= 104200 #if BOOST_VERSION >= 104200
// If this line doesn't compile, upgrade your Boost. // If this line doesn't compile, upgrade your Boost.

4
moses/PhraseDictionary.cpp
View File

@ -77,10 +77,10 @@ PhraseDictionaryFeature::PhraseDictionaryFeature
m_sparsePhraseDictionaryFeature(spdf) m_sparsePhraseDictionaryFeature(spdf)
{ {
if (implementation == Memory || implementation == SCFG || implementation == SuffixArray || if (implementation == Memory || implementation == SCFG || implementation == SuffixArray ||
implementation==Compact ) { implementation==Compact) { // || implementation==FuzzyMatch ) {
m_useThreadSafePhraseDictionary = true; m_useThreadSafePhraseDictionary = true;
if (implementation == SuffixArray) { if (implementation == SuffixArray) {
cerr << "Warning: implementation holds chached weights!" << endl; cerr << "Warning: implementation holds cached weights!" << endl;
exit(1); exit(1);
} }
} else { } else {

1
moses/fuzzy-match/FuzzyMatchWrapper.h
View File

@ -29,7 +29,6 @@ public:
protected: protected:
// tm-mt // tm-mt
//std::vector< std::vector< tmmt::WORD_ID > > source;
std::vector< std::vector< tmmt::SentenceAlignment > > targetAndAlignment; std::vector< std::vector< tmmt::SentenceAlignment > > targetAndAlignment;
tmmt::SuffixArray *suffixArray; tmmt::SuffixArray *suffixArray;
std::map< WORD_ID,std::vector< int > > single_word_index; std::map< WORD_ID,std::vector< int > > single_word_index;

19
moses/fuzzy-match/Vocabulary.cpp
View File

@ -1,5 +1,8 @@
// $Id: Vocabulary.cpp 1565 2008-02-22 14:42:01Z bojar $ // $Id: Vocabulary.cpp 1565 2008-02-22 14:42:01Z bojar $
#include "Vocabulary.h" #include "Vocabulary.h"
#ifdef WITH_THREADS
#include <boost/thread/locks.hpp>
#endif
using namespace std; using namespace std;
@ -30,11 +33,17 @@ vector<WORD_ID> Vocabulary::Tokenize( const char input[] ) {
} }
WORD_ID Vocabulary::StoreIfNew( const WORD& word ) { WORD_ID Vocabulary::StoreIfNew( const WORD& word ) {
map<WORD, WORD_ID>::iterator i = lookup.find( word );
if( i != lookup.end() )
return i->second;
{ // read=lock scope
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
map<WORD, WORD_ID>::iterator i = lookup.find( word );
if( i != lookup.end() )
return i->second;
}
boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
WORD_ID id = vocab.size(); WORD_ID id = vocab.size();
vocab.push_back( word ); vocab.push_back( word );
lookup[ word ] = id; lookup[ word ] = id;
@ -42,6 +51,8 @@ WORD_ID Vocabulary::StoreIfNew( const WORD& word ) {
} }
WORD_ID Vocabulary::GetWordID( const WORD &word ) { WORD_ID Vocabulary::GetWordID( const WORD &word ) {
boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
map<WORD, WORD_ID>::iterator i = lookup.find( word ); map<WORD, WORD_ID>::iterator i = lookup.find( word );
if( i == lookup.end() ) if( i == lookup.end() )
return 0; return 0;

12
moses/fuzzy-match/Vocabulary.h
View File

@ -11,6 +11,10 @@
#include <map> #include <map>
#include <cmath> #include <cmath>
#ifdef WITH_THREADS
#include <boost/thread/shared_mutex.hpp>
#endif
namespace tmmt namespace tmmt
{ {
@ -38,6 +42,14 @@ class Vocabulary {
WORD_ID GetWordID( const WORD& ); WORD_ID GetWordID( const WORD& );
std::vector<WORD_ID> Tokenize( const char[] ); std::vector<WORD_ID> Tokenize( const char[] );
inline WORD &GetWord( WORD_ID id ) const { WORD &i = (WORD&) vocab[ id ]; return i; } inline WORD &GetWord( WORD_ID id ) const { WORD &i = (WORD&) vocab[ id ]; return i; }
protected:
#ifdef WITH_THREADS
//reader-writer lock
mutable boost::shared_mutex m_accessLock;
#endif
}; };
} }