moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-28 14:32:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Hieu Hoang 2014-11-24 14:11:23 +00:00
commit a2967ab206
3 changed files with 50 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
moses/LM/oxlm/OxLM.cpp
View File

@ -37,6 +37,11 @@ OxLM<Model>::OxLM(const string &line)
template<class Model> template<class Model>
OxLM<Model>::~OxLM() { OxLM<Model>::~OxLM() {
if (persistentCache) { if (persistentCache) {
if (cache.get()) {
string cache_file = m_filePath + ".phrases.cache.bin";
savePersistentCache(cache_file);
}
double cache_hit_ratio = 100.0 * cacheHits / totalHits; double cache_hit_ratio = 100.0 * cacheHits / totalHits;
cerr << "Cache hit ratio: " << cache_hit_ratio << endl; cerr << "Cache hit ratio: " << cache_hit_ratio << endl;
} }
@ -91,6 +96,8 @@ LMResult OxLM<Model>::GetValue(
const vector<const Word*> &contextFactor, State* finalState) const { const vector<const Word*> &contextFactor, State* finalState) const {
if (!cache.get()) { if (!cache.get()) {
cache.reset(new QueryCache()); cache.reset(new QueryCache());
string cache_file = m_filePath + ".phrases.cache.bin";
loadPersistentCache(cache_file);
} }
vector<int> context; vector<int> context;
@ -136,6 +143,30 @@ LMResult OxLM<Model>::GetValue(
return ret; return ret;
} }
template<class Model>
void OxLM<Model>::loadPersistentCache(const string& cache_file) const {
if (boost::filesystem::exists(cache_file)) {
ifstream f(cache_file);
boost::archive::binary_iarchive iar(f);
cerr << "Loading n-gram probability cache from " << cache_file << endl;
iar >> *cache;
cerr << "Done loading " << cache->size()
<< " n-gram probabilities..." << endl;
} else {
cerr << "Cache file not found" << endl;
}
}
template<class Model>
void OxLM<Model>::savePersistentCache(const string& cache_file) const {
ofstream f(cache_file);
boost::archive::binary_oarchive oar(f);
cerr << "Saving persistent cache to " << cache_file << endl;
oar << *cache;
cerr << "Done saving " << cache->size()
<< " n-gram probabilities..." << endl;
}
template<class Model> template<class Model>
void OxLM<Model>::InitializeForInput(const InputType& source) { void OxLM<Model>::InitializeForInput(const InputType& source) {
LanguageModelSingleFactor::InitializeForInput(source); LanguageModelSingleFactor::InitializeForInput(source);
@ -146,17 +177,8 @@ void OxLM<Model>::InitializeForInput(const InputType& source) {
} }
int sentence_id = source.GetTranslationId(); int sentence_id = source.GetTranslationId();
string cacheFile = m_filePath + "." + to_string(sentence_id) + ".cache.bin"; string cache_file = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
if (boost::filesystem::exists(cacheFile)) { loadPersistentCache(cache_file);
ifstream f(cacheFile);
boost::archive::binary_iarchive iar(f);
cerr << "Loading n-gram probability cache from " << cacheFile << endl;
iar >> *cache;
cerr << "Done loading " << cache->size()
<< " n-gram probabilities..." << endl;
} else {
cerr << "Cache file not found!" << endl;
}
} }
} }
@ -167,13 +189,8 @@ void OxLM<Model>::CleanUpAfterSentenceProcessing(const InputType& source) {
if (persistentCache) { if (persistentCache) {
int sentence_id = source.GetTranslationId(); int sentence_id = source.GetTranslationId();
string cacheFile = m_filePath + "." + to_string(sentence_id) + ".cache.bin"; string cache_file = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
ofstream f(cacheFile); savePersistentCache(cache_file);
boost::archive::binary_oarchive oar(f);
cerr << "Saving persistent cache to " << cacheFile << endl;
oar << *cache;
cerr << "Done saving " << cache->size()
<< " n-gram probabilities..." << endl;
cache->clear(); cache->clear();
} }

4
moses/LM/oxlm/OxLM.h
View File

@ -35,6 +35,10 @@ class OxLM : public LanguageModelSingleFactor {
private: private:
double GetScore(int word, const vector<int>& context) const; double GetScore(int word, const vector<int>& context) const;
void loadPersistentCache(const string& cache_file) const;
void savePersistentCache(const string& cache_file) const;
protected: protected:
Model model; Model model;
boost::shared_ptr<OxLMMapper> mapper; boost::shared_ptr<OxLMMapper> mapper;

18
scripts/training/train-model.perl
View File

@ -2115,12 +2115,16 @@ sub create_ini {
my $path = `pwd`; chop($path); my $path = `pwd`; chop($path);
$fn = $path."/".$fn; $fn = $path."/".$fn;
} }
$type = 0 unless $type; $type = "SRILM" unless defined $type; # default to SRILM if no type given
my $type_name = "UnknownLM";
$type_name = "SRILM" if $type == 0; if ($type =~ /^\d+$/) {
$type_name = "IRSTLM" if $type == 1; # backwards compatibility if the type is given not as string but as a number
$type_name = "KENLM lazyken=0" if $type == 8; $type = "SRILM" if $type == 0;
$type_name = "KENLM lazyken=1" if $type == 9; $type = "IRSTLM" if $type == 1;
$type = "KENLM lazyken=0" if $type == 8;
$type = "KENLM lazyken=1" if $type == 9;
die "Unknown numeric LM type given: $type" if $type =~ /^\d+$/;
}
my $lm_oov_prob = 0.1; my $lm_oov_prob = 0.1;
@ -2129,7 +2133,7 @@ sub create_ini {
$_LMODEL_OOV_FEATURE = "yes"; $_LMODEL_OOV_FEATURE = "yes";
} }
$feature_spec .= "$type_name name=LM$i factor=$f path=$fn order=$o\n"; $feature_spec .= "$type name=LM$i factor=$f path=$fn order=$o\n";
$weight_spec .= "LM$i= 0.5".($_LMODEL_OOV_FEATURE?" $lm_oov_prob":"")."\n"; $weight_spec .= "LM$i= 0.5".($_LMODEL_OOV_FEATURE?" $lm_oov_prob":"")."\n";
$i++; $i++;
} }