moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-28 14:32:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Hieu Hoang 2014-11-24 14:11:23 +00:00
commit a2967ab206
3 changed files with 50 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
moses/LM/oxlm/OxLM.cpp
View File

@ -37,6 +37,11 @@ OxLM<Model>::OxLM(const string &line)
template<class Model>
OxLM<Model>::~OxLM() {
if (persistentCache) {
if (cache.get()) {
string cache_file = m_filePath + ".phrases.cache.bin";
savePersistentCache(cache_file);
}
double cache_hit_ratio = 100.0 * cacheHits / totalHits;
cerr << "Cache hit ratio: " << cache_hit_ratio << endl;
}
@ -91,6 +96,8 @@ LMResult OxLM<Model>::GetValue(
const vector<const Word*> &contextFactor, State* finalState) const {
if (!cache.get()) {
cache.reset(new QueryCache());
string cache_file = m_filePath + ".phrases.cache.bin";
loadPersistentCache(cache_file);
}
vector<int> context;
@ -136,6 +143,30 @@ LMResult OxLM<Model>::GetValue(
return ret;
}
template<class Model>
void OxLM<Model>::loadPersistentCache(const string& cache_file) const {
if (boost::filesystem::exists(cache_file)) {
ifstream f(cache_file);
boost::archive::binary_iarchive iar(f);
cerr << "Loading n-gram probability cache from " << cache_file << endl;
iar >> *cache;
cerr << "Done loading " << cache->size()
<< " n-gram probabilities..." << endl;
} else {
cerr << "Cache file not found" << endl;
}
}
template<class Model>
void OxLM<Model>::savePersistentCache(const string& cache_file) const {
ofstream f(cache_file);
boost::archive::binary_oarchive oar(f);
cerr << "Saving persistent cache to " << cache_file << endl;
oar << *cache;
cerr << "Done saving " << cache->size()
<< " n-gram probabilities..." << endl;
}
template<class Model>
void OxLM<Model>::InitializeForInput(const InputType& source) {
LanguageModelSingleFactor::InitializeForInput(source);
@ -146,17 +177,8 @@ void OxLM<Model>::InitializeForInput(const InputType& source) {
}
int sentence_id = source.GetTranslationId();
string cacheFile = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
if (boost::filesystem::exists(cacheFile)) {
ifstream f(cacheFile);
boost::archive::binary_iarchive iar(f);
cerr << "Loading n-gram probability cache from " << cacheFile << endl;
iar >> *cache;
cerr << "Done loading " << cache->size()
<< " n-gram probabilities..." << endl;
} else {
cerr << "Cache file not found!" << endl;
}
string cache_file = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
loadPersistentCache(cache_file);
}
}
@ -167,13 +189,8 @@ void OxLM<Model>::CleanUpAfterSentenceProcessing(const InputType& source) {
if (persistentCache) {
int sentence_id = source.GetTranslationId();
string cacheFile = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
ofstream f(cacheFile);
boost::archive::binary_oarchive oar(f);
cerr << "Saving persistent cache to " << cacheFile << endl;
oar << *cache;
cerr << "Done saving " << cache->size()
<< " n-gram probabilities..." << endl;
string cache_file = m_filePath + "." + to_string(sentence_id) + ".cache.bin";
savePersistentCache(cache_file);
cache->clear();
}

4
moses/LM/oxlm/OxLM.h
View File

@ -35,6 +35,10 @@ class OxLM : public LanguageModelSingleFactor {
private:
double GetScore(int word, const vector<int>& context) const;
void loadPersistentCache(const string& cache_file) const;
void savePersistentCache(const string& cache_file) const;
protected:
Model model;
boost::shared_ptr<OxLMMapper> mapper;

18
scripts/training/train-model.perl
View File

@ -2115,12 +2115,16 @@ sub create_ini {
my $path = `pwd`; chop($path);
$fn = $path."/".$fn;
}
$type = 0 unless $type;
my $type_name = "UnknownLM";
$type_name = "SRILM" if $type == 0;
$type_name = "IRSTLM" if $type == 1;
$type_name = "KENLM lazyken=0" if $type == 8;
$type_name = "KENLM lazyken=1" if $type == 9;
$type = "SRILM" unless defined $type; # default to SRILM if no type given
if ($type =~ /^\d+$/) {
# backwards compatibility if the type is given not as string but as a number
$type = "SRILM" if $type == 0;
$type = "IRSTLM" if $type == 1;
$type = "KENLM lazyken=0" if $type == 8;
$type = "KENLM lazyken=1" if $type == 9;
die "Unknown numeric LM type given: $type" if $type =~ /^\d+$/;
}
my $lm_oov_prob = 0.1;
@ -2129,7 +2133,7 @@ sub create_ini {
$_LMODEL_OOV_FEATURE = "yes";
}
$feature_spec .= "$type_name name=LM$i factor=$f path=$fn order=$o\n";
$feature_spec .= "$type name=LM$i factor=$f path=$fn order=$o\n";
$weight_spec .= "LM$i= 0.5".($_LMODEL_OOV_FEATURE?" $lm_oov_prob":"")."\n";
$i++;
}