moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-17 07:20:48 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed a subtle possibly thread-related thread

Browse Source
This commit is contained in:
Lane Schwartz 2017-01-05 15:48:30 -06:00
parent a18b6676b1
commit 1f744ecd9b
2 changed files with 63 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
moses/LM/InMemoryPerSentenceOnDemandLM.cpp
View File

@ -17,7 +17,7 @@ using namespace std;
namespace Moses
{
InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line), initialized(false)
InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line), m_factorType(0)
{
ReadParameters();
}
@ -25,7 +25,7 @@ namespace Moses
InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM()
{
}
void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask) {
// The context scope object for this translation task
@ -56,15 +56,11 @@ void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask) {
tmp.close();
LanguageModelKen<lm::ngram::ProbingModel> & lm = GetPerThreadLM();
lm.LoadModel("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm", util::POPULATE_OR_READ);
// m_tmpFilename.reset(new std::string("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm"));
m_tmpFilename.reset(new std::string(filename));
initialized = true;
VERBOSE(1, filename);
if (initialized) {
VERBOSE(1, "\tLM initialized\n");
}
//LanguageModelKen<lm::ngram::ProbingModel> & lm =
GetPerThreadLM();
// std::remove(filename);
@ -76,9 +72,21 @@ LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPer
lm = m_perThreadLM.get();
if (lm == NULL) {
lm = new LanguageModelKen<lm::ngram::ProbingModel>();
string* filename = m_tmpFilename.get();
if (filename == NULL) {
UTIL_THROW(util::Exception, "Can't get a thread-specific LM because no temporary filename has been set for this thread\n");
} else {
lm->LoadModel(*filename, util::POPULATE_OR_READ);
}
VERBOSE(1, filename);
VERBOSE(1, "\tLM initialized\n");
m_perThreadLM.reset(lm);
}
assert(lm);
return *lm;
}

59
moses/LM/InMemoryPerSentenceOnDemandLM.h
View File

@ -1,6 +1,7 @@
// $Id$
#pragma once
#include <string>
#include <vector>
#include "SingleFactor.h"
#include <boost/thread/tss.hpp>
@ -38,7 +39,7 @@ public:
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
if (initialized) {
if (isInitialized()) {
return GetPerThreadLM().EmptyHypothesisState(input);
} else {
return new InMemoryPerSentenceOnDemandLMState();
@ -46,7 +47,7 @@ public:
}
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
if (initialized) {
if (isInitialized()) {
return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out);
} else {
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
@ -54,7 +55,7 @@ public:
}
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
if (initialized) {
if (isInitialized()) {
return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator);
} else {
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
@ -62,7 +63,7 @@ public:
}
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
if (initialized) {
if (isInitialized()) {
return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator);
} else {
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
@ -71,40 +72,58 @@ public:
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScore called prior to being initialized");
}
}
virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScoreFromCache called prior to being initialized");
}
}
virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) {
GetPerThreadLM().IssueRequestsFor(hypo, input_state);
if (isInitialized()) {
GetPerThreadLM().IssueRequestsFor(hypo, input_state);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IssueRequestsFor called prior to being initialized");
}
}
virtual void sync() {
GetPerThreadLM().sync();
if (isInitialized()) {
GetPerThreadLM().sync();
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::sync called prior to being initialized");
}
}
virtual void SetFFStateIdx(int state_idx) {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().SetFFStateIdx(state_idx);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::SetFFStateIdx called prior to being initialized");
}
}
virtual void IncrementalCallback(Incremental::Manager &manager) const {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().IncrementalCallback(manager);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IncrementalCallback called prior to being initialized");
}
}
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().ReportHistoryOrder(out, phrase);
} else {
UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::ReportHistoryOrder called prior to being initialized");
}
}
@ -112,13 +131,16 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedScores) const {
if (initialized) {
if (isInitialized()) {
GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores);
} else {
// UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::EvaluateInIsolation called prior to being initialized");
}
}
bool IsUseable(const FactorMask &mask) const {
return GetPerThreadLM().IsUseable(mask);
bool ret = mask[m_factorType];
return ret;
}
@ -126,8 +148,17 @@ protected:
LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const;
mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM;
mutable boost::thread_specific_ptr<std::string> m_tmpFilename;
bool initialized;
FactorType m_factorType;
bool isInitialized() const {
if (m_tmpFilename.get() == NULL) {
return false;
} else {
return true;
}
}
};