mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Add InMemoryPerSentenceOnDemandLM
This commit is contained in:
parent
999d6b6371
commit
578e65298f
@ -68,6 +68,7 @@
|
|||||||
#include "moses/FF/SkeletonStatelessFF.h"
|
#include "moses/FF/SkeletonStatelessFF.h"
|
||||||
#include "moses/FF/SkeletonStatefulFF.h"
|
#include "moses/FF/SkeletonStatefulFF.h"
|
||||||
#include "moses/LM/SkeletonLM.h"
|
#include "moses/LM/SkeletonLM.h"
|
||||||
|
#include "moses/LM/InMemoryPerSentenceOnDemandLM.h"
|
||||||
#include "moses/FF/SkeletonTranslationOptionListFeature.h"
|
#include "moses/FF/SkeletonTranslationOptionListFeature.h"
|
||||||
#include "moses/LM/BilingualLM.h"
|
#include "moses/LM/BilingualLM.h"
|
||||||
#include "moses/TranslationModel/SkeletonPT.h"
|
#include "moses/TranslationModel/SkeletonPT.h"
|
||||||
@ -299,6 +300,7 @@ FeatureRegistry::FeatureRegistry()
|
|||||||
MOSES_FNAME(SkeletonStatelessFF);
|
MOSES_FNAME(SkeletonStatelessFF);
|
||||||
MOSES_FNAME(SkeletonStatefulFF);
|
MOSES_FNAME(SkeletonStatefulFF);
|
||||||
MOSES_FNAME(SkeletonLM);
|
MOSES_FNAME(SkeletonLM);
|
||||||
|
MOSES_FNAME(InMemoryPerSentenceOnDemandLM);
|
||||||
MOSES_FNAME(SkeletonTranslationOptionListFeature);
|
MOSES_FNAME(SkeletonTranslationOptionListFeature);
|
||||||
MOSES_FNAME(SkeletonPT);
|
MOSES_FNAME(SkeletonPT);
|
||||||
|
|
||||||
|
@ -61,7 +61,7 @@ void LanguageModelImplementation::ShiftOrPush(std::vector<const Word*> &contextF
|
|||||||
{
|
{
|
||||||
if (contextFactor.size() < GetNGramOrder()) {
|
if (contextFactor.size() < GetNGramOrder()) {
|
||||||
contextFactor.push_back(&word);
|
contextFactor.push_back(&word);
|
||||||
} else {
|
} else if (GetNGramOrder() > 0) {
|
||||||
// shift
|
// shift
|
||||||
for (size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder() - 1 ; currNGramOrder++) {
|
for (size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder() - 1 ; currNGramOrder++) {
|
||||||
contextFactor[currNGramOrder] = contextFactor[currNGramOrder + 1];
|
contextFactor[currNGramOrder] = contextFactor[currNGramOrder + 1];
|
||||||
|
91
moses/LM/InMemoryPerSentenceOnDemandLM.cpp
Normal file
91
moses/LM/InMemoryPerSentenceOnDemandLM.cpp
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
#include <boost/foreach.hpp>
|
||||||
|
#include "InMemoryPerSentenceOnDemandLM.h"
|
||||||
|
#include "moses/FactorCollection.h"
|
||||||
|
#include "moses/Util.h"
|
||||||
|
#include "moses/StaticData.h"
|
||||||
|
#include "moses/TranslationTask.h"
|
||||||
|
#include "moses/ContextScope.h"
|
||||||
|
#include "moses/LM/Ken.h"
|
||||||
|
#include "lm/model.hh"
|
||||||
|
#include "util/mmap.hh"
|
||||||
|
|
||||||
|
#include <cstdio>
|
||||||
|
#include <iostream>
|
||||||
|
#include <fstream>
|
||||||
|
|
||||||
|
using namespace std;
|
||||||
|
|
||||||
|
namespace Moses
|
||||||
|
{
|
||||||
|
InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line), initialized(false)
|
||||||
|
{
|
||||||
|
ReadParameters();
|
||||||
|
}
|
||||||
|
|
||||||
|
InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask) {
|
||||||
|
|
||||||
|
// The context scope object for this translation task
|
||||||
|
// contains a map of translation task-specific data
|
||||||
|
boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
|
||||||
|
|
||||||
|
// The key to the map is this object
|
||||||
|
void const* key = static_cast<void const*>(this);
|
||||||
|
|
||||||
|
// The value stored in the map is a string representing a phrase table
|
||||||
|
boost::shared_ptr<string> value = contextScope->get<string>(key);
|
||||||
|
|
||||||
|
// Create a stream to read the phrase table data
|
||||||
|
stringstream strme(*(value.get()));
|
||||||
|
|
||||||
|
char * nullpointer = (char *) 0;
|
||||||
|
const char * filename = std::tmpnam(nullpointer);
|
||||||
|
ofstream tmp;
|
||||||
|
tmp.open(filename);
|
||||||
|
|
||||||
|
// Read the phrase table data, one line at a time
|
||||||
|
string line;
|
||||||
|
while (getline(strme, line)) {
|
||||||
|
|
||||||
|
tmp << line << "\n";
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
tmp.close();
|
||||||
|
|
||||||
|
LanguageModelKen<lm::ngram::ProbingModel> & lm = GetPerThreadLM();
|
||||||
|
lm.LoadModel("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm", util::POPULATE_OR_READ);
|
||||||
|
|
||||||
|
initialized = true;
|
||||||
|
|
||||||
|
VERBOSE(1, filename);
|
||||||
|
if (initialized) {
|
||||||
|
VERBOSE(1, "\tLM initialized\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// std::remove(filename);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPerThreadLM() const {
|
||||||
|
|
||||||
|
LanguageModelKen<lm::ngram::ProbingModel> *lm;
|
||||||
|
lm = m_perThreadLM.get();
|
||||||
|
if (lm == NULL) {
|
||||||
|
lm = new LanguageModelKen<lm::ngram::ProbingModel>();
|
||||||
|
m_perThreadLM.reset(lm);
|
||||||
|
}
|
||||||
|
assert(lm);
|
||||||
|
return *lm;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
135
moses/LM/InMemoryPerSentenceOnDemandLM.h
Normal file
135
moses/LM/InMemoryPerSentenceOnDemandLM.h
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
// $Id$
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <vector>
|
||||||
|
#include "SingleFactor.h"
|
||||||
|
#include <boost/thread/tss.hpp>
|
||||||
|
#include "lm/model.hh"
|
||||||
|
#include "moses/LM/Ken.h"
|
||||||
|
#include "moses/FF/FFState.h"
|
||||||
|
|
||||||
|
namespace Moses
|
||||||
|
{
|
||||||
|
|
||||||
|
struct InMemoryPerSentenceOnDemandLMState : public FFState {
|
||||||
|
lm::ngram::State state;
|
||||||
|
virtual size_t hash() const {
|
||||||
|
size_t ret = hash_value(state);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
virtual bool operator==(const FFState& o) const {
|
||||||
|
const InMemoryPerSentenceOnDemandLMState &other = static_cast<const InMemoryPerSentenceOnDemandLMState &>(o);
|
||||||
|
bool ret = state == other.state;
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
class InMemoryPerSentenceOnDemandLM : public LanguageModel
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
InMemoryPerSentenceOnDemandLM(const std::string &line);
|
||||||
|
~InMemoryPerSentenceOnDemandLM();
|
||||||
|
|
||||||
|
void InitializeForInput(ttasksptr const& ttask);
|
||||||
|
|
||||||
|
virtual void SetParameter(const std::string& key, const std::string& value) {
|
||||||
|
GetPerThreadLM().SetParameter(key, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
|
||||||
|
if (initialized) {
|
||||||
|
return GetPerThreadLM().EmptyHypothesisState(input);
|
||||||
|
} else {
|
||||||
|
return new InMemoryPerSentenceOnDemandLMState();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
|
||||||
|
if (initialized) {
|
||||||
|
return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out);
|
||||||
|
} else {
|
||||||
|
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
|
||||||
|
if (initialized) {
|
||||||
|
return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator);
|
||||||
|
} else {
|
||||||
|
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
|
||||||
|
if (initialized) {
|
||||||
|
return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator);
|
||||||
|
} else {
|
||||||
|
UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) {
|
||||||
|
GetPerThreadLM().IssueRequestsFor(hypo, input_state);
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void sync() {
|
||||||
|
GetPerThreadLM().sync();
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void SetFFStateIdx(int state_idx) {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().SetFFStateIdx(state_idx);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void IncrementalCallback(Incremental::Manager &manager) const {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().IncrementalCallback(manager);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().ReportHistoryOrder(out, phrase);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
virtual void EvaluateInIsolation(const Phrase &source
|
||||||
|
, const TargetPhrase &targetPhrase
|
||||||
|
, ScoreComponentCollection &scoreBreakdown
|
||||||
|
, ScoreComponentCollection &estimatedScores) const {
|
||||||
|
if (initialized) {
|
||||||
|
GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IsUseable(const FactorMask &mask) const {
|
||||||
|
return GetPerThreadLM().IsUseable(mask);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected:
|
||||||
|
LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const;
|
||||||
|
|
||||||
|
mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM;
|
||||||
|
|
||||||
|
bool initialized;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
}
|
@ -138,7 +138,7 @@ if $(with-dalm) {
|
|||||||
|
|
||||||
#Top-level LM library. If you've added a file that doesn't depend on external
|
#Top-level LM library. If you've added a file that doesn't depend on external
|
||||||
#libraries, put it here.
|
#libraries, put it here.
|
||||||
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
|
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
|
||||||
../../lm//kenlm ..//headers $(dependencies) ;
|
../../lm//kenlm ..//headers $(dependencies) ;
|
||||||
|
|
||||||
alias macros : : : : <define>$(lmmacros) ;
|
alias macros : : : : <define>$(lmmacros) ;
|
||||||
|
@ -105,6 +105,7 @@ template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string
|
|||||||
config.load_method = load_method;
|
config.load_method = load_method;
|
||||||
|
|
||||||
m_ngram.reset(new Model(file.c_str(), config));
|
m_ngram.reset(new Model(file.c_str(), config));
|
||||||
|
VERBOSE(2, "LanguageModelKen " << m_description << " reset to " << file << "\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
|
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
|
||||||
@ -116,6 +117,15 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::stri
|
|||||||
LoadModel(file, load_method);
|
LoadModel(file, load_method);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <class Model> LanguageModelKen<Model>::LanguageModelKen()
|
||||||
|
:LanguageModel("KENLM")
|
||||||
|
,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_))
|
||||||
|
,m_factorType(0)
|
||||||
|
{
|
||||||
|
ReadParameters();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> ©_from)
|
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> ©_from)
|
||||||
:LanguageModel(copy_from.GetArgLine()),
|
:LanguageModel(copy_from.GetArgLine()),
|
||||||
m_ngram(copy_from.m_ngram),
|
m_ngram(copy_from.m_ngram),
|
||||||
|
@ -33,11 +33,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#include "moses/TypeDef.h"
|
#include "moses/TypeDef.h"
|
||||||
#include "moses/Word.h"
|
#include "moses/Word.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
namespace Moses
|
namespace Moses
|
||||||
{
|
{
|
||||||
|
|
||||||
//class LanguageModel;
|
//class LanguageModel;
|
||||||
class FFState;
|
class FFState;
|
||||||
|
class InMemoryPerSentenceOnDemandLM;
|
||||||
|
|
||||||
LanguageModel *ConstructKenLM(const std::string &line);
|
LanguageModel *ConstructKenLM(const std::string &line);
|
||||||
|
|
||||||
@ -67,6 +70,8 @@ public:
|
|||||||
|
|
||||||
virtual bool IsUseable(const FactorMask &mask) const;
|
virtual bool IsUseable(const FactorMask &mask) const;
|
||||||
|
|
||||||
|
friend class InMemoryPerSentenceOnDemandLM;
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
boost::shared_ptr<Model> m_ngram;
|
boost::shared_ptr<Model> m_ngram;
|
||||||
|
|
||||||
@ -84,6 +89,7 @@ protected:
|
|||||||
std::vector<lm::WordIndex> m_lmIdLookup;
|
std::vector<lm::WordIndex> m_lmIdLookup;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
LanguageModelKen();
|
||||||
LanguageModelKen(const LanguageModelKen<Model> ©_from);
|
LanguageModelKen(const LanguageModelKen<Model> ©_from);
|
||||||
|
|
||||||
// Convert last words of hypothesis into vocab ids, returning an end pointer.
|
// Convert last words of hypothesis into vocab ids, returning an end pointer.
|
||||||
|
Loading…
Reference in New Issue
Block a user