mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-26 19:37:58 +03:00
ReloadingLM compiles and runs, but gives double the LM score it should
This commit is contained in:
parent
70b20909b1
commit
d58db287c3
@ -92,6 +92,7 @@
|
||||
#endif
|
||||
|
||||
#include "moses/LM/Ken.h"
|
||||
#include "moses/LM/Reloading.h"
|
||||
#ifdef LM_IRST
|
||||
#include "moses/LM/IRST.h"
|
||||
#endif
|
||||
@ -330,7 +331,7 @@ FeatureRegistry::FeatureRegistry()
|
||||
MOSES_FNAME2("OxSourceFactoredLM", SourceOxLM);
|
||||
MOSES_FNAME2("OxTreeLM", OxLM<oxlm::FactoredTreeLM>);
|
||||
#endif
|
||||
|
||||
MOSES_FNAME2("ReloadingLM", ReloadingLanguageModel);
|
||||
Add("KENLM", new KenFactory());
|
||||
}
|
||||
|
||||
|
@ -138,7 +138,7 @@ if $(with-dalm) {
|
||||
|
||||
#Top-level LM library. If you've added a file that doesn't depend on external
|
||||
#libraries, put it here.
|
||||
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Reloading.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
|
||||
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
|
||||
../../lm//kenlm ..//headers $(dependencies) ;
|
||||
|
||||
alias macros : : : : <define>$(lmmacros) ;
|
||||
|
@ -35,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
//#include "moses/Util.h"
|
||||
//#include "moses/StaticData.h"
|
||||
//#include <iostream>
|
||||
|
||||
/*
|
||||
namespace Moses
|
||||
{
|
||||
namespace
|
||||
@ -53,21 +53,14 @@ struct ReloadingLMState : public FFState {
|
||||
};
|
||||
} // namespace
|
||||
|
||||
/** Constructs a new reloading language model. */
|
||||
|
||||
template <class Model> ReloadingLanguageModel<Model>::ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line,file,factorType,lazy)
|
||||
{
|
||||
//
|
||||
// This space intentionally left blank
|
||||
//
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs an empty reloading language model state.
|
||||
*
|
||||
* This state will correspond with a translation hypothesis
|
||||
* where no source words have been translated.
|
||||
*/
|
||||
template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &/*input*/) const
|
||||
template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &input) const
|
||||
{
|
||||
ReloadingLMState *ret = new ReloadingLMState();
|
||||
ret->state = m_ngram->BeginSentenceState();
|
||||
@ -77,14 +70,6 @@ template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypoth
|
||||
|
||||
template <class Model> FFState *ReloadingLanguageModel<Model>::EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const
|
||||
{
|
||||
/*
|
||||
const lm::ngram::State &in_state = static_cast<const ReloadingLMState&>(*ps).state;
|
||||
|
||||
std::auto_ptr<KenLMState> kenlmState(new KenLMState());
|
||||
kenlmState->state = in_state;
|
||||
|
||||
std::auto_ptr<KenLMState> kenlmReturn(LanguageModelKen::EvaluateWhenApplied(hypo, kenlmState, out));
|
||||
*/
|
||||
|
||||
std::auto_ptr<FFState> kenlmState(LanguageModelKen<Model>::EvaluateWhenApplied(hypo, ps, out));
|
||||
const lm::ngram::State &out_state = static_cast<const ReloadingLMState&>(*kenlmState).state;
|
||||
@ -124,3 +109,4 @@ LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &
|
||||
}
|
||||
|
||||
} // namespace Moses
|
||||
*/
|
||||
|
@ -24,38 +24,64 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "moses/LM/Base.h"
|
||||
#include "moses/LM/Ken.h"
|
||||
|
||||
#include "lm/state.hh"
|
||||
|
||||
#include <iostream>
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
//! This will also load. Returns a templated reloading LM.
|
||||
LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
|
||||
|
||||
class FFState;
|
||||
|
||||
/*
|
||||
* An implementation of single factor reloading LM using Kenneth's code.
|
||||
*/
|
||||
template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model>
|
||||
class ReloadingLanguageModel : public LanguageModel
|
||||
{
|
||||
public:
|
||||
ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
|
||||
|
||||
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
||||
ReloadingLanguageModel(const std::string &line) : LanguageModel(line), m_lm(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) {
|
||||
std::cout << "ReloadingLM constructor" << std::endl;
|
||||
std::cout << std::string(line).replace(0,11,"KENLM") << std::endl;
|
||||
}
|
||||
|
||||
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
|
||||
~ReloadingLanguageModel() {
|
||||
delete m_lm;
|
||||
}
|
||||
|
||||
virtual const FFState *EmptyHypothesisState(const InputType &input) const {
|
||||
return m_lm->EmptyHypothesisState(input);
|
||||
}
|
||||
|
||||
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
|
||||
m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount);
|
||||
}
|
||||
|
||||
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
|
||||
return m_lm->EvaluateWhenApplied(hypo, ps, out);
|
||||
}
|
||||
|
||||
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
|
||||
return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator);
|
||||
}
|
||||
|
||||
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
|
||||
return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator);
|
||||
}
|
||||
|
||||
virtual void IncrementalCallback(Incremental::Manager &manager) const {
|
||||
m_lm->IncrementalCallback(manager);
|
||||
}
|
||||
|
||||
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
|
||||
m_lm->ReportHistoryOrder(out, phrase);
|
||||
}
|
||||
|
||||
virtual bool IsUseable(const FactorMask &mask) const {
|
||||
return m_lm->IsUseable(mask);
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
|
||||
// These lines are required to make the parent class's protected members visible to this class
|
||||
using LanguageModelKen<Model>::m_ngram;
|
||||
// using LanguageModelKen<Model>::m_beginSentenceFactor;
|
||||
//using LanguageModelKen<Model>::m_factorType;
|
||||
//using LanguageModelKen<Model>::TranslateID;
|
||||
LanguageModel *m_lm;
|
||||
|
||||
};
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user