moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-26 19:37:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

ReloadingLM compiles and runs, but gives double the LM score it should

Browse Source
This commit is contained in:
Lane Schwartz 2016-01-12 14:36:14 -06:00
parent 70b20909b1
commit d58db287c3
4 changed files with 50 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
moses/FF/Factory.cpp
View File

@ -92,6 +92,7 @@
#endif
#include "moses/LM/Ken.h"
#include "moses/LM/Reloading.h"
#ifdef LM_IRST
#include "moses/LM/IRST.h"
#endif
@ -330,7 +331,7 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME2("OxSourceFactoredLM", SourceOxLM);
MOSES_FNAME2("OxTreeLM", OxLM<oxlm::FactoredTreeLM>);
#endif
MOSES_FNAME2("ReloadingLM", ReloadingLanguageModel);
Add("KENLM", new KenFactory());
}

2
moses/LM/Jamfile
View File

@ -138,7 +138,7 @@ if $(with-dalm) {
#Top-level LM library. If you've added a file that doesn't depend on external
#libraries, put it here.
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Reloading.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
../../lm//kenlm ..//headers $(dependencies) ;
alias macros : : : : <define>$(lmmacros) ;

22
moses/LM/Reloading.cpp
View File

@ -35,7 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//#include "moses/Util.h"
//#include "moses/StaticData.h"
//#include <iostream>
/*
namespace Moses
{
namespace
@ -53,21 +53,14 @@ struct ReloadingLMState : public FFState {
};
} // namespace
/** Constructs a new reloading language model. */
template <class Model> ReloadingLanguageModel<Model>::ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line,file,factorType,lazy)
{
//
// This space intentionally left blank
//
}
/**
* Constructs an empty reloading language model state.
*
* This state will correspond with a translation hypothesis
* where no source words have been translated.
*/
template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &/*input*/) const
template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypothesisState(const InputType &input) const
{
ReloadingLMState *ret = new ReloadingLMState();
ret->state = m_ngram->BeginSentenceState();
@ -77,14 +70,6 @@ template <class Model> const FFState *ReloadingLanguageModel<Model>::EmptyHypoth
template <class Model> FFState *ReloadingLanguageModel<Model>::EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const
{
/*
const lm::ngram::State &in_state = static_cast<const ReloadingLMState&>(*ps).state;
std::auto_ptr<KenLMState> kenlmState(new KenLMState());
kenlmState->state = in_state;
std::auto_ptr<KenLMState> kenlmReturn(LanguageModelKen::EvaluateWhenApplied(hypo, kenlmState, out));
*/
std::auto_ptr<FFState> kenlmState(LanguageModelKen<Model>::EvaluateWhenApplied(hypo, ps, out));
const lm::ngram::State &out_state = static_cast<const ReloadingLMState&>(*kenlmState).state;
@ -124,3 +109,4 @@ LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &
}
} // namespace Moses
*/

60
moses/LM/Reloading.h
View File

@ -24,38 +24,64 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <string>
#include "moses/LM/Base.h"
#include "moses/LM/Ken.h"
#include "lm/state.hh"
#include <iostream>
namespace Moses
{
//! This will also load. Returns a templated reloading LM.
LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
class FFState;
/*
* An implementation of single factor reloading LM using Kenneth's code.
*/
template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model>
class ReloadingLanguageModel : public LanguageModel
{
public:
ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
ReloadingLanguageModel(const std::string &line) : LanguageModel(line), m_lm(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) {
std::cout << "ReloadingLM constructor" << std::endl;
std::cout << std::string(line).replace(0,11,"KENLM") << std::endl;
}
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
~ReloadingLanguageModel() {
delete m_lm;
}
virtual const FFState *EmptyHypothesisState(const InputType &input) const {
return m_lm->EmptyHypothesisState(input);
}
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount);
}
virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
return m_lm->EvaluateWhenApplied(hypo, ps, out);
}
virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator);
}
virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator);
}
virtual void IncrementalCallback(Incremental::Manager &manager) const {
m_lm->IncrementalCallback(manager);
}
virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
m_lm->ReportHistoryOrder(out, phrase);
}
virtual bool IsUseable(const FactorMask &mask) const {
return m_lm->IsUseable(mask);
}
private:
// These lines are required to make the parent class's protected members visible to this class
using LanguageModelKen<Model>::m_ngram;
// using LanguageModelKen<Model>::m_beginSentenceFactor;
//using LanguageModelKen<Model>::m_factorType;
//using LanguageModelKen<Model>::TranslateID;
LanguageModel *m_lm;
};