moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 13:23:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

add NeuralLM

Browse Source
This commit is contained in:
Hieu Hoang 2013-10-28 23:21:02 +00:00
parent 0af1df9f8b
commit 37896ed854
6 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/other-builds/moses/.cproject
View File

@ -50,7 +50,7 @@
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="HAVE_NPLM"/>
<listOptionValue builtIn="false" value="LM_NPLM"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BIT=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>

8
contrib/other-builds/moses/.project
View File

@ -1427,14 +1427,14 @@
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/MultiFactor.h</locationURI>
</link>
<link>
<name>LM/NeuralLM.cpp</name>
<name>LM/NeuralLMWrapper.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLM.cpp</locationURI>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLMWrapper.cpp</locationURI>
</link>
<link>
<name>LM/NeuralLM.h</name>
<name>LM/NeuralLMWrapper.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLM.h</locationURI>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLMWrapper.h</locationURI>
</link>
<link>
<name>LM/ORLM.cpp</name>

8
moses/FF/Factory.cpp
View File

@ -59,8 +59,8 @@
#include "moses/SyntacticLanguageModel.h"
#endif
#ifdef HAVE_NPLM
#include "moses/LM/NeuralLM.h"
#ifdef LM_NEURAL
#include "moses/LM/NeuralLMWrapper.h"
#endif
#include "util/exception.hh"
@ -176,8 +176,8 @@ FeatureRegistry::FeatureRegistry()
#ifdef LM_RAND
MOSES_FNAME2("RANDLM", LanguageModelRandLM);
#endif
#ifdef HAVE_NPLM
MOSES_FNAME(NeuralLM);
#ifdef LM_NEURAL
MOSES_FNAME2("NeuralLM", NeuralLMWrapper);
#endif
Add("KENLM", new KenFactory());

10
moses/LM/Jamfile
View File

@ -70,6 +70,16 @@ if $(with-ldhtlm) {
lmmacros += LM_LDHT ;
}
#NPLM
local with-nplm = [ option.get "with-nplm" ] ;
if $(with-nplm) {
lib neuralLM : : <search>$(with-nplm)/lib <search>$(with-nplm)/lib64 ;
obj NeuralLMWrapper.o : NeuralLMWrapper.cpp neuralLM ..//headers : <include>$(with-nplm)/src <include>$(with-nplm)/3rdparty/eigen ;
alias nplm : NeuralLMWrapper.o neuralLM : : : <define>LM_NEURAL ;
dependencies += nplm ;
lmmacros += LM_NEURAL ;
}
#ORLM is always compiled but needs special headers
obj ORLM.o : ORLM.cpp ..//headers ../TranslationModel/DynSAInclude//dynsa : : : <include>../TranslationModel/DynSAInclude ;

14
moses/LM/NeuralLM.cpp → moses/LM/NeuralLMWrapper.cpp
View File

@ -1,28 +1,28 @@
#include "moses/StaticData.h"
#include "moses/FactorCollection.h"
#include "NeuralLM.h"
#include "NeuralLMWrapper.h"
#include "neuralLM.h"
#include "model.h"
#include <model.h>
using namespace std;
namespace Moses
{
NeuralLM::NeuralLM(const std::string &line)
NeuralLMWrapper::NeuralLMWrapper(const std::string &line)
:LanguageModelSingleFactor("NeuralLM", line)
{
// This space intentionally left blank
}
NeuralLM::~NeuralLM()
NeuralLMWrapper::~NeuralLMWrapper()
{
delete m_neuralLM;
}
bool NeuralLM::Load(const std::string &filePath, FactorType factorType, size_t nGramOrder)
bool NeuralLMWrapper::Load(const std::string &filePath, FactorType factorType, size_t nGramOrder)
{
TRACE_ERR("Loading NeuralLM " << filePath << endl);
@ -42,7 +42,7 @@ bool NeuralLM::Load(const std::string &filePath, FactorType factorType, size_t n
m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, EOS_);
m_sentenceEndWord[m_factorType] = m_sentenceEnd;
m_neuralLM = new nplm::neuralLM(24234);
m_neuralLM = new nplm::neuralLM();
m_neuralLM->read(m_filePath);
m_neuralLM->set_log_base(10);
@ -51,7 +51,7 @@ bool NeuralLM::Load(const std::string &filePath, FactorType factorType, size_t n
}
LMResult NeuralLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
LMResult NeuralLMWrapper::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
unsigned int hashCode = 0;

6
moses/LM/NeuralLM.h → moses/LM/NeuralLMWrapper.h
View File

@ -11,15 +11,15 @@ namespace Moses
/** Implementation of single factor LM using IRST's code.
*/
class NeuralLM : public LanguageModelSingleFactor
class NeuralLMWrapper : public LanguageModelSingleFactor
{
protected:
nplm::neuralLM *m_neuralLM;
public:
NeuralLM(const std::string &line);
NeuralLMWrapper(const std::string &line);
// NeuralLM(const std::string &line);
~NeuralLM();
~NeuralLMWrapper();
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;