moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-28 14:32:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Hieu Hoang 2013-10-29 12:54:55 +00:00
commit 5eb95fc4a6
8 changed files with 168 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/other-builds/moses-chart-cmd/.cproject
View File

@ -48,6 +48,7 @@
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.755343734" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.816413868" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
<option id="gnu.cpp.link.option.paths.330225535" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../nplm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../cmph/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib&quot;"/>
@ -65,6 +66,7 @@
<listOptionValue builtIn="false" value="/opt/local/lib"/>
</option>
<option id="gnu.cpp.link.option.libs.1177721357" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="neuralLM"/>
<listOptionValue builtIn="false" value="dstruct"/>
<listOptionValue builtIn="false" value="flm"/>
<listOptionValue builtIn="false" value="oolm"/>

2
contrib/other-builds/moses-cmd/.cproject
View File

@ -47,6 +47,7 @@
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.1546774818" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
<option id="gnu.cpp.link.option.paths.523170942" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../irstlm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../nplm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../randlm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../cmph/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
@ -61,6 +62,7 @@
<listOptionValue builtIn="false" value="/opt/local/lib"/>
</option>
<option id="gnu.cpp.link.option.libs.998577284" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="neuralLM"/>
<listOptionValue builtIn="false" value="dstruct"/>
<listOptionValue builtIn="false" value="flm"/>
<listOptionValue builtIn="false" value="oolm"/>

6
contrib/other-builds/moses/.cproject
View File

@ -34,6 +34,8 @@
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/include&quot;"/>
<listOptionValue builtIn="false" value="/opt/local/include/"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../irstlm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../nplm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../eigen&quot;"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../srilm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../randlm/include/RandLM&quot;"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../"/>
@ -48,6 +50,7 @@
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_NPLM"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BIT=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>
@ -71,9 +74,6 @@
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.511477442" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1742823107" name="ChartTranslationOption.cpp" rcbsApplicability="disable" resourcePath="ChartTranslationOption.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327"/>
</fileInfo>
<sourceEntries>
<entry excluding="FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>

20
contrib/other-builds/moses/.project
View File

@ -1156,16 +1156,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h</locationURI>
</link>
<link>
<name>FF/InternalStructStatelessFF.cpp</name>
<type>1</type>
<location>/Users/mnadejde/Documents/workspace/moses_120913/mosesdecoder/moses/FF/InternalStructStatelessFF.cpp</location>
</link>
<link>
<name>FF/InternalStructStatelessFF.h</name>
<type>1</type>
<location>/Users/mnadejde/Documents/workspace/moses_120913/mosesdecoder/moses/FF/InternalStructStatelessFF.h</location>
</link>
<link>
<name>FF/LexicalReordering</name>
<type>2</type>
@ -1436,6 +1426,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/MultiFactor.h</locationURI>
</link>
<link>
<name>LM/NeuralLMWrapper.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLMWrapper.cpp</locationURI>
</link>
<link>
<name>LM/NeuralLMWrapper.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/NeuralLMWrapper.h</locationURI>
</link>
<link>
<name>LM/ORLM.cpp</name>
<type>1</type>

10
moses/FF/Factory.cpp
View File

@ -31,7 +31,6 @@
#include "moses/FF/OSM-Feature/OpSequenceModel.h"
#include "moses/FF/ControlRecombination.h"
#include "moses/FF/ExternalFeature.h"
#include "moses/FF/InternalStructStatelessFF.h"
#include "moses/FF/ConstrainedDecoding.h"
#include "moses/FF/SkeletonStatelessFF.h"
@ -60,6 +59,10 @@
#include "moses/SyntacticLanguageModel.h"
#endif
#ifdef LM_NEURAL
#include "moses/LM/NeuralLMWrapper.h"
#endif
#include "util/exception.hh"
#include <vector>
@ -155,7 +158,6 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(SkeletonStatefulFF);
MOSES_FNAME(InternalStructStatelessFF);
MOSES_FNAME(SkeletonLM);
MOSES_FNAME(SkeletonPT);
@ -174,6 +176,10 @@ FeatureRegistry::FeatureRegistry()
#ifdef LM_RAND
MOSES_FNAME2("RANDLM", LanguageModelRandLM);
#endif
#ifdef LM_NEURAL
MOSES_FNAME2("NeuralLM", NeuralLMWrapper);
#endif
Add("KENLM", new KenFactory());
}

10
moses/LM/Jamfile
View File

@ -70,6 +70,16 @@ if $(with-ldhtlm) {
lmmacros += LM_LDHT ;
}
#NPLM
local with-nplm = [ option.get "with-nplm" ] ;
if $(with-nplm) {
lib neuralLM : : <search>$(with-nplm)/lib <search>$(with-nplm)/lib64 ;
obj NeuralLMWrapper.o : NeuralLMWrapper.cpp neuralLM ..//headers : <include>$(with-nplm)/src <include>$(with-nplm)/3rdparty/eigen ;
alias nplm : NeuralLMWrapper.o neuralLM : : : <define>LM_NEURAL ;
dependencies += nplm ;
lmmacros += LM_NEURAL ;
}
#ORLM is always compiled but needs special headers
obj ORLM.o : ORLM.cpp ..//headers ../TranslationModel/DynSAInclude//dynsa : : : <include>../TranslationModel/DynSAInclude ;

97
moses/LM/NeuralLMWrapper.cpp Normal file
View File

@ -0,0 +1,97 @@
#include "moses/StaticData.h"
#include "moses/FactorCollection.h"
#include "NeuralLMWrapper.h"
#include "neuralLM.h"
#include <model.h>
using namespace std;
namespace Moses
{
NeuralLMWrapper::NeuralLMWrapper(const std::string &line)
:LanguageModelSingleFactor("NeuralLM", line)
{
// This space intentionally left blank
}
NeuralLMWrapper::~NeuralLMWrapper()
{
delete m_neuralLM;
}
void NeuralLMWrapper::Load()
{
TRACE_ERR("Loading NeuralLM " << m_filePath << endl);
// Set parameters required by ancestor classes
FactorCollection &factorCollection = FactorCollection::Instance();
m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, BOS_);
m_sentenceStartWord[m_factorType] = m_sentenceStart;
m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, EOS_);
m_sentenceEndWord[m_factorType] = m_sentenceEnd;
m_neuralLM = new nplm::neuralLM();
m_neuralLM->read(m_filePath);
m_neuralLM->set_log_base(10);
//TODO: Implement this
}
LMResult NeuralLMWrapper::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
unsigned int hashCode = 0;
vector<int> words(contextFactor.size());
// TRACE_ERR("NeuralLM words:");
for (size_t i=0, n=contextFactor.size(); i<n; i+=1) {
const Word* word = contextFactor[i];
const Factor* factor = word->GetFactor(m_factorType);
const std::string string= factor->GetString().as_string();
int neuralLM_wordID = m_neuralLM->lookup_word(string);
words[i] = neuralLM_wordID;
hashCode += neuralLM_wordID;
// TRACE_ERR(" " << string << "(" << neuralLM_wordID << ")" );
}
double value = m_neuralLM->lookup_ngram(words);
// TRACE_ERR("\t=\t" << value);
// TRACE_ERR(endl);
// Create a new struct to hold the result
LMResult ret;
ret.score = value;
ret.unknown = false;
// State* finalState is a void pointer
//
// Construct a hash value from the vector of words (contextFactor)
//
// The hash value must be the same size as sizeof(void*)
//
// TODO Set finalState to the above hash value
// use last word as state info
// const Factor *factor;
// size_t hash_value(const Factor &f);
// if (contextFactor.size()) {
// factor = contextFactor.back()->GetFactor(m_factorType);
// } else {
// factor = NULL;
// }
//
// (*finalState) = (State*) factor;
(*finalState) = (State*) hashCode;
return ret;
}
}

36
moses/LM/NeuralLMWrapper.h Normal file
View File

@ -0,0 +1,36 @@
#pragma once
#include "SingleFactor.h"
namespace nplm {
class neuralLM;
}
namespace Moses
{
/** Implementation of single factor LM using IRST's code.
*/
class NeuralLMWrapper : public LanguageModelSingleFactor
{
protected:
nplm::neuralLM *m_neuralLM;
public:
NeuralLMWrapper(const std::string &line);
// NeuralLM(const std::string &line);
~NeuralLMWrapper();
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
virtual void Load();
};
} // namespace