moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 05:55:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

intergrate randlm

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@1935 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972 2008-11-04 18:03:03 +00:00
parent 5539e81209
commit 789d6d96d1
9 changed files with 326 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
misc/.cproject
View File

@ -19,35 +19,37 @@
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.968564372.540679682" name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.9940797" name="GCC Tool Chain" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.2030392618" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
<builder buildPath="${workspace_loc:/misc/Debug}" id="cdt.managedbuild.target.gnu.builder.exe.debug.1148261757" name="Gnu Make Builder.Debug" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
<builder buildPath="${workspace_loc:/misc/Debug}" id="cdt.managedbuild.target.gnu.builder.exe.debug.1148261757" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
<option id="gnu.cpp.compiler.option.include.paths.634924686" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<option id="gnu.cpp.compiler.option.include.paths.634924686" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../moses/src&quot;"/>
</option>
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.516647124" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1854615959" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.516647124" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1854615959" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1264389336" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.694708706" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.1818506362" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.exe.debug.option.debugging.level.310245060" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.1818506362" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.exe.debug.option.debugging.level.310245060" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1239923844" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1774294955" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.1519446567" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
<option id="gnu.cpp.link.option.userobjs.1214926161" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">
<option id="gnu.cpp.link.option.userobjs.1214926161" name="Other objects" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">
<listOptionValue builtIn="false" value="&quot;../../moses/${ConfigName}/libmoses.a&quot;"/>
</option>
<option id="gnu.cpp.link.option.libs.1358709998" superClass="gnu.cpp.link.option.libs" valueType="libs">
<option id="gnu.cpp.link.option.libs.1358709998" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="randlm"/>
<listOptionValue builtIn="false" value="irstlm"/>
<listOptionValue builtIn="false" value="lattice"/>
<listOptionValue builtIn="false" value="misc"/>
<listOptionValue builtIn="false" value="dstruct"/>
<listOptionValue builtIn="false" value="oolm"/>
</option>
<option id="gnu.cpp.link.option.paths.512401148" superClass="gnu.cpp.link.option.paths" valueType="stringList">
<option id="gnu.cpp.link.option.paths.512401148" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="stringList">
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/i686&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../randlm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../irstlm/lib/i686-redhat-linux-gnu&quot;"/>
</option>
</tool>
@ -59,15 +61,15 @@
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.968564372./misc/processLexicalTable.cpp" name="processLexicalTable.cpp" rcbsApplicability="disable" resourcePath="processLexicalTable.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323./misc/processLexicalTable.cpp">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323./misc/processLexicalTable.cpp" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323">
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.998573704" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.109384265" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.998573704" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.109384265" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.884067908" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
</fileInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.968564372./misc/queryLexicalTable.cpp" name="queryLexicalTable.cpp" rcbsApplicability="disable" resourcePath="queryLexicalTable.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323./misc/queryLexicalTable.cpp">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323./misc/queryLexicalTable.cpp" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1174838323">
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1238173685" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.105211765" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1238173685" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.105211765" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1749305049" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
</fileInfo>
@ -76,6 +78,8 @@
</sourceEntries>
</configuration>
</storageModule>
<storageModule moduleId="scannerConfiguration"/>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
<cconfiguration id="cdt.managedbuild.config.gnu.exe.release.740659024">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.740659024" moduleId="org.eclipse.cdt.core.settings" name="Release">
@ -93,35 +97,37 @@
<folderInfo id="cdt.managedbuild.config.gnu.exe.release.740659024.1221432227" name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.548693740" name="GCC Tool Chain" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.1675623137" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
<builder buildPath="${workspace_loc:/misc/Release}" id="cdt.managedbuild.target.gnu.builder.exe.release.1102466478" name="Gnu Make Builder.Release" superClass="cdt.managedbuild.target.gnu.builder.exe.release"/>
<builder buildPath="${workspace_loc:/misc/Release}" id="cdt.managedbuild.target.gnu.builder.exe.release.1102466478" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.exe.release"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release">
<option id="gnu.cpp.compiler.option.include.paths.1215088252" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<option id="gnu.cpp.compiler.option.include.paths.1215088252" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../moses/src&quot;"/>
</option>
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.1166195501" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.384708418" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.1166195501" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.384708418" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1468360366" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.release.871712778" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.release">
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.exe.release.option.optimization.level.153638233" superClass="gnu.c.compiler.exe.release.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.exe.release.option.debugging.level.399452460" superClass="gnu.c.compiler.exe.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.exe.release.option.optimization.level.153638233" name="Optimization Level" superClass="gnu.c.compiler.exe.release.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.exe.release.option.debugging.level.399452460" name="Debug Level" superClass="gnu.c.compiler.exe.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.366352650" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.release.143684865" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.release"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.release.792459607" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.release">
<option id="gnu.cpp.link.option.userobjs.1902362839" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">
<option id="gnu.cpp.link.option.userobjs.1902362839" name="Other objects" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">
<listOptionValue builtIn="false" value="&quot;../../moses/${ConfigName}/libmoses.a&quot;"/>
</option>
<option id="gnu.cpp.link.option.libs.679824006" superClass="gnu.cpp.link.option.libs" valueType="libs">
<option id="gnu.cpp.link.option.libs.679824006" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="randlm"/>
<listOptionValue builtIn="false" value="lattice"/>
<listOptionValue builtIn="false" value="misc"/>
<listOptionValue builtIn="false" value="dstruct"/>
<listOptionValue builtIn="false" value="oolm"/>
<listOptionValue builtIn="false" value="irstlm"/>
</option>
<option id="gnu.cpp.link.option.paths.1956617803" superClass="gnu.cpp.link.option.paths" valueType="stringList">
<option id="gnu.cpp.link.option.paths.1956617803" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="stringList">
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/i686&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../randlm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../irstlm/lib/i686-redhat-linux-gnu&quot;"/>
</option>
</tool>
@ -133,22 +139,22 @@
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.release.740659024./misc/GenerateTuples.cpp" name="GenerateTuples.cpp" rcbsApplicability="disable" resourcePath="GenerateTuples.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/GenerateTuples.cpp">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/GenerateTuples.cpp" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067">
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.952213694" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.1956331211" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.952213694" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.1956331211" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1501021312" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
</fileInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.release.740659024./misc/processLexicalTable.cpp" name="processLexicalTable.cpp" rcbsApplicability="disable" resourcePath="processLexicalTable.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/processLexicalTable.cpp">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/processLexicalTable.cpp" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067">
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.2140608126" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.1705910784" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.2140608126" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.1705910784" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1360203886" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
</fileInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.release.740659024./misc/queryLexicalTable.cpp" name="queryLexicalTable.cpp" rcbsApplicability="disable" resourcePath="queryLexicalTable.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/queryLexicalTable.cpp">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067./misc/queryLexicalTable.cpp" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1949270067">
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.655834898" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.959500279" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.655834898" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.959500279" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.125175402" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
</fileInfo>
@ -157,6 +163,8 @@
</sourceEntries>
</configuration>
</storageModule>
<storageModule moduleId="scannerConfiguration"/>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">

72
misc/.project
View File

@ -11,14 +11,6 @@
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
<dictionary>
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
<value>clean</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>?name?</key>
<value></value>
@ -28,44 +20,52 @@
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.stopOnError</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildCommand</key>
<value>make</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.contents</key>
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildLocation</key>
<value>${workspace_loc:/misc/Debug}</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
<value>false</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
<value>true</value>
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildArguments</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildCommand</key>
<value>make</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildLocation</key>
<value>${workspace_loc:/misc/Release}</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
<value>clean</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.contents</key>
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
<value>false</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
<value>all</value>
<key>org.eclipse.cdt.make.core.stopOnError</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
<value>true</value>
</dictionary>
</arguments>
</buildCommand>

6
moses-cmd/.cproject
View File

@ -36,6 +36,7 @@
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.1028025969" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="LM_SRI"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_INTERNAL"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
@ -51,6 +52,7 @@
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.888044188" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
<option id="gnu.cpp.link.option.libs.427302130" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="irstlm"/>
<listOptionValue builtIn="false" value="randlm"/>
<listOptionValue builtIn="false" value="pthread"/>
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="lattice"/>
@ -62,6 +64,7 @@
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/cygwin&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../irstlm/lib/i686-redhat-linux-gnu&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/i686&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../randlm/lib&quot;"/>
</option>
<option id="gnu.cpp.link.option.userobjs.551693347" name="Other objects" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">
<listOptionValue builtIn="false" value="&quot;../../moses/${ConfigName}/libmoses.a&quot;"/>
@ -115,6 +118,7 @@
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.542441515" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="LM_SRI"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_INTERNAL"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
@ -130,6 +134,7 @@
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.release.1153317521" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.release">
<option id="gnu.cpp.link.option.libs.1100640875" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="pthread"/>
<listOptionValue builtIn="false" value="randlm"/>
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="lattice"/>
<listOptionValue builtIn="false" value="misc"/>
@ -140,6 +145,7 @@
<option id="gnu.cpp.link.option.paths.724264639" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="stringList">
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/cygwin&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../srilm/lib/i686&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../randlm/lib&quot;"/>
<listOptionValue builtIn="false" value="&quot;../../irstlm/lib/i686-redhat-linux-gnu&quot;"/>
</option>
<option id="gnu.cpp.link.option.userobjs.1365891611" name="Other objects" superClass="gnu.cpp.link.option.userobjs" valueType="userObjs">

50
moses/.cproject
View File

@ -24,32 +24,34 @@
<folderInfo id="cdt.managedbuild.config.gnu.lib.debug.2082569407.280491634" name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.lib.debug.2031285777" name="GCC Tool Chain" superClass="cdt.managedbuild.toolchain.gnu.lib.debug">
<targetPlatform id="cdt.managedbuild.target.gnu.platform.lib.debug.157058940" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.lib.debug"/>
<builder buildPath="${workspace_loc:/moses/Debug}" id="cdt.managedbuild.target.gnu.builder.lib.debug.188556568" name="Gnu Make Builder.Debug" superClass="cdt.managedbuild.target.gnu.builder.lib.debug"/>
<builder buildPath="${workspace_loc:/moses/Debug}" id="cdt.managedbuild.target.gnu.builder.lib.debug.188556568" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.lib.debug"/>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.lib.debug.1319234555" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.lib.debug">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.lib.debug.option.optimization.level.1576819994" superClass="gnu.c.compiler.lib.debug.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.lib.debug.option.debugging.level.532786472" superClass="gnu.c.compiler.lib.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.lib.debug.option.optimization.level.1576819994" name="Optimization Level" superClass="gnu.c.compiler.lib.debug.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.lib.debug.option.debugging.level.532786472" name="Debug Level" superClass="gnu.c.compiler.lib.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1713267187" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.lib.debug.1022475428" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.lib.debug">
<option id="gnu.cpp.compiler.option.preprocessor.def.2071633498" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<option id="gnu.cpp.compiler.option.preprocessor.def.2071633498" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="LM_SRI"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_INTERNAL"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BITS=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>
<option id="gnu.cpp.compiler.option.include.paths.560695438" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<option id="gnu.cpp.compiler.option.include.paths.560695438" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../irstlm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../randlm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/misc/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/dstruct/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/lm/src&quot;"/>
</option>
<option id="gnu.cpp.compiler.option.debugging.gprof.1598624147" superClass="gnu.cpp.compiler.option.debugging.gprof" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.allwarn.1855070516" superClass="gnu.cpp.compiler.option.warnings.allwarn" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.lib.debug.option.optimization.level.1543054613" superClass="gnu.cpp.compiler.lib.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.lib.debug.option.debugging.level.1229946089" superClass="gnu.cpp.compiler.lib.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.gprof.1598624147" name="Generate gprof information (-pg)" superClass="gnu.cpp.compiler.option.debugging.gprof" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.allwarn.1855070516" name="All warnings (-Wall)" superClass="gnu.cpp.compiler.option.warnings.allwarn" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.lib.debug.option.optimization.level.1543054613" name="Optimization Level" superClass="gnu.cpp.compiler.lib.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.lib.debug.option.debugging.level.1229946089" name="Debug Level" superClass="gnu.cpp.compiler.lib.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1215038936" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.archiver.lib.debug.187676627" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.lib.debug"/>
@ -62,6 +64,10 @@
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.language.mapping"/>
<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
<storageModule moduleId="scannerConfiguration"/>
</cconfiguration>
<cconfiguration id="cdt.managedbuild.config.gnu.lib.release.875756117">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.lib.release.875756117" moduleId="org.eclipse.cdt.core.settings" name="Release">
@ -84,33 +90,35 @@
<folderInfo id="cdt.managedbuild.config.gnu.lib.release.875756117.719008845" name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.lib.release.1956672127" name="GCC Tool Chain" superClass="cdt.managedbuild.toolchain.gnu.lib.release">
<targetPlatform id="cdt.managedbuild.target.gnu.platform.lib.release.1093853083" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.lib.release"/>
<builder buildPath="${workspace_loc:/moses/Release}" id="cdt.managedbuild.target.gnu.builder.lib.release.484433680" name="Gnu Make Builder.Release" superClass="cdt.managedbuild.target.gnu.builder.lib.release"/>
<builder buildPath="${workspace_loc:/moses/Release}" id="cdt.managedbuild.target.gnu.builder.lib.release.484433680" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.lib.release"/>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.lib.release.1930771681" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.lib.release">
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.lib.release.option.optimization.level.1063318824" superClass="gnu.c.compiler.lib.release.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.lib.release.option.debugging.level.672202361" superClass="gnu.c.compiler.lib.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.lib.release.option.optimization.level.1063318824" name="Optimization Level" superClass="gnu.c.compiler.lib.release.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.lib.release.option.debugging.level.672202361" name="Debug Level" superClass="gnu.c.compiler.lib.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.70231818" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.lib.release.703142952" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.lib.release">
<option id="gnu.cpp.compiler.option.include.paths.28728819" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<option id="gnu.cpp.compiler.option.include.paths.28728819" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../irstlm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../randlm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/misc/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/dstruct/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${ProjDirPath}/../srilm/lm/src&quot;"/>
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.444585015" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<option id="gnu.cpp.compiler.option.preprocessor.def.444585015" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="LM_SRI"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_INTERNAL"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BITS=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>
<option id="gnu.cpp.compiler.option.debugging.gprof.1449614968" superClass="gnu.cpp.compiler.option.debugging.gprof" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.allwarn.1037990342" superClass="gnu.cpp.compiler.option.warnings.allwarn" value="true" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.pedantic.280967211" superClass="gnu.cpp.compiler.option.warnings.pedantic" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.lib.release.option.optimization.level.481705900" superClass="gnu.cpp.compiler.lib.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.lib.release.option.debugging.level.1978909896" superClass="gnu.cpp.compiler.lib.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.gprof.1449614968" name="Generate gprof information (-pg)" superClass="gnu.cpp.compiler.option.debugging.gprof" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.allwarn.1037990342" name="All warnings (-Wall)" superClass="gnu.cpp.compiler.option.warnings.allwarn" value="true" valueType="boolean"/>
<option id="gnu.cpp.compiler.option.warnings.pedantic.280967211" name="Pedantic (-pedantic)" superClass="gnu.cpp.compiler.option.warnings.pedantic" value="false" valueType="boolean"/>
<option id="gnu.cpp.compiler.lib.release.option.optimization.level.481705900" name="Optimization Level" superClass="gnu.cpp.compiler.lib.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.lib.release.option.debugging.level.1978909896" name="Debug Level" superClass="gnu.cpp.compiler.lib.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1037157329" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.archiver.lib.release.1731276863" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.lib.release"/>
@ -123,6 +131,10 @@
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.language.mapping"/>
<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
<storageModule moduleId="scannerConfiguration"/>
</cconfiguration>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">

18
moses/src/LanguageModelFactory.cpp
View File

@ -32,6 +32,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifdef LM_IRST
# include "LanguageModelIRST.h"
#endif
#ifdef LM_RAND
# include "LanguageModelRandLM.h"
#endif
#include "LanguageModelInternal.h"
#include "LanguageModelSkip.h"
@ -44,7 +47,7 @@ namespace LanguageModelFactory
{
LanguageModel* CreateLanguageModel(LMImplementation lmImplementation
, const std::vector<FactorType> &factorTypes
, const std::vector<FactorType> &factorTypes
, size_t nGramOrder
, const std::string &languageModelFile
, float weight
@ -54,6 +57,13 @@ namespace LanguageModelFactory
LanguageModel *lm = NULL;
switch (lmImplementation)
{
case RandLM:
#ifdef LM_RAND
lm = new LanguageModelRandLM(true,
scoreIndexManager);
#endif
break;
case SRI:
#ifdef LM_SRI
lm = new LanguageModelSRI(true, scoreIndexManager);
@ -94,7 +104,7 @@ namespace LanguageModelFactory
#endif
break;
}
if (lm == NULL)
{
UserMessage::Add("Language model type unknown. Probably not compiled into library");
@ -109,7 +119,7 @@ namespace LanguageModelFactory
delete lm;
lm = NULL;
}
break;
break;
case MultiFactor:
if (! static_cast<LanguageModelMultiFactor*>(lm)->Load(languageModelFile, factorTypes, weight, nGramOrder))
{
@ -119,7 +129,7 @@ namespace LanguageModelFactory
break;
}
}
return lm;
}
}

114
moses/src/LanguageModelRandLM.cpp Normal file
View File

@ -0,0 +1,114 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <cassert>
#include <limits>
#include <iostream>
#include <fstream>
#include "LanguageModelRandLM.h"
#include "FactorCollection.h"
#include "Phrase.h"
#include "InputFileStream.h"
#include "StaticData.h"
namespace Moses
{
bool LanguageModelRandLM::Load(const std::string &filePath, FactorType factorType, float weight,
size_t nGramOrder) {
cerr << "Loading LanguageModelRandLM..." << endl;
FactorCollection &factorCollection = FactorCollection::Instance();
m_filePath = filePath;
m_factorType = factorType;
m_weight = weight;
m_nGramOrder = nGramOrder;
int cache_MB = 50; // increase cache size
m_lm = randlm::RandLM::initRandLM(filePath, nGramOrder, cache_MB);
assert(m_lm != NULL);
// get special word ids
m_oov_id = m_lm->getWordID(m_lm->getOOV());
CreateFactors(factorCollection);
return true;
}
void LanguageModelRandLM::CreateFactors(FactorCollection &factorCollection) { // add factors which have randlm id
// code copied & paste from SRI LM class. should do template function
// first get all bf vocab in map
std::map<size_t, randlm::WordID> randlm_ids_map; // map from factor id -> randlm id
size_t maxFactorId = 0; // to create lookup vector later on
for(std::map<randlm::Word, randlm::WordID>::const_iterator vIter = m_lm->vocabStart();
vIter != m_lm->vocabEnd(); vIter++){
// get word from randlm vocab and associate with (new) factor id
size_t factorId=factorCollection.AddFactor(Output,m_factorType,vIter->first)->GetId();
randlm_ids_map[factorId] = vIter->second;
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
}
// add factors for BOS and EOS and store bf word ids
size_t factorId;
m_sentenceStart = factorCollection.AddFactor(Output, m_factorType, m_lm->getBOS());
factorId = m_sentenceStart->GetId();
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
m_sentenceStartArray[m_factorType] = m_sentenceStart;
m_sentenceEnd = factorCollection.AddFactor(Output, m_factorType, m_lm->getEOS());
factorId = m_sentenceEnd->GetId();
maxFactorId = (factorId > maxFactorId) ? factorId : maxFactorId;
m_sentenceEndArray[m_factorType] = m_sentenceEnd;
// add to lookup vector in object
m_randlm_ids_vec.resize(maxFactorId+1);
// fill with OOV code
fill(m_randlm_ids_vec.begin(), m_randlm_ids_vec.end(), m_oov_id);
for (map<size_t, randlm::WordID>::const_iterator iter = randlm_ids_map.begin();
iter != randlm_ids_map.end() ; ++iter)
m_randlm_ids_vec[iter->first] = iter->second;
}
randlm::WordID LanguageModelRandLM::GetLmID( const std::string &str ) const {
return m_lm->getWordID(str);
}
float LanguageModelRandLM::GetValue(const vector<const Word*> &contextFactor,
State* finalState, unsigned int* len) const {
unsigned int dummy; // is this needed ?
if (!len) { len = &dummy; }
FactorType factorType = GetFactorType();
// set up context
randlm::WordID ngram[MAX_NGRAM_SIZE];
int count = contextFactor.size();
for (int i = 0 ; i < count ; i++) {
ngram[i] = GetLmID((*contextFactor[i])[factorType]);
//std::cerr << m_lm->getWord(ngram[i]) << " ";
}
int found = 0;
float logprob = FloorScore(TransformSRIScore(m_lm->getProb(&ngram[0], count, &found, finalState)));
*len = 0; // not available
//if (finalState)
// std::cerr << " = " << logprob << "(" << *finalState << ", " << *len <<")"<< std::endl;
//else
// std::cerr << " = " << logprob << std::endl;
return logprob;
}
}

65
moses/src/LanguageModelRandLM.h Normal file
View File

@ -0,0 +1,65 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#include <string>
#include <vector>
#include "Factor.h"
#include "Util.h"
#include "LanguageModelSingleFactor.h"
#include "RandLM.h"
class randlm::RandLM;
namespace Moses
{
class Factor;
class Phrase;
// RandLM wrapper (single factor LM)
class LanguageModelRandLM : public LanguageModelSingleFactor {
public:
LanguageModelRandLM(bool registerScore, ScoreIndexManager &scoreIndexManager)
: LanguageModelSingleFactor(registerScore, scoreIndexManager), m_lm(0) {}
bool Load(const std::string &filePath, FactorType factorType, float weight, size_t nGramOrder);
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const;
~LanguageModelRandLM() {
delete m_lm;
}
void CleanUpAfterSentenceProcessing() {
m_lm->clearCaches(); // clear caches
}
void InitializeBeforeSentenceProcessing() {} // nothing to do
protected:
std::vector<randlm::WordID> m_randlm_ids_vec;
randlm::RandLM* m_lm;
randlm::WordID m_oov_id;
void CreateFactors(FactorCollection &factorCollection);
randlm::WordID GetLmID( const std::string &str ) const;
randlm::WordID GetLmID( const Factor *factor ) const{
size_t factorId = factor->GetId();
return ( factorId >= m_randlm_ids_vec.size()) ? m_oov_id : m_randlm_ids_vec[factorId];
};
};
}

4
moses/src/Makefile.am
View File

@ -104,6 +104,10 @@ if IRST_LM
libmoses_a_SOURCES += LanguageModelIRST.cpp
endif
if RAND_LM
libmoses_a_SOURCES += LanguageModelRandLM.cpp
endif
if INTERNAL_LM
libmoses_a_SOURCES += LanguageModelInternal.cpp \
NGramCollection.cpp \

34
moses/src/TypeDef.h
View File

@ -34,7 +34,7 @@ namespace Moses
#ifndef BOS_
#define BOS_ "<s>" //Beginning of sentence symbol
#endif
#ifndef EOS_
#ifndef EOS_
#define EOS_ "</s>" //End of sentence symbol
#endif
@ -55,7 +55,7 @@ const float LOWEST_SCORE = -100.0f;
const float DEFAULT_BEAM_WIDTH = 0.00001f;
const size_t DEFAULT_VERBOSE_LEVEL = 1;
/////////////////////////////////////////////////
/////////////////////////////////////////////////
// for those using autoconf/automake
#if HAVE_CONFIG_H
#include "config.h"
@ -72,10 +72,14 @@ const size_t DEFAULT_VERBOSE_LEVEL = 1;
# define LM_IRST 1
# endif
#endif
/////////////////////////////////////////////////
# ifdef HAVE_RANDLM
# define LM_RAND 1
# endif
// enums.
#endif
/////////////////////////////////////////////////
// enums.
// must be 0, 1, 2, ..., unless otherwise stated
// can only be 2 at the moment
@ -84,7 +88,7 @@ const int NUM_LANGUAGES = 2;
const size_t MAX_NUM_FACTORS = 4;
enum FactorDirection
{
{
Input, //! Source factors
Output //! Target factors
};
@ -114,21 +118,23 @@ namespace DistortionOrientationType
{
Monotone, //distinguish only between monotone and non-monotone as possible orientations
Msd //further separate non-monotone into swapped and discontinuous
};
};
}
enum LMType
{
{
SingleFactor
,MultiFactor
};
enum LMImplementation
{
SRI = 0
,IRST = 1
,Skip = 2
{
SRI = 0
,IRST = 1
,Skip = 2
,Joint = 3
,Internal = 4
,RandLM = 5
};
@ -148,13 +154,13 @@ enum XmlInputType
};
enum DictionaryFind
{
{
Best = 0
,All = 1
};
enum SearchAlgorithm
{
{
Normal = 0
,CubePruning = 1
,CubeGrowing = 2