moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

merge master into this branch

Browse Source
This commit is contained in:
Nicola Bertoldi 2014-01-14 08:50:18 +01:00
commit 50970b2b59
231 changed files with 6746 additions and 4779 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -67,3 +67,7 @@ contrib/other-builds/*.xcodeproj/xcuserdata/
*/*.xcodeproj/xcuserdata
mert/sentence-bleu
._*
.DS_Store
*.pbxuser
*.mode1v3

13
Jamroot
View File

@ -19,7 +19,8 @@
#--with-cmph=/path/to/cmph
#
#Thread-caching malloc (if present, used for multi-threaded builds by default)
#--without-tcmalloc
#--without-tcmalloc does not compile with tcmalloc even if present
#--full-tcmalloc links against the full version (useful for memory profiling)
#
#REGRESSION TESTING
#--with-regtest=/path/to/moses-reg-test-data
@ -76,8 +77,14 @@ boost 103600 ;
external-lib z ;
if ! [ option.get "without-tcmalloc" : : "yes" ] && [ test_library "tcmalloc_minimal" ] {
external-lib tcmalloc_minimal ;
requirements += <threading>multi:<library>tcmalloc_minimal ;
if [ option.get "full-tcmalloc" : : "yes" ] {
external-lib unwind ;
external-lib tcmalloc_and_profiler : : unwind ;
requirements += <library>tcmalloc_and_profiler <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
} else {
external-lib tcmalloc_minimal ;
requirements += <threading>multi:<library>$(tcmalloc) ;
}
} else {
echo "Tip: install tcmalloc for faster threading. See BUILD-INSTRUCTIONS.txt for more information." ;
}

8
OnDiskPt/OnDiskWrapper.cpp
View File

@ -191,13 +191,7 @@ UINT64 OnDiskWrapper::GetMisc(const std::string &key) const
return iter->second;
}
PhraseNode &OnDiskWrapper::GetRootSourceNode()
{
return *m_rootSourceNode;
}
Word *OnDiskWrapper::ConvertFromMoses(Moses::FactorDirection /* direction */
, const std::vector<Moses::FactorType> &factorsVec
Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
, const Moses::Word &origWord) const
{
bool isNonTerminal = origWord.IsNonTerminal();

10
OnDiskPt/OnDiskWrapper.h
View File

@ -95,12 +95,16 @@ public:
return 1;
}
PhraseNode &GetRootSourceNode();
PhraseNode &GetRootSourceNode() {
return *m_rootSourceNode;
}
const PhraseNode &GetRootSourceNode() const {
return *m_rootSourceNode;
}
UINT64 GetMisc(const std::string &key) const;
Word *ConvertFromMoses(Moses::FactorDirection direction
, const std::vector<Moses::FactorType> &factorsVec
Word *ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
, const Moses::Word &origWord) const;
};

16
OnDiskPt/TargetPhrase.cpp
View File

@ -227,14 +227,17 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
, const std::vector<Moses::FactorType> &outputFactors
, const Vocab &vocab
, const Moses::PhraseDictionary &phraseDict
, const std::vector<float> &weightT) const
, const std::vector<float> &weightT
, bool isSyntax) const
{
Moses::TargetPhrase *ret = new Moses::TargetPhrase();
// words
size_t phraseSize = GetSize();
CHECK(phraseSize > 0); // last word is lhs
--phraseSize;
if (isSyntax) {
--phraseSize;
}
for (size_t pos = 0; pos < phraseSize; ++pos) {
GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord());
@ -261,16 +264,17 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
ret->SetAlignTerm(alignTerm);
ret->SetAlignNonTerm(alignNonTerm);
Moses::Word *lhsTarget = new Moses::Word(true);
GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
ret->SetTargetLHS(lhsTarget);
if (isSyntax) {
Moses::Word *lhsTarget = new Moses::Word(true);
GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
ret->SetTargetLHS(lhsTarget);
}
// set source phrase
Moses::Phrase mosesSP(Moses::Input);
for (size_t pos = 0; pos < sp->GetSize(); ++pos) {
sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord());
}
ret->SetSourcePhrase(mosesSP);
// scores
ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);

3
OnDiskPt/TargetPhrase.h
View File

@ -103,7 +103,8 @@ public:
, const std::vector<Moses::FactorType> &outputFactors
, const Vocab &vocab
, const Moses::PhraseDictionary &phraseDict
, const std::vector<float> &weightT) const;
, const std::vector<float> &weightT
, bool isSyntax) const;
UINT64 ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl);
UINT64 ReadFromFile(std::fstream &fileTP);

7
OnDiskPt/TargetPhraseCollection.cpp
View File

@ -117,8 +117,8 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std:
, const std::vector<Moses::FactorType> &outputFactors
, const Moses::PhraseDictionary &phraseDict
, const std::vector<float> &weightT
, const std::string & /* filePath */
, Vocab &vocab) const
, Vocab &vocab
, bool isSyntax) const
{
Moses::TargetPhraseCollection *ret = new Moses::TargetPhraseCollection();
@ -128,7 +128,8 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std:
Moses::TargetPhrase *mosesPhrase = tp.ConvertToMoses(inputFactors, outputFactors
, vocab
, phraseDict
, weightT);
, weightT
, isSyntax);
/*
// debugging output

4
OnDiskPt/TargetPhraseCollection.h
View File

@ -73,8 +73,8 @@ public:
, const std::vector<Moses::FactorType> &outputFactors
, const Moses::PhraseDictionary &phraseDict
, const std::vector<float> &weightT
, const std::string &filePath
, Vocab &vocab) const;
, Vocab &vocab
, bool isSyntax) const;
void ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper);
const std::string GetDebugStr() const;

11
contrib/other-builds/extract-ghkm/.cproject
View File

@ -18,7 +18,7 @@
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1410559002." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.1035891586" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.242178856" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract-ghkm/Debug}" id="cdt.managedbuild.builder.gnu.cross.430400318" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract-ghkm/Debug}" id="cdt.managedbuild.builder.gnu.cross.430400318" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.builder.gnu.cross"/>
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.251687262" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.962699619" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.option.debugging.level.230503798" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
@ -126,5 +126,12 @@
</scannerConfigBuildInfo>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
<storageModule moduleId="refreshScope"/>
<storageModule moduleId="refreshScope" versionNumber="2">
<configuration configurationName="Release">
<resource resourceType="PROJECT" workspacePath="/extract-ghkm"/>
</configuration>
<configuration configurationName="Debug">
<resource resourceType="PROJECT" workspacePath="/extract-ghkm"/>
</configuration>
</storageModule>
</cproject>

129
contrib/other-builds/extract-rules/.cproject Normal file
View File

@ -0,0 +1,129 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292" moduleId="org.eclipse.cdt.core.settings" name="Debug">
<externalSettings/>
<extensions>
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
</extensions>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.124769989" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.266544803" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract-rules}/Debug" id="cdt.managedbuild.builder.gnu.cross.335858926" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1376077469" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.947547329" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.option.debugging.level.426953885" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<option id="gnu.c.compiler.option.include.paths.1671695899" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" valueType="includePath"/>
<option id="gnu.c.compiler.option.include.files.1838960067" name="Include files (-include)" superClass="gnu.c.compiler.option.include.files" valueType="includeFiles"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.985831394" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.53480540" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
<option id="gnu.cpp.compiler.option.optimization.level.1726371873" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.level.899893408" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.include.paths.1099087456" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.88958138" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1616232021" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1411857637" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
<option id="gnu.cpp.link.option.libs.109133121" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="boost_iostreams-mt"/>
<listOptionValue builtIn="false" value="boost_system-mt"/>
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
</option>
<option id="gnu.cpp.link.option.paths.1030374421" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.272393234" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
</inputType>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1391783790" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.2066621509" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1945638157" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
</tool>
</toolChain>
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544" moduleId="org.eclipse.cdt.core.settings" name="Release">
<externalSettings/>
<extensions>
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
</extensions>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1113964425" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1722595316" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract-rules}/Release" id="cdt.managedbuild.builder.gnu.cross.691589832" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.593530229" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.1320426973" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.option.debugging.level.947026588" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1217031668" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1401773863" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
<option id="gnu.cpp.compiler.option.optimization.level.1504181086" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.level.1645775798" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1484987112" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1807515346" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.44234391" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1468234013" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
</inputType>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.467923425" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1673313707" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.518252425" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
</tool>
</toolChain>
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<project id="extract-rules.cdt.managedbuild.target.gnu.cross.exe.1916763759" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
</storageModule>
<storageModule moduleId="scannerConfiguration">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292;cdt.managedbuild.config.gnu.cross.exe.debug.1438215292.;cdt.managedbuild.tool.gnu.cross.c.compiler.1376077469;cdt.managedbuild.tool.gnu.c.compiler.input.985831394">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292;cdt.managedbuild.config.gnu.cross.exe.debug.1438215292.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.53480540;cdt.managedbuild.tool.gnu.cpp.compiler.input.88958138">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1200693544;cdt.managedbuild.config.gnu.cross.exe.release.1200693544.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1401773863;cdt.managedbuild.tool.gnu.cpp.compiler.input.1484987112">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1200693544;cdt.managedbuild.config.gnu.cross.exe.release.1200693544.;cdt.managedbuild.tool.gnu.cross.c.compiler.593530229;cdt.managedbuild.tool.gnu.c.compiler.input.1217031668">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
</scannerConfigBuildInfo>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
<storageModule moduleId="refreshScope"/>
</cproject>

1
contrib/other-builds/extract-rules/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/Debug

134
contrib/other-builds/extract-rules/.project Normal file
View File

@ -0,0 +1,134 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>extract-rules</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
<triggers>full,incremental,</triggers>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.cdt.core.cnature</nature>
<nature>org.eclipse.cdt.core.ccnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
</natures>
<linkedResources>
<link>
<name>ExtractedRule.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/ExtractedRule.cpp</locationURI>
</link>
<link>
<name>ExtractedRule.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/ExtractedRule.h</locationURI>
</link>
<link>
<name>Hole.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/Hole.h</locationURI>
</link>
<link>
<name>HoleCollection.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.cpp</locationURI>
</link>
<link>
<name>HoleCollection.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.h</locationURI>
</link>
<link>
<name>InputFileStream.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp</locationURI>
</link>
<link>
<name>InputFileStream.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h</locationURI>
</link>
<link>
<name>OutputFileStream.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp</locationURI>
</link>
<link>
<name>OutputFileStream.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h</locationURI>
</link>
<link>
<name>SentenceAlignment.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.cpp</locationURI>
</link>
<link>
<name>SentenceAlignment.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.h</locationURI>
</link>
<link>
<name>SentenceAlignmentWithSyntax.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignmentWithSyntax.cpp</locationURI>
</link>
<link>
<name>SentenceAlignmentWithSyntax.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignmentWithSyntax.h</locationURI>
</link>
<link>
<name>SyntaxTree.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SyntaxTree.cpp</locationURI>
</link>
<link>
<name>SyntaxTree.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SyntaxTree.h</locationURI>
</link>
<link>
<name>XmlTree.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/XmlTree.cpp</locationURI>
</link>
<link>
<name>XmlTree.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/XmlTree.h</locationURI>
</link>
<link>
<name>extract-rules-main.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/extract-rules-main.cpp</locationURI>
</link>
<link>
<name>gzfilebuf.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/gzfilebuf.h</locationURI>
</link>
<link>
<name>tables-core.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp</locationURI>
</link>
<link>
<name>tables-core.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h</locationURI>
</link>
</linkedResources>
</projectDescription>

0
phrase-extract/extract.xcodeproj/project.pbxproj → contrib/other-builds/extract.xcodeproj/project.pbxproj
View File

127
contrib/other-builds/extract/.cproject Normal file
View File

@ -0,0 +1,127 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689" moduleId="org.eclipse.cdt.core.settings" name="Debug">
<externalSettings/>
<extensions>
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
</extensions>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.671913278" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1231657738" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract}/Debug" id="cdt.managedbuild.builder.gnu.cross.571044108" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.332036857" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.1292572253" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.option.debugging.level.1873227592" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1165888615" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1342023600" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
<option id="gnu.cpp.compiler.option.optimization.level.698819695" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.level.1451916947" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.include.paths.1702398011" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.579278848" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1856691234" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1699542791" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
<option id="gnu.cpp.link.option.libs.1880730637" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="boost_iostreams-mt"/>
<listOptionValue builtIn="false" value="boost_system-mt"/>
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
</option>
<option id="gnu.cpp.link.option.paths.298225069" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1339210059" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
</inputType>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.976825054" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1971927463" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.704926167" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
</tool>
</toolChain>
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.140124152">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.140124152" moduleId="org.eclipse.cdt.core.settings" name="Release">
<externalSettings/>
<extensions>
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
</extensions>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.140124152" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.140124152." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1250240843" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.597335968" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
<builder buildPath="${workspace_loc:/extract}/Release" id="cdt.managedbuild.builder.gnu.cross.95066247" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.2096762162" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.88795016" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.option.debugging.level.383328020" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.681105644" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1806684544" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
<option id="gnu.cpp.compiler.option.optimization.level.553394848" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.debugging.level.1420596769" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1726759263" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.234409052" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.320346578" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.2045242811" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
</inputType>
</tool>
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.417132714" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1944597759" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.203400619" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
</tool>
</toolChain>
</folderInfo>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<project id="extract.cdt.managedbuild.target.gnu.cross.exe.1220534104" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
</storageModule>
<storageModule moduleId="scannerConfiguration">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.140124152;cdt.managedbuild.config.gnu.cross.exe.release.140124152.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1806684544;cdt.managedbuild.tool.gnu.cpp.compiler.input.1726759263">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.140124152;cdt.managedbuild.config.gnu.cross.exe.release.140124152.;cdt.managedbuild.tool.gnu.cross.c.compiler.2096762162;cdt.managedbuild.tool.gnu.c.compiler.input.681105644">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.386290689;cdt.managedbuild.config.gnu.cross.exe.debug.386290689.;cdt.managedbuild.tool.gnu.cross.c.compiler.332036857;cdt.managedbuild.tool.gnu.c.compiler.input.1165888615">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
</scannerConfigBuildInfo>
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.386290689;cdt.managedbuild.config.gnu.cross.exe.debug.386290689.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1342023600;cdt.managedbuild.tool.gnu.cpp.compiler.input.579278848">
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
</scannerConfigBuildInfo>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
<storageModule moduleId="refreshScope"/>
</cproject>

74
contrib/other-builds/extract/.project Normal file
View File

@ -0,0 +1,74 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>extract</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
<triggers>full,incremental,</triggers>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.cdt.core.cnature</nature>
<nature>org.eclipse.cdt.core.ccnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
</natures>
<linkedResources>
<link>
<name>InputFileStream.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp</locationURI>
</link>
<link>
<name>InputFileStream.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h</locationURI>
</link>
<link>
<name>OutputFileStream.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp</locationURI>
</link>
<link>
<name>OutputFileStream.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h</locationURI>
</link>
<link>
<name>SentenceAlignment.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.cpp</locationURI>
</link>
<link>
<name>SentenceAlignment.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.h</locationURI>
</link>
<link>
<name>extract-main.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/extract-main.cpp</locationURI>
</link>
<link>
<name>tables-core.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp</locationURI>
</link>
<link>
<name>tables-core.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h</locationURI>
</link>
</linkedResources>
</projectDescription>

11
contrib/other-builds/lm.xcodeproj/project.pbxproj
View File

@ -34,7 +34,6 @@
1EBA44E314B97E22003CC0EA /* model_type.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449114B97E22003CC0EA /* model_type.hh */; };
1EBA44E414B97E22003CC0EA /* model.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449214B97E22003CC0EA /* model.cc */; };
1EBA44E514B97E22003CC0EA /* model.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449314B97E22003CC0EA /* model.hh */; };
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449414B97E22003CC0EA /* ngram_query.cc */; };
1EBA44E714B97E22003CC0EA /* ngram_query.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449514B97E22003CC0EA /* ngram_query.hh */; };
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449614B97E22003CC0EA /* quantize.cc */; };
1EBA44E914B97E22003CC0EA /* quantize.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449714B97E22003CC0EA /* quantize.hh */; };
@ -149,7 +148,6 @@
1EBA449114B97E22003CC0EA /* model_type.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model_type.hh; path = ../../lm/model_type.hh; sourceTree = "<group>"; };
1EBA449214B97E22003CC0EA /* model.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = model.cc; path = ../../lm/model.cc; sourceTree = "<group>"; };
1EBA449314B97E22003CC0EA /* model.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model.hh; path = ../../lm/model.hh; sourceTree = "<group>"; };
1EBA449414B97E22003CC0EA /* ngram_query.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ngram_query.cc; path = ../../lm/ngram_query.cc; sourceTree = "<group>"; };
1EBA449514B97E22003CC0EA /* ngram_query.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = ngram_query.hh; path = ../../lm/ngram_query.hh; sourceTree = "<group>"; };
1EBA449614B97E22003CC0EA /* quantize.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = quantize.cc; path = ../../lm/quantize.cc; sourceTree = "<group>"; };
1EBA449714B97E22003CC0EA /* quantize.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = quantize.hh; path = ../../lm/quantize.hh; sourceTree = "<group>"; };
@ -266,7 +264,6 @@
1EBA449114B97E22003CC0EA /* model_type.hh */,
1EBA449214B97E22003CC0EA /* model.cc */,
1EBA449314B97E22003CC0EA /* model.hh */,
1EBA449414B97E22003CC0EA /* ngram_query.cc */,
1EBA449514B97E22003CC0EA /* ngram_query.hh */,
1EBA449614B97E22003CC0EA /* quantize.cc */,
1EBA449714B97E22003CC0EA /* quantize.hh */,
@ -535,7 +532,6 @@
1EBA44DF14B97E22003CC0EA /* lm_exception.cc in Sources */,
1EBA44E214B97E22003CC0EA /* model_test.cc in Sources */,
1EBA44E414B97E22003CC0EA /* model.cc in Sources */,
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */,
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */,
1EBA44EA14B97E22003CC0EA /* read_arpa.cc in Sources */,
1EBA44ED14B97E22003CC0EA /* search_hashed.cc in Sources */,
@ -589,6 +585,7 @@
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_DYNAMIC_NO_PIC = NO;
GCC_ENABLE_OBJC_EXCEPTIONS = YES;
GCC_INPUT_FILETYPE = automatic;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
@ -601,10 +598,11 @@
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = (
"$(LIBRARY_MY_BOOST)",
../..,
/opt/local/include,
);
MACOSX_DEPLOYMENT_TARGET = 10.7;
MACOSX_DEPLOYMENT_TARGET = 10.6;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
};
@ -625,10 +623,11 @@
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = (
"$(XCODE_MOSES_BOOST)",
../..,
/opt/local/include,
);
MACOSX_DEPLOYMENT_TARGET = 10.7;
MACOSX_DEPLOYMENT_TARGET = 10.6;
SDKROOT = macosx;
};
name = Release;

4
contrib/other-builds/moses-chart-cmd/.cproject
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<?fileVersion 4.0.0?>
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.162355801">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.162355801" moduleId="org.eclipse.cdt.core.settings" name="Debug">

4
contrib/other-builds/moses-cmd/.cproject
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<?fileVersion 4.0.0?>
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.461114338">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.461114338" moduleId="org.eclipse.cdt.core.settings" name="Debug">

4562
contrib/other-builds/moses.xcodeproj/project.pbxproj
View File

File diff suppressed because it is too large Load Diff

3
contrib/other-builds/moses/.cproject
View File

@ -71,6 +71,9 @@
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.511477442" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1742823107" name="ChartTranslationOption.cpp" rcbsApplicability="disable" resourcePath="ChartTranslationOption.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050">
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327"/>
</fileInfo>
<sourceEntries>
<entry excluding="FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>

128
contrib/other-builds/moses/.project
View File

@ -191,11 +191,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartParserCallback.h</locationURI>
</link>
<link>
<name>ChartRuleLookupManager.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartRuleLookupManager.cpp</locationURI>
</link>
<link>
<name>ChartRuleLookupManager.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartRuleLookupManager.h</locationURI>
</link>
<link>
<name>ChartTranslationOption.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartTranslationOption.cpp</locationURI>
</link>
<link>
<name>ChartTranslationOption.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartTranslationOption.h</locationURI>
</link>
<link>
<name>ChartTranslationOptionList.cpp</name>
<type>1</type>
@ -271,16 +286,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ConfusionNet.h</locationURI>
</link>
<link>
<name>DecodeFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.cpp</locationURI>
</link>
<link>
<name>DecodeFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.h</locationURI>
</link>
<link>
<name>DecodeGraph.cpp</name>
<type>1</type>
@ -471,6 +476,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/InputFileStream.h</locationURI>
</link>
<link>
<name>InputPath.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/InputPath.cpp</locationURI>
</link>
<link>
<name>InputPath.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/InputPath.h</locationURI>
</link>
<link>
<name>InputType.cpp</name>
<type>1</type>
@ -501,36 +516,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LVoc.h</locationURI>
</link>
<link>
<name>LexicalReordering.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.cpp</locationURI>
</link>
<link>
<name>LexicalReordering.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.h</locationURI>
</link>
<link>
<name>LexicalReorderingState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.cpp</locationURI>
</link>
<link>
<name>LexicalReorderingState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.h</locationURI>
</link>
<link>
<name>LexicalReorderingTable.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.cpp</locationURI>
</link>
<link>
<name>LexicalReorderingTable.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.h</locationURI>
</link>
<link>
<name>Manager.cpp</name>
<type>1</type>
@ -1051,16 +1036,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.h</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.h</locationURI>
</link>
<link>
<name>FF/ControlRecombination.cpp</name>
<type>1</type>
@ -1071,6 +1046,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ControlRecombination.h</locationURI>
</link>
<link>
<name>FF/DecodeFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.cpp</locationURI>
</link>
<link>
<name>FF/DecodeFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.h</locationURI>
</link>
<link>
<name>FF/DistortionScoreProducer.cpp</name>
<type>1</type>
@ -1142,19 +1127,14 @@
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h</locationURI>
</link>
<link>
<name>FF/OSM-Feature</name>
<name>FF/LexicalReordering</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.h</locationURI>
<name>FF/OSM-Feature</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>FF/PhraseBoundaryFeature.cpp</name>
@ -1626,6 +1606,36 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReordering.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReordering.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingTable.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingTable.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.h</locationURI>
</link>
<link>
<name>FF/OSM-Feature/OpSequenceModel.cpp</name>
<type>1</type>

3
contrib/server/mosesserver.cpp
View File

@ -246,9 +246,6 @@ public:
if (multiModelWeights.size() > 0) {
PhraseDictionaryMultiModel* pdmm = (PhraseDictionaryMultiModel*) staticData.GetPhraseDictionaries()[0]; //TODO: only works if multimodel is first phrase table
pdmm->SetTemporaryMultiModelWeightsVector(multiModelWeights);
if (staticData.GetUseTransOptCache()) {
cerr << "Warning: -use-persistent-cache is set to true; sentence-specific weights may be ignored. Disable cache for true results.\n";
}
}
stringstream out, graphInfo, transCollOpts;

33
jam-files/sanity.jam
View File

@ -73,17 +73,30 @@ rule test_library ( name ) {
constant CLEANING : $(cleaning) ;
}
shared-command-line = ;
local argv = [ modules.peek : ARGV ] ;
while $(argv) {
if $(argv[1]) = "link=shared" {
shared-command-line = <link>shared ;
}
argv = $(argv[2-]) ;
}
#Determine if a library can be compiled statically.
rule auto-shared ( name : additional * ) {
additional ?= "" ;
if [ test_flags $(additional)" -static -l"$(name) ] {
return ;
if $(shared-command-line) = "<link>shared" {
return "<link>shared" ;
} else {
if $(FORCE-STATIC) {
echo "Could not statically link against lib $(name). Your build will probably fail." ;
if [ test_flags $(additional)" -static -l"$(name) ] {
return ;
} else {
return "<link>shared" ;
if $(FORCE-STATIC) {
echo "Could not statically link against lib $(name). Your build will probably fail." ;
return ;
} else {
return "<link>shared" ;
}
}
}
}
@ -133,8 +146,8 @@ rule boost-lib ( name macro : deps * ) {
flags += " -static" ;
}
if [ test_flags $(flags) : $(main) ] {
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : <link>static : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : <link>static : <library>$(deps) ;
} else {
lib inner_boost_$(name) : : $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
}
@ -143,7 +156,7 @@ rule boost-lib ( name macro : deps * ) {
alias boost_$(name) : inner_boost_$(name) : <link>shared ;
requirements += <define>BOOST_$(macro) ;
} else {
alias boost_$(name) : inner_boost_$(name) : : : <link>shared:<define>BOOST_$(macro) ;
alias boost_$(name) : inner_boost_$(name) : <link>static ;
}
}
@ -184,8 +197,8 @@ rule boost ( min-version ) {
}
#Link normally to a library, but sometimes static isn't installed so fall back to dynamic.
rule external-lib ( name : search-path * ) {
lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) ;
rule external-lib ( name : search-path * : deps * ) {
lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) <use>$(deps) ;
}
#Write the current command line to previous.sh. This does not do shell escaping.

4
mira/Decoder.cpp
View File

@ -144,7 +144,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
m_manager->CalcNBest(nBestSize, nBestList, distinct);
// optionally print nbest to file (to extract scores and features.. currently just for sentence bleu scoring)
if (filename != "") {
/*if (filename != "") {
ofstream out(filename.c_str());
if (!out) {
ostringstream msg;
@ -154,7 +154,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
// TODO: handle sentence id (for now always 0)
//OutputNBest(out, nBestList, StaticData::Instance().GetOutputFactorOrder(), 0, false);
out.close();
}
}*/
// read off the feature values and bleu scores for each sentence in the nbest list
Moses::TrellisPathList::const_iterator iter;

159
mira/Main.cpp
View File

@ -41,7 +41,6 @@ namespace mpi = boost::mpi;
#include "moses/ChartTrellisPath.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/ThreadPool.h"
#include "moses/LexicalReordering.h"
#include "mert/BleuScorer.h"
#include "moses/FeatureVector.h"
@ -78,8 +77,8 @@ int main(int argc, char** argv)
size_t mixingFrequency;
size_t weightDumpFrequency;
string weightDumpStem;
bool scale_margin, scale_margin_precision;
bool scale_update, scale_update_precision;
bool scale_margin;
bool scale_update;
size_t n;
size_t batchSize;
bool distinctNbest;
@ -103,7 +102,7 @@ int main(int argc, char** argv)
bool perceptron_update;
bool hope_fear;
bool model_hope_fear;
int hope_n, fear_n;
size_t hope_n, fear_n;
size_t bleu_smoothing_scheme;
float min_oracle_bleu;
float minBleuRatio, maxBleuRatio;
@ -122,17 +121,19 @@ int main(int argc, char** argv)
string moses_src;
float sigmoidParam;
float bleuWeight, bleuWeight_hope, bleuWeight_fear;
bool bleu_weight_lm, bleu_weight_lm_adjust;
bool bleu_weight_lm;
float bleu_weight_lm_factor;
bool l1_regularize, l2_regularize, l1_reg_sparse, l2_reg_sparse;
float l1_lambda, l2_lambda;
bool most_violated, most_violated_reg, all_violated, max_bleu_diff, one_against_all;
bool most_violated, most_violated_reg, all_violated, max_bleu_diff;
bool feature_confidence, signed_counts;
float decay_core, decay_sparse, core_r0, sparse_r0;
bool selective, summed;
float bleu_weight_fear_factor;
bool hildreth;
float add2lm;
// compute real sentence Bleu scores on complete translations, disable Bleu feature
bool realBleu, disableBleuFeature;
bool rescaleSlack;
bool makePairs;
@ -147,12 +148,14 @@ int main(int argc, char** argv)
("l1sparse", po::value<bool>(&l1_reg_sparse)->default_value(true), "L1-regularization for sparse weights only")
("l2sparse", po::value<bool>(&l2_reg_sparse)->default_value(true), "L2-regularization for sparse weights only")
("mv-reg", po::value<bool>(&most_violated_reg)->default_value(false), "Regularize most violated constraint")
("most-violated", po::value<bool>(&most_violated)->default_value(false), "Add most violated constraint")
("all-violated", po::value<bool>(&all_violated)->default_value(false), "Add all violated constraints")
("feature-confidence", po::value<bool>(&feature_confidence)->default_value(false), "Confidence-weighted learning")
("signed-counts", po::value<bool>(&signed_counts)->default_value(false), "Use signed feature counts for CWL")
("dbg", po::value<bool>(&debug)->default_value(true), "More debug output")
("make-pairs", po::value<bool>(&makePairs)->default_value(true), "Make pairs of hypotheses for 1slack")
("debug", po::value<bool>(&debug)->default_value(true), "More debug output")
("rescale-slack", po::value<bool>(&rescaleSlack)->default_value(false), "Rescale slack in 1-slack formulation")
("disable-bleu-feature", po::value<bool>(&disableBleuFeature)->default_value(false), "Disable the Bleu feature")
("real-bleu", po::value<bool>(&realBleu)->default_value(false), "Compute real sentence Bleu on complete translations")
("add2lm", po::value<float>(&add2lm)->default_value(0.0), "Add the specified amount to all LM weights")
("hildreth", po::value<bool>(&hildreth)->default_value(false), "Prefer Hildreth over analytical update")
("selective", po::value<bool>(&selective)->default_value(false), "Build constraints for every feature")
@ -166,9 +169,10 @@ int main(int argc, char** argv)
("core-r0", po::value<float>(&core_r0)->default_value(1.0), "Start learning rate for core features")
("sparse-r0", po::value<float>(&sparse_r0)->default_value(1.0), "Start learning rate for sparse features")
("decay-core", po::value<float>(&decay_core)->default_value(0.01), "Decay for core feature learning rate")
("decay-sparse", po::value<float>(&decay_sparse)->default_value(0.01), "Decay for sparse feature learning rate")
("tie-bw-to-lm", po::value<bool>(&bleu_weight_lm)->default_value(false), "Make bleu weight depend on lm weight")
("adjust-bw", po::value<bool>(&bleu_weight_lm_adjust)->default_value(false), "Adjust bleu weight when lm weight changes")
("tie-bw-to-lm", po::value<bool>(&bleu_weight_lm)->default_value(true), "Make bleu weight depend on lm weight")
("bw-lm-factor", po::value<float>(&bleu_weight_lm_factor)->default_value(2.0), "Make bleu weight depend on lm weight by this factor")
("bw-factor-fear", po::value<float>(&bleu_weight_fear_factor)->default_value(1.0), "Multiply fear weight by this factor")
("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
@ -190,12 +194,12 @@ int main(int argc, char** argv)
("dump-mixed-weights", po::value<bool>(&dumpMixedWeights)->default_value(false), "Dump mixed weights instead of averaged weights")
("epochs,e", po::value<size_t>(&epochs)->default_value(10), "Number of epochs")
("feature-cutoff", po::value<int>(&featureCutoff)->default_value(-1), "Feature cutoff as additional regularization for sparse features")
("fear-n", po::value<int>(&fear_n)->default_value(1), "Number of fear translations used")
("fear-n", po::value<size_t>(&fear_n)->default_value(1), "Number of fear translations used")
("help", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
("history-bleu", po::value<bool>(&historyBleu)->default_value(false), "Use 1best translations to update the history")
("history-smoothing", po::value<float>(&historySmoothing)->default_value(0.9), "Adjust the factor for history smoothing")
("hope-fear", po::value<bool>(&hope_fear)->default_value(true), "Use only hope and fear translations for optimisation (not model)")
("hope-n", po::value<int>(&hope_n)->default_value(2), "Number of hope translations used")
("hope-n", po::value<size_t>(&hope_n)->default_value(2), "Number of hope translations used")
("input-file,i", po::value<string>(&inputFile), "Input file containing tokenised source")
("input-files-folds", po::value<vector<string> >(&inputFilesFolds), "Input files containing tokenised source, one for each fold")
("learner,l", po::value<string>(&learner)->default_value("mira"), "Learning algorithm")
@ -209,10 +213,10 @@ int main(int argc, char** argv)
("min-oracle-bleu", po::value<float>(&min_oracle_bleu)->default_value(0), "Set a minimum oracle BLEU score")
("min-weight-change", po::value<float>(&min_weight_change)->default_value(0.0001), "Set minimum weight change for stopping criterion")
("mira-learning-rate", po::value<float>(&mira_learning_rate)->default_value(1), "Learning rate for MIRA (fixed or flexible)")
("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(1), "How often per epoch to mix weights, when using mpi")
("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(10), "How often per epoch to mix weights, when using mpi")
("model-hope-fear", po::value<bool>(&model_hope_fear)->default_value(false), "Use model, hope and fear translations for optimisation")
("moses-src", po::value<string>(&moses_src)->default_value(""), "Moses source directory")
("nbest,n", po::value<size_t>(&n)->default_value(1), "Number of translations in n-best list")
("nbest,n", po::value<size_t>(&n)->default_value(30), "Number of translations in n-best list")
("normalise-weights", po::value<bool>(&normaliseWeights)->default_value(false), "Whether to normalise the updated weights before passing them to the decoder")
("normalise-margin", po::value<bool>(&normaliseMargin)->default_value(false), "Normalise the margin: squash between 0 and 1")
("perceptron-learning-rate", po::value<float>(&perceptron_learning_rate)->default_value(0.01), "Perceptron learning rate")
@ -224,30 +228,27 @@ int main(int argc, char** argv)
("prune-zero-weights", po::value<bool>(&pruneZeroWeights)->default_value(false), "Prune zero-valued sparse feature weights")
("reference-files,r", po::value<vector<string> >(&referenceFiles), "Reference translation files for training")
("reference-files-folds", po::value<vector<string> >(&referenceFilesFolds), "Reference translation files for training, one for each fold")
("kbest", po::value<bool>(&kbest)->default_value(false), "Select hope/fear pairs from a list of nbest translations")
("kbest", po::value<bool>(&kbest)->default_value(true), "Select hope/fear pairs from a list of nbest translations")
("scale-by-inverse-length", po::value<bool>(&scaleByInverseLength)->default_value(false), "Scale BLEU by (history of) inverse input length")
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(false), "Scale BLEU by (history of) input length")
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale BLEU by (history of) input length")
("scale-by-avg-input-length", po::value<bool>(&scaleByAvgInputLength)->default_value(false), "Scale BLEU by average input length")
("scale-by-avg-inverse-length", po::value<bool>(&scaleByAvgInverseLength)->default_value(false), "Scale BLEU by average inverse input length")
("scale-by-x", po::value<float>(&scaleByX)->default_value(1), "Scale the BLEU score by value x")
("scale-lm", po::value<bool>(&scale_lm)->default_value(false), "Scale the language model feature")
("scale-factor-lm", po::value<float>(&scale_lm_factor)->default_value(2), "Scale the language model feature by this factor")
("scale-by-x", po::value<float>(&scaleByX)->default_value(0.1), "Scale the BLEU score by value x")
("scale-lm", po::value<bool>(&scale_lm)->default_value(true), "Scale the language model feature")
("scale-factor-lm", po::value<float>(&scale_lm_factor)->default_value(0.5), "Scale the language model feature by this factor")
("scale-wp", po::value<bool>(&scale_wp)->default_value(false), "Scale the word penalty feature")
("scale-factor-wp", po::value<float>(&scale_wp_factor)->default_value(2), "Scale the word penalty feature by this factor")
("scale-margin", po::value<bool>(&scale_margin)->default_value(0), "Scale the margin by the Bleu score of the oracle translation")
("scale-margin-precision", po::value<bool>(&scale_margin_precision)->default_value(0), "Scale margin by precision of oracle")
("scale-update", po::value<bool>(&scale_update)->default_value(0), "Scale update by Bleu score of oracle")
("scale-update-precision", po::value<bool>(&scale_update_precision)->default_value(0), "Scale update by precision of oracle")
("sentence-level-bleu", po::value<bool>(&sentenceBleu)->default_value(true), "Use a sentences level Bleu scoring function")
("shuffle", po::value<bool>(&shuffle)->default_value(false), "Shuffle input sentences before processing")
("sigmoid-param", po::value<float>(&sigmoidParam)->default_value(1), "y=sigmoidParam is the axis that this sigmoid approaches")
("slack", po::value<float>(&slack)->default_value(0.01), "Use slack in optimiser")
("slack", po::value<float>(&slack)->default_value(0.05), "Use slack in optimiser")
("sparse-average", po::value<bool>(&sparseAverage)->default_value(false), "Average weights by the number of processes")
("sparse-no-average", po::value<bool>(&sparseNoAverage)->default_value(false), "Don't average sparse weights, just sum")
("stop-weights", po::value<bool>(&weightConvergence)->default_value(true), "Stop when weights converge")
("verbosity,v", po::value<int>(&verbosity)->default_value(0), "Verbosity level")
("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(1), "How often per epoch to dump weights (mpi)")
("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(2), "How often per epoch to dump weights (mpi)")
("weight-dump-stem", po::value<string>(&weightDumpStem)->default_value("weights"), "Stem of filename to use for dumping weights");
po::options_description cmdline_options;
@ -396,20 +397,26 @@ int main(int argc, char** argv)
// initialise Moses
// add references to initialize Bleu feature
boost::trim(decoder_settings);
decoder_settings += " -mira -distinct-nbest -references";
if (trainWithMultipleFolds) {
decoder_settings += " ";
decoder_settings += referenceFilesFolds[myFold];
} else {
for (size_t i=0; i < referenceFiles.size(); ++i) {
decoder_settings += " ";
decoder_settings += referenceFiles[i];
}
}
decoder_settings += " -mira -n-best-list - " + boost::lexical_cast<string>(n) + " distinct";
vector<string> decoder_params;
boost::split(decoder_params, decoder_settings, boost::is_any_of("\t "));
// bleu feature
decoder_params.push_back("-feature-add");
decoder_settings = "BleuScoreFeature tuneable=false references=";
if (trainWithMultipleFolds) {
decoder_settings += referenceFilesFolds[myFold];
} else {
decoder_settings += referenceFiles[0];
for (size_t i=1; i < referenceFiles.size(); ++i) {
decoder_settings += ",";
decoder_settings += referenceFiles[i];
}
}
decoder_params.push_back(decoder_settings);
string configFile = trainWithMultipleFolds? mosesConfigFilesFolds[myFold] : mosesConfigFile;
VERBOSE(1, "Rank " << rank << " reading config file from " << configFile << endl);
MosesDecoder* decoder = new MosesDecoder(configFile, verbosity, decoder_params.size(), decoder_params);
@ -442,8 +449,7 @@ int main(int argc, char** argv)
if (normaliseMargin)
cerr << "sigmoid parameter: " << sigmoidParam << endl;
}
optimiser = new MiraOptimiser(slack, scale_margin, scale_margin_precision,
scale_update, scale_update_precision, boost, normaliseMargin, sigmoidParam);
optimiser = new MiraOptimiser(slack, scale_margin, scale_update, boost, normaliseMargin, sigmoidParam);
learning_rate = mira_learning_rate;
perceptron_update = false;
} else if (learner == "perceptron") {
@ -469,9 +475,9 @@ int main(int argc, char** argv)
cerr << "Info: Setting batch size to 1 for perceptron update" << endl;
}
if (hope_n == -1)
if (hope_n == 0)
hope_n = n;
if (fear_n == -1)
if (fear_n == 0)
fear_n = n;
if (model_hope_fear || kbest)
@ -556,9 +562,10 @@ int main(int argc, char** argv)
}
bleuWeight = lmSum * bleu_weight_lm_factor;
cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl;
if (!kbest) cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl;
}
// bleu weights can be set separately for hope and fear; otherwise they are both set to 'lm weight * bleu_weight_lm_factor'
if (bleuWeight_hope == -1) {
bleuWeight_hope = bleuWeight;
}
@ -566,8 +573,10 @@ int main(int argc, char** argv)
bleuWeight_fear = bleuWeight;
}
bleuWeight_fear *= bleu_weight_fear_factor;
cerr << "Bleu weight: " << bleuWeight << endl;
cerr << "Bleu weight fear: " << bleuWeight_fear << endl;
if (!kbest) {
cerr << "Bleu weight: " << bleuWeight << endl;
cerr << "Bleu weight fear: " << bleuWeight_fear << endl;
}
if (decode_hope || decode_fear || decode_model) {
size_t decode = 1;
@ -697,7 +706,6 @@ int main(int argc, char** argv)
vector<size_t> ref_ids;
size_t actualBatchSize = 0;
vector<size_t>::const_iterator current_sid_start = sid;
size_t examples_in_batch = 0;
bool skip_example = false;
for (size_t batchPosition = 0; batchPosition < batchSize && sid
@ -802,27 +810,11 @@ int main(int argc, char** argv)
ref_length = decoder->getClosestReferenceLength(*sid, oracle.size());
avg_ref_length = ref_length;
float hope_length_ratio = (float)oracle.size()/ref_length;
int oracleSize = (int)oracle.size();
cerr << endl;
// count sparse features occurring in hope translation
featureValuesHope[batchPosition][0].IncrementSparseHopeFeatures();
float precision = bleuScoresHope[batchPosition][0];
if (historyBleu || simpleHistoryBleu) {
precision /= decoder->getTargetLengthHistory();
} else {
if (scaleByAvgInputLength) precision /= decoder->getAverageInputLength();
else if (scaleByAvgInverseLength) precision /= (100/decoder->getAverageInputLength());
precision /= scaleByX;
}
if (scale_margin_precision || scale_update_precision) {
if (historyBleu || simpleHistoryBleu || scaleByAvgInputLength || scaleByAvgInverseLength) {
cerr << "Rank " << rank << ", epoch " << epoch << ", set hope precision: " << precision << endl;
((MiraOptimiser*) optimiser)->setPrecision(precision);
}
}
vector<const Word*> bestModel;
if (debug_model || historyBleu || simpleHistoryBleu) {
// MODEL (for updating the history only, using dummy vectors)
@ -837,9 +829,9 @@ int main(int argc, char** argv)
}
// FEAR
float fear_length_ratio = 0;
//float fear_length_ratio = 0;
float bleuRatioHopeFear = 0;
int fearSize = 0;
//int fearSize = 0;
cerr << "Rank " << rank << ", epoch " << epoch << ", " << fear_n << "best fear translations" << endl;
vector< vector<const Word*> > outputFear = decoder->getNBest(input, *sid, fear_n, -1.0, bleuWeight_fear,
featureValuesFear[batchPosition], bleuScoresFear[batchPosition], modelScoresFear[batchPosition],
@ -849,8 +841,8 @@ int main(int argc, char** argv)
ref_length = decoder->getClosestReferenceLength(*sid, fear.size());
avg_ref_length += ref_length;
avg_ref_length /= 2;
fear_length_ratio = (float)fear.size()/ref_length;
fearSize = (int)fear.size();
//fear_length_ratio = (float)fear.size()/ref_length;
//fearSize = (int)fear.size();
cerr << endl;
for (size_t i = 0; i < fear.size(); ++i)
delete fear[i];
@ -994,11 +986,11 @@ int main(int argc, char** argv)
HypothesisQueue queueHope(hope_n);
HypothesisQueue queueFear(fear_n);
cerr << endl;
if (most_violated || all_violated || one_against_all) {
if (most_violated || all_violated) {
float bleuHope = -1000;
float bleuFear = 1000;
size_t indexHope = -1;
size_t indexFear = -1;
int indexHope = -1;
int indexFear = -1;
vector<float> bleuHopeList;
vector<float> bleuFearList;
@ -1029,32 +1021,23 @@ int main(int argc, char** argv)
}
float currentViolation = 0;
float minimum_bleu_diff = 0.01;
for (size_t i=0; i<bleuScores[batchPosition].size(); ++i) {
float bleuDiff = bleuHope - bleuScores[batchPosition][i];
float modelDiff = modelScores[batchPosition][indexHope] - modelScores[batchPosition][i];
if (bleuDiff > epsilon) {
if (one_against_all && bleuDiff > minimum_bleu_diff) {
cerr << ".. adding pair";
bleuHopeList.push_back(bleuHope);
bleuFearList.push_back(bleuScores[batchPosition][i]);
indexHopeList.push_back(indexHope);
indexFearList.push_back(i);
} else if (modelDiff < bleuDiff) {
float diff = bleuDiff - modelDiff;
if (diff > epsilon) {
if (all_violated) {
cerr << ".. adding pair";
bleuHopeList.push_back(bleuHope);
bleuFearList.push_back(bleuScores[batchPosition][i]);
indexHopeList.push_back(indexHope);
indexFearList.push_back(i);
} else if (most_violated && diff > currentViolation) {
currentViolation = diff;
bleuFear = bleuScores[batchPosition][i];
indexFear = i;
cerr << "Rank " << rank << ", epoch " << epoch << ", current violation: " << currentViolation << " (" << modelDiff << " >= " << bleuDiff << ")" << endl;
}
if ((bleuDiff > epsilon) && (modelDiff < bleuDiff)) {
float diff = bleuDiff - modelDiff;
if (diff > epsilon) {
if (all_violated) {
cerr << ".. adding pair";
bleuHopeList.push_back(bleuHope);
bleuFearList.push_back(bleuScores[batchPosition][i]);
indexHopeList.push_back(indexHope);
indexFearList.push_back(i);
} else if (most_violated && diff > currentViolation) {
currentViolation = diff;
bleuFear = bleuScores[batchPosition][i];
indexFear = i;
cerr << "Rank " << rank << ", epoch " << epoch << ", current violation: " << currentViolation << " (" << modelDiff << " >= " << bleuDiff << ")" << endl;
}
}
}

13
mira/MiraOptimiser.cpp
View File

@ -356,15 +356,6 @@ size_t MiraOptimiser::updateWeightsAnalytically(
cerr << "Rank " << rank << ", epoch " << epoch << ", normalised constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl;
}
if (m_scale_margin) {
diff *= bleuScoreHope;
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with oracle bleu score " << bleuScoreHope << endl;
}
if (m_scale_margin_precision) {
diff *= (1+m_precision);
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with 1+precision: " << (1+m_precision) << endl;
}
if (diff > epsilon) {
// squash it between 0 and 1
//diff = tanh(diff);
@ -401,10 +392,6 @@ size_t MiraOptimiser::updateWeightsAnalytically(
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with oracle bleu score " << bleuScoreHope << endl;
alpha *= bleuScoreHope;
}
if (m_scale_update_precision) {
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with 1+precision: " << (1+m_precision) << endl;
alpha *= (1+m_precision);
}
cerr << "Rank " << rank << ", epoch " << epoch << ", clipped/scaled alpha: " << alpha << endl;

30
mira/Optimiser.h
View File

@ -69,16 +69,21 @@ public:
MiraOptimiser() :
Optimiser() { }
MiraOptimiser(
float slack, bool scale_margin, bool scale_margin_precision,
bool scale_update, bool scale_update_precision, bool boost, bool normaliseMargin, float sigmoidParam) :
MiraOptimiser(float slack) :
Optimiser(),
m_slack(slack),
m_scale_margin(false),
m_scale_update(false),
m_boost(false),
m_normaliseMargin(false),
m_sigmoidParam(1.0) { }
MiraOptimiser(float slack, bool scale_margin, bool scale_update,
bool boost, bool normaliseMargin, float sigmoidParam) :
Optimiser(),
m_slack(slack),
m_scale_margin(scale_margin),
m_scale_margin_precision(scale_margin_precision),
m_scale_update(scale_update),
m_scale_update_precision(scale_update_precision),
m_precision(1),
m_boost(boost),
m_normaliseMargin(normaliseMargin),
m_sigmoidParam(sigmoidParam) { }
@ -148,21 +153,16 @@ public:
m_slack = slack;
}
void setPrecision(float precision) {
m_precision = precision;
}
private:
// regularise Hildreth updates
float m_slack;
// scale margin with BLEU score or precision
bool m_scale_margin, m_scale_margin_precision;
// scale update with oracle BLEU score or precision
bool m_scale_update, m_scale_update_precision;
// scale margin with BLEU score
bool m_scale_margin;
float m_precision;
// scale update with oracle BLEU score
bool m_scale_update;
// boosting of updates on misranked candidates
bool m_boost;

4
mira/training-expt.perl
View File

@ -289,8 +289,8 @@ my $learner = &param("train.learner", "mira");
my $batch = &param("train.batch", 1); # don't print this param twice (when printing training file)
my $extra_args = &param("train.extra-args");
my $by_node = &param("train.by-node");
my $slots = &param("train.slots",8);
my $jobs = &param("train.jobs",8);
my $slots = &param("train.slots",10);
my $jobs = &param("train.jobs",10);
my $mixing_frequency = &param("train.mixing-frequency", 1); # don't print this param twice
my $weight_dump_frequency = &param("train.weight-dump-frequency", 1); # don't print this param twice
my $burn_in = &param("train.burn-in");

2
misc/processLexicalTable.cpp
View File

@ -3,7 +3,7 @@
#include "moses/Timer.h"
#include "moses/InputFileStream.h"
#include "moses/LexicalReorderingTable.h"
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
using namespace Moses;

2
misc/queryLexicalTable.cpp
View File

@ -5,7 +5,7 @@
#include "moses/FactorCollection.h"
#include "moses/Timer.h"
#include "moses/InputFileStream.h"
#include "moses/LexicalReorderingTable.h"
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
using namespace Moses;

27
moses-cmd/IOWrapper.cpp
View File

@ -196,15 +196,32 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
{
CHECK(outputFactorOrder.size() > 0);
const Phrase& phrase = edge.GetCurrTargetPhrase();
bool markUnknown = StaticData::Instance().GetMarkUnknown();
if (reportAllFactors == true) {
out << phrase;
} else {
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().second;
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
out << *factor;
if (placeholderFactor != NOT_FOUND) {
const Factor *origFactor = phrase.GetFactor(pos, placeholderFactor);
if (origFactor) {
factor = origFactor;
}
}
CHECK(factor);
//preface surface form with UNK if marking unknowns
const Word &word = phrase.GetWord(pos);
if(markUnknown && word.IsOOV()) {
out << "UNK" << *factor;
} else {
out << *factor;
}
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
CHECK(factor);
@ -349,7 +366,7 @@ void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
{
if (hypo->GetPrevHypo()) {
OutputInput(map, hypo->GetPrevHypo());
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = hypo->GetSourcePhrase();
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
}
}
@ -370,6 +387,10 @@ void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, c
Backtrack(hypo);
VERBOSE(3,"0" << std::endl);
if (!m_surpressSingleBestOutput) {
if (StaticData::Instance().GetOutputHypoScore()) {
cout << hypo->GetTotalScore() << " ";
}
if (StaticData::Instance().IsPathRecoveryEnabled()) {
OutputInput(cout, hypo);
cout << "||| ";
@ -450,7 +471,7 @@ void OutputNBest(std::ostream& out
}
if (StaticData::Instance().IsPathRecoveryEnabled()) {
out << "|||";
out << " ||| ";
OutputInput(out, edges[0]);
}

2
moses-cmd/Main.cpp
View File

@ -402,7 +402,7 @@ public:
//list of unknown words
if (m_unknownsCollector) {
const vector<Phrase*>& unknowns = manager.getSntTranslationOptions()->GetUnknownSources();
const vector<const Phrase*>& unknowns = manager.getSntTranslationOptions()->GetUnknownSources();
ostringstream out;
for (size_t i = 0; i < unknowns.size(); ++i) {
out << *(unknowns[i]);

15
moses/AlignmentInfo.cpp
View File

@ -53,9 +53,22 @@ void AlignmentInfo::BuildNonTermIndexMap()
}
m_nonTermIndexMap[p->second] = i++;
}
}
std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
{
std::set<size_t> ret;
CollType::const_iterator iter;
for (iter = begin(); iter != end(); ++iter) {
const std::pair<size_t,size_t> &align = *iter;
if (iter->first == sourcePos) {
ret.insert(iter->second);
}
}
return ret;
}
bool compare_target(const std::pair<size_t,size_t> *a, const std::pair<size_t,size_t> *b)
{
if(a->second < b->second) return true;

2
moses/AlignmentInfo.h
View File

@ -69,6 +69,8 @@ public:
return m_collection;
}
std::set<size_t> GetAlignmentsForSource(size_t sourcePos) const;
size_t GetSize() const {
return m_collection.size();
}

4
moses/ChartHypothesis.cpp
View File

@ -47,7 +47,7 @@ ObjectPool<ChartHypothesis> ChartHypothesis::s_objectPool("ChartHypothesis", 300
ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
const RuleCubeItem &item,
ChartManager &manager)
:m_targetPhrase(*(item.GetTranslationDimension().GetTargetPhrase()))
:m_transOpt(item.GetTranslationDimension().GetTranslationOption())
,m_currSourceWordsRange(transOpt.GetSourceWordsRange())
,m_ffStates(StatefulFeatureFunction::GetStatefulFeatureFunctions().size())
,m_arcList(NULL)
@ -162,7 +162,7 @@ void ChartHypothesis::Evaluate()
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown);
sfs[i]->EvaluateChart(*this,&m_scoreBreakdown);
}
}

8
moses/ChartHypothesis.h
View File

@ -49,7 +49,7 @@ protected:
static ObjectPool<ChartHypothesis> s_objectPool;
#endif
const TargetPhrase &m_targetPhrase;
boost::shared_ptr<ChartTranslationOption> m_transOpt;
WordsRange m_currSourceWordsRange;
std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
@ -100,9 +100,13 @@ public:
return m_id;
}
const ChartTranslationOption &GetTranslationOption()const {
return *m_transOpt;
}
//! Get the rule that created this hypothesis
const TargetPhrase &GetCurrTargetPhrase()const {
return m_targetPhrase;
return m_transOpt->GetPhrase();
}
//! the source range that this hypothesis spans

30
moses/ChartManager.cpp
View File

@ -31,7 +31,6 @@
#include "StaticData.h"
#include "DecodeStep.h"
#include "TreeInput.h"
#include "moses/FF/PhrasePenaltyProducer.h"
#include "moses/FF/WordPenaltyProducer.h"
using namespace std;
@ -51,8 +50,14 @@ ChartManager::ChartManager(InputType const& source)
,m_start(clock())
,m_hypothesisId(0)
,m_parser(source, m_hypoStackColl)
,m_translationOptionList(StaticData::Instance().GetRuleLimit())
,m_translationOptionList(StaticData::Instance().GetRuleLimit(), source)
{
const StaticData &staticData = StaticData::Instance();
long sentenceID = source.GetTranslationId();
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
if (m_constraint) {
VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl);
}
}
ChartManager::~ChartManager()
@ -127,6 +132,8 @@ void ChartManager::ProcessSentence()
void ChartManager::AddXmlChartOptions()
{
const StaticData &staticData = StaticData::Instance();
const Phrase *constraint = GetConstraint();
const std::vector <ChartTranslationOptions*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
IFVERBOSE(2) {
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
@ -137,14 +144,24 @@ void ChartManager::AddXmlChartOptions()
i != xmlChartOptionsList.end(); ++i) {
ChartTranslationOptions* opt = *i;
TargetPhrase &targetPhrase = *opt->GetTargetPhraseCollection().GetCollection()[0];
targetPhrase.GetScoreBreakdown().Assign(staticData.GetPhrasePenaltyProducer(), -1);
targetPhrase.GetScoreBreakdown().Assign(staticData.GetWordPenaltyProducer(), -1);
const TargetPhrase &targetPhrase = opt->GetTargetPhrases()[0]->GetPhrase();
const WordsRange &range = opt->GetSourceWordsRange();
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
if (constraint) {
Phrase hypoPhrase = hypo->GetOutputPhrase();
if (!constraint->Contains(hypoPhrase)) {
delete item;
delete hypo;
continue;
}
}
hypo->Evaluate();
const Word &targetLHS = hypo->GetTargetLHS();
ChartCell &cell = m_hypoStackColl.Get(range);
cell.AddHypothesis(hypo);
}
@ -342,5 +359,4 @@ void ChartManager::CreateDeviantPaths(
}
}
} // namespace Moses

3
moses/ChartManager.h
View File

@ -59,6 +59,7 @@ private:
std::auto_ptr<SentenceStats> m_sentenceStats;
clock_t m_start; /**< starting time, used for logging */
unsigned m_hypothesisId; /* For handing out hypothesis ids to ChartHypothesis */
const Phrase *m_constraint;
ChartParser m_parser;
@ -101,6 +102,8 @@ public:
return m_hypothesisId++;
}
const Phrase *GetConstraint() const
{ return m_constraint; }
};
}

76
moses/ChartParser.cpp
View File

@ -24,6 +24,7 @@
#include "ChartRuleLookupManager.h"
#include "StaticData.h"
#include "TreeInput.h"
#include "Sentence.h"
#include "moses/FF/UnknownWordPenaltyProducer.h"
using namespace std;
@ -93,7 +94,6 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetSourcePhrase(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);
targetPhrase->SetAlignmentInfo("0-0");
@ -119,7 +119,6 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetSourcePhrase(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);
// chart rule
@ -141,14 +140,27 @@ ChartParser::ChartParser(InputType const &source, ChartCellCollectionBase &cells
p != dictionaries.end(); ++p) {
const PhraseDictionary *dict = *p;
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(source, cells));
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(*this, cells));
}
CreateInputPaths(m_source);
}
ChartParser::~ChartParser()
{
RemoveAllInColl(m_ruleLookupManagers);
StaticData::Instance().CleanUpAfterSentenceProcessing(m_source);
InputPathMatrix::const_iterator iterOuter;
for (iterOuter = m_inputPathMatrix.begin(); iterOuter != m_inputPathMatrix.end(); ++iterOuter) {
const std::vector<InputPath*> &outer = *iterOuter;
std::vector<InputPath*>::const_iterator iterInner;
for (iterInner = outer.begin(); iterInner != outer.end(); ++iterInner) {
InputPath *path = *iterInner;
delete path;
}
}
}
void ChartParser::Create(const WordsRange &wordsRange, ChartParserCallback &to)
@ -177,4 +189,62 @@ void ChartParser::Create(const WordsRange &wordsRange, ChartParserCallback &to)
}
}
void ChartParser::CreateInputPaths(const InputType &input)
{
size_t size = input.GetSize();
m_inputPathMatrix.resize(size);
CHECK(input.GetType() == SentenceInput || input.GetType() == TreeInputType);
for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
size_t endPos = startPos + phaseSize -1;
vector<InputPath*> &vec = m_inputPathMatrix[startPos];
WordsRange range(startPos, endPos);
Phrase subphrase(input.GetSubString(WordsRange(startPos, endPos)));
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
InputPath *node;
if (range.GetNumWordsCovered() == 1) {
node = new InputPath(subphrase, labels, range, NULL, NULL);
vec.push_back(node);
} else {
const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
node = new InputPath(subphrase, labels, range, &prevNode, NULL);
vec.push_back(node);
}
//m_phraseDictionaryQueue.push_back(node);
}
}
}
const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const
{
size_t offset = endPos - startPos;
CHECK(offset < m_inputPathMatrix[startPos].size());
return *m_inputPathMatrix[startPos][offset];
}
InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos)
{
size_t offset = endPos - startPos;
CHECK(offset < m_inputPathMatrix[startPos].size());
return *m_inputPathMatrix[startPos][offset];
}
/*
const Sentence &ChartParser::GetSentence() const {
const Sentence &sentence = static_cast<const Sentence&>(m_source);
return sentence;
}
*/
size_t ChartParser::GetSize() const
{
return m_source.GetSize();
}
long ChartParser::GetTranslationId() const
{
return m_source.GetTranslationId();
}
} // namespace Moses

21
moses/ChartParser.h
View File

@ -21,11 +21,11 @@
#pragma once
#include "WordsRange.h"
#include "StackVec.h"
#include <list>
#include <vector>
#include "WordsRange.h"
#include "StackVec.h"
#include "InputPath.h"
namespace Moses
{
@ -33,6 +33,7 @@ namespace Moses
class ChartParserCallback;
class ChartRuleLookupManager;
class InputType;
class Sentence;
class ChartCellCollectionBase;
class Word;
class Phrase;
@ -50,7 +51,6 @@ public:
private:
std::vector<Phrase*> m_unksrcs;
std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection;
StackVec m_emptyStackVec;
};
class ChartParser
@ -61,11 +61,24 @@ public:
void Create(const WordsRange &range, ChartParserCallback &to);
//! the sentence being decoded
//const Sentence &GetSentence() const;
long GetTranslationId() const;
size_t GetSize() const;
const InputPath &GetInputPath(size_t startPos, size_t endPos) const;
private:
ChartParserUnknown m_unknown;
std::vector <DecodeGraph*> m_decodeGraphList;
std::vector<ChartRuleLookupManager*> m_ruleLookupManagers;
InputType const& m_source; /**< source sentence to be translated */
typedef std::vector< std::vector<InputPath*> > InputPathMatrix;
InputPathMatrix m_inputPathMatrix;
void CreateInputPaths(const InputType &input);
InputPath &GetInputPath(size_t startPos, size_t endPos);
};
}

4
moses/ChartParserCallback.h
View File

@ -10,6 +10,8 @@ namespace Moses
class TargetPhraseCollection;
class WordsRange;
class TargetPhrase;
class InputPath;
class InputType;
class ChartParserCallback
{
@ -21,6 +23,8 @@ public:
virtual bool Empty() const = 0;
virtual void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range) = 0;
virtual void Evaluate(const InputType &input, const InputPath &inputPath) = 0;
};
} // namespace Moses

13
moses/ChartRuleLookupManager.cpp Normal file
View File

@ -0,0 +1,13 @@
#include "ChartRuleLookupManager.h"
#include "ChartParser.h"
namespace Moses
{
/*
const Sentence &ChartRuleLookupManager::GetSentence() const
{
return m_parser.GetSentence();
}
*/
} // namespace Moses

19
moses/ChartRuleLookupManager.h
View File

@ -26,9 +26,10 @@
namespace Moses
{
class ChartParser;
class ChartParserCallback;
class WordsRange;
class Sentence;
/** Defines an interface for looking up rules in a rule table. Concrete
* implementation classes should correspond to specific PhraseDictionary
@ -39,22 +40,22 @@ class WordsRange;
class ChartRuleLookupManager
{
public:
ChartRuleLookupManager(const InputType &sentence,
ChartRuleLookupManager(const ChartParser &parser,
const ChartCellCollectionBase &cellColl)
: m_sentence(sentence)
: m_parser(parser)
, m_cellCollection(cellColl) {}
virtual ~ChartRuleLookupManager() {}
//! the sentence being decoded
const InputType &GetSentence() const {
return m_sentence;
}
const ChartCellLabelSet &GetTargetLabelSet(size_t begin, size_t end) const {
return m_cellCollection.GetBase(WordsRange(begin, end)).GetTargetLabelSet();
}
const ChartParser &GetParser() const {
return m_parser;
}
//const Sentence &GetSentence() const;
const ChartCellLabel &GetSourceAt(size_t at) const {
return m_cellCollection.GetSourceWordLabel(at);
}
@ -73,7 +74,7 @@ private:
//! Non-copyable: copy constructor and assignment operator not implemented.
ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
const InputType &m_sentence;
const ChartParser &m_parser;
const ChartCellCollectionBase &m_cellCollection;
};

23
moses/ChartTranslationOption.cpp Normal file
View File

@ -0,0 +1,23 @@
#include "ChartTranslationOptions.h"
#include "InputType.h"
#include "InputPath.h"
namespace Moses
{
ChartTranslationOption::ChartTranslationOption(const TargetPhrase &targetPhrase)
:m_targetPhrase(targetPhrase)
,m_scoreBreakdown(targetPhrase.GetScoreBreakdown())
{
}
void ChartTranslationOption::Evaluate(const InputType &input, const InputPath &inputPath)
{
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
ff.Evaluate(input, inputPath, m_scoreBreakdown);
}
}
}

32
moses/ChartTranslationOption.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#include "ScoreComponentCollection.h"
namespace Moses
{
class TargetPhrase;
class InputPath;
class InputType;
class ChartTranslationOption
{
protected:
const TargetPhrase &m_targetPhrase;
ScoreComponentCollection m_scoreBreakdown;
public:
ChartTranslationOption(const TargetPhrase &targetPhrase);
const TargetPhrase &GetPhrase() const {
return m_targetPhrase;
}
const ScoreComponentCollection &GetScores() const {
return m_scoreBreakdown;
}
void Evaluate(const InputType &input, const InputPath &inputPath);
};
}

16
moses/ChartTranslationOptionList.cpp
View File

@ -19,16 +19,21 @@
#include <algorithm>
#include <iostream>
#include <vector>
#include "StaticData.h"
#include "ChartTranslationOptionList.h"
#include "ChartTranslationOptions.h"
#include "ChartCellCollection.h"
#include "WordsRange.h"
#include "InputType.h"
#include "InputPath.h"
using namespace std;
namespace Moses
{
ChartTranslationOptionList::ChartTranslationOptionList(size_t ruleLimit)
ChartTranslationOptionList::ChartTranslationOptionList(size_t ruleLimit, const InputType &input)
: m_size(0)
, m_ruleLimit(ruleLimit)
{
@ -142,4 +147,13 @@ void ChartTranslationOptionList::ApplyThreshold()
m_size = std::distance(m_collection.begin(), bound);
}
void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPath &inputPath)
{
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOptions &transOpts = **iter;
transOpts.Evaluate(input, inputPath);
}
}
}

6
moses/ChartTranslationOptionList.h
View File

@ -30,12 +30,14 @@ namespace Moses
class TargetPhraseCollection;
class WordsRange;
class InputType;
class InputPath;
//! a vector of translations options for a specific range, in a specific sentence
class ChartTranslationOptionList : public ChartParserCallback
{
public:
ChartTranslationOptionList(size_t);
ChartTranslationOptionList(size_t ruleLimit, const InputType &input);
~ChartTranslationOptionList();
const ChartTranslationOptions &Get(size_t i) const {
@ -58,6 +60,7 @@ public:
void Clear();
void ApplyThreshold();
void Evaluate(const InputType &input, const InputPath &inputPath);
private:
typedef std::vector<ChartTranslationOptions*> CollType;
@ -74,6 +77,7 @@ private:
size_t m_size;
float m_scoreThreshold;
const size_t m_ruleLimit;
};
}

35
moses/ChartTranslationOptions.cpp
View File

@ -18,14 +18,35 @@
***********************************************************************/
#include "ChartTranslationOptions.h"
#include "ChartHypothesis.h"
#include "ChartCellLabel.h"
#include "ChartTranslationOption.h"
namespace Moses
{
ChartTranslationOptions::ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
const StackVec &stackVec,
const WordsRange &wordsRange,
float score)
: m_stackVec(stackVec)
, m_wordsRange(&wordsRange)
, m_estimateOfBestScore(score)
{
TargetPhraseCollection::const_iterator iter;
for (iter = targetPhraseColl.begin(); iter != targetPhraseColl.end(); ++iter) {
const TargetPhrase *origTP = *iter;
boost::shared_ptr<ChartTranslationOption> ptr(new ChartTranslationOption(*origTP));
m_collection.push_back(ptr);
}
}
ChartTranslationOptions::~ChartTranslationOptions()
{
}
float ChartTranslationOptions::CalcEstimateOfBestScore(
const TargetPhraseCollection &tpc,
const StackVec &stackVec)
@ -43,4 +64,14 @@ float ChartTranslationOptions::CalcEstimateOfBestScore(
return estimateOfBestScore;
}
void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &inputPath)
{
CollType::iterator iter;
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
ChartTranslationOption &transOpt = **iter;
transOpt.Evaluate(input, inputPath);
}
}
}

26
moses/ChartTranslationOptions.h
View File

@ -26,9 +26,14 @@
#include "util/check.hh"
#include <vector>
#include <boost/shared_ptr.hpp>
#include "ChartTranslationOption.h"
namespace Moses
{
class ChartTranslationOption;
class InputPath;
class InputType;
/** Similar to a DottedRule, but contains a direct reference to a list
* of translations and provdes an estimate of the best score. For a specific range in the input sentence
@ -36,6 +41,8 @@ namespace Moses
class ChartTranslationOptions
{
public:
typedef std::vector<boost::shared_ptr<ChartTranslationOption> > CollType;
/** Constructor
\param targetPhraseColl @todo dunno
\param stackVec @todo dunno
@ -45,14 +52,8 @@ public:
ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
const StackVec &stackVec,
const WordsRange &wordsRange,
float score)
: m_stackVec(stackVec)
, m_targetPhraseCollection(&targetPhraseColl)
, m_wordsRange(&wordsRange)
, m_estimateOfBestScore(score) {
}
~ChartTranslationOptions() {}
float score);
~ChartTranslationOptions();
static float CalcEstimateOfBestScore(const TargetPhraseCollection &,
const StackVec &);
@ -63,8 +64,8 @@ public:
}
//! @todo isn't the translation suppose to just contain 1 target phrase, not a whole collection of them?
const TargetPhraseCollection &GetTargetPhraseCollection() const {
return *m_targetPhraseCollection;
const CollType &GetTargetPhrases() const {
return m_collection;
}
//! the range in the source sentence this translation option covers
@ -80,10 +81,13 @@ public:
return m_estimateOfBestScore;
}
void Evaluate(const InputType &input, const InputPath &inputPath);
private:
StackVec m_stackVec; //! vector of hypothesis list!
const TargetPhraseCollection *m_targetPhraseCollection;
CollType m_collection;
const WordsRange *m_wordsRange;
float m_estimateOfBestScore;
};

5
moses/ConfusionNet.cpp
View File

@ -64,6 +64,11 @@ ConfusionNet::ConfusionNet()
: InputType()
{
stats.createOne();
const StaticData& staticData = StaticData::Instance();
if (staticData.IsChart()) {
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
}
}
ConfusionNet::~ConfusionNet()
{

5
moses/ConfusionNet.h
View File

@ -7,6 +7,7 @@
#include <iostream>
#include "Word.h"
#include "InputType.h"
#include "NonTerminal.h"
namespace Moses
{
@ -25,6 +26,7 @@ public:
protected:
std::vector<Column> data;
NonTerminalSet m_defaultLabelSet;
bool ReadFormat0(std::istream&,const std::vector<FactorType>& factorOrder);
bool ReadFormat1(std::istream&,const std::vector<FactorType>& factorOrder);
@ -71,8 +73,7 @@ public:
TranslationOptionCollection* CreateTranslationOptionCollection() const;
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
CHECK(false);
return *(new NonTerminalSet());
return m_defaultLabelSet;
}
};

8
moses/DecodeStep.h
View File

@ -103,14 +103,6 @@ public:
const GenerationDictionary* GetGenerationDictionaryFeature() const;
/*! Given an input TranslationOption, extend it in some way (put results in outputPartialTranslOptColl) */
virtual void Process(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit
, const Phrase &src) const = 0;
void RemoveFeature(const FeatureFunction *ff);
};

8
moses/DecodeStepGeneration.cpp
View File

@ -65,8 +65,7 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection * /* toc */
, bool /*adhereTableLimit*/
, const Phrase &src) const
, bool /*adhereTableLimit*/) const
{
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
// word deletion
@ -81,6 +80,7 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
const GenerationDictionary* generationDictionary = decodeStep.GetGenerationDictionaryFeature();
const Phrase &targetPhrase = inputPartialTranslOpt.GetTargetPhrase();
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
size_t targetLength = targetPhrase.GetSize();
// generation list for each word in phrase
@ -148,13 +148,15 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
outPhrase.GetScoreBreakdown().PlusEquals(generationScore);
outPhrase.MergeFactors(genPhrase, m_newOutputFactors);
outPhrase.Evaluate(src, m_featuresToApply);
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply);
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
assert(newTransOpt);
newTransOpt->SetInputPath(inputPath);
outputPartialTranslOptColl.Add(newTransOpt);
// increment iterators

11
moses/DecodeStepGeneration.h
View File

@ -40,12 +40,11 @@ public:
const std::vector<FeatureFunction*> &features);
virtual void Process(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit
, const Phrase &src) const;
void Process(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const;
private:
};

151
moses/DecodeStepTranslation.cpp
View File

@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "TranslationOptionCollection.h"
#include "PartialTranslOptColl.h"
#include "FactorCollection.h"
#include "util/exception.hh"
using namespace std;
@ -48,7 +49,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit
, const Phrase &src) const
, const TargetPhraseCollection *phraseColl) const
{
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
// word deletion
@ -58,15 +59,13 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
// normal trans step
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
const PhraseDictionary* phraseDictionary =
decodeStep.GetPhraseDictionaryFeature();
const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
const size_t currSize = inPhrase.GetSize();
const size_t tableLimit = phraseDictionary->GetTableLimit();
const TargetPhraseCollection *phraseColl=
phraseDictionary->GetTargetPhraseCollection(toc->GetSource(),sourceWordsRange);
if (phraseColl != NULL) {
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
@ -85,13 +84,14 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
}
outPhrase.Merge(targetPhrase, m_newOutputFactors);
cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(src, m_featuresToApply)" << endl;
outPhrase.Evaluate(src, m_featuresToApply); // need to do this as all non-transcores would be screwed up
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply)" << endl;
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
assert(newTransOpt != NULL);
newTransOpt->SetInputPath(inputPath);
outputPartialTranslOptColl.Add(newTransOpt );
}
@ -101,17 +101,17 @@ cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(src, m_f
}
}
void DecodeStepTranslation::ProcessInitialTranslation(
const InputType &source
,PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit) const
, size_t startPos, size_t endPos, bool adhereTableLimit
, const InputPath &inputPath
, const TargetPhraseCollection *phraseColl) const
{
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
const size_t tableLimit = phraseDictionary->GetTableLimit();
const WordsRange wordsRange(startPos, endPos);
const TargetPhraseCollection *phraseColl = phraseDictionary->GetTargetPhraseCollection(source,wordsRange);
if (phraseColl != NULL) {
IFVERBOSE(3) {
@ -128,6 +128,8 @@ void DecodeStepTranslation::ProcessInitialTranslation(
const TargetPhrase &targetPhrase = **iterTargetPhrase;
TranslationOption *transOpt = new TranslationOption(wordsRange, targetPhrase);
transOpt->SetInputPath(inputPath);
outputPartialTranslOptColl.Add (transOpt);
VERBOSE(3,"\t" << targetPhrase << "\n");
@ -136,6 +138,135 @@ void DecodeStepTranslation::ProcessInitialTranslation(
}
}
void DecodeStepTranslation::ProcessInitialTranslationLEGACY(
const InputType &source
,PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit
, const InputPathList &inputPathList) const
{
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
const size_t tableLimit = phraseDictionary->GetTableLimit();
const WordsRange wordsRange(startPos, endPos);
const TargetPhraseCollectionWithSourcePhrase *phraseColl = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,wordsRange);
if (phraseColl != NULL) {
IFVERBOSE(3) {
if(StaticData::Instance().GetInputType() == SentenceInput)
TRACE_ERR("[" << source.GetSubString(wordsRange) << "; " << startPos << "-" << endPos << "]\n");
else
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
}
const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
std::vector<Phrase>::const_iterator iterSourcePhrase;
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
for (iterTargetPhrase = phraseColl->begin(), iterSourcePhrase = sourcePhrases.begin()
; iterTargetPhrase != iterEnd
; ++iterTargetPhrase, ++iterSourcePhrase) {
CHECK(iterSourcePhrase != sourcePhrases.end());
const TargetPhrase &targetPhrase = **iterTargetPhrase;
const Phrase &sourcePhrase = *iterSourcePhrase;
const InputPath &inputPath = GetInputPathLEGACY(targetPhrase, sourcePhrase, inputPathList);
TranslationOption *transOpt = new TranslationOption(wordsRange, targetPhrase);
transOpt->SetInputPath(inputPath);
outputPartialTranslOptColl.Add (transOpt);
VERBOSE(3,"\t" << targetPhrase << "\n");
}
VERBOSE(3,std::endl);
}
}
const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
const TargetPhrase targetPhrase,
const Phrase sourcePhrase,
const InputPathList &inputPathList) const
{
const Word &wordFromPt = sourcePhrase.GetWord(0);
InputPathList::const_iterator iter;
for (iter = inputPathList.begin(); iter != inputPathList.end(); ++iter) {
const InputPath &inputPath = **iter;
const Phrase &phraseFromIP = inputPath.GetPhrase();
const Word &wordIP = phraseFromIP.GetWord(0);
const WordsRange &range = inputPath.GetWordsRange();
if (wordFromPt == wordIP) {
return inputPath;
}
}
UTIL_THROW(util::Exception, "Input path not found");
}
void DecodeStepTranslation::ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const
{
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
// word deletion
outputPartialTranslOptColl.Add(new TranslationOption(inputPartialTranslOpt));
return;
}
// normal trans step
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
const PhraseDictionary* phraseDictionary =
decodeStep.GetPhraseDictionaryFeature();
const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
const size_t currSize = inPhrase.GetSize();
const size_t tableLimit = phraseDictionary->GetTableLimit();
const TargetPhraseCollectionWithSourcePhrase *phraseColl
= phraseDictionary->GetTargetPhraseCollectionLEGACY(toc->GetSource(),sourceWordsRange);
if (phraseColl != NULL) {
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
const TargetPhrase& targetPhrase = **iterTargetPhrase;
const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
// skip if the
if (targetPhrase.GetSize() != currSize) continue;
TargetPhrase outPhrase(inPhrase);
if (IsFilteringStep()) {
if (!inputPartialTranslOpt.IsCompatible(targetPhrase, m_conflictFactors))
continue;
}
outPhrase.Merge(targetPhrase, m_newOutputFactors);
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
assert(newTransOpt != NULL);
newTransOpt->SetInputPath(inputPath);
outputPartialTranslOptColl.Add(newTransOpt );
}
} else if (sourceWordsRange.GetNumWordsCovered() == 1) {
// unknown handler
//toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection);
}
}
}

25
moses/DecodeStepTranslation.h
View File

@ -24,12 +24,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "DecodeStep.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "InputPath.h"
namespace Moses
{
class PhraseDictionary;
class TargetPhrase;
class InputPath;
//! subclass of DecodeStep for translation step
class DecodeStepTranslation : public DecodeStep
@ -46,7 +48,7 @@ public:
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit
, const Phrase &src) const;
, const TargetPhraseCollection *phraseColl) const;
/*! initialize list of partial translation options by applying the first translation step
@ -54,9 +56,28 @@ public:
*/
void ProcessInitialTranslation(const InputType &source
, PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit) const;
, size_t startPos, size_t endPos, bool adhereTableLimit
, const InputPath &inputPath
, const TargetPhraseCollection *phraseColl) const;
// legacy
void ProcessInitialTranslationLEGACY(const InputType &source
, PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit
, const InputPathList &inputPathList) const;
void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const;
private:
// I'm not sure whether this actually works or not for binary phrase table.
// The source phrase only appears to contain the 1st word, therefore, this function
// only compares the 1st word
const InputPath &GetInputPathLEGACY(const TargetPhrase targetPhrase,
const Phrase sourcePhrase,
const InputPathList &inputPathList) const;
};

83
moses/FF/BleuScoreFeature.cpp
View File

@ -91,45 +91,60 @@ BleuScoreFeature::BleuScoreFeature(const std::string &line)
m_historySmoothing(0.9),
m_smoothing_scheme(PLUS_POINT_ONE)
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
std::cerr << "Initializing BleuScoreFeature." << std::endl;
m_tuneable = false;
if (args[0] == "references") {
vector<string> referenceFiles = Tokenize(args[1]);
CHECK(referenceFiles.size());
vector<vector<string> > references(referenceFiles.size());
ReadParameters();
std::cerr << "Finished initializing BleuScoreFeature." << std::endl;
}
for (size_t i =0; i < referenceFiles.size(); ++i) {
ifstream in(referenceFiles[i].c_str());
if (!in) {
stringstream strme;
strme << "Unable to load references from " << referenceFiles[i];
UserMessage::Add(strme.str());
void BleuScoreFeature::SetParameter(const std::string& key, const std::string& value)
{
if (key == "references") {
vector<string> referenceFiles = Tokenize(value, ",");
CHECK(referenceFiles.size());
vector<vector<string> > references(referenceFiles.size());
for (size_t i =0; i < referenceFiles.size(); ++i) {
ifstream in(referenceFiles[i].c_str());
if (!in) {
stringstream strme;
strme << "Unable to load references from " << referenceFiles[i];
UserMessage::Add(strme.str());
abort();
}
string line;
while (getline(in,line)) {
/* if (GetSearchAlgorithm() == ChartDecoding) {
stringstream tmp;
tmp << "<s> " << line << " </s>";
line = tmp.str();
}
*/
references[i].push_back(line);
}
if (i > 0) {
if (references[i].size() != references[i-1].size()) {
UserMessage::Add("Reference files are of different lengths");
abort();
}
string line;
while (getline(in,line)) {
/* if (GetSearchAlgorithm() == ChartDecoding) {
stringstream tmp;
tmp << "<s> " << line << " </s>";
line = tmp.str();
}
*/
references[i].push_back(line);
}
if (i > 0) {
if (references[i].size() != references[i-1].size()) {
UserMessage::Add("Reference files are of different lengths");
abort();
}
}
in.close();
} // for (size_t i =0; i < referenceFiles.size(); ++i) {
}
in.close();
} // for (size_t i =0; i < referenceFiles.size(); ++i) {
//Set the references in the bleu feature
LoadReferences(references);
} // if (args[0] == "references") {
} // for (size_t i = 0; i < toks.size(); ++i) {
//Set the references in the bleu feature
LoadReferences(references);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
std::vector<float> BleuScoreFeature::DefaultWeights() const
{
std::vector<float> ret(m_numScoreComponents, 1);
return ret;
}
void BleuScoreFeature::PrintHistory(std::ostream& out) const

14
moses/FF/BleuScoreFeature.h
View File

@ -67,6 +67,10 @@ public:
BleuScoreFeature(const std::string &line);
void SetParameter(const std::string& key, const std::string& value);
std::vector<float> DefaultWeights() const;
void PrintHistory(std::ostream& out) const;
void LoadReferences(const std::vector< std::vector< std::string > > &);
void SetCurrSourceLength(size_t);
@ -114,6 +118,16 @@ public:
FFState* EvaluateChart(const ChartHypothesis& cur_hypo,
int featureID,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool Enabled() const {
return m_enabled;
}

24
moses/FF/ChartBasedFeatureContext.cpp
View File

@ -1,24 +0,0 @@
#include "ChartBasedFeatureContext.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
namespace Moses
{
ChartBasedFeatureContext::ChartBasedFeatureContext
(const ChartHypothesis* hypothesis):
m_hypothesis(hypothesis),
m_targetPhrase(hypothesis->GetCurrTargetPhrase()),
m_source(hypothesis->GetManager().GetSource())
{}
ChartBasedFeatureContext::ChartBasedFeatureContext(
const TargetPhrase& targetPhrase,
const InputType& source):
m_hypothesis(NULL),
m_targetPhrase(targetPhrase),
m_source(source)
{}
}

39
moses/FF/ChartBasedFeatureContext.h
View File

@ -1,39 +0,0 @@
#pragma once
namespace Moses
{
class ChartHypothesis;
class InputType;
class TargetPhrase;
/**
* Same as PhraseBasedFeatureContext, but for chart-based Moses.
**/
class ChartBasedFeatureContext
{
//The context either has a hypothesis (during search) or a
//TargetPhrase and source sentence (during pre-calculation)
//TODO: should the context also include some info on where the TargetPhrase
//is anchored (assuming it's lexicalised), which is available at pre-calc?
const ChartHypothesis* m_hypothesis;
const TargetPhrase& m_targetPhrase;
const InputType& m_source;
public:
ChartBasedFeatureContext(const ChartHypothesis* hypothesis);
ChartBasedFeatureContext(const TargetPhrase& targetPhrase,
const InputType& source);
const InputType& GetSource() const {
return m_source;
}
const TargetPhrase& GetTargetPhrase() const {
return m_targetPhrase;
}
};
} // namespace

10
moses/FF/ControlRecombination.h
View File

@ -35,6 +35,16 @@ public:
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
//! return the state associated with the empty hypothesis for a given sentence
virtual const FFState* EmptyHypothesisState(const InputType &input) const;

2
moses/DecodeFeature.cpp → moses/FF/DecodeFeature.cpp
View File

@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <iostream>
#include "DecodeFeature.h"
#include "StaticData.h"
#include "moses/StaticData.h"
using namespace std;

20
moses/DecodeFeature.h → moses/FF/DecodeFeature.h
View File

@ -24,9 +24,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <vector>
#include "FactorTypeSet.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "TypeDef.h"
#include "moses/FactorTypeSet.h"
#include "moses/TypeDef.h"
namespace Moses
{
@ -63,6 +63,22 @@ public:
bool IsUseable(const FactorMask &mask) const;
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
protected:
std::vector<FactorType> m_input;
std::vector<FactorType> m_output;

11
moses/FF/DistortionScoreProducer.h
View File

@ -40,6 +40,17 @@ public:
ScoreComponentCollection*) const {
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
}

21
moses/FF/DynamicCacheBasedLanguageModel.cpp
View File

@ -80,7 +80,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
, ScoreComponentCollection &estimatedFutureScore) const
{
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) START" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: |" << sp << "|" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << sp << "|" << std::endl);
float score;
switch(m_query_type) {
case CBLM_QUERY_TYPE_WHOLESTRING:
@ -94,7 +94,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
}
score=-0.333333;
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: |" << tp << "| score:|" << score << "|" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << tp << "| score:|" << score << "|" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) scoreBreakdown before:|" << scoreBreakdown << "|" << std::endl);
// scoreBreakdown.Assign(this, score);
scoreBreakdown.PlusEquals(this, score);
@ -102,6 +102,22 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) END" << std::endl);
}
void DynamicCacheBasedLanguageModel::Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &input, ....) START" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << input << "|" << std::endl);
float score=-0.222222;
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: score:|" << score << "|" << std::endl);
// scoreBreakdown.Assign(this, score);
scoreBreakdown.PlusEquals(this, score);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &input, ....) END" << std::endl);
}
/*
void DynamicCacheBasedLanguageModel::Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const
{
@ -114,6 +130,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const InputType &source
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &source, ....) END" << std::endl);
}
*/
float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase& tp) const
{

25
moses/FF/DynamicCacheBasedLanguageModel.h
View File

@ -90,20 +90,29 @@ public:
void Insert(std::string &entries);
/*
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
*/
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual void Evaluate(const InputType &source
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const;
/*
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
*/
void SetQueryType(size_t type);
void SetScoreType(size_t type);

4
moses/FF/Factory.cpp
View File

@ -11,7 +11,7 @@
#include "moses/TranslationModel/PhraseDictionaryDynSuffixArray.h"
#include "moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h"
#include "moses/LexicalReordering.h"
#include "moses/FF/LexicalReordering/LexicalReordering.h"
#include "moses/FF/BleuScoreFeature.h"
#include "moses/FF/TargetWordInsertionFeature.h"
@ -127,7 +127,6 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME2("Generation", GenerationDictionary);
MOSES_FNAME(BleuScoreFeature);
MOSES_FNAME2("Distortion", DistortionScoreProducer);
MOSES_FNAME2("PhrasePenalty", PhrasePenaltyProducer);
MOSES_FNAME2("WordPenalty", WordPenaltyProducer);
MOSES_FNAME(InputFeature);
MOSES_FNAME2("PhraseDictionaryBinary", PhraseDictionaryTreeAdaptor);
@ -140,6 +139,7 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(PhraseDictionaryDynSuffixArray);
MOSES_FNAME(PhraseDictionaryDynamicCacheBased);
MOSES_FNAME(OpSequenceModel);
MOSES_FNAME(PhrasePenalty);
MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer);
MOSES_FNAME(ControlRecombination);
MOSES_FNAME(DynamicCacheBasedLanguageModel);

23
moses/FF/FeatureFunction.h
View File

@ -4,8 +4,6 @@
#include <vector>
#include <set>
#include <string>
#include "PhraseBasedFeatureContext.h"
#include "ChartBasedFeatureContext.h"
#include "moses/TypeDef.h"
namespace Moses
@ -22,8 +20,7 @@ class ScoreComponentCollection;
class WordsBitmap;
class WordsRange;
class FactorMask;
class InputPath;
/** base class for all feature functions.
*/
@ -96,16 +93,22 @@ public:
// return true if the feature function can be evaluated
virtual bool IsUseable(const FactorMask &mask) const = 0;
// used by stateless ff. And stateful ff to make initial score estimate during loading of phrase table
// used by stateless ff and stateful ff. Calculate initial score estimate during loading of phrase table
// source phrase is the substring that the phrase table uses to look up the target phrase,
// may have more factors than actually need, but not guaranteed.
// For SCFG decoding, the source contains non-terminals, NOT the raw source from the input sentence
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const {
}
, ScoreComponentCollection &estimatedFutureScore) const = 0;
virtual void Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const {
}
// This method is called once all the translation options are retrieved from the phrase table, and
// just before search.
// 'inputPath' is guaranteed to be the raw substring from the input. No factors were added or taken away
// Currently not used by any FF. Not called by moses_chart
virtual void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const = 0;
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void ReadParameters();

4
moses/FF/GlobalLexicalModel.cpp
View File

@ -167,11 +167,11 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP
}
void GlobalLexicalModel::Evaluate
(const PhraseBasedFeatureContext& context,
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
accumulator->PlusEquals( this,
GetFromCacheOrScorePhrase(context.GetTargetPhrase()) );
GetFromCacheOrScorePhrase(hypo.GetCurrTargetPhrase()) );
}
bool GlobalLexicalModel::IsUseable(const FactorMask &mask) const

18
moses/FF/GlobalLexicalModel.h
View File

@ -64,20 +64,32 @@ public:
GlobalLexicalModel(const std::string &line);
virtual ~GlobalLexicalModel();
void SetParameter(const std::string& key, const std::string& value);
void InitializeForInput( Sentence const& in );
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const PhraseBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(
const ChartBasedFeatureContext& context,
const ChartHypothesis& hypo,
ScoreComponentCollection* accumulator) const {
throw std::logic_error("GlobalLexicalModel not supported in chart decoder, yet");
}
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

10
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -90,6 +90,16 @@ public:
throw std::logic_error("GlobalLexicalModelUnlimited not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void AddFeature(ScoreComponentCollection* accumulator,
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
StringPiece targetWord) const;

32
moses/FF/InputFeature.cpp
View File

@ -1,6 +1,8 @@
#include <stdexcept>
#include "InputFeature.h"
#include "moses/Util.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/InputPath.h"
#include "util/check.hh"
using namespace std;
@ -10,20 +12,30 @@ namespace Moses
InputFeature::InputFeature(const std::string &line)
:StatelessFeatureFunction("InputFeature", line)
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
CHECK(args.size() == 2);
ReadParameters();
}
if (args[0] == "num-input-features") {
m_numInputScores = Scan<size_t>(args[1]);
} else if (args[0] == "real-word-count") {
m_numRealWordCount = Scan<size_t>(args[1]);
} else {
throw "Unknown argument " + args[0];
}
void InputFeature::SetParameter(const std::string& key, const std::string& value)
{
if (key == "num-input-features") {
m_numInputScores = Scan<size_t>(value);
} else if (key == "real-word-count") {
m_numRealWordCount = Scan<size_t>(value);
} else {
StatelessFeatureFunction::SetParameter(key, value);
}
}
void InputFeature::Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{
const ScoreComponentCollection *scores = inputPath.GetInputScore();
if (scores) {
}
}
} // namespace

18
moses/FF/InputFeature.h
View File

@ -16,6 +16,8 @@ protected:
public:
InputFeature(const std::string &line);
void SetParameter(const std::string& key, const std::string& value);
bool IsUseable(const FactorMask &mask) const {
return true;
}
@ -27,6 +29,22 @@ public:
return m_numRealWordCount;
}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
};

2
moses/LexicalReordering.cpp → moses/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -3,7 +3,7 @@
#include "moses/FF/FFState.h"
#include "LexicalReordering.h"
#include "LexicalReorderingState.h"
#include "StaticData.h"
#include "moses/StaticData.h"
using namespace std;

24
moses/LexicalReordering.h → moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -3,15 +3,16 @@
#include <string>
#include <vector>
#include "Factor.h"
#include "Phrase.h"
#include "TypeDef.h"
#include "Util.h"
#include "WordsRange.h"
#include "moses/Factor.h"
#include "moses/Phrase.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "LexicalReorderingState.h"
#include "LexicalReorderingTable.h"
#include "moses/FF/StatefulFeatureFunction.h"
#include "util/exception.hh"
namespace Moses
@ -48,10 +49,17 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis&,
int /* featureID */,
ScoreComponentCollection*) const {
CHECK(0); // not valid for chart decoder
std::cerr << "LexicalReordering::EvaluateChartr(const ChartHypothesis& hypo,...) START" << std::endl;
return NULL;
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
private:
bool DecodeCondition(std::string s);

8
moses/LexicalReorderingState.cpp → moses/FF/LexicalReordering/LexicalReorderingState.cpp
View File

@ -4,10 +4,10 @@
#include "util/check.hh"
#include "moses/FF/FFState.h"
#include "Hypothesis.h"
#include "WordsRange.h"
#include "ReorderingStack.h"
#include "TranslationOption.h"
#include "moses/Hypothesis.h"
#include "moses/WordsRange.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "LexicalReordering.h"
#include "LexicalReorderingState.h"

10
moses/LexicalReorderingState.h → moses/FF/LexicalReordering/LexicalReorderingState.h
View File

@ -4,12 +4,12 @@
#include <vector>
#include <string>
#include "Hypothesis.h"
#include "moses/Hypothesis.h"
#include "LexicalReordering.h"
#include "WordsRange.h"
#include "WordsBitmap.h"
#include "ReorderingStack.h"
#include "TranslationOption.h"
#include "moses/WordsRange.h"
#include "moses/WordsBitmap.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "moses/FF/FFState.h"

12
moses/LexicalReorderingTable.cpp → moses/FF/LexicalReordering/LexicalReorderingTable.cpp
View File

@ -1,15 +1,15 @@
#include "LexicalReorderingTable.h"
#include "InputFileStream.h"
#include "moses/InputFileStream.h"
//#include "LVoc.h" //need IPhrase
#include "StaticData.h"
#include "moses/StaticData.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "GenerationDictionary.h"
#include "TargetPhrase.h"
#include "TargetPhraseCollection.h"
#include "moses/GenerationDictionary.h"
#include "moses/TargetPhrase.h"
#include "moses/TargetPhraseCollection.h"
#ifndef WIN32
#include "TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#endif
namespace Moses

12
moses/LexicalReorderingTable.h → moses/FF/LexicalReordering/LexicalReorderingTable.h
View File

@ -13,12 +13,12 @@
#endif
//moses dependencies:
#include "TypeDef.h"
#include "Phrase.h"
#include "InputType.h"
#include "ConfusionNet.h"
#include "Sentence.h"
#include "PrefixTreeMap.h"
#include "moses/TypeDef.h"
#include "moses/Phrase.h"
#include "moses/InputType.h"
#include "moses/ConfusionNet.h"
#include "moses/Sentence.h"
#include "moses/PrefixTreeMap.h"
namespace Moses
{

49
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -13,7 +13,9 @@ namespace Moses
OpSequenceModel::OpSequenceModel(const std::string &line)
:StatefulFeatureFunction("OpSequenceModel", 5, line )
{
sFactor = 0;
tFactor = 0;
numFeatures = 5;
ReadParameters();
}
@ -48,7 +50,7 @@ void OpSequenceModel:: Evaluate(const Phrase &source
WordsBitmap myBitmap(source.GetSize());
vector <string> mySourcePhrase;
vector <string> myTargetPhrase;
vector<float> scores(5);
vector<float> scores;
vector <int> alignments;
int startIndex = 0;
int endIndex = source.GetSize();
@ -56,28 +58,27 @@ void OpSequenceModel:: Evaluate(const Phrase &source
const AlignmentInfo &align = targetPhrase.GetAlignTerm();
AlignmentInfo::const_iterator iter;
for (iter = align.begin(); iter != align.end(); ++iter) {
alignments.push_back(iter->first);
alignments.push_back(iter->second);
}
for (int i = 0; i < targetPhrase.GetSize(); i++) {
if (targetPhrase.GetWord(i).IsOOV())
if (targetPhrase.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
myTargetPhrase.push_back("_TRANS_SLF_");
else
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(0)->GetString().as_string());
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(tFactor)->GetString().as_string());
}
for (int i = 0; i < source.GetSize(); i++) {
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
}
obj.setPhrases(mySourcePhrase , myTargetPhrase);
obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
obj.computeOSMFeature(startIndex,myBitmap);
obj.calculateOSMProb(*OSM);
obj.populateScores(scores);
obj.populateScores(scores,numFeatures);
estimatedFutureScore.PlusEquals(this, scores);
}
@ -97,7 +98,7 @@ FFState* OpSequenceModel::Evaluate(
osmHypothesis obj;
vector <string> mySourcePhrase;
vector <string> myTargetPhrase;
vector<float> scores(5);
vector<float> scores;
//target.GetWord(0)
@ -141,16 +142,16 @@ FFState* OpSequenceModel::Evaluate(
for (int i = startIndex; i <= endIndex; i++) {
myBitmap.SetValue(i,0); // resetting coverage of this phrase ...
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
// cerr<<mySourcePhrase[i]<<endl;
}
for (int i = 0; i < target.GetSize(); i++) {
if (target.GetWord(i).IsOOV())
if (target.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
myTargetPhrase.push_back("_TRANS_SLF_");
else
myTargetPhrase.push_back(target.GetWord(i).GetFactor(0)->GetString().as_string());
myTargetPhrase.push_back(target.GetWord(i).GetFactor(tFactor)->GetString().as_string());
}
@ -162,7 +163,8 @@ FFState* OpSequenceModel::Evaluate(
obj.setPhrases(mySourcePhrase , myTargetPhrase);
obj.computeOSMFeature(startIndex,myBitmap);
obj.calculateOSMProb(*OSM);
obj.populateScores(scores);
obj.populateScores(scores,numFeatures);
//obj.print();
/*
if (bitmap.GetFirstGapPos() == NOT_FOUND)
@ -176,14 +178,7 @@ FFState* OpSequenceModel::Evaluate(
}
*/
/*
vector<float> scores(5);
scores[0] = 0.343423f;
scores[1] = 1.343423f;
scores[2] = 2.343423f;
scores[3] = 3.343423f;
scores[4] = 4.343423f;
*/
accumulator->PlusEquals(this, scores);
@ -226,7 +221,7 @@ std::vector<float> OpSequenceModel::GetFutureScores(const Phrase &source, const
iter = m_futureCost.find(pp);
//iter = m_coll.find(pp);
if (iter == m_futureCost.end()) {
vector<float> scores(5, 0);
vector<float> scores(numFeatures, 0);
scores[0] = unkOpProb;
return scores;
} else {
@ -240,11 +235,23 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
if (key == "path") {
m_lmPath = value;
} else if (key == "numFeatures") {
numFeatures = Scan<int>(value);
} else if (key == "order") {
lmOrder = Scan<int>(value);
} else if (key == "sFactor") {
sFactor = Scan<int>(value);
} else if (key == "tFactor") {
tFactor = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
bool OpSequenceModel::IsUseable(const FactorMask &mask) const
{
bool ret = mask[0];
return ret;
}
} // namespace

21
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -21,6 +21,9 @@ public:
int lmOrder;
float unkOpProb;
int sFactor; // Source Factor ...
int tFactor; // Target Factor ...
int numFeatures; // Number of features used ...
OpSequenceModel(const std::string &line);
@ -32,16 +35,20 @@ public:
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
virtual std::string GetScoreProducerWeightShortName(unsigned idx=0) const;
@ -49,9 +56,7 @@ public:
std::vector<float> GetFutureScores(const Phrase &source, const Phrase &target) const;
void SetParameter(const std::string& key, const std::string& value);
bool IsUseable(const FactorMask &mask) const {
return true;
}
bool IsUseable(const FactorMask &mask) const;
protected:
typedef std::pair<Phrase, Phrase> ParallelPhrase;

7
moses/FF/OSM-Feature/osmHyp.cpp
View File

@ -105,7 +105,6 @@ void osmHypothesis :: removeReorderingOperations()
deletionCount = 0;
openGapCount = 0;
gapWidth = 0;
//cout<<"I came here"<<endl;
std::vector <std::string> tupleSequence;
@ -581,10 +580,14 @@ void osmHypothesis :: constructCepts(vector <int> & align , int startIndex , int
}
void osmHypothesis :: populateScores(vector <float> & scores)
void osmHypothesis :: populateScores(vector <float> & scores , const int numFeatures)
{
scores.clear();
scores.push_back(opProb);
if (numFeatures == 1)
return;
scores.push_back(gapWidth);
scores.push_back(gapCount);
scores.push_back(openGapCount);

2
moses/FF/OSM-Feature/osmHyp.h
View File

@ -89,7 +89,7 @@ public:
void setState(const FFState* prev_state);
osmState * saveState();
void print();
void populateScores(std::vector <float> & scores);
void populateScores(std::vector <float> & scores , const int numFeatures);
void setState(const lm::ngram::State & val) {
lmState = val;
}

33
moses/FF/PhraseBasedFeatureContext.cpp
View File

@ -1,33 +0,0 @@
#include "PhraseBasedFeatureContext.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/TranslationOption.h"
namespace Moses
{
PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) :
m_hypothesis(hypothesis),
m_translationOption(m_hypothesis->GetTranslationOption()),
m_source(m_hypothesis->GetManager().GetSource()) {}
PhraseBasedFeatureContext::PhraseBasedFeatureContext
(const TranslationOption& translationOption, const InputType& source) :
m_hypothesis(NULL),
m_translationOption(translationOption),
m_source(source)
{}
const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const
{
return m_translationOption.GetTargetPhrase();
}
const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const
{
if (!m_hypothesis) {
throw std::logic_error("Coverage vector not available during pre-calculation");
}
return m_hypothesis->GetWordsBitmap();
}
}

44
moses/FF/PhraseBasedFeatureContext.h
View File

@ -1,44 +0,0 @@
#pragma once
namespace Moses
{
class Hypothesis;
class TranslationOption;
class InputType;
class TargetPhrase;
class WordsBitmap;
/**
* Contains all that a feature function can access without affecting recombination.
* For stateless features, this is all that it can access. Currently this is not
* used for stateful features, as it would need to be retro-fitted to the LM feature.
* TODO: Expose source segmentation,lattice path.
* XXX Don't add anything to the context that would break recombination XXX
**/
class PhraseBasedFeatureContext
{
// The context either has a hypothesis (during search), or a TranslationOption and
// source sentence (during pre-calculation).
const Hypothesis* m_hypothesis;
const TranslationOption& m_translationOption;
const InputType& m_source;
public:
PhraseBasedFeatureContext(const Hypothesis* hypothesis);
PhraseBasedFeatureContext(const TranslationOption& translationOption,
const InputType& source);
const TranslationOption& GetTranslationOption() const {
return m_translationOption;
}
const InputType& GetSource() const {
return m_source;
}
const TargetPhrase& GetTargetPhrase() const; //convenience method
const WordsBitmap& GetWordsBitmap() const;
};
} // namespace

7
moses/FF/PhraseBoundaryFeature.cpp
View File

@ -1,6 +1,7 @@
#include "PhraseBoundaryFeature.h"
#include "moses/Hypothesis.h"
#include "moses/TranslationOption.h"
using namespace std;
@ -77,12 +78,12 @@ FFState* PhraseBoundaryFeature::Evaluate
const Word* rightTargetWord = &(targetPhrase.GetWord(0));
AddFeatures(leftTargetWord,rightTargetWord,m_targetFactors,"tgt",scores);
const Phrase* sourcePhrase = cur_hypo.GetSourcePhrase();
const Phrase& sourcePhrase = cur_hypo.GetTranslationOption().GetInputPath().GetPhrase();
const Word* leftSourceWord = pbState->GetSourceWord();
const Word* rightSourceWord = &(sourcePhrase->GetWord(0));
const Word* rightSourceWord = &(sourcePhrase.GetWord(0));
AddFeatures(leftSourceWord,rightSourceWord,m_sourceFactors,"src",scores);
const Word* endSourceWord = &(sourcePhrase->GetWord(sourcePhrase->GetSize()-1));
const Word* endSourceWord = &(sourcePhrase.GetWord(sourcePhrase.GetSize()-1));
const Word* endTargetWord = &(targetPhrase.GetWord(targetPhrase.GetSize()-1));
//if end of sentence add EOS

11
moses/FF/PhraseBoundaryFeature.h
View File

@ -52,6 +52,17 @@ public:
ScoreComponentCollection* ) const {
throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

11
moses/FF/PhraseLengthFeature.h
View File

@ -24,11 +24,20 @@ public:
return true;
}
void EvaluateChart(const ChartBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown

10
moses/FF/PhrasePairFeature.cpp
View File

@ -106,11 +106,11 @@ void PhrasePairFeature::Load()
}
void PhrasePairFeature::Evaluate(
const PhraseBasedFeatureContext& context,
const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& target = context.GetTargetPhrase();
const Phrase& source = *(context.GetTranslationOption().GetSourcePhrase());
const TargetPhrase& target = hypo.GetCurrTargetPhrase();
const Phrase& source = hypo.GetTranslationOption().GetInputPath().GetPhrase();
if (m_simple) {
ostringstream namestr;
namestr << "pp_";
@ -131,7 +131,7 @@ void PhrasePairFeature::Evaluate(
accumulator->SparsePlusEquals(namestr.str(),1);
}
if (m_domainTrigger) {
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const bool use_topicid = input.GetUseTopicId();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
@ -199,7 +199,7 @@ void PhrasePairFeature::Evaluate(
}
}
if (m_sourceContext) {
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {

14
moses/FF/PhrasePairFeature.h
View File

@ -37,14 +37,24 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const PhraseBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartBasedFeatureContext& context,
void EvaluateChart(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Load();
void SetParameter(const std::string& key, const std::string& value);

20
moses/FF/PhrasePenaltyProducer.cpp
View File

@ -6,32 +6,34 @@ using namespace std;
namespace Moses
{
PhrasePenaltyProducer::PhrasePenaltyProducer(const std::string &line)
PhrasePenalty::PhrasePenalty(const std::string &line)
: StatelessFeatureFunction("PhrasePenalty",1, line)
{
ReadParameters();
}
void PhrasePenaltyProducer::Evaluate(const Phrase &source
void PhrasePenalty::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
std::cerr << "void PhrasePenaltyProducer::Evaluate(const Phrase &source, ...) START" << std::endl;
std::cerr << "void PhrasePenalty::Evaluate(const Phrase &source, ...) START" << std::endl;
// scoreBreakdown.Assign(this, - 1.0f);
scoreBreakdown.PlusEquals(this, - 1.0f);
std::cerr << "void PhrasePenaltyProducer::Evaluate(const Phrase &source, ...) END" << std::endl;
std::cerr << "void PhrasePenalty::Evaluate(const Phrase &source, ...) END" << std::endl;
}
void PhrasePenaltyProducer::Evaluate(const InputType &source
/*
void PhrasePenalty::Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const
{
std::cerr << "void PhrasePenaltyProducer::Evaluate(const InputType &source, ...) START" << std::endl;
std::cerr << "void PhrasePenalty::Evaluate(const InputType &source, ...) START" << std::endl;
// scoreBreakdown.Assign(this, - 1.0f);
scoreBreakdown.PlusEquals(this, - 1.0f);
std::cerr << "void PhrasePenaltyProducer::Evaluate(const InputType &source, ...) END" << std::endl;
}
std::cerr << "void PhrasePenalty::Evaluate(const InputType &source, ...) END" << std::endl;
}
*/
}

25
moses/FF/PhrasePenaltyProducer.h
View File

@ -1,18 +1,14 @@
#pragma once
#include <string>
#include "StatelessFeatureFunction.h"
#include "util/check.hh"
namespace Moses
{
class TargetPhrase;
class ScoreComponentCollection;
class PhrasePenaltyProducer : public StatelessFeatureFunction
class PhrasePenalty : public StatelessFeatureFunction
{
public:
PhrasePenaltyProducer(const std::string &line);
PhrasePenalty(const std::string &line);
bool IsUseable(const FactorMask &mask) const {
return true;
@ -23,9 +19,20 @@ public:
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual void Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
};
}
} //namespace

12
moses/FF/SourceWordDeletionFeature.h
View File

@ -28,10 +28,20 @@ public:
bool IsUseable(const FactorMask &mask) const;
virtual void Evaluate(const Phrase &source
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,

10
moses/FF/StatelessFeatureFunction.h
View File

@ -23,16 +23,14 @@ public:
/**
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const {
}
virtual void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const = 0;
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const = 0;
virtual bool IsStateless() const {
return true;

12
moses/FF/TargetBigramFeature.h
View File

@ -45,8 +45,18 @@ public:
virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const {
abort();
throw std::logic_error("TargetBigramFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

11
moses/FF/TargetNgramFeature.h
View File

@ -191,6 +191,17 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

10
moses/FF/TargetWordInsertionFeature.h
View File

@ -32,6 +32,16 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,

16
moses/FF/UnknownWordPenaltyProducer.h
View File

@ -22,6 +22,22 @@ public:
}
std::vector<float> DefaultWeights() const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
}

10
moses/FF/WordPenaltyProducer.h
View File

@ -22,6 +22,16 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
/*

155
moses/FF/WordTranslationFeature.cpp
View File

@ -137,16 +137,17 @@ void WordTranslationFeature::Load()
}
void WordTranslationFeature::Evaluate
(const PhraseBasedFeatureContext& context,
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const TranslationOption& transOpt = hypo.GetTranslationOption();
const TargetPhrase& targetPhrase = hypo.GetCurrTargetPhrase();
const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();
// process aligned words
for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) {
const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase();
const Phrase& sourcePhrase = transOpt.GetInputPath().GetPhrase();
int sourceIndex = alignmentPoint->first;
int targetIndex = alignmentPoint->second;
Word ws = sourcePhrase.GetWord(sourceIndex);
@ -242,7 +243,7 @@ void WordTranslationFeature::Evaluate
}
}
if (m_sourceContext) {
size_t globalSourceIndex = context.GetTranslationOption().GetStartPos() + sourceIndex;
size_t globalSourceIndex = hypo.GetTranslationOption().GetStartPos() + sourceIndex;
if (!m_domainTrigger && globalSourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
@ -348,150 +349,10 @@ void WordTranslationFeature::Evaluate
}
void WordTranslationFeature::EvaluateChart(
const ChartBasedFeatureContext& context,
const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
// process aligned words
for (AlignmentInfo::const_iterator alignmentPoint = alignmentInfo.begin(); alignmentPoint != alignmentInfo.end(); alignmentPoint++) {
const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase();
int sourceIndex = alignmentPoint->first;
int targetIndex = alignmentPoint->second;
Word ws = sourcePhrase.GetWord(sourceIndex);
if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue;
Word wt = targetPhrase.GetWord(targetIndex);
if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue;
StringPiece sourceWord = ws.GetFactor(m_factorTypeSource)->GetString();
StringPiece targetWord = wt.GetFactor(m_factorTypeTarget)->GetString();
if (m_ignorePunctuation) {
// check if source or target are punctuation
char firstChar = sourceWord[0];
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
firstChar = targetWord[0];
charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
if (!m_unrestricted) {
if (FindStringPiece(m_vocabSource, sourceWord) == m_vocabSource.end())
sourceWord = "OTHER";
if (FindStringPiece(m_vocabTarget, targetWord) == m_vocabTarget.end())
targetWord = "OTHER";
}
if (m_simple) {
// construct feature name
stringstream featureName;
featureName << m_description << "_";
//featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER");
featureName << sourceWord;
featureName << "~";
//featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER");
featureName << targetWord;
accumulator->SparsePlusEquals(featureName.str(), 1);
}
/* if (m_sourceContext) {
size_t globalSourceIndex = cur_hypo.GetCurrSourceRange().GetStartPos() + sourceIndex;
if (globalSourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
feature << "wt_";
feature << "<s>,";
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
cerr << feature.str() << endl;
}
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
if (contextIndex == globalSourceIndex) continue;
string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString();
if (m_ignorePunctuation) {
// check if trigger is punctuation
char firstChar = sourceTrigger.at(0);
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
bool sourceTriggerExists = false;
if (!m_unrestricted)
sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end();
if (m_unrestricted || sourceTriggerExists) {
stringstream feature;
feature << "wt_";
if (contextIndex < globalSourceIndex) {
feature << sourceTrigger;
feature << ",";
feature << sourceWord;
}
else {
feature << sourceWord;
feature << ",";
feature << sourceTrigger;
}
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
cerr << feature.str() << endl;
}
}
}*/
/* if (m_targetContext) {
size_t globalTargetIndex = 0; // TODO
// size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex;
if (globalTargetIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
feature << "wt_";
feature << sourceWord;
feature << "~";
feature << "<s>,";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
cerr << feature.str() << endl;
}
// range over target words (up to current position) to get context
for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) {
Phrase outputPhrase = cur_hypo.GetOutputPhrase();
string targetTrigger = outputPhrase.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
//string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
if (m_ignorePunctuation) {
// check if trigger is punctuation
char firstChar = targetTrigger.at(0);
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
if(charIterator != m_punctuationHash.end())
continue;
}
bool targetTriggerExists = false;
if (!m_unrestricted)
targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end();
if (m_unrestricted || targetTriggerExists) {
stringstream feature;
feature << "wt_";
feature << sourceWord;
feature << "~";
feature << targetTrigger;
feature << ",";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
cerr << feature.str() << endl;
}
}
}*/
}
UTIL_THROW(util::Exception, "Need source phrase. Can't be arsed at the moment");
}
bool WordTranslationFeature::IsUseable(const FactorMask &mask) const

Some files were not shown because too many files have changed in this diff Show More