mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
merge master into this branch
This commit is contained in:
commit
50970b2b59
4
.gitignore
vendored
4
.gitignore
vendored
@ -67,3 +67,7 @@ contrib/other-builds/*.xcodeproj/xcuserdata/
|
||||
*/*.xcodeproj/xcuserdata
|
||||
|
||||
mert/sentence-bleu
|
||||
._*
|
||||
.DS_Store
|
||||
*.pbxuser
|
||||
*.mode1v3
|
||||
|
13
Jamroot
13
Jamroot
@ -19,7 +19,8 @@
|
||||
#--with-cmph=/path/to/cmph
|
||||
#
|
||||
#Thread-caching malloc (if present, used for multi-threaded builds by default)
|
||||
#--without-tcmalloc
|
||||
#--without-tcmalloc does not compile with tcmalloc even if present
|
||||
#--full-tcmalloc links against the full version (useful for memory profiling)
|
||||
#
|
||||
#REGRESSION TESTING
|
||||
#--with-regtest=/path/to/moses-reg-test-data
|
||||
@ -76,8 +77,14 @@ boost 103600 ;
|
||||
external-lib z ;
|
||||
|
||||
if ! [ option.get "without-tcmalloc" : : "yes" ] && [ test_library "tcmalloc_minimal" ] {
|
||||
external-lib tcmalloc_minimal ;
|
||||
requirements += <threading>multi:<library>tcmalloc_minimal ;
|
||||
if [ option.get "full-tcmalloc" : : "yes" ] {
|
||||
external-lib unwind ;
|
||||
external-lib tcmalloc_and_profiler : : unwind ;
|
||||
requirements += <library>tcmalloc_and_profiler <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
|
||||
} else {
|
||||
external-lib tcmalloc_minimal ;
|
||||
requirements += <threading>multi:<library>$(tcmalloc) ;
|
||||
}
|
||||
} else {
|
||||
echo "Tip: install tcmalloc for faster threading. See BUILD-INSTRUCTIONS.txt for more information." ;
|
||||
}
|
||||
|
@ -191,13 +191,7 @@ UINT64 OnDiskWrapper::GetMisc(const std::string &key) const
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
PhraseNode &OnDiskWrapper::GetRootSourceNode()
|
||||
{
|
||||
return *m_rootSourceNode;
|
||||
}
|
||||
|
||||
Word *OnDiskWrapper::ConvertFromMoses(Moses::FactorDirection /* direction */
|
||||
, const std::vector<Moses::FactorType> &factorsVec
|
||||
Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
|
||||
, const Moses::Word &origWord) const
|
||||
{
|
||||
bool isNonTerminal = origWord.IsNonTerminal();
|
||||
|
@ -95,12 +95,16 @@ public:
|
||||
return 1;
|
||||
}
|
||||
|
||||
PhraseNode &GetRootSourceNode();
|
||||
PhraseNode &GetRootSourceNode() {
|
||||
return *m_rootSourceNode;
|
||||
}
|
||||
const PhraseNode &GetRootSourceNode() const {
|
||||
return *m_rootSourceNode;
|
||||
}
|
||||
|
||||
UINT64 GetMisc(const std::string &key) const;
|
||||
|
||||
Word *ConvertFromMoses(Moses::FactorDirection direction
|
||||
, const std::vector<Moses::FactorType> &factorsVec
|
||||
Word *ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
|
||||
, const Moses::Word &origWord) const;
|
||||
|
||||
};
|
||||
|
@ -227,14 +227,17 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
|
||||
, const std::vector<Moses::FactorType> &outputFactors
|
||||
, const Vocab &vocab
|
||||
, const Moses::PhraseDictionary &phraseDict
|
||||
, const std::vector<float> &weightT) const
|
||||
, const std::vector<float> &weightT
|
||||
, bool isSyntax) const
|
||||
{
|
||||
Moses::TargetPhrase *ret = new Moses::TargetPhrase();
|
||||
|
||||
// words
|
||||
size_t phraseSize = GetSize();
|
||||
CHECK(phraseSize > 0); // last word is lhs
|
||||
--phraseSize;
|
||||
if (isSyntax) {
|
||||
--phraseSize;
|
||||
}
|
||||
|
||||
for (size_t pos = 0; pos < phraseSize; ++pos) {
|
||||
GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord());
|
||||
@ -261,16 +264,17 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
|
||||
ret->SetAlignTerm(alignTerm);
|
||||
ret->SetAlignNonTerm(alignNonTerm);
|
||||
|
||||
Moses::Word *lhsTarget = new Moses::Word(true);
|
||||
GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
|
||||
ret->SetTargetLHS(lhsTarget);
|
||||
if (isSyntax) {
|
||||
Moses::Word *lhsTarget = new Moses::Word(true);
|
||||
GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
|
||||
ret->SetTargetLHS(lhsTarget);
|
||||
}
|
||||
|
||||
// set source phrase
|
||||
Moses::Phrase mosesSP(Moses::Input);
|
||||
for (size_t pos = 0; pos < sp->GetSize(); ++pos) {
|
||||
sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord());
|
||||
}
|
||||
ret->SetSourcePhrase(mosesSP);
|
||||
|
||||
// scores
|
||||
ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);
|
||||
|
@ -103,7 +103,8 @@ public:
|
||||
, const std::vector<Moses::FactorType> &outputFactors
|
||||
, const Vocab &vocab
|
||||
, const Moses::PhraseDictionary &phraseDict
|
||||
, const std::vector<float> &weightT) const;
|
||||
, const std::vector<float> &weightT
|
||||
, bool isSyntax) const;
|
||||
UINT64 ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl);
|
||||
UINT64 ReadFromFile(std::fstream &fileTP);
|
||||
|
||||
|
@ -117,8 +117,8 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std:
|
||||
, const std::vector<Moses::FactorType> &outputFactors
|
||||
, const Moses::PhraseDictionary &phraseDict
|
||||
, const std::vector<float> &weightT
|
||||
, const std::string & /* filePath */
|
||||
, Vocab &vocab) const
|
||||
, Vocab &vocab
|
||||
, bool isSyntax) const
|
||||
{
|
||||
Moses::TargetPhraseCollection *ret = new Moses::TargetPhraseCollection();
|
||||
|
||||
@ -128,7 +128,8 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std:
|
||||
Moses::TargetPhrase *mosesPhrase = tp.ConvertToMoses(inputFactors, outputFactors
|
||||
, vocab
|
||||
, phraseDict
|
||||
, weightT);
|
||||
, weightT
|
||||
, isSyntax);
|
||||
|
||||
/*
|
||||
// debugging output
|
||||
|
@ -73,8 +73,8 @@ public:
|
||||
, const std::vector<Moses::FactorType> &outputFactors
|
||||
, const Moses::PhraseDictionary &phraseDict
|
||||
, const std::vector<float> &weightT
|
||||
, const std::string &filePath
|
||||
, Vocab &vocab) const;
|
||||
, Vocab &vocab
|
||||
, bool isSyntax) const;
|
||||
void ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper);
|
||||
|
||||
const std::string GetDebugStr() const;
|
||||
|
@ -18,7 +18,7 @@
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1410559002." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.1035891586" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.242178856" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract-ghkm/Debug}" id="cdt.managedbuild.builder.gnu.cross.430400318" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract-ghkm/Debug}" id="cdt.managedbuild.builder.gnu.cross.430400318" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.251687262" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.962699619" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.debugging.level.230503798" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
@ -126,5 +126,12 @@
|
||||
</scannerConfigBuildInfo>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
|
||||
<storageModule moduleId="refreshScope"/>
|
||||
<storageModule moduleId="refreshScope" versionNumber="2">
|
||||
<configuration configurationName="Release">
|
||||
<resource resourceType="PROJECT" workspacePath="/extract-ghkm"/>
|
||||
</configuration>
|
||||
<configuration configurationName="Debug">
|
||||
<resource resourceType="PROJECT" workspacePath="/extract-ghkm"/>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
</cproject>
|
||||
|
129
contrib/other-builds/extract-rules/.cproject
Normal file
129
contrib/other-builds/extract-rules/.cproject
Normal file
@ -0,0 +1,129 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<storageModule moduleId="org.eclipse.cdt.core.settings">
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.124769989" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.266544803" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract-rules}/Debug" id="cdt.managedbuild.builder.gnu.cross.335858926" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1376077469" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.947547329" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.debugging.level.426953885" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.include.paths.1671695899" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" valueType="includePath"/>
|
||||
<option id="gnu.c.compiler.option.include.files.1838960067" name="Include files (-include)" superClass="gnu.c.compiler.option.include.files" valueType="includeFiles"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.985831394" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.53480540" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
|
||||
<option id="gnu.cpp.compiler.option.optimization.level.1726371873" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.debugging.level.899893408" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.include.paths.1099087456" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../boost/include""/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.88958138" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1616232021" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1411857637" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
|
||||
<option id="gnu.cpp.link.option.libs.109133121" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="z"/>
|
||||
<listOptionValue builtIn="false" value="boost_iostreams-mt"/>
|
||||
<listOptionValue builtIn="false" value="boost_system-mt"/>
|
||||
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.paths.1030374421" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.272393234" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1391783790" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.2066621509" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1945638157" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.1200693544." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1113964425" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1722595316" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract-rules}/Release" id="cdt.managedbuild.builder.gnu.cross.691589832" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.593530229" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
|
||||
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.1320426973" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.debugging.level.947026588" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1217031668" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1401773863" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
|
||||
<option id="gnu.cpp.compiler.option.optimization.level.1504181086" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.debugging.level.1645775798" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1484987112" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1807515346" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.44234391" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1468234013" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.467923425" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1673313707" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.518252425" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<project id="extract-rules.cdt.managedbuild.target.gnu.cross.exe.1916763759" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
|
||||
</storageModule>
|
||||
<storageModule moduleId="scannerConfiguration">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292;cdt.managedbuild.config.gnu.cross.exe.debug.1438215292.;cdt.managedbuild.tool.gnu.cross.c.compiler.1376077469;cdt.managedbuild.tool.gnu.c.compiler.input.985831394">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1438215292;cdt.managedbuild.config.gnu.cross.exe.debug.1438215292.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.53480540;cdt.managedbuild.tool.gnu.cpp.compiler.input.88958138">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1200693544;cdt.managedbuild.config.gnu.cross.exe.release.1200693544.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1401773863;cdt.managedbuild.tool.gnu.cpp.compiler.input.1484987112">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1200693544;cdt.managedbuild.config.gnu.cross.exe.release.1200693544.;cdt.managedbuild.tool.gnu.cross.c.compiler.593530229;cdt.managedbuild.tool.gnu.c.compiler.input.1217031668">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
|
||||
</scannerConfigBuildInfo>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
|
||||
<storageModule moduleId="refreshScope"/>
|
||||
</cproject>
|
1
contrib/other-builds/extract-rules/.gitignore
vendored
Normal file
1
contrib/other-builds/extract-rules/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
/Debug
|
134
contrib/other-builds/extract-rules/.project
Normal file
134
contrib/other-builds/extract-rules/.project
Normal file
@ -0,0 +1,134 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>extract-rules</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<triggers>full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
</natures>
|
||||
<linkedResources>
|
||||
<link>
|
||||
<name>ExtractedRule.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/ExtractedRule.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ExtractedRule.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/ExtractedRule.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>Hole.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/Hole.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>HoleCollection.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>HoleCollection.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputFileStream.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputFileStream.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignment.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignment.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignmentWithSyntax.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignmentWithSyntax.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignmentWithSyntax.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignmentWithSyntax.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SyntaxTree.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SyntaxTree.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SyntaxTree.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SyntaxTree.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>XmlTree.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/XmlTree.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>XmlTree.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/XmlTree.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>extract-rules-main.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/extract-rules-main.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>gzfilebuf.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/gzfilebuf.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tables-core.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tables-core.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h</locationURI>
|
||||
</link>
|
||||
</linkedResources>
|
||||
</projectDescription>
|
127
contrib/other-builds/extract/.cproject
Normal file
127
contrib/other-builds/extract/.cproject
Normal file
@ -0,0 +1,127 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<storageModule moduleId="org.eclipse.cdt.core.settings">
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.386290689." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.671913278" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1231657738" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract}/Debug" id="cdt.managedbuild.builder.gnu.cross.571044108" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.332036857" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.1292572253" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.debugging.level.1873227592" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1165888615" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1342023600" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
|
||||
<option id="gnu.cpp.compiler.option.optimization.level.698819695" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.debugging.level.1451916947" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.include.paths.1702398011" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../boost/include""/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.579278848" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1856691234" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1699542791" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
|
||||
<option id="gnu.cpp.link.option.libs.1880730637" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="z"/>
|
||||
<listOptionValue builtIn="false" value="boost_iostreams-mt"/>
|
||||
<listOptionValue builtIn="false" value="boost_system-mt"/>
|
||||
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.paths.298225069" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1339210059" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.976825054" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1971927463" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.704926167" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.140124152">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.140124152" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.140124152" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.140124152." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1250240843" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.597335968" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
<builder buildPath="${workspace_loc:/extract}/Release" id="cdt.managedbuild.builder.gnu.cross.95066247" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.2096762162" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
|
||||
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.88795016" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.debugging.level.383328020" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.681105644" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1806684544" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
|
||||
<option id="gnu.cpp.compiler.option.optimization.level.553394848" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.debugging.level.1420596769" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1726759263" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.234409052" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.320346578" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.2045242811" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.archiver.417132714" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1944597759" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.203400619" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<project id="extract.cdt.managedbuild.target.gnu.cross.exe.1220534104" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
|
||||
</storageModule>
|
||||
<storageModule moduleId="scannerConfiguration">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.140124152;cdt.managedbuild.config.gnu.cross.exe.release.140124152.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1806684544;cdt.managedbuild.tool.gnu.cpp.compiler.input.1726759263">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.140124152;cdt.managedbuild.config.gnu.cross.exe.release.140124152.;cdt.managedbuild.tool.gnu.cross.c.compiler.2096762162;cdt.managedbuild.tool.gnu.c.compiler.input.681105644">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.386290689;cdt.managedbuild.config.gnu.cross.exe.debug.386290689.;cdt.managedbuild.tool.gnu.cross.c.compiler.332036857;cdt.managedbuild.tool.gnu.c.compiler.input.1165888615">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileC"/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.386290689;cdt.managedbuild.config.gnu.cross.exe.debug.386290689.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1342023600;cdt.managedbuild.tool.gnu.cpp.compiler.input.579278848">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId="org.eclipse.cdt.managedbuilder.core.GCCManagedMakePerProjectProfileCPP"/>
|
||||
</scannerConfigBuildInfo>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
|
||||
<storageModule moduleId="refreshScope"/>
|
||||
</cproject>
|
74
contrib/other-builds/extract/.project
Normal file
74
contrib/other-builds/extract/.project
Normal file
@ -0,0 +1,74 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>extract</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<triggers>full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
</natures>
|
||||
<linkedResources>
|
||||
<link>
|
||||
<name>InputFileStream.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputFileStream.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/InputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/OutputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignment.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>SentenceAlignment.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/SentenceAlignment.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>extract-main.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/extract-main.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tables-core.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>tables-core.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/tables-core.h</locationURI>
|
||||
</link>
|
||||
</linkedResources>
|
||||
</projectDescription>
|
@ -34,7 +34,6 @@
|
||||
1EBA44E314B97E22003CC0EA /* model_type.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449114B97E22003CC0EA /* model_type.hh */; };
|
||||
1EBA44E414B97E22003CC0EA /* model.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449214B97E22003CC0EA /* model.cc */; };
|
||||
1EBA44E514B97E22003CC0EA /* model.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449314B97E22003CC0EA /* model.hh */; };
|
||||
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449414B97E22003CC0EA /* ngram_query.cc */; };
|
||||
1EBA44E714B97E22003CC0EA /* ngram_query.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449514B97E22003CC0EA /* ngram_query.hh */; };
|
||||
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449614B97E22003CC0EA /* quantize.cc */; };
|
||||
1EBA44E914B97E22003CC0EA /* quantize.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449714B97E22003CC0EA /* quantize.hh */; };
|
||||
@ -149,7 +148,6 @@
|
||||
1EBA449114B97E22003CC0EA /* model_type.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model_type.hh; path = ../../lm/model_type.hh; sourceTree = "<group>"; };
|
||||
1EBA449214B97E22003CC0EA /* model.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = model.cc; path = ../../lm/model.cc; sourceTree = "<group>"; };
|
||||
1EBA449314B97E22003CC0EA /* model.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model.hh; path = ../../lm/model.hh; sourceTree = "<group>"; };
|
||||
1EBA449414B97E22003CC0EA /* ngram_query.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ngram_query.cc; path = ../../lm/ngram_query.cc; sourceTree = "<group>"; };
|
||||
1EBA449514B97E22003CC0EA /* ngram_query.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = ngram_query.hh; path = ../../lm/ngram_query.hh; sourceTree = "<group>"; };
|
||||
1EBA449614B97E22003CC0EA /* quantize.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = quantize.cc; path = ../../lm/quantize.cc; sourceTree = "<group>"; };
|
||||
1EBA449714B97E22003CC0EA /* quantize.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = quantize.hh; path = ../../lm/quantize.hh; sourceTree = "<group>"; };
|
||||
@ -266,7 +264,6 @@
|
||||
1EBA449114B97E22003CC0EA /* model_type.hh */,
|
||||
1EBA449214B97E22003CC0EA /* model.cc */,
|
||||
1EBA449314B97E22003CC0EA /* model.hh */,
|
||||
1EBA449414B97E22003CC0EA /* ngram_query.cc */,
|
||||
1EBA449514B97E22003CC0EA /* ngram_query.hh */,
|
||||
1EBA449614B97E22003CC0EA /* quantize.cc */,
|
||||
1EBA449714B97E22003CC0EA /* quantize.hh */,
|
||||
@ -535,7 +532,6 @@
|
||||
1EBA44DF14B97E22003CC0EA /* lm_exception.cc in Sources */,
|
||||
1EBA44E214B97E22003CC0EA /* model_test.cc in Sources */,
|
||||
1EBA44E414B97E22003CC0EA /* model.cc in Sources */,
|
||||
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */,
|
||||
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */,
|
||||
1EBA44EA14B97E22003CC0EA /* read_arpa.cc in Sources */,
|
||||
1EBA44ED14B97E22003CC0EA /* search_hashed.cc in Sources */,
|
||||
@ -589,6 +585,7 @@
|
||||
GCC_C_LANGUAGE_STANDARD = gnu99;
|
||||
GCC_DYNAMIC_NO_PIC = NO;
|
||||
GCC_ENABLE_OBJC_EXCEPTIONS = YES;
|
||||
GCC_INPUT_FILETYPE = automatic;
|
||||
GCC_OPTIMIZATION_LEVEL = 0;
|
||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
||||
"DEBUG=1",
|
||||
@ -601,10 +598,11 @@
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
HEADER_SEARCH_PATHS = (
|
||||
"$(LIBRARY_MY_BOOST)",
|
||||
../..,
|
||||
/opt/local/include,
|
||||
);
|
||||
MACOSX_DEPLOYMENT_TARGET = 10.7;
|
||||
MACOSX_DEPLOYMENT_TARGET = 10.6;
|
||||
ONLY_ACTIVE_ARCH = YES;
|
||||
SDKROOT = macosx;
|
||||
};
|
||||
@ -625,10 +623,11 @@
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
HEADER_SEARCH_PATHS = (
|
||||
"$(XCODE_MOSES_BOOST)",
|
||||
../..,
|
||||
/opt/local/include,
|
||||
);
|
||||
MACOSX_DEPLOYMENT_TARGET = 10.7;
|
||||
MACOSX_DEPLOYMENT_TARGET = 10.6;
|
||||
SDKROOT = macosx;
|
||||
};
|
||||
name = Release;
|
||||
|
@ -1,5 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<?fileVersion 4.0.0?>
|
||||
|
||||
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<storageModule moduleId="org.eclipse.cdt.core.settings">
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.162355801">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.162355801" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
|
@ -1,5 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<?fileVersion 4.0.0?>
|
||||
|
||||
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<storageModule moduleId="org.eclipse.cdt.core.settings">
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.461114338">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.461114338" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -71,6 +71,9 @@
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.511477442" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
|
||||
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1742823107" name="ChartTranslationOption.cpp" rcbsApplicability="disable" resourcePath="ChartTranslationOption.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050">
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327.1616881050" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327"/>
|
||||
</fileInfo>
|
||||
<sourceEntries>
|
||||
<entry excluding="FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
|
||||
</sourceEntries>
|
||||
|
@ -191,11 +191,26 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartParserCallback.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ChartRuleLookupManager.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartRuleLookupManager.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ChartRuleLookupManager.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartRuleLookupManager.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ChartTranslationOption.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartTranslationOption.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ChartTranslationOption.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ChartTranslationOption.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>ChartTranslationOptionList.cpp</name>
|
||||
<type>1</type>
|
||||
@ -271,16 +286,6 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/ConfusionNet.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>DecodeFeature.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>DecodeFeature.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>DecodeGraph.cpp</name>
|
||||
<type>1</type>
|
||||
@ -471,6 +476,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/InputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputPath.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/InputPath.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputPath.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/InputPath.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>InputType.cpp</name>
|
||||
<type>1</type>
|
||||
@ -501,36 +516,6 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LVoc.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReordering.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReordering.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReorderingState.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReorderingState.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReorderingTable.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LexicalReorderingTable.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>Manager.cpp</name>
|
||||
<type>1</type>
|
||||
@ -1051,16 +1036,6 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/ChartBasedFeatureContext.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/ChartBasedFeatureContext.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/ControlRecombination.cpp</name>
|
||||
<type>1</type>
|
||||
@ -1071,6 +1046,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ControlRecombination.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/DecodeFeature.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/DecodeFeature.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/DistortionScoreProducer.cpp</name>
|
||||
<type>1</type>
|
||||
@ -1142,19 +1127,14 @@
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/OSM-Feature</name>
|
||||
<name>FF/LexicalReordering</name>
|
||||
<type>2</type>
|
||||
<locationURI>virtual:/virtual</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/PhraseBasedFeatureContext.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/PhraseBasedFeatureContext.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.h</locationURI>
|
||||
<name>FF/OSM-Feature</name>
|
||||
<type>2</type>
|
||||
<locationURI>virtual:/virtual</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/PhraseBoundaryFeature.cpp</name>
|
||||
@ -1626,6 +1606,36 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReordering.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReordering.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReorderingState.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReorderingState.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReorderingTable.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/LexicalReordering/LexicalReorderingTable.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>FF/OSM-Feature/OpSequenceModel.cpp</name>
|
||||
<type>1</type>
|
||||
|
@ -246,9 +246,6 @@ public:
|
||||
if (multiModelWeights.size() > 0) {
|
||||
PhraseDictionaryMultiModel* pdmm = (PhraseDictionaryMultiModel*) staticData.GetPhraseDictionaries()[0]; //TODO: only works if multimodel is first phrase table
|
||||
pdmm->SetTemporaryMultiModelWeightsVector(multiModelWeights);
|
||||
if (staticData.GetUseTransOptCache()) {
|
||||
cerr << "Warning: -use-persistent-cache is set to true; sentence-specific weights may be ignored. Disable cache for true results.\n";
|
||||
}
|
||||
}
|
||||
|
||||
stringstream out, graphInfo, transCollOpts;
|
||||
|
@ -73,17 +73,30 @@ rule test_library ( name ) {
|
||||
constant CLEANING : $(cleaning) ;
|
||||
}
|
||||
|
||||
shared-command-line = ;
|
||||
local argv = [ modules.peek : ARGV ] ;
|
||||
while $(argv) {
|
||||
if $(argv[1]) = "link=shared" {
|
||||
shared-command-line = <link>shared ;
|
||||
}
|
||||
argv = $(argv[2-]) ;
|
||||
}
|
||||
|
||||
#Determine if a library can be compiled statically.
|
||||
rule auto-shared ( name : additional * ) {
|
||||
additional ?= "" ;
|
||||
if [ test_flags $(additional)" -static -l"$(name) ] {
|
||||
return ;
|
||||
if $(shared-command-line) = "<link>shared" {
|
||||
return "<link>shared" ;
|
||||
} else {
|
||||
if $(FORCE-STATIC) {
|
||||
echo "Could not statically link against lib $(name). Your build will probably fail." ;
|
||||
if [ test_flags $(additional)" -static -l"$(name) ] {
|
||||
return ;
|
||||
} else {
|
||||
return "<link>shared" ;
|
||||
if $(FORCE-STATIC) {
|
||||
echo "Could not statically link against lib $(name). Your build will probably fail." ;
|
||||
return ;
|
||||
} else {
|
||||
return "<link>shared" ;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -133,8 +146,8 @@ rule boost-lib ( name macro : deps * ) {
|
||||
flags += " -static" ;
|
||||
}
|
||||
if [ test_flags $(flags) : $(main) ] {
|
||||
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
|
||||
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : : <library>$(deps) ;
|
||||
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : <link>static : <library>$(deps) ;
|
||||
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : <link>static : <library>$(deps) ;
|
||||
} else {
|
||||
lib inner_boost_$(name) : : $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
|
||||
}
|
||||
@ -143,7 +156,7 @@ rule boost-lib ( name macro : deps * ) {
|
||||
alias boost_$(name) : inner_boost_$(name) : <link>shared ;
|
||||
requirements += <define>BOOST_$(macro) ;
|
||||
} else {
|
||||
alias boost_$(name) : inner_boost_$(name) : : : <link>shared:<define>BOOST_$(macro) ;
|
||||
alias boost_$(name) : inner_boost_$(name) : <link>static ;
|
||||
}
|
||||
}
|
||||
|
||||
@ -184,8 +197,8 @@ rule boost ( min-version ) {
|
||||
}
|
||||
|
||||
#Link normally to a library, but sometimes static isn't installed so fall back to dynamic.
|
||||
rule external-lib ( name : search-path * ) {
|
||||
lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) ;
|
||||
rule external-lib ( name : search-path * : deps * ) {
|
||||
lib $(name) : : [ auto-shared $(name) : "-L"$(search-path) ] <search>$(search-path) <use>$(deps) ;
|
||||
}
|
||||
|
||||
#Write the current command line to previous.sh. This does not do shell escaping.
|
||||
|
@ -144,7 +144,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
|
||||
m_manager->CalcNBest(nBestSize, nBestList, distinct);
|
||||
|
||||
// optionally print nbest to file (to extract scores and features.. currently just for sentence bleu scoring)
|
||||
if (filename != "") {
|
||||
/*if (filename != "") {
|
||||
ofstream out(filename.c_str());
|
||||
if (!out) {
|
||||
ostringstream msg;
|
||||
@ -154,7 +154,7 @@ vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source
|
||||
// TODO: handle sentence id (for now always 0)
|
||||
//OutputNBest(out, nBestList, StaticData::Instance().GetOutputFactorOrder(), 0, false);
|
||||
out.close();
|
||||
}
|
||||
}*/
|
||||
|
||||
// read off the feature values and bleu scores for each sentence in the nbest list
|
||||
Moses::TrellisPathList::const_iterator iter;
|
||||
|
159
mira/Main.cpp
159
mira/Main.cpp
@ -41,7 +41,6 @@ namespace mpi = boost::mpi;
|
||||
#include "moses/ChartTrellisPath.h"
|
||||
#include "moses/ScoreComponentCollection.h"
|
||||
#include "moses/ThreadPool.h"
|
||||
#include "moses/LexicalReordering.h"
|
||||
#include "mert/BleuScorer.h"
|
||||
#include "moses/FeatureVector.h"
|
||||
|
||||
@ -78,8 +77,8 @@ int main(int argc, char** argv)
|
||||
size_t mixingFrequency;
|
||||
size_t weightDumpFrequency;
|
||||
string weightDumpStem;
|
||||
bool scale_margin, scale_margin_precision;
|
||||
bool scale_update, scale_update_precision;
|
||||
bool scale_margin;
|
||||
bool scale_update;
|
||||
size_t n;
|
||||
size_t batchSize;
|
||||
bool distinctNbest;
|
||||
@ -103,7 +102,7 @@ int main(int argc, char** argv)
|
||||
bool perceptron_update;
|
||||
bool hope_fear;
|
||||
bool model_hope_fear;
|
||||
int hope_n, fear_n;
|
||||
size_t hope_n, fear_n;
|
||||
size_t bleu_smoothing_scheme;
|
||||
float min_oracle_bleu;
|
||||
float minBleuRatio, maxBleuRatio;
|
||||
@ -122,17 +121,19 @@ int main(int argc, char** argv)
|
||||
string moses_src;
|
||||
float sigmoidParam;
|
||||
float bleuWeight, bleuWeight_hope, bleuWeight_fear;
|
||||
bool bleu_weight_lm, bleu_weight_lm_adjust;
|
||||
bool bleu_weight_lm;
|
||||
float bleu_weight_lm_factor;
|
||||
bool l1_regularize, l2_regularize, l1_reg_sparse, l2_reg_sparse;
|
||||
float l1_lambda, l2_lambda;
|
||||
bool most_violated, most_violated_reg, all_violated, max_bleu_diff, one_against_all;
|
||||
bool most_violated, most_violated_reg, all_violated, max_bleu_diff;
|
||||
bool feature_confidence, signed_counts;
|
||||
float decay_core, decay_sparse, core_r0, sparse_r0;
|
||||
bool selective, summed;
|
||||
float bleu_weight_fear_factor;
|
||||
bool hildreth;
|
||||
float add2lm;
|
||||
|
||||
// compute real sentence Bleu scores on complete translations, disable Bleu feature
|
||||
bool realBleu, disableBleuFeature;
|
||||
bool rescaleSlack;
|
||||
bool makePairs;
|
||||
@ -147,12 +148,14 @@ int main(int argc, char** argv)
|
||||
("l1sparse", po::value<bool>(&l1_reg_sparse)->default_value(true), "L1-regularization for sparse weights only")
|
||||
("l2sparse", po::value<bool>(&l2_reg_sparse)->default_value(true), "L2-regularization for sparse weights only")
|
||||
("mv-reg", po::value<bool>(&most_violated_reg)->default_value(false), "Regularize most violated constraint")
|
||||
("most-violated", po::value<bool>(&most_violated)->default_value(false), "Add most violated constraint")
|
||||
("all-violated", po::value<bool>(&all_violated)->default_value(false), "Add all violated constraints")
|
||||
("feature-confidence", po::value<bool>(&feature_confidence)->default_value(false), "Confidence-weighted learning")
|
||||
("signed-counts", po::value<bool>(&signed_counts)->default_value(false), "Use signed feature counts for CWL")
|
||||
("dbg", po::value<bool>(&debug)->default_value(true), "More debug output")
|
||||
("make-pairs", po::value<bool>(&makePairs)->default_value(true), "Make pairs of hypotheses for 1slack")
|
||||
("debug", po::value<bool>(&debug)->default_value(true), "More debug output")
|
||||
("rescale-slack", po::value<bool>(&rescaleSlack)->default_value(false), "Rescale slack in 1-slack formulation")
|
||||
("disable-bleu-feature", po::value<bool>(&disableBleuFeature)->default_value(false), "Disable the Bleu feature")
|
||||
("real-bleu", po::value<bool>(&realBleu)->default_value(false), "Compute real sentence Bleu on complete translations")
|
||||
("add2lm", po::value<float>(&add2lm)->default_value(0.0), "Add the specified amount to all LM weights")
|
||||
("hildreth", po::value<bool>(&hildreth)->default_value(false), "Prefer Hildreth over analytical update")
|
||||
("selective", po::value<bool>(&selective)->default_value(false), "Build constraints for every feature")
|
||||
@ -166,9 +169,10 @@ int main(int argc, char** argv)
|
||||
|
||||
("core-r0", po::value<float>(&core_r0)->default_value(1.0), "Start learning rate for core features")
|
||||
("sparse-r0", po::value<float>(&sparse_r0)->default_value(1.0), "Start learning rate for sparse features")
|
||||
("decay-core", po::value<float>(&decay_core)->default_value(0.01), "Decay for core feature learning rate")
|
||||
("decay-sparse", po::value<float>(&decay_sparse)->default_value(0.01), "Decay for sparse feature learning rate")
|
||||
|
||||
("tie-bw-to-lm", po::value<bool>(&bleu_weight_lm)->default_value(false), "Make bleu weight depend on lm weight")
|
||||
("adjust-bw", po::value<bool>(&bleu_weight_lm_adjust)->default_value(false), "Adjust bleu weight when lm weight changes")
|
||||
("tie-bw-to-lm", po::value<bool>(&bleu_weight_lm)->default_value(true), "Make bleu weight depend on lm weight")
|
||||
("bw-lm-factor", po::value<float>(&bleu_weight_lm_factor)->default_value(2.0), "Make bleu weight depend on lm weight by this factor")
|
||||
("bw-factor-fear", po::value<float>(&bleu_weight_fear_factor)->default_value(1.0), "Multiply fear weight by this factor")
|
||||
("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs")
|
||||
@ -190,12 +194,12 @@ int main(int argc, char** argv)
|
||||
("dump-mixed-weights", po::value<bool>(&dumpMixedWeights)->default_value(false), "Dump mixed weights instead of averaged weights")
|
||||
("epochs,e", po::value<size_t>(&epochs)->default_value(10), "Number of epochs")
|
||||
("feature-cutoff", po::value<int>(&featureCutoff)->default_value(-1), "Feature cutoff as additional regularization for sparse features")
|
||||
("fear-n", po::value<int>(&fear_n)->default_value(1), "Number of fear translations used")
|
||||
("fear-n", po::value<size_t>(&fear_n)->default_value(1), "Number of fear translations used")
|
||||
("help", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
|
||||
("history-bleu", po::value<bool>(&historyBleu)->default_value(false), "Use 1best translations to update the history")
|
||||
("history-smoothing", po::value<float>(&historySmoothing)->default_value(0.9), "Adjust the factor for history smoothing")
|
||||
("hope-fear", po::value<bool>(&hope_fear)->default_value(true), "Use only hope and fear translations for optimisation (not model)")
|
||||
("hope-n", po::value<int>(&hope_n)->default_value(2), "Number of hope translations used")
|
||||
("hope-n", po::value<size_t>(&hope_n)->default_value(2), "Number of hope translations used")
|
||||
("input-file,i", po::value<string>(&inputFile), "Input file containing tokenised source")
|
||||
("input-files-folds", po::value<vector<string> >(&inputFilesFolds), "Input files containing tokenised source, one for each fold")
|
||||
("learner,l", po::value<string>(&learner)->default_value("mira"), "Learning algorithm")
|
||||
@ -209,10 +213,10 @@ int main(int argc, char** argv)
|
||||
("min-oracle-bleu", po::value<float>(&min_oracle_bleu)->default_value(0), "Set a minimum oracle BLEU score")
|
||||
("min-weight-change", po::value<float>(&min_weight_change)->default_value(0.0001), "Set minimum weight change for stopping criterion")
|
||||
("mira-learning-rate", po::value<float>(&mira_learning_rate)->default_value(1), "Learning rate for MIRA (fixed or flexible)")
|
||||
("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(1), "How often per epoch to mix weights, when using mpi")
|
||||
("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(10), "How often per epoch to mix weights, when using mpi")
|
||||
("model-hope-fear", po::value<bool>(&model_hope_fear)->default_value(false), "Use model, hope and fear translations for optimisation")
|
||||
("moses-src", po::value<string>(&moses_src)->default_value(""), "Moses source directory")
|
||||
("nbest,n", po::value<size_t>(&n)->default_value(1), "Number of translations in n-best list")
|
||||
("nbest,n", po::value<size_t>(&n)->default_value(30), "Number of translations in n-best list")
|
||||
("normalise-weights", po::value<bool>(&normaliseWeights)->default_value(false), "Whether to normalise the updated weights before passing them to the decoder")
|
||||
("normalise-margin", po::value<bool>(&normaliseMargin)->default_value(false), "Normalise the margin: squash between 0 and 1")
|
||||
("perceptron-learning-rate", po::value<float>(&perceptron_learning_rate)->default_value(0.01), "Perceptron learning rate")
|
||||
@ -224,30 +228,27 @@ int main(int argc, char** argv)
|
||||
("prune-zero-weights", po::value<bool>(&pruneZeroWeights)->default_value(false), "Prune zero-valued sparse feature weights")
|
||||
("reference-files,r", po::value<vector<string> >(&referenceFiles), "Reference translation files for training")
|
||||
("reference-files-folds", po::value<vector<string> >(&referenceFilesFolds), "Reference translation files for training, one for each fold")
|
||||
("kbest", po::value<bool>(&kbest)->default_value(false), "Select hope/fear pairs from a list of nbest translations")
|
||||
("kbest", po::value<bool>(&kbest)->default_value(true), "Select hope/fear pairs from a list of nbest translations")
|
||||
|
||||
("scale-by-inverse-length", po::value<bool>(&scaleByInverseLength)->default_value(false), "Scale BLEU by (history of) inverse input length")
|
||||
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(false), "Scale BLEU by (history of) input length")
|
||||
("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(true), "Scale BLEU by (history of) input length")
|
||||
("scale-by-avg-input-length", po::value<bool>(&scaleByAvgInputLength)->default_value(false), "Scale BLEU by average input length")
|
||||
("scale-by-avg-inverse-length", po::value<bool>(&scaleByAvgInverseLength)->default_value(false), "Scale BLEU by average inverse input length")
|
||||
("scale-by-x", po::value<float>(&scaleByX)->default_value(1), "Scale the BLEU score by value x")
|
||||
("scale-lm", po::value<bool>(&scale_lm)->default_value(false), "Scale the language model feature")
|
||||
("scale-factor-lm", po::value<float>(&scale_lm_factor)->default_value(2), "Scale the language model feature by this factor")
|
||||
("scale-by-x", po::value<float>(&scaleByX)->default_value(0.1), "Scale the BLEU score by value x")
|
||||
("scale-lm", po::value<bool>(&scale_lm)->default_value(true), "Scale the language model feature")
|
||||
("scale-factor-lm", po::value<float>(&scale_lm_factor)->default_value(0.5), "Scale the language model feature by this factor")
|
||||
("scale-wp", po::value<bool>(&scale_wp)->default_value(false), "Scale the word penalty feature")
|
||||
("scale-factor-wp", po::value<float>(&scale_wp_factor)->default_value(2), "Scale the word penalty feature by this factor")
|
||||
("scale-margin", po::value<bool>(&scale_margin)->default_value(0), "Scale the margin by the Bleu score of the oracle translation")
|
||||
("scale-margin-precision", po::value<bool>(&scale_margin_precision)->default_value(0), "Scale margin by precision of oracle")
|
||||
("scale-update", po::value<bool>(&scale_update)->default_value(0), "Scale update by Bleu score of oracle")
|
||||
("scale-update-precision", po::value<bool>(&scale_update_precision)->default_value(0), "Scale update by precision of oracle")
|
||||
("sentence-level-bleu", po::value<bool>(&sentenceBleu)->default_value(true), "Use a sentences level Bleu scoring function")
|
||||
("shuffle", po::value<bool>(&shuffle)->default_value(false), "Shuffle input sentences before processing")
|
||||
("sigmoid-param", po::value<float>(&sigmoidParam)->default_value(1), "y=sigmoidParam is the axis that this sigmoid approaches")
|
||||
("slack", po::value<float>(&slack)->default_value(0.01), "Use slack in optimiser")
|
||||
("slack", po::value<float>(&slack)->default_value(0.05), "Use slack in optimiser")
|
||||
("sparse-average", po::value<bool>(&sparseAverage)->default_value(false), "Average weights by the number of processes")
|
||||
("sparse-no-average", po::value<bool>(&sparseNoAverage)->default_value(false), "Don't average sparse weights, just sum")
|
||||
("stop-weights", po::value<bool>(&weightConvergence)->default_value(true), "Stop when weights converge")
|
||||
("verbosity,v", po::value<int>(&verbosity)->default_value(0), "Verbosity level")
|
||||
("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(1), "How often per epoch to dump weights (mpi)")
|
||||
("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(2), "How often per epoch to dump weights (mpi)")
|
||||
("weight-dump-stem", po::value<string>(&weightDumpStem)->default_value("weights"), "Stem of filename to use for dumping weights");
|
||||
|
||||
po::options_description cmdline_options;
|
||||
@ -396,20 +397,26 @@ int main(int argc, char** argv)
|
||||
// initialise Moses
|
||||
// add references to initialize Bleu feature
|
||||
boost::trim(decoder_settings);
|
||||
decoder_settings += " -mira -distinct-nbest -references";
|
||||
if (trainWithMultipleFolds) {
|
||||
decoder_settings += " ";
|
||||
decoder_settings += referenceFilesFolds[myFold];
|
||||
} else {
|
||||
for (size_t i=0; i < referenceFiles.size(); ++i) {
|
||||
decoder_settings += " ";
|
||||
decoder_settings += referenceFiles[i];
|
||||
}
|
||||
}
|
||||
decoder_settings += " -mira -n-best-list - " + boost::lexical_cast<string>(n) + " distinct";
|
||||
|
||||
vector<string> decoder_params;
|
||||
boost::split(decoder_params, decoder_settings, boost::is_any_of("\t "));
|
||||
|
||||
// bleu feature
|
||||
decoder_params.push_back("-feature-add");
|
||||
|
||||
decoder_settings = "BleuScoreFeature tuneable=false references=";
|
||||
if (trainWithMultipleFolds) {
|
||||
decoder_settings += referenceFilesFolds[myFold];
|
||||
} else {
|
||||
decoder_settings += referenceFiles[0];
|
||||
for (size_t i=1; i < referenceFiles.size(); ++i) {
|
||||
decoder_settings += ",";
|
||||
decoder_settings += referenceFiles[i];
|
||||
}
|
||||
}
|
||||
decoder_params.push_back(decoder_settings);
|
||||
|
||||
string configFile = trainWithMultipleFolds? mosesConfigFilesFolds[myFold] : mosesConfigFile;
|
||||
VERBOSE(1, "Rank " << rank << " reading config file from " << configFile << endl);
|
||||
MosesDecoder* decoder = new MosesDecoder(configFile, verbosity, decoder_params.size(), decoder_params);
|
||||
@ -442,8 +449,7 @@ int main(int argc, char** argv)
|
||||
if (normaliseMargin)
|
||||
cerr << "sigmoid parameter: " << sigmoidParam << endl;
|
||||
}
|
||||
optimiser = new MiraOptimiser(slack, scale_margin, scale_margin_precision,
|
||||
scale_update, scale_update_precision, boost, normaliseMargin, sigmoidParam);
|
||||
optimiser = new MiraOptimiser(slack, scale_margin, scale_update, boost, normaliseMargin, sigmoidParam);
|
||||
learning_rate = mira_learning_rate;
|
||||
perceptron_update = false;
|
||||
} else if (learner == "perceptron") {
|
||||
@ -469,9 +475,9 @@ int main(int argc, char** argv)
|
||||
cerr << "Info: Setting batch size to 1 for perceptron update" << endl;
|
||||
}
|
||||
|
||||
if (hope_n == -1)
|
||||
if (hope_n == 0)
|
||||
hope_n = n;
|
||||
if (fear_n == -1)
|
||||
if (fear_n == 0)
|
||||
fear_n = n;
|
||||
|
||||
if (model_hope_fear || kbest)
|
||||
@ -556,9 +562,10 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
bleuWeight = lmSum * bleu_weight_lm_factor;
|
||||
cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl;
|
||||
if (!kbest) cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl;
|
||||
}
|
||||
|
||||
// bleu weights can be set separately for hope and fear; otherwise they are both set to 'lm weight * bleu_weight_lm_factor'
|
||||
if (bleuWeight_hope == -1) {
|
||||
bleuWeight_hope = bleuWeight;
|
||||
}
|
||||
@ -566,8 +573,10 @@ int main(int argc, char** argv)
|
||||
bleuWeight_fear = bleuWeight;
|
||||
}
|
||||
bleuWeight_fear *= bleu_weight_fear_factor;
|
||||
cerr << "Bleu weight: " << bleuWeight << endl;
|
||||
cerr << "Bleu weight fear: " << bleuWeight_fear << endl;
|
||||
if (!kbest) {
|
||||
cerr << "Bleu weight: " << bleuWeight << endl;
|
||||
cerr << "Bleu weight fear: " << bleuWeight_fear << endl;
|
||||
}
|
||||
|
||||
if (decode_hope || decode_fear || decode_model) {
|
||||
size_t decode = 1;
|
||||
@ -697,7 +706,6 @@ int main(int argc, char** argv)
|
||||
vector<size_t> ref_ids;
|
||||
size_t actualBatchSize = 0;
|
||||
|
||||
vector<size_t>::const_iterator current_sid_start = sid;
|
||||
size_t examples_in_batch = 0;
|
||||
bool skip_example = false;
|
||||
for (size_t batchPosition = 0; batchPosition < batchSize && sid
|
||||
@ -802,27 +810,11 @@ int main(int argc, char** argv)
|
||||
ref_length = decoder->getClosestReferenceLength(*sid, oracle.size());
|
||||
avg_ref_length = ref_length;
|
||||
float hope_length_ratio = (float)oracle.size()/ref_length;
|
||||
int oracleSize = (int)oracle.size();
|
||||
cerr << endl;
|
||||
|
||||
// count sparse features occurring in hope translation
|
||||
featureValuesHope[batchPosition][0].IncrementSparseHopeFeatures();
|
||||
|
||||
float precision = bleuScoresHope[batchPosition][0];
|
||||
if (historyBleu || simpleHistoryBleu) {
|
||||
precision /= decoder->getTargetLengthHistory();
|
||||
} else {
|
||||
if (scaleByAvgInputLength) precision /= decoder->getAverageInputLength();
|
||||
else if (scaleByAvgInverseLength) precision /= (100/decoder->getAverageInputLength());
|
||||
precision /= scaleByX;
|
||||
}
|
||||
if (scale_margin_precision || scale_update_precision) {
|
||||
if (historyBleu || simpleHistoryBleu || scaleByAvgInputLength || scaleByAvgInverseLength) {
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", set hope precision: " << precision << endl;
|
||||
((MiraOptimiser*) optimiser)->setPrecision(precision);
|
||||
}
|
||||
}
|
||||
|
||||
vector<const Word*> bestModel;
|
||||
if (debug_model || historyBleu || simpleHistoryBleu) {
|
||||
// MODEL (for updating the history only, using dummy vectors)
|
||||
@ -837,9 +829,9 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
// FEAR
|
||||
float fear_length_ratio = 0;
|
||||
//float fear_length_ratio = 0;
|
||||
float bleuRatioHopeFear = 0;
|
||||
int fearSize = 0;
|
||||
//int fearSize = 0;
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", " << fear_n << "best fear translations" << endl;
|
||||
vector< vector<const Word*> > outputFear = decoder->getNBest(input, *sid, fear_n, -1.0, bleuWeight_fear,
|
||||
featureValuesFear[batchPosition], bleuScoresFear[batchPosition], modelScoresFear[batchPosition],
|
||||
@ -849,8 +841,8 @@ int main(int argc, char** argv)
|
||||
ref_length = decoder->getClosestReferenceLength(*sid, fear.size());
|
||||
avg_ref_length += ref_length;
|
||||
avg_ref_length /= 2;
|
||||
fear_length_ratio = (float)fear.size()/ref_length;
|
||||
fearSize = (int)fear.size();
|
||||
//fear_length_ratio = (float)fear.size()/ref_length;
|
||||
//fearSize = (int)fear.size();
|
||||
cerr << endl;
|
||||
for (size_t i = 0; i < fear.size(); ++i)
|
||||
delete fear[i];
|
||||
@ -994,11 +986,11 @@ int main(int argc, char** argv)
|
||||
HypothesisQueue queueHope(hope_n);
|
||||
HypothesisQueue queueFear(fear_n);
|
||||
cerr << endl;
|
||||
if (most_violated || all_violated || one_against_all) {
|
||||
if (most_violated || all_violated) {
|
||||
float bleuHope = -1000;
|
||||
float bleuFear = 1000;
|
||||
size_t indexHope = -1;
|
||||
size_t indexFear = -1;
|
||||
int indexHope = -1;
|
||||
int indexFear = -1;
|
||||
|
||||
vector<float> bleuHopeList;
|
||||
vector<float> bleuFearList;
|
||||
@ -1029,32 +1021,23 @@ int main(int argc, char** argv)
|
||||
}
|
||||
|
||||
float currentViolation = 0;
|
||||
float minimum_bleu_diff = 0.01;
|
||||
for (size_t i=0; i<bleuScores[batchPosition].size(); ++i) {
|
||||
float bleuDiff = bleuHope - bleuScores[batchPosition][i];
|
||||
float modelDiff = modelScores[batchPosition][indexHope] - modelScores[batchPosition][i];
|
||||
if (bleuDiff > epsilon) {
|
||||
if (one_against_all && bleuDiff > minimum_bleu_diff) {
|
||||
cerr << ".. adding pair";
|
||||
bleuHopeList.push_back(bleuHope);
|
||||
bleuFearList.push_back(bleuScores[batchPosition][i]);
|
||||
indexHopeList.push_back(indexHope);
|
||||
indexFearList.push_back(i);
|
||||
} else if (modelDiff < bleuDiff) {
|
||||
float diff = bleuDiff - modelDiff;
|
||||
if (diff > epsilon) {
|
||||
if (all_violated) {
|
||||
cerr << ".. adding pair";
|
||||
bleuHopeList.push_back(bleuHope);
|
||||
bleuFearList.push_back(bleuScores[batchPosition][i]);
|
||||
indexHopeList.push_back(indexHope);
|
||||
indexFearList.push_back(i);
|
||||
} else if (most_violated && diff > currentViolation) {
|
||||
currentViolation = diff;
|
||||
bleuFear = bleuScores[batchPosition][i];
|
||||
indexFear = i;
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", current violation: " << currentViolation << " (" << modelDiff << " >= " << bleuDiff << ")" << endl;
|
||||
}
|
||||
if ((bleuDiff > epsilon) && (modelDiff < bleuDiff)) {
|
||||
float diff = bleuDiff - modelDiff;
|
||||
if (diff > epsilon) {
|
||||
if (all_violated) {
|
||||
cerr << ".. adding pair";
|
||||
bleuHopeList.push_back(bleuHope);
|
||||
bleuFearList.push_back(bleuScores[batchPosition][i]);
|
||||
indexHopeList.push_back(indexHope);
|
||||
indexFearList.push_back(i);
|
||||
} else if (most_violated && diff > currentViolation) {
|
||||
currentViolation = diff;
|
||||
bleuFear = bleuScores[batchPosition][i];
|
||||
indexFear = i;
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", current violation: " << currentViolation << " (" << modelDiff << " >= " << bleuDiff << ")" << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -356,15 +356,6 @@ size_t MiraOptimiser::updateWeightsAnalytically(
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", normalised constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl;
|
||||
}
|
||||
|
||||
if (m_scale_margin) {
|
||||
diff *= bleuScoreHope;
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with oracle bleu score " << bleuScoreHope << endl;
|
||||
}
|
||||
if (m_scale_margin_precision) {
|
||||
diff *= (1+m_precision);
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with 1+precision: " << (1+m_precision) << endl;
|
||||
}
|
||||
|
||||
if (diff > epsilon) {
|
||||
// squash it between 0 and 1
|
||||
//diff = tanh(diff);
|
||||
@ -401,10 +392,6 @@ size_t MiraOptimiser::updateWeightsAnalytically(
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with oracle bleu score " << bleuScoreHope << endl;
|
||||
alpha *= bleuScoreHope;
|
||||
}
|
||||
if (m_scale_update_precision) {
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with 1+precision: " << (1+m_precision) << endl;
|
||||
alpha *= (1+m_precision);
|
||||
}
|
||||
|
||||
cerr << "Rank " << rank << ", epoch " << epoch << ", clipped/scaled alpha: " << alpha << endl;
|
||||
|
||||
|
@ -69,16 +69,21 @@ public:
|
||||
MiraOptimiser() :
|
||||
Optimiser() { }
|
||||
|
||||
MiraOptimiser(
|
||||
float slack, bool scale_margin, bool scale_margin_precision,
|
||||
bool scale_update, bool scale_update_precision, bool boost, bool normaliseMargin, float sigmoidParam) :
|
||||
MiraOptimiser(float slack) :
|
||||
Optimiser(),
|
||||
m_slack(slack),
|
||||
m_scale_margin(false),
|
||||
m_scale_update(false),
|
||||
m_boost(false),
|
||||
m_normaliseMargin(false),
|
||||
m_sigmoidParam(1.0) { }
|
||||
|
||||
MiraOptimiser(float slack, bool scale_margin, bool scale_update,
|
||||
bool boost, bool normaliseMargin, float sigmoidParam) :
|
||||
Optimiser(),
|
||||
m_slack(slack),
|
||||
m_scale_margin(scale_margin),
|
||||
m_scale_margin_precision(scale_margin_precision),
|
||||
m_scale_update(scale_update),
|
||||
m_scale_update_precision(scale_update_precision),
|
||||
m_precision(1),
|
||||
m_boost(boost),
|
||||
m_normaliseMargin(normaliseMargin),
|
||||
m_sigmoidParam(sigmoidParam) { }
|
||||
@ -148,21 +153,16 @@ public:
|
||||
m_slack = slack;
|
||||
}
|
||||
|
||||
void setPrecision(float precision) {
|
||||
m_precision = precision;
|
||||
}
|
||||
|
||||
private:
|
||||
// regularise Hildreth updates
|
||||
float m_slack;
|
||||
|
||||
// scale margin with BLEU score or precision
|
||||
bool m_scale_margin, m_scale_margin_precision;
|
||||
|
||||
// scale update with oracle BLEU score or precision
|
||||
bool m_scale_update, m_scale_update_precision;
|
||||
// scale margin with BLEU score
|
||||
bool m_scale_margin;
|
||||
|
||||
float m_precision;
|
||||
// scale update with oracle BLEU score
|
||||
bool m_scale_update;
|
||||
|
||||
// boosting of updates on misranked candidates
|
||||
bool m_boost;
|
||||
|
@ -289,8 +289,8 @@ my $learner = ¶m("train.learner", "mira");
|
||||
my $batch = ¶m("train.batch", 1); # don't print this param twice (when printing training file)
|
||||
my $extra_args = ¶m("train.extra-args");
|
||||
my $by_node = ¶m("train.by-node");
|
||||
my $slots = ¶m("train.slots",8);
|
||||
my $jobs = ¶m("train.jobs",8);
|
||||
my $slots = ¶m("train.slots",10);
|
||||
my $jobs = ¶m("train.jobs",10);
|
||||
my $mixing_frequency = ¶m("train.mixing-frequency", 1); # don't print this param twice
|
||||
my $weight_dump_frequency = ¶m("train.weight-dump-frequency", 1); # don't print this param twice
|
||||
my $burn_in = ¶m("train.burn-in");
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
#include "moses/Timer.h"
|
||||
#include "moses/InputFileStream.h"
|
||||
#include "moses/LexicalReorderingTable.h"
|
||||
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
|
||||
|
||||
using namespace Moses;
|
||||
|
||||
|
@ -5,7 +5,7 @@
|
||||
#include "moses/FactorCollection.h"
|
||||
#include "moses/Timer.h"
|
||||
#include "moses/InputFileStream.h"
|
||||
#include "moses/LexicalReorderingTable.h"
|
||||
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
|
||||
|
||||
using namespace Moses;
|
||||
|
||||
|
@ -196,15 +196,32 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
|
||||
{
|
||||
CHECK(outputFactorOrder.size() > 0);
|
||||
const Phrase& phrase = edge.GetCurrTargetPhrase();
|
||||
bool markUnknown = StaticData::Instance().GetMarkUnknown();
|
||||
if (reportAllFactors == true) {
|
||||
out << phrase;
|
||||
} else {
|
||||
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().second;
|
||||
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++) {
|
||||
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
|
||||
out << *factor;
|
||||
|
||||
if (placeholderFactor != NOT_FOUND) {
|
||||
const Factor *origFactor = phrase.GetFactor(pos, placeholderFactor);
|
||||
if (origFactor) {
|
||||
factor = origFactor;
|
||||
}
|
||||
}
|
||||
CHECK(factor);
|
||||
|
||||
//preface surface form with UNK if marking unknowns
|
||||
const Word &word = phrase.GetWord(pos);
|
||||
if(markUnknown && word.IsOOV()) {
|
||||
out << "UNK" << *factor;
|
||||
} else {
|
||||
out << *factor;
|
||||
}
|
||||
|
||||
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
|
||||
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
|
||||
CHECK(factor);
|
||||
@ -349,7 +366,7 @@ void OutputInput(std::vector<const Phrase*>& map, const Hypothesis* hypo)
|
||||
{
|
||||
if (hypo->GetPrevHypo()) {
|
||||
OutputInput(map, hypo->GetPrevHypo());
|
||||
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = hypo->GetSourcePhrase();
|
||||
map[hypo->GetCurrSourceWordsRange().GetStartPos()] = &hypo->GetTranslationOption().GetInputPath().GetPhrase();
|
||||
}
|
||||
}
|
||||
|
||||
@ -370,6 +387,10 @@ void IOWrapper::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, c
|
||||
Backtrack(hypo);
|
||||
VERBOSE(3,"0" << std::endl);
|
||||
if (!m_surpressSingleBestOutput) {
|
||||
if (StaticData::Instance().GetOutputHypoScore()) {
|
||||
cout << hypo->GetTotalScore() << " ";
|
||||
}
|
||||
|
||||
if (StaticData::Instance().IsPathRecoveryEnabled()) {
|
||||
OutputInput(cout, hypo);
|
||||
cout << "||| ";
|
||||
@ -450,7 +471,7 @@ void OutputNBest(std::ostream& out
|
||||
}
|
||||
|
||||
if (StaticData::Instance().IsPathRecoveryEnabled()) {
|
||||
out << "|||";
|
||||
out << " ||| ";
|
||||
OutputInput(out, edges[0]);
|
||||
}
|
||||
|
||||
|
@ -402,7 +402,7 @@ public:
|
||||
|
||||
//list of unknown words
|
||||
if (m_unknownsCollector) {
|
||||
const vector<Phrase*>& unknowns = manager.getSntTranslationOptions()->GetUnknownSources();
|
||||
const vector<const Phrase*>& unknowns = manager.getSntTranslationOptions()->GetUnknownSources();
|
||||
ostringstream out;
|
||||
for (size_t i = 0; i < unknowns.size(); ++i) {
|
||||
out << *(unknowns[i]);
|
||||
|
@ -53,9 +53,22 @@ void AlignmentInfo::BuildNonTermIndexMap()
|
||||
}
|
||||
m_nonTermIndexMap[p->second] = i++;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::set<size_t> AlignmentInfo::GetAlignmentsForSource(size_t sourcePos) const
|
||||
{
|
||||
std::set<size_t> ret;
|
||||
CollType::const_iterator iter;
|
||||
for (iter = begin(); iter != end(); ++iter) {
|
||||
const std::pair<size_t,size_t> &align = *iter;
|
||||
if (iter->first == sourcePos) {
|
||||
ret.insert(iter->second);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
bool compare_target(const std::pair<size_t,size_t> *a, const std::pair<size_t,size_t> *b)
|
||||
{
|
||||
if(a->second < b->second) return true;
|
||||
|
@ -69,6 +69,8 @@ public:
|
||||
return m_collection;
|
||||
}
|
||||
|
||||
std::set<size_t> GetAlignmentsForSource(size_t sourcePos) const;
|
||||
|
||||
size_t GetSize() const {
|
||||
return m_collection.size();
|
||||
}
|
||||
|
@ -47,7 +47,7 @@ ObjectPool<ChartHypothesis> ChartHypothesis::s_objectPool("ChartHypothesis", 300
|
||||
ChartHypothesis::ChartHypothesis(const ChartTranslationOptions &transOpt,
|
||||
const RuleCubeItem &item,
|
||||
ChartManager &manager)
|
||||
:m_targetPhrase(*(item.GetTranslationDimension().GetTargetPhrase()))
|
||||
:m_transOpt(item.GetTranslationDimension().GetTranslationOption())
|
||||
,m_currSourceWordsRange(transOpt.GetSourceWordsRange())
|
||||
,m_ffStates(StatefulFeatureFunction::GetStatefulFeatureFunctions().size())
|
||||
,m_arcList(NULL)
|
||||
@ -162,7 +162,7 @@ void ChartHypothesis::Evaluate()
|
||||
StatelessFeatureFunction::GetStatelessFeatureFunctions();
|
||||
for (unsigned i = 0; i < sfs.size(); ++i) {
|
||||
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
|
||||
sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown);
|
||||
sfs[i]->EvaluateChart(*this,&m_scoreBreakdown);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -49,7 +49,7 @@ protected:
|
||||
static ObjectPool<ChartHypothesis> s_objectPool;
|
||||
#endif
|
||||
|
||||
const TargetPhrase &m_targetPhrase;
|
||||
boost::shared_ptr<ChartTranslationOption> m_transOpt;
|
||||
|
||||
WordsRange m_currSourceWordsRange;
|
||||
std::vector<const FFState*> m_ffStates; /*! stateful feature function states */
|
||||
@ -100,9 +100,13 @@ public:
|
||||
return m_id;
|
||||
}
|
||||
|
||||
const ChartTranslationOption &GetTranslationOption()const {
|
||||
return *m_transOpt;
|
||||
}
|
||||
|
||||
//! Get the rule that created this hypothesis
|
||||
const TargetPhrase &GetCurrTargetPhrase()const {
|
||||
return m_targetPhrase;
|
||||
return m_transOpt->GetPhrase();
|
||||
}
|
||||
|
||||
//! the source range that this hypothesis spans
|
||||
|
@ -31,7 +31,6 @@
|
||||
#include "StaticData.h"
|
||||
#include "DecodeStep.h"
|
||||
#include "TreeInput.h"
|
||||
#include "moses/FF/PhrasePenaltyProducer.h"
|
||||
#include "moses/FF/WordPenaltyProducer.h"
|
||||
|
||||
using namespace std;
|
||||
@ -51,8 +50,14 @@ ChartManager::ChartManager(InputType const& source)
|
||||
,m_start(clock())
|
||||
,m_hypothesisId(0)
|
||||
,m_parser(source, m_hypoStackColl)
|
||||
,m_translationOptionList(StaticData::Instance().GetRuleLimit())
|
||||
,m_translationOptionList(StaticData::Instance().GetRuleLimit(), source)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
long sentenceID = source.GetTranslationId();
|
||||
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
|
||||
if (m_constraint) {
|
||||
VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl);
|
||||
}
|
||||
}
|
||||
|
||||
ChartManager::~ChartManager()
|
||||
@ -127,6 +132,8 @@ void ChartManager::ProcessSentence()
|
||||
void ChartManager::AddXmlChartOptions()
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const Phrase *constraint = GetConstraint();
|
||||
|
||||
const std::vector <ChartTranslationOptions*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
|
||||
IFVERBOSE(2) {
|
||||
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
|
||||
@ -137,14 +144,24 @@ void ChartManager::AddXmlChartOptions()
|
||||
i != xmlChartOptionsList.end(); ++i) {
|
||||
ChartTranslationOptions* opt = *i;
|
||||
|
||||
TargetPhrase &targetPhrase = *opt->GetTargetPhraseCollection().GetCollection()[0];
|
||||
targetPhrase.GetScoreBreakdown().Assign(staticData.GetPhrasePenaltyProducer(), -1);
|
||||
targetPhrase.GetScoreBreakdown().Assign(staticData.GetWordPenaltyProducer(), -1);
|
||||
|
||||
const TargetPhrase &targetPhrase = opt->GetTargetPhrases()[0]->GetPhrase();
|
||||
const WordsRange &range = opt->GetSourceWordsRange();
|
||||
|
||||
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
|
||||
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
|
||||
if (constraint) {
|
||||
Phrase hypoPhrase = hypo->GetOutputPhrase();
|
||||
if (!constraint->Contains(hypoPhrase)) {
|
||||
delete item;
|
||||
delete hypo;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
hypo->Evaluate();
|
||||
|
||||
const Word &targetLHS = hypo->GetTargetLHS();
|
||||
|
||||
ChartCell &cell = m_hypoStackColl.Get(range);
|
||||
cell.AddHypothesis(hypo);
|
||||
}
|
||||
@ -342,5 +359,4 @@ void ChartManager::CreateDeviantPaths(
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace Moses
|
||||
|
@ -59,6 +59,7 @@ private:
|
||||
std::auto_ptr<SentenceStats> m_sentenceStats;
|
||||
clock_t m_start; /**< starting time, used for logging */
|
||||
unsigned m_hypothesisId; /* For handing out hypothesis ids to ChartHypothesis */
|
||||
const Phrase *m_constraint;
|
||||
|
||||
ChartParser m_parser;
|
||||
|
||||
@ -101,6 +102,8 @@ public:
|
||||
return m_hypothesisId++;
|
||||
}
|
||||
|
||||
const Phrase *GetConstraint() const
|
||||
{ return m_constraint; }
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -24,6 +24,7 @@
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "StaticData.h"
|
||||
#include "TreeInput.h"
|
||||
#include "Sentence.h"
|
||||
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
||||
|
||||
using namespace std;
|
||||
@ -93,7 +94,6 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
|
||||
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->Evaluate(*unksrc);
|
||||
|
||||
targetPhrase->SetSourcePhrase(*unksrc);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
targetPhrase->SetAlignmentInfo("0-0");
|
||||
|
||||
@ -119,7 +119,6 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
|
||||
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
|
||||
targetPhrase->Evaluate(*unksrc);
|
||||
|
||||
targetPhrase->SetSourcePhrase(*unksrc);
|
||||
targetPhrase->SetTargetLHS(targetLHS);
|
||||
|
||||
// chart rule
|
||||
@ -141,14 +140,27 @@ ChartParser::ChartParser(InputType const &source, ChartCellCollectionBase &cells
|
||||
p != dictionaries.end(); ++p) {
|
||||
const PhraseDictionary *dict = *p;
|
||||
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
|
||||
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(source, cells));
|
||||
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(*this, cells));
|
||||
}
|
||||
|
||||
CreateInputPaths(m_source);
|
||||
}
|
||||
|
||||
ChartParser::~ChartParser()
|
||||
{
|
||||
RemoveAllInColl(m_ruleLookupManagers);
|
||||
StaticData::Instance().CleanUpAfterSentenceProcessing(m_source);
|
||||
|
||||
InputPathMatrix::const_iterator iterOuter;
|
||||
for (iterOuter = m_inputPathMatrix.begin(); iterOuter != m_inputPathMatrix.end(); ++iterOuter) {
|
||||
const std::vector<InputPath*> &outer = *iterOuter;
|
||||
|
||||
std::vector<InputPath*>::const_iterator iterInner;
|
||||
for (iterInner = outer.begin(); iterInner != outer.end(); ++iterInner) {
|
||||
InputPath *path = *iterInner;
|
||||
delete path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ChartParser::Create(const WordsRange &wordsRange, ChartParserCallback &to)
|
||||
@ -177,4 +189,62 @@ void ChartParser::Create(const WordsRange &wordsRange, ChartParserCallback &to)
|
||||
}
|
||||
}
|
||||
|
||||
void ChartParser::CreateInputPaths(const InputType &input)
|
||||
{
|
||||
size_t size = input.GetSize();
|
||||
m_inputPathMatrix.resize(size);
|
||||
|
||||
CHECK(input.GetType() == SentenceInput || input.GetType() == TreeInputType);
|
||||
for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
|
||||
for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
|
||||
size_t endPos = startPos + phaseSize -1;
|
||||
vector<InputPath*> &vec = m_inputPathMatrix[startPos];
|
||||
|
||||
WordsRange range(startPos, endPos);
|
||||
Phrase subphrase(input.GetSubString(WordsRange(startPos, endPos)));
|
||||
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
|
||||
|
||||
InputPath *node;
|
||||
if (range.GetNumWordsCovered() == 1) {
|
||||
node = new InputPath(subphrase, labels, range, NULL, NULL);
|
||||
vec.push_back(node);
|
||||
} else {
|
||||
const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
|
||||
node = new InputPath(subphrase, labels, range, &prevNode, NULL);
|
||||
vec.push_back(node);
|
||||
}
|
||||
|
||||
//m_phraseDictionaryQueue.push_back(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos) const
|
||||
{
|
||||
size_t offset = endPos - startPos;
|
||||
CHECK(offset < m_inputPathMatrix[startPos].size());
|
||||
return *m_inputPathMatrix[startPos][offset];
|
||||
}
|
||||
|
||||
InputPath &ChartParser::GetInputPath(size_t startPos, size_t endPos)
|
||||
{
|
||||
size_t offset = endPos - startPos;
|
||||
CHECK(offset < m_inputPathMatrix[startPos].size());
|
||||
return *m_inputPathMatrix[startPos][offset];
|
||||
}
|
||||
/*
|
||||
const Sentence &ChartParser::GetSentence() const {
|
||||
const Sentence &sentence = static_cast<const Sentence&>(m_source);
|
||||
return sentence;
|
||||
}
|
||||
*/
|
||||
size_t ChartParser::GetSize() const
|
||||
{
|
||||
return m_source.GetSize();
|
||||
}
|
||||
|
||||
long ChartParser::GetTranslationId() const
|
||||
{
|
||||
return m_source.GetTranslationId();
|
||||
}
|
||||
} // namespace Moses
|
||||
|
@ -21,11 +21,11 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "WordsRange.h"
|
||||
#include "StackVec.h"
|
||||
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include "WordsRange.h"
|
||||
#include "StackVec.h"
|
||||
#include "InputPath.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -33,6 +33,7 @@ namespace Moses
|
||||
class ChartParserCallback;
|
||||
class ChartRuleLookupManager;
|
||||
class InputType;
|
||||
class Sentence;
|
||||
class ChartCellCollectionBase;
|
||||
class Word;
|
||||
class Phrase;
|
||||
@ -50,7 +51,6 @@ public:
|
||||
private:
|
||||
std::vector<Phrase*> m_unksrcs;
|
||||
std::list<TargetPhraseCollection*> m_cacheTargetPhraseCollection;
|
||||
StackVec m_emptyStackVec;
|
||||
};
|
||||
|
||||
class ChartParser
|
||||
@ -61,11 +61,24 @@ public:
|
||||
|
||||
void Create(const WordsRange &range, ChartParserCallback &to);
|
||||
|
||||
//! the sentence being decoded
|
||||
//const Sentence &GetSentence() const;
|
||||
long GetTranslationId() const;
|
||||
size_t GetSize() const;
|
||||
const InputPath &GetInputPath(size_t startPos, size_t endPos) const;
|
||||
|
||||
private:
|
||||
ChartParserUnknown m_unknown;
|
||||
std::vector <DecodeGraph*> m_decodeGraphList;
|
||||
std::vector<ChartRuleLookupManager*> m_ruleLookupManagers;
|
||||
InputType const& m_source; /**< source sentence to be translated */
|
||||
|
||||
typedef std::vector< std::vector<InputPath*> > InputPathMatrix;
|
||||
InputPathMatrix m_inputPathMatrix;
|
||||
|
||||
void CreateInputPaths(const InputType &input);
|
||||
InputPath &GetInputPath(size_t startPos, size_t endPos);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -10,6 +10,8 @@ namespace Moses
|
||||
class TargetPhraseCollection;
|
||||
class WordsRange;
|
||||
class TargetPhrase;
|
||||
class InputPath;
|
||||
class InputType;
|
||||
|
||||
class ChartParserCallback
|
||||
{
|
||||
@ -21,6 +23,8 @@ public:
|
||||
virtual bool Empty() const = 0;
|
||||
|
||||
virtual void AddPhraseOOV(TargetPhrase &phrase, std::list<TargetPhraseCollection*> &waste_memory, const WordsRange &range) = 0;
|
||||
|
||||
virtual void Evaluate(const InputType &input, const InputPath &inputPath) = 0;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
13
moses/ChartRuleLookupManager.cpp
Normal file
13
moses/ChartRuleLookupManager.cpp
Normal file
@ -0,0 +1,13 @@
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "ChartParser.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
/*
|
||||
const Sentence &ChartRuleLookupManager::GetSentence() const
|
||||
{
|
||||
return m_parser.GetSentence();
|
||||
}
|
||||
*/
|
||||
} // namespace Moses
|
||||
|
@ -26,9 +26,10 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ChartParser;
|
||||
class ChartParserCallback;
|
||||
class WordsRange;
|
||||
class Sentence;
|
||||
|
||||
/** Defines an interface for looking up rules in a rule table. Concrete
|
||||
* implementation classes should correspond to specific PhraseDictionary
|
||||
@ -39,22 +40,22 @@ class WordsRange;
|
||||
class ChartRuleLookupManager
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManager(const InputType &sentence,
|
||||
ChartRuleLookupManager(const ChartParser &parser,
|
||||
const ChartCellCollectionBase &cellColl)
|
||||
: m_sentence(sentence)
|
||||
: m_parser(parser)
|
||||
, m_cellCollection(cellColl) {}
|
||||
|
||||
virtual ~ChartRuleLookupManager() {}
|
||||
|
||||
//! the sentence being decoded
|
||||
const InputType &GetSentence() const {
|
||||
return m_sentence;
|
||||
}
|
||||
|
||||
const ChartCellLabelSet &GetTargetLabelSet(size_t begin, size_t end) const {
|
||||
return m_cellCollection.GetBase(WordsRange(begin, end)).GetTargetLabelSet();
|
||||
}
|
||||
|
||||
const ChartParser &GetParser() const {
|
||||
return m_parser;
|
||||
}
|
||||
//const Sentence &GetSentence() const;
|
||||
|
||||
const ChartCellLabel &GetSourceAt(size_t at) const {
|
||||
return m_cellCollection.GetSourceWordLabel(at);
|
||||
}
|
||||
@ -73,7 +74,7 @@ private:
|
||||
//! Non-copyable: copy constructor and assignment operator not implemented.
|
||||
ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
|
||||
|
||||
const InputType &m_sentence;
|
||||
const ChartParser &m_parser;
|
||||
const ChartCellCollectionBase &m_cellCollection;
|
||||
};
|
||||
|
||||
|
23
moses/ChartTranslationOption.cpp
Normal file
23
moses/ChartTranslationOption.cpp
Normal file
@ -0,0 +1,23 @@
|
||||
#include "ChartTranslationOptions.h"
|
||||
#include "InputType.h"
|
||||
#include "InputPath.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
ChartTranslationOption::ChartTranslationOption(const TargetPhrase &targetPhrase)
|
||||
:m_targetPhrase(targetPhrase)
|
||||
,m_scoreBreakdown(targetPhrase.GetScoreBreakdown())
|
||||
{
|
||||
}
|
||||
|
||||
void ChartTranslationOption::Evaluate(const InputType &input, const InputPath &inputPath)
|
||||
{
|
||||
const std::vector<FeatureFunction*> &ffs = FeatureFunction::GetFeatureFunctions();
|
||||
|
||||
for (size_t i = 0; i < ffs.size(); ++i) {
|
||||
const FeatureFunction &ff = *ffs[i];
|
||||
ff.Evaluate(input, inputPath, m_scoreBreakdown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
32
moses/ChartTranslationOption.h
Normal file
32
moses/ChartTranslationOption.h
Normal file
@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "ScoreComponentCollection.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class TargetPhrase;
|
||||
class InputPath;
|
||||
class InputType;
|
||||
|
||||
class ChartTranslationOption
|
||||
{
|
||||
protected:
|
||||
const TargetPhrase &m_targetPhrase;
|
||||
ScoreComponentCollection m_scoreBreakdown;
|
||||
|
||||
public:
|
||||
ChartTranslationOption(const TargetPhrase &targetPhrase);
|
||||
|
||||
const TargetPhrase &GetPhrase() const {
|
||||
return m_targetPhrase;
|
||||
}
|
||||
|
||||
const ScoreComponentCollection &GetScores() const {
|
||||
return m_scoreBreakdown;
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input, const InputPath &inputPath);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -19,16 +19,21 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include "StaticData.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "ChartTranslationOptions.h"
|
||||
#include "ChartCellCollection.h"
|
||||
#include "WordsRange.h"
|
||||
#include "InputType.h"
|
||||
#include "InputPath.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
ChartTranslationOptionList::ChartTranslationOptionList(size_t ruleLimit)
|
||||
ChartTranslationOptionList::ChartTranslationOptionList(size_t ruleLimit, const InputType &input)
|
||||
: m_size(0)
|
||||
, m_ruleLimit(ruleLimit)
|
||||
{
|
||||
@ -142,4 +147,13 @@ void ChartTranslationOptionList::ApplyThreshold()
|
||||
m_size = std::distance(m_collection.begin(), bound);
|
||||
}
|
||||
|
||||
void ChartTranslationOptionList::Evaluate(const InputType &input, const InputPath &inputPath)
|
||||
{
|
||||
CollType::iterator iter;
|
||||
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
|
||||
ChartTranslationOptions &transOpts = **iter;
|
||||
transOpts.Evaluate(input, inputPath);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -30,12 +30,14 @@ namespace Moses
|
||||
|
||||
class TargetPhraseCollection;
|
||||
class WordsRange;
|
||||
class InputType;
|
||||
class InputPath;
|
||||
|
||||
//! a vector of translations options for a specific range, in a specific sentence
|
||||
class ChartTranslationOptionList : public ChartParserCallback
|
||||
{
|
||||
public:
|
||||
ChartTranslationOptionList(size_t);
|
||||
ChartTranslationOptionList(size_t ruleLimit, const InputType &input);
|
||||
~ChartTranslationOptionList();
|
||||
|
||||
const ChartTranslationOptions &Get(size_t i) const {
|
||||
@ -58,6 +60,7 @@ public:
|
||||
|
||||
void Clear();
|
||||
void ApplyThreshold();
|
||||
void Evaluate(const InputType &input, const InputPath &inputPath);
|
||||
|
||||
private:
|
||||
typedef std::vector<ChartTranslationOptions*> CollType;
|
||||
@ -74,6 +77,7 @@ private:
|
||||
size_t m_size;
|
||||
float m_scoreThreshold;
|
||||
const size_t m_ruleLimit;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -18,14 +18,35 @@
|
||||
***********************************************************************/
|
||||
|
||||
#include "ChartTranslationOptions.h"
|
||||
|
||||
#include "ChartHypothesis.h"
|
||||
|
||||
#include "ChartCellLabel.h"
|
||||
#include "ChartTranslationOption.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
ChartTranslationOptions::ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
|
||||
const StackVec &stackVec,
|
||||
const WordsRange &wordsRange,
|
||||
float score)
|
||||
: m_stackVec(stackVec)
|
||||
, m_wordsRange(&wordsRange)
|
||||
, m_estimateOfBestScore(score)
|
||||
{
|
||||
TargetPhraseCollection::const_iterator iter;
|
||||
for (iter = targetPhraseColl.begin(); iter != targetPhraseColl.end(); ++iter) {
|
||||
const TargetPhrase *origTP = *iter;
|
||||
|
||||
boost::shared_ptr<ChartTranslationOption> ptr(new ChartTranslationOption(*origTP));
|
||||
m_collection.push_back(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
ChartTranslationOptions::~ChartTranslationOptions()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
float ChartTranslationOptions::CalcEstimateOfBestScore(
|
||||
const TargetPhraseCollection &tpc,
|
||||
const StackVec &stackVec)
|
||||
@ -43,4 +64,14 @@ float ChartTranslationOptions::CalcEstimateOfBestScore(
|
||||
return estimateOfBestScore;
|
||||
}
|
||||
|
||||
void ChartTranslationOptions::Evaluate(const InputType &input, const InputPath &inputPath)
|
||||
{
|
||||
CollType::iterator iter;
|
||||
for (iter = m_collection.begin(); iter != m_collection.end(); ++iter) {
|
||||
ChartTranslationOption &transOpt = **iter;
|
||||
transOpt.Evaluate(input, inputPath);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -26,9 +26,14 @@
|
||||
|
||||
#include "util/check.hh"
|
||||
#include <vector>
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include "ChartTranslationOption.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ChartTranslationOption;
|
||||
class InputPath;
|
||||
class InputType;
|
||||
|
||||
/** Similar to a DottedRule, but contains a direct reference to a list
|
||||
* of translations and provdes an estimate of the best score. For a specific range in the input sentence
|
||||
@ -36,6 +41,8 @@ namespace Moses
|
||||
class ChartTranslationOptions
|
||||
{
|
||||
public:
|
||||
typedef std::vector<boost::shared_ptr<ChartTranslationOption> > CollType;
|
||||
|
||||
/** Constructor
|
||||
\param targetPhraseColl @todo dunno
|
||||
\param stackVec @todo dunno
|
||||
@ -45,14 +52,8 @@ public:
|
||||
ChartTranslationOptions(const TargetPhraseCollection &targetPhraseColl,
|
||||
const StackVec &stackVec,
|
||||
const WordsRange &wordsRange,
|
||||
float score)
|
||||
: m_stackVec(stackVec)
|
||||
, m_targetPhraseCollection(&targetPhraseColl)
|
||||
, m_wordsRange(&wordsRange)
|
||||
, m_estimateOfBestScore(score) {
|
||||
}
|
||||
|
||||
~ChartTranslationOptions() {}
|
||||
float score);
|
||||
~ChartTranslationOptions();
|
||||
|
||||
static float CalcEstimateOfBestScore(const TargetPhraseCollection &,
|
||||
const StackVec &);
|
||||
@ -63,8 +64,8 @@ public:
|
||||
}
|
||||
|
||||
//! @todo isn't the translation suppose to just contain 1 target phrase, not a whole collection of them?
|
||||
const TargetPhraseCollection &GetTargetPhraseCollection() const {
|
||||
return *m_targetPhraseCollection;
|
||||
const CollType &GetTargetPhrases() const {
|
||||
return m_collection;
|
||||
}
|
||||
|
||||
//! the range in the source sentence this translation option covers
|
||||
@ -80,10 +81,13 @@ public:
|
||||
return m_estimateOfBestScore;
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input, const InputPath &inputPath);
|
||||
|
||||
private:
|
||||
|
||||
StackVec m_stackVec; //! vector of hypothesis list!
|
||||
const TargetPhraseCollection *m_targetPhraseCollection;
|
||||
CollType m_collection;
|
||||
|
||||
const WordsRange *m_wordsRange;
|
||||
float m_estimateOfBestScore;
|
||||
};
|
||||
|
@ -64,6 +64,11 @@ ConfusionNet::ConfusionNet()
|
||||
: InputType()
|
||||
{
|
||||
stats.createOne();
|
||||
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
if (staticData.IsChart()) {
|
||||
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
|
||||
}
|
||||
}
|
||||
ConfusionNet::~ConfusionNet()
|
||||
{
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include <iostream>
|
||||
#include "Word.h"
|
||||
#include "InputType.h"
|
||||
#include "NonTerminal.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -25,6 +26,7 @@ public:
|
||||
|
||||
protected:
|
||||
std::vector<Column> data;
|
||||
NonTerminalSet m_defaultLabelSet;
|
||||
|
||||
bool ReadFormat0(std::istream&,const std::vector<FactorType>& factorOrder);
|
||||
bool ReadFormat1(std::istream&,const std::vector<FactorType>& factorOrder);
|
||||
@ -71,8 +73,7 @@ public:
|
||||
TranslationOptionCollection* CreateTranslationOptionCollection() const;
|
||||
|
||||
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const {
|
||||
CHECK(false);
|
||||
return *(new NonTerminalSet());
|
||||
return m_defaultLabelSet;
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -103,14 +103,6 @@ public:
|
||||
const GenerationDictionary* GetGenerationDictionaryFeature() const;
|
||||
|
||||
|
||||
/*! Given an input TranslationOption, extend it in some way (put results in outputPartialTranslOptColl) */
|
||||
virtual void Process(const TranslationOption &inputPartialTranslOpt
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit
|
||||
, const Phrase &src) const = 0;
|
||||
|
||||
void RemoveFeature(const FeatureFunction *ff);
|
||||
|
||||
};
|
||||
|
@ -65,8 +65,7 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection * /* toc */
|
||||
, bool /*adhereTableLimit*/
|
||||
, const Phrase &src) const
|
||||
, bool /*adhereTableLimit*/) const
|
||||
{
|
||||
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
|
||||
// word deletion
|
||||
@ -81,6 +80,7 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
|
||||
const GenerationDictionary* generationDictionary = decodeStep.GetGenerationDictionaryFeature();
|
||||
|
||||
const Phrase &targetPhrase = inputPartialTranslOpt.GetTargetPhrase();
|
||||
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
|
||||
size_t targetLength = targetPhrase.GetSize();
|
||||
|
||||
// generation list for each word in phrase
|
||||
@ -148,13 +148,15 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
|
||||
outPhrase.GetScoreBreakdown().PlusEquals(generationScore);
|
||||
|
||||
outPhrase.MergeFactors(genPhrase, m_newOutputFactors);
|
||||
outPhrase.Evaluate(src, m_featuresToApply);
|
||||
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply);
|
||||
|
||||
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
|
||||
|
||||
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
|
||||
assert(newTransOpt);
|
||||
|
||||
newTransOpt->SetInputPath(inputPath);
|
||||
|
||||
outputPartialTranslOptColl.Add(newTransOpt);
|
||||
|
||||
// increment iterators
|
||||
|
@ -40,12 +40,11 @@ public:
|
||||
const std::vector<FeatureFunction*> &features);
|
||||
|
||||
|
||||
virtual void Process(const TranslationOption &inputPartialTranslOpt
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit
|
||||
, const Phrase &src) const;
|
||||
void Process(const TranslationOption &inputPartialTranslOpt
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit) const;
|
||||
|
||||
private:
|
||||
};
|
||||
|
@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "TranslationOptionCollection.h"
|
||||
#include "PartialTranslOptColl.h"
|
||||
#include "FactorCollection.h"
|
||||
#include "util/exception.hh"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -48,7 +49,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit
|
||||
, const Phrase &src) const
|
||||
, const TargetPhraseCollection *phraseColl) const
|
||||
{
|
||||
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
|
||||
// word deletion
|
||||
@ -58,15 +59,13 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
|
||||
|
||||
// normal trans step
|
||||
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
|
||||
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
|
||||
const PhraseDictionary* phraseDictionary =
|
||||
decodeStep.GetPhraseDictionaryFeature();
|
||||
const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
|
||||
const size_t currSize = inPhrase.GetSize();
|
||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||
|
||||
const TargetPhraseCollection *phraseColl=
|
||||
phraseDictionary->GetTargetPhraseCollection(toc->GetSource(),sourceWordsRange);
|
||||
|
||||
if (phraseColl != NULL) {
|
||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
|
||||
@ -85,13 +84,14 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
|
||||
}
|
||||
|
||||
outPhrase.Merge(targetPhrase, m_newOutputFactors);
|
||||
cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(src, m_featuresToApply)" << endl;
|
||||
outPhrase.Evaluate(src, m_featuresToApply); // need to do this as all non-transcores would be screwed up
|
||||
|
||||
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
|
||||
cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply)" << endl;
|
||||
|
||||
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
|
||||
assert(newTransOpt != NULL);
|
||||
|
||||
newTransOpt->SetInputPath(inputPath);
|
||||
|
||||
outputPartialTranslOptColl.Add(newTransOpt );
|
||||
|
||||
}
|
||||
@ -101,17 +101,17 @@ cerr << "DecodeStepTranslation::Process is calling outPhrase.Evaluate(src, m_f
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void DecodeStepTranslation::ProcessInitialTranslation(
|
||||
const InputType &source
|
||||
,PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit) const
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhraseCollection *phraseColl) const
|
||||
{
|
||||
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||
|
||||
const WordsRange wordsRange(startPos, endPos);
|
||||
const TargetPhraseCollection *phraseColl = phraseDictionary->GetTargetPhraseCollection(source,wordsRange);
|
||||
|
||||
if (phraseColl != NULL) {
|
||||
IFVERBOSE(3) {
|
||||
@ -128,6 +128,8 @@ void DecodeStepTranslation::ProcessInitialTranslation(
|
||||
const TargetPhrase &targetPhrase = **iterTargetPhrase;
|
||||
TranslationOption *transOpt = new TranslationOption(wordsRange, targetPhrase);
|
||||
|
||||
transOpt->SetInputPath(inputPath);
|
||||
|
||||
outputPartialTranslOptColl.Add (transOpt);
|
||||
|
||||
VERBOSE(3,"\t" << targetPhrase << "\n");
|
||||
@ -136,6 +138,135 @@ void DecodeStepTranslation::ProcessInitialTranslation(
|
||||
}
|
||||
}
|
||||
|
||||
void DecodeStepTranslation::ProcessInitialTranslationLEGACY(
|
||||
const InputType &source
|
||||
,PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit
|
||||
, const InputPathList &inputPathList) const
|
||||
{
|
||||
const PhraseDictionary* phraseDictionary = GetPhraseDictionaryFeature();
|
||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||
|
||||
const WordsRange wordsRange(startPos, endPos);
|
||||
const TargetPhraseCollectionWithSourcePhrase *phraseColl = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,wordsRange);
|
||||
|
||||
if (phraseColl != NULL) {
|
||||
IFVERBOSE(3) {
|
||||
if(StaticData::Instance().GetInputType() == SentenceInput)
|
||||
TRACE_ERR("[" << source.GetSubString(wordsRange) << "; " << startPos << "-" << endPos << "]\n");
|
||||
else
|
||||
TRACE_ERR("[" << startPos << "-" << endPos << "]" << std::endl);
|
||||
}
|
||||
|
||||
const std::vector<Phrase> &sourcePhrases = phraseColl->GetSourcePhrases();
|
||||
|
||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||
std::vector<Phrase>::const_iterator iterSourcePhrase;
|
||||
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
|
||||
|
||||
for (iterTargetPhrase = phraseColl->begin(), iterSourcePhrase = sourcePhrases.begin()
|
||||
; iterTargetPhrase != iterEnd
|
||||
; ++iterTargetPhrase, ++iterSourcePhrase) {
|
||||
CHECK(iterSourcePhrase != sourcePhrases.end());
|
||||
|
||||
const TargetPhrase &targetPhrase = **iterTargetPhrase;
|
||||
const Phrase &sourcePhrase = *iterSourcePhrase;
|
||||
|
||||
const InputPath &inputPath = GetInputPathLEGACY(targetPhrase, sourcePhrase, inputPathList);
|
||||
|
||||
TranslationOption *transOpt = new TranslationOption(wordsRange, targetPhrase);
|
||||
transOpt->SetInputPath(inputPath);
|
||||
|
||||
outputPartialTranslOptColl.Add (transOpt);
|
||||
|
||||
VERBOSE(3,"\t" << targetPhrase << "\n");
|
||||
}
|
||||
VERBOSE(3,std::endl);
|
||||
}
|
||||
}
|
||||
|
||||
const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
|
||||
const TargetPhrase targetPhrase,
|
||||
const Phrase sourcePhrase,
|
||||
const InputPathList &inputPathList) const
|
||||
{
|
||||
const Word &wordFromPt = sourcePhrase.GetWord(0);
|
||||
|
||||
InputPathList::const_iterator iter;
|
||||
for (iter = inputPathList.begin(); iter != inputPathList.end(); ++iter) {
|
||||
const InputPath &inputPath = **iter;
|
||||
const Phrase &phraseFromIP = inputPath.GetPhrase();
|
||||
const Word &wordIP = phraseFromIP.GetWord(0);
|
||||
|
||||
const WordsRange &range = inputPath.GetWordsRange();
|
||||
|
||||
if (wordFromPt == wordIP) {
|
||||
return inputPath;
|
||||
}
|
||||
}
|
||||
|
||||
UTIL_THROW(util::Exception, "Input path not found");
|
||||
}
|
||||
|
||||
void DecodeStepTranslation::ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit) const
|
||||
{
|
||||
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
|
||||
// word deletion
|
||||
outputPartialTranslOptColl.Add(new TranslationOption(inputPartialTranslOpt));
|
||||
return;
|
||||
}
|
||||
|
||||
// normal trans step
|
||||
const WordsRange &sourceWordsRange = inputPartialTranslOpt.GetSourceWordsRange();
|
||||
const InputPath &inputPath = inputPartialTranslOpt.GetInputPath();
|
||||
const PhraseDictionary* phraseDictionary =
|
||||
decodeStep.GetPhraseDictionaryFeature();
|
||||
const TargetPhrase &inPhrase = inputPartialTranslOpt.GetTargetPhrase();
|
||||
const size_t currSize = inPhrase.GetSize();
|
||||
const size_t tableLimit = phraseDictionary->GetTableLimit();
|
||||
|
||||
const TargetPhraseCollectionWithSourcePhrase *phraseColl
|
||||
= phraseDictionary->GetTargetPhraseCollectionLEGACY(toc->GetSource(),sourceWordsRange);
|
||||
|
||||
|
||||
if (phraseColl != NULL) {
|
||||
TargetPhraseCollection::const_iterator iterTargetPhrase, iterEnd;
|
||||
iterEnd = (!adhereTableLimit || tableLimit == 0 || phraseColl->GetSize() < tableLimit) ? phraseColl->end() : phraseColl->begin() + tableLimit;
|
||||
|
||||
for (iterTargetPhrase = phraseColl->begin(); iterTargetPhrase != iterEnd; ++iterTargetPhrase) {
|
||||
const TargetPhrase& targetPhrase = **iterTargetPhrase;
|
||||
const ScoreComponentCollection &transScores = targetPhrase.GetScoreBreakdown();
|
||||
// skip if the
|
||||
if (targetPhrase.GetSize() != currSize) continue;
|
||||
|
||||
TargetPhrase outPhrase(inPhrase);
|
||||
|
||||
if (IsFilteringStep()) {
|
||||
if (!inputPartialTranslOpt.IsCompatible(targetPhrase, m_conflictFactors))
|
||||
continue;
|
||||
}
|
||||
|
||||
outPhrase.Merge(targetPhrase, m_newOutputFactors);
|
||||
outPhrase.Evaluate(inputPath.GetPhrase(), m_featuresToApply); // need to do this as all non-transcores would be screwed up
|
||||
|
||||
|
||||
TranslationOption *newTransOpt = new TranslationOption(sourceWordsRange, outPhrase);
|
||||
assert(newTransOpt != NULL);
|
||||
|
||||
newTransOpt->SetInputPath(inputPath);
|
||||
|
||||
outputPartialTranslOptColl.Add(newTransOpt );
|
||||
|
||||
}
|
||||
} else if (sourceWordsRange.GetNumWordsCovered() == 1) {
|
||||
// unknown handler
|
||||
//toc->ProcessUnknownWord(sourceWordsRange.GetStartPos(), factorCollection);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
@ -24,12 +24,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
#include "DecodeStep.h"
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
#include "InputPath.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class PhraseDictionary;
|
||||
class TargetPhrase;
|
||||
class InputPath;
|
||||
|
||||
//! subclass of DecodeStep for translation step
|
||||
class DecodeStepTranslation : public DecodeStep
|
||||
@ -46,7 +48,7 @@ public:
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit
|
||||
, const Phrase &src) const;
|
||||
, const TargetPhraseCollection *phraseColl) const;
|
||||
|
||||
|
||||
/*! initialize list of partial translation options by applying the first translation step
|
||||
@ -54,9 +56,28 @@ public:
|
||||
*/
|
||||
void ProcessInitialTranslation(const InputType &source
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit) const;
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit
|
||||
, const InputPath &inputPath
|
||||
, const TargetPhraseCollection *phraseColl) const;
|
||||
|
||||
// legacy
|
||||
void ProcessInitialTranslationLEGACY(const InputType &source
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, size_t startPos, size_t endPos, bool adhereTableLimit
|
||||
, const InputPathList &inputPathList) const;
|
||||
void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
|
||||
, const DecodeStep &decodeStep
|
||||
, PartialTranslOptColl &outputPartialTranslOptColl
|
||||
, TranslationOptionCollection *toc
|
||||
, bool adhereTableLimit) const;
|
||||
|
||||
private:
|
||||
// I'm not sure whether this actually works or not for binary phrase table.
|
||||
// The source phrase only appears to contain the 1st word, therefore, this function
|
||||
// only compares the 1st word
|
||||
const InputPath &GetInputPathLEGACY(const TargetPhrase targetPhrase,
|
||||
const Phrase sourcePhrase,
|
||||
const InputPathList &inputPathList) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -91,45 +91,60 @@ BleuScoreFeature::BleuScoreFeature(const std::string &line)
|
||||
m_historySmoothing(0.9),
|
||||
m_smoothing_scheme(PLUS_POINT_ONE)
|
||||
{
|
||||
for (size_t i = 0; i < m_args.size(); ++i) {
|
||||
const vector<string> &args = m_args[i];
|
||||
std::cerr << "Initializing BleuScoreFeature." << std::endl;
|
||||
m_tuneable = false;
|
||||
|
||||
if (args[0] == "references") {
|
||||
vector<string> referenceFiles = Tokenize(args[1]);
|
||||
CHECK(referenceFiles.size());
|
||||
vector<vector<string> > references(referenceFiles.size());
|
||||
ReadParameters();
|
||||
std::cerr << "Finished initializing BleuScoreFeature." << std::endl;
|
||||
}
|
||||
|
||||
for (size_t i =0; i < referenceFiles.size(); ++i) {
|
||||
ifstream in(referenceFiles[i].c_str());
|
||||
if (!in) {
|
||||
stringstream strme;
|
||||
strme << "Unable to load references from " << referenceFiles[i];
|
||||
UserMessage::Add(strme.str());
|
||||
void BleuScoreFeature::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "references") {
|
||||
vector<string> referenceFiles = Tokenize(value, ",");
|
||||
CHECK(referenceFiles.size());
|
||||
vector<vector<string> > references(referenceFiles.size());
|
||||
|
||||
for (size_t i =0; i < referenceFiles.size(); ++i) {
|
||||
ifstream in(referenceFiles[i].c_str());
|
||||
if (!in) {
|
||||
stringstream strme;
|
||||
strme << "Unable to load references from " << referenceFiles[i];
|
||||
UserMessage::Add(strme.str());
|
||||
abort();
|
||||
}
|
||||
string line;
|
||||
while (getline(in,line)) {
|
||||
/* if (GetSearchAlgorithm() == ChartDecoding) {
|
||||
stringstream tmp;
|
||||
tmp << "<s> " << line << " </s>";
|
||||
line = tmp.str();
|
||||
}
|
||||
*/
|
||||
references[i].push_back(line);
|
||||
}
|
||||
if (i > 0) {
|
||||
if (references[i].size() != references[i-1].size()) {
|
||||
UserMessage::Add("Reference files are of different lengths");
|
||||
abort();
|
||||
}
|
||||
string line;
|
||||
while (getline(in,line)) {
|
||||
/* if (GetSearchAlgorithm() == ChartDecoding) {
|
||||
stringstream tmp;
|
||||
tmp << "<s> " << line << " </s>";
|
||||
line = tmp.str();
|
||||
}
|
||||
*/
|
||||
references[i].push_back(line);
|
||||
}
|
||||
if (i > 0) {
|
||||
if (references[i].size() != references[i-1].size()) {
|
||||
UserMessage::Add("Reference files are of different lengths");
|
||||
abort();
|
||||
}
|
||||
}
|
||||
in.close();
|
||||
} // for (size_t i =0; i < referenceFiles.size(); ++i) {
|
||||
}
|
||||
in.close();
|
||||
} // for (size_t i =0; i < referenceFiles.size(); ++i) {
|
||||
|
||||
//Set the references in the bleu feature
|
||||
LoadReferences(references);
|
||||
} // if (args[0] == "references") {
|
||||
} // for (size_t i = 0; i < toks.size(); ++i) {
|
||||
//Set the references in the bleu feature
|
||||
LoadReferences(references);
|
||||
|
||||
} else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::vector<float> BleuScoreFeature::DefaultWeights() const
|
||||
{
|
||||
std::vector<float> ret(m_numScoreComponents, 1);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void BleuScoreFeature::PrintHistory(std::ostream& out) const
|
||||
|
@ -67,6 +67,10 @@ public:
|
||||
|
||||
BleuScoreFeature(const std::string &line);
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
std::vector<float> DefaultWeights() const;
|
||||
|
||||
void PrintHistory(std::ostream& out) const;
|
||||
void LoadReferences(const std::vector< std::vector< std::string > > &);
|
||||
void SetCurrSourceLength(size_t);
|
||||
@ -114,6 +118,16 @@ public:
|
||||
FFState* EvaluateChart(const ChartHypothesis& cur_hypo,
|
||||
int featureID,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
bool Enabled() const {
|
||||
return m_enabled;
|
||||
}
|
||||
|
@ -1,24 +0,0 @@
|
||||
#include "ChartBasedFeatureContext.h"
|
||||
#include "moses/ChartHypothesis.h"
|
||||
#include "moses/ChartManager.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
ChartBasedFeatureContext::ChartBasedFeatureContext
|
||||
(const ChartHypothesis* hypothesis):
|
||||
m_hypothesis(hypothesis),
|
||||
m_targetPhrase(hypothesis->GetCurrTargetPhrase()),
|
||||
m_source(hypothesis->GetManager().GetSource())
|
||||
{}
|
||||
|
||||
ChartBasedFeatureContext::ChartBasedFeatureContext(
|
||||
const TargetPhrase& targetPhrase,
|
||||
const InputType& source):
|
||||
m_hypothesis(NULL),
|
||||
m_targetPhrase(targetPhrase),
|
||||
m_source(source)
|
||||
{}
|
||||
|
||||
|
||||
}
|
||||
|
@ -1,39 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class ChartHypothesis;
|
||||
class InputType;
|
||||
class TargetPhrase;
|
||||
|
||||
/**
|
||||
* Same as PhraseBasedFeatureContext, but for chart-based Moses.
|
||||
**/
|
||||
class ChartBasedFeatureContext
|
||||
{
|
||||
//The context either has a hypothesis (during search) or a
|
||||
//TargetPhrase and source sentence (during pre-calculation)
|
||||
//TODO: should the context also include some info on where the TargetPhrase
|
||||
//is anchored (assuming it's lexicalised), which is available at pre-calc?
|
||||
const ChartHypothesis* m_hypothesis;
|
||||
const TargetPhrase& m_targetPhrase;
|
||||
const InputType& m_source;
|
||||
|
||||
public:
|
||||
ChartBasedFeatureContext(const ChartHypothesis* hypothesis);
|
||||
ChartBasedFeatureContext(const TargetPhrase& targetPhrase,
|
||||
const InputType& source);
|
||||
|
||||
const InputType& GetSource() const {
|
||||
return m_source;
|
||||
}
|
||||
|
||||
const TargetPhrase& GetTargetPhrase() const {
|
||||
return m_targetPhrase;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -35,6 +35,16 @@ public:
|
||||
int /* featureID - used to index the state in the previous hypotheses */,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
//! return the state associated with the empty hypothesis for a given sentence
|
||||
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
|
||||
|
||||
|
@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include <iostream>
|
||||
|
||||
#include "DecodeFeature.h"
|
||||
#include "StaticData.h"
|
||||
#include "moses/StaticData.h"
|
||||
|
||||
using namespace std;
|
||||
|
@ -24,9 +24,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "FactorTypeSet.h"
|
||||
#include "moses/FF/StatelessFeatureFunction.h"
|
||||
#include "TypeDef.h"
|
||||
#include "moses/FactorTypeSet.h"
|
||||
#include "moses/TypeDef.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -63,6 +63,22 @@ public:
|
||||
bool IsUseable(const FactorMask &mask) const;
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
protected:
|
||||
std::vector<FactorType> m_input;
|
||||
std::vector<FactorType> m_output;
|
@ -40,6 +40,17 @@ public:
|
||||
ScoreComponentCollection*) const {
|
||||
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -80,7 +80,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) START" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: |" << sp << "|" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << sp << "|" << std::endl);
|
||||
float score;
|
||||
switch(m_query_type) {
|
||||
case CBLM_QUERY_TYPE_WHOLESTRING:
|
||||
@ -94,7 +94,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
|
||||
}
|
||||
|
||||
score=-0.333333;
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: |" << tp << "| score:|" << score << "|" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << tp << "| score:|" << score << "|" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) scoreBreakdown before:|" << scoreBreakdown << "|" << std::endl);
|
||||
// scoreBreakdown.Assign(this, score);
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
@ -102,6 +102,22 @@ void DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const Phrase &sp, ....) END" << std::endl);
|
||||
}
|
||||
|
||||
void DynamicCacheBasedLanguageModel::Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &input, ....) START" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate: |" << input << "|" << std::endl);
|
||||
float score=-0.222222;
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModeldd::Evaluate: score:|" << score << "|" << std::endl);
|
||||
// scoreBreakdown.Assign(this, score);
|
||||
scoreBreakdown.PlusEquals(this, score);
|
||||
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &input, ....) END" << std::endl);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
void DynamicCacheBasedLanguageModel::Evaluate(const InputType &source
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{
|
||||
@ -114,6 +130,7 @@ void DynamicCacheBasedLanguageModel::Evaluate(const InputType &source
|
||||
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Evaluate(const InputType &source, ....) END" << std::endl);
|
||||
}
|
||||
*/
|
||||
|
||||
float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase& tp) const
|
||||
{
|
||||
|
@ -90,20 +90,29 @@ public:
|
||||
|
||||
void Insert(std::string &entries);
|
||||
|
||||
/*
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
*/
|
||||
|
||||
virtual void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual void Evaluate(const InputType &source
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const;
|
||||
/*
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
*/
|
||||
|
||||
|
||||
void SetQueryType(size_t type);
|
||||
void SetScoreType(size_t type);
|
||||
|
@ -11,7 +11,7 @@
|
||||
#include "moses/TranslationModel/PhraseDictionaryDynSuffixArray.h"
|
||||
#include "moses/TranslationModel/PhraseDictionaryDynamicCacheBased.h"
|
||||
|
||||
#include "moses/LexicalReordering.h"
|
||||
#include "moses/FF/LexicalReordering/LexicalReordering.h"
|
||||
|
||||
#include "moses/FF/BleuScoreFeature.h"
|
||||
#include "moses/FF/TargetWordInsertionFeature.h"
|
||||
@ -127,7 +127,6 @@ FeatureRegistry::FeatureRegistry()
|
||||
MOSES_FNAME2("Generation", GenerationDictionary);
|
||||
MOSES_FNAME(BleuScoreFeature);
|
||||
MOSES_FNAME2("Distortion", DistortionScoreProducer);
|
||||
MOSES_FNAME2("PhrasePenalty", PhrasePenaltyProducer);
|
||||
MOSES_FNAME2("WordPenalty", WordPenaltyProducer);
|
||||
MOSES_FNAME(InputFeature);
|
||||
MOSES_FNAME2("PhraseDictionaryBinary", PhraseDictionaryTreeAdaptor);
|
||||
@ -140,6 +139,7 @@ FeatureRegistry::FeatureRegistry()
|
||||
MOSES_FNAME(PhraseDictionaryDynSuffixArray);
|
||||
MOSES_FNAME(PhraseDictionaryDynamicCacheBased);
|
||||
MOSES_FNAME(OpSequenceModel);
|
||||
MOSES_FNAME(PhrasePenalty);
|
||||
MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer);
|
||||
MOSES_FNAME(ControlRecombination);
|
||||
MOSES_FNAME(DynamicCacheBasedLanguageModel);
|
||||
|
@ -4,8 +4,6 @@
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include "PhraseBasedFeatureContext.h"
|
||||
#include "ChartBasedFeatureContext.h"
|
||||
#include "moses/TypeDef.h"
|
||||
|
||||
namespace Moses
|
||||
@ -22,8 +20,7 @@ class ScoreComponentCollection;
|
||||
class WordsBitmap;
|
||||
class WordsRange;
|
||||
class FactorMask;
|
||||
|
||||
|
||||
class InputPath;
|
||||
|
||||
/** base class for all feature functions.
|
||||
*/
|
||||
@ -96,16 +93,22 @@ public:
|
||||
// return true if the feature function can be evaluated
|
||||
virtual bool IsUseable(const FactorMask &mask) const = 0;
|
||||
|
||||
// used by stateless ff. And stateful ff to make initial score estimate during loading of phrase table
|
||||
// used by stateless ff and stateful ff. Calculate initial score estimate during loading of phrase table
|
||||
// source phrase is the substring that the phrase table uses to look up the target phrase,
|
||||
// may have more factors than actually need, but not guaranteed.
|
||||
// For SCFG decoding, the source contains non-terminals, NOT the raw source from the input sentence
|
||||
virtual void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const {
|
||||
}
|
||||
, ScoreComponentCollection &estimatedFutureScore) const = 0;
|
||||
|
||||
virtual void Evaluate(const InputType &source
|
||||
, ScoreComponentCollection &scoreBreakdown) const {
|
||||
}
|
||||
// This method is called once all the translation options are retrieved from the phrase table, and
|
||||
// just before search.
|
||||
// 'inputPath' is guaranteed to be the raw substring from the input. No factors were added or taken away
|
||||
// Currently not used by any FF. Not called by moses_chart
|
||||
virtual void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const = 0;
|
||||
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
virtual void ReadParameters();
|
||||
|
@ -167,11 +167,11 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP
|
||||
}
|
||||
|
||||
void GlobalLexicalModel::Evaluate
|
||||
(const PhraseBasedFeatureContext& context,
|
||||
(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
accumulator->PlusEquals( this,
|
||||
GetFromCacheOrScorePhrase(context.GetTargetPhrase()) );
|
||||
GetFromCacheOrScorePhrase(hypo.GetCurrTargetPhrase()) );
|
||||
}
|
||||
|
||||
bool GlobalLexicalModel::IsUseable(const FactorMask &mask) const
|
||||
|
@ -64,20 +64,32 @@ public:
|
||||
GlobalLexicalModel(const std::string &line);
|
||||
virtual ~GlobalLexicalModel();
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
void InitializeForInput( Sentence const& in );
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const;
|
||||
|
||||
void Evaluate(const PhraseBasedFeatureContext& context,
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
|
||||
void EvaluateChart(
|
||||
const ChartBasedFeatureContext& context,
|
||||
const ChartHypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const {
|
||||
throw std::logic_error("GlobalLexicalModel not supported in chart decoder, yet");
|
||||
}
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
@ -90,6 +90,16 @@ public:
|
||||
throw std::logic_error("GlobalLexicalModelUnlimited not supported in chart decoder, yet");
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void AddFeature(ScoreComponentCollection* accumulator,
|
||||
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
|
||||
StringPiece targetWord) const;
|
||||
|
@ -1,6 +1,8 @@
|
||||
#include <stdexcept>
|
||||
#include "InputFeature.h"
|
||||
#include "moses/Util.h"
|
||||
#include "moses/ScoreComponentCollection.h"
|
||||
#include "moses/InputPath.h"
|
||||
#include "util/check.hh"
|
||||
|
||||
using namespace std;
|
||||
@ -10,20 +12,30 @@ namespace Moses
|
||||
InputFeature::InputFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("InputFeature", line)
|
||||
{
|
||||
for (size_t i = 0; i < m_args.size(); ++i) {
|
||||
const vector<string> &args = m_args[i];
|
||||
CHECK(args.size() == 2);
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
if (args[0] == "num-input-features") {
|
||||
m_numInputScores = Scan<size_t>(args[1]);
|
||||
} else if (args[0] == "real-word-count") {
|
||||
m_numRealWordCount = Scan<size_t>(args[1]);
|
||||
} else {
|
||||
throw "Unknown argument " + args[0];
|
||||
}
|
||||
void InputFeature::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "num-input-features") {
|
||||
m_numInputScores = Scan<size_t>(value);
|
||||
} else if (key == "real-word-count") {
|
||||
m_numRealWordCount = Scan<size_t>(value);
|
||||
} else {
|
||||
StatelessFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void InputFeature::Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{
|
||||
const ScoreComponentCollection *scores = inputPath.GetInputScore();
|
||||
if (scores) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -16,6 +16,8 @@ protected:
|
||||
public:
|
||||
InputFeature(const std::string &line);
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const {
|
||||
return true;
|
||||
}
|
||||
@ -27,6 +29,22 @@ public:
|
||||
return m_numRealWordCount;
|
||||
}
|
||||
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const;
|
||||
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
@ -3,7 +3,7 @@
|
||||
#include "moses/FF/FFState.h"
|
||||
#include "LexicalReordering.h"
|
||||
#include "LexicalReorderingState.h"
|
||||
#include "StaticData.h"
|
||||
#include "moses/StaticData.h"
|
||||
|
||||
using namespace std;
|
||||
|
@ -3,15 +3,16 @@
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "Factor.h"
|
||||
#include "Phrase.h"
|
||||
#include "TypeDef.h"
|
||||
#include "Util.h"
|
||||
#include "WordsRange.h"
|
||||
#include "moses/Factor.h"
|
||||
#include "moses/Phrase.h"
|
||||
#include "moses/TypeDef.h"
|
||||
#include "moses/Util.h"
|
||||
#include "moses/WordsRange.h"
|
||||
|
||||
#include "LexicalReorderingState.h"
|
||||
#include "LexicalReorderingTable.h"
|
||||
#include "moses/FF/StatefulFeatureFunction.h"
|
||||
#include "util/exception.hh"
|
||||
|
||||
|
||||
namespace Moses
|
||||
@ -48,10 +49,17 @@ public:
|
||||
virtual FFState* EvaluateChart(const ChartHypothesis&,
|
||||
int /* featureID */,
|
||||
ScoreComponentCollection*) const {
|
||||
CHECK(0); // not valid for chart decoder
|
||||
std::cerr << "LexicalReordering::EvaluateChartr(const ChartHypothesis& hypo,...) START" << std::endl;
|
||||
return NULL;
|
||||
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
|
||||
}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
private:
|
||||
bool DecodeCondition(std::string s);
|
@ -4,10 +4,10 @@
|
||||
#include "util/check.hh"
|
||||
|
||||
#include "moses/FF/FFState.h"
|
||||
#include "Hypothesis.h"
|
||||
#include "WordsRange.h"
|
||||
#include "ReorderingStack.h"
|
||||
#include "TranslationOption.h"
|
||||
#include "moses/Hypothesis.h"
|
||||
#include "moses/WordsRange.h"
|
||||
#include "moses/ReorderingStack.h"
|
||||
#include "moses/TranslationOption.h"
|
||||
|
||||
#include "LexicalReordering.h"
|
||||
#include "LexicalReorderingState.h"
|
@ -4,12 +4,12 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "Hypothesis.h"
|
||||
#include "moses/Hypothesis.h"
|
||||
#include "LexicalReordering.h"
|
||||
#include "WordsRange.h"
|
||||
#include "WordsBitmap.h"
|
||||
#include "ReorderingStack.h"
|
||||
#include "TranslationOption.h"
|
||||
#include "moses/WordsRange.h"
|
||||
#include "moses/WordsBitmap.h"
|
||||
#include "moses/ReorderingStack.h"
|
||||
#include "moses/TranslationOption.h"
|
||||
#include "moses/FF/FFState.h"
|
||||
|
||||
|
@ -1,15 +1,15 @@
|
||||
#include "LexicalReorderingTable.h"
|
||||
#include "InputFileStream.h"
|
||||
#include "moses/InputFileStream.h"
|
||||
//#include "LVoc.h" //need IPhrase
|
||||
|
||||
#include "StaticData.h"
|
||||
#include "moses/StaticData.h"
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
#include "GenerationDictionary.h"
|
||||
#include "TargetPhrase.h"
|
||||
#include "TargetPhraseCollection.h"
|
||||
#include "moses/GenerationDictionary.h"
|
||||
#include "moses/TargetPhrase.h"
|
||||
#include "moses/TargetPhraseCollection.h"
|
||||
|
||||
#ifndef WIN32
|
||||
#include "TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
|
||||
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
|
||||
#endif
|
||||
|
||||
namespace Moses
|
@ -13,12 +13,12 @@
|
||||
#endif
|
||||
|
||||
//moses dependencies:
|
||||
#include "TypeDef.h"
|
||||
#include "Phrase.h"
|
||||
#include "InputType.h"
|
||||
#include "ConfusionNet.h"
|
||||
#include "Sentence.h"
|
||||
#include "PrefixTreeMap.h"
|
||||
#include "moses/TypeDef.h"
|
||||
#include "moses/Phrase.h"
|
||||
#include "moses/InputType.h"
|
||||
#include "moses/ConfusionNet.h"
|
||||
#include "moses/Sentence.h"
|
||||
#include "moses/PrefixTreeMap.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
@ -13,7 +13,9 @@ namespace Moses
|
||||
OpSequenceModel::OpSequenceModel(const std::string &line)
|
||||
:StatefulFeatureFunction("OpSequenceModel", 5, line )
|
||||
{
|
||||
|
||||
sFactor = 0;
|
||||
tFactor = 0;
|
||||
numFeatures = 5;
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
@ -48,7 +50,7 @@ void OpSequenceModel:: Evaluate(const Phrase &source
|
||||
WordsBitmap myBitmap(source.GetSize());
|
||||
vector <string> mySourcePhrase;
|
||||
vector <string> myTargetPhrase;
|
||||
vector<float> scores(5);
|
||||
vector<float> scores;
|
||||
vector <int> alignments;
|
||||
int startIndex = 0;
|
||||
int endIndex = source.GetSize();
|
||||
@ -56,28 +58,27 @@ void OpSequenceModel:: Evaluate(const Phrase &source
|
||||
const AlignmentInfo &align = targetPhrase.GetAlignTerm();
|
||||
AlignmentInfo::const_iterator iter;
|
||||
|
||||
|
||||
for (iter = align.begin(); iter != align.end(); ++iter) {
|
||||
alignments.push_back(iter->first);
|
||||
alignments.push_back(iter->second);
|
||||
}
|
||||
|
||||
for (int i = 0; i < targetPhrase.GetSize(); i++) {
|
||||
if (targetPhrase.GetWord(i).IsOOV())
|
||||
if (targetPhrase.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
|
||||
myTargetPhrase.push_back("_TRANS_SLF_");
|
||||
else
|
||||
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(0)->GetString().as_string());
|
||||
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(tFactor)->GetString().as_string());
|
||||
}
|
||||
|
||||
for (int i = 0; i < source.GetSize(); i++) {
|
||||
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
|
||||
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
|
||||
}
|
||||
|
||||
obj.setPhrases(mySourcePhrase , myTargetPhrase);
|
||||
obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
|
||||
obj.computeOSMFeature(startIndex,myBitmap);
|
||||
obj.calculateOSMProb(*OSM);
|
||||
obj.populateScores(scores);
|
||||
obj.populateScores(scores,numFeatures);
|
||||
estimatedFutureScore.PlusEquals(this, scores);
|
||||
|
||||
}
|
||||
@ -97,7 +98,7 @@ FFState* OpSequenceModel::Evaluate(
|
||||
osmHypothesis obj;
|
||||
vector <string> mySourcePhrase;
|
||||
vector <string> myTargetPhrase;
|
||||
vector<float> scores(5);
|
||||
vector<float> scores;
|
||||
|
||||
|
||||
//target.GetWord(0)
|
||||
@ -141,16 +142,16 @@ FFState* OpSequenceModel::Evaluate(
|
||||
|
||||
for (int i = startIndex; i <= endIndex; i++) {
|
||||
myBitmap.SetValue(i,0); // resetting coverage of this phrase ...
|
||||
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
|
||||
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
|
||||
// cerr<<mySourcePhrase[i]<<endl;
|
||||
}
|
||||
|
||||
for (int i = 0; i < target.GetSize(); i++) {
|
||||
|
||||
if (target.GetWord(i).IsOOV())
|
||||
if (target.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
|
||||
myTargetPhrase.push_back("_TRANS_SLF_");
|
||||
else
|
||||
myTargetPhrase.push_back(target.GetWord(i).GetFactor(0)->GetString().as_string());
|
||||
myTargetPhrase.push_back(target.GetWord(i).GetFactor(tFactor)->GetString().as_string());
|
||||
|
||||
}
|
||||
|
||||
@ -162,7 +163,8 @@ FFState* OpSequenceModel::Evaluate(
|
||||
obj.setPhrases(mySourcePhrase , myTargetPhrase);
|
||||
obj.computeOSMFeature(startIndex,myBitmap);
|
||||
obj.calculateOSMProb(*OSM);
|
||||
obj.populateScores(scores);
|
||||
obj.populateScores(scores,numFeatures);
|
||||
//obj.print();
|
||||
|
||||
/*
|
||||
if (bitmap.GetFirstGapPos() == NOT_FOUND)
|
||||
@ -176,14 +178,7 @@ FFState* OpSequenceModel::Evaluate(
|
||||
}
|
||||
*/
|
||||
|
||||
/*
|
||||
vector<float> scores(5);
|
||||
scores[0] = 0.343423f;
|
||||
scores[1] = 1.343423f;
|
||||
scores[2] = 2.343423f;
|
||||
scores[3] = 3.343423f;
|
||||
scores[4] = 4.343423f;
|
||||
*/
|
||||
|
||||
|
||||
accumulator->PlusEquals(this, scores);
|
||||
|
||||
@ -226,7 +221,7 @@ std::vector<float> OpSequenceModel::GetFutureScores(const Phrase &source, const
|
||||
iter = m_futureCost.find(pp);
|
||||
//iter = m_coll.find(pp);
|
||||
if (iter == m_futureCost.end()) {
|
||||
vector<float> scores(5, 0);
|
||||
vector<float> scores(numFeatures, 0);
|
||||
scores[0] = unkOpProb;
|
||||
return scores;
|
||||
} else {
|
||||
@ -240,11 +235,23 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
|
||||
|
||||
if (key == "path") {
|
||||
m_lmPath = value;
|
||||
} else if (key == "numFeatures") {
|
||||
numFeatures = Scan<int>(value);
|
||||
} else if (key == "order") {
|
||||
lmOrder = Scan<int>(value);
|
||||
} else if (key == "sFactor") {
|
||||
sFactor = Scan<int>(value);
|
||||
} else if (key == "tFactor") {
|
||||
tFactor = Scan<int>(value);
|
||||
} else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
bool OpSequenceModel::IsUseable(const FactorMask &mask) const
|
||||
{
|
||||
bool ret = mask[0];
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
@ -21,6 +21,9 @@ public:
|
||||
|
||||
int lmOrder;
|
||||
float unkOpProb;
|
||||
int sFactor; // Source Factor ...
|
||||
int tFactor; // Target Factor ...
|
||||
int numFeatures; // Number of features used ...
|
||||
|
||||
OpSequenceModel(const std::string &line);
|
||||
|
||||
@ -32,16 +35,20 @@ public:
|
||||
const FFState* prev_state,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual FFState* EvaluateChart(
|
||||
const ChartHypothesis& /* cur_hypo */,
|
||||
int /* featureID - used to index the state in the previous hypotheses */,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
|
||||
|
||||
virtual std::string GetScoreProducerWeightShortName(unsigned idx=0) const;
|
||||
@ -49,9 +56,7 @@ public:
|
||||
std::vector<float> GetFutureScores(const Phrase &source, const Phrase &target) const;
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const {
|
||||
return true;
|
||||
}
|
||||
bool IsUseable(const FactorMask &mask) const;
|
||||
|
||||
protected:
|
||||
typedef std::pair<Phrase, Phrase> ParallelPhrase;
|
||||
|
@ -105,7 +105,6 @@ void osmHypothesis :: removeReorderingOperations()
|
||||
deletionCount = 0;
|
||||
openGapCount = 0;
|
||||
gapWidth = 0;
|
||||
//cout<<"I came here"<<endl;
|
||||
|
||||
std::vector <std::string> tupleSequence;
|
||||
|
||||
@ -581,10 +580,14 @@ void osmHypothesis :: constructCepts(vector <int> & align , int startIndex , int
|
||||
|
||||
}
|
||||
|
||||
void osmHypothesis :: populateScores(vector <float> & scores)
|
||||
void osmHypothesis :: populateScores(vector <float> & scores , const int numFeatures)
|
||||
{
|
||||
scores.clear();
|
||||
scores.push_back(opProb);
|
||||
|
||||
if (numFeatures == 1)
|
||||
return;
|
||||
|
||||
scores.push_back(gapWidth);
|
||||
scores.push_back(gapCount);
|
||||
scores.push_back(openGapCount);
|
||||
|
@ -89,7 +89,7 @@ public:
|
||||
void setState(const FFState* prev_state);
|
||||
osmState * saveState();
|
||||
void print();
|
||||
void populateScores(std::vector <float> & scores);
|
||||
void populateScores(std::vector <float> & scores , const int numFeatures);
|
||||
void setState(const lm::ngram::State & val) {
|
||||
lmState = val;
|
||||
}
|
||||
|
@ -1,33 +0,0 @@
|
||||
#include "PhraseBasedFeatureContext.h"
|
||||
#include "moses/Hypothesis.h"
|
||||
#include "moses/Manager.h"
|
||||
#include "moses/TranslationOption.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) :
|
||||
m_hypothesis(hypothesis),
|
||||
m_translationOption(m_hypothesis->GetTranslationOption()),
|
||||
m_source(m_hypothesis->GetManager().GetSource()) {}
|
||||
|
||||
PhraseBasedFeatureContext::PhraseBasedFeatureContext
|
||||
(const TranslationOption& translationOption, const InputType& source) :
|
||||
m_hypothesis(NULL),
|
||||
m_translationOption(translationOption),
|
||||
m_source(source)
|
||||
{}
|
||||
|
||||
const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const
|
||||
{
|
||||
return m_translationOption.GetTargetPhrase();
|
||||
}
|
||||
|
||||
const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const
|
||||
{
|
||||
if (!m_hypothesis) {
|
||||
throw std::logic_error("Coverage vector not available during pre-calculation");
|
||||
}
|
||||
return m_hypothesis->GetWordsBitmap();
|
||||
}
|
||||
|
||||
}
|
@ -1,44 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class Hypothesis;
|
||||
class TranslationOption;
|
||||
class InputType;
|
||||
class TargetPhrase;
|
||||
class WordsBitmap;
|
||||
|
||||
/**
|
||||
* Contains all that a feature function can access without affecting recombination.
|
||||
* For stateless features, this is all that it can access. Currently this is not
|
||||
* used for stateful features, as it would need to be retro-fitted to the LM feature.
|
||||
* TODO: Expose source segmentation,lattice path.
|
||||
* XXX Don't add anything to the context that would break recombination XXX
|
||||
**/
|
||||
class PhraseBasedFeatureContext
|
||||
{
|
||||
// The context either has a hypothesis (during search), or a TranslationOption and
|
||||
// source sentence (during pre-calculation).
|
||||
const Hypothesis* m_hypothesis;
|
||||
const TranslationOption& m_translationOption;
|
||||
const InputType& m_source;
|
||||
|
||||
public:
|
||||
PhraseBasedFeatureContext(const Hypothesis* hypothesis);
|
||||
PhraseBasedFeatureContext(const TranslationOption& translationOption,
|
||||
const InputType& source);
|
||||
|
||||
const TranslationOption& GetTranslationOption() const {
|
||||
return m_translationOption;
|
||||
}
|
||||
const InputType& GetSource() const {
|
||||
return m_source;
|
||||
}
|
||||
const TargetPhrase& GetTargetPhrase() const; //convenience method
|
||||
const WordsBitmap& GetWordsBitmap() const;
|
||||
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
#include "PhraseBoundaryFeature.h"
|
||||
|
||||
#include "moses/Hypothesis.h"
|
||||
#include "moses/TranslationOption.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -77,12 +78,12 @@ FFState* PhraseBoundaryFeature::Evaluate
|
||||
const Word* rightTargetWord = &(targetPhrase.GetWord(0));
|
||||
AddFeatures(leftTargetWord,rightTargetWord,m_targetFactors,"tgt",scores);
|
||||
|
||||
const Phrase* sourcePhrase = cur_hypo.GetSourcePhrase();
|
||||
const Phrase& sourcePhrase = cur_hypo.GetTranslationOption().GetInputPath().GetPhrase();
|
||||
const Word* leftSourceWord = pbState->GetSourceWord();
|
||||
const Word* rightSourceWord = &(sourcePhrase->GetWord(0));
|
||||
const Word* rightSourceWord = &(sourcePhrase.GetWord(0));
|
||||
AddFeatures(leftSourceWord,rightSourceWord,m_sourceFactors,"src",scores);
|
||||
|
||||
const Word* endSourceWord = &(sourcePhrase->GetWord(sourcePhrase->GetSize()-1));
|
||||
const Word* endSourceWord = &(sourcePhrase.GetWord(sourcePhrase.GetSize()-1));
|
||||
const Word* endTargetWord = &(targetPhrase.GetWord(targetPhrase.GetSize()-1));
|
||||
|
||||
//if end of sentence add EOS
|
||||
|
@ -52,6 +52,17 @@ public:
|
||||
ScoreComponentCollection* ) const {
|
||||
throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
private:
|
||||
|
@ -24,11 +24,20 @@ public:
|
||||
return true;
|
||||
}
|
||||
|
||||
void EvaluateChart(const ChartBasedFeatureContext& context,
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void EvaluateChart(const ChartHypothesis& hypo,
|
||||
ScoreComponentCollection*) const {
|
||||
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
|
||||
virtual void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
|
@ -106,11 +106,11 @@ void PhrasePairFeature::Load()
|
||||
}
|
||||
|
||||
void PhrasePairFeature::Evaluate(
|
||||
const PhraseBasedFeatureContext& context,
|
||||
const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
const TargetPhrase& target = context.GetTargetPhrase();
|
||||
const Phrase& source = *(context.GetTranslationOption().GetSourcePhrase());
|
||||
const TargetPhrase& target = hypo.GetCurrTargetPhrase();
|
||||
const Phrase& source = hypo.GetTranslationOption().GetInputPath().GetPhrase();
|
||||
if (m_simple) {
|
||||
ostringstream namestr;
|
||||
namestr << "pp_";
|
||||
@ -131,7 +131,7 @@ void PhrasePairFeature::Evaluate(
|
||||
accumulator->SparsePlusEquals(namestr.str(),1);
|
||||
}
|
||||
if (m_domainTrigger) {
|
||||
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
|
||||
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
|
||||
const bool use_topicid = input.GetUseTopicId();
|
||||
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
|
||||
|
||||
@ -199,7 +199,7 @@ void PhrasePairFeature::Evaluate(
|
||||
}
|
||||
}
|
||||
if (m_sourceContext) {
|
||||
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
|
||||
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
|
||||
|
||||
// range over source words to get context
|
||||
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
|
||||
|
@ -37,14 +37,24 @@ public:
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const;
|
||||
|
||||
void Evaluate(const PhraseBasedFeatureContext& context,
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
void EvaluateChart(const ChartBasedFeatureContext& context,
|
||||
void EvaluateChart(const ChartHypothesis& hypo,
|
||||
ScoreComponentCollection*) const {
|
||||
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
|
||||
}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void Load();
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
|
@ -6,32 +6,34 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
PhrasePenaltyProducer::PhrasePenaltyProducer(const std::string &line)
|
||||
PhrasePenalty::PhrasePenalty(const std::string &line)
|
||||
: StatelessFeatureFunction("PhrasePenalty",1, line)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
void PhrasePenaltyProducer::Evaluate(const Phrase &source
|
||||
void PhrasePenalty::Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
std::cerr << "void PhrasePenaltyProducer::Evaluate(const Phrase &source, ...) START" << std::endl;
|
||||
std::cerr << "void PhrasePenalty::Evaluate(const Phrase &source, ...) START" << std::endl;
|
||||
// scoreBreakdown.Assign(this, - 1.0f);
|
||||
scoreBreakdown.PlusEquals(this, - 1.0f);
|
||||
std::cerr << "void PhrasePenaltyProducer::Evaluate(const Phrase &source, ...) END" << std::endl;
|
||||
std::cerr << "void PhrasePenalty::Evaluate(const Phrase &source, ...) END" << std::endl;
|
||||
}
|
||||
|
||||
void PhrasePenaltyProducer::Evaluate(const InputType &source
|
||||
/*
|
||||
void PhrasePenalty::Evaluate(const InputType &source
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{
|
||||
std::cerr << "void PhrasePenaltyProducer::Evaluate(const InputType &source, ...) START" << std::endl;
|
||||
std::cerr << "void PhrasePenalty::Evaluate(const InputType &source, ...) START" << std::endl;
|
||||
// scoreBreakdown.Assign(this, - 1.0f);
|
||||
scoreBreakdown.PlusEquals(this, - 1.0f);
|
||||
std::cerr << "void PhrasePenaltyProducer::Evaluate(const InputType &source, ...) END" << std::endl;
|
||||
|
||||
}
|
||||
std::cerr << "void PhrasePenalty::Evaluate(const InputType &source, ...) END" << std::endl;
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
|
@ -1,18 +1,14 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include "StatelessFeatureFunction.h"
|
||||
#include "util/check.hh"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class TargetPhrase;
|
||||
class ScoreComponentCollection;
|
||||
|
||||
class PhrasePenaltyProducer : public StatelessFeatureFunction
|
||||
class PhrasePenalty : public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
PhrasePenaltyProducer(const std::string &line);
|
||||
PhrasePenalty(const std::string &line);
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const {
|
||||
return true;
|
||||
@ -23,9 +19,20 @@ public:
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
|
||||
virtual void Evaluate(const InputType &source
|
||||
, ScoreComponentCollection &scoreBreakdown) const;
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
} //namespace
|
||||
|
||||
|
@ -28,10 +28,20 @@ public:
|
||||
|
||||
bool IsUseable(const FactorMask &mask) const;
|
||||
|
||||
virtual void Evaluate(const Phrase &source
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void ComputeFeatures(const Phrase &source,
|
||||
const TargetPhrase& targetPhrase,
|
||||
|
@ -23,16 +23,14 @@ public:
|
||||
/**
|
||||
* This should be implemented for features that apply to phrase-based models.
|
||||
**/
|
||||
virtual void Evaluate(const PhraseBasedFeatureContext& context,
|
||||
ScoreComponentCollection* accumulator) const {
|
||||
}
|
||||
virtual void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const = 0;
|
||||
|
||||
/**
|
||||
* Same for chart-based features.
|
||||
**/
|
||||
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
|
||||
ScoreComponentCollection* accumulator) const {
|
||||
}
|
||||
virtual void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const = 0;
|
||||
|
||||
virtual bool IsStateless() const {
|
||||
return true;
|
||||
|
@ -45,8 +45,18 @@ public:
|
||||
virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */,
|
||||
int /* featureID */,
|
||||
ScoreComponentCollection* ) const {
|
||||
abort();
|
||||
throw std::logic_error("TargetBigramFeature not valid in chart decoder");
|
||||
}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
private:
|
||||
|
@ -191,6 +191,17 @@ public:
|
||||
|
||||
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
|
||||
ScoreComponentCollection* accumulator) const;
|
||||
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
private:
|
||||
|
@ -32,6 +32,16 @@ public:
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
|
||||
void ComputeFeatures(const Phrase &source,
|
||||
const TargetPhrase& targetPhrase,
|
||||
|
@ -22,6 +22,22 @@ public:
|
||||
}
|
||||
std::vector<float> DefaultWeights() const;
|
||||
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
void Evaluate(const Phrase &source
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -22,6 +22,16 @@ public:
|
||||
, const TargetPhrase &targetPhrase
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const;
|
||||
void Evaluate(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void EvaluateChart(const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{}
|
||||
void Evaluate(const InputType &input
|
||||
, const InputPath &inputPath
|
||||
, ScoreComponentCollection &scoreBreakdown) const
|
||||
{}
|
||||
|
||||
|
||||
/*
|
||||
|
@ -137,16 +137,17 @@ void WordTranslationFeature::Load()
|
||||
}
|
||||
|
||||
void WordTranslationFeature::Evaluate
|
||||
(const PhraseBasedFeatureContext& context,
|
||||
(const Hypothesis& hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
|
||||
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
|
||||
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
|
||||
const TranslationOption& transOpt = hypo.GetTranslationOption();
|
||||
const TargetPhrase& targetPhrase = hypo.GetCurrTargetPhrase();
|
||||
const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();
|
||||
|
||||
// process aligned words
|
||||
for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) {
|
||||
const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase();
|
||||
const Phrase& sourcePhrase = transOpt.GetInputPath().GetPhrase();
|
||||
int sourceIndex = alignmentPoint->first;
|
||||
int targetIndex = alignmentPoint->second;
|
||||
Word ws = sourcePhrase.GetWord(sourceIndex);
|
||||
@ -242,7 +243,7 @@ void WordTranslationFeature::Evaluate
|
||||
}
|
||||
}
|
||||
if (m_sourceContext) {
|
||||
size_t globalSourceIndex = context.GetTranslationOption().GetStartPos() + sourceIndex;
|
||||
size_t globalSourceIndex = hypo.GetTranslationOption().GetStartPos() + sourceIndex;
|
||||
if (!m_domainTrigger && globalSourceIndex == 0) {
|
||||
// add <s> trigger feature for source
|
||||
stringstream feature;
|
||||
@ -348,150 +349,10 @@ void WordTranslationFeature::Evaluate
|
||||
}
|
||||
|
||||
void WordTranslationFeature::EvaluateChart(
|
||||
const ChartBasedFeatureContext& context,
|
||||
const ChartHypothesis &hypo,
|
||||
ScoreComponentCollection* accumulator) const
|
||||
{
|
||||
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
|
||||
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
|
||||
|
||||
// process aligned words
|
||||
for (AlignmentInfo::const_iterator alignmentPoint = alignmentInfo.begin(); alignmentPoint != alignmentInfo.end(); alignmentPoint++) {
|
||||
const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase();
|
||||
int sourceIndex = alignmentPoint->first;
|
||||
int targetIndex = alignmentPoint->second;
|
||||
Word ws = sourcePhrase.GetWord(sourceIndex);
|
||||
if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue;
|
||||
Word wt = targetPhrase.GetWord(targetIndex);
|
||||
if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue;
|
||||
StringPiece sourceWord = ws.GetFactor(m_factorTypeSource)->GetString();
|
||||
StringPiece targetWord = wt.GetFactor(m_factorTypeTarget)->GetString();
|
||||
if (m_ignorePunctuation) {
|
||||
// check if source or target are punctuation
|
||||
char firstChar = sourceWord[0];
|
||||
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
|
||||
if(charIterator != m_punctuationHash.end())
|
||||
continue;
|
||||
firstChar = targetWord[0];
|
||||
charIterator = m_punctuationHash.find( firstChar );
|
||||
if(charIterator != m_punctuationHash.end())
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!m_unrestricted) {
|
||||
if (FindStringPiece(m_vocabSource, sourceWord) == m_vocabSource.end())
|
||||
sourceWord = "OTHER";
|
||||
if (FindStringPiece(m_vocabTarget, targetWord) == m_vocabTarget.end())
|
||||
targetWord = "OTHER";
|
||||
}
|
||||
|
||||
if (m_simple) {
|
||||
// construct feature name
|
||||
stringstream featureName;
|
||||
featureName << m_description << "_";
|
||||
//featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER");
|
||||
featureName << sourceWord;
|
||||
featureName << "~";
|
||||
//featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER");
|
||||
featureName << targetWord;
|
||||
accumulator->SparsePlusEquals(featureName.str(), 1);
|
||||
}
|
||||
/* if (m_sourceContext) {
|
||||
size_t globalSourceIndex = cur_hypo.GetCurrSourceRange().GetStartPos() + sourceIndex;
|
||||
if (globalSourceIndex == 0) {
|
||||
// add <s> trigger feature for source
|
||||
stringstream feature;
|
||||
feature << "wt_";
|
||||
feature << "<s>,";
|
||||
feature << sourceWord;
|
||||
feature << "~";
|
||||
feature << targetWord;
|
||||
accumulator->SparsePlusEquals(feature.str(), 1);
|
||||
cerr << feature.str() << endl;
|
||||
}
|
||||
|
||||
// range over source words to get context
|
||||
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
|
||||
if (contextIndex == globalSourceIndex) continue;
|
||||
string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString();
|
||||
if (m_ignorePunctuation) {
|
||||
// check if trigger is punctuation
|
||||
char firstChar = sourceTrigger.at(0);
|
||||
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
|
||||
if(charIterator != m_punctuationHash.end())
|
||||
continue;
|
||||
}
|
||||
|
||||
bool sourceTriggerExists = false;
|
||||
if (!m_unrestricted)
|
||||
sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end();
|
||||
|
||||
if (m_unrestricted || sourceTriggerExists) {
|
||||
stringstream feature;
|
||||
feature << "wt_";
|
||||
if (contextIndex < globalSourceIndex) {
|
||||
feature << sourceTrigger;
|
||||
feature << ",";
|
||||
feature << sourceWord;
|
||||
}
|
||||
else {
|
||||
feature << sourceWord;
|
||||
feature << ",";
|
||||
feature << sourceTrigger;
|
||||
}
|
||||
feature << "~";
|
||||
feature << targetWord;
|
||||
accumulator->SparsePlusEquals(feature.str(), 1);
|
||||
cerr << feature.str() << endl;
|
||||
}
|
||||
}
|
||||
}*/
|
||||
/* if (m_targetContext) {
|
||||
size_t globalTargetIndex = 0; // TODO
|
||||
// size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex;
|
||||
if (globalTargetIndex == 0) {
|
||||
// add <s> trigger feature for source
|
||||
stringstream feature;
|
||||
feature << "wt_";
|
||||
feature << sourceWord;
|
||||
feature << "~";
|
||||
feature << "<s>,";
|
||||
feature << targetWord;
|
||||
accumulator->SparsePlusEquals(feature.str(), 1);
|
||||
cerr << feature.str() << endl;
|
||||
}
|
||||
|
||||
// range over target words (up to current position) to get context
|
||||
for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) {
|
||||
Phrase outputPhrase = cur_hypo.GetOutputPhrase();
|
||||
string targetTrigger = outputPhrase.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
|
||||
//string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString();
|
||||
if (m_ignorePunctuation) {
|
||||
// check if trigger is punctuation
|
||||
char firstChar = targetTrigger.at(0);
|
||||
CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar );
|
||||
if(charIterator != m_punctuationHash.end())
|
||||
continue;
|
||||
}
|
||||
|
||||
bool targetTriggerExists = false;
|
||||
if (!m_unrestricted)
|
||||
targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end();
|
||||
|
||||
if (m_unrestricted || targetTriggerExists) {
|
||||
stringstream feature;
|
||||
feature << "wt_";
|
||||
feature << sourceWord;
|
||||
feature << "~";
|
||||
feature << targetTrigger;
|
||||
feature << ",";
|
||||
feature << targetWord;
|
||||
accumulator->SparsePlusEquals(feature.str(), 1);
|
||||
cerr << feature.str() << endl;
|
||||
}
|
||||
}
|
||||
}*/
|
||||
}
|
||||
|
||||
UTIL_THROW(util::Exception, "Need source phrase. Can't be arsed at the moment");
|
||||
}
|
||||
|
||||
bool WordTranslationFeature::IsUseable(const FactorMask &mask) const
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user