moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 05:14:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://github.com/moses-smt/mosesdecoder

Browse Source
This commit is contained in:
phikoehn 2014-07-23 15:40:04 +01:00
commit 2d11fe3916
216 changed files with 7945 additions and 1904 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -79,3 +79,4 @@ nbproject/
mingw/MosesGUI/MosesGUI.e4p
mingw/MosesGUI/_eric4project/
contrib/m4m/merge-sorted

11
Jamroot
View File

@ -114,6 +114,13 @@ requirements += [ option.get "with-mm" : : <define>PT_UG ] ;
requirements += [ option.get "with-mm" : : <define>MAX_NUM_FACTORS=4 ] ;
requirements += [ option.get "unlabelled-source" : : <define>UNLABELLED_SOURCE ] ;
if [ option.get "with-lbllm" ] {
external-lib boost_serialization ;
external-lib gomp ;
requirements += <library>boost_serialization ;
requirements += <library>gomp ;
}
if [ option.get "with-cmph" ] {
requirements += <define>HAVE_CMPH ;
}
@ -152,13 +159,15 @@ build-projects lm util phrase-extract search moses moses/LM mert moses-cmd moses
if [ option.get "with-mm" : : "yes" ]
{
alias mm :
moses/TranslationModel/UG//spe-check-coverage2
moses/TranslationModel/UG//ptable-lookup
moses/TranslationModel/UG//sim-pe
moses/TranslationModel/UG//spe-check-coverage
moses/TranslationModel/UG/mm//mtt-build
moses/TranslationModel/UG/mm//mtt-dump
moses/TranslationModel/UG/mm//symal2mam
moses/TranslationModel/UG/mm//mam2symal
moses/TranslationModel/UG/mm//mam_verify
moses/TranslationModel/UG/mm//custom-pt
moses/TranslationModel/UG/mm//mmlex-build
moses/TranslationModel/UG/mm//mmlex-lookup
moses/TranslationModel/UG/mm//mtt-count-words

20
OnDiskPt/Word.cpp
View File

@ -104,14 +104,20 @@ void Word::ConvertToMoses(
Moses::FactorCollection &factorColl = Moses::FactorCollection::Instance();
overwrite = Moses::Word(m_isNonTerminal);
// TODO: this conversion should have been done at load time.
util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');
for (std::vector<Moses::FactorType>::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) {
UTIL_THROW_IF2(!tok, "Too few factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
overwrite.SetFactor(*t, factorColl.AddFactor(*tok, m_isNonTerminal));
if (m_isNonTerminal) {
const std::string &tok = vocab.GetString(m_vocabId);
overwrite.SetFactor(0, factorColl.AddFactor(tok, m_isNonTerminal));
}
else {
// TODO: this conversion should have been done at load time.
util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');
for (std::vector<Moses::FactorType>::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) {
UTIL_THROW_IF2(!tok, "Too few factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
overwrite.SetFactor(*t, factorColl.AddFactor(*tok, m_isNonTerminal));
}
UTIL_THROW_IF2(tok, "Too many factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
}
UTIL_THROW_IF2(tok, "Too many factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
}
int Word::Compare(const Word &compare) const

2
OnDiskPt/queryOnDiskPt.cpp
View File

@ -22,7 +22,7 @@ int main(int argc, char **argv)
{
int tableLimit = 20;
std::string ttable = "";
bool useAlignments = false;
// bool useAlignments = false;
for(int i = 1; i < argc; i++) {
if(!strcmp(argv[i], "-tlimit")) {

19
contrib/other-builds/extract-rules/.project
View File

@ -25,6 +25,21 @@
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
</natures>
<linkedResources>
<link>
<name>Hole.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/Hole.h</locationURI>
</link>
<link>
<name>HoleCollection.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.cpp</locationURI>
</link>
<link>
<name>HoleCollection.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/HoleCollection.h</locationURI>
</link>
<link>
<name>InputFileStream.cpp</name>
<type>1</type>
@ -86,9 +101,9 @@
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/XmlTree.h</locationURI>
</link>
<link>
<name>extract-main.cpp</name>
<name>extract-rules-main.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/phrase-extract/extract-main.cpp</locationURI>
<locationURI>PARENT-1-ECLIPSE_HOME/github/mosesdecoder/phrase-extract/extract-rules-main.cpp</locationURI>
</link>
<link>
<name>tables-core.cpp</name>

107
contrib/other-builds/moses/.cproject
View File

@ -36,7 +36,6 @@
<listOptionValue builtIn="false" value="/opt/local/include/"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../irstlm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../nplm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../eigen&quot;"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../srilm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/darts-clone&quot;"/>
@ -53,11 +52,11 @@
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_DALM"/>
<listOptionValue builtIn="false" value="LM_RAND"/>
<listOptionValue builtIn="false" value="LM_NPLM"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BIT=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>
<option id="gnu.cpp.compiler.option.dialect.std.1734198568" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" value="gnu.cpp.compiler.dialect.c++11" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1905116220" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.2126314903" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
@ -80,8 +79,18 @@
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.511477442" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1211280539" name="DALMWrapper.h" rcbsApplicability="disable" resourcePath="LM/DALMWrapper.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.790052015" name="IRST.h" rcbsApplicability="disable" resourcePath="LM/IRST.h" toolsToInvoke=""/>
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.2147044266" name="/" resourcePath="LM/oxlm">
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.369597529" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug" unusedChildren="">
<tool id="cdt.managedbuild.tool.gnu.archiver.base.1919082719" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base.1976472988"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.451148054" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1774992327"/>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1343615699" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.2126314903"/>
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1929750597" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1168585173"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.1235749054" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.2074660557"/>
<tool id="cdt.managedbuild.tool.gnu.assembler.exe.debug.8387388" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.debug.933467113"/>
</toolChain>
</folderInfo>
<sourceEntries>
<entry excluding="TranslationModel/fuzzy-match/FuzzyMatchWrapper.cpp|TranslationModel/CompactPT|LM/NeuralLMWrapper.cpp|FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
<entry excluding="LM/oxlm|LM/Rand.h|LM/Rand.cpp|TranslationModel/CompactPT|LM/NeuralLMWrapper.cpp|FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>
</configuration>
</storageModule>
@ -133,6 +142,95 @@
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380" moduleId="org.eclipse.cdt.core.settings" name="Debug with oxlm">
<externalSettings>
<externalSetting>
<entry flags="VALUE_WORKSPACE_PATH" kind="includePath" name="/moses"/>
<entry flags="VALUE_WORKSPACE_PATH" kind="libraryPath" name="/moses/Debug with oxlm"/>
<entry flags="RESOLVED" kind="libraryFile" name="moses" srcPrefixMapping="" srcRootPath=""/>
</externalSetting>
</externalSettings>
<extensions>
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
</extensions>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380" name="Debug with oxlm" parent="cdt.managedbuild.config.gnu.exe.debug">
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380." name="/" resourcePath="">
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.58016517" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.debug.1519676809" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
<builder buildPath="${workspace_loc:/moses/Debug}" id="cdt.managedbuild.target.gnu.builder.exe.debug.210713286" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
<tool id="cdt.managedbuild.tool.gnu.archiver.base.352461864" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1930334119" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.287782778" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1878892542" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.include.paths.1954109101" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../probingPT/helpers&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/include&quot;"/>
<listOptionValue builtIn="false" value="/opt/local/include/"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../irstlm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../nplm/src&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../eigen&quot;"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../srilm/include"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/darts-clone&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../randlm/include/RandLM&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../eigen-3&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../oxlm/src&quot;"/>
<listOptionValue builtIn="false" value="${workspace_loc}/../../"/>
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.274233516" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="IS_ECLIPSE"/>
<listOptionValue builtIn="false" value="HAVE_PROBINGPT"/>
<listOptionValue builtIn="false" value="HAVE_BOOST"/>
<listOptionValue builtIn="false" value="MAX_NUM_FACTORS=4"/>
<listOptionValue builtIn="false" value="WITH_THREADS"/>
<listOptionValue builtIn="false" value="KENLM_MAX_ORDER=7"/>
<listOptionValue builtIn="false" value="TRACE_ENABLE"/>
<listOptionValue builtIn="false" value="LM_IRST"/>
<listOptionValue builtIn="false" value="LM_DALM"/>
<listOptionValue builtIn="false" value="LM_NPLM"/>
<listOptionValue builtIn="false" value="LM_LBL"/>
<listOptionValue builtIn="false" value="_FILE_OFFSET_BIT=64"/>
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
</option>
<option id="gnu.cpp.compiler.option.dialect.std.1353163586" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" value="gnu.cpp.compiler.dialect.c++11" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1413141770" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1040012873" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.260276259" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
<option id="gnu.c.compiler.exe.debug.option.debugging.level.744208673" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.699852884" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1114065632" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.373508964" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1873470979" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
</inputType>
</tool>
<tool id="cdt.managedbuild.tool.gnu.assembler.exe.debug.1046426871" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.debug">
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1482215763" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
</tool>
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380.LM/Rand.h" name="Rand.h" rcbsApplicability="disable" resourcePath="LM/Rand.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380.LM/IRST.h" name="IRST.h" rcbsApplicability="disable" resourcePath="LM/IRST.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.916939380.LM/DALMWrapper.h" name="DALMWrapper.h" rcbsApplicability="disable" resourcePath="LM/DALMWrapper.h" toolsToInvoke=""/>
<sourceEntries>
<entry excluding="LM/Rand.h|LM/Rand.cpp|TranslationModel/CompactPT|LM/NeuralLMWrapper.cpp|FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
</cconfiguration>
</storageModule>
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
<project id="moses.cdt.managedbuild.target.gnu.exe.1375079569" name="Executable" projectType="cdt.managedbuild.target.gnu.exe"/>
@ -171,6 +269,9 @@
<configuration configurationName="Debug">
<resource resourceType="PROJECT" workspacePath="/moses"/>
</configuration>
<configuration configurationName="Debug with oxlm">
<resource resourceType="PROJECT" workspacePath="/moses"/>
</configuration>
</storageModule>
<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>

75
contrib/other-builds/moses/.project
View File

@ -601,16 +601,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ReorderingConstraint.h</locationURI>
</link>
<link>
<name>ReorderingStack.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ReorderingStack.cpp</locationURI>
</link>
<link>
<name>ReorderingStack.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ReorderingStack.h</locationURI>
</link>
<link>
<name>RuleCube.cpp</name>
<type>1</type>
@ -1331,6 +1321,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SpanLength.h</locationURI>
</link>
<link>
<name>FF/SparseHieroReorderingFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SparseHieroReorderingFeature.cpp</locationURI>
</link>
<link>
<name>FF/SparseHieroReorderingFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SparseHieroReorderingFeature.h</locationURI>
</link>
<link>
<name>FF/StatefulFeatureFunction.cpp</name>
<type>1</type>
@ -1636,6 +1636,11 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/backward.arpa</locationURI>
</link>
<link>
<name>LM/oxlm</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>PP/CountsPhraseProperty.cpp</name>
<type>1</type>
@ -1656,6 +1661,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PP/Factory.h</locationURI>
</link>
<link>
<name>PP/NonTermContextProperty.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PP/NonTermContextProperty.cpp</locationURI>
</link>
<link>
<name>PP/NonTermContextProperty.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PP/NonTermContextProperty.h</locationURI>
</link>
<link>
<name>PP/PhraseProperty.cpp</name>
<type>1</type>
@ -1916,6 +1931,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/ReorderingStack.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/ReorderingStack.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/ReorderingStack.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/ReorderingStack.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/SparseReordering.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/SparseReordering.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/SparseReordering.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/SparseReordering.h</locationURI>
</link>
<link>
<name>FF/OSM-Feature/OpSequenceModel.cpp</name>
<type>1</type>
@ -1936,6 +1971,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/OSM-Feature/osmHyp.h</locationURI>
</link>
<link>
<name>LM/oxlm/LBLLM.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/LBLLM.cpp</locationURI>
</link>
<link>
<name>LM/oxlm/LBLLM.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/LBLLM.h</locationURI>
</link>
<link>
<name>LM/oxlm/Mapper.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/Mapper.cpp</locationURI>
</link>
<link>
<name>LM/oxlm/Mapper.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/oxlm/Mapper.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerCYKPlus.cpp</name>
<type>1</type>

39
contrib/server/mosesserver.cpp
View File

@ -4,6 +4,7 @@
#include <algorithm>
#include "moses/Util.h"
#include "moses/ChartManager.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
@ -59,7 +60,7 @@ public:
if(add2ORLM_) {
//updateORLM();
}
cerr << "Done inserting\n";
XVERBOSE(1,"Done inserting\n");
//PhraseDictionary* pdsa = (PhraseDictionary*) pdf->GetDictionary(*dummy);
map<string, xmlrpc_c::value> retData;
//*retvalP = xmlrpc_c::value_struct(retData);
@ -120,17 +121,17 @@ public:
if(si == params.end())
throw xmlrpc_c::fault("Missing source sentence", xmlrpc_c::fault::CODE_PARSE);
source_ = xmlrpc_c::value_string(si->second);
cerr << "source = " << source_ << endl;
XVERBOSE(1,"source = " << source_ << endl);
si = params.find("target");
if(si == params.end())
throw xmlrpc_c::fault("Missing target sentence", xmlrpc_c::fault::CODE_PARSE);
target_ = xmlrpc_c::value_string(si->second);
cerr << "target = " << target_ << endl;
XVERBOSE(1,"target = " << target_ << endl);
si = params.find("alignment");
if(si == params.end())
throw xmlrpc_c::fault("Missing alignment", xmlrpc_c::fault::CODE_PARSE);
alignment_ = xmlrpc_c::value_string(si->second);
cerr << "alignment = " << alignment_ << endl;
XVERBOSE(1,"alignment = " << alignment_ << endl);
si = params.find("bounded");
bounded_ = (si != params.end());
si = params.find("updateORLM");
@ -224,7 +225,7 @@ public:
}
const string source((xmlrpc_c::value_string(si->second)));
cerr << "Input: " << source << endl;
XVERBOSE(1,"Input: " << source << endl);
si = params.find("align");
bool addAlignInfo = (si != params.end());
si = params.find("word-align");
@ -287,13 +288,13 @@ public:
}
} else {
Sentence sentence;
const vector<FactorType> &inputFactorOrder =
staticData.GetInputFactorOrder();
const vector<FactorType> &
inputFactorOrder = staticData.GetInputFactorOrder();
stringstream in(source + "\n");
sentence.Read(in,inputFactorOrder);
size_t lineNumber = 0; // TODO: Include sentence request number here?
Manager manager(lineNumber, sentence, staticData.GetSearchAlgorithm());
manager.ProcessSentence();
manager.ProcessSentence();
const Hypothesis* hypo = manager.GetBestHypothesis();
vector<xmlrpc_c::value> alignInfo;
@ -331,7 +332,7 @@ public:
pair<string, xmlrpc_c::value>
text("text", xmlrpc_c::value_string(out.str()));
retData.insert(text);
cerr << "Output: " << out.str() << endl;
XVERBOSE(1,"Output: " << out.str() << endl);
*retvalP = xmlrpc_c::value_struct(retData);
}
@ -574,7 +575,7 @@ int main(int argc, char** argv)
{
//Extract port and log, send other args to moses
char** mosesargv = new char*[argc+2];
char** mosesargv = new char*[argc+2]; // why "+2" [UG]
int mosesargc = 0;
int port = 8080;
const char* logfile = "/dev/null";
@ -634,11 +635,11 @@ int main(int argc, char** argv)
myRegistry.addMethod("updater", updater);
myRegistry.addMethod("optimize", optimizer);
xmlrpc_c::serverAbyss myAbyssServer(
myRegistry,
port, // TCP port on which to listen
logfile
);
xmlrpc_c::serverAbyss myAbyssServer(
myRegistry,
port, // TCP port on which to listen
logfile
);
/* doesn't work with xmlrpc-c v. 1.16.33 - ie very old lib on Ubuntu 12.04
xmlrpc_c::serverAbyss myAbyssServer(
xmlrpc_c::serverAbyss::constrOpt()
@ -648,12 +649,10 @@ int main(int argc, char** argv)
.allowOrigin("*")
);
*/
cerr << "Listening on port " << port << endl;
XVERBOSE(1,"Listening on port " << port << endl);
if (isSerial) {
while(1) {
myAbyssServer.runOnce();
}
while(1) myAbyssServer.runOnce();
} else {
myAbyssServer.run();
}

31
doc/Mmsapt.howto Normal file
View File

@ -0,0 +1,31 @@
How to use memory-mapped suffix array phrase tables in the moses decoder
(phrase-based decoding only)
1. Compile with the bjam switch --with-mm
2. You need
- sentences aligned text files
- the word alignment between these files in symal output format
3. Build binary files
Let
${L1} be the extension of the language that you are translating from,
${L2} the extension of the language that you want to translate into, and
${CORPUS} the name of the word-aligned training corpus
% zcat ${CORPUS}.${L1}.gz | mtt-build -i -o /some/path/${CORPUS}.${L1}
% zcat ${CORPUS}.${L2}.gz | mtt-build -i -o /some/path/${CORPUS}.${L2}
% zcat ${CORPUS}.${L1}-${L2}.symal.gz | symal2mam /some/path/${CORPUS}.${L1}-${L2}.mam
% mmlex-build /some/path/${CORPUS} ${L1} ${L2} -o /some/path/${CORPUS}.${L1}-${L2}.lex -c /some/path/${CORPUS}.${L1}-${L2}.coc
4. Define line in moses.ini
The best configuration of phrase table features is still under investigation.
For the time being, try this:
Mmsapt name=PT0 output-factor=0 num-features=9 base=/some/path/${CORPUS} L1=${L1} L2=${L2} pfwd=g pbwd=g smooth=0 sample=1000 workers=1
You can increase the number of workers for sampling (a bit faster),
but you'll lose replicability of the translation output.

2
lm/Jamfile
View File

@ -37,4 +37,4 @@ for local p in [ glob *_main.cc ] {
exes += $(name) ;
}
alias programs : $(exes) filter//filter : <threading>multi:<source>builder//lmplz ;
alias programs : $(exes) filter//filter builder//dump_counts : <threading>multi:<source>builder//lmplz ;

4
lm/builder/Jamfile
View File

@ -4,6 +4,10 @@ fakelib builder : [ glob *.cc : *test.cc *main.cc ]
exe lmplz : lmplz_main.cc builder /top//boost_program_options ;
exe dump_counts : dump_counts_main.cc builder ;
alias programs : lmplz dump_counts ;
import testing ;
unit-test corpus_count_test : corpus_count_test.cc builder /top//boost_unit_test_framework ;
unit-test adjust_counts_test : adjust_counts_test.cc builder /top//boost_unit_test_framework ;

36
lm/builder/dump_counts_main.cc Normal file
View File

@ -0,0 +1,36 @@
#include "lm/builder/print.hh"
#include "lm/word_index.hh"
#include "util/file.hh"
#include "util/read_compressed.hh"
#include <boost/lexical_cast.hpp>
#include <iostream>
#include <vector>
int main(int argc, char *argv[]) {
if (argc != 4) {
std::cerr << "Usage: " << argv[0] << " counts vocabulary order\n"
"The counts file contains records with 4-byte vocabulary ids followed by 8-byte\n"
"counts. Each record has order many vocabulary ids.\n"
"The vocabulary file contains the words delimited by NULL in order of id.\n"
"The vocabulary file may not be compressed because it is mmapped but the counts\n"
"file can be compressed.\n";
return 1;
}
util::ReadCompressed counts(util::OpenReadOrThrow(argv[1]));
util::scoped_fd vocab_file(util::OpenReadOrThrow(argv[2]));
lm::builder::VocabReconstitute vocab(vocab_file.get());
unsigned int order = boost::lexical_cast<unsigned int>(argv[3]);
std::vector<char> record(sizeof(uint32_t) * order + sizeof(uint64_t));
while (std::size_t got = counts.ReadOrEOF(&*record.begin(), record.size())) {
UTIL_THROW_IF(got != record.size(), util::Exception, "Read " << got << " bytes at the end of file, which is not a complete record of length " << record.size());
const lm::WordIndex *words = reinterpret_cast<const lm::WordIndex*>(&*record.begin());
for (const lm::WordIndex *i = words; i != words + order; ++i) {
UTIL_THROW_IF(*i >= vocab.Size(), util::Exception, "Vocab ID " << *i << " is larger than the vocab file's maximum of " << vocab.Size() << ". Are you sure you have the right order and vocab file for these counts?");
std::cout << vocab.Lookup(*i) << ' ';
}
// TODO don't use std::cout because it is slow. Add fast uint64_t printing support to FakeOFStream.
std::cout << *reinterpret_cast<const uint64_t*>(words + order) << '\n';
}
}

5
lm/builder/print.cc
View File

@ -54,9 +54,8 @@ void PrintARPA::Run(const util::stream::ChainPositions &positions) {
for (const WordIndex *i = stream->begin() + 1; i != stream->end(); ++i) {
out << ' ' << vocab_.Lookup(*i);
}
float backoff = stream->Value().complete.backoff;
if (backoff != 0.0)
out << '\t' << backoff;
if (order != positions.size())
out << '\t' << stream->Value().complete.backoff;
out << '\n';
}

2
lm/ngram_query.hh
View File

@ -36,7 +36,7 @@ struct FullPrint : public BasicPrint {
"Perplexity including OOVs:\t" << ppl_including_oov << "\n"
"Perplexity excluding OOVs:\t" << ppl_excluding_oov << "\n"
"OOVs:\t" << corpus_oov << "\n"
"Tokenss:\t" << corpus_tokens << '\n'
"Tokens:\t" << corpus_tokens << '\n'
;
}
};

3
lm/wrappers/README Normal file
View File

@ -0,0 +1,3 @@
This directory is for wrappers around other people's LMs, presenting an interface similar to KenLM's. You will need to have their LM installed.
NPLM is a work in progress.

90
lm/wrappers/nplm.cc Normal file
View File

@ -0,0 +1,90 @@
#include "lm/wrappers/nplm.hh"
#include "util/exception.hh"
#include "util/file.hh"
#include <algorithm>
#include <string.h>
#include "neuralLM.h"
namespace lm {
namespace np {
Vocabulary::Vocabulary(const nplm::vocabulary &vocab)
: base::Vocabulary(vocab.lookup_word("<s>"), vocab.lookup_word("</s>"), vocab.lookup_word("<unk>")),
vocab_(vocab), null_word_(vocab.lookup_word("<null>")) {}
Vocabulary::~Vocabulary() {}
WordIndex Vocabulary::Index(const std::string &str) const {
return vocab_.lookup_word(str);
}
bool Model::Recognize(const std::string &name) {
try {
util::scoped_fd file(util::OpenReadOrThrow(name.c_str()));
char magic_check[16];
util::ReadOrThrow(file.get(), magic_check, sizeof(magic_check));
const char nnlm_magic[] = "\\config\nversion ";
return !memcmp(magic_check, nnlm_magic, 16);
} catch (const util::Exception &) {
return false;
}
}
Model::Model(const std::string &file, std::size_t cache)
: base_instance_(new nplm::neuralLM(file)), vocab_(base_instance_->get_vocabulary()), cache_size_(cache) {
UTIL_THROW_IF(base_instance_->get_order() > NPLM_MAX_ORDER, util::Exception, "This NPLM has order " << (unsigned int)base_instance_->get_order() << " but the KenLM wrapper was compiled with " << NPLM_MAX_ORDER << ". Change the defintion of NPLM_MAX_ORDER and recompile.");
// log10 compatible with backoff models.
base_instance_->set_log_base(10.0);
State begin_sentence, null_context;
std::fill(begin_sentence.words, begin_sentence.words + NPLM_MAX_ORDER - 1, base_instance_->lookup_word("<s>"));
null_word_ = base_instance_->lookup_word("<null>");
std::fill(null_context.words, null_context.words + NPLM_MAX_ORDER - 1, null_word_);
Init(begin_sentence, null_context, vocab_, base_instance_->get_order());
}
Model::~Model() {}
FullScoreReturn Model::FullScore(const State &from, const WordIndex new_word, State &out_state) const {
nplm::neuralLM *lm = backend_.get();
if (!lm) {
lm = new nplm::neuralLM(*base_instance_);
backend_.reset(lm);
lm->set_cache(cache_size_);
}
// State is in natural word order.
FullScoreReturn ret;
for (int i = 0; i < lm->get_order() - 1; ++i) {
lm->staging_ngram()(i) = from.words[i];
}
lm->staging_ngram()(lm->get_order() - 1) = new_word;
ret.prob = lm->lookup_from_staging();
// Always say full order.
ret.ngram_length = lm->get_order();
// Shift everything down by one.
memcpy(out_state.words, from.words + 1, sizeof(WordIndex) * (lm->get_order() - 2));
out_state.words[lm->get_order() - 2] = new_word;
// Fill in trailing words with zeros so state comparison works.
memset(out_state.words + lm->get_order() - 1, 0, sizeof(WordIndex) * (NPLM_MAX_ORDER - lm->get_order()));
return ret;
}
// TODO: optimize with direct call?
FullScoreReturn Model::FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const {
// State is in natural word order. The API here specifies reverse order.
std::size_t state_length = std::min<std::size_t>(Order() - 1, context_rend - context_rbegin);
State state;
// Pad with null words.
for (lm::WordIndex *i = state.words; i < state.words + Order() - 1 - state_length; ++i) {
*i = null_word_;
}
// Put new words at the end.
std::reverse_copy(context_rbegin, context_rbegin + state_length, state.words + Order() - 1 - state_length);
return FullScore(state, new_word, out_state);
}
} // namespace np
} // namespace lm

83
lm/wrappers/nplm.hh Normal file
View File

@ -0,0 +1,83 @@
#ifndef LM_WRAPPERS_NPLM_H
#define LM_WRAPPERS_NPLM_H
#include "lm/facade.hh"
#include "lm/max_order.hh"
#include "util/string_piece.hh"
#include <boost/thread/tss.hpp>
#include <boost/scoped_ptr.hpp>
/* Wrapper to NPLM "by Ashish Vaswani, with contributions from David Chiang
* and Victoria Fossum."
* http://nlg.isi.edu/software/nplm/
*/
namespace nplm {
class vocabulary;
class neuralLM;
} // namespace nplm
namespace lm {
namespace np {
class Vocabulary : public base::Vocabulary {
public:
Vocabulary(const nplm::vocabulary &vocab);
~Vocabulary();
WordIndex Index(const std::string &str) const;
// TODO: lobby them to support StringPiece
WordIndex Index(const StringPiece &str) const {
return Index(std::string(str.data(), str.size()));
}
lm::WordIndex NullWord() const { return null_word_; }
private:
const nplm::vocabulary &vocab_;
const lm::WordIndex null_word_;
};
// Sorry for imposing my limitations on your code.
#define NPLM_MAX_ORDER 7
struct State {
WordIndex words[NPLM_MAX_ORDER - 1];
};
class Model : public lm::base::ModelFacade<Model, State, Vocabulary> {
private:
typedef lm::base::ModelFacade<Model, State, Vocabulary> P;
public:
// Does this look like an NPLM?
static bool Recognize(const std::string &file);
explicit Model(const std::string &file, std::size_t cache_size = 1 << 20);
~Model();
FullScoreReturn FullScore(const State &from, const WordIndex new_word, State &out_state) const;
FullScoreReturn FullScoreForgotState(const WordIndex *context_rbegin, const WordIndex *context_rend, const WordIndex new_word, State &out_state) const;
private:
boost::scoped_ptr<nplm::neuralLM> base_instance_;
mutable boost::thread_specific_ptr<nplm::neuralLM> backend_;
Vocabulary vocab_;
lm::WordIndex null_word_;
const std::size_t cache_size_;
};
} // namespace np
} // namespace lm
#endif // LM_WRAPPERS_NPLM_H

4
mert/BleuScorer.cpp
View File

@ -266,12 +266,12 @@ float smoothedSentenceBleu
float sentenceLevelBackgroundBleu(const std::vector<float>& sent, const std::vector<float>& bg)
{
// Sum sent and background
std::vector<float> stats;
UTIL_THROW_IF(sent.size()!=bg.size(), util::Exception, "Error");
UTIL_THROW_IF(sent.size() != kBleuNgramOrder * 2 + 1, util::Exception, "Error");
std::vector<float> stats(sent.size());
for(size_t i=0; i<sent.size(); i++)
stats.push_back(sent[i]+bg[i]);
stats[i] = sent[i]+bg[i];
// Calculate BLEU
float logbleu = 0.0;

2
mert/BleuScorer.h
View File

@ -13,7 +13,7 @@
namespace MosesTuning
{
const int kBleuNgramOrder = 4;
const size_t kBleuNgramOrder = 4;
class NgramCounts;
class Reference;

30
mert/FeatureStats.cpp
View File

@ -14,6 +14,8 @@
#include <boost/functional/hash.hpp>
#include "util/murmur_hash.hh"
#include "Util.h"
using namespace std;
@ -59,6 +61,11 @@ void SparseVector::set(const string& name, FeatureStatsType value)
m_fvector[id] = value;
}
void SparseVector::set(size_t id, FeatureStatsType value) {
assert(m_id_to_name.size() > id);
m_fvector[id] = value;
}
void SparseVector::write(ostream& out, const string& sep) const
{
for (fvector_t::const_iterator i = m_fvector.begin(); i != m_fvector.end(); ++i) {
@ -91,6 +98,16 @@ void SparseVector::load(const string& file)
}
}
SparseVector& SparseVector::operator+=(const SparseVector& rhs)
{
for (fvector_t::const_iterator i = rhs.m_fvector.begin();
i != rhs.m_fvector.end(); ++i) {
m_fvector[i->first] = get(i->first) + (i->second);
}
return *this;
}
SparseVector& SparseVector::operator-=(const SparseVector& rhs)
{
@ -162,12 +179,18 @@ bool operator==(SparseVector const& item1, SparseVector const& item2)
return item1.m_fvector==item2.m_fvector;
}
std::size_t hash_value(SparseVector const& item)
{
boost::hash<SparseVector::fvector_t> hasher;
return hasher(item.m_fvector);
size_t seed = 0;
for (SparseVector::fvector_t::const_iterator i = item.m_fvector.begin(); i != item.m_fvector.end(); ++i) {
seed = util::MurmurHashNative(&(i->first), sizeof(i->first), seed);
seed = util::MurmurHashNative(&(i->second), sizeof(i->second), seed);
}
return seed;
}
FeatureStats::FeatureStats()
: m_available_size(kAvailableSize), m_entries(0),
m_array(new FeatureStatsType[m_available_size]) {}
@ -181,8 +204,7 @@ FeatureStats::FeatureStats(const size_t size)
FeatureStats::~FeatureStats()
{
delete [] m_array;
m_array = NULL;
delete [] m_array;
}
void FeatureStats::Copy(const FeatureStats &stats)

7
mert/FeatureStats.h
View File

@ -14,6 +14,11 @@
#include <map>
#include <string>
#include <vector>
#include <boost/unordered_map.hpp>
#include <util/string_piece.hh>
#include "Types.h"
namespace MosesTuning
@ -31,6 +36,7 @@ public:
FeatureStatsType get(const std::string& name) const;
FeatureStatsType get(std::size_t id) const;
void set(const std::string& name, FeatureStatsType value);
void set(size_t id, FeatureStatsType value);
void clear();
void load(const std::string& file);
std::size_t size() const {
@ -40,6 +46,7 @@ public:
void write(std::ostream& out, const std::string& sep = " ") const;
SparseVector& operator-=(const SparseVector& rhs);
SparseVector& operator+=(const SparseVector& rhs);
FeatureStatsType inner_product(const SparseVector& rhs) const;
// Added by cherryc

432
mert/ForestRescore.cpp Normal file
View File

@ -0,0 +1,432 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <cmath>
#include <limits>
#include <list>
#include <boost/unordered_set.hpp>
#include "util/file_piece.hh"
#include "util/tokenize_piece.hh"
#include "BleuScorer.h"
#include "ForestRescore.h"
using namespace std;
namespace MosesTuning {
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec) {
out << "[";
for (size_t i = 0; i < wordVec.size(); ++i) {
out << wordVec[i]->first;
if (i+1< wordVec.size()) out << " ";
}
out << "]";
return out;
}
void ReferenceSet::Load(const vector<string>& files, Vocab& vocab) {
for (size_t i = 0; i < files.size(); ++i) {
util::FilePiece fh(files[i].c_str());
size_t sentenceId = 0;
while(true) {
StringPiece line;
try {
line = fh.ReadLine();
} catch (util::EndOfFileException &e) {
break;
}
AddLine(sentenceId, line, vocab);
++sentenceId;
}
}
}
void ReferenceSet::AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab) {
//cerr << line << endl;
NgramCounter ngramCounts;
list<WordVec> openNgrams;
size_t length = 0;
//tokenize & count
for (util::TokenIter<util::SingleCharacter, true> j(line, util::SingleCharacter(' ')); j; ++j) {
const Vocab::Entry* nextTok = &(vocab.FindOrAdd(*j));
++length;
openNgrams.push_front(WordVec());
for (list<WordVec>::iterator k = openNgrams.begin(); k != openNgrams.end(); ++k) {
k->push_back(nextTok);
++ngramCounts[*k];
}
if (openNgrams.size() >= kBleuNgramOrder) openNgrams.pop_back();
}
//merge into overall ngram map
for (NgramCounter::const_iterator ni = ngramCounts.begin();
ni != ngramCounts.end(); ++ni) {
size_t count = ni->second;
//cerr << *ni << " " << count << endl;
if (ngramCounts_.size() <= sentenceId) ngramCounts_.resize(sentenceId+1);
NgramMap::iterator totalsIter = ngramCounts_[sentenceId].find(ni->first);
if (totalsIter == ngramCounts_[sentenceId].end()) {
ngramCounts_[sentenceId][ni->first] = pair<size_t,size_t>(count,count);
} else {
ngramCounts_[sentenceId][ni->first].first = max(count, ngramCounts_[sentenceId][ni->first].first); //clip
ngramCounts_[sentenceId][ni->first].second += count; //no clip
}
}
//length
if (lengths_.size() <= sentenceId) lengths_.resize(sentenceId+1);
//TODO - length strategy - this is MIN
if (!lengths_[sentenceId]) {
lengths_[sentenceId] = length;
} else {
lengths_[sentenceId] = min(length,lengths_[sentenceId]);
}
//cerr << endl;
}
size_t ReferenceSet::NgramMatches(size_t sentenceId, const WordVec& ngram, bool clip) const {
const NgramMap& ngramCounts = ngramCounts_.at(sentenceId);
NgramMap::const_iterator ngi = ngramCounts.find(ngram);
if (ngi == ngramCounts.end()) return 0;
return clip ? ngi->second.first : ngi->second.second;
}
VertexState::VertexState(): bleuStats(kBleuNgramOrder), targetLength(0) {}
void HgBleuScorer::UpdateMatches(const NgramCounter& counts, vector<FeatureStatsType>& bleuStats ) const {
for (NgramCounter::const_iterator ngi = counts.begin(); ngi != counts.end(); ++ngi) {
//cerr << "Checking: " << *ngi << " matches " << references_.NgramMatches(sentenceId_,*ngi,false) << endl;
size_t order = ngi->first.size();
size_t count = ngi->second;
bleuStats[(order-1)*2 + 1] += count;
bleuStats[(order-1) * 2] += min(count, references_.NgramMatches(sentenceId_,ngi->first,false));
}
}
size_t HgBleuScorer::GetTargetLength(const Edge& edge) const {
size_t targetLength = 0;
for (size_t i = 0; i < edge.Words().size(); ++i) {
const Vocab::Entry* word = edge.Words()[i];
if (word) ++targetLength;
}
for (size_t i = 0; i < edge.Children().size(); ++i) {
const VertexState& state = vertexStates_[edge.Children()[i]];
targetLength += state.targetLength;
}
return targetLength;
}
FeatureStatsType HgBleuScorer::Score(const Edge& edge, const Vertex& head, vector<FeatureStatsType>& bleuStats) {
NgramCounter ngramCounts;
size_t childId = 0;
size_t wordId = 0;
size_t contextId = 0; //position within left or right context
const VertexState* vertexState = NULL;
bool inLeftContext = false;
bool inRightContext = false;
list<WordVec> openNgrams;
const Vocab::Entry* currentWord = NULL;
while (wordId < edge.Words().size()) {
currentWord = edge.Words()[wordId];
if (currentWord != NULL) {
++wordId;
} else {
if (!inLeftContext && !inRightContext) {
//entering a vertex
assert(!vertexState);
vertexState = &(vertexStates_[edge.Children()[childId]]);
++childId;
if (vertexState->leftContext.size()) {
inLeftContext = true;
contextId = 0;
currentWord = vertexState->leftContext[contextId];
} else {
//empty context
vertexState = NULL;
++wordId;
continue;
}
} else {
//already in a vertex
++contextId;
if (inLeftContext && contextId < vertexState->leftContext.size()) {
//still in left context
currentWord = vertexState->leftContext[contextId];
} else if (inLeftContext) {
//at end of left context
if (vertexState->leftContext.size() == kBleuNgramOrder-1) {
//full size context, jump to right state
openNgrams.clear();
inLeftContext = false;
inRightContext = true;
contextId = 0;
currentWord = vertexState->rightContext[contextId];
} else {
//short context, just ignore right context
inLeftContext = false;
vertexState = NULL;
++wordId;
continue;
}
} else {
//in right context
if (contextId < vertexState->rightContext.size()) {
currentWord = vertexState->rightContext[contextId];
} else {
//leaving vertex
inRightContext = false;
vertexState = NULL;
++wordId;
continue;
}
}
}
}
assert(currentWord);
if (graph_.IsBoundary(currentWord)) continue;
openNgrams.push_front(WordVec());
openNgrams.front().reserve(kBleuNgramOrder);
for (list<WordVec>::iterator k = openNgrams.begin(); k != openNgrams.end(); ++k) {
k->push_back(currentWord);
//Only insert ngrams that cross boundaries
if (!vertexState || (inLeftContext && k->size() > contextId+1)) ++ngramCounts[*k];
}
if (openNgrams.size() >= kBleuNgramOrder) openNgrams.pop_back();
}
//Collect matches
//This edge
//cerr << "edge ngrams" << endl;
UpdateMatches(ngramCounts, bleuStats);
//Child vertexes
for (size_t i = 0; i < edge.Children().size(); ++i) {
//cerr << "vertex ngrams " << edge.Children()[i] << endl;
for (size_t j = 0; j < bleuStats.size(); ++j) {
bleuStats[j] += vertexStates_[edge.Children()[i]].bleuStats[j];
}
}
FeatureStatsType sourceLength = head.SourceCovered();
size_t referenceLength = references_.Length(sentenceId_);
FeatureStatsType effectiveReferenceLength =
sourceLength / totalSourceLength_ * referenceLength;
bleuStats[bleuStats.size()-1] = effectiveReferenceLength;
//backgroundBleu_[backgroundBleu_.size()-1] =
// backgroundRefLength_ * sourceLength / totalSourceLength_;
FeatureStatsType bleu = sentenceLevelBackgroundBleu(bleuStats, backgroundBleu_);
return bleu;
}
void HgBleuScorer::UpdateState(const Edge& winnerEdge, size_t vertexId, const vector<FeatureStatsType>& bleuStats) {
//TODO: Maybe more efficient to absorb into the Score() method
VertexState& vertexState = vertexStates_[vertexId];
//cerr << "Updating state for " << vertexId << endl;
//leftContext
int wi = 0;
const VertexState* childState = NULL;
int contexti = 0; //index within child context
int childi = 0;
while (vertexState.leftContext.size() < (kBleuNgramOrder-1)) {
if ((size_t)wi >= winnerEdge.Words().size()) break;
const Vocab::Entry* word = winnerEdge.Words()[wi];
if (word != NULL) {
vertexState.leftContext.push_back(word);
++wi;
} else {
if (childState == NULL) {
//start of child state
childState = &(vertexStates_[winnerEdge.Children()[childi++]]);
contexti = 0;
}
if ((size_t)contexti < childState->leftContext.size()) {
vertexState.leftContext.push_back(childState->leftContext[contexti++]);
} else {
//end of child context
childState = NULL;
++wi;
}
}
}
//rightContext
wi = winnerEdge.Words().size() - 1;
childState = NULL;
childi = winnerEdge.Children().size() - 1;
while (vertexState.rightContext.size() < (kBleuNgramOrder-1)) {
if (wi < 0) break;
const Vocab::Entry* word = winnerEdge.Words()[wi];
if (word != NULL) {
vertexState.rightContext.push_back(word);
--wi;
} else {
if (childState == NULL) {
//start (ie rhs) of child state
childState = &(vertexStates_[winnerEdge.Children()[childi--]]);
contexti = childState->rightContext.size()-1;
}
if (contexti >= 0) {
vertexState.rightContext.push_back(childState->rightContext[contexti--]);
} else {
//end (ie lhs) of child context
childState = NULL;
--wi;
}
}
}
reverse(vertexState.rightContext.begin(), vertexState.rightContext.end());
//length + counts
vertexState.targetLength = GetTargetLength(winnerEdge);
vertexState.bleuStats = bleuStats;
}
typedef pair<const Edge*,FeatureStatsType> BackPointer;
/**
* Recurse through back pointers
**/
static void GetBestHypothesis(size_t vertexId, const Graph& graph, const vector<BackPointer>& bps,
HgHypothesis* bestHypo) {
//cerr << "Expanding " << vertexId << endl;
//UTIL_THROW_IF(bps[vertexId].second == kMinScore+1, HypergraphException, "Landed at vertex " << vertexId << " which is a dead end");
if (!bps[vertexId].first) return;
const Edge* prevEdge = bps[vertexId].first;
bestHypo->featureVector += *(prevEdge->Features().get());
size_t childId = 0;
for (size_t i = 0; i < prevEdge->Words().size(); ++i) {
if (prevEdge->Words()[i] != NULL) {
bestHypo->text.push_back(prevEdge->Words()[i]);
} else {
size_t childVertexId = prevEdge->Children()[childId++];
HgHypothesis childHypo;
GetBestHypothesis(childVertexId,graph,bps,&childHypo);
bestHypo->text.insert(bestHypo->text.end(), childHypo.text.begin(), childHypo.text.end());
bestHypo->featureVector += childHypo.featureVector;
}
}
}
void Viterbi(const Graph& graph, const SparseVector& weights, float bleuWeight, const ReferenceSet& references , size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu, HgHypothesis* bestHypo)
{
BackPointer init(NULL,kMinScore);
vector<BackPointer> backPointers(graph.VertexSize(),init);
HgBleuScorer bleuScorer(references, graph, sentenceId, backgroundBleu);
vector<FeatureStatsType> winnerStats(kBleuNgramOrder*2+1);
for (size_t vi = 0; vi < graph.VertexSize(); ++vi) {
//cerr << "vertex id " << vi << endl;
FeatureStatsType winnerScore = kMinScore;
const Vertex& vertex = graph.GetVertex(vi);
const vector<const Edge*>& incoming = vertex.GetIncoming();
if (!incoming.size()) {
//UTIL_THROW(HypergraphException, "Vertex " << vi << " has no incoming edges");
//If no incoming edges, vertex is a dead end
backPointers[vi].first = NULL;
backPointers[vi].second = kMinScore/2;
} else {
//cerr << "\nVertex: " << vi << endl;
for (size_t ei = 0; ei < incoming.size(); ++ei) {
//cerr << "edge id " << ei << endl;
FeatureStatsType incomingScore = incoming[ei]->GetScore(weights);
for (size_t i = 0; i < incoming[ei]->Children().size(); ++i) {
size_t childId = incoming[ei]->Children()[i];
UTIL_THROW_IF(backPointers[childId].second == kMinScore,
HypergraphException, "Graph was not topologically sorted. curr=" << vi << " prev=" << childId);
incomingScore += backPointers[childId].second;
}
vector<FeatureStatsType> bleuStats(kBleuNgramOrder*2+1);
// cerr << "Score: " << incomingScore << " Bleu: ";
// if (incomingScore > nonbleuscore) {nonbleuscore = incomingScore; nonbleuid = ei;}
FeatureStatsType totalScore = incomingScore;
if (bleuWeight) {
FeatureStatsType bleuScore = bleuScorer.Score(*(incoming[ei]), vertex, bleuStats);
if (isnan(bleuScore)) {
cerr << "WARN: bleu score undefined" << endl;
cerr << "\tVertex id : " << vi << endl;
cerr << "\tBleu stats : ";
for (size_t i = 0; i < bleuStats.size(); ++i) {
cerr << bleuStats[i] << ",";
}
cerr << endl;
bleuScore = 0;
}
//UTIL_THROW_IF(isnan(bleuScore), util::Exception, "Bleu score undefined, smoothing problem?");
totalScore += bleuWeight * bleuScore;
// cerr << bleuScore << " Total: " << incomingScore << endl << endl;
//cerr << "is " << incomingScore << " bs " << bleuScore << endl;
}
if (totalScore >= winnerScore) {
//We only store the feature score (not the bleu score) with the vertex,
//since the bleu score is always cumulative, ie from counts for the whole span.
winnerScore = totalScore;
backPointers[vi].first = incoming[ei];
backPointers[vi].second = incomingScore;
winnerStats = bleuStats;
}
}
//update with winner
//if (bleuWeight) {
//TODO: Not sure if we need this when computing max-model solution
bleuScorer.UpdateState(*(backPointers[vi].first), vi, winnerStats);
}
}
//expand back pointers
GetBestHypothesis(graph.VertexSize()-1, graph, backPointers, bestHypo);
//bleu stats and fv
//Need the actual (clipped) stats
//TODO: This repeats code in bleu scorer - factor out
bestHypo->bleuStats.resize(kBleuNgramOrder*2+1);
NgramCounter counts;
list<WordVec> openNgrams;
for (size_t i = 0; i < bestHypo->text.size(); ++i) {
const Vocab::Entry* entry = bestHypo->text[i];
if (graph.IsBoundary(entry)) continue;
openNgrams.push_front(WordVec());
for (list<WordVec>::iterator k = openNgrams.begin(); k != openNgrams.end(); ++k) {
k->push_back(entry);
++counts[*k];
}
if (openNgrams.size() >= kBleuNgramOrder) openNgrams.pop_back();
}
for (NgramCounter::const_iterator ngi = counts.begin(); ngi != counts.end(); ++ngi) {
size_t order = ngi->first.size();
size_t count = ngi->second;
bestHypo->bleuStats[(order-1)*2 + 1] += count;
bestHypo->bleuStats[(order-1) * 2] += min(count, references.NgramMatches(sentenceId,ngi->first,true));
}
bestHypo->bleuStats[kBleuNgramOrder*2] = references.Length(sentenceId);
}
};

120
mert/ForestRescore.h Normal file
View File

@ -0,0 +1,120 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef MERT_FOREST_RESCORE_H
#define MERT_FOREST_RESCORE_H
#include <valarray>
#include <vector>
#include <boost/unordered_set.hpp>
#include "BleuScorer.h"
#include "Hypergraph.h"
namespace MosesTuning {
std::ostream& operator<<(std::ostream& out, const WordVec& wordVec);
struct NgramHash : public std::unary_function<const WordVec&, std::size_t> {
std::size_t operator()(const WordVec& ngram) const {
return util::MurmurHashNative(&(ngram[0]), ngram.size() * sizeof(WordVec::value_type));
}
};
struct NgramEquals : public std::binary_function<const WordVec&, const WordVec&, bool> {
bool operator()(const WordVec& first, const WordVec& second) const {
if (first.size() != second.size()) return false;
return memcmp(&(first[0]), &(second[0]), first.size() * sizeof(WordVec::value_type)) == 0;
}
};
typedef boost::unordered_map<WordVec, size_t, NgramHash, NgramEquals> NgramCounter;
class ReferenceSet {
public:
void AddLine(size_t sentenceId, const StringPiece& line, Vocab& vocab);
void Load(const std::vector<std::string>& files, Vocab& vocab);
size_t NgramMatches(size_t sentenceId, const WordVec&, bool clip) const;
size_t Length(size_t sentenceId) const {return lengths_[sentenceId];}
private:
//ngrams to (clipped,unclipped) counts
typedef boost::unordered_map<WordVec, std::pair<std::size_t,std::size_t>, NgramHash,NgramEquals> NgramMap;
std::vector<NgramMap> ngramCounts_;
std::vector<size_t> lengths_;
};
struct VertexState {
VertexState();
std::vector<FeatureStatsType> bleuStats;
WordVec leftContext;
WordVec rightContext;
size_t targetLength;
};
/**
* Used to score an rule (ie edge) when we are applying it.
**/
class HgBleuScorer {
public:
HgBleuScorer(const ReferenceSet& references, const Graph& graph, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu):
references_(references), sentenceId_(sentenceId), graph_(graph), backgroundBleu_(backgroundBleu),
backgroundRefLength_(backgroundBleu[kBleuNgramOrder*2]) {
vertexStates_.resize(graph.VertexSize());
totalSourceLength_ = graph.GetVertex(graph.VertexSize()-1).SourceCovered();
}
FeatureStatsType Score(const Edge& edge, const Vertex& head, std::vector<FeatureStatsType>& bleuStats) ;
void UpdateState(const Edge& winnerEdge, size_t vertexId, const std::vector<FeatureStatsType>& bleuStats);
private:
const ReferenceSet& references_;
std::vector<VertexState> vertexStates_;
size_t sentenceId_;
size_t totalSourceLength_;
const Graph& graph_;
std::vector<FeatureStatsType> backgroundBleu_;
FeatureStatsType backgroundRefLength_;
void UpdateMatches(const NgramCounter& counter, std::vector<FeatureStatsType>& bleuStats) const;
size_t GetTargetLength(const Edge& edge) const;
};
struct HgHypothesis {
SparseVector featureVector;
WordVec text;
std::vector<FeatureStatsType> bleuStats;
};
void Viterbi(const Graph& graph, const SparseVector& weights, float bleuWeight, const ReferenceSet& references, size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu, HgHypothesis* bestHypo);
};
#endif

246
mert/ForestRescoreTest.cpp Normal file
View File

@ -0,0 +1,246 @@
#include <iostream>
#include "ForestRescore.h"
#define BOOST_TEST_MODULE MertForestRescore
#include <boost/test/unit_test.hpp>
using namespace std;
using namespace MosesTuning;
BOOST_AUTO_TEST_CASE(viterbi_simple_lattice)
{
Vocab vocab;
WordVec words;
string wordStrings[] =
{"<s>", "</s>", "a", "b", "c", "d", "e", "f", "g"};
for (size_t i = 0; i < 9; ++i) {
words.push_back(&(vocab.FindOrAdd((wordStrings[i]))));
}
const string f1 = "foo";
const string f2 = "bar";
Graph graph(vocab);
graph.SetCounts(5,5);
Edge* e0 = graph.NewEdge();
e0->AddWord(words[0]);
e0->AddFeature(f1, 2.0);
Vertex* v0 = graph.NewVertex();
v0->AddEdge(e0);
Edge* e1 = graph.NewEdge();
e1->AddWord(NULL);
e1->AddChild(0);
e1->AddWord(words[2]);
e1->AddWord(words[3]);
e1->AddFeature(f1, 1.0);
e1->AddFeature(f2, 3.0);
Vertex* v1 = graph.NewVertex();
v1->AddEdge(e1);
Edge* e2 = graph.NewEdge();
e2->AddWord(NULL);
e2->AddChild(1);
e2->AddWord(words[4]);
e2->AddWord(words[5]);
e2->AddFeature(f2, 2.5);
Vertex* v2 = graph.NewVertex();
v2->AddEdge(e2);
Edge* e3 = graph.NewEdge();
e3->AddWord(NULL);
e3->AddChild(2);
e3->AddWord(words[6]);
e3->AddWord(words[7]);
e3->AddWord(words[8]);
e3->AddFeature(f1, -1);
Vertex* v3 = graph.NewVertex();
v3->AddEdge(e3);
Edge* e4 = graph.NewEdge();
e4->AddWord(NULL);
e4->AddChild(3);
e4->AddWord(words[1]);
e3->AddFeature(f2, 0.5);
Vertex* v4 = graph.NewVertex();
v4->AddEdge(e4);
ReferenceSet references;
references.AddLine(0, "a b c k e f o", vocab);
HgHypothesis modelHypo;
vector<FeatureStatsType> bg(kBleuNgramOrder*2+1);
SparseVector weights;
weights.set(f1,2);
weights.set(f2,1);
Viterbi(graph, weights, 0, references, 0, bg, &modelHypo);
BOOST_CHECK_CLOSE(2.0,modelHypo.featureVector.get(f1), 0.0001);
BOOST_CHECK_CLOSE(6.0,modelHypo.featureVector.get(f2), 0.0001);
BOOST_CHECK_EQUAL(words[0]->first, modelHypo.text[0]->first);
BOOST_CHECK_EQUAL(words[2]->first, modelHypo.text[1]->first);
BOOST_CHECK_EQUAL(words[3]->first, modelHypo.text[2]->first);
BOOST_CHECK_EQUAL(words[4]->first, modelHypo.text[3]->first);
BOOST_CHECK_EQUAL(words[5]->first, modelHypo.text[4]->first);
BOOST_CHECK_EQUAL(words[6]->first, modelHypo.text[5]->first);
BOOST_CHECK_EQUAL(words[7]->first, modelHypo.text[6]->first);
BOOST_CHECK_EQUAL(words[8]->first, modelHypo.text[7]->first);
BOOST_CHECK_EQUAL(words[1]->first, modelHypo.text[8]->first);
}
BOOST_AUTO_TEST_CASE(viterbi_3branch_lattice)
{
Vocab vocab;
WordVec words;
string wordStrings[] =
{"<s>", "</s>", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"};
for (size_t i = 0; i < 13; ++i) {
words.push_back(&(vocab.FindOrAdd((wordStrings[i]))));
}
const string f1 = "foo";
const string f2 = "bar";
Graph graph(vocab);
graph.SetCounts(5,8);
Edge* e0 = graph.NewEdge();
e0->AddWord(words[0]);
Vertex* v0 = graph.NewVertex();
v0->AddEdge(e0);
Edge* e1 = graph.NewEdge();
e1->AddWord(NULL);
e1->AddChild(0);
e1->AddWord(words[2]);
e1->AddWord(words[3]);
e1->AddFeature(f1,1);
e1->AddFeature(f2,1);
Edge* e5 = graph.NewEdge();
e5->AddWord(NULL);
e5->AddChild(0);
e5->AddWord(words[9]);
e5->AddWord(words[10]);
e5->AddFeature(f1,2);
e5->AddFeature(f2,-2);
Vertex* v1 = graph.NewVertex();
v1->AddEdge(e1);
v1->AddEdge(e5);
v1->SetSourceCovered(1);
Edge* e2 = graph.NewEdge();
e2->AddWord(NULL);
e2->AddChild(1);
e2->AddWord(words[4]);
e2->AddWord(words[5]);
e2->AddFeature(f2,3);
Vertex* v2 = graph.NewVertex();
v2->AddEdge(e2);
v2->SetSourceCovered(3);
Edge* e3 = graph.NewEdge();
e3->AddWord(NULL);
e3->AddChild(2);
e3->AddWord(words[6]);
e3->AddWord(words[7]);
e3->AddWord(words[8]);
e3->AddFeature(f1,1);
Edge* e6 = graph.NewEdge();
e6->AddWord(NULL);
e6->AddChild(2);
e6->AddWord(words[9]);
e6->AddWord(words[12]);
e6->AddFeature(f2,1);
Edge* e7 = graph.NewEdge();
e7->AddWord(NULL);
e7->AddChild(1);
e7->AddWord(words[11]);
e7->AddWord(words[12]);
e7->AddFeature(f1,2);
e7->AddFeature(f2,3);
Vertex* v3 = graph.NewVertex();
v3->AddEdge(e3);
v3->AddEdge(e6);
v3->AddEdge(e7);
v3->SetSourceCovered(5);
Edge* e4 = graph.NewEdge();
e4->AddWord(NULL);
e4->AddChild(3);
e4->AddWord(words[1]);
Vertex* v4 = graph.NewVertex();
v4->AddEdge(e4);
v4->SetSourceCovered(6);
/*Paths || foo || bar || s(2,1)
ab cd hk || 1 || 5 || 7
hi cd hk || 2 || 2 || 6
ab jk || 3 || 4 || 10
hi jk || 4 || 1 || 9
ab cd efg || 2 || 4 || 8
hi cd efg || 3 || 1 || 7
*/
ReferenceSet references;
references.AddLine(0, "a b c d h k", vocab);
HgHypothesis modelHypo;
vector<FeatureStatsType> bg(kBleuNgramOrder*2+1, 0.1);
SparseVector weights;
weights.set(f1,2);
weights.set(f2,1);
Viterbi(graph, weights, 0, references, 0, bg, &modelHypo);
BOOST_CHECK_CLOSE(3.0,modelHypo.featureVector.get(f1), 0.0001);
BOOST_CHECK_CLOSE(4.0,modelHypo.featureVector.get(f2), 0.0001);
BOOST_CHECK_EQUAL(6, modelHypo.text.size());
//expect ab jk
BOOST_CHECK_EQUAL(words[0]->first, modelHypo.text[0]->first);
BOOST_CHECK_EQUAL(words[2]->first, modelHypo.text[1]->first);
BOOST_CHECK_EQUAL(words[3]->first, modelHypo.text[2]->first);
BOOST_CHECK_EQUAL(words[11]->first, modelHypo.text[3]->first);
BOOST_CHECK_EQUAL(words[12]->first, modelHypo.text[4]->first);
BOOST_CHECK_EQUAL(words[1]->first, modelHypo.text[5]->first);
HgHypothesis hopeHypo;
Viterbi(graph, weights, 1, references, 0, bg, &hopeHypo);
//expect abcdhk
BOOST_CHECK_EQUAL(8, hopeHypo.text.size());
BOOST_CHECK_EQUAL(words[0]->first, hopeHypo.text[0]->first);
BOOST_CHECK_EQUAL(words[2]->first, hopeHypo.text[1]->first);
BOOST_CHECK_EQUAL(words[3]->first, hopeHypo.text[2]->first);
BOOST_CHECK_EQUAL(words[4]->first, hopeHypo.text[3]->first);
BOOST_CHECK_EQUAL(words[5]->first, hopeHypo.text[4]->first);
BOOST_CHECK_EQUAL(words[9]->first, hopeHypo.text[5]->first);
BOOST_CHECK_EQUAL(words[12]->first, hopeHypo.text[6]->first);
BOOST_CHECK_EQUAL(words[1]->first, hopeHypo.text[7]->first);
BOOST_CHECK_EQUAL(kBleuNgramOrder*2+1, hopeHypo.bleuStats.size());
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[0]);
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[1]);
BOOST_CHECK_EQUAL(5, hopeHypo.bleuStats[2]);
BOOST_CHECK_EQUAL(5, hopeHypo.bleuStats[3]);
BOOST_CHECK_EQUAL(4, hopeHypo.bleuStats[4]);
BOOST_CHECK_EQUAL(4, hopeHypo.bleuStats[5]);
BOOST_CHECK_EQUAL(3, hopeHypo.bleuStats[6]);
BOOST_CHECK_EQUAL(3, hopeHypo.bleuStats[7]);
BOOST_CHECK_EQUAL(6, hopeHypo.bleuStats[8]);
}

330
mert/HopeFearDecoder.cpp Normal file
View File

@ -0,0 +1,330 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <cmath>
#include <iterator>
#include <boost/filesystem.hpp>
#include <boost/lexical_cast.hpp>
#include "util/exception.hh"
#include "util/file_piece.hh"
#include "BleuScorer.h"
#include "HopeFearDecoder.h"
using namespace std;
namespace fs = boost::filesystem;
namespace MosesTuning {
static const ValType BLEU_RATIO = 5;
ValType HopeFearDecoder::Evaluate(const AvgWeightVector& wv) {
vector<ValType> stats(kBleuNgramOrder*2+1,0);
for(reset(); !finished(); next()) {
vector<ValType> sent;
MaxModel(wv,&sent);
for(size_t i=0; i<sent.size(); i++) {
stats[i]+=sent[i];
}
}
return unsmoothedBleu(stats);
}
NbestHopeFearDecoder::NbestHopeFearDecoder(
const vector<string>& featureFiles,
const vector<string>& scoreFiles,
bool streaming,
bool no_shuffle,
bool safe_hope
) : safe_hope_(safe_hope) {
if (streaming) {
train_.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
} else {
train_.reset(new RandomAccessHypPackEnumerator(featureFiles, scoreFiles, no_shuffle));
}
}
void NbestHopeFearDecoder::next() {
train_->next();
}
bool NbestHopeFearDecoder::finished() {
return train_->finished();
}
void NbestHopeFearDecoder::reset() {
train_->reset();
}
void NbestHopeFearDecoder::HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) {
// Hope / fear decode
ValType hope_scale = 1.0;
size_t hope_index=0, fear_index=0, model_index=0;
ValType hope_score=0, fear_score=0, model_score=0;
for(size_t safe_loop=0; safe_loop<2; safe_loop++) {
ValType hope_bleu, hope_model;
for(size_t i=0; i< train_->cur_size(); i++) {
const MiraFeatureVector& vec=train_->featuresAt(i);
ValType score = wv.score(vec);
ValType bleu = sentenceLevelBackgroundBleu(train_->scoresAt(i),backgroundBleu);
// Hope
if(i==0 || (hope_scale*score + bleu) > hope_score) {
hope_score = hope_scale*score + bleu;
hope_index = i;
hope_bleu = bleu;
hope_model = score;
}
// Fear
if(i==0 || (score - bleu) > fear_score) {
fear_score = score - bleu;
fear_index = i;
}
// Model
if(i==0 || score > model_score) {
model_score = score;
model_index = i;
}
}
// Outer loop rescales the contribution of model score to 'hope' in antagonistic cases
// where model score is having far more influence than BLEU
hope_bleu *= BLEU_RATIO; // We only care about cases where model has MUCH more influence than BLEU
if(safe_hope_ && safe_loop==0 && abs(hope_model)>1e-8 && abs(hope_bleu)/abs(hope_model)<hope_scale)
hope_scale = abs(hope_bleu) / abs(hope_model);
else break;
}
hopeFear->modelFeatures = train_->featuresAt(model_index);
hopeFear->hopeFeatures = train_->featuresAt(hope_index);
hopeFear->fearFeatures = train_->featuresAt(fear_index);
hopeFear->hopeStats = train_->scoresAt(hope_index);
hopeFear->hopeBleu = sentenceLevelBackgroundBleu(hopeFear->hopeStats, backgroundBleu);
const vector<float>& fear_stats = train_->scoresAt(fear_index);
hopeFear->fearBleu = sentenceLevelBackgroundBleu(fear_stats, backgroundBleu);
hopeFear->modelStats = train_->scoresAt(model_index);
hopeFear->hopeFearEqual = (hope_index == fear_index);
}
void NbestHopeFearDecoder::MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats) {
// Find max model
size_t max_index=0;
ValType max_score=0;
for(size_t i=0; i<train_->cur_size(); i++) {
MiraFeatureVector vec(train_->featuresAt(i));
ValType score = wv.score(vec);
if(i==0 || score > max_score) {
max_index = i;
max_score = score;
}
}
*stats = train_->scoresAt(max_index);
}
HypergraphHopeFearDecoder::HypergraphHopeFearDecoder
(
const string& hypergraphDir,
const vector<string>& referenceFiles,
size_t num_dense,
bool streaming,
bool no_shuffle,
bool safe_hope,
size_t hg_pruning,
const MiraWeightVector& wv
) :
num_dense_(num_dense) {
UTIL_THROW_IF(streaming, util::Exception, "Streaming not currently supported for hypergraphs");
UTIL_THROW_IF(!fs::exists(hypergraphDir), HypergraphException, "Directory '" << hypergraphDir << "' does not exist");
UTIL_THROW_IF(!referenceFiles.size(), util::Exception, "No reference files supplied");
references_.Load(referenceFiles, vocab_);
SparseVector weights;
wv.ToSparse(&weights);
static const string kWeights = "weights";
fs::directory_iterator dend;
size_t fileCount = 0;
cerr << "Reading hypergraphs" << endl;
for (fs::directory_iterator di(hypergraphDir); di != dend; ++di) {
if (di->path().filename() == kWeights) continue;
Graph graph(vocab_);
size_t id = boost::lexical_cast<size_t>(di->path().stem().string());
util::scoped_fd fd(util::OpenReadOrThrow(di->path().string().c_str()));
//util::FilePiece file(di->path().string().c_str());
util::FilePiece file(fd.release());
ReadGraph(file,graph);
//cerr << "ref length " << references_.Length(id) << endl;
size_t edgeCount = hg_pruning * references_.Length(id);
boost::shared_ptr<Graph> prunedGraph;
prunedGraph.reset(new Graph(vocab_));
graph.Prune(prunedGraph.get(), weights, edgeCount);
graphs_[id] = prunedGraph;
//cerr << "Pruning to v=" << graphs_[id]->VertexSize() << " e=" << graphs_[id]->EdgeSize() << endl;
++fileCount;
if (fileCount % 10 == 0) cerr << ".";
if (fileCount % 400 == 0) cerr << " [count=" << fileCount << "]\n";
}
cerr << endl << "Done" << endl;
}
void HypergraphHopeFearDecoder::reset() {
graphIter_ = graphs_.begin();
}
void HypergraphHopeFearDecoder::next() {
++graphIter_;
}
bool HypergraphHopeFearDecoder::finished() {
return graphIter_ == graphs_.end();
}
void HypergraphHopeFearDecoder::HopeFear(
const vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) {
size_t sentenceId = graphIter_->first;
SparseVector weights;
wv.ToSparse(&weights);
const Graph& graph = *(graphIter_->second);
ValType hope_scale = 1.0;
HgHypothesis hopeHypo, fearHypo, modelHypo;
for(size_t safe_loop=0; safe_loop<2; safe_loop++) {
//hope decode
Viterbi(graph, weights, 1, references_, sentenceId, backgroundBleu, &hopeHypo);
//fear decode
Viterbi(graph, weights, -1, references_, sentenceId, backgroundBleu, &fearHypo);
//Model decode
Viterbi(graph, weights, 0, references_, sentenceId, backgroundBleu, &modelHypo);
// Outer loop rescales the contribution of model score to 'hope' in antagonistic cases
// where model score is having far more influence than BLEU
// hope_bleu *= BLEU_RATIO; // We only care about cases where model has MUCH more influence than BLEU
// if(safe_hope_ && safe_loop==0 && abs(hope_model)>1e-8 && abs(hope_bleu)/abs(hope_model)<hope_scale)
// hope_scale = abs(hope_bleu) / abs(hope_model);
// else break;
//TODO: Don't currently get model and bleu so commented this out for now.
break;
}
//modelFeatures, hopeFeatures and fearFeatures
hopeFear->modelFeatures = MiraFeatureVector(modelHypo.featureVector, num_dense_);
hopeFear->hopeFeatures = MiraFeatureVector(hopeHypo.featureVector, num_dense_);
hopeFear->fearFeatures = MiraFeatureVector(fearHypo.featureVector, num_dense_);
//Need to know which are to be mapped to dense features!
//Only C++11
//hopeFear->modelStats.assign(std::begin(modelHypo.bleuStats), std::end(modelHypo.bleuStats));
vector<ValType> fearStats(kBleuNgramOrder*2+1);
hopeFear->hopeStats.reserve(kBleuNgramOrder*2+1);
hopeFear->modelStats.reserve(kBleuNgramOrder*2+1);
for (size_t i = 0; i < fearStats.size(); ++i) {
hopeFear->modelStats.push_back(modelHypo.bleuStats[i]);
hopeFear->hopeStats.push_back(hopeHypo.bleuStats[i]);
fearStats[i] = fearHypo.bleuStats[i];
}
/*
cerr << "hope" << endl;;
for (size_t i = 0; i < hopeHypo.text.size(); ++i) {
cerr << hopeHypo.text[i]->first << " ";
}
cerr << endl;
for (size_t i = 0; i < fearStats.size(); ++i) {
cerr << hopeHypo.bleuStats[i] << " ";
}
cerr << endl;
cerr << "fear";
for (size_t i = 0; i < fearHypo.text.size(); ++i) {
cerr << fearHypo.text[i]->first << " ";
}
cerr << endl;
for (size_t i = 0; i < fearStats.size(); ++i) {
cerr << fearHypo.bleuStats[i] << " ";
}
cerr << endl;
cerr << "model";
for (size_t i = 0; i < modelHypo.text.size(); ++i) {
cerr << modelHypo.text[i]->first << " ";
}
cerr << endl;
for (size_t i = 0; i < fearStats.size(); ++i) {
cerr << modelHypo.bleuStats[i] << " ";
}
cerr << endl;
*/
hopeFear->hopeBleu = sentenceLevelBackgroundBleu(hopeFear->hopeStats, backgroundBleu);
hopeFear->fearBleu = sentenceLevelBackgroundBleu(fearStats, backgroundBleu);
//If fv and bleu stats are equal, then assume equal
hopeFear->hopeFearEqual = true; //(hopeFear->hopeBleu - hopeFear->fearBleu) >= 1e-8;
if (hopeFear->hopeFearEqual) {
for (size_t i = 0; i < fearStats.size(); ++i) {
if (fearStats[i] != hopeFear->hopeStats[i]) {
hopeFear->hopeFearEqual = false;
break;
}
}
}
hopeFear->hopeFearEqual = hopeFear->hopeFearEqual && (hopeFear->fearFeatures == hopeFear->hopeFeatures);
}
void HypergraphHopeFearDecoder::MaxModel(const AvgWeightVector& wv, vector<ValType>* stats) {
assert(!finished());
HgHypothesis bestHypo;
size_t sentenceId = graphIter_->first;
SparseVector weights;
wv.ToSparse(&weights);
vector<ValType> bg(kBleuNgramOrder*2+1);
Viterbi(*(graphIter_->second), weights, 0, references_, sentenceId, bg, &bestHypo);
stats->resize(bestHypo.bleuStats.size());
/*
for (size_t i = 0; i < bestHypo.text.size(); ++i) {
cerr << bestHypo.text[i]->first << " ";
}
cerr << endl;
*/
for (size_t i = 0; i < bestHypo.bleuStats.size(); ++i) {
(*stats)[i] = bestHypo.bleuStats[i];
}
}
};

151
mert/HopeFearDecoder.h Normal file
View File

@ -0,0 +1,151 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef MERT_HOPEFEARDECODER_H
#define MERT_HOPEFEARDECODER_H
#include <vector>
#include <boost/scoped_ptr.hpp>
#include <boost/shared_ptr.hpp>
#include "ForestRescore.h"
#include "Hypergraph.h"
#include "HypPackEnumerator.h"
#include "MiraFeatureVector.h"
#include "MiraWeightVector.h"
//
// Used by batch mira to get the hope, fear and model hypothesis. This wraps
// the n-best list and lattice/hypergraph implementations
//
namespace MosesTuning {
/** To be filled in by the decoder */
struct HopeFearData {
MiraFeatureVector modelFeatures;
MiraFeatureVector hopeFeatures;
MiraFeatureVector fearFeatures;
std::vector<float> modelStats;
std::vector<float> hopeStats;
ValType hopeBleu;
ValType fearBleu;
bool hopeFearEqual;
};
//Abstract base class
class HopeFearDecoder {
public:
//iterator methods
virtual void reset() = 0;
virtual void next() = 0;
virtual bool finished() = 0;
/**
* Calculate hope, fear and model hypotheses
**/
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
) = 0;
/** Max score decoding */
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats)
= 0;
/** Calculate bleu on training set */
ValType Evaluate(const AvgWeightVector& wv);
};
/** Gets hope-fear from nbest lists */
class NbestHopeFearDecoder : public virtual HopeFearDecoder {
public:
NbestHopeFearDecoder(const std::vector<std::string>& featureFiles,
const std::vector<std::string>& scoreFiles,
bool streaming,
bool no_shuffle,
bool safe_hope
);
virtual void reset();
virtual void next();
virtual bool finished();
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
);
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats);
private:
boost::scoped_ptr<HypPackEnumerator> train_;
bool safe_hope_;
};
/** Gets hope-fear from hypergraphs */
class HypergraphHopeFearDecoder : public virtual HopeFearDecoder {
public:
HypergraphHopeFearDecoder(
const std::string& hypergraphDir,
const std::vector<std::string>& referenceFiles,
size_t num_dense,
bool streaming,
bool no_shuffle,
bool safe_hope,
size_t hg_pruning,
const MiraWeightVector& wv
);
virtual void reset();
virtual void next();
virtual bool finished();
virtual void HopeFear(
const std::vector<ValType>& backgroundBleu,
const MiraWeightVector& wv,
HopeFearData* hopeFear
);
virtual void MaxModel(const AvgWeightVector& wv, std::vector<ValType>* stats);
private:
size_t num_dense_;
//maps sentence Id to graph ptr
typedef std::map<size_t, boost::shared_ptr<Graph> > GraphColl;
GraphColl graphs_;
GraphColl::const_iterator graphIter_;
ReferenceSet references_;
Vocab vocab_;
};
};
#endif

286
mert/Hypergraph.cpp Normal file
View File

@ -0,0 +1,286 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <iostream>
#include <set>
#include <boost/lexical_cast.hpp>
#include "util/double-conversion/double-conversion.h"
#include "util/string_piece.hh"
#include "util/tokenize_piece.hh"
#include "Hypergraph.h"
using namespace std;
static const string kBOS = "<s>";
static const string kEOS = "</s>";
namespace MosesTuning {
StringPiece NextLine(util::FilePiece& from) {
StringPiece line;
while ((line = from.ReadLine()).starts_with("#"));
return line;
}
Vocab::Vocab() : eos_( FindOrAdd(kEOS)), bos_(FindOrAdd(kBOS)){
}
const Vocab::Entry &Vocab::FindOrAdd(const StringPiece &str) {
#if BOOST_VERSION >= 104200
Map::const_iterator i= map_.find(str, Hash(), Equals());
#else
std::string copied_str(str.data(), str.size());
Map::const_iterator i = map_.find(copied_str.c_str());
#endif
if (i != map_.end()) return *i;
char *copied = static_cast<char*>(piece_backing_.Allocate(str.size() + 1));
memcpy(copied, str.data(), str.size());
copied[str.size()] = 0;
return *map_.insert(Entry(copied, map_.size())).first;
}
double_conversion::StringToDoubleConverter converter(double_conversion::StringToDoubleConverter::NO_FLAGS, NAN, NAN, "inf", "nan");
/**
* Reads an incoming edge. Returns edge and source words covered.
**/
static pair<Edge*,size_t> ReadEdge(util::FilePiece &from, Graph &graph) {
Edge* edge = graph.NewEdge();
StringPiece line = NextLine(from);
util::TokenIter<util::MultiCharacter> pipes(line, util::MultiCharacter(" ||| "));
//Target
for (util::TokenIter<util::SingleCharacter, true> i(*pipes, util::SingleCharacter(' ')); i; ++i) {
StringPiece got = *i;
if ('[' == *got.data() && ']' == got.data()[got.size() - 1]) {
// non-terminal
char *end_ptr;
unsigned long int child = std::strtoul(got.data() + 1, &end_ptr, 10);
UTIL_THROW_IF(end_ptr != got.data() + got.size() - 1, HypergraphException, "Bad non-terminal" << got);
UTIL_THROW_IF(child >= graph.VertexSize(), HypergraphException, "Reference to vertex " << child << " but we only have " << graph.VertexSize() << " vertices. Is the file in bottom-up format?");
edge->AddWord(NULL);
edge->AddChild(child);
} else {
const Vocab::Entry &found = graph.MutableVocab().FindOrAdd(got);
edge->AddWord(&found);
}
}
//Features
++pipes;
for (util::TokenIter<util::SingleCharacter, true> i(*pipes, util::SingleCharacter(' ')); i; ++i) {
StringPiece fv = *i;
if (!fv.size()) break;
size_t equals = fv.find_last_of("=");
UTIL_THROW_IF(equals == fv.npos, HypergraphException, "Failed to parse feature '" << fv << "'");
StringPiece name = fv.substr(0,equals);
StringPiece value = fv.substr(equals+1);
int processed;
float score = converter.StringToFloat(value.data(), value.length(), &processed);
UTIL_THROW_IF(isnan(score), HypergraphException, "Failed to parse weight '" << value << "'");
edge->AddFeature(name,score);
}
//Covered words
++pipes;
size_t sourceCovered = boost::lexical_cast<size_t>(*pipes);
return pair<Edge*,size_t>(edge,sourceCovered);
}
void Graph::Prune(Graph* pNewGraph, const SparseVector& weights, size_t minEdgeCount) const {
Graph& newGraph = *pNewGraph;
//TODO: Optimise case where no pruning required
//For debug
/*
map<const Edge*, string> edgeIds;
for (size_t i = 0; i < edges_.Size(); ++i) {
stringstream str;
for (size_t j = 0; j < edges_[i].Words().size(); ++j) {
if (edges_[i].Words()[j]) str << edges_[i].Words()[j]->first;
}
edgeIds[&(edges_[i])] = str.str();
}*/
//end For debug
map<const Edge*, FeatureStatsType> edgeBackwardScores;
map<const Edge*, size_t> edgeHeads;
vector<FeatureStatsType> vertexBackwardScores(vertices_.Size(), kMinScore);
vector<vector<const Edge*> > outgoing(vertices_.Size());
//Compute backward scores
for (size_t vi = 0; vi < vertices_.Size(); ++vi) {
//cerr << "Vertex " << vi << endl;
const Vertex& vertex = vertices_[vi];
const vector<const Edge*>& incoming = vertex.GetIncoming();
if (!incoming.size()) {
vertexBackwardScores[vi] = 0;
} else {
for (size_t ei = 0; ei < incoming.size(); ++ei) {
//cerr << "Edge " << edgeIds[incoming[ei]] << endl;
edgeHeads[incoming[ei]]= vi;
FeatureStatsType incomingScore = incoming[ei]->GetScore(weights);
for (size_t i = 0; i < incoming[ei]->Children().size(); ++i) {
// cerr << "\tChild " << incoming[ei]->Children()[i] << endl;
size_t childId = incoming[ei]->Children()[i];
UTIL_THROW_IF(vertexBackwardScores[childId] == kMinScore,
HypergraphException, "Graph was not topologically sorted. curr=" << vi << " prev=" << childId);
outgoing[childId].push_back(incoming[ei]);
incomingScore += vertexBackwardScores[childId];
}
edgeBackwardScores[incoming[ei]]= incomingScore;
// cerr << "Backward score: " << incomingScore << endl;
if (incomingScore > vertexBackwardScores[vi]) vertexBackwardScores[vi] = incomingScore;
}
}
}
//Compute forward scores
vector<FeatureStatsType> vertexForwardScores(vertices_.Size(), kMinScore);
map<const Edge*, FeatureStatsType> edgeForwardScores;
for (size_t i = 1; i <= vertices_.Size(); ++i) {
size_t vi = vertices_.Size() - i;
//cerr << "Vertex " << vi << endl;
if (!outgoing[vi].size()) {
vertexForwardScores[vi] = 0;
} else {
for (size_t ei = 0; ei < outgoing[vi].size(); ++ei) {
//cerr << "Edge " << edgeIds[outgoing[vi][ei]] << endl;
FeatureStatsType outgoingScore = 0;
//sum scores of siblings
for (size_t i = 0; i < outgoing[vi][ei]->Children().size(); ++i) {
size_t siblingId = outgoing[vi][ei]->Children()[i];
if (siblingId != vi) {
// cerr << "\tSibling " << siblingId << endl;
outgoingScore += vertexBackwardScores[siblingId];
}
}
//add score of head
outgoingScore += vertexForwardScores[edgeHeads[outgoing[vi][ei]]];
//cerr << "Forward score " << outgoingScore << endl;
edgeForwardScores[outgoing[vi][ei]] = outgoingScore;
outgoingScore += outgoing[vi][ei]->GetScore(weights);
if (outgoingScore > vertexForwardScores[vi]) vertexForwardScores[vi] = outgoingScore;
}
}
}
multimap<FeatureStatsType, const Edge*> edgeScores;
for (size_t i = 0; i < edges_.Size(); ++i) {
const Edge* edge = &(edges_[i]);
if (edgeForwardScores.find(edge) == edgeForwardScores.end()) {
//This edge has no children, so didn't get a forward score. Its forward score
//is that of its head
edgeForwardScores[edge] = vertexForwardScores[edgeHeads[edge]];
}
FeatureStatsType score = edgeForwardScores[edge] + edgeBackwardScores[edge];
edgeScores.insert(pair<FeatureStatsType, const Edge*>(score,edge));
// cerr << edgeIds[edge] << " " << score << endl;
}
multimap<FeatureStatsType, const Edge*>::const_reverse_iterator ei = edgeScores.rbegin();
size_t edgeCount = 1;
while(edgeCount < minEdgeCount && ei != edgeScores.rend()) {
++ei;
++edgeCount;
}
multimap<FeatureStatsType, const Edge*>::const_iterator lowest = edgeScores.begin();
if (ei != edgeScores.rend()) lowest = edgeScores.lower_bound(ei->first);
//cerr << "Retained edges" << endl;
set<size_t> retainedVertices;
set<const Edge*> retainedEdges;
for (; lowest != edgeScores.end(); ++lowest) {
//cerr << lowest->first << " " << edgeIds[lowest->second] << endl;
retainedEdges.insert(lowest->second);
retainedVertices.insert(edgeHeads[lowest->second]);
for (size_t i = 0; i < lowest->second->Children().size(); ++i) {
retainedVertices.insert(lowest->second->Children()[i]);
}
}
newGraph.SetCounts(retainedVertices.size(), retainedEdges.size());
//cerr << "Retained vertices" << endl;
map<size_t,size_t> oldIdToNew;
size_t vi = 0;
for (set<size_t>::const_iterator i = retainedVertices.begin(); i != retainedVertices.end(); ++i, ++vi) {
//cerr << *i << endl;
oldIdToNew[*i] = vi;
Vertex* vertex = newGraph.NewVertex();
vertex->SetSourceCovered(vertices_[*i].SourceCovered());
}
for (set<const Edge*>::const_iterator i = retainedEdges.begin(); i != retainedEdges.end(); ++i) {
Edge* newEdge = newGraph.NewEdge();
const Edge* oldEdge = *i;
for (size_t j = 0; j < oldEdge->Words().size(); ++j) {
newEdge->AddWord(oldEdge->Words()[j]);
}
for (size_t j = 0; j < oldEdge->Children().size(); ++j) {
newEdge->AddChild(oldIdToNew[oldEdge->Children()[j]]);
}
newEdge->SetFeatures(oldEdge->Features());
Vertex& newHead = newGraph.vertices_[oldIdToNew[edgeHeads[oldEdge]]];
newHead.AddEdge(newEdge);
}
}
/**
* Read from "Kenneth's hypergraph" aka cdec target_graph format (with comments)
**/
void ReadGraph(util::FilePiece &from, Graph &graph) {
//First line should contain field names
StringPiece line = from.ReadLine();
UTIL_THROW_IF(line.compare("# target ||| features ||| source-covered") != 0, HypergraphException, "Incorrect format spec on first line: '" << line << "'");
line = NextLine(from);
//Then expect numbers of vertices
util::TokenIter<util::SingleCharacter, false> i(line, util::SingleCharacter(' '));
unsigned long int vertices = boost::lexical_cast<unsigned long int>(*i);
++i;
unsigned long int edges = boost::lexical_cast<unsigned long int>(*i);
graph.SetCounts(vertices, edges);
//cerr << "vertices: " << vertices << "; edges: " << edges << endl;
for (size_t i = 0; i < vertices; ++i) {
line = NextLine(from);
unsigned long int edge_count = boost::lexical_cast<unsigned long int>(line);
Vertex* vertex = graph.NewVertex();
for (unsigned long int e = 0; e < edge_count; ++e) {
pair<Edge*,size_t> edge = ReadEdge(from, graph);
vertex->AddEdge(edge.first);
//Note: the file format attaches this to the edge, but it's really a property
//of the vertex.
if (!e) {vertex->SetSourceCovered(edge.second);}
}
}
}
};

251
mert/Hypergraph.h Normal file
View File

@ -0,0 +1,251 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2014- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef MERT_HYPERGRAPH_H
#define MERT_HYPERGRAPH_H
#include <string>
#include <boost/noncopyable.hpp>
#include <boost/scoped_array.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/functional/hash/hash.hpp>
#include <boost/unordered_map.hpp>
#include "util/exception.hh"
#include "util/file_piece.hh"
#include "util/murmur_hash.hh"
#include "util/pool.hh"
#include "util/string_piece.hh"
#include "FeatureStats.h"
namespace MosesTuning {
typedef unsigned int WordIndex;
const WordIndex kMaxWordIndex = UINT_MAX;
const FeatureStatsType kMinScore = -std::numeric_limits<FeatureStatsType>::max();
template <class T> class FixedAllocator : boost::noncopyable {
public:
FixedAllocator() : current_(NULL), end_(NULL) {}
void Init(std::size_t count) {
assert(!current_);
array_.reset(new T[count]);
current_ = array_.get();
end_ = current_ + count;
}
T &operator[](std::size_t idx) {
return array_.get()[idx];
}
const T &operator[](std::size_t idx) const {
return array_.get()[idx];
}
T *New() {
T *ret = current_++;
UTIL_THROW_IF(ret >= end_, util::Exception, "Allocating past end");
return ret;
}
std::size_t Capacity() const {
return end_ - array_.get();
}
std::size_t Size() const {
return current_ - array_.get();
}
private:
boost::scoped_array<T> array_;
T *current_, *end_;
};
class Vocab {
public:
Vocab();
typedef std::pair<const char *const, WordIndex> Entry;
const Entry &FindOrAdd(const StringPiece &str);
const Entry& Bos() const {return bos_;}
const Entry& Eos() const {return eos_;}
private:
util::Pool piece_backing_;
struct Hash : public std::unary_function<const char *, std::size_t> {
std::size_t operator()(StringPiece str) const {
return util::MurmurHashNative(str.data(), str.size());
}
};
struct Equals : public std::binary_function<const char *, const char *, bool> {
bool operator()(StringPiece first, StringPiece second) const {
return first == second;
}
};
typedef boost::unordered_map<const char *, WordIndex, Hash, Equals> Map;
Map map_;
Entry eos_;
Entry bos_;
};
typedef std::vector<const Vocab::Entry*> WordVec;
class Vertex;
//Use shared pointer to save copying when we prune
typedef boost::shared_ptr<SparseVector> FeaturePtr;
/**
* An edge has 1 head vertex, 0..n child (tail) vertices, a list of words and a feature vector.
**/
class Edge {
public:
Edge() {features_.reset(new SparseVector());}
void AddWord(const Vocab::Entry *word) {
words_.push_back(word);
}
void AddChild(size_t child) {
children_.push_back(child);
}
void AddFeature(const StringPiece& name, FeatureStatsType value) {
//TODO StringPiece interface
features_->set(name.as_string(),value);
}
const WordVec &Words() const {
return words_;
}
const FeaturePtr& Features() const {
return features_;
}
void SetFeatures(const FeaturePtr& features) {
features_ = features;
}
const std::vector<size_t>& Children() const {
return children_;
}
FeatureStatsType GetScore(const SparseVector& weights) const {
return inner_product(*(features_.get()), weights);
}
private:
// NULL for non-terminals.
std::vector<const Vocab::Entry*> words_;
std::vector<size_t> children_;
boost::shared_ptr<SparseVector> features_;
};
/*
* A vertex has 0..n incoming edges
**/
class Vertex {
public:
Vertex() : sourceCovered_(0) {}
void AddEdge(const Edge* edge) {incoming_.push_back(edge);}
void SetSourceCovered(size_t sourceCovered) {sourceCovered_ = sourceCovered;}
const std::vector<const Edge*>& GetIncoming() const {return incoming_;}
size_t SourceCovered() const {return sourceCovered_;}
private:
std::vector<const Edge*> incoming_;
size_t sourceCovered_;
};
class Graph : boost::noncopyable {
public:
Graph(Vocab& vocab) : vocab_(vocab) {}
void SetCounts(std::size_t vertices, std::size_t edges) {
vertices_.Init(vertices);
edges_.Init(edges);
}
Vocab &MutableVocab() { return vocab_; }
Edge *NewEdge() {
return edges_.New();
}
Vertex *NewVertex() {
return vertices_.New();
}
const Vertex &GetVertex(std::size_t index) const {
return vertices_[index];
}
Edge &GetEdge(std::size_t index) {
return edges_[index];
}
/* Created a pruned copy of this graph with minEdgeCount edges. Uses
the scores in the max-product semiring to rank edges, as suggested by
Colin Cherry */
void Prune(Graph* newGraph, const SparseVector& weights, size_t minEdgeCount) const;
std::size_t VertexSize() const { return vertices_.Size(); }
std::size_t EdgeSize() const { return edges_.Size(); }
bool IsBoundary(const Vocab::Entry* word) const {
return word->second == vocab_.Bos().second || word->second == vocab_.Eos().second;
}
private:
FixedAllocator<Edge> edges_;
FixedAllocator<Vertex> vertices_;
Vocab& vocab_;
};
class HypergraphException : public util::Exception {
public:
HypergraphException() {}
~HypergraphException() throw() {}
};
void ReadGraph(util::FilePiece &from, Graph &graph);
};
#endif

151
mert/HypergraphTest.cpp Normal file
View File

@ -0,0 +1,151 @@
#include <iostream>
#define BOOST_TEST_MODULE MertForestRescore
#include <boost/test/unit_test.hpp>
#include "Hypergraph.h"
using namespace std;
using namespace MosesTuning;
BOOST_AUTO_TEST_CASE(prune)
{
Vocab vocab;
WordVec words;
string wordStrings[] =
{"<s>", "</s>", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"};
for (size_t i = 0; i < 13; ++i) {
words.push_back(&(vocab.FindOrAdd((wordStrings[i]))));
}
const string f1 = "foo";
const string f2 = "bar";
Graph graph(vocab);
graph.SetCounts(5,8);
Edge* e0 = graph.NewEdge();
e0->AddWord(words[0]);
Vertex* v0 = graph.NewVertex();
v0->AddEdge(e0);
Edge* e1 = graph.NewEdge();
e1->AddWord(NULL);
e1->AddChild(0);
e1->AddWord(words[2]);
e1->AddWord(words[3]);
e1->AddFeature(f1,1);
e1->AddFeature(f2,1);
Edge* e5 = graph.NewEdge();
e5->AddWord(NULL);
e5->AddChild(0);
e5->AddWord(words[9]);
e5->AddWord(words[10]);
e5->AddFeature(f1,2);
e5->AddFeature(f2,-2);
Vertex* v1 = graph.NewVertex();
v1->AddEdge(e1);
v1->AddEdge(e5);
v1->SetSourceCovered(1);
Edge* e2 = graph.NewEdge();
e2->AddWord(NULL);
e2->AddChild(1);
e2->AddWord(words[4]);
e2->AddWord(words[5]);
e2->AddFeature(f2,3);
Vertex* v2 = graph.NewVertex();
v2->AddEdge(e2);
v2->SetSourceCovered(3);
Edge* e3 = graph.NewEdge();
e3->AddWord(NULL);
e3->AddChild(2);
e3->AddWord(words[6]);
e3->AddWord(words[7]);
e3->AddWord(words[8]);
e3->AddFeature(f1,1);
Edge* e6 = graph.NewEdge();
e6->AddWord(NULL);
e6->AddChild(2);
e6->AddWord(words[9]);
e6->AddWord(words[12]);
e6->AddFeature(f2,1);
Edge* e7 = graph.NewEdge();
e7->AddWord(NULL);
e7->AddChild(1);
e7->AddWord(words[11]);
e7->AddWord(words[12]);
e7->AddFeature(f1,2);
e7->AddFeature(f2,3);
Vertex* v3 = graph.NewVertex();
v3->AddEdge(e3);
v3->AddEdge(e6);
v3->AddEdge(e7);
v3->SetSourceCovered(5);
Edge* e4 = graph.NewEdge();
e4->AddWord(NULL);
e4->AddChild(3);
e4->AddWord(words[1]);
Vertex* v4 = graph.NewVertex();
v4->AddEdge(e4);
v4->SetSourceCovered(6);
SparseVector weights;
weights.set(f1,2);
weights.set(f2,1);
Graph pruned(vocab);
graph.Prune(&pruned, weights, 5);
BOOST_CHECK_EQUAL(5, pruned.EdgeSize());
BOOST_CHECK_EQUAL(4, pruned.VertexSize());
//edges retained should be best path (<s> ab jk </s>) and hi
BOOST_CHECK_EQUAL(1, pruned.GetVertex(0).GetIncoming().size());
BOOST_CHECK_EQUAL(2, pruned.GetVertex(1).GetIncoming().size());
BOOST_CHECK_EQUAL(1, pruned.GetVertex(2).GetIncoming().size());
BOOST_CHECK_EQUAL(1, pruned.GetVertex(3).GetIncoming().size());
const Edge* edge;
edge = pruned.GetVertex(0).GetIncoming()[0];
BOOST_CHECK_EQUAL(1, edge->Words().size());
BOOST_CHECK_EQUAL(words[0], edge->Words()[0]);
edge = pruned.GetVertex(1).GetIncoming()[0];
BOOST_CHECK_EQUAL(3, edge->Words().size());
BOOST_CHECK_EQUAL((Vocab::Entry*)NULL, edge->Words()[0]);
BOOST_CHECK_EQUAL(words[2]->first, edge->Words()[1]->first);
BOOST_CHECK_EQUAL(words[3]->first, edge->Words()[2]->first);
edge = pruned.GetVertex(1).GetIncoming()[1];
BOOST_CHECK_EQUAL(3, edge->Words().size());
BOOST_CHECK_EQUAL((Vocab::Entry*)NULL, edge->Words()[0]);
BOOST_CHECK_EQUAL(words[9]->first, edge->Words()[1]->first);
BOOST_CHECK_EQUAL(words[10]->first, edge->Words()[2]->first);
edge = pruned.GetVertex(2).GetIncoming()[0];
BOOST_CHECK_EQUAL(3, edge->Words().size());
BOOST_CHECK_EQUAL((Vocab::Entry*)NULL, edge->Words()[0]);
BOOST_CHECK_EQUAL(words[11]->first, edge->Words()[1]->first);
BOOST_CHECK_EQUAL(words[12]->first, edge->Words()[2]->first);
edge = pruned.GetVertex(3).GetIncoming()[0];
BOOST_CHECK_EQUAL(2, edge->Words().size());
BOOST_CHECK_EQUAL((Vocab::Entry*)NULL, edge->Words()[0]);
BOOST_CHECK_EQUAL(words[1]->first, edge->Words()[1]->first);
// BOOST_CHECK_EQUAL(words[0], pruned.GetVertex(0).GetIncoming()[0].Words()[0]);
}

7
mert/Jamfile
View File

@ -15,6 +15,9 @@ FeatureStats.cpp
FeatureArray.cpp
FeatureData.cpp
FeatureDataIterator.cpp
ForestRescore.cpp
HopeFearDecoder.cpp
Hypergraph.cpp
MiraFeatureVector.cpp
MiraWeightVector.cpp
HypPackEnumerator.cpp
@ -62,13 +65,15 @@ exe sentence-bleu : sentence-bleu.cpp mert_lib ;
exe pro : pro.cpp mert_lib ..//boost_program_options ;
exe kbmira : kbmira.cpp mert_lib ..//boost_program_options ;
exe kbmira : kbmira.cpp mert_lib ..//boost_program_options ..//boost_filesystem ;
alias programs : mert extractor evaluator pro kbmira sentence-bleu ;
unit-test bleu_scorer_test : BleuScorerTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test feature_data_test : FeatureDataTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test data_test : DataTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test forest_rescore_test : ForestRescoreTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test hypergraph_test : HypergraphTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test ngram_test : NgramTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test optimizer_factory_test : OptimizerFactoryTest.cpp mert_lib ..//boost_unit_test_framework ;
unit-test point_test : PointTest.cpp mert_lib ..//boost_unit_test_framework ;

40
mert/MiraFeatureVector.cpp
View File

@ -9,18 +9,17 @@ namespace MosesTuning
{
MiraFeatureVector::MiraFeatureVector(const FeatureDataItem& vec)
: m_dense(vec.dense)
{
vector<size_t> sparseFeats = vec.sparse.feats();
void MiraFeatureVector::InitSparse(const SparseVector& sparse, size_t ignoreLimit) {
vector<size_t> sparseFeats = sparse.feats();
bool bFirst = true;
size_t lastFeat = 0;
m_sparseFeats.reserve(sparseFeats.size());
m_sparseVals.reserve(sparseFeats.size());
for(size_t i=0; i<sparseFeats.size(); i++) {
if (sparseFeats[i] < ignoreLimit) continue;
size_t feat = m_dense.size() + sparseFeats[i];
m_sparseFeats.push_back(feat);
m_sparseVals.push_back(vec.sparse.get(sparseFeats[i]));
m_sparseVals.push_back(sparse.get(sparseFeats[i]));
// Check ordered property
if(bFirst) {
@ -35,6 +34,21 @@ MiraFeatureVector::MiraFeatureVector(const FeatureDataItem& vec)
}
}
MiraFeatureVector::MiraFeatureVector(const FeatureDataItem& vec)
: m_dense(vec.dense)
{
InitSparse(vec.sparse);
}
MiraFeatureVector::MiraFeatureVector(const SparseVector& sparse, size_t num_dense) {
m_dense.resize(num_dense);
//Assume that features with id [0,num_dense) are the dense features
for (size_t id = 0; id < num_dense; ++id) {
m_dense[id] = sparse.get(id);
}
InitSparse(sparse,num_dense);
}
MiraFeatureVector::MiraFeatureVector(const MiraFeatureVector& other)
: m_dense(other.m_dense),
m_sparseFeats(other.m_sparseFeats),
@ -148,6 +162,22 @@ MiraFeatureVector operator-(const MiraFeatureVector& a, const MiraFeatureVector&
return MiraFeatureVector(dense,sparseFeats,sparseVals);
}
bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b) {
ValType eps = 1e-8;
//dense features
if (a.m_dense.size() != b.m_dense.size()) return false;
for (size_t i = 0; i < a.m_dense.size(); ++i) {
if (fabs(a.m_dense[i]-b.m_dense[i]) < eps) return false;
}
if (a.m_sparseFeats.size() != b.m_sparseFeats.size()) return false;
for (size_t i = 0; i < a.m_sparseFeats.size(); ++i) {
if (a.m_sparseFeats[i] != b.m_sparseFeats[i]) return false;
if (fabs(a.m_sparseVals[i] != b.m_sparseVals[i])) return false;
}
return true;
}
ostream& operator<<(ostream& o, const MiraFeatureVector& e)
{
for(size_t i=0; i<e.size(); i++) {

8
mert/MiraFeatureVector.h
View File

@ -26,7 +26,10 @@ typedef FeatureStatsType ValType;
class MiraFeatureVector
{
public:
MiraFeatureVector() {}
MiraFeatureVector(const FeatureDataItem& vec);
//Assumes that features in sparse with id < num_dense are dense features
MiraFeatureVector(const SparseVector& sparse, size_t num_dense);
MiraFeatureVector(const MiraFeatureVector& other);
MiraFeatureVector(const std::vector<ValType>& dense,
const std::vector<std::size_t>& sparseFeats,
@ -42,7 +45,12 @@ public:
friend std::ostream& operator<<(std::ostream& o, const MiraFeatureVector& e);
friend bool operator==(const MiraFeatureVector& a,const MiraFeatureVector& b);
private:
//Ignore any sparse features with id < ignoreLimit
void InitSparse(const SparseVector& sparse, size_t ignoreLimit = 0);
std::vector<ValType> m_dense;
std::vector<std::size_t> m_sparseFeats;
std::vector<ValType> m_sparseVals;

17
mert/MiraWeightVector.cpp
View File

@ -93,6 +93,14 @@ void MiraWeightVector::update(size_t index, ValType delta)
m_lastUpdated[index] = m_numUpdates;
}
void MiraWeightVector::ToSparse(SparseVector* sparse) const {
for (size_t i = 0; i < m_weights.size(); ++i) {
if(abs(m_weights[i])>1e-8) {
sparse->set(i,m_weights[i]);
}
}
}
/**
* Make sure everyone's total is up-to-date
*/
@ -163,6 +171,15 @@ size_t AvgWeightVector::size() const
return m_wv.m_weights.size();
}
void AvgWeightVector::ToSparse(SparseVector* sparse) const {
for (size_t i = 0; i < size(); ++i) {
ValType w = weight(i);
if(abs(w)>1e-8) {
sparse->set(i,w);
}
}
}
// --Emacs trickery--
// Local Variables:
// mode:c++

8
mert/MiraWeightVector.h
View File

@ -63,6 +63,11 @@ public:
*/
AvgWeightVector avg();
/**
* Convert to sparse vector, interpreting all features as sparse.
**/
void ToSparse(SparseVector* sparse) const;
friend class AvgWeightVector;
friend std::ostream& operator<<(std::ostream& o, const MiraWeightVector& e);
@ -99,12 +104,12 @@ public:
ValType score(const MiraFeatureVector& fv) const;
ValType weight(std::size_t index) const;
std::size_t size() const;
void ToSparse(SparseVector* sparse) const;
private:
const MiraWeightVector& m_wv;
};
#endif // MERT_WEIGHT_VECTOR_H
// --Emacs trickery--
// Local Variables:
@ -113,3 +118,4 @@ private:
// End:
}
#endif // MERT_WEIGHT_VECTOR_H

187
mert/kbmira.cpp
View File

@ -39,8 +39,10 @@ de recherches du Canada
#include <boost/program_options.hpp>
#include <boost/scoped_ptr.hpp>
#include "util/exception.hh"
#include "BleuScorer.h"
#include "HypPackEnumerator.h"
#include "HopeFearDecoder.h"
#include "MiraFeatureVector.h"
#include "MiraWeightVector.h"
@ -49,38 +51,16 @@ using namespace MosesTuning;
namespace po = boost::program_options;
ValType evaluate(HypPackEnumerator* train, const AvgWeightVector& wv)
{
vector<ValType> stats(kBleuNgramOrder*2+1,0);
for(train->reset(); !train->finished(); train->next()) {
// Find max model
size_t max_index=0;
ValType max_score=0;
for(size_t i=0; i<train->cur_size(); i++) {
MiraFeatureVector vec(train->featuresAt(i));
ValType score = wv.score(vec);
if(i==0 || score > max_score) {
max_index = i;
max_score = score;
}
}
// Update stats
const vector<float>& sent = train->scoresAt(max_index);
for(size_t i=0; i<sent.size(); i++) {
stats[i]+=sent[i];
}
}
return unsmoothedBleu(stats);
}
int main(int argc, char** argv)
{
const ValType BLEU_RATIO = 5;
bool help;
string denseInitFile;
string sparseInitFile;
string type = "nbest";
vector<string> scoreFiles;
vector<string> featureFiles;
vector<string> referenceFiles; //for hg mira
string hgDir;
int seed;
string outputFile;
float c = 0.01; // Step-size cap C
@ -91,25 +71,30 @@ int main(int argc, char** argv)
bool model_bg = false; // Use model for background corpus
bool verbose = false; // Verbose updates
bool safe_hope = false; // Model score cannot have more than BLEU_RATIO times more influence than BLEU
size_t hgPruning = 50; //prune hypergraphs to have this many edges per reference word
// Command-line processing follows pro.cpp
po::options_description desc("Allowed options");
desc.add_options()
("help,h", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit")
("type,t", po::value<string>(&type), "Either nbest or hypergraph")
("scfile,S", po::value<vector<string> >(&scoreFiles), "Scorer data files")
("ffile,F", po::value<vector<string> > (&featureFiles), "Feature data files")
("hgdir,H", po::value<string> (&hgDir), "Directory containing hypergraphs")
("reference,R", po::value<vector<string> > (&referenceFiles), "Reference files, only required for hypergraph mira")
("random-seed,r", po::value<int>(&seed), "Seed for random number generation")
("output-file,o", po::value<string>(&outputFile), "Output file")
("cparam,C", po::value<float>(&c), "MIRA C-parameter, lower for more regularization (default 0.01)")
("decay,D", po::value<float>(&decay), "BLEU background corpus decay rate (default 0.999)")
("iters,J", po::value<int>(&n_iters), "Number of MIRA iterations to run (default 60)")
("dense-init,d", po::value<string>(&denseInitFile), "Weight file for dense features")
("dense-init,d", po::value<string>(&denseInitFile), "Weight file for dense features. This should have 'name= value' on each line, or (legacy) should be the Moses mert 'init.opt' format.")
("sparse-init,s", po::value<string>(&sparseInitFile), "Weight file for sparse features")
("streaming", po::value(&streaming)->zero_tokens()->default_value(false), "Stream n-best lists to save memory, implies --no-shuffle")
("no-shuffle", po::value(&no_shuffle)->zero_tokens()->default_value(false), "Don't shuffle hypotheses before each epoch")
("model-bg", po::value(&model_bg)->zero_tokens()->default_value(false), "Use model instead of hope for BLEU background")
("verbose", po::value(&verbose)->zero_tokens()->default_value(false), "Verbose updates")
("safe-hope", po::value(&safe_hope)->zero_tokens()->default_value(false), "Mode score's influence on hope decoding is limited")
("hg-prune", po::value<size_t>(&hgPruning), "Prune hypergraphs to have this many edges per reference word")
;
po::options_description cmdline_options;
@ -145,12 +130,56 @@ int main(int argc, char** argv)
cerr << "could not open dense initfile: " << denseInitFile << endl;
exit(3);
}
if (verbose) cerr << "Reading dense features:" << endl;
parameter_t val;
getline(opt,buffer);
istringstream strstrm(buffer);
while(strstrm >> val) {
initParams.push_back(val);
if (buffer.find_first_of("=") == buffer.npos) {
UTIL_THROW_IF(type == "hypergraph", util::Exception, "For hypergraph version, require dense features in 'name= value' format");
cerr << "WARN: dense features in deprecated Moses mert format. Prefer 'name= value' format." << endl;
istringstream strstrm(buffer);
while(strstrm >> val) {
initParams.push_back(val);
if(verbose) cerr << val << endl;
}
} else {
vector<string> names;
string last_name = "";
size_t feature_ctr = 0;
do {
size_t equals = buffer.find_last_of("=");
UTIL_THROW_IF(equals == buffer.npos, util::Exception, "Incorrect format in dense feature file: '"
<< buffer << "'");
string name = buffer.substr(0,equals);
names.push_back(name);
initParams.push_back(boost::lexical_cast<ValType>(buffer.substr(equals+2)));
//Names for features with several values need to have their id added
if (name != last_name) feature_ctr = 0;
last_name = name;
if (feature_ctr) {
stringstream namestr;
namestr << names.back() << feature_ctr;
names[names.size()-1] = namestr.str();
if (feature_ctr == 1) {
stringstream namestr;
namestr << names[names.size()-2] << (feature_ctr-1);
names[names.size()-2] = namestr.str();
}
}
++feature_ctr;
} while(getline(opt,buffer));
//Make sure that SparseVector encodes dense feature names as 0..n-1.
for (size_t i = 0; i < names.size(); ++i) {
size_t id = SparseVector::encode(names[i]);
assert(id == i);
if (verbose) cerr << names[i] << " " << initParams[i] << endl;
}
}
opt.close();
}
size_t initDenseSize = initParams.size();
@ -188,82 +217,45 @@ int main(int argc, char** argv)
}
bg.push_back(kBleuNgramOrder);
boost::scoped_ptr<HopeFearDecoder> decoder;
if (type == "nbest") {
decoder.reset(new NbestHopeFearDecoder(featureFiles, scoreFiles, streaming, no_shuffle, safe_hope));
} else if (type == "hypergraph") {
decoder.reset(new HypergraphHopeFearDecoder(hgDir, referenceFiles, initDenseSize, streaming, no_shuffle, safe_hope, hgPruning, wv));
} else {
UTIL_THROW(util::Exception, "Unknown batch mira type: '" << type << "'");
}
// Training loop
boost::scoped_ptr<HypPackEnumerator> train;
if(streaming)
train.reset(new StreamingHypPackEnumerator(featureFiles, scoreFiles));
else
train.reset(new RandomAccessHypPackEnumerator(featureFiles, scoreFiles, no_shuffle));
cerr << "Initial BLEU = " << evaluate(train.get(), wv.avg()) << endl;
cerr << "Initial BLEU = " << decoder->Evaluate(wv.avg()) << endl;
ValType bestBleu = 0;
for(int j=0; j<n_iters; j++) {
// MIRA train for one epoch
int iNumHyps = 0;
int iNumExamples = 0;
int iNumUpdates = 0;
ValType totalLoss = 0.0;
for(train->reset(); !train->finished(); train->next()) {
// Hope / fear decode
ValType hope_scale = 1.0;
size_t hope_index=0, fear_index=0, model_index=0;
ValType hope_score=0, fear_score=0, model_score=0;
int iNumHypsBackup = iNumHyps;
for(size_t safe_loop=0; safe_loop<2; safe_loop++) {
iNumHyps = iNumHypsBackup;
ValType hope_bleu, hope_model;
for(size_t i=0; i< train->cur_size(); i++) {
const MiraFeatureVector& vec=train->featuresAt(i);
ValType score = wv.score(vec);
ValType bleu = sentenceLevelBackgroundBleu(train->scoresAt(i),bg);
// Hope
if(i==0 || (hope_scale*score + bleu) > hope_score) {
hope_score = hope_scale*score + bleu;
hope_index = i;
hope_bleu = bleu;
hope_model = score;
}
// Fear
if(i==0 || (score - bleu) > fear_score) {
fear_score = score - bleu;
fear_index = i;
}
// Model
if(i==0 || score > model_score) {
model_score = score;
model_index = i;
}
iNumHyps++;
}
// Outer loop rescales the contribution of model score to 'hope' in antagonistic cases
// where model score is having far more influence than BLEU
hope_bleu *= BLEU_RATIO; // We only care about cases where model has MUCH more influence than BLEU
if(safe_hope && safe_loop==0 && abs(hope_model)>1e-8 && abs(hope_bleu)/abs(hope_model)<hope_scale)
hope_scale = abs(hope_bleu) / abs(hope_model);
else break;
}
size_t sentenceIndex = 0;
for(decoder->reset();!decoder->finished(); decoder->next()) {
HopeFearData hfd;
decoder->HopeFear(bg,wv,&hfd);
// Update weights
if(hope_index!=fear_index) {
if (!hfd.hopeFearEqual && hfd.hopeBleu > hfd.fearBleu) {
// Vector difference
const MiraFeatureVector& hope=train->featuresAt(hope_index);
const MiraFeatureVector& fear=train->featuresAt(fear_index);
MiraFeatureVector diff = hope - fear;
MiraFeatureVector diff = hfd.hopeFeatures - hfd.fearFeatures;
// Bleu difference
const vector<float>& hope_stats = train->scoresAt(hope_index);
ValType hopeBleu = sentenceLevelBackgroundBleu(hope_stats, bg);
const vector<float>& fear_stats = train->scoresAt(fear_index);
ValType fearBleu = sentenceLevelBackgroundBleu(fear_stats, bg);
assert(hopeBleu + 1e-8 >= fearBleu);
ValType delta = hopeBleu - fearBleu;
//assert(hfd.hopeBleu + 1e-8 >= hfd.fearBleu);
ValType delta = hfd.hopeBleu - hfd.fearBleu;
// Loss and update
ValType diff_score = wv.score(diff);
ValType loss = delta - diff_score;
if(verbose) {
cerr << "Updating sent " << train->cur_id() << endl;
cerr << "Updating sent " << sentenceIndex << endl;
cerr << "Wght: " << wv << endl;
cerr << "Hope: " << hope << " BLEU:" << hopeBleu << " Score:" << wv.score(hope) << endl;
cerr << "Fear: " << fear << " BLEU:" << fearBleu << " Score:" << wv.score(fear) << endl;
cerr << "Hope: " << hfd.hopeFeatures << " BLEU:" << hfd.hopeBleu << " Score:" << wv.score(hfd.hopeFeatures) << endl;
cerr << "Fear: " << hfd.fearFeatures << " BLEU:" << hfd.fearBleu << " Score:" << wv.score(hfd.fearFeatures) << endl;
cerr << "Diff: " << diff << " BLEU:" << delta << " Score:" << diff_score << endl;
cerr << "Loss: " << loss << " Scale: " << hope_scale << endl;
cerr << "Loss: " << loss << " Scale: " << 1 << endl;
cerr << endl;
}
if(loss > 0) {
@ -273,16 +265,16 @@ int main(int argc, char** argv)
iNumUpdates++;
}
// Update BLEU statistics
const vector<float>& model_stats = train->scoresAt(model_index);
for(size_t k=0; k<bg.size(); k++) {
bg[k]*=decay;
if(model_bg)
bg[k]+=model_stats[k];
bg[k]+=hfd.modelStats[k];
else
bg[k]+=hope_stats[k];
bg[k]+=hfd.hopeStats[k];
}
}
iNumExamples++;
++sentenceIndex;
}
// Training Epoch summary
cerr << iNumUpdates << "/" << iNumExamples << " updates"
@ -291,15 +283,16 @@ int main(int argc, char** argv)
// Evaluate current average weights
AvgWeightVector avg = wv.avg();
ValType bleu = evaluate(train.get(), avg);
ValType bleu = decoder->Evaluate(avg);
cerr << ", BLEU = " << bleu << endl;
if(bleu > bestBleu) {
/*
size_t num_dense = train->num_dense();
if(initDenseSize>0 && initDenseSize!=num_dense) {
cerr << "Error: Initial dense feature count and dense feature count from n-best do not match: "
<< initDenseSize << "!=" << num_dense << endl;
exit(1);
}
}*/
// Write to a file
ostream* out;
ofstream outFile;
@ -314,11 +307,11 @@ int main(int argc, char** argv)
out = &cout;
}
for(size_t i=0; i<avg.size(); i++) {
if(i<num_dense)
if(i<initDenseSize)
*out << "F" << i << " " << avg.weight(i) << endl;
else {
if(abs(avg.weight(i))>1e-8)
*out << SparseVector::decode(i-num_dense) << " " << avg.weight(i) << endl;
*out << SparseVector::decode(i-initDenseSize) << " " << avg.weight(i) << endl;
}
}
outFile.close();

2
mira/Main.cpp
View File

@ -665,7 +665,7 @@ int main(int argc, char** argv)
}
// number of weight dumps this epoch
// size_t weightMixingThisEpoch = 0;
size_t weightMixingThisEpoch = 0;
size_t weightEpochDump = 0;
size_t shardPosition = 0;

10
misc/Jamfile
View File

@ -36,4 +36,12 @@ else {
alias programsProbing ;
}
alias programs : 1-1-Extraction TMining generateSequences processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin programsProbing ;
exe merge-sorted :
merge-sorted.cc
../moses//moses
../moses/TranslationModel/UG/generic//generic
$(TOP)//boost_iostreams
$(TOP)//boost_program_options
;
alias programs : 1-1-Extraction TMining generateSequences processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin programsProbing merge-sorted ;

0
contrib/m4m/util/merge-sorted.cc → misc/merge-sorted.cc
View File

12
moses-cmd/Jamfile
View File

@ -3,4 +3,16 @@ alias deps : IOWrapper.cpp mbr.cpp LatticeMBR.cpp TranslationAnalysis.cpp ..//z
exe moses : Main.cpp deps ;
exe lmbrgrid : LatticeMBRGrid.cpp deps ;
if [ option.get "with-mm" : : "yes" ] {
exe simulate-pe :
simulate-pe.cc
$(TOP)/moses/TranslationModel/UG/generic//generic
$(TOP)//boost_program_options
deps
;
alias programs : moses lmbrgrid simulate-pe ;
} else {
alias programs : moses lmbrgrid ;
}

856
moses-cmd/simulate-pe.cc Normal file
View File

@ -0,0 +1,856 @@
// Fork of Main.cpp, to simulate post-editing sessions.
// Written by Ulrich Germann.
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2009 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <boost/algorithm/string/predicate.hpp>
#include <boost/filesystem.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filter/bzip2.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/foreach.hpp>
#include <exception>
#include <fstream>
#include <sstream>
#include <vector>
#include "util/usage.hh"
#include "util/exception.hh"
#include "moses/Util.h"
#include "moses/TranslationModel/UG/mmsapt.h"
#include "moses/TranslationModel/UG/generic/program_options/ug_splice_arglist.h"
#ifdef WIN32
// Include Visual Leak Detector
//#include <vld.h>
#endif
#include "TranslationAnalysis.h"
#include "IOWrapper.h"
#include "mbr.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/StaticData.h"
#include "moses/Util.h"
#include "moses/Timer.h"
#include "moses/ThreadPool.h"
#include "moses/OutputCollector.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/FF/StatefulFeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h"
#ifdef HAVE_PROTOBUF
#include "hypergraph.pb.h"
#endif
using namespace std;
using namespace Moses;
using namespace MosesCmd;
namespace MosesCmd
{
// output floats with five significant digits
static const size_t PRECISION = 3;
/** Enforce rounding */
void fix(std::ostream& stream, size_t size)
{
stream.setf(std::ios::fixed);
stream.precision(size);
}
/** Translates a sentence.
* - calls the search (Manager)
* - applies the decision rule
* - outputs best translation and additional reporting
**/
class TranslationTask : public Task
{
public:
TranslationTask(size_t lineNumber,
InputType* source,
OutputCollector* outputCollector,
OutputCollector* nbestCollector,
OutputCollector* latticeSamplesCollector,
OutputCollector* wordGraphCollector,
OutputCollector* searchGraphCollector,
OutputCollector* detailedTranslationCollector,
OutputCollector* alignmentInfoCollector,
OutputCollector* unknownsCollector,
bool outputSearchGraphSLF,
bool outputSearchGraphHypergraph)
: m_source(source)
, m_lineNumber(lineNumber)
, m_outputCollector(outputCollector)
, m_nbestCollector(nbestCollector)
, m_latticeSamplesCollector(latticeSamplesCollector)
, m_wordGraphCollector(wordGraphCollector)
, m_searchGraphCollector(searchGraphCollector)
, m_detailedTranslationCollector(detailedTranslationCollector)
, m_alignmentInfoCollector(alignmentInfoCollector)
, m_unknownsCollector(unknownsCollector)
, m_outputSearchGraphSLF(outputSearchGraphSLF)
, m_outputSearchGraphHypergraph(outputSearchGraphHypergraph)
{ }
/** Translate one sentence
* gets called by main function implemented at end of this source file */
void Run() {
// shorthand for "global data"
const StaticData &staticData = StaticData::Instance();
// input sentence
Sentence sentence;
// report wall time spent on translation
Timer translationTime;
translationTime.start();
// report thread number
#if defined(WITH_THREADS) && defined(BOOST_HAS_PTHREADS)
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl);
#endif
// execute the translation
// note: this executes the search, resulting in a search graph
// we still need to apply the decision rule (MAP, MBR, ...)
Timer initTime;
initTime.start();
Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm());
VERBOSE(1, "Line " << m_lineNumber << ": Initialize search took " << initTime << " seconds total" << endl);
manager.ProcessSentence();
// we are done with search, let's look what we got
Timer additionalReportingTime;
additionalReportingTime.start();
// output word graph
if (m_wordGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.GetWordGraph(m_lineNumber, out);
m_wordGraphCollector->Write(m_lineNumber, out.str());
}
// output search graph
if (m_searchGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraph(m_lineNumber, out);
m_searchGraphCollector->Write(m_lineNumber, out.str());
#ifdef HAVE_PROTOBUF
if (staticData.GetOutputSearchGraphPB()) {
ostringstream sfn;
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
string fn = sfn.str();
VERBOSE(2, "Writing search graph to " << fn << endl);
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
manager.SerializeSearchGraphPB(m_lineNumber, output);
}
#endif
}
// Output search graph in HTK standard lattice format (SLF)
if (m_outputSearchGraphSLF) {
stringstream fileName;
fileName << staticData.GetParam("output-search-graph-slf")[0] << "/" << m_lineNumber << ".slf";
std::ofstream *file = new std::ofstream;
file->open(fileName.str().c_str());
if (file->is_open() && file->good()) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraphAsSLF(m_lineNumber, out);
*file << out.str();
file -> flush();
} else {
TRACE_ERR("Cannot output HTK standard lattice for line " << m_lineNumber << " because the output file is not open or not ready for writing" << std::endl);
}
delete file;
}
// Output search graph in hypergraph format for Kenneth Heafield's lazy hypergraph decoder
if (m_outputSearchGraphHypergraph) {
vector<string> hypergraphParameters = staticData.GetParam("output-search-graph-hypergraph");
bool appendSuffix;
if (hypergraphParameters.size() > 0 && hypergraphParameters[0] == "true") {
appendSuffix = true;
} else {
appendSuffix = false;
}
string compression;
if (hypergraphParameters.size() > 1) {
compression = hypergraphParameters[1];
} else {
compression = "txt";
}
string hypergraphDir;
if ( hypergraphParameters.size() > 2 ) {
hypergraphDir = hypergraphParameters[2];
} else {
string nbestFile = staticData.GetNBestFilePath();
if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) {
boost::filesystem::path nbestPath(nbestFile);
// In the Boost filesystem API version 2,
// which was the default prior to Boost 1.46,
// the filename() method returned a string.
//
// In the Boost filesystem API version 3,
// which is the default starting with Boost 1.46,
// the filename() method returns a path object.
//
// To get a string from the path object,
// the native() method must be called.
// hypergraphDir = nbestPath.parent_path().filename()
//#if BOOST_VERSION >= 104600
// .native()
//#endif
//;
// Hopefully the following compiles under all versions of Boost.
//
// If this line gives you compile errors,
// contact Lane Schwartz on the Moses mailing list
hypergraphDir = nbestPath.parent_path().string();
} else {
stringstream hypergraphDirName;
hypergraphDirName << boost::filesystem::current_path().string() << "/hypergraph";
hypergraphDir = hypergraphDirName.str();
}
}
if ( ! boost::filesystem::exists(hypergraphDir) ) {
boost::filesystem::create_directory(hypergraphDir);
}
if ( ! boost::filesystem::exists(hypergraphDir) ) {
TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because the directory does not exist" << std::endl);
} else if ( ! boost::filesystem::is_directory(hypergraphDir) ) {
TRACE_ERR("Cannot output hypergraphs to " << hypergraphDir << " because that path exists, but is not a directory" << std::endl);
} else {
stringstream fileName;
fileName << hypergraphDir << "/" << m_lineNumber;
if ( appendSuffix ) {
fileName << "." << compression;
}
boost::iostreams::filtering_ostream *file
= new boost::iostreams::filtering_ostream;
if ( compression == "gz" ) {
file->push( boost::iostreams::gzip_compressor() );
} else if ( compression == "bz2" ) {
file->push( boost::iostreams::bzip2_compressor() );
} else if ( compression != "txt" ) {
TRACE_ERR("Unrecognized hypergraph compression format ("
<< compression
<< ") - using uncompressed plain txt" << std::endl);
compression = "txt";
}
file->push( boost::iostreams::file_sink(fileName.str(), ios_base::out) );
if (file->is_complete() && file->good()) {
fix(*file,PRECISION);
manager.OutputSearchGraphAsHypergraph(m_lineNumber, *file);
file -> flush();
} else {
TRACE_ERR("Cannot output hypergraph for line " << m_lineNumber
<< " because the output file " << fileName.str()
<< " is not open or not ready for writing"
<< std::endl);
}
file -> pop();
delete file;
}
}
additionalReportingTime.stop();
// apply decision rule and output best translation(s)
if (m_outputCollector) {
ostringstream out;
ostringstream debug;
fix(debug,PRECISION);
// all derivations - send them to debug stream
if (staticData.PrintAllDerivations()) {
additionalReportingTime.start();
manager.PrintAllDerivations(m_lineNumber, debug);
additionalReportingTime.stop();
}
Timer decisionRuleTime;
decisionRuleTime.start();
// MAP decoding: best hypothesis
const Hypothesis* bestHypo = NULL;
if (!staticData.UseMBR()) {
bestHypo = manager.GetBestHypothesis();
if (bestHypo) {
if (StaticData::Instance().GetOutputHypoScore()) {
out << bestHypo->GetTotalScore() << ' ';
}
if (staticData.IsPathRecoveryEnabled()) {
OutputInput(out, bestHypo);
out << "||| ";
}
if (staticData.GetParam("print-id").size() && Scan<bool>(staticData.GetParam("print-id")[0]) ) {
out << m_source->GetTranslationId() << " ";
}
if (staticData.GetReportSegmentation() == 2) {
manager.GetOutputLanguageModelOrder(out, bestHypo);
}
OutputBestSurface(
out,
bestHypo,
staticData.GetOutputFactorOrder(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors());
if (staticData.PrintAlignmentInfo()) {
out << "||| ";
OutputAlignment(out, bestHypo);
}
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
} else {
VERBOSE(1, "NO BEST TRANSLATION" << endl);
}
out << endl;
}
// MBR decoding (n-best MBR, lattice MBR, consensus)
else {
// we first need the n-best translations
size_t nBestSize = staticData.GetMBRSize();
if (nBestSize <= 0) {
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
exit(1);
}
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
IFVERBOSE(2) {
PrintUserTime("calculated n-best list for (L)MBR decoding");
}
// lattice MBR
if (staticData.UseLatticeMBR()) {
if (m_nbestCollector) {
//lattice mbr nbest
vector<LatticeMBRSolution> solutions;
size_t n = min(nBestSize, staticData.GetNBestSize());
getLatticeMBRNBest(manager,nBestList,solutions,n);
ostringstream out;
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
m_nbestCollector->Write(m_lineNumber, out.str());
} else {
//Lattice MBR decoding
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
IFVERBOSE(2) {
PrintUserTime("finished Lattice MBR decoding");
}
}
}
// consensus decoding
else if (staticData.UseConsensusDecoding()) {
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
OutputBestHypo(conBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished Consensus decoding");
}
}
// n-best MBR decoding
else {
const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished MBR decoding");
}
}
}
// report best translation to output collector
m_outputCollector->Write(m_lineNumber,out.str(),debug.str());
decisionRuleTime.stop();
VERBOSE(1, "Line " << m_lineNumber << ": Decision rule took " << decisionRuleTime << " seconds total" << endl);
}
additionalReportingTime.start();
// output n-best list
if (m_nbestCollector && !staticData.UseLatticeMBR()) {
TrellisPathList nBestList;
ostringstream out;
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
OutputNBest(out, nBestList, staticData.GetOutputFactorOrder(), m_lineNumber,
staticData.GetReportSegmentation());
m_nbestCollector->Write(m_lineNumber, out.str());
}
//lattice samples
if (m_latticeSamplesCollector) {
TrellisPathList latticeSamples;
ostringstream out;
manager.CalcLatticeSamples(staticData.GetLatticeSamplesSize(), latticeSamples);
OutputNBest(out,latticeSamples, staticData.GetOutputFactorOrder(), m_lineNumber,
staticData.GetReportSegmentation());
m_latticeSamplesCollector->Write(m_lineNumber, out.str());
}
// detailed translation reporting
if (m_detailedTranslationCollector) {
ostringstream out;
fix(out,PRECISION);
TranslationAnalysis::PrintTranslationAnalysis(out, manager.GetBestHypothesis());
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
}
//list of unknown words
if (m_unknownsCollector) {
const vector<const Phrase*>& unknowns = manager.getSntTranslationOptions()->GetUnknownSources();
ostringstream out;
for (size_t i = 0; i < unknowns.size(); ++i) {
out << *(unknowns[i]);
}
out << endl;
m_unknownsCollector->Write(m_lineNumber, out.str());
}
// report additional statistics
manager.CalcDecoderStatistics();
VERBOSE(1, "Line " << m_lineNumber << ": Additional reporting took " << additionalReportingTime << " seconds total" << endl);
VERBOSE(1, "Line " << m_lineNumber << ": Translation took " << translationTime << " seconds total" << endl);
IFVERBOSE(2) {
PrintUserTime("Sentence Decoding Time:");
}
}
~TranslationTask() {
delete m_source;
}
private:
InputType* m_source;
size_t m_lineNumber;
OutputCollector* m_outputCollector;
OutputCollector* m_nbestCollector;
OutputCollector* m_latticeSamplesCollector;
OutputCollector* m_wordGraphCollector;
OutputCollector* m_searchGraphCollector;
OutputCollector* m_detailedTranslationCollector;
OutputCollector* m_alignmentInfoCollector;
OutputCollector* m_unknownsCollector;
bool m_outputSearchGraphSLF;
bool m_outputSearchGraphHypergraph;
std::ofstream *m_alignmentStream;
};
static void PrintFeatureWeight(const FeatureFunction* ff)
{
cout << ff->GetScoreProducerDescription() << "=";
size_t numScoreComps = ff->GetNumScoreComponents();
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
for (size_t i = 0; i < numScoreComps; ++i) {
cout << " " << values[i];
}
cout << endl;
}
static void ShowWeights()
{
//TODO: Find a way of ensuring this order is synced with the nbest
fix(cout,6);
const vector<const StatelessFeatureFunction*>& slf = StatelessFeatureFunction::GetStatelessFeatureFunctions();
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < sff.size(); ++i) {
const StatefulFeatureFunction *ff = sff[i];
if (ff->IsTuneable()) {
PrintFeatureWeight(ff);
}
else {
cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl;
}
}
for (size_t i = 0; i < slf.size(); ++i) {
const StatelessFeatureFunction *ff = slf[i];
if (ff->IsTuneable()) {
PrintFeatureWeight(ff);
}
else {
cout << ff->GetScoreProducerDescription() << " UNTUNEABLE" << endl;
}
}
}
size_t OutputFeatureWeightsForHypergraph(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream)
{
size_t numScoreComps = ff->GetNumScoreComponents();
if (numScoreComps != 0) {
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
if (numScoreComps > 1) {
for (size_t i = 0; i < numScoreComps; ++i) {
outputSearchGraphStream << ff->GetScoreProducerDescription()
<< i
<< "=" << values[i] << endl;
}
} else {
outputSearchGraphStream << ff->GetScoreProducerDescription()
<< "=" << values[0] << endl;
}
return index+numScoreComps;
} else {
UTIL_THROW2("Sparse features are not yet supported when outputting hypergraph format");
}
}
void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream)
{
outputSearchGraphStream.setf(std::ios::fixed);
outputSearchGraphStream.precision(6);
const vector<const StatelessFeatureFunction*>& slf =StatelessFeatureFunction::GetStatelessFeatureFunctions();
const vector<const StatefulFeatureFunction*>& sff = StatefulFeatureFunction::GetStatefulFeatureFunctions();
size_t featureIndex = 1;
for (size_t i = 0; i < sff.size(); ++i) {
featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, sff[i], outputSearchGraphStream);
}
for (size_t i = 0; i < slf.size(); ++i) {
/*
if (slf[i]->GetScoreProducerWeightShortName() != "u" &&
slf[i]->GetScoreProducerWeightShortName() != "tm" &&
slf[i]->GetScoreProducerWeightShortName() != "I" &&
slf[i]->GetScoreProducerWeightShortName() != "g")
*/
{
featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, slf[i], outputSearchGraphStream);
}
}
const vector<PhraseDictionary*>& pds = PhraseDictionary::GetColl();
for( size_t i=0; i<pds.size(); i++ ) {
featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, pds[i], outputSearchGraphStream);
}
const vector<GenerationDictionary*>& gds = GenerationDictionary::GetColl();
for( size_t i=0; i<gds.size(); i++ ) {
featureIndex = OutputFeatureWeightsForHypergraph(featureIndex, gds[i], outputSearchGraphStream);
}
}
} //namespace
/** main function of the command line version of the decoder **/
int main(int argc, char** argv)
{
try {
#ifdef HAVE_PROTOBUF
GOOGLE_PROTOBUF_VERIFY_VERSION;
#endif
// echo command line, if verbose
IFVERBOSE(1) {
TRACE_ERR("command: ");
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
TRACE_ERR(endl);
}
// set number of significant decimals in output
fix(cout,PRECISION);
fix(cerr,PRECISION);
vector<pair<string,int> > argfilter(4);
argfilter[0] = std::make_pair(string("--spe-src"),1);
argfilter[1] = std::make_pair(string("--spe-trg"),1);
argfilter[2] = std::make_pair(string("--spe-aln"),1);
char** my_args; int my_acnt;
char** mo_args; int mo_acnt;
filter_arguments(argc, argv, mo_acnt, &mo_args, my_acnt, &my_args, argfilter);
ifstream spe_src,spe_trg,spe_aln;
// instead of translating show coverage by phrase tables
for (int i = 0; i < my_acnt; i += 2)
{
if (!strcmp(my_args[i],"--spe-src"))
spe_src.open(my_args[i+1]);
else if (!strcmp(my_args[i],"--spe-trg"))
spe_trg.open(my_args[i+1]);
else if (!strcmp(my_args[i],"--spe-aln"))
spe_aln.open(my_args[i+1]);
}
// load all the settings into the Parameter class
// (stores them as strings, or array of strings)
Parameter params;
if (!params.LoadParam(mo_acnt,mo_args)) {
exit(1);
}
// initialize all "global" variables, which are stored in StaticData
// note: this also loads models such as the language model, etc.
if (!StaticData::LoadDataStatic(&params, argv[0])) {
exit(1);
}
// setting "-show-weights" -> just dump out weights and exit
if (params.isParamSpecified("show-weights")) {
ShowWeights();
exit(0);
}
// shorthand for accessing information in StaticData
const StaticData& staticData = StaticData::Instance();
//initialise random numbers
srand(time(NULL));
// set up read/writing class
IOWrapper* ioWrapper = GetIOWrapper(staticData);
if (!ioWrapper) {
cerr << "Error; Failed to create IO object" << endl;
exit(1);
}
// check on weights
const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights);
TRACE_ERR("\n");
}
if (staticData.GetOutputSearchGraphHypergraph()) {
ofstream* weightsOut = new std::ofstream;
stringstream weightsFilename;
if (staticData.GetParam("output-search-graph-hypergraph").size() > 3) {
weightsFilename << staticData.GetParam("output-search-graph-hypergraph")[3];
} else {
string nbestFile = staticData.GetNBestFilePath();
if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) {
boost::filesystem::path nbestPath(nbestFile);
weightsFilename << nbestPath.parent_path().filename() << "/weights";
} else {
weightsFilename << boost::filesystem::current_path().string() << "/hypergraph/weights";
}
}
boost::filesystem::path weightsFilePath(weightsFilename.str());
if ( ! boost::filesystem::exists(weightsFilePath.parent_path()) ) {
boost::filesystem::create_directory(weightsFilePath.parent_path());
}
TRACE_ERR("The weights file is " << weightsFilename.str() << "\n");
weightsOut->open(weightsFilename.str().c_str());
OutputFeatureWeightsForHypergraph(*weightsOut);
weightsOut->flush();
weightsOut->close();
delete weightsOut;
}
// initialize output streams
// note: we can't just write to STDOUT or files
// because multithreading may return sentences in shuffled order
auto_ptr<OutputCollector> outputCollector; // for translations
auto_ptr<OutputCollector> nbestCollector; // for n-best lists
auto_ptr<OutputCollector> latticeSamplesCollector; //for lattice samples
auto_ptr<ofstream> nbestOut;
auto_ptr<ofstream> latticeSamplesOut;
size_t nbestSize = staticData.GetNBestSize();
string nbestFile = staticData.GetNBestFilePath();
bool output1best = true;
if (nbestSize) {
if (nbestFile == "-" || nbestFile == "/dev/stdout") {
// nbest to stdout, no 1-best
nbestCollector.reset(new OutputCollector());
output1best = false;
} else {
// nbest to file, 1-best to stdout
nbestOut.reset(new ofstream(nbestFile.c_str()));
if (!nbestOut->good()) {
TRACE_ERR("ERROR: Failed to open " << nbestFile << " for nbest lists" << endl);
exit(1);
}
nbestCollector.reset(new OutputCollector(nbestOut.get()));
}
}
size_t latticeSamplesSize = staticData.GetLatticeSamplesSize();
string latticeSamplesFile = staticData.GetLatticeSamplesFilePath();
if (latticeSamplesSize) {
if (latticeSamplesFile == "-" || latticeSamplesFile == "/dev/stdout") {
latticeSamplesCollector.reset(new OutputCollector());
output1best = false;
} else {
latticeSamplesOut.reset(new ofstream(latticeSamplesFile.c_str()));
if (!latticeSamplesOut->good()) {
TRACE_ERR("ERROR: Failed to open " << latticeSamplesFile << " for lattice samples" << endl);
exit(1);
}
latticeSamplesCollector.reset(new OutputCollector(latticeSamplesOut.get()));
}
}
if (output1best) {
outputCollector.reset(new OutputCollector());
}
// initialize stream for word graph (aka: output lattice)
auto_ptr<OutputCollector> wordGraphCollector;
if (staticData.GetOutputWordGraph()) {
wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream())));
}
// initialize stream for search graph
// note: this is essentially the same as above, but in a different format
auto_ptr<OutputCollector> searchGraphCollector;
if (staticData.GetOutputSearchGraph()) {
searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream())));
}
// initialize stram for details about the decoder run
auto_ptr<OutputCollector> detailedTranslationCollector;
if (staticData.IsDetailedTranslationReportingEnabled()) {
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
}
// initialize stram for word alignment between input and output
auto_ptr<OutputCollector> alignmentInfoCollector;
if (!staticData.GetAlignmentOutputFile().empty()) {
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
}
//initialise stream for unknown (oov) words
auto_ptr<OutputCollector> unknownsCollector;
auto_ptr<ofstream> unknownsStream;
if (!staticData.GetOutputUnknownsFile().empty()) {
unknownsStream.reset(new ofstream(staticData.GetOutputUnknownsFile().c_str()));
if (!unknownsStream->good()) {
TRACE_ERR("Unable to open " << staticData.GetOutputUnknownsFile() << " for unknowns");
exit(1);
}
unknownsCollector.reset(new OutputCollector(unknownsStream.get()));
}
#ifdef WITH_THREADS
ThreadPool pool(staticData.ThreadCount());
#endif
// main loop over set of input sentences
InputType* source = NULL;
size_t lineCount = staticData.GetStartTranslationId();
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
IFVERBOSE(1) {
ResetUserTime();
}
// set up task of translating one sentence
TranslationTask* task =
new TranslationTask(lineCount,source, outputCollector.get(),
nbestCollector.get(),
latticeSamplesCollector.get(),
wordGraphCollector.get(),
searchGraphCollector.get(),
detailedTranslationCollector.get(),
alignmentInfoCollector.get(),
unknownsCollector.get(),
staticData.GetOutputSearchGraphSLF(),
staticData.GetOutputSearchGraphHypergraph());
// execute task
#ifdef WITH_THREADS
if (my_acnt)
{
task->Run();
delete task;
string src,trg,aln;
UTIL_THROW_IF2(!getline(spe_src,src), "[" << HERE << "] "
<< "missing update data for simulated post-editing.");
UTIL_THROW_IF2(!getline(spe_trg,trg), "[" << HERE << "] "
<< "missing update data for simulated post-editing.");
UTIL_THROW_IF2(!getline(spe_aln,aln), "[" << HERE << "] "
<< "missing update data for simulated post-editing.");
BOOST_FOREACH (PhraseDictionary* pd, PhraseDictionary::GetColl())
{
Mmsapt* sapt = dynamic_cast<Mmsapt*>(pd);
if (sapt) sapt->add(src,trg,aln);
VERBOSE(1,"[" << HERE << " added src] " << src << endl);
VERBOSE(1,"[" << HERE << " added trg] " << trg << endl);
VERBOSE(1,"[" << HERE << " added aln] " << aln << endl);
}
}
else pool.Submit(task);
#else
task->Run();
delete task;
#endif
source = NULL; //make sure it doesn't get deleted
++lineCount;
}
// we are done, finishing up
#ifdef WITH_THREADS
pool.Stop(true); //flush remaining jobs
#endif
delete ioWrapper;
FeatureFunction::Destroy();
} catch (const std::exception &e) {
std::cerr << "Exception: " << e.what() << std::endl;
return EXIT_FAILURE;
}
IFVERBOSE(1) util::PrintUsage(std::cerr);
#ifndef EXIT_RETURN
//This avoids that destructors are called (it can take a long time)
exit(EXIT_SUCCESS);
#else
return EXIT_SUCCESS;
#endif
}

14
moses/BitmapContainer.cpp
View File

@ -161,13 +161,17 @@ BackwardsEdge::BackwardsEdge(const BitmapContainer &prevBitmapContainer
}
if (m_translations.size() > 1) {
UTIL_THROW_IF2(m_translations.Get(0)->GetFutureScore() < m_translations.Get(1)->GetFutureScore(),
"Non-monotonic future score");
UTIL_THROW_IF2(m_translations.Get(0)->GetFutureScore() < m_translations.Get(1)->GetFutureScore(),
"Non-monotonic future score: "
<< m_translations.Get(0)->GetFutureScore() << " vs. "
<< m_translations.Get(1)->GetFutureScore());
}
if (m_hypotheses.size() > 1) {
UTIL_THROW_IF2(m_hypotheses[0]->GetTotalScore() < m_hypotheses[1]->GetTotalScore(),
"Non-monotonic total score");
"Non-monotonic total score"
<< m_hypotheses[0]->GetTotalScore() << " vs. "
<< m_hypotheses[1]->GetTotalScore());
}
HypothesisScoreOrdererWithDistortion orderer (&transOptRange);
@ -442,7 +446,9 @@ BitmapContainer::ProcessBestHypothesis()
if (!Empty()) {
HypothesisQueueItem *check = Dequeue(true);
UTIL_THROW_IF2(item->GetHypothesis()->GetTotalScore() < check->GetHypothesis()->GetTotalScore(),
"Non-monotonic total score");
"Non-monotonic total score: "
<< item->GetHypothesis()->GetTotalScore() << " vs. "
<< check->GetHypothesis()->GetTotalScore());
}
// Logging for the criminally insane

38
moses/ChartHypothesis.cpp
View File

@ -149,6 +149,40 @@ Phrase ChartHypothesis::GetOutputPhrase() const
return outPhrase;
}
void ChartHypothesis::GetOutputPhrase(int leftRightMost, int numWords, Phrase &outPhrase) const
{
const TargetPhrase &tp = GetCurrTargetPhrase();
int targetSize = tp.GetSize();
for (int i = 0; i < targetSize; ++i) {
int pos;
if (leftRightMost == 1) {
pos = i;
}
else if (leftRightMost == 2) {
pos = targetSize - i - 1;
}
else {
abort();
}
const Word &word = tp.GetWord(pos);
if (word.IsNonTerminal()) {
// non-term. fill out with prev hypo
size_t nonTermInd = tp.GetAlignNonTerm().GetNonTermIndexMap()[pos];
const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd];
prevHypo->GetOutputPhrase(outPhrase);
} else {
outPhrase.AddWord(word);
}
if (outPhrase.GetSize() >= numWords) {
return;
}
}
}
/** check, if two hypothesis can be recombined.
this is actually a sorting function that allows us to
keep an ordered list of hypotheses. This makes recombination
@ -200,7 +234,7 @@ void ChartHypothesis::Evaluate()
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
sfs[i]->EvaluateChart(*this,&m_scoreBreakdown);
sfs[i]->EvaluateWhenApplied(*this,&m_scoreBreakdown);
}
}
@ -208,7 +242,7 @@ void ChartHypothesis::Evaluate()
StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (unsigned i = 0; i < ffs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *ffs[i] )) {
m_ffStates[i] = ffs[i]->EvaluateChart(*this,i,&m_scoreBreakdown);
m_ffStates[i] = ffs[i]->EvaluateWhenApplied(*this,i,&m_scoreBreakdown);
}
}

4
moses/ChartHypothesis.h
View File

@ -138,6 +138,10 @@ public:
void GetOutputPhrase(Phrase &outPhrase) const;
Phrase GetOutputPhrase() const;
// get leftmost/rightmost words only
// leftRightMost: 1=left, 2=right
void GetOutputPhrase(int leftRightMost, int numWords, Phrase &outPhrase) const;
int RecombineCompare(const ChartHypothesis &compare) const;
void Evaluate();

2
moses/ChartTranslationOption.cpp
View File

@ -18,7 +18,7 @@ void ChartTranslationOption::Evaluate(const InputType &input,
for (size_t i = 0; i < ffs.size(); ++i) {
const FeatureFunction &ff = *ffs[i];
ff.Evaluate(input, inputPath, m_targetPhrase, &stackVec, m_scoreBreakdown);
ff.EvaluateWithSourceContext(input, inputPath, m_targetPhrase, &stackVec, m_scoreBreakdown);
}
}

4
moses/FF/BleuScoreFeature.cpp
View File

@ -502,7 +502,7 @@ void BleuScoreFeature::GetClippedNgramMatchesAndCounts(Phrase& phrase,
* Given a previous state, compute Bleu score for the updated state with an additional target
* phrase translated.
*/
FFState* BleuScoreFeature::Evaluate(const Hypothesis& cur_hypo,
FFState* BleuScoreFeature::EvaluateWhenApplied(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
@ -563,7 +563,7 @@ FFState* BleuScoreFeature::Evaluate(const Hypothesis& cur_hypo,
return new_state;
}
FFState* BleuScoreFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureID,
FFState* BleuScoreFeature::EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID,
ScoreComponentCollection* accumulator ) const
{
if (!m_enabled) return new BleuScoreState();

8
moses/FF/BleuScoreFeature.h
View File

@ -115,20 +115,20 @@ public:
std::vector< size_t >&,
size_t skip = 0) const;
FFState* Evaluate( const Hypothesis& cur_hypo,
FFState* EvaluateWhenApplied( const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(const ChartHypothesis& cur_hypo,
FFState* EvaluateWhenApplied(const ChartHypothesis& cur_hypo,
int featureID,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

4
moses/FF/ConstrainedDecoding.cpp
View File

@ -100,7 +100,7 @@ const std::vector<Phrase> *GetConstraint(const std::map<long,std::vector<Phrase>
}
}
FFState* ConstrainedDecoding::Evaluate(
FFState* ConstrainedDecoding::EvaluateWhenApplied(
const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -143,7 +143,7 @@ FFState* ConstrainedDecoding::Evaluate(
return ret;
}
FFState* ConstrainedDecoding::EvaluateChart(
FFState* ConstrainedDecoding::EvaluateWhenApplied(
const ChartHypothesis &hypo,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/ConstrainedDecoding.h
View File

@ -41,13 +41,13 @@ public:
return true;
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -55,12 +55,12 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;

4
moses/FF/ControlRecombination.cpp
View File

@ -56,7 +56,7 @@ std::vector<float> ControlRecombination::DefaultWeights() const
return ret;
}
FFState* ControlRecombination::Evaluate(
FFState* ControlRecombination::EvaluateWhenApplied(
const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -64,7 +64,7 @@ FFState* ControlRecombination::Evaluate(
return new ControlRecombinationState(hypo, *this);
}
FFState* ControlRecombination::EvaluateChart(
FFState* ControlRecombination::EvaluateWhenApplied(
const ChartHypothesis &hypo,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/ControlRecombination.h
View File

@ -57,24 +57,24 @@ public:
return true;
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;

2
moses/FF/CountNonTerms.cpp
View File

@ -16,7 +16,7 @@ CountNonTerms::CountNonTerms(const std::string &line)
ReadParameters();
}
void CountNonTerms::Evaluate(const Phrase &sourcePhrase
void CountNonTerms::EvaluateInIsolation(const Phrase &sourcePhrase
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/CountNonTerms.h
View File

@ -12,12 +12,12 @@ public:
bool IsUseable(const FactorMask &mask) const
{ return true; }
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -25,11 +25,11 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(
void EvaluateWhenApplied(
const ChartHypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}

8
moses/FF/CoveredReferenceFeature.cpp
View File

@ -40,13 +40,13 @@ int CoveredReferenceState::Compare(const FFState& other) const
// return (m_coveredRef.size() < otherState.m_coveredRef.size()) ? -1 : +1;
}
void CoveredReferenceFeature::Evaluate(const Phrase &source
void CoveredReferenceFeature::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void CoveredReferenceFeature::Evaluate(const InputType &input
void CoveredReferenceFeature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -90,7 +90,7 @@ void CoveredReferenceFeature::SetParameter(const std::string& key, const std::st
}
}
FFState* CoveredReferenceFeature::Evaluate(
FFState* CoveredReferenceFeature::EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -131,7 +131,7 @@ FFState* CoveredReferenceFeature::Evaluate(
return ret;
}
FFState* CoveredReferenceFeature::EvaluateChart(
FFState* CoveredReferenceFeature::EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/CoveredReferenceFeature.h
View File

@ -52,21 +52,21 @@ public:
return new CoveredReferenceState();
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;

8
moses/FF/DecodeFeature.h
View File

@ -62,20 +62,20 @@ public:
bool IsUseable(const FactorMask &mask) const;
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

2
moses/FF/DistortionScoreProducer.cpp
View File

@ -87,7 +87,7 @@ float DistortionScoreProducer::CalculateDistortionScore(const Hypothesis& hypo,
}
FFState* DistortionScoreProducer::Evaluate(
FFState* DistortionScoreProducer::EvaluateWhenApplied(
const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* out) const

8
moses/FF/DistortionScoreProducer.h
View File

@ -28,26 +28,26 @@ public:
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
virtual FFState* Evaluate(
virtual FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(
virtual FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection*) const {
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

4
moses/FF/ExternalFeature.cpp
View File

@ -51,7 +51,7 @@ void ExternalFeature::SetParameter(const std::string& key, const std::string& va
}
}
FFState* ExternalFeature::Evaluate(
FFState* ExternalFeature::EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -59,7 +59,7 @@ FFState* ExternalFeature::Evaluate(
return new ExternalFeatureState(m_stateSize);
}
FFState* ExternalFeature::EvaluateChart(
FFState* ExternalFeature::EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/ExternalFeature.h
View File

@ -51,24 +51,24 @@ public:
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;

20
moses/FF/Factory.cpp
View File

@ -27,6 +27,7 @@
#include "moses/FF/PhrasePairFeature.h"
#include "moses/FF/PhraseLengthFeature.h"
#include "moses/FF/DistortionScoreProducer.h"
#include "moses/FF/SparseHieroReorderingFeature.h"
#include "moses/FF/WordPenaltyProducer.h"
#include "moses/FF/InputFeature.h"
#include "moses/FF/PhrasePenalty.h"
@ -92,6 +93,10 @@
#include "moses/LM/DALMWrapper.h"
#endif
#ifdef LM_LBL
#include "moses/LM/oxlm/LBLLM.h"
#endif
#include "util/exception.hh"
#include <vector>
@ -200,6 +205,7 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(RuleScope);
MOSES_FNAME(MaxSpanFreeNonTermSource);
MOSES_FNAME(NieceTerminal);
MOSES_FNAME(SparseHieroReorderingFeature);
MOSES_FNAME(SpanLength);
MOSES_FNAME(SyntaxRHS);
@ -239,6 +245,11 @@ FeatureRegistry::FeatureRegistry()
#ifdef LM_DALM
MOSES_FNAME2("DALM", LanguageModelDALM);
#endif
#ifdef LM_LBL
MOSES_FNAME2("LBLLM-LM", LBLLM<oxlm::LM>);
MOSES_FNAME2("LBLLM-FactoredLM", LBLLM<oxlm::FactoredLM>);
MOSES_FNAME2("LBLLM-FactoredMaxentLM", LBLLM<oxlm::FactoredMaxentLM>);
#endif
Add("KENLM", new KenFactory());
}
@ -267,12 +278,21 @@ void FeatureRegistry::Construct(const std::string &name, const std::string &line
void FeatureRegistry::PrintFF() const
{
vector<string> ffs;
std::cerr << "Available feature functions:" << std::endl;
Map::const_iterator iter;
for (iter = registry_.begin(); iter != registry_.end(); ++iter) {
const string &ffName = iter->first;
ffs.push_back(ffName);
}
vector<string>::const_iterator iterVec;
std::sort(ffs.begin(), ffs.end());
for (iterVec = ffs.begin(); iterVec != ffs.end(); ++iterVec) {
const string &ffName = *iterVec;
std::cerr << ffName << " ";
}
std::cerr << std::endl;
}

4
moses/FF/FeatureFunction.h
View File

@ -98,7 +98,7 @@ public:
// source phrase is the substring that the phrase table uses to look up the target phrase,
// may have more factors than actually need, but not guaranteed.
// For SCFG decoding, the source contains non-terminals, NOT the raw source from the input sentence
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const = 0;
@ -110,7 +110,7 @@ public:
// It is guaranteed to be in the same order as the non-terms in the source phrase.
// For pb models, stackvec is NULL.
// No FF should set estimatedFutureScore in both overloads!
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec

2
moses/FF/GlobalLexicalModel.cpp
View File

@ -165,7 +165,7 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP
return score;
}
void GlobalLexicalModel::Evaluate
void GlobalLexicalModel::EvaluateWhenApplied
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{

8
moses/FF/GlobalLexicalModel.h
View File

@ -70,24 +70,24 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(
void EvaluateWhenApplied(
const ChartHypothesis& hypo,
ScoreComponentCollection* accumulator) const {
throw std::logic_error("GlobalLexicalModel not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

2
moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -108,7 +108,7 @@ void GlobalLexicalModelUnlimited::InitializeForInput( Sentence const& in )
m_local->input = &in;
}
void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const
void GlobalLexicalModelUnlimited::EvaluateWhenApplied(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const
{
const Sentence& input = *(m_local->input);
const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase();

8
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -81,23 +81,23 @@ public:
//TODO: This implements the old interface, but cannot be updated because
//it appears to be stateful
void Evaluate(const Hypothesis& cur_hypo,
void EvaluateWhenApplied(const Hypothesis& cur_hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis& /* cur_hypo */,
void EvaluateWhenApplied(const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const {
throw std::logic_error("GlobalLexicalModelUnlimited not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/HyperParameterAsWeight.h
View File

@ -17,13 +17,13 @@ public:
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -31,14 +31,14 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartHypothesis &hypo,
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

2
moses/FF/InputFeature.cpp
View File

@ -44,7 +44,7 @@ void InputFeature::SetParameter(const std::string& key, const std::string& value
}
void InputFeature::Evaluate(const InputType &input
void InputFeature::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec

9
moses/FF/InputFeature.h
View File

@ -41,22 +41,23 @@ public:
return m_numRealWordCount;
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

4
moses/FF/InternalStructStatelessFF.cpp
View File

@ -5,7 +5,7 @@ using namespace std;
namespace Moses
{
void InternalStructStatelessFF::Evaluate(const Phrase &source
void InternalStructStatelessFF::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -15,7 +15,7 @@ void InternalStructStatelessFF::Evaluate(const Phrase &source
}
void InternalStructStatelessFF::Evaluate(const InputType &input
void InternalStructStatelessFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec

8
moses/FF/InternalStructStatelessFF.h
View File

@ -16,21 +16,21 @@ public:
bool IsUseable(const FactorMask &mask) const
{ return true; }
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

25
moses/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -14,11 +14,12 @@ LexicalReordering::LexicalReordering(const std::string &line)
{
std::cerr << "Initializing LexicalReordering.." << std::endl;
map<string,string> sparseArgs;
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "type") {
m_configuration = new LexicalReorderingConfiguration(args[1]);
m_configuration.reset(new LexicalReorderingConfiguration(args[1]));
m_configuration->SetScoreProducer(this);
m_modelTypeString = m_configuration->GetModelString();
} else if (args[0] == "input-factor") {
@ -27,8 +28,10 @@ LexicalReordering::LexicalReordering(const std::string &line)
m_factorsE =Tokenize<FactorType>(args[1]);
} else if (args[0] == "path") {
m_filePath = args[1];
} else if (args[0].substr(0,7) == "sparse-") {
sparseArgs[args[0].substr(7)] = args[1];
} else {
throw "Unknown argument " + args[0];
UTIL_THROW(util::Exception,"Unknown argument " + args[0]);
}
}
@ -36,29 +39,29 @@ LexicalReordering::LexicalReordering(const std::string &line)
case LexicalReorderingConfiguration::FE:
case LexicalReorderingConfiguration::E:
if(m_factorsE.empty()) {
throw "TL factor mask for lexical reordering is unexpectedly empty";
UTIL_THROW(util::Exception,"TL factor mask for lexical reordering is unexpectedly empty");
}
if(m_configuration->GetCondition() == LexicalReorderingConfiguration::E)
break; // else fall through
case LexicalReorderingConfiguration::F:
if(m_factorsF.empty()) {
throw "SL factor mask for lexical reordering is unexpectedly empty";
UTIL_THROW(util::Exception,"SL factor mask for lexical reordering is unexpectedly empty");
}
break;
default:
throw "Unknown conditioning option!";
UTIL_THROW(util::Exception,"Unknown conditioning option!");
}
m_configuration->ConfigureSparse(sparseArgs, this);
}
LexicalReordering::~LexicalReordering()
{
delete m_table;
delete m_configuration;
}
void LexicalReordering::Load()
{
m_table = LexicalReorderingTable::LoadAvailable(m_filePath, m_factorsF, m_factorsE, std::vector<FactorType>());
m_table.reset(LexicalReorderingTable::LoadAvailable(m_filePath, m_factorsF, m_factorsE, std::vector<FactorType>()));
}
Scores LexicalReordering::GetProb(const Phrase& f, const Phrase& e) const
@ -66,13 +69,13 @@ Scores LexicalReordering::GetProb(const Phrase& f, const Phrase& e) const
return m_table->GetScore(f, e, Phrase(ARRAY_SIZE_INCR));
}
FFState* LexicalReordering::Evaluate(const Hypothesis& hypo,
FFState* LexicalReordering::EvaluateWhenApplied(const Hypothesis& hypo,
const FFState* prev_state,
ScoreComponentCollection* out) const
{
Scores score(GetNumScoreComponents(), 0);
const LexicalReorderingState *prev = static_cast<const LexicalReorderingState *>(prev_state);
LexicalReorderingState *next_state = prev->Expand(hypo.GetTranslationOption(), score);
const LexicalReorderingState *prev = dynamic_cast<const LexicalReorderingState *>(prev_state);
LexicalReorderingState *next_state = prev->Expand(hypo.GetTranslationOption(), hypo.GetInput(), out);
out->PlusEquals(this, score);

19
moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -3,17 +3,20 @@
#include <string>
#include <vector>
#include <boost/scoped_ptr.hpp>
#include "moses/Factor.h"
#include "moses/Phrase.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "LexicalReorderingState.h"
#include "LexicalReorderingTable.h"
#include "moses/FF/StatefulFeatureFunction.h"
#include "util/exception.hh"
#include "LexicalReorderingState.h"
#include "LexicalReorderingTable.h"
#include "SparseReordering.h"
namespace Moses
{
@ -42,23 +45,23 @@ public:
Scores GetProb(const Phrase& f, const Phrase& e) const;
virtual FFState* Evaluate(const Hypothesis& cur_hypo,
virtual FFState* EvaluateWhenApplied(const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(const ChartHypothesis&,
virtual FFState* EvaluateWhenApplied(const ChartHypothesis&,
int /* featureID */,
ScoreComponentCollection*) const {
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -69,10 +72,10 @@ private:
bool DecodeDirection(std::string s);
bool DecodeNumFeatureFunctions(std::string s);
LexicalReorderingConfiguration *m_configuration;
boost::scoped_ptr<LexicalReorderingConfiguration> m_configuration;
std::string m_modelTypeString;
std::vector<std::string> m_modelType;
LexicalReorderingTable* m_table;
boost::scoped_ptr<LexicalReorderingTable> m_table;
//std::vector<Direction> m_direction;
std::vector<LexicalReorderingConfiguration::Condition> m_condition;
//std::vector<size_t> m_scoreOffset;

110
moses/FF/LexicalReordering/LexicalReorderingState.cpp
View File

@ -5,11 +5,11 @@
#include "moses/FF/FFState.h"
#include "moses/Hypothesis.h"
#include "moses/WordsRange.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "LexicalReordering.h"
#include "LexicalReorderingState.h"
#include "ReorderingStack.h"
namespace Moses
{
@ -38,6 +38,14 @@ size_t LexicalReorderingConfiguration::GetNumScoreComponents() const
}
}
void LexicalReorderingConfiguration::ConfigureSparse
(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer)
{
if (sparseArgs.size()) {
m_sparse.reset(new SparseReordering(sparseArgs, producer));
}
}
void LexicalReorderingConfiguration::SetAdditionalScoreComponents(size_t number)
{
m_additionalScoreComponents = number;
@ -122,52 +130,52 @@ LexicalReorderingState *LexicalReorderingConfiguration::CreateLexicalReorderingS
return new BidirectionalReorderingState(*this, bwd, fwd, 0);
}
void LexicalReorderingState::CopyScores(Scores& scores, const TranslationOption &topt, ReorderingType reoType) const
void LexicalReorderingState::CopyScores(ScoreComponentCollection* accum, const TranslationOption &topt, const InputType& input, ReorderingType reoType) const
{
// don't call this on a bidirectional object
UTIL_THROW_IF2(m_direction != LexicalReorderingConfiguration::Backward && m_direction != LexicalReorderingConfiguration::Forward,
"Unknown direction: " << m_direction);
const Scores *cachedScores = (m_direction == LexicalReorderingConfiguration::Backward) ?
topt.GetLexReorderingScores(m_configuration.GetScoreProducer()) : m_prevScore;
const TranslationOption* relevantOpt = &topt;
if (m_direction != LexicalReorderingConfiguration::Backward) relevantOpt = m_prevOption;
const Scores *cachedScores = relevantOpt->GetLexReorderingScores(m_configuration.GetScoreProducer());
// No scores available. TODO: Using a good prior distribution would be nicer.
if(cachedScores == NULL)
return;
if(cachedScores) {
Scores scores(m_configuration.GetScoreProducer()->GetNumScoreComponents(),0);
const Scores &scoreSet = *cachedScores;
if(m_configuration.CollapseScores())
scores[m_offset] = scoreSet[m_offset + reoType];
else {
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
scores[m_offset + reoType] = scoreSet[m_offset + reoType];
const Scores &scoreSet = *cachedScores;
if(m_configuration.CollapseScores())
scores[m_offset] = scoreSet[m_offset + reoType];
else {
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
scores[m_offset + reoType] = scoreSet[m_offset + reoType];
}
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
}
const SparseReordering* sparse = m_configuration.GetSparseReordering();
if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType, m_direction, accum);
}
void LexicalReorderingState::ClearScores(Scores& scores) const
{
if(m_configuration.CollapseScores())
scores[m_offset] = 0;
else
std::fill(scores.begin() + m_offset, scores.begin() + m_offset + m_configuration.GetNumberOfTypes(), 0);
}
int LexicalReorderingState::ComparePrevScores(const Scores *other) const
int LexicalReorderingState::ComparePrevScores(const TranslationOption *other) const
{
if(m_prevScore == other)
const Scores* myPrevScores = m_prevOption->GetLexReorderingScores(m_configuration.GetScoreProducer());
const Scores* otherPrevScores = other->GetLexReorderingScores(m_configuration.GetScoreProducer());
if(myPrevScores == otherPrevScores)
return 0;
// The pointers are NULL if a phrase pair isn't found in the reordering table.
if(other == NULL)
if(otherPrevScores == NULL)
return -1;
if(m_prevScore == NULL)
if(myPrevScores == NULL)
return 1;
const Scores &my = *m_prevScore;
const Scores &their = *other;
for(size_t i = m_offset; i < m_offset + m_configuration.GetNumberOfTypes(); i++)
if(my[i] < their[i])
if((*myPrevScores)[i] < (*otherPrevScores)[i])
return -1;
else if(my[i] > their[i])
else if((*myPrevScores)[i] > (*otherPrevScores)[i])
return 1;
return 0;
@ -192,7 +200,7 @@ int PhraseBasedReorderingState::Compare(const FFState& o) const
const PhraseBasedReorderingState* other = static_cast<const PhraseBasedReorderingState*>(&o);
if (m_prevRange == other->m_prevRange) {
if (m_direction == LexicalReorderingConfiguration::Forward) {
return ComparePrevScores(other->m_prevScore);
return ComparePrevScores(other->m_prevOption);
} else {
return 0;
}
@ -202,27 +210,23 @@ int PhraseBasedReorderingState::Compare(const FFState& o) const
return 1;
}
LexicalReorderingState* PhraseBasedReorderingState::Expand(const TranslationOption& topt, Scores& scores) const
LexicalReorderingState* PhraseBasedReorderingState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const
{
ReorderingType reoType;
const WordsRange currWordsRange = topt.GetSourceWordsRange();
const LexicalReorderingConfiguration::ModelType modelType = m_configuration.GetModelType();
if (m_direction == LexicalReorderingConfiguration::Forward && m_first) {
ClearScores(scores);
} else {
if (!m_first || m_useFirstBackwardScore) {
if (modelType == LexicalReorderingConfiguration::MSD) {
reoType = GetOrientationTypeMSD(currWordsRange);
} else if (modelType == LexicalReorderingConfiguration::MSLR) {
reoType = GetOrientationTypeMSLR(currWordsRange);
} else if (modelType == LexicalReorderingConfiguration::Monotonic) {
reoType = GetOrientationTypeMonotonic(currWordsRange);
} else {
reoType = GetOrientationTypeLeftRight(currWordsRange);
}
CopyScores(scores, topt, reoType);
if ((m_direction != LexicalReorderingConfiguration::Forward && m_useFirstBackwardScore) || !m_first) {
if (modelType == LexicalReorderingConfiguration::MSD) {
reoType = GetOrientationTypeMSD(currWordsRange);
} else if (modelType == LexicalReorderingConfiguration::MSLR) {
reoType = GetOrientationTypeMSLR(currWordsRange);
} else if (modelType == LexicalReorderingConfiguration::Monotonic) {
reoType = GetOrientationTypeMonotonic(currWordsRange);
} else {
reoType = GetOrientationTypeLeftRight(currWordsRange);
}
CopyScores(scores, topt, input, reoType);
}
return new PhraseBasedReorderingState(this, topt);
@ -300,10 +304,10 @@ int BidirectionalReorderingState::Compare(const FFState& o) const
return m_forward->Compare(*other.m_forward);
}
LexicalReorderingState* BidirectionalReorderingState::Expand(const TranslationOption& topt, Scores& scores) const
LexicalReorderingState* BidirectionalReorderingState::Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const
{
LexicalReorderingState *newbwd = m_backward->Expand(topt, scores);
LexicalReorderingState *newfwd = m_forward->Expand(topt, scores);
LexicalReorderingState *newbwd = m_backward->Expand(topt,input, scores);
LexicalReorderingState *newfwd = m_forward->Expand(topt, input, scores);
return new BidirectionalReorderingState(m_configuration, newbwd, newfwd, m_offset);
}
@ -324,7 +328,7 @@ int HierarchicalReorderingBackwardState::Compare(const FFState& o) const
return m_reoStack.Compare(other.m_reoStack);
}
LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const TranslationOption& topt, Scores& scores) const
LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const
{
HierarchicalReorderingBackwardState* nextState = new HierarchicalReorderingBackwardState(this, topt, m_reoStack);
@ -343,7 +347,7 @@ LexicalReorderingState* HierarchicalReorderingBackwardState::Expand(const Transl
reoType = GetOrientationTypeMonotonic(reoDistance);
}
CopyScores(scores, topt, reoType);
CopyScores(scores, topt, input, reoType);
return nextState;
}
@ -409,7 +413,7 @@ int HierarchicalReorderingForwardState::Compare(const FFState& o) const
const HierarchicalReorderingForwardState* other = static_cast<const HierarchicalReorderingForwardState*>(&o);
if (m_prevRange == other->m_prevRange) {
return ComparePrevScores(other->m_prevScore);
return ComparePrevScores(other->m_prevOption);
} else if (m_prevRange < other->m_prevRange) {
return -1;
}
@ -427,7 +431,7 @@ int HierarchicalReorderingForwardState::Compare(const FFState& o) const
// dright: if the next phrase follows the conditioning phrase and other stuff comes in between
// dleft: if the next phrase precedes the conditioning phrase and other stuff comes in between
LexicalReorderingState* HierarchicalReorderingForwardState::Expand(const TranslationOption& topt, Scores& scores) const
LexicalReorderingState* HierarchicalReorderingForwardState::Expand(const TranslationOption& topt, const InputType& input,ScoreComponentCollection* scores) const
{
const LexicalReorderingConfiguration::ModelType modelType = m_configuration.GetModelType();
const WordsRange currWordsRange = topt.GetSourceWordsRange();
@ -438,7 +442,7 @@ LexicalReorderingState* HierarchicalReorderingForwardState::Expand(const Transla
ReorderingType reoType;
if (m_first) {
ClearScores(scores);
} else {
if (modelType == LexicalReorderingConfiguration::MSD) {
reoType = GetOrientationTypeMSD(currWordsRange, coverage);
@ -450,7 +454,7 @@ LexicalReorderingState* HierarchicalReorderingForwardState::Expand(const Transla
reoType = GetOrientationTypeLeftRight(currWordsRange, coverage);
}
CopyScores(scores, topt, reoType);
CopyScores(scores, topt, input, reoType);
}
return new HierarchicalReorderingForwardState(this, topt);

42
moses/FF/LexicalReordering/LexicalReorderingState.h
View File

@ -4,22 +4,25 @@
#include <vector>
#include <string>
#include <boost/scoped_ptr.hpp>
#include "moses/Hypothesis.h"
#include "LexicalReordering.h"
//#include "LexicalReordering.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/WordsRange.h"
#include "moses/WordsBitmap.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "moses/FF/FFState.h"
#include "ReorderingStack.h"
namespace Moses
{
class LexicalReorderingState;
class LexicalReordering;
class SparseReordering;
/** Factory class for lexical reordering states
* @todo There's a lot of classes for lexicalized reordering. Perhaps put them in a separate dir
*/
class LexicalReorderingConfiguration
{
@ -31,6 +34,8 @@ public:
LexicalReorderingConfiguration(const std::string &modelType);
void ConfigureSparse(const std::map<std::string,std::string>& sparseArgs, const LexicalReordering* producer);
LexicalReorderingState *CreateLexicalReorderingState(const InputType &input) const;
size_t GetNumScoreComponents() const;
@ -62,6 +67,10 @@ public:
return m_collapseScores;
}
const SparseReordering* GetSparseReordering() const {
return m_sparse.get();
}
private:
void SetScoreProducer(LexicalReordering* scoreProducer) {
m_scoreProducer = scoreProducer;
@ -79,6 +88,7 @@ private:
Direction m_direction;
Condition m_condition;
size_t m_additionalScoreComponents;
boost::scoped_ptr<SparseReordering> m_sparse;
};
//! Abstract class for lexical reordering model states
@ -86,34 +96,35 @@ class LexicalReorderingState : public FFState
{
public:
virtual int Compare(const FFState& o) const = 0;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, Scores& scores) const = 0;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const = 0;
static LexicalReorderingState* CreateLexicalReorderingState(const std::vector<std::string>& config,
LexicalReorderingConfiguration::Direction dir, const InputType &input);
typedef int ReorderingType;
protected:
typedef int ReorderingType;
const LexicalReorderingConfiguration &m_configuration;
// The following is the true direction of the object, which can be Backward or Forward even if the Configuration has Bidirectional.
LexicalReorderingConfiguration::Direction m_direction;
size_t m_offset;
const Scores *m_prevScore;
//forward scores are conditioned on prev option, so need to remember it
const TranslationOption *m_prevOption;
inline LexicalReorderingState(const LexicalReorderingState *prev, const TranslationOption &topt) :
m_configuration(prev->m_configuration), m_direction(prev->m_direction), m_offset(prev->m_offset),
m_prevScore(topt.GetLexReorderingScores(m_configuration.GetScoreProducer())) {}
m_prevOption(&topt) {}
inline LexicalReorderingState(const LexicalReorderingConfiguration &config, LexicalReorderingConfiguration::Direction dir, size_t offset)
: m_configuration(config), m_direction(dir), m_offset(offset), m_prevScore(NULL) {}
: m_configuration(config), m_direction(dir), m_offset(offset), m_prevOption(NULL) {}
// copy the right scores in the right places, taking into account forward/backward, offset, collapse
void CopyScores(Scores& scores, const TranslationOption& topt, ReorderingType reoType) const;
void ClearScores(Scores& scores) const;
int ComparePrevScores(const Scores *other) const;
void CopyScores(ScoreComponentCollection* scores, const TranslationOption& topt, const InputType& input, ReorderingType reoType) const;
int ComparePrevScores(const TranslationOption *other) const;
//constants for the different type of reorderings (corresponding to indexes in the table file)
public:
static const ReorderingType M = 0; // monotonic
static const ReorderingType NM = 1; // non-monotonic
static const ReorderingType S = 1; // swap
@ -122,6 +133,7 @@ protected:
static const ReorderingType DR = 3; // discontinuous, right
static const ReorderingType R = 0; // right
static const ReorderingType L = 1; // left
static const ReorderingType MAX = 3; //largest possible
};
//! @todo what is this?
@ -140,7 +152,7 @@ public:
}
virtual int Compare(const FFState& o) const;
virtual LexicalReorderingState* Expand(const TranslationOption& topt, Scores& scores) const;
virtual LexicalReorderingState* Expand(const TranslationOption& topt, const InputType& input, ScoreComponentCollection* scores) const;
};
//! State for the standard Moses implementation of lexical reordering models
@ -156,7 +168,7 @@ public:
PhraseBasedReorderingState(const PhraseBasedReorderingState *prev, const TranslationOption &topt);
virtual int Compare(const FFState& o) const;
virtual LexicalReorderingState* Expand(const TranslationOption& topt, Scores& scores) const;
virtual LexicalReorderingState* Expand(const TranslationOption& topt,const InputType& input, ScoreComponentCollection* scores) const;
ReorderingType GetOrientationTypeMSD(WordsRange currRange) const;
ReorderingType GetOrientationTypeMSLR(WordsRange currRange) const;
@ -177,7 +189,7 @@ public:
const TranslationOption &topt, ReorderingStack reoStack);
virtual int Compare(const FFState& o) const;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, Scores& scores) const;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const;
private:
ReorderingType GetOrientationTypeMSD(int reoDistance) const;
@ -200,7 +212,7 @@ public:
HierarchicalReorderingForwardState(const HierarchicalReorderingForwardState *prev, const TranslationOption &topt);
virtual int Compare(const FFState& o) const;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, Scores& scores) const;
virtual LexicalReorderingState* Expand(const TranslationOption& hypo, const InputType& input, ScoreComponentCollection* scores) const;
private:
ReorderingType GetOrientationTypeMSD(WordsRange currRange, WordsBitmap coverage) const;

0
moses/ReorderingStack.cpp → moses/FF/LexicalReordering/ReorderingStack.cpp
View File

2
moses/ReorderingStack.h → moses/FF/LexicalReordering/ReorderingStack.h
View File

@ -12,7 +12,7 @@
//#include "Phrase.h"
//#include "TypeDef.h"
//#include "Util.h"
#include "WordsRange.h"
#include "moses/WordsRange.h"
namespace Moses
{

252
moses/FF/LexicalReordering/SparseReordering.cpp Normal file
View File

@ -0,0 +1,252 @@
#include <fstream>
#include "moses/FactorCollection.h"
#include "moses/InputPath.h"
#include "moses/Util.h"
#include "util/exception.hh"
#include "util/file_piece.hh"
#include "util/string_piece.hh"
#include "util/tokenize_piece.hh"
#include "LexicalReordering.h"
#include "SparseReordering.h"
using namespace std;
namespace Moses
{
const std::string& SparseReorderingFeatureKey::Name(const string& wordListId) {
static string kSep = "-";
static string name;
ostringstream buf;
// type side position id word reotype
if (type == Phrase) {
buf << "phr";
} else if (type == Stack) {
buf << "stk";
} else if (type == Between) {
buf << "btn";
}
buf << kSep;
if (side == Source) {
buf << "src";
} else if (side == Target) {
buf << "tgt";
}
buf << kSep;
if (position == First) {
buf << "first";
} else if (position == Last) {
buf << "last";
}
buf << kSep;
buf << wordListId;
buf << kSep;
if (isCluster) buf << "cluster_";
buf << word->GetString();
buf << kSep;
buf << reoType;
name = buf.str();
return name;
}
SparseReordering::SparseReordering(const map<string,string>& config, const LexicalReordering* producer)
: m_producer(producer)
{
static const string kSource= "source";
static const string kTarget = "target";
for (map<string,string>::const_iterator i = config.begin(); i != config.end(); ++i) {
vector<string> fields = Tokenize(i->first, "-");
if (fields[0] == "words") {
UTIL_THROW_IF(!(fields.size() == 3), util::Exception, "Sparse reordering word list name should be sparse-words-(source|target)-<id>");
if (fields[1] == kSource) {
ReadWordList(i->second,fields[2], SparseReorderingFeatureKey::Source, &m_sourceWordLists);
} else if (fields[1] == kTarget) {
ReadWordList(i->second,fields[2],SparseReorderingFeatureKey::Target, &m_targetWordLists);
} else {
UTIL_THROW(util::Exception, "Sparse reordering requires source or target, not " << fields[1]);
}
} else if (fields[0] == "clusters") {
UTIL_THROW_IF(!(fields.size() == 3), util::Exception, "Sparse reordering cluster name should be sparse-clusters-(source|target)-<id>");
if (fields[1] == kSource) {
ReadClusterMap(i->second,fields[2], SparseReorderingFeatureKey::Source, &m_sourceClusterMaps);
} else if (fields[1] == kTarget) {
ReadClusterMap(i->second,fields[2],SparseReorderingFeatureKey::Target, &m_targetClusterMaps);
} else {
UTIL_THROW(util::Exception, "Sparse reordering requires source or target, not " << fields[1]);
}
} else if (fields[0] == "phrase") {
m_usePhrase = true;
} else if (fields[0] == "stack") {
m_useStack = true;
} else if (fields[0] == "between") {
m_useBetween = true;
} else {
UTIL_THROW(util::Exception, "Unable to parse sparse reordering option: " << i->first);
}
}
}
void SparseReordering::PreCalculateFeatureNames(size_t index, const string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster) {
for (size_t type = SparseReorderingFeatureKey::Stack;
type <= SparseReorderingFeatureKey::Between; ++type) {
for (size_t position = SparseReorderingFeatureKey::First;
position <= SparseReorderingFeatureKey::Last; ++position) {
for (int reoType = 0; reoType <= LexicalReorderingState::MAX; ++reoType) {
SparseReorderingFeatureKey key(
index, static_cast<SparseReorderingFeatureKey::Type>(type), factor, isCluster,
static_cast<SparseReorderingFeatureKey::Position>(position), side, reoType);
m_featureMap[key] = key.Name(id);
}
}
}
}
void SparseReordering::ReadWordList(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<WordList>* pWordLists) {
ifstream fh(filename.c_str());
UTIL_THROW_IF(!fh, util::Exception, "Unable to open: " << filename);
string line;
pWordLists->push_back(WordList());
pWordLists->back().first = id;
while (getline(fh,line)) {
//TODO: StringPiece
const Factor* factor = FactorCollection::Instance().AddFactor(line);
pWordLists->back().second.insert(factor);
PreCalculateFeatureNames(pWordLists->size()-1, id, side, factor, false);
}
}
void SparseReordering::ReadClusterMap(const string& filename, const string& id, SparseReorderingFeatureKey::Side side, vector<ClusterMap>* pClusterMaps) {
pClusterMaps->push_back(ClusterMap());
pClusterMaps->back().first = id;
util::FilePiece file(filename.c_str());
StringPiece line;
while (true) {
try {
line = file.ReadLine();
} catch (const util::EndOfFileException &e) {
break;
}
util::TokenIter<util::SingleCharacter, true> lineIter(line,util::SingleCharacter('\t'));
if (!lineIter) UTIL_THROW(util::Exception, "Malformed cluster line (missing word): '" << line << "'");
const Factor* wordFactor = FactorCollection::Instance().AddFactor(*lineIter);
++lineIter;
if (!lineIter) UTIL_THROW(util::Exception, "Malformed cluster line (missing cluster id): '" << line << "'");
const Factor* idFactor = FactorCollection::Instance().AddFactor(*lineIter);
pClusterMaps->back().second[wordFactor] = idFactor;
PreCalculateFeatureNames(pClusterMaps->size()-1, id, side, idFactor, true);
}
}
void SparseReordering::AddFeatures(
SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side,
const Word& word, SparseReorderingFeatureKey::Position position,
LexicalReorderingState::ReorderingType reoType,
ScoreComponentCollection* scores) const {
const Factor* wordFactor = word.GetFactor(0);
const vector<WordList>* wordLists;
const vector<ClusterMap>* clusterMaps;
if (side == SparseReorderingFeatureKey::Source) {
wordLists = &m_sourceWordLists;
clusterMaps = &m_sourceClusterMaps;
} else {
wordLists = &m_targetWordLists;
clusterMaps = &m_targetClusterMaps;
}
for (size_t id = 0; id < wordLists->size(); ++id) {
if ((*wordLists)[id].second.find(wordFactor) == (*wordLists)[id].second.end()) continue;
SparseReorderingFeatureKey key(id, type, wordFactor, false, position, side, reoType);
FeatureMap::const_iterator fmi = m_featureMap.find(key);
assert(fmi != m_featureMap.end());
scores->PlusEquals(m_producer, fmi->second, 1.0);
}
for (size_t id = 0; id < clusterMaps->size(); ++id) {
const ClusterMap& clusterMap = (*clusterMaps)[id];
boost::unordered_map<const Factor*, const Factor*>::const_iterator clusterIter
= clusterMap.second.find(wordFactor);
if (clusterIter != clusterMap.second.end()) {
SparseReorderingFeatureKey key(id, type, clusterIter->second, true, position, side, reoType);
FeatureMap::const_iterator fmi = m_featureMap.find(key);
assert(fmi != m_featureMap.end());
scores->PlusEquals(m_producer, fmi->second, 1.0);
}
}
}
void SparseReordering::CopyScores(
const TranslationOption& currentOpt,
const TranslationOption* previousOpt,
const InputType& input,
LexicalReorderingState::ReorderingType reoType,
LexicalReorderingConfiguration::Direction direction,
ScoreComponentCollection* scores) const
{
if (m_useBetween && direction == LexicalReorderingConfiguration::Backward &&
(reoType == LexicalReorderingState::D || reoType == LexicalReorderingState::DL ||
reoType == LexicalReorderingState::DR)) {
size_t gapStart, gapEnd;
const Sentence& sentence = dynamic_cast<const Sentence&>(input);
const WordsRange& currentRange = currentOpt.GetSourceWordsRange();
if (previousOpt) {
const WordsRange& previousRange = previousOpt->GetSourceWordsRange();
if (previousRange < currentRange) {
gapStart = previousRange.GetEndPos() + 1;
gapEnd = currentRange.GetStartPos();
} else {
gapStart = currentRange.GetEndPos() + 1;
gapEnd = previousRange.GetStartPos();
}
} else {
//start of sentence
gapStart = 0;
gapEnd = currentRange.GetStartPos();
}
assert(gapStart < gapEnd);
for (size_t i = gapStart; i < gapEnd; ++i) {
AddFeatures(SparseReorderingFeatureKey::Between,
SparseReorderingFeatureKey::Source, sentence.GetWord(i),
SparseReorderingFeatureKey::First, reoType, scores);
}
}
//std::cerr << "SR " << topt << " " << reoType << " " << direction << std::endl;
//phrase (backward)
//stack (forward)
SparseReorderingFeatureKey::Type type;
if (direction == LexicalReorderingConfiguration::Forward) {
if (!m_useStack) return;
type = SparseReorderingFeatureKey::Stack;
} else if (direction == LexicalReorderingConfiguration::Backward) {
if (!m_usePhrase) return;
type = SparseReorderingFeatureKey::Phrase;
} else {
//Shouldn't be called for bidirectional
//keep compiler happy
type = SparseReorderingFeatureKey::Phrase;
assert(!"Shouldn't call CopyScores() with bidirectional direction");
}
const Phrase& sourcePhrase = currentOpt.GetInputPath().GetPhrase();
AddFeatures(type, SparseReorderingFeatureKey::Source, sourcePhrase.GetWord(0),
SparseReorderingFeatureKey::First, reoType, scores);
AddFeatures(type, SparseReorderingFeatureKey::Source, sourcePhrase.GetWord(sourcePhrase.GetSize()-1), SparseReorderingFeatureKey::Last, reoType, scores);
const Phrase& targetPhrase = currentOpt.GetTargetPhrase();
AddFeatures(type, SparseReorderingFeatureKey::Target, targetPhrase.GetWord(0),
SparseReorderingFeatureKey::First, reoType, scores);
AddFeatures(type, SparseReorderingFeatureKey::Target, targetPhrase.GetWord(targetPhrase.GetSize()-1), SparseReorderingFeatureKey::Last, reoType, scores);
}
} //namespace

132
moses/FF/LexicalReordering/SparseReordering.h Normal file
View File

@ -0,0 +1,132 @@
#ifndef moses_FF_LexicalReordering_SparseReordering_h
#define moses_FF_LexicalReordering_SparseReordering_h
/**
* Sparse reordering features for phrase-based MT, following Cherry (NAACL, 2013)
**/
#include <functional>
#include <map>
#include <string>
#include <vector>
#include <boost/unordered_set.hpp>
#include "util/murmur_hash.hh"
#include "util/pool.hh"
#include "util/string_piece.hh"
#include "moses/ScoreComponentCollection.h"
#include "LexicalReorderingState.h"
/**
Configuration of sparse reordering:
The sparse reordering feature is configured using sparse-* configs in the lexical reordering line.
sparse-words-(source|target)-<id>=<filename> -- Features which fire for the words in the list
sparse-clusters-(source|target)-<id>=<filename> -- Features which fire for clusters in the list. Format
of cluster file TBD
sparse-phrase -- Add features which depend on the current phrase (backward)
sparse-stack -- Add features which depend on the previous phrase, or
top of stack. (forward)
sparse-between -- Add features which depend on words between previous phrase
(or top of stack) and current phrase.
**/
namespace Moses
{
/**
* Used to store pre-calculated feature names.
**/
struct SparseReorderingFeatureKey {
size_t id;
enum Type {Stack, Phrase, Between} type;
const Factor* word;
bool isCluster;
enum Position {First, Last} position;
enum Side {Source, Target} side;
LexicalReorderingState::ReorderingType reoType;
SparseReorderingFeatureKey(size_t id_, Type type_, const Factor* word_, bool isCluster_,
Position position_, Side side_, LexicalReorderingState::ReorderingType reoType_)
: id(id_), type(type_), word(word_), isCluster(isCluster_),
position(position_), side(side_), reoType(reoType_)
{}
const std::string& Name(const std::string& wordListId) ;
};
struct HashSparseReorderingFeatureKey : public std::unary_function<SparseReorderingFeatureKey, std::size_t> {
std::size_t operator()(const SparseReorderingFeatureKey& key) const {
//TODO: can we just hash the memory?
//not sure, there could be random padding
std::size_t seed = 0;
seed = util::MurmurHashNative(&key.id, sizeof(key.id), seed);
seed = util::MurmurHashNative(&key.type, sizeof(key.type), seed);
seed = util::MurmurHashNative(&key.word, sizeof(key.word), seed);
seed = util::MurmurHashNative(&key.isCluster, sizeof(key.isCluster), seed);
seed = util::MurmurHashNative(&key.position, sizeof(key.position), seed);
seed = util::MurmurHashNative(&key.side, sizeof(key.side), seed);
seed = util::MurmurHashNative(&key.reoType, sizeof(key.reoType), seed);
return seed;
}
};
struct EqualsSparseReorderingFeatureKey :
public std::binary_function<SparseReorderingFeatureKey, SparseReorderingFeatureKey, bool> {
bool operator()(const SparseReorderingFeatureKey& left, const SparseReorderingFeatureKey& right) const {
//TODO: Can we just compare the memory?
return left.id == right.id && left.type == right.type && left.word == right.word &&
left.position == right.position && left.side == right.side &&
left.reoType == right.reoType;
}
};
class SparseReordering
{
public:
SparseReordering(const std::map<std::string,std::string>& config, const LexicalReordering* producer);
//If direction is backward the options will be different, for forward they will be the same
void CopyScores(const TranslationOption& currentOpt,
const TranslationOption* previousOpt,
const InputType& input,
LexicalReorderingState::ReorderingType reoType,
LexicalReorderingConfiguration::Direction direction,
ScoreComponentCollection* scores) const ;
private:
const LexicalReordering* m_producer;
typedef std::pair<std::string, boost::unordered_set<const Factor*> > WordList; //id and list
std::vector<WordList> m_sourceWordLists;
std::vector<WordList> m_targetWordLists;
typedef std::pair<std::string, boost::unordered_map<const Factor*, const Factor*> > ClusterMap; //id and map
std::vector<ClusterMap> m_sourceClusterMaps;
std::vector<ClusterMap> m_targetClusterMaps;
bool m_usePhrase;
bool m_useBetween;
bool m_useStack;
typedef boost::unordered_map<SparseReorderingFeatureKey, std::string, HashSparseReorderingFeatureKey, EqualsSparseReorderingFeatureKey> FeatureMap;
FeatureMap m_featureMap;
void ReadWordList(const std::string& filename, const std::string& id,
SparseReorderingFeatureKey::Side side, std::vector<WordList>* pWordLists);
void ReadClusterMap(const std::string& filename, const std::string& id, SparseReorderingFeatureKey::Side side, std::vector<ClusterMap>* pClusterMaps);
void PreCalculateFeatureNames(size_t index, const std::string& id, SparseReorderingFeatureKey::Side side, const Factor* factor, bool isCluster);
void AddFeatures(
SparseReorderingFeatureKey::Type type, SparseReorderingFeatureKey::Side side,
const Word& word, SparseReorderingFeatureKey::Position position,
LexicalReorderingState::ReorderingType reoType,
ScoreComponentCollection* scores) const;
};
} //namespace
#endif

4
moses/FF/MaxSpanFreeNonTermSource.cpp
View File

@ -27,7 +27,7 @@ MaxSpanFreeNonTermSource::MaxSpanFreeNonTermSource(const std::string &line)
m_glueTargetLHS.SetFactor(0, factor);
}
void MaxSpanFreeNonTermSource::Evaluate(const Phrase &source
void MaxSpanFreeNonTermSource::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -35,7 +35,7 @@ void MaxSpanFreeNonTermSource::Evaluate(const Phrase &source
targetPhrase.SetRuleSource(source);
}
void MaxSpanFreeNonTermSource::Evaluate(const InputType &input
void MaxSpanFreeNonTermSource::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec

8
moses/FF/MaxSpanFreeNonTermSource.h
View File

@ -15,23 +15,23 @@ public:
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
virtual void EvaluateChart(const ChartHypothesis &hypo,
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

8
moses/FF/NieceTerminal.cpp
View File

@ -25,7 +25,7 @@ std::vector<float> NieceTerminal::DefaultWeights() const
return ret;
}
void NieceTerminal::Evaluate(const Phrase &source
void NieceTerminal::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -33,7 +33,7 @@ void NieceTerminal::Evaluate(const Phrase &source
targetPhrase.SetRuleSource(source);
}
void NieceTerminal::Evaluate(const InputType &input
void NieceTerminal::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -71,11 +71,11 @@ void NieceTerminal::Evaluate(const InputType &input
}
void NieceTerminal::Evaluate(const Hypothesis& hypo,
void NieceTerminal::EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void NieceTerminal::EvaluateChart(const ChartHypothesis &hypo,
void NieceTerminal::EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

8
moses/FF/NieceTerminal.h
View File

@ -19,19 +19,19 @@ public:
return true;
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis &hypo,
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
void SetParameter(const std::string& key, const std::string& value);

6
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -42,7 +42,7 @@ void OpSequenceModel::Load()
void OpSequenceModel:: Evaluate(const Phrase &source
void OpSequenceModel:: EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -87,7 +87,7 @@ void OpSequenceModel:: Evaluate(const Phrase &source
}
FFState* OpSequenceModel::Evaluate(
FFState* OpSequenceModel::EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -194,7 +194,7 @@ FFState* OpSequenceModel::Evaluate(
// return NULL;
}
FFState* OpSequenceModel::EvaluateChart(
FFState* OpSequenceModel::EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -29,24 +29,24 @@ public:
void readLanguageModel(const char *);
void Load();
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(
virtual FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;

2
moses/FF/PhraseBoundaryFeature.cpp
View File

@ -66,7 +66,7 @@ void PhraseBoundaryFeature::AddFeatures(
}
FFState* PhraseBoundaryFeature::Evaluate
FFState* PhraseBoundaryFeature::EvaluateWhenApplied
(const Hypothesis& cur_hypo, const FFState* prev_state,
ScoreComponentCollection* scores) const
{

8
moses/FF/PhraseBoundaryFeature.h
View File

@ -44,23 +44,23 @@ public:
virtual const FFState* EmptyHypothesisState(const InputType &) const;
virtual FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state,
virtual FFState* EvaluateWhenApplied(const Hypothesis& cur_hypo, const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */,
virtual FFState* EvaluateWhenApplied( const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const {
throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

2
moses/FF/PhraseLengthFeature.cpp
View File

@ -15,7 +15,7 @@ PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
ReadParameters();
}
void PhraseLengthFeature::Evaluate(const Phrase &source
void PhraseLengthFeature::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/PhraseLengthFeature.h
View File

@ -24,16 +24,16 @@ public:
return true;
}
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis& hypo,
void EvaluateWhenApplied(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -41,7 +41,7 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;

2
moses/FF/PhrasePairFeature.cpp
View File

@ -106,7 +106,7 @@ void PhrasePairFeature::Load()
}
}
void PhrasePairFeature::Evaluate(
void PhrasePairFeature::EvaluateWhenApplied(
const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{

8
moses/FF/PhrasePairFeature.h
View File

@ -37,22 +37,22 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartHypothesis& hypo,
void EvaluateWhenApplied(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

2
moses/FF/PhrasePenalty.cpp
View File

@ -10,7 +10,7 @@ PhrasePenalty::PhrasePenalty(const std::string &line)
ReadParameters();
}
void PhrasePenalty::Evaluate(const Phrase &source
void PhrasePenalty::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/PhrasePenalty.h
View File

@ -14,19 +14,19 @@ public:
return true;
}
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec

8
moses/FF/ReferenceComparison.h
View File

@ -15,13 +15,13 @@ public:
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -29,11 +29,11 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
virtual void EvaluateChart(const ChartHypothesis &hypo,
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

2
moses/FF/RuleScope.cpp
View File

@ -16,7 +16,7 @@ bool IsAmbiguous(const Word &word, bool sourceSyntax)
return word.IsNonTerminal() && (!sourceSyntax || word == inputDefaultNonTerminal);
}
void RuleScope::Evaluate(const Phrase &source
void RuleScope::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/RuleScope.h
View File

@ -14,12 +14,12 @@ public:
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -27,11 +27,11 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
virtual void EvaluateChart(const ChartHypothesis &hypo,
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

2
moses/FF/SetSourcePhrase.cpp
View File

@ -10,7 +10,7 @@ SetSourcePhrase::SetSourcePhrase(const std::string &line)
ReadParameters();
}
void SetSourcePhrase::Evaluate(const Phrase &source
void SetSourcePhrase::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const

8
moses/FF/SetSourcePhrase.h
View File

@ -14,12 +14,12 @@ public:
virtual bool IsUseable(const FactorMask &mask) const
{ return true; }
virtual void Evaluate(const Phrase &source
virtual void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual void Evaluate(const InputType &input
virtual void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -27,11 +27,11 @@ public:
, ScoreComponentCollection *estimatedFutureScore = NULL) const
{}
virtual void Evaluate(const Hypothesis& hypo,
virtual void EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
virtual void EvaluateChart(const ChartHypothesis &hypo,
virtual void EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

8
moses/FF/SkeletonStatefulFF.cpp
View File

@ -23,13 +23,13 @@ SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
ReadParameters();
}
void SkeletonStatefulFF::Evaluate(const Phrase &source
void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SkeletonStatefulFF::Evaluate(const InputType &input
void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -37,7 +37,7 @@ void SkeletonStatefulFF::Evaluate(const InputType &input
, ScoreComponentCollection *estimatedFutureScore) const
{}
FFState* SkeletonStatefulFF::Evaluate(
FFState* SkeletonStatefulFF::EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -56,7 +56,7 @@ FFState* SkeletonStatefulFF::Evaluate(
return new SkeletonState(0);
}
FFState* SkeletonStatefulFF::EvaluateChart(
FFState* SkeletonStatefulFF::EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const

8
moses/FF/SkeletonStatefulFF.h
View File

@ -30,21 +30,21 @@ public:
return new SkeletonState(0);
}
void Evaluate(const Phrase &source
void EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
void EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection *estimatedFutureScore = NULL) const;
FFState* Evaluate(
FFState* EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
FFState* EvaluateChart(
FFState* EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;

8
moses/FF/SkeletonStatelessFF.cpp
View File

@ -13,7 +13,7 @@ SkeletonStatelessFF::SkeletonStatelessFF(const std::string &line)
ReadParameters();
}
void SkeletonStatelessFF::Evaluate(const Phrase &source
void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
@ -29,7 +29,7 @@ void SkeletonStatelessFF::Evaluate(const Phrase &source
}
void SkeletonStatelessFF::Evaluate(const InputType &input
void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -44,11 +44,11 @@ void SkeletonStatelessFF::Evaluate(const InputType &input
}
void SkeletonStatelessFF::Evaluate(const Hypothesis& hypo,
void SkeletonStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void SkeletonStatelessFF::EvaluateChart(const ChartHypothesis &hypo,
void SkeletonStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}

Some files were not shown because too many files have changed in this diff Show More