moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-28 14:32:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Ian Johnson 2013-05-28 13:54:35 +01:00
commit 219d51754c
153 changed files with 1514 additions and 2403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
OnDiskPt/TargetPhrase.cpp
View File

@ -23,7 +23,6 @@
#include "moses/Util.h"
#include "moses/TargetPhrase.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/DummyScoreProducers.h"
#include "TargetPhrase.h"
#include "OnDiskWrapper.h"
@ -241,10 +240,6 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord());
}
// scores
ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);
ret->Evaluate();
// alignments
int index = 0;
Moses::AlignmentInfo::CollType alignTerm, alignNonTerm;
@ -270,14 +265,18 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
Moses::Word *lhsTarget = new Moses::Word(true);
GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
ret->SetTargetLHS(lhsTarget);
// set source phrase
Moses::Phrase mosesSP(Moses::Input);
for (size_t pos = 0; pos < sp->GetSize(); ++pos) {
sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord());
}
ret->SetSourcePhrase(mosesSP);
// scores
ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);
ret->Evaluate(mosesSP);
return ret;
}

5
contrib/other-builds/moses/.cproject
View File

@ -68,11 +68,8 @@
</tool>
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1495793082" name="SRI.h" rcbsApplicability="disable" resourcePath="LM/SRI.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1830045369" name="Backward.h" rcbsApplicability="disable" resourcePath="LM/Backward.h" toolsToInvoke=""/>
<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.656913512.1283955357" name="BackwardLMState.h" rcbsApplicability="disable" resourcePath="LM/BackwardLMState.h" toolsToInvoke=""/>
<sourceEntries>
<entry excluding="PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/Rand.h|LM/Rand.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
<entry excluding="FF/PhraseLengthFeatureTest.cpp|PhraseLengthFeatureTest.cpp|LM/BackwardTest.cpp|LM/BackwardLMState.h|LM/BackwardLMState.cpp|LM/Backward.h|LM/Backward.cpp|FeatureVectorTest.cpp|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|src/SyntacticLanguageModelState.h|src/SyntacticLanguageModelFiles.h|src/SyntacticLanguageModel.h|src/SyntacticLanguageModel.cpp|src/LM/SRI.h|src/LM/SRI.cpp|src/LM/Rand.h|src/LM/Rand.cpp|src/LM/LDHT.h|src/LM/LDHT.cpp|SyntacticLanguageModelState.h|SyntacticLanguageModelFiles.h|SyntacticLanguageModel.h|SyntacticLanguageModel.cpp|LM/Rand.h|LM/Rand.cpp|LM/LDHT.h|LM/LDHT.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>
</configuration>
</storageModule>

570
contrib/other-builds/moses/.project
View File

@ -116,16 +116,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/BitmapContainer.h</locationURI>
</link>
<link>
<name>BleuScoreFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/BleuScoreFeature.cpp</locationURI>
</link>
<link>
<name>BleuScoreFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/BleuScoreFeature.h</locationURI>
</link>
<link>
<name>ChartCell.cpp</name>
<type>1</type>
@ -332,24 +322,9 @@
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeStepTranslation.h</locationURI>
</link>
<link>
<name>DummyScoreProducers.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DummyScoreProducers.cpp</locationURI>
</link>
<link>
<name>DummyScoreProducers.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DummyScoreProducers.h</locationURI>
</link>
<link>
<name>FFState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FFState.cpp</locationURI>
</link>
<link>
<name>FFState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FFState.h</locationURI>
<name>FF</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>Factor.cpp</name>
@ -381,16 +356,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FactorTypeSet.h</locationURI>
</link>
<link>
<name>FeatureFunction.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FeatureFunction.cpp</locationURI>
</link>
<link>
<name>FeatureFunction.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FeatureFunction.h</locationURI>
</link>
<link>
<name>FeatureVector.cpp</name>
<type>1</type>
@ -441,26 +406,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/GenerationDictionary.h</locationURI>
</link>
<link>
<name>GlobalLexicalModel.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/GlobalLexicalModel.cpp</locationURI>
</link>
<link>
<name>GlobalLexicalModel.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/GlobalLexicalModel.h</locationURI>
</link>
<link>
<name>GlobalLexicalModelUnlimited.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/GlobalLexicalModelUnlimited.cpp</locationURI>
</link>
<link>
<name>GlobalLexicalModelUnlimited.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/GlobalLexicalModelUnlimited.h</locationURI>
</link>
<link>
<name>HypoList.h</name>
<type>1</type>
@ -546,11 +491,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LMList.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LMList.h</locationURI>
</link>
<link>
<name>LVoc.cpp</name>
<type>1</type>
@ -681,41 +621,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/Phrase.h</locationURI>
</link>
<link>
<name>PhraseBoundaryFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhraseBoundaryFeature.cpp</locationURI>
</link>
<link>
<name>PhraseBoundaryFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhraseBoundaryFeature.h</locationURI>
</link>
<link>
<name>PhraseLengthFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhraseLengthFeature.cpp</locationURI>
</link>
<link>
<name>PhraseLengthFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhraseLengthFeature.h</locationURI>
</link>
<link>
<name>PhraseLengthFeatureTest.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhraseLengthFeatureTest.cpp</locationURI>
</link>
<link>
<name>PhrasePairFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhrasePairFeature.cpp</locationURI>
</link>
<link>
<name>PhrasePairFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/PhrasePairFeature.h</locationURI>
</link>
<link>
<name>PrefixTree.h</name>
<type>1</type>
@ -856,16 +761,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/SentenceStats.h</locationURI>
</link>
<link>
<name>SourceWordDeletionFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/SourceWordDeletionFeature.cpp</locationURI>
</link>
<link>
<name>SourceWordDeletionFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/SourceWordDeletionFeature.h</locationURI>
</link>
<link>
<name>SquareMatrix.cpp</name>
<type>1</type>
@ -911,26 +806,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/SyntacticLanguageModelState.h</locationURI>
</link>
<link>
<name>TargetBigramFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetBigramFeature.cpp</locationURI>
</link>
<link>
<name>TargetBigramFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetBigramFeature.h</locationURI>
</link>
<link>
<name>TargetNgramFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetNgramFeature.cpp</locationURI>
</link>
<link>
<name>TargetNgramFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetNgramFeature.h</locationURI>
</link>
<link>
<name>TargetPhrase.cpp</name>
<type>1</type>
@ -951,16 +826,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetPhraseCollection.h</locationURI>
</link>
<link>
<name>TargetWordInsertionFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetWordInsertionFeature.cpp</locationURI>
</link>
<link>
<name>TargetWordInsertionFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TargetWordInsertionFeature.h</locationURI>
</link>
<link>
<name>Terminal.h</name>
<type>1</type>
@ -1126,16 +991,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/WordLattice.h</locationURI>
</link>
<link>
<name>WordTranslationFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/WordTranslationFeature.cpp</locationURI>
</link>
<link>
<name>WordTranslationFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/WordTranslationFeature.h</locationURI>
</link>
<link>
<name>WordsBitmap.cpp</name>
<type>1</type>
@ -1186,6 +1041,211 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/rule.proto</locationURI>
</link>
<link>
<name>FF/BleuScoreFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.cpp</locationURI>
</link>
<link>
<name>FF/BleuScoreFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.h</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.h</locationURI>
</link>
<link>
<name>FF/DistortionScoreProducer.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DistortionScoreProducer.cpp</locationURI>
</link>
<link>
<name>FF/DistortionScoreProducer.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DistortionScoreProducer.h</locationURI>
</link>
<link>
<name>FF/FFState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/FFState.cpp</locationURI>
</link>
<link>
<name>FF/FFState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/FFState.h</locationURI>
</link>
<link>
<name>FF/FeatureFunction.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/FeatureFunction.cpp</locationURI>
</link>
<link>
<name>FF/FeatureFunction.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/FeatureFunction.h</locationURI>
</link>
<link>
<name>FF/GlobalLexicalModel.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/GlobalLexicalModel.cpp</locationURI>
</link>
<link>
<name>FF/GlobalLexicalModel.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/GlobalLexicalModel.h</locationURI>
</link>
<link>
<name>FF/GlobalLexicalModelUnlimited.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/GlobalLexicalModelUnlimited.cpp</locationURI>
</link>
<link>
<name>FF/GlobalLexicalModelUnlimited.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/GlobalLexicalModelUnlimited.h</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.h</locationURI>
</link>
<link>
<name>FF/PhraseBoundaryFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBoundaryFeature.cpp</locationURI>
</link>
<link>
<name>FF/PhraseBoundaryFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBoundaryFeature.h</locationURI>
</link>
<link>
<name>FF/PhraseLengthFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseLengthFeature.cpp</locationURI>
</link>
<link>
<name>FF/PhraseLengthFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseLengthFeature.h</locationURI>
</link>
<link>
<name>FF/PhraseLengthFeatureTest.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseLengthFeatureTest.cpp</locationURI>
</link>
<link>
<name>FF/PhrasePairFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhrasePairFeature.cpp</locationURI>
</link>
<link>
<name>FF/PhrasePairFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhrasePairFeature.h</locationURI>
</link>
<link>
<name>FF/SourceWordDeletionFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SourceWordDeletionFeature.cpp</locationURI>
</link>
<link>
<name>FF/SourceWordDeletionFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SourceWordDeletionFeature.h</locationURI>
</link>
<link>
<name>FF/StatefulFeatureFunction.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/StatefulFeatureFunction.cpp</locationURI>
</link>
<link>
<name>FF/StatefulFeatureFunction.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/StatefulFeatureFunction.h</locationURI>
</link>
<link>
<name>FF/StatelessFeatureFunction.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/StatelessFeatureFunction.cpp</locationURI>
</link>
<link>
<name>FF/StatelessFeatureFunction.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/StatelessFeatureFunction.h</locationURI>
</link>
<link>
<name>FF/TargetBigramFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetBigramFeature.cpp</locationURI>
</link>
<link>
<name>FF/TargetBigramFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetBigramFeature.h</locationURI>
</link>
<link>
<name>FF/TargetNgramFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetNgramFeature.cpp</locationURI>
</link>
<link>
<name>FF/TargetNgramFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetNgramFeature.h</locationURI>
</link>
<link>
<name>FF/TargetWordInsertionFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetWordInsertionFeature.cpp</locationURI>
</link>
<link>
<name>FF/TargetWordInsertionFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/TargetWordInsertionFeature.h</locationURI>
</link>
<link>
<name>FF/UnknownWordPenaltyProducer.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/UnknownWordPenaltyProducer.cpp</locationURI>
</link>
<link>
<name>FF/UnknownWordPenaltyProducer.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/UnknownWordPenaltyProducer.h</locationURI>
</link>
<link>
<name>FF/WordPenaltyProducer.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/WordPenaltyProducer.cpp</locationURI>
</link>
<link>
<name>FF/WordPenaltyProducer.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/WordPenaltyProducer.h</locationURI>
</link>
<link>
<name>FF/WordTranslationFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/WordTranslationFeature.cpp</locationURI>
</link>
<link>
<name>FF/WordTranslationFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/WordTranslationFeature.h</locationURI>
</link>
<link>
<name>LM/Backward.cpp</name>
<type>1</type>
@ -1361,11 +1421,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/backward.arpa</locationURI>
</link>
<link>
<name>LM/bin</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/BilingualDynSuffixArray.cpp</name>
<type>1</type>
@ -1421,6 +1476,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryDynSuffixArray.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMemory.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemory.cpp</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMemory.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemory.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryMultiModel.cpp</name>
<type>1</type>
@ -1441,6 +1506,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMultiModelCounts.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryNodeMemory.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryNodeMemory.cpp</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryNodeMemory.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryNodeMemory.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionaryTree.cpp</name>
<type>1</type>
@ -1496,21 +1571,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/lm.log</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerCYKPlus.cpp</name>
<type>1</type>
@ -1861,16 +1921,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.cpp</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.h</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp</name>
<type>1</type>
@ -1881,16 +1931,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/PhraseDictionarySCFG.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionarySCFG.cpp</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/PhraseDictionarySCFG.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/RuleTable/PhraseDictionarySCFG.h</locationURI>
</link>
<link>
<name>TranslationModel/RuleTable/Trie.cpp</name>
<type>1</type>
@ -2061,16 +2101,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7</name>
<type>2</type>
@ -2091,16 +2121,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7/release</name>
<type>2</type>
@ -2116,16 +2136,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on</name>
<type>2</type>
@ -2141,16 +2151,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static</name>
<type>2</type>
@ -2776,36 +2776,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/moses_test.passed</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Factory.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Factory.o</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/IRST.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/IRST.o</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ORLM.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ORLM.o</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ParallelBackoff.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/ParallelBackoff.o</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/SRI.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/SRI.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi</name>
<type>2</type>
@ -2956,76 +2926,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Backward.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Backward.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardLMState.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardLMState.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.output</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.output</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.run</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.run</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.test</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BackwardTest.test</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Base.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Base.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Implementation.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Implementation.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Joint.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Joint.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Ken.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Ken.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/MultiFactor.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/MultiFactor.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Remote.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/Remote.o</locationURI>
</link>
<link>
<name>LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/SingleFactor.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/BackwardTest.test/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/SingleFactor.o</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/BlockHashIndex.o</name>
<type>1</type>
@ -3147,20 +3047,20 @@
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.o</locationURI>
</link>
<link>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.o</name>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryMemory.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryNodeSCFG.o</locationURI>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryMemory.o</locationURI>
</link>
<link>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryNodeMemory.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryNodeMemory.o</locationURI>
</link>
<link>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryOnDisk.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionaryOnDisk.o</locationURI>
</link>
<link>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionarySCFG.o</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/PhraseDictionarySCFG.o</locationURI>
</link>
<link>
<name>bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/TranslationModel/RuleTable/Trie.o</name>
<type>1</type>

6
contrib/server/mosesserver.cpp
View File

@ -12,7 +12,6 @@
#include "moses/TranslationModel/PhraseDictionaryDynSuffixArray.h"
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
#include "moses/TreeInput.h"
#include "moses/LMList.h"
#include "moses/LM/ORLM.h"
#ifdef WITH_THREADS
@ -48,7 +47,7 @@ public:
cerr << "Inserting into address " << pdsa << endl;
pdsa->insertSnt(source_, target_, alignment_);
if(add2ORLM_) {
updateORLM();
//updateORLM();
}
cerr << "Done inserting\n";
//PhraseDictionary* pdsa = (PhraseDictionary*) pdf->GetDictionary(*dummy);
@ -60,6 +59,7 @@ public:
}
string source_, target_, alignment_;
bool bounded_, add2ORLM_;
/*
void updateORLM() {
// TODO(level101): this belongs in the language model, not in moseserver.cpp
vector<string> vl;
@ -101,6 +101,8 @@ public:
}
}
}
*/
void breakOutParams(const params_t& params) {
params_t::const_iterator si = params.find("source");
if(si == params.end())

10
moses/PhraseLengthFeatureTest.cpp → defer/PhraseLengthFeatureTest.cpp
View File

@ -18,11 +18,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <boost/test/unit_test.hpp>
#include "FactorCollection.h"
#include "PhraseLengthFeature.h"
#include "Sentence.h"
#include "TargetPhrase.h"
#include "TranslationOption.h"
#include "moses/FF/PhraseLengthFeature.h"
#include "moses/FactorCollection.h"
#include "moses/Sentence.h"
#include "moses/TargetPhrase.h"
#include "moses/TranslationOption.h"
using namespace Moses;
using namespace std;

2
mira/Decoder.h
View File

@ -24,13 +24,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <sstream>
#include "moses/BleuScoreFeature.h"
#include "moses/ChartTrellisPathList.h"
#include "moses/Hypothesis.h"
#include "moses/Parameter.h"
#include "moses/SearchNormal.h"
#include "moses/Sentence.h"
#include "moses/StaticData.h"
#include "moses/FF/BleuScoreFeature.h"
//
// Wrapper functions and objects for the decoder.

93
mira/Main.cpp
View File

@ -36,17 +36,18 @@ namespace mpi = boost::mpi;
#include "Optimiser.h"
#include "Hildreth.h"
#include "HypothesisQueue.h"
#include "moses/FeatureVector.h"
#include "moses/StaticData.h"
#include "moses/ChartTrellisPathList.h"
#include "moses/ChartTrellisPath.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/ThreadPool.h"
#include "moses/DummyScoreProducers.h"
#include "moses/LexicalReordering.h"
#include "moses/WordTranslationFeature.h"
#include "moses/PhrasePairFeature.h"
#include "mert/BleuScorer.h"
#include "moses/FeatureVector.h"
#include "moses/FF/WordTranslationFeature.h"
#include "moses/FF/PhrasePairFeature.h"
#include "moses/FF/WordPenaltyProducer.h"
using namespace Mira;
using namespace std;
@ -526,12 +527,17 @@ int main(int argc, char** argv) {
ScoreComponentCollection initialWeights = decoder->getWeights();
if (add2lm != 0) {
const LMList& lmList_new = staticData.GetLMList();
for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) {
float lmWeight = initialWeights.GetScoreForProducer(*i) + add2lm;
initialWeights.Assign(*i, lmWeight);
cerr << "Rank " << rank << ", add " << add2lm << " to lm weight." << endl;
}
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
float lmWeight = initialWeights.GetScoreForProducer(lm) + add2lm;
initialWeights.Assign(lm, lmWeight);
cerr << "Rank " << rank << ", add " << add2lm << " to lm weight." << endl;
}
}
}
if (normaliseWeights) {
@ -542,11 +548,18 @@ int main(int argc, char** argv) {
decoder->setWeights(initialWeights);
// set bleu weight to twice the size of the language model weight(s)
const LMList& lmList = staticData.GetLMList();
if (bleu_weight_lm) {
float lmSum = 0;
for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i)
lmSum += abs(initialWeights.GetScoreForProducer(*i));
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
lmSum += abs(initialWeights.GetScoreForProducer(lm));
}
}
bleuWeight = lmSum * bleu_weight_lm_factor;
cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl;
}
@ -765,15 +778,20 @@ int main(int argc, char** argv) {
}
// check LM weight
const LMList& lmList_new = staticData.GetLMList();
for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) {
float lmWeight = mosesWeights.GetScoreForProducer(*i);
cerr << "Rank " << rank << ", epoch " << epoch << ", lm weight: " << lmWeight << endl;
if (lmWeight <= 0) {
cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: language model weight should never be <= 0." << endl;
mosesWeights.Assign(*i, 0.1);
cerr << "Rank " << rank << ", epoch " << epoch << ", assign lm weights of 0.1" << endl;
}
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
float lmWeight = mosesWeights.GetScoreForProducer(lm);
cerr << "Rank " << rank << ", epoch " << epoch << ", lm weight: " << lmWeight << endl;
if (lmWeight <= 0) {
cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: language model weight should never be <= 0." << endl;
mosesWeights.Assign(lm, 0.1);
cerr << "Rank " << rank << ", epoch " << epoch << ", assign lm weights of 0.1" << endl;
}
}
}
// select inference scheme
@ -1167,17 +1185,22 @@ int main(int argc, char** argv) {
// scale LM feature (to avoid rapid changes)
if (scale_lm) {
cerr << "scale lm" << endl;
const LMList& lmList_new = staticData.GetLMList();
for (LMList::const_iterator iter = lmList_new.begin(); iter != lmList_new.end(); ++iter) {
// scale down score
if (model_hope_fear) {
scaleFeatureScore(*iter, scale_lm_factor, featureValues, rank, epoch);
}
else {
scaleFeatureScore(*iter, scale_lm_factor, featureValuesHope, rank, epoch);
scaleFeatureScore(*iter, scale_lm_factor, featureValuesFear, rank, epoch);
}
}
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
// scale down score
if (model_hope_fear) {
scaleFeatureScore(lm, scale_lm_factor, featureValues, rank, epoch);
}
else {
scaleFeatureScore(lm, scale_lm_factor, featureValuesHope, rank, epoch);
scaleFeatureScore(lm, scale_lm_factor, featureValuesFear, rank, epoch);
}
}
}
}
// scale WP
@ -1848,7 +1871,7 @@ void applyPerFeatureLearningRates(vector<vector<ScoreComponentCollection> > &fea
featureValues[i][j].MultiplyEqualsBackoff(featureLearningRates, sparse_r0);
}
void scaleFeatureScore(FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) {
void scaleFeatureScore(const FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) {
string name = sp->GetScoreProducerDescription();
// scale down score
@ -1862,7 +1885,7 @@ void scaleFeatureScore(FeatureFunction *sp, float scaling_factor, vector<vector<
}
}
void scaleFeatureScores(FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) {
void scaleFeatureScores(const FeatureFunction *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) {
string name = sp->GetScoreProducerDescription();
// scale down score

6
mira/Main.h
View File

@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/ScoreComponentCollection.h"
#include "moses/Word.h"
#include "moses/FeatureFunction.h"
#include "moses/FF/FeatureFunction.h"
#include "Decoder.h"
typedef std::map<const Moses::FeatureFunction*, std::vector< float > > ProducerWeightMap;
@ -52,7 +52,7 @@ void deleteTranslations(std::vector<std::vector<const Moses::Word*> > &translati
void decodeHopeOrFear(size_t rank, size_t size, size_t decode, std::string decode_filename, std::vector<std::string> &inputSentences, Mira::MosesDecoder* decoder, size_t n, float bleuWeight);
void applyLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, float core_r0, float sparse_r0);
void applyPerFeatureLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, Moses::ScoreComponentCollection featureLearningRates, float sparse_r0);
void scaleFeatureScore(Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch);
void scaleFeatureScores(Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch);
void scaleFeatureScore(const Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch);
void scaleFeatureScores(const Moses::FeatureFunction *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch);
#endif /* MAIN_H_ */

2
misc/Jamfile
View File

@ -19,4 +19,4 @@ else {
alias programsMin ;
}
alias programs : processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin create-ini//create-ini ;
alias programs : processPhraseTable processLexicalTable queryPhraseTable queryLexicalTable programsMin ;

3
misc/create-ini/Distortion.cpp
View File

@ -1,3 +0,0 @@
#include "Distortion.h"
int Distortion::s_index = 0;

26
misc/create-ini/Distortion.h
View File

@ -1,26 +0,0 @@
#pragma once
#include "FF.h"
class Distortion : public FF
{
static int s_index;
float GetWeight() const
{ return 0.3; }
void Output(std::ostream &out) const
{
out << name
<< std::endl;
}
public:
Distortion(const std::string &line)
:FF(line)
{
index = s_index++;
name = "Distortion";
numFeatures = 1;
}
};

27
misc/create-ini/FF.cpp
View File

@ -1,27 +0,0 @@
#include "FF.h"
#include <sstream>
using namespace std;
std::string FF::OutputFactors(const std::vector<int> &factors) const
{
if (factors.size() == 0)
return "";
stringstream ret;
ret << factors[0];
for (size_t i = 1; i < factors.size(); ++i) {
ret << "," << factors[i];
}
return ret.str();
}
void FF::OutputWeights(std::ostream &out) const
{
out << name << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}

32
misc/create-ini/FF.h
View File

@ -1,32 +0,0 @@
#pragma once
#include <iostream>
#include <vector>
#include <string>
#include "Util.h"
class FF
{
virtual float GetWeight() const = 0;
protected:
std::string OutputFactors(const std::vector<int> &factors) const;
public:
std::vector<std::string> toks;
Factors inFactors, outFactors;
std::string name;
std::string path;
int index, numFeatures;
FF(const std::string &line)
{
toks = Tokenize(line, ":");
}
virtual void Output(std::ostream &out) const = 0;
virtual void OutputWeights(std::ostream &out) const;
};

3
misc/create-ini/Jamfile
View File

@ -1,3 +0,0 @@
exe create-ini : FF.cpp LM.cpp PT.cpp RO.cpp WP.cpp Distortion.cpp UnknownWP.cpp Main.cpp ;

23
misc/create-ini/LM.cpp
View File

@ -1,23 +0,0 @@
#include "LM.h"
int LM::s_index = 0;
void LM::Output(std::ostream &out) const
{
out << name
<< " name=LM" << index
<< " order=" << order
<< " factor=" << OutputFactors(outFactors)
<< " path=" << path
<< " " << otherArgs
<< std::endl;
}
void LM::OutputWeights(std::ostream &out) const
{
out << "LM" << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}

45
misc/create-ini/LM.h
View File

@ -1,45 +0,0 @@
#pragma once
#include <iostream>
#include <vector>
#include <string>
#include "Util.h"
#include "FF.h"
class LM : public FF
{
static int s_index;
float GetWeight() const
{ return 0.5; }
void Output(std::ostream &out) const;
void OutputWeights(std::ostream &out) const;
public:
std::string otherArgs;
int order;
LM(const std::string &line)
:FF(line)
{
index = s_index++;
numFeatures = 1;
outFactors.push_back(Scan<int>(toks[0]));
order = Scan<int>(toks[1]);
path = toks[2];
int implNum = 0;
if (toks.size() >= 4)
implNum = Scan<int>(toks[3]);
switch (implNum)
{
case 0: name = "SRILM"; break;
case 1: name = "IRSTLM"; break;
case 8: name = "KENLM"; otherArgs = "lazyken=0"; break;
case 9: name = "KENLM"; otherArgs = "lazyken=1"; break;
}
}
};

226
misc/create-ini/Main.cpp
View File

@ -1,226 +0,0 @@
#include <typeinfo>
#include <iostream>
#include <fstream>
#include <sstream>
#include <cassert>
#include <vector>
#include <stdlib.h>
#include "FF.h"
#include "LM.h"
#include "RO.h"
#include "PT.h"
#include "WP.h"
#include "UnknownWP.h"
#include "Distortion.h"
using namespace std;
string iniPath;
vector<FF*> ffVec;
vector<string> additionalIni;
bool isHierarchical = false;
int inputFactorMax = 0;
void OutputIni();
void OutputAdditionalIni(ofstream &out);
void OutputAdditionalIni(ofstream &out, const string &path);
void ParseFactors(const string &line, vector< pair<Factors, Factors> > &ret);
int main(int argc, char **argv)
{
vector< pair<Factors, Factors> > transFactors, roFactors;
int indTrans = 0, indRO = 0;
FF *model;
for (int i = 1; i < argc; ++i) {
string key(argv[i]);
if (key == "-phrase-translation-table") {
++i;
pair<Factors, Factors> *factors = transFactors.size() > indTrans ? &transFactors[indTrans] : NULL;
model = new PT(argv[i], 5, isHierarchical, factors);
ffVec.push_back(model);
++indTrans;
}
else if (key == "-glue-grammar-file") {
++i;
pair<Factors, Factors> *factors = transFactors.size() > indTrans ? &transFactors[indTrans] : NULL;
model = new PT(argv[i], 1, isHierarchical, factors);
ffVec.push_back(model);
++indTrans;
}
else if (key == "-reordering-table") {
++i;
pair<Factors, Factors> *factors = roFactors.size() > indRO ? &roFactors[indRO] : NULL;
model = new RO(argv[i], factors);
ffVec.push_back(model);
++indRO;
}
else if (key == "-lm") {
++i;
model = new LM(argv[i]);
ffVec.push_back(model);
}
else if (key == "-config") {
++i;
iniPath = argv[i];
}
else if (key == "-hierarchical") {
isHierarchical = true;
}
else if (key == "-translation-factors") {
++i;
ParseFactors(argv[i], transFactors);
}
else if (key == "-reordering-factors") {
++i;
ParseFactors(argv[i], roFactors);
}
else if (key == "-input-factor-max") {
++i;
inputFactorMax = Scan<int>(argv[i]);
}
else if (key == "-additional-ini-file") {
++i;
additionalIni.push_back(argv[i]);
}
else if (key == "-sparse-translation-table") {
// ignore. TODO check that all pt can handle sparse features
}
else if (key == "-score-options") {
// ignore. TODO
++i;
}
else if (key == "-additional-ini") {
// ignore. TODO
++i;
}
else {
cerr << "Unknown arg " << key << endl;
abort();
}
}
model = new WP("");
ffVec.insert(ffVec.begin(), model);
model = new UnknownWP("");
ffVec.insert(ffVec.begin(), model);
if (!isHierarchical) {
model = new Distortion("");
ffVec.insert(ffVec.begin(), model);
}
OutputIni();
}
// parse input & output factors for
// -translation-factors & -reordering-factors
void ParseFactorsPair(const string &line, vector< pair<Factors, Factors> > &ret)
{
vector<string> toks = Tokenize(line, "-");
assert(toks.size() == 2);
Factors input, output;
input = Tokenize<int>(toks[0], ",");
output = Tokenize<int>(toks[1], ",");
ret.push_back(pair<Factors, Factors>(input, output) );
}
void ParseFactors(const string &line, vector< pair<Factors, Factors> > &ret)
{
vector<string> toks = Tokenize(line, ":");
for (size_t i = 0; i < toks.size(); ++i) {
const string &tok = toks[i];
ParseFactorsPair(tok, ret);
}
}
// output ini file, with features and weights, and everything else
void OutputIni()
{
ofstream strme(iniPath.c_str());
stringstream weightStrme;
weightStrme << "\n\n[weight]" << endl;
strme << "[input-factors]" << endl;
for (size_t i = 0; i <= inputFactorMax; ++i) {
strme << i << endl;
}
strme << endl;
strme << "[mapping]" << endl;
if (isHierarchical) {
strme << "0 T 0" << endl
<< "1 T 1" << endl;
}
else {
strme << "0 T 0" << endl;
}
if (!isHierarchical) {
strme << "[distortion-limit]" << endl;
strme << "6" << endl;
}
else {
strme << "[cube-pruning-pop-limit]" << endl;
strme << "1000" << endl;
strme << "[non-terminals]" << endl;
strme << "X" << endl;
strme << "[search-algorithm]" << endl;
strme << "3" << endl;
strme << "[max-chart-span]" << endl;
strme << "20" << endl;
strme << "1000" << endl;
}
strme << "\n\n[feature]" << endl;
for (size_t i = 0; i < ffVec.size(); ++i) {
const FF &ff = *ffVec[i];
ff.Output(strme);
ff.OutputWeights(weightStrme);
}
strme << weightStrme.str();
strme << endl;
OutputAdditionalIni(strme);
strme << endl << endl << endl;
strme.close();
}
void OutputAdditionalIni(ofstream &out)
{
for (size_t i = 0; i < additionalIni.size(); ++i) {
string &path = additionalIni[i];
OutputAdditionalIni(out, path);
}
}
void OutputAdditionalIni(ofstream &out, const string &path)
{
ifstream in(path.c_str());
string line;
while (getline(in, line)) {
out << line << endl;
}
in.close();
}

0
misc/create-ini/Main.h
View File

65
misc/create-ini/PT.cpp
View File

@ -1,65 +0,0 @@
#include "PT.h"
using namespace std;
int PT::s_index = 0;
PT::PT(const std::string &line, int numFeatures, bool isHierarchical, const pair<Factors, Factors> *factors)
:FF(line)
{
index = s_index++;
this->numFeatures = numFeatures;
path = toks[0];
if (factors) {
inFactors = factors->first;
outFactors = factors->second;
}
if (inFactors.size() == 0) {
inFactors.push_back(0);
}
if (outFactors.size() == 0) {
outFactors.push_back(0);
}
int implementation;
if (toks.size() > 1)
implementation = Scan<int>(toks[1]);
else if (isHierarchical)
implementation = 6;
else
implementation = 0;
switch (implementation)
{
case 0: name = "PhraseDictionaryMemory"; break;
case 1: name = "PhraseDictionaryBinary"; break;
case 2: name = "PhraseDictionaryOnDisk"; break;
case 6: name = "PhraseDictionaryMemory"; break;
default:name = "UnknownPtImplementation"; break;
}
}
void PT::Output(std::ostream &out) const
{
out << name
<< " name=TranslationModel" << index
<< " num-features=" << numFeatures
<< " path=" << path;
out << " input-factor=" << OutputFactors(inFactors);
out << " output-factor=" << OutputFactors(outFactors);
out << std::endl;
}
void PT::OutputWeights(std::ostream &out) const
{
out << "TranslationModel" << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}

22
misc/create-ini/PT.h
View File

@ -1,22 +0,0 @@
#pragma once
#include <vector>
#include <string>
#include "Util.h"
#include "FF.h"
class PT : public FF
{
static int s_index;
float GetWeight() const
{ return 0.2; }
void Output(std::ostream &out) const;
public:
PT(const std::string &line, int numFeatures, bool isHierarchical, const std::pair<Factors, Factors> *factors);
void OutputWeights(std::ostream &out) const;
};

27
misc/create-ini/RO.cpp
View File

@ -1,27 +0,0 @@
#include "RO.h"
int RO::s_index = 0;
RO::RO(const std::string &line, const std::pair<Factors, Factors> *factors)
:FF(line)
{
index = s_index++;
name = "LexicalReordering";
numFeatures = 6;
path = toks[0];
type = "wbe-msd-bidirectional-fe-allff"; // TODO what is this?
fileTypeSuffix = "wbe-msd-bidirectional-fe";
if (factors) {
inFactors = factors->first;
outFactors = factors->second;
}
if (inFactors.size() == 0) {
inFactors.push_back(0);
}
if (outFactors.size() == 0) {
outFactors.push_back(0);
}
}

29
misc/create-ini/RO.h
View File

@ -1,29 +0,0 @@
#pragma once
#include "Util.h"
#include "FF.h"
class RO : public FF
{
static int s_index;
std::string type, fileTypeSuffix;
float GetWeight() const
{ return 0.3; }
void Output(std::ostream &out) const
{
out << name
<< " num-features=" << numFeatures
<< " type=" << type
<< " input-factor=" << OutputFactors(inFactors)
<< " output-factor=" << OutputFactors(outFactors)
<< " path=" << path << "." << fileTypeSuffix << ".gz"
<< std::endl;
}
public:
RO(const std::string &line, const std::pair<Factors, Factors> *factors);
};

3
misc/create-ini/UnknownWP.cpp
View File

@ -1,3 +0,0 @@
#include "UnknownWP.h"
int UnknownWP::s_index = 0;

26
misc/create-ini/UnknownWP.h
View File

@ -1,26 +0,0 @@
#pragma once
#include "FF.h"
class UnknownWP : public FF
{
static int s_index;
virtual float GetWeight() const
{ return 1.0; }
void Output(std::ostream &out) const
{
out << name
<< std::endl;
}
public:
UnknownWP(const std::string &line)
:FF(line)
{
index = s_index++;
name = "UnknownWordPenalty";
numFeatures = 1;
}
};

60
misc/create-ini/Util.h
View File

@ -1,60 +0,0 @@
#pragma once
#include <string>
#include <vector>
#include <sstream>
typedef std::vector<int> Factors;
template<typename T>
inline T Scan(const std::string &input)
{
std::stringstream stream(input);
T ret;
stream >> ret;
return ret;
}
template<typename T>
inline std::vector<T> Scan(const std::vector< std::string > &input)
{
std::vector<T> output(input.size());
for (size_t i = 0 ; i < input.size() ; i++) {
output[i] = Scan<T>( input[i] );
}
return output;
}
inline std::vector<std::string> Tokenize(const std::string& str,
const std::string& delimiters)
{
std::vector<std::string> tokens;
// Skip delimiters at beginning.
std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
// Find first "non-delimiter".
std::string::size_type pos = str.find_first_of(delimiters, lastPos);
while (std::string::npos != pos || std::string::npos != lastPos) {
// Found a token, add it to the vector.
tokens.push_back(str.substr(lastPos, pos - lastPos));
// Skip delimiters. Note the "not_of"
lastPos = str.find_first_not_of(delimiters, pos);
// Find next "non-delimiter"
pos = str.find_first_of(delimiters, lastPos);
}
return tokens;
}
//! tokenise input string to vector of type T
template<typename T>
inline std::vector<T> Tokenize( const std::string &input
, const std::string& delimiters = " \t")
{
std::vector<std::string> stringVector = Tokenize(input, delimiters);
return Scan<T>( stringVector );
}

3
misc/create-ini/WP.cpp
View File

@ -1,3 +0,0 @@
#include "WP.h"
int WP::s_index = 0;

26
misc/create-ini/WP.h
View File

@ -1,26 +0,0 @@
#pragma once
#include "FF.h"
class WP : public FF
{
static int s_index;
virtual float GetWeight() const
{ return -1.0; }
void Output(std::ostream &out) const
{
out << name
<< std::endl;
}
public:
WP(const std::string &line)
:FF(line)
{
index = s_index++;
name = "WordPenalty";
numFeatures = 1;
}
};

1
misc/create-ini/compile.sh
View File

@ -1 +0,0 @@
g++ FF.cpp LM.cpp RO.cpp PT.cpp WP.cpp Distortion.cpp UnknownWP.cpp Main.cpp -o ../../bin/create-ini

4
misc/queryPhraseTableMin.cpp
View File

@ -47,9 +47,7 @@ int main(int argc, char **argv)
std::vector<FactorType> input(1, 0);
std::vector<FactorType> output(1, 0);
std::vector<float> weight(nscores, 0);
LMList lmList;
Parameter *parameter = new Parameter();
const_cast<std::vector<std::string>&>(parameter->GetParam("factor-delimiter")).resize(1, "||dummy_string||");
const_cast<std::vector<std::string>&>(parameter->GetParam("input-factors")).resize(1, "0");

1
moses-chart-cmd/IOWrapper.cpp
View File

@ -38,7 +38,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "moses/StaticData.h"
#include "moses/DummyScoreProducers.h"
#include "moses/InputFileStream.h"
#include "moses/Incremental.h"
#include "moses/TranslationModel/PhraseDictionary.h"

1
moses-chart-cmd/Main.cpp
View File

@ -44,7 +44,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "mbr.h"
#include "IOWrapper.h"
#include "moses/DummyScoreProducers.h"
#include "moses/FactorCollection.h"
#include "moses/Manager.h"
#include "moses/Phrase.h"

1
moses-cmd/IOWrapper.cpp
View File

@ -42,7 +42,6 @@ POSSIBILITY OF SUCH DAMAGE.
#include "moses/WordsRange.h"
#include "moses/TrellisPathList.h"
#include "moses/StaticData.h"
#include "moses/DummyScoreProducers.h"
#include "moses/FeatureVector.h"
#include "moses/InputFileStream.h"
#include "IOWrapper.h"

23
moses-cmd/TranslationAnalysis.cpp
View File

@ -101,14 +101,21 @@ void PrintTranslationAnalysis(std::ostream &os, const Hypothesis* hypo)
}
os << std::endl << std::endl;
if (doLMStats && lmCalls > 0) {
std::vector<unsigned int>::iterator acc = lmAcc.begin();
const LMList &lmList = StaticData::Instance().GetLMList();
LMList::const_iterator i = lmList.begin();
for (; acc != lmAcc.end(); ++acc, ++i) {
char buf[256];
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
}
std::vector<unsigned int>::iterator acc = lmAcc.begin();
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
char buf[256];
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
os << lm->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
++acc;
}
}
}
if (droppedWords.size() > 0) {

2
moses/BitmapContainer.cpp
View File

@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "BitmapContainer.h"
#include "HypothesisStackCubePruning.h"
#include "DummyScoreProducers.h"
#include "moses/FF/DistortionScoreProducer.h"
#include "TranslationOptionList.h"
namespace Moses

9
moses/ChartHypothesis.cpp
View File

@ -27,9 +27,8 @@
#include "TargetPhrase.h"
#include "Phrase.h"
#include "StaticData.h"
#include "DummyScoreProducers.h"
#include "ChartTranslationOptions.h"
#include "FFState.h"
#include "moses/FF/FFState.h"
using namespace std;
@ -166,9 +165,9 @@ void ChartHypothesis::CalcScore()
// cached in the translation option-- there is no principled distinction
const std::vector<const StatelessFeatureFunction*>& sfs =
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i)
if (sfs[i]->GetStatelessFeatureType() == RequiresSegmentation)
sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown);
for (unsigned i = 0; i < sfs.size(); ++i) {
sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown);
}
const std::vector<const StatefulFeatureFunction*>& ffs =
StatefulFeatureFunction::GetStatefulFeatureFunctions();

6
moses/ChartManager.cpp
View File

@ -31,7 +31,7 @@
#include "StaticData.h"
#include "DecodeStep.h"
#include "TreeInput.h"
#include "DummyScoreProducers.h"
#include "moses/FF/WordPenaltyProducer.h"
using namespace std;
using namespace Moses;
@ -354,9 +354,7 @@ void ChartManager::PreCalculateScores()
StatelessFeatureFunction::GetStatelessFeatureFunctions();
ScoreComponentCollection& breakdown = m_precalculatedScores[*targetPhrase];
for (size_t k = 0; k < sfs.size(); ++k) {
if (sfs[k]->GetStatelessFeatureType() == RequiresSource) {
sfs[k]->EvaluateChart(context,&breakdown);
}
sfs[k]->EvaluateChart(context,&breakdown);
}
}
}

6
moses/ChartParser.cpp
View File

@ -22,9 +22,9 @@
#include "ChartParser.h"
#include "ChartParserCallback.h"
#include "ChartRuleLookupManager.h"
#include "DummyScoreProducers.h"
#include "StaticData.h"
#include "TreeInput.h"
#include "moses/FF/UnknownWordPenaltyProducer.h"
using namespace std;
using namespace Moses;
@ -86,7 +86,7 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
float unknownScore = FloorScore(TransformScore(prob));
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate();
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetSourcePhrase(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);
@ -112,7 +112,7 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
CHECK(targetLHS->GetFactor(0) != NULL);
targetPhrase->GetScoreBreakdown().Assign(unknownWordPenaltyProducer, unknownScore);
targetPhrase->Evaluate();
targetPhrase->Evaluate(*unksrc);
targetPhrase->SetSourcePhrase(*unksrc);
targetPhrase->SetTargetLHS(targetLHS);

7
moses/DecodeFeature.h
View File

@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <vector>
#include "FactorTypeSet.h"
#include "FeatureFunction.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "TypeDef.h"
namespace Moses
@ -59,11 +59,6 @@ class DecodeFeature : public StatelessFeatureFunction {
const std::vector<FactorType>& GetInput() const;
const std::vector<FactorType>& GetOutput() const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool IsDecodeFeature() const
{ return true; }

4
moses/DecodeStep.h
View File

@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "util/check.hh"
#include "TypeDef.h"
#include "FactorTypeSet.h"
#include "Phrase.h"
namespace Moses
{
@ -99,7 +100,8 @@ public:
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const = 0;
, bool adhereTableLimit
, const Phrase &src) const = 0;
};

3
moses/DecodeStepGeneration.cpp
View File

@ -63,7 +63,8 @@ void DecodeStepGeneration::Process(const TranslationOption &inputPartialTranslOp
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection * /* toc */
, bool /*adhereTableLimit*/) const
, bool /*adhereTableLimit*/
, const Phrase &src) const
{
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
// word deletion

3
moses/DecodeStepGeneration.h
View File

@ -42,7 +42,8 @@ public:
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const;
, bool adhereTableLimit
, const Phrase &src) const;
private:
};

7
moses/DecodeStepTranslation.cpp
View File

@ -38,13 +38,12 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const
, bool adhereTableLimit
, const Phrase &src) const
{
if (inputPartialTranslOpt.GetTargetPhrase().GetSize() == 0) {
// word deletion
outputPartialTranslOptColl.Add(new TranslationOption(inputPartialTranslOpt));
return;
}
@ -77,7 +76,7 @@ void DecodeStepTranslation::Process(const TranslationOption &inputPartialTranslO
}
outPhrase.GetScoreBreakdown().PlusEquals(transScores);
outPhrase.Evaluate(); // need to do this as all non-transcores would be screwed up
outPhrase.Evaluate(src); // need to do this as all non-transcores would be screwed up
outPhrase.MergeFactors(targetPhrase, m_newOutputFactors);

3
moses/DecodeStepTranslation.h
View File

@ -43,7 +43,8 @@ public:
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
, bool adhereTableLimit) const;
, bool adhereTableLimit
, const Phrase &src) const;
/*! initialize list of partial translation options by applying the first translation step

106
moses/DummyScoreProducers.h
View File

@ -1,106 +0,0 @@
// $Id$
#ifndef moses_DummyScoreProducers_h
#define moses_DummyScoreProducers_h
#include "FeatureFunction.h"
#include "util/check.hh"
namespace Moses
{
class WordsRange;
/** Calculates Distortion scores
*/
class DistortionScoreProducer : public StatefulFeatureFunction
{
public:
DistortionScoreProducer(const std::string &line)
: StatefulFeatureFunction("Distortion", 1, line)
{}
static float CalculateDistortionScore(const Hypothesis& hypo,
const WordsRange &prev, const WordsRange &curr, const int FirstGapPosition);
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
virtual FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection*) const {
CHECK(0); // feature function not valid in chart decoder
return NULL;
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
};
/** Doesn't do anything but provide a key into the global
* score array to store the word penalty in.
*/
class WordPenaltyProducer : public StatelessFeatureFunction
{
public:
WordPenaltyProducer(const std::string &line) : StatelessFeatureFunction("WordPenalty",1, line) {}
virtual void Evaluate(
const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{}
virtual void EvaluateChart(
const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
//required but does nothing.
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
};
/** unknown word penalty */
class UnknownWordPenaltyProducer : public StatelessFeatureFunction
{
public:
UnknownWordPenaltyProducer(const std::string &line)
: StatelessFeatureFunction("UnknownWordPenalty",1, line)
{
m_tuneable = false;
}
void Evaluate( const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
//do nothing - not a real feature
}
void EvaluateChart(
const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
//do nothing - not a real feature
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
StatelessFeatureType GetStatelessFeatureType() const
{ return SetByOriginator; }
};
}
#endif

11
moses/BleuScoreFeature.cpp → moses/FF/BleuScoreFeature.cpp
View File

@ -1,7 +1,7 @@
#include "BleuScoreFeature.h"
#include "StaticData.h"
#include "UserMessage.h"
#include "moses/StaticData.h"
#include "moses/UserMessage.h"
using namespace std;
@ -858,12 +858,5 @@ const FFState* BleuScoreFeature::EmptyHypothesisState(const InputType& input) co
return new BleuScoreState();
}
void BleuScoreFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
} // namespace.

12
moses/BleuScoreFeature.h → moses/FF/BleuScoreFeature.h
View File

@ -7,11 +7,11 @@
#include <boost/unordered_map.hpp>
#include "FeatureFunction.h"
#include "StatefulFeatureFunction.h"
#include "FFState.h"
#include "Phrase.h"
#include "ChartHypothesis.h"
#include "moses/FF/FFState.h"
#include "moses/Phrase.h"
#include "moses/ChartHypothesis.h"
namespace Moses {
@ -118,10 +118,6 @@ public:
float GetTargetLengthHistory() { return m_target_length_history; }
float GetAverageInputLength() { return m_avg_input_length; }
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
private:
bool m_enabled;
bool m_sentence_bleu;

24
moses/FF/ChartBasedFeatureContext.cpp Normal file
View File

@ -0,0 +1,24 @@
#include "ChartBasedFeatureContext.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
namespace Moses
{
ChartBasedFeatureContext::ChartBasedFeatureContext
(const ChartHypothesis* hypothesis):
m_hypothesis(hypothesis),
m_targetPhrase(hypothesis->GetCurrTargetPhrase()),
m_source(hypothesis->GetManager().GetSource())
{}
ChartBasedFeatureContext::ChartBasedFeatureContext(
const TargetPhrase& targetPhrase,
const InputType& source):
m_hypothesis(NULL),
m_targetPhrase(targetPhrase),
m_source(source)
{}
}

37
moses/FF/ChartBasedFeatureContext.h Normal file
View File

@ -0,0 +1,37 @@
#pragma once
namespace Moses
{
class ChartHypothesis;
class InputType;
class TargetPhrase;
/**
* Same as PhraseBasedFeatureContext, but for chart-based Moses.
**/
class ChartBasedFeatureContext
{
//The context either has a hypothesis (during search) or a
//TargetPhrase and source sentence (during pre-calculation)
//TODO: should the context also include some info on where the TargetPhrase
//is anchored (assuming it's lexicalised), which is available at pre-calc?
const ChartHypothesis* m_hypothesis;
const TargetPhrase& m_targetPhrase;
const InputType& m_source;
public:
ChartBasedFeatureContext(const ChartHypothesis* hypothesis);
ChartBasedFeatureContext(const TargetPhrase& targetPhrase,
const InputType& source);
const InputType& GetSource() const
{ return m_source; }
const TargetPhrase& GetTargetPhrase() const
{ return m_targetPhrase; }
};
} // namespace

37
moses/DummyScoreProducers.cpp → moses/FF/DistortionScoreProducer.cpp
View File

@ -1,15 +1,11 @@
// $Id$
#include "util/check.hh"
#include "DistortionScoreProducer.h"
#include "FFState.h"
#include "StaticData.h"
#include "DummyScoreProducers.h"
#include "WordsRange.h"
#include "TranslationOption.h"
#include "moses/WordsRange.h"
#include "moses/StaticData.h"
namespace Moses
{
struct DistortionState_traditional : public FFState {
WordsRange range;
int first_gap;
@ -46,7 +42,7 @@ float DistortionScoreProducer::CalculateDistortionScore(const Hypothesis& hypo,
}
else {
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
Definitions:
Definitions:
S : current source range
S' : last translated source phrase range
S'' : longest fully-translated initial segment
@ -78,7 +74,7 @@ float DistortionScoreProducer::CalculateDistortionScore(const Hypothesis& hypo,
// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());
}
}
@ -101,27 +97,6 @@ FFState* DistortionScoreProducer::Evaluate(
return res;
}
void DistortionScoreProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// no score
}
void WordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
float score = - (float) targetPhrase.GetNumTerminals();
scoreBreakdown.Assign(this, score);
}
void UnknownWordPenaltyProducer::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// shouldn't be called
CHECK(false);
}
}

42
moses/FF/DistortionScoreProducer.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include <stdexcept>
#include <string>
#include "StatefulFeatureFunction.h"
namespace Moses
{
class FFState;
class ScoreComponentCollection;
class Hypothesis;
class ChartHypothesis;
class WordsRange;
/** Calculates Distortion scores
*/
class DistortionScoreProducer : public StatefulFeatureFunction
{
public:
DistortionScoreProducer(const std::string &line)
: StatefulFeatureFunction("Distortion", 1, line)
{}
static float CalculateDistortionScore(const Hypothesis& hypo,
const WordsRange &prev, const WordsRange &curr, const int FirstGapPosition);
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
virtual FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection*) const {
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
}
};
}

2
moses/FFState.cpp → moses/FF/FFState.cpp
View File

@ -1,4 +1,4 @@
#include "FFState.h"
#include "moses/FF/FFState.h"
namespace Moses
{

0
moses/FFState.h → moses/FF/FFState.h
View File

92
moses/FF/FeatureFunction.cpp Normal file
View File

@ -0,0 +1,92 @@
#include <stdexcept>
#include "util/check.hh"
#include "FeatureFunction.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/TranslationOption.h"
using namespace std;
namespace Moses
{
multiset<string> FeatureFunction::description_counts;
std::vector<FeatureFunction*> FeatureFunction::m_producers;
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
std::vector<const StatefulFeatureFunction*> StatefulFeatureFunction::m_statefulFFs;
FeatureFunction::FeatureFunction(const std::string& description, const std::string &line)
: m_tuneable(true)
{
ParseLine(description, line);
if (m_description == "") {
// not been given a name. Make a unique name
size_t index = description_counts.count(description);
ostringstream dstream;
dstream << description;
dstream << index;
description_counts.insert(description);
m_description = dstream.str();
}
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
}
FeatureFunction::FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
: m_numScoreComponents(numScoreComponents)
, m_tuneable(true)
{
ParseLine(description, line);
if (m_description == "") {
size_t index = description_counts.count(description);
ostringstream dstream;
dstream << description;
dstream << index;
description_counts.insert(description);
m_description = dstream.str();
}
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
}
FeatureFunction::~FeatureFunction() {}
void FeatureFunction::ParseLine(const std::string& description, const std::string &line)
{
vector<string> toks = Tokenize(line);
CHECK(toks.size());
//CHECK(toks[0] == description);
for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = Tokenize(toks[i], "=");
CHECK(args.size() == 2);
if (args[0] == "num-features") {
m_numScoreComponents = Scan<size_t>(args[1]);
}
else if (args[0] == "name") {
m_description = args[1];
}
else if (args[0] == "tuneable") {
m_tuneable = Scan<bool>(args[1]);
}
else {
m_args.push_back(args);
}
}
}
}

90
moses/FF/FeatureFunction.h Normal file
View File

@ -0,0 +1,90 @@
#ifndef moses_FeatureFunction_h
#define moses_FeatureFunction_h
#include <vector>
#include <set>
#include <string>
#include "PhraseBasedFeatureContext.h"
#include "ChartBasedFeatureContext.h"
#include "moses/TypeDef.h"
namespace Moses
{
class Phrase;
class TargetPhrase;
class TranslationOption;
class Hypothesis;
class ChartHypothesis;
class FFState;
class InputType;
class ScoreComponentCollection;
class WordsBitmap;
class WordsRange;
/** base class for all feature functions.
*/
class FeatureFunction
{
protected:
/**< all the score producers in this run */
static std::vector<FeatureFunction*> m_producers;
std::string m_description, m_argLine;
std::vector<std::vector<std::string> > m_args;
bool m_tuneable;
size_t m_numScoreComponents;
//In case there's multiple producers with the same description
static std::multiset<std::string> description_counts;
void ParseLine(const std::string& description, const std::string &line);
public:
static const std::vector<FeatureFunction*>& GetFeatureFunctions() { return m_producers; }
FeatureFunction(const std::string& description, const std::string &line);
FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
virtual bool IsStateless() const = 0;
virtual ~FeatureFunction();
static void ResetDescriptionCounts() {
description_counts.clear();
}
//! returns the number of scores that a subclass produces.
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
size_t GetNumScoreComponents() const {return m_numScoreComponents;}
//! returns a string description of this producer
const std::string& GetScoreProducerDescription() const
{ return m_description; }
virtual bool IsTuneable() const { return m_tuneable; }
//!
virtual void InitializeForInput(InputType const& source)
{}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing(const InputType& source)
{}
const std::string &GetArgLine() const
{ return m_argLine; }
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
virtual void Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const
{}
};
}
#endif

15
moses/GlobalLexicalModel.cpp → moses/FF/GlobalLexicalModel.cpp
View File

@ -1,9 +1,9 @@
#include <fstream>
#include "GlobalLexicalModel.h"
#include "StaticData.h"
#include "InputFileStream.h"
#include "TranslationOption.h"
#include "UserMessage.h"
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
using namespace std;
@ -186,11 +186,4 @@ void GlobalLexicalModel::Evaluate
GetFromCacheOrScorePhrase(context.GetTargetPhrase()) );
}
void GlobalLexicalModel::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
}

28
moses/GlobalLexicalModel.h → moses/FF/GlobalLexicalModel.h
View File

@ -1,17 +1,18 @@
#ifndef moses_GlobalLexicalModel_h
#define moses_GlobalLexicalModel_h
#include <stdexcept>
#include <string>
#include <vector>
#include <memory>
#include "Factor.h"
#include "Phrase.h"
#include "TypeDef.h"
#include "Util.h"
#include "WordsRange.h"
#include "FeatureFunction.h"
#include "FactorTypeSet.h"
#include "Sentence.h"
#include "StatelessFeatureFunction.h"
#include "moses/Factor.h"
#include "moses/Phrase.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "moses/FactorTypeSet.h"
#include "moses/Sentence.h"
#ifdef WITH_THREADS
#include <boost/thread/tss.hpp>
@ -75,18 +76,9 @@ public:
const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
std::cerr << "EvaluateChart not implemented." << std::endl;
exit(1);
throw std::logic_error("GlobalLexicalModel not supported in chart decoder, yet");
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSource; }
};
}

13
moses/GlobalLexicalModelUnlimited.cpp → moses/FF/GlobalLexicalModelUnlimited.cpp
View File

@ -1,8 +1,8 @@
#include "GlobalLexicalModelUnlimited.h"
#include <fstream>
#include "StaticData.h"
#include "InputFileStream.h"
#include "UserMessage.h"
#include "moses/StaticData.h"
#include "moses/InputFileStream.h"
#include "moses/UserMessage.h"
#include "util/string_piece_hash.hh"
using namespace std;
@ -334,11 +334,4 @@ void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulat
}
void GlobalLexicalModelUnlimited::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
}

36
moses/GlobalLexicalModelUnlimited.h → moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -1,18 +1,22 @@
#ifndef GLOBALLEXICALMODELUNLIMITED_H_
#define GLOBALLEXICALMODELUNLIMITED_H_
#include <stdexcept>
#include <string>
#include <vector>
#include "Factor.h"
#include "Phrase.h"
#include "TypeDef.h"
#include "Util.h"
#include "WordsRange.h"
#include "FeatureFunction.h"
#include "FactorTypeSet.h"
#include "Sentence.h"
#include <boost/unordered_set.hpp>
#include <boost/unordered_map.hpp>
#include "FFState.h"
#include "StatelessFeatureFunction.h"
#include "moses/Factor.h"
#include "moses/Phrase.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "moses/FactorTypeSet.h"
#include "moses/Sentence.h"
#include "moses/FF/FFState.h"
#ifdef WITH_THREADS
#include <boost/thread/tss.hpp>
@ -62,8 +66,8 @@ private:
bool m_biasFeature;
bool m_ignorePunctuation;
std::set<std::string> m_vocabSource;
std::set<std::string> m_vocabTarget;
boost::unordered_set<std::string> m_vocabSource;
boost::unordered_set<std::string> m_vocabTarget;
public:
GlobalLexicalModelUnlimited(const std::string &line);
@ -84,21 +88,13 @@ public:
void EvaluateChart(const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const {
/* Not implemented */
assert(0);
throw std::logic_error("GlobalLexicalModelUnlimited not supported in chart decoder, yet");
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void AddFeature(ScoreComponentCollection* accumulator,
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
StringPiece targetWord) const;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSource; }
};
}

33
moses/FF/PhraseBasedFeatureContext.cpp Normal file
View File

@ -0,0 +1,33 @@
#include "PhraseBasedFeatureContext.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/TranslationOption.h"
namespace Moses
{
PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) :
m_hypothesis(hypothesis),
m_translationOption(m_hypothesis->GetTranslationOption()),
m_source(m_hypothesis->GetManager().GetSource()) {}
PhraseBasedFeatureContext::PhraseBasedFeatureContext
(const TranslationOption& translationOption, const InputType& source) :
m_hypothesis(NULL),
m_translationOption(translationOption),
m_source(source)
{}
const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const
{
return m_translationOption.GetTargetPhrase();
}
const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const
{
if (!m_hypothesis) {
throw std::logic_error("Coverage vector not available during pre-calculation");
}
return m_hypothesis->GetWordsBitmap();
}
}

42
moses/FF/PhraseBasedFeatureContext.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
namespace Moses
{
class Hypothesis;
class TranslationOption;
class InputType;
class TargetPhrase;
class WordsBitmap;
/**
* Contains all that a feature function can access without affecting recombination.
* For stateless features, this is all that it can access. Currently this is not
* used for stateful features, as it would need to be retro-fitted to the LM feature.
* TODO: Expose source segmentation,lattice path.
* XXX Don't add anything to the context that would break recombination XXX
**/
class PhraseBasedFeatureContext
{
// The context either has a hypothesis (during search), or a TranslationOption and
// source sentence (during pre-calculation).
const Hypothesis* m_hypothesis;
const TranslationOption& m_translationOption;
const InputType& m_source;
public:
PhraseBasedFeatureContext(const Hypothesis* hypothesis);
PhraseBasedFeatureContext(const TranslationOption& translationOption,
const InputType& source);
const TranslationOption& GetTranslationOption() const
{ return m_translationOption; }
const InputType& GetSource() const
{ return m_source; }
const TargetPhrase& GetTargetPhrase() const; //convenience method
const WordsBitmap& GetWordsBitmap() const;
};
} // namespace

9
moses/PhraseBoundaryFeature.cpp → moses/FF/PhraseBoundaryFeature.cpp
View File

@ -1,6 +1,6 @@
#include "PhraseBoundaryFeature.h"
#include "Hypothesis.h"
#include "moses/Hypothesis.h"
using namespace std;
@ -94,12 +94,5 @@ FFState* PhraseBoundaryFeature::Evaluate
return new PhraseBoundaryState(endSourceWord,endTargetWord);
}
void PhraseBoundaryFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
}

19
moses/PhraseBoundaryFeature.h → moses/FF/PhraseBoundaryFeature.h
View File

@ -1,12 +1,13 @@
#ifndef moses_PhraseBoundaryFeature_h
#define moses_PhraseBoundaryFeature_h
#include <stdexcept>
#include <sstream>
#include <string>
#include "FeatureFunction.h"
#include "FFState.h"
#include "Word.h"
#include "StatefulFeatureFunction.h"
#include "moses/FF/FFState.h"
#include "moses/Word.h"
namespace Moses
{
@ -42,14 +43,10 @@ public:
virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const
{
abort();
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
ScoreComponentCollection* ) const {
throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
}
private:
void AddFeatures(
const Word* leftWord, const Word* rightWord, const FactorList& factors,

44
moses/FF/PhraseLengthFeature.cpp Normal file
View File

@ -0,0 +1,44 @@
#include <sstream>
#include "PhraseLengthFeature.h"
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
namespace Moses {
using namespace std;
PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
:StatelessFeatureFunction("PhraseLengthFeature", 0, line)
{
}
void PhraseLengthFeature::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
// get length of source and target phrase
size_t targetLength = targetPhrase.GetSize();
size_t sourceLength = source.GetSize();
// create feature names
stringstream nameSource;
nameSource << "s" << sourceLength;
stringstream nameTarget;
nameTarget << "t" << targetLength;
stringstream nameBoth;
nameBoth << sourceLength << "," << targetLength;
// increase feature counts
scoreBreakdown.PlusEquals(this,nameSource.str(),1);
scoreBreakdown.PlusEquals(this,nameTarget.str(),1);
scoreBreakdown.PlusEquals(this,nameBoth.str(),1);
//cerr << nameSource.str() << " " << nameTarget.str() << " " << nameBoth.str() << endl;
}
}

19
moses/PhraseLengthFeature.h → moses/FF/PhraseLengthFeature.h
View File

@ -5,10 +5,10 @@
#include <string>
#include <map>
#include "FactorCollection.h"
#include "FeatureFunction.h"
#include "FFState.h"
#include "Word.h"
#include "StatelessFeatureFunction.h"
#include "moses/FF/FFState.h"
#include "moses/Word.h"
#include "moses/FactorCollection.h"
namespace Moses
{
@ -19,18 +19,13 @@ class PhraseLengthFeature : public StatelessFeatureFunction {
public:
PhraseLengthFeature(const std::string &line);
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSegmentation; }
void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection*) const {
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
}
}
virtual void Evaluate(const TargetPhrase &targetPhrase
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;

15
moses/PhrasePairFeature.cpp → moses/FF/PhrasePairFeature.cpp
View File

@ -1,10 +1,10 @@
#include <boost/algorithm/string.hpp>
#include "AlignmentInfo.h"
#include "PhrasePairFeature.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "TranslationOption.h"
#include "moses/AlignmentInfo.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/TranslationOption.h"
#include "util/string_piece_hash.hh"
using namespace std;
@ -243,11 +243,4 @@ void PhrasePairFeature::Evaluate(
}
}
void PhrasePairFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
}

14
moses/PhrasePairFeature.h → moses/FF/PhrasePairFeature.h
View File

@ -2,10 +2,11 @@
#define moses_PhrasePairFeature_h
#include <stdexcept>
#include <boost/unordered_set.hpp>
#include "Factor.h"
#include "FeatureFunction.h"
#include "Sentence.h"
#include "StatelessFeatureFunction.h"
#include "moses/Factor.h"
#include "moses/Sentence.h"
namespace Moses {
@ -17,8 +18,7 @@ class PhrasePairFeature: public StatelessFeatureFunction {
typedef std::map< char, short > CharHash;
typedef std::vector< std::set<std::string> > DocumentVector;
std::set<std::string> m_vocabSource;
//std::set<std::string> m_vocabTarget;
boost::unordered_set<std::string> m_vocabSource;
DocumentVector m_vocabDomain;
FactorType m_sourceFactorId;
FactorType m_targetFactorId;
@ -40,10 +40,6 @@ class PhrasePairFeature: public StatelessFeatureFunction {
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/);
};

49
moses/SourceWordDeletionFeature.cpp → moses/FF/SourceWordDeletionFeature.cpp
View File

@ -1,13 +1,14 @@
#include <sstream>
#include "SourceWordDeletionFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "TranslationOption.h"
#include "UserMessage.h"
#include "Util.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ChartHypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
#include "moses/Util.h"
#include "util/string_piece_hash.hh"
namespace Moses {
@ -65,37 +66,23 @@ bool SourceWordDeletionFeature::Load(const std::string &filePath)
return true;
}
void SourceWordDeletionFeature::Evaluate(
const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
ComputeFeatures(targetPhrase, accumulator, alignmentInfo);
}
void SourceWordDeletionFeature::EvaluateChart(
const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
const AlignmentInfo &alignmentInfo = context.GetTargetPhrase().GetAlignTerm();
ComputeFeatures(context.GetTargetPhrase(), accumulator, alignmentInfo);
}
void SourceWordDeletionFeature::Evaluate(const TargetPhrase &targetPhrase
void SourceWordDeletionFeature::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
ComputeFeatures(source, targetPhrase, &scoreBreakdown, alignmentInfo);
}
void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase,
void SourceWordDeletionFeature::ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const
{
// handle special case: unknown words (they have no word alignment)
size_t targetLength = targetPhrase.GetSize();
size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize();
size_t sourceLength = source.GetSize();
if (targetLength == 1 && sourceLength == 1 && !alignmentInfo.GetSize()) return;
// flag aligned words
@ -109,7 +96,7 @@ void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase
// process unaligned source words
for(size_t i=0; i<sourceLength; i++) {
if (!aligned[i]) {
Word w = targetPhrase.GetSourcePhrase().GetWord(i);
const Word &w = source.GetWord(i);
if (!w.IsNonTerminal()) {
const StringPiece word = w.GetFactor(m_factorType)->GetString();
if (word != "<s>" && word != "</s>") {
@ -117,7 +104,7 @@ void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase
accumulator->PlusEquals(this, StringPiece("OTHER"),1);
}
else {
accumulator->PlusEquals(this,word,1);
accumulator->PlusEquals(this,word,1);
}
}
}

24
moses/SourceWordDeletionFeature.h → moses/FF/SourceWordDeletionFeature.h
View File

@ -2,11 +2,11 @@
#define moses_SourceWordDeletionFeature_h
#include <string>
#include <map>
#include <boost/unordered_set.hpp>
#include "FeatureFunction.h"
#include "FactorCollection.h"
#include "AlignmentInfo.h"
#include "StatelessFeatureFunction.h"
#include "moses/FactorCollection.h"
#include "moses/AlignmentInfo.h"
namespace Moses
{
@ -15,7 +15,7 @@ namespace Moses
*/
class SourceWordDeletionFeature : public StatelessFeatureFunction {
private:
std::set<std::string> m_vocab;
boost::unordered_set<std::string> m_vocab;
FactorType m_factorType;
bool m_unrestricted;
@ -23,22 +23,16 @@ public:
SourceWordDeletionFeature(const std::string &line);
bool Load(const std::string &filePath);
void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSegmentation; }
};
}

19
moses/FF/StatefulFeatureFunction.cpp Normal file
View File

@ -0,0 +1,19 @@
#include "StatefulFeatureFunction.h"
namespace Moses
{
StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, const std::string &line)
: FeatureFunction(description, line)
{
m_statefulFFs.push_back(this);
}
StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
: FeatureFunction(description,numScoreComponents, line)
{
m_statefulFFs.push_back(this);
}
}

51
moses/FF/StatefulFeatureFunction.h Normal file
View File

@ -0,0 +1,51 @@
#pragma once
#include "FeatureFunction.h"
namespace Moses
{
/** base class for all stateful feature functions.
* eg. LM, distortion penalty
*/
class StatefulFeatureFunction: public FeatureFunction
{
//All statefull FFs
static std::vector<const StatefulFeatureFunction*> m_statefulFFs;
public:
static const std::vector<const StatefulFeatureFunction*>& GetStatefulFeatureFunctions() {return m_statefulFFs;}
StatefulFeatureFunction(const std::string& description, const std::string &line);
StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
/**
* \brief This interface should be implemented.
* Notes: When evaluating the value of this feature function, you should avoid
* calling hypo.GetPrevHypo(). If you need something from the "previous"
* hypothesis, you should store it in an FFState object which will be passed
* in as prev_state. If you don't do this, you will get in trouble.
*/
virtual FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const = 0;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const = 0;
//! return the state associated with the empty hypothesis for a given sentence
virtual const FFState* EmptyHypothesisState(const InputType &input) const = 0;
bool IsStateless() const
{ return false; }
};
}

19
moses/FF/StatelessFeatureFunction.cpp Normal file
View File

@ -0,0 +1,19 @@
#include "StatelessFeatureFunction.h"
namespace Moses
{
StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, const std::string &line)
:FeatureFunction(description, line)
{
m_statelessFFs.push_back(this);
}
StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
:FeatureFunction(description, numScoreComponents, line)
{
m_statelessFFs.push_back(this);
}
}

42
moses/FF/StatelessFeatureFunction.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#include "FeatureFunction.h"
namespace Moses
{
/** base class for all stateless feature functions.
* eg. phrase table, word penalty, phrase penalty
*/
class StatelessFeatureFunction: public FeatureFunction
{
//All stateless FFs, except those that cache scores in T-Option
static std::vector<const StatelessFeatureFunction*> m_statelessFFs;
public:
static const std::vector<const StatelessFeatureFunction*>& GetStatelessFeatureFunctions() {return m_statelessFFs;}
StatelessFeatureFunction(const std::string& description, const std::string &line);
StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
/**
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{}
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{}
virtual bool IsStateless() const
{ return true; }
};
} // namespace

15
moses/TargetBigramFeature.cpp → moses/FF/TargetBigramFeature.cpp
View File

@ -1,8 +1,8 @@
#include "TargetBigramFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ScoreComponentCollection.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "util/string_piece_hash.hh"
using namespace std;
@ -112,12 +112,5 @@ FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
return new TargetBigramState(targetPhrase.GetWord(targetPhrase.GetSize()-1));
}
void TargetBigramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
}

15
moses/TargetBigramFeature.h → moses/FF/TargetBigramFeature.h
View File

@ -3,11 +3,12 @@
#include <string>
#include <map>
#include <boost/unordered_set.hpp>
#include "FactorCollection.h"
#include "FeatureFunction.h"
#include "FFState.h"
#include "Word.h"
#include "moses/FF/FFState.h"
#include "StatefulFeatureFunction.h"
#include "moses/FactorCollection.h"
#include "moses/Word.h"
namespace Moses
{
@ -42,14 +43,10 @@ public:
abort();
}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
private:
FactorType m_factorType;
Word m_bos;
std::set<std::string> m_vocab;
boost::unordered_set<std::string> m_vocab;
};
}

17
moses/TargetNgramFeature.cpp → moses/FF/TargetNgramFeature.cpp
View File

@ -1,9 +1,9 @@
#include "TargetNgramFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ScoreComponentCollection.h"
#include "ChartHypothesis.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/ChartHypothesis.h"
#include "util/string_piece_hash.hh"
@ -393,13 +393,6 @@ FFState* TargetNgramFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int
return new TargetNgramChartState(cur_hypo, featureId, m_n);
}
void TargetNgramFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
CHECK(false);
}
void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const {
stringstream ngram;
size_t size = contextFactor.size();

22
moses/TargetNgramFeature.h → moses/FF/TargetNgramFeature.h
View File

@ -3,15 +3,15 @@
#include <string>
#include <map>
#include <boost/unordered_set.hpp>
#include "FactorCollection.h"
#include "FeatureFunction.h"
#include "FFState.h"
#include "Word.h"
#include "LM/SingleFactor.h"
#include "ChartHypothesis.h"
#include "ChartManager.h"
#include "StatefulFeatureFunction.h"
#include "moses/FF/FFState.h"
#include "moses/Word.h"
#include "moses/FactorCollection.h"
#include "moses/LM/SingleFactor.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
namespace Moses
{
@ -192,14 +192,10 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
private:
FactorType m_factorType;
Word m_bos;
std::set<std::string> m_vocab;
boost::unordered_set<std::string> m_vocab;
size_t m_n;
bool m_lower_ngrams;

43
moses/TargetWordInsertionFeature.cpp → moses/FF/TargetWordInsertionFeature.cpp
View File

@ -1,12 +1,12 @@
#include <sstream>
#include "TargetWordInsertionFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "TranslationOption.h"
#include "UserMessage.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ChartHypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
#include "util/string_piece_hash.hh"
namespace Moses {
@ -66,38 +66,23 @@ bool TargetWordInsertionFeature::Load(const std::string &filePath)
return true;
}
void TargetWordInsertionFeature::Evaluate(
const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
ComputeFeatures(targetPhrase, accumulator, alignmentInfo);
}
void TargetWordInsertionFeature::EvaluateChart(
const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const AlignmentInfo &alignmentInfo = context.GetTargetPhrase().GetAlignTerm();
ComputeFeatures(targetPhrase, accumulator, alignmentInfo);
}
void TargetWordInsertionFeature::Evaluate(const TargetPhrase &targetPhrase
void TargetWordInsertionFeature::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignTerm();
ComputeFeatures(source, targetPhrase, &scoreBreakdown, alignmentInfo);
}
void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhrase,
void TargetWordInsertionFeature::ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const
{
// handle special case: unknown words (they have no word alignment)
size_t targetLength = targetPhrase.GetSize();
size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize();
size_t sourceLength = source.GetSize();
if (targetLength == 1 && sourceLength == 1 && !alignmentInfo.GetSize()) return;
// flag aligned words

24
moses/TargetWordInsertionFeature.h → moses/FF/TargetWordInsertionFeature.h
View File

@ -2,11 +2,11 @@
#define moses_TargetWordInsertionFeature_h
#include <string>
#include <map>
#include <boost/unordered_set.hpp>
#include "FeatureFunction.h"
#include "FactorCollection.h"
#include "AlignmentInfo.h"
#include "StatelessFeatureFunction.h"
#include "moses/FactorCollection.h"
#include "moses/AlignmentInfo.h"
namespace Moses
{
@ -15,7 +15,7 @@ namespace Moses
*/
class TargetWordInsertionFeature : public StatelessFeatureFunction {
private:
std::set<std::string> m_vocab;
boost::unordered_set<std::string> m_vocab;
FactorType m_factorType;
bool m_unrestricted;
@ -23,23 +23,17 @@ public:
TargetWordInsertionFeature(const std::string &line);
bool Load(const std::string &filePath);
void Evaluate( const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
void EvaluateChart( const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void ComputeFeatures(const TargetPhrase& targetPhrase,
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSegmentation; }
};
}

7
moses/FF/UnknownWordPenaltyProducer.cpp Normal file
View File

@ -0,0 +1,7 @@
namespace Moses
{
}

26
moses/FF/UnknownWordPenaltyProducer.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
// $Id$
#include "StatelessFeatureFunction.h"
namespace Moses
{
class WordsRange;
/** unknown word penalty */
class UnknownWordPenaltyProducer : public StatelessFeatureFunction
{
public:
UnknownWordPenaltyProducer(const std::string &line)
: StatelessFeatureFunction("UnknownWordPenalty",1, line)
{
m_tuneable = false;
}
};
}

17
moses/FF/WordPenaltyProducer.cpp Normal file
View File

@ -0,0 +1,17 @@
#include "WordPenaltyProducer.h"
#include "moses/TargetPhrase.h"
#include "moses/ScoreComponentCollection.h"
namespace Moses
{
void WordPenaltyProducer::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
float score = - (float) targetPhrase.GetNumTerminals();
scoreBreakdown.Assign(this, score);
}
}

27
moses/FF/WordPenaltyProducer.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <string>
#include "StatelessFeatureFunction.h"
namespace Moses
{
class TargetPhrase;
class ScoreComponentCollection;
/** Doesn't do anything but provide a key into the global
* score array to store the word penalty in.
*/
class WordPenaltyProducer : public StatelessFeatureFunction
{
public:
WordPenaltyProducer(const std::string &line) : StatelessFeatureFunction("WordPenalty",1, line) {}
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
};
}

39
moses/WordTranslationFeature.cpp → moses/FF/WordTranslationFeature.cpp
View File

@ -1,13 +1,13 @@
#include <sstream>
#include <boost/algorithm/string.hpp>
#include "WordTranslationFeature.h"
#include "Phrase.h"
#include "TargetPhrase.h"
#include "Hypothesis.h"
#include "ChartHypothesis.h"
#include "ScoreComponentCollection.h"
#include "TranslationOption.h"
#include "UserMessage.h"
#include "moses/Phrase.h"
#include "moses/TargetPhrase.h"
#include "moses/Hypothesis.h"
#include "moses/ChartHypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
#include "util/string_piece_hash.hh"
namespace Moses {
@ -211,19 +211,19 @@ void WordTranslationFeature::Evaluate
featureName << targetWord;
accumulator->SparsePlusEquals(featureName.str(), 1);
}
if (m_domainTrigger && !m_sourceContext) {
if (m_domainTrigger && !m_sourceContext) {
const bool use_topicid = input.GetUseTopicId();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
if (use_topicid || use_topicid_prob) {
if(use_topicid) {
// use topicid as trigger
const long topicid = input.GetTopicId();
stringstream feature;
feature << m_description << "_";
if (topicid == -1)
if (topicid == -1)
feature << "unk";
else
feature << topicid;
else
feature << topicid;
feature << "_";
feature << sourceWord;
@ -268,7 +268,7 @@ void WordTranslationFeature::Evaluate
feature << sourceWord;
feature << "~";
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
accumulator->SparsePlusEquals(feature.str(), 1);
}
}
}
@ -284,7 +284,7 @@ void WordTranslationFeature::Evaluate
feature << targetWord;
accumulator->SparsePlusEquals(feature.str(), 1);
}
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {
if (contextIndex == globalSourceIndex) continue;
@ -296,14 +296,14 @@ void WordTranslationFeature::Evaluate
if(charIterator != m_punctuationHash.end())
continue;
}
const long docid = input.GetDocumentId();
bool sourceTriggerExists = false;
if (m_domainTrigger)
sourceTriggerExists = FindStringPiece(m_vocabDomain[docid], sourceTrigger ) != m_vocabDomain[docid].end();
else if (!m_unrestricted)
sourceTriggerExists = FindStringPiece(m_vocabSource, sourceTrigger ) != m_vocabSource.end();
if (m_domainTrigger) {
if (sourceTriggerExists) {
stringstream feature;
@ -527,11 +527,4 @@ void WordTranslationFeature::EvaluateChart(
}
void WordTranslationFeature::Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
}
}

14
moses/WordTranslationFeature.h → moses/FF/WordTranslationFeature.h
View File

@ -4,11 +4,10 @@
#include <string>
#include <boost/unordered_set.hpp>
#include "FeatureFunction.h"
#include "FactorCollection.h"
#include "Sentence.h"
#include "moses/FactorCollection.h"
#include "moses/Sentence.h"
#include "FFState.h"
#include "StatelessFeatureFunction.h"
namespace Moses
{
@ -48,13 +47,6 @@ public:
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresSegmentation; }
};
}

182
moses/FeatureFunction.cpp
View File

@ -1,182 +0,0 @@
#include <stdexcept>
#include "util/check.hh"
#include "ChartHypothesis.h"
#include "ChartManager.h"
#include "FeatureFunction.h"
#include "Hypothesis.h"
#include "Manager.h"
#include "TranslationOption.h"
using namespace std;
namespace Moses
{
PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) :
m_hypothesis(hypothesis),
m_translationOption(m_hypothesis->GetTranslationOption()),
m_source(m_hypothesis->GetManager().GetSource()) {}
PhraseBasedFeatureContext::PhraseBasedFeatureContext
(const TranslationOption& translationOption, const InputType& source) :
m_hypothesis(NULL),
m_translationOption(translationOption),
m_source(source) {}
const TranslationOption& PhraseBasedFeatureContext::GetTranslationOption() const
{
return m_translationOption;
}
const InputType& PhraseBasedFeatureContext::GetSource() const
{
return m_source;
}
const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const
{
return m_translationOption.GetTargetPhrase();
}
const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const
{
if (!m_hypothesis) {
throw std::logic_error("Coverage vector not available during pre-calculation");
}
return m_hypothesis->GetWordsBitmap();
}
ChartBasedFeatureContext::ChartBasedFeatureContext
(const ChartHypothesis* hypothesis):
m_hypothesis(hypothesis),
m_targetPhrase(hypothesis->GetCurrTargetPhrase()),
m_source(hypothesis->GetManager().GetSource()) {}
ChartBasedFeatureContext::ChartBasedFeatureContext(
const TargetPhrase& targetPhrase,
const InputType& source):
m_hypothesis(NULL),
m_targetPhrase(targetPhrase),
m_source(source) {}
const InputType& ChartBasedFeatureContext::GetSource() const
{
return m_source;
}
const TargetPhrase& ChartBasedFeatureContext::GetTargetPhrase() const
{
return m_targetPhrase;
}
multiset<string> FeatureFunction::description_counts;
std::vector<FeatureFunction*> FeatureFunction::m_producers;
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
std::vector<const StatefulFeatureFunction*> StatefulFeatureFunction::m_statefulFFs;
FeatureFunction::FeatureFunction(const std::string& description, const std::string &line)
: m_tuneable(true)
{
ParseLine(description, line);
if (m_description == "") {
// not been given a name. Make a unique name
size_t index = description_counts.count(description);
ostringstream dstream;
dstream << description;
dstream << index;
description_counts.insert(description);
m_description = dstream.str();
}
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
}
FeatureFunction::FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
: m_numScoreComponents(numScoreComponents)
, m_tuneable(true)
{
ParseLine(description, line);
if (m_description == "") {
size_t index = description_counts.count(description);
ostringstream dstream;
dstream << description;
dstream << index;
description_counts.insert(description);
m_description = dstream.str();
}
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
}
FeatureFunction::~FeatureFunction() {}
void FeatureFunction::ParseLine(const std::string& description, const std::string &line)
{
vector<string> toks = Tokenize(line);
CHECK(toks.size());
//CHECK(toks[0] == description);
for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = Tokenize(toks[i], "=");
CHECK(args.size() == 2);
if (args[0] == "num-features") {
m_numScoreComponents = Scan<size_t>(args[1]);
}
else if (args[0] == "name") {
m_description = args[1];
}
else if (args[0] == "tuneable") {
m_tuneable = Scan<bool>(args[1]);
}
else {
m_args.push_back(args);
}
}
}
StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, const std::string &line)
:FeatureFunction(description, line)
{
m_statelessFFs.push_back(this);
}
StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
:FeatureFunction(description, numScoreComponents, line)
{
m_statelessFFs.push_back(this);
}
StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, const std::string &line)
: FeatureFunction(description, line)
{
m_statefulFFs.push_back(this);
}
StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
: FeatureFunction(description,numScoreComponents, line)
{
m_statefulFFs.push_back(this);
}
bool StatefulFeatureFunction::IsStateless() const
{
return false;
}
}

203
moses/FeatureFunction.h
View File

@ -1,203 +0,0 @@
#ifndef moses_FeatureFunction_h
#define moses_FeatureFunction_h
#include <vector>
#include <set>
#include <string>
#include "TypeDef.h"
namespace Moses
{
class TargetPhrase;
class TranslationOption;
class Hypothesis;
class ChartHypothesis;
class FFState;
class InputType;
class ScoreComponentCollection;
class WordsBitmap;
class WordsRange;
/**
* Contains all that a feature function can access without affecting recombination.
* For stateless features, this is all that it can access. Currently this is not
* used for stateful features, as it would need to be retro-fitted to the LM feature.
* TODO: Expose source segmentation,lattice path.
* XXX Don't add anything to the context that would break recombination XXX
**/
class PhraseBasedFeatureContext
{
// The context either has a hypothesis (during search), or a TranslationOption and
// source sentence (during pre-calculation).
const Hypothesis* m_hypothesis;
const TranslationOption& m_translationOption;
const InputType& m_source;
public:
PhraseBasedFeatureContext(const Hypothesis* hypothesis);
PhraseBasedFeatureContext(const TranslationOption& translationOption,
const InputType& source);
const TranslationOption& GetTranslationOption() const;
const InputType& GetSource() const;
const TargetPhrase& GetTargetPhrase() const; //convenience method
const WordsBitmap& GetWordsBitmap() const;
};
/**
* Same as PhraseBasedFeatureContext, but for chart-based Moses.
**/
class ChartBasedFeatureContext
{
//The context either has a hypothesis (during search) or a
//TargetPhrase and source sentence (during pre-calculation)
//TODO: should the context also include some info on where the TargetPhrase
//is anchored (assuming it's lexicalised), which is available at pre-calc?
const ChartHypothesis* m_hypothesis;
const TargetPhrase& m_targetPhrase;
const InputType& m_source;
public:
ChartBasedFeatureContext(const ChartHypothesis* hypothesis);
ChartBasedFeatureContext(const TargetPhrase& targetPhrase,
const InputType& source);
const InputType& GetSource() const;
const TargetPhrase& GetTargetPhrase() const;
};
/** base class for all feature functions.
* @todo is this for pb & hiero too?
*/
class FeatureFunction
{
protected:
/**< all the score producers in this run */
static std::vector<FeatureFunction*> m_producers;
std::string m_description, m_argLine;
std::vector<std::vector<std::string> > m_args;
bool m_tuneable;
size_t m_numScoreComponents;
//In case there's multiple producers with the same description
static std::multiset<std::string> description_counts;
void ParseLine(const std::string& description, const std::string &line);
public:
static const std::vector<FeatureFunction*>& GetFeatureFunctions() { return m_producers; }
FeatureFunction(const std::string& description, const std::string &line);
FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
virtual bool IsStateless() const = 0;
virtual ~FeatureFunction();
static void ResetDescriptionCounts() {
description_counts.clear();
}
//! returns the number of scores that a subclass produces.
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
size_t GetNumScoreComponents() const {return m_numScoreComponents;}
//! returns a string description of this producer
const std::string& GetScoreProducerDescription() const
{ return m_description; }
virtual bool IsTuneable() const { return m_tuneable; }
//!
virtual void InitializeForInput(InputType const& source)
{}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing(const InputType& source)
{}
const std::string &GetArgLine() const
{ return m_argLine; }
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const = 0;
};
/** base class for all stateless feature functions.
* eg. phrase table, word penalty, phrase penalty
*/
class StatelessFeatureFunction: public FeatureFunction
{
//All stateless FFs, except those that cache scores in T-Option
static std::vector<const StatelessFeatureFunction*> m_statelessFFs;
public:
static const std::vector<const StatelessFeatureFunction*>& GetStatelessFeatureFunctions() {return m_statelessFFs;}
StatelessFeatureFunction(const std::string& description, const std::string &line);
StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
/**
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const = 0;
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const = 0;
virtual StatelessFeatureType GetStatelessFeatureType() const
{ return RequiresTargetPhrase; }
bool IsStateless() const
{ return true; }
};
/** base class for all stateful feature functions.
* eg. LM, distortion penalty
*/
class StatefulFeatureFunction: public FeatureFunction
{
//All statefull FFs
static std::vector<const StatefulFeatureFunction*> m_statefulFFs;
public:
static const std::vector<const StatefulFeatureFunction*>& GetStatefulFeatureFunctions() {return m_statefulFFs;}
StatefulFeatureFunction(const std::string& description, const std::string &line);
StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
/**
* \brief This interface should be implemented.
* Notes: When evaluating the value of this feature function, you should avoid
* calling hypo.GetPrevHypo(). If you need something from the "previous"
* hypothesis, you should store it in an FFState object which will be passed
* in as prev_state. If you don't do this, you will get in trouble.
*/
virtual FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const = 0;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const = 0;
//! return the state associated with the empty hypothesis for a given sentence
virtual const FFState* EmptyHypothesisState(const InputType &input) const = 0;
bool IsStateless() const;
};
}
#endif

13
moses/GenerationDictionary.h
View File

@ -70,19 +70,6 @@ public:
*/
const OutputWordCollection *FindWord(const Word &word) const;
//Usual feature function methods are not implemented
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
throw std::logic_error("GenerationDictionary::Evaluate() Not implemented");
}
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{
throw std::logic_error("GenerationDictionary.Evaluate() Not implemented");
}
};

8
moses/Hypothesis.cpp
View File

@ -25,10 +25,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <vector>
#include <algorithm>
#include "FFState.h"
#include "TranslationOption.h"
#include "TranslationOptionCollection.h"
#include "DummyScoreProducers.h"
#include "Hypothesis.h"
#include "Util.h"
#include "SquareMatrix.h"
@ -36,6 +34,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "StaticData.h"
#include "InputType.h"
#include "Manager.h"
#include "moses/FF/FFState.h"
using namespace std;
@ -271,7 +270,6 @@ void Hypothesis::EvaluateWith(const StatelessFeatureFunction& slff) {
*/
void Hypothesis::CalcScore(const SquareMatrix &futureScore)
{
const StaticData &staticData = StaticData::Instance();
clock_t t=0; // used to track time
// some stateless score producers cache their values in the translation
@ -285,9 +283,7 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore)
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i) {
const StatelessFeatureFunction &ff = *sfs[i];
if (ff.GetStatelessFeatureType() == RequiresSegmentation) {
EvaluateWith(ff);
}
EvaluateWith(ff);
}
const vector<const StatefulFeatureFunction*>& ffs =

2
moses/Hypothesis.h
View File

@ -45,6 +45,8 @@ class TranslationOption;
class WordsRange;
class Hypothesis;
class FFState;
class StatelessFeatureFunction;
class StatefulFeatureFunction;
class Manager;
class LexicalReordering;

37
moses/Incremental.cpp
View File

@ -1,3 +1,5 @@
#include <stdexcept>
#include "moses/Incremental.h"
#include "moses/ChartCell.h"
@ -5,6 +7,7 @@
#include "moses/FeatureVector.h"
#include "moses/StaticData.h"
#include "moses/Util.h"
#include "moses/LM/Base.h"
#include "lm/model.hh"
#include "search/applied.hh"
@ -181,8 +184,7 @@ Manager::~Manager() {
}
template <class Model, class Best> search::History Manager::PopulateBest(const Model &model, const std::vector<lm::WordIndex> &words, Best &out) {
const LMList &lmList = StaticData::Instance().GetLMList();
const LanguageModel &abstract = **lmList.begin();
const LanguageModel &abstract = GetFirstLM();
const float oov_weight = abstract.OOVFeatureEnabled() ? abstract.GetOOVWeight() : 0.0;
const StaticData &data = StaticData::Instance();
search::Config config(abstract.GetWeight(), data.GetCubePruningPopLimit(), search::NBestConfig(data.GetNBestSize()));
@ -236,10 +238,7 @@ template void Manager::LMCallback<lm::ngram::ArrayTrieModel>(const lm::ngram::Ar
template void Manager::LMCallback<lm::ngram::QuantArrayTrieModel>(const lm::ngram::QuantArrayTrieModel &model, const std::vector<lm::WordIndex> &words);
const std::vector<search::Applied> &Manager::ProcessSentence() {
const LMList &lmList = StaticData::Instance().GetLMList();
UTIL_THROW_IF(lmList.size() != 1, util::Exception, "Incremental search only supports one language model.");
(*lmList.begin())->IncrementalCallback(*this);
GetFirstLM().IncrementalCallback(*this);
return *completed_nbest_;
}
@ -286,12 +285,34 @@ void PhraseAndFeatures(const search::Applied final, Phrase &phrase, ScoreCompone
float full, ignored_ngram;
std::size_t ignored_oov;
const LMList &lmList = StaticData::Instance().GetLMList();
const LanguageModel &model = **lmList.begin();
const LanguageModel &model = GetFirstLM();
model.CalcScore(phrase, full, ignored_ngram, ignored_oov);
// CalcScore transforms, but EvaluateChart doesn't.
features.Assign(&model, full);
}
const LanguageModel &GetFirstLM()
{
static const LanguageModel *lmStatic = NULL;
if (lmStatic) {
return *lmStatic;
}
// 1st time looking up lm
const std::vector<const StatefulFeatureFunction*> &statefulFFs = StatefulFeatureFunction::GetStatefulFeatureFunctions();
for (size_t i = 0; i < statefulFFs.size(); ++i) {
const StatefulFeatureFunction *ff = statefulFFs[i];
const LanguageModel *lm = dynamic_cast<const LanguageModel*>(ff);
if (lm) {
lmStatic = lm;
return *lmStatic;
}
}
throw std::logic_error("Incremental search only supports one language model.");
}
} // namespace Incremental
} // namespace Moses

4
moses/Incremental.h
View File

@ -13,6 +13,8 @@
namespace Moses {
class ScoreComponentCollection;
class InputType;
class LanguageModel;
namespace Incremental {
class Manager {
@ -53,6 +55,8 @@ void ToPhrase(const search::Applied final, Phrase &out);
// Get the phrase and the features.
void PhraseAndFeatures(const search::Applied final, Phrase &phrase, ScoreComponentCollection &features);
const LanguageModel &GetFirstLM();
} // namespace Incremental
} // namespace Moses

1
moses/Jamfile
View File

@ -40,6 +40,7 @@ lib moses :
TranslationModel/RuleTable/*.cpp
TranslationModel/Scope3Parser/*.cpp
TranslationModel/CYKPlusParser/*.cpp
FF/*.cpp
: #exceptions
ThreadPool.cpp
SyntacticLanguageModel.cpp

3
moses/LM/Base.cpp
View File

@ -60,7 +60,8 @@ void LanguageModel::IncrementalCallback(Incremental::Manager &manager) const {
UTIL_THROW(util::Exception, "Incremental search is only supported by KenLM.");
}
void LanguageModel::Evaluate(const TargetPhrase &targetPhrase
void LanguageModel::Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{

5
moses/LM/Base.h
View File

@ -25,7 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <string>
#include <cstddef>
#include "moses/FeatureFunction.h"
#include "moses/FF/StatefulFeatureFunction.h"
namespace Moses
{
@ -86,7 +86,8 @@ public:
// KenLM only (others throw an exception): call incremental search with the model and mapping.
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual void Evaluate(const TargetPhrase &targetPhrase
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;

2
moses/LM/ChartState.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"

2
moses/LM/Implementation.cpp
View File

@ -24,7 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <memory>
#include <sstream>
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
#include "Implementation.h"
#include "ChartState.h"
#include "moses/TypeDef.h"

2
moses/LM/Implementation.h
View File

@ -27,7 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/Factor.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/FeatureFunction.h"
#include "moses/FF/FeatureFunction.h"
#include "moses/Word.h"
#include "Base.h"

5
moses/LM/Ken.cpp
View File

@ -32,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Ken.h"
#include "Base.h"
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/FactorCollection.h"
@ -337,7 +337,6 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const Cha
LanguageModel *ConstructKenLM(const std::string &description, const std::string &line)
{
FactorType factorType;
size_t nGramOrder;
string filePath;
bool lazy;
@ -350,7 +349,7 @@ LanguageModel *ConstructKenLM(const std::string &description, const std::string
factorType = Scan<FactorType>(args[1]);
}
else if (args[0] == "order") {
nGramOrder = Scan<size_t>(args[1]);
//nGramOrder = Scan<size_t>(args[1]);
}
else if (args[0] == "path") {
filePath = args[1];

2
moses/LM/PointerState.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
namespace Moses
{

2
moses/LM/SingleFactor.cpp
View File

@ -25,10 +25,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "SingleFactor.h"
#include "PointerState.h"
#include "moses/FF/FFState.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/FactorCollection.h"
#include "moses/FFState.h"
#include "moses/Phrase.h"
#include "moses/StaticData.h"

Some files were not shown because too many files have changed in this diff Show More