moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-26 21:42:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Barry Haddow 2013-09-06 13:29:29 +01:00
commit 867c6efe6c
132 changed files with 3704 additions and 3555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -67,3 +67,7 @@ contrib/other-builds/*.xcodeproj/xcuserdata/
*/*.xcodeproj/xcuserdata
mert/sentence-bleu
._*
.DS_Store
*.pbxuser
*.mode1v3

6
Jamroot
View File

@ -78,12 +78,10 @@ external-lib z ;
if ! [ option.get "without-tcmalloc" : : "yes" ] && [ test_library "tcmalloc_minimal" ] {
if [ option.get "full-tcmalloc" : : "yes" ] {
tcmalloc = "tcmalloc" ;
external-lib unwind ;
external-lib tcmalloc : : unwind ;
requirements += <library>tcmalloc <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
external-lib tcmalloc_and_profiler : : unwind ;
requirements += <library>tcmalloc_and_profiler <library>unwind <cflags>-fno-omit-frame-pointer <cxxflags>-fno-omit-frame-pointer ;
} else {
tcmalloc = "tcmalloc_minimal" ;
external-lib tcmalloc_minimal ;
requirements += <threading>multi:<library>$(tcmalloc) ;
}

11
contrib/other-builds/lm.xcodeproj/project.pbxproj
View File

@ -34,7 +34,6 @@
1EBA44E314B97E22003CC0EA /* model_type.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449114B97E22003CC0EA /* model_type.hh */; };
1EBA44E414B97E22003CC0EA /* model.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449214B97E22003CC0EA /* model.cc */; };
1EBA44E514B97E22003CC0EA /* model.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449314B97E22003CC0EA /* model.hh */; };
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449414B97E22003CC0EA /* ngram_query.cc */; };
1EBA44E714B97E22003CC0EA /* ngram_query.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449514B97E22003CC0EA /* ngram_query.hh */; };
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */ = {isa = PBXBuildFile; fileRef = 1EBA449614B97E22003CC0EA /* quantize.cc */; };
1EBA44E914B97E22003CC0EA /* quantize.hh in Headers */ = {isa = PBXBuildFile; fileRef = 1EBA449714B97E22003CC0EA /* quantize.hh */; };
@ -149,7 +148,6 @@
1EBA449114B97E22003CC0EA /* model_type.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model_type.hh; path = ../../lm/model_type.hh; sourceTree = "<group>"; };
1EBA449214B97E22003CC0EA /* model.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = model.cc; path = ../../lm/model.cc; sourceTree = "<group>"; };
1EBA449314B97E22003CC0EA /* model.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = model.hh; path = ../../lm/model.hh; sourceTree = "<group>"; };
1EBA449414B97E22003CC0EA /* ngram_query.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = ngram_query.cc; path = ../../lm/ngram_query.cc; sourceTree = "<group>"; };
1EBA449514B97E22003CC0EA /* ngram_query.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = ngram_query.hh; path = ../../lm/ngram_query.hh; sourceTree = "<group>"; };
1EBA449614B97E22003CC0EA /* quantize.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = quantize.cc; path = ../../lm/quantize.cc; sourceTree = "<group>"; };
1EBA449714B97E22003CC0EA /* quantize.hh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; name = quantize.hh; path = ../../lm/quantize.hh; sourceTree = "<group>"; };
@ -266,7 +264,6 @@
1EBA449114B97E22003CC0EA /* model_type.hh */,
1EBA449214B97E22003CC0EA /* model.cc */,
1EBA449314B97E22003CC0EA /* model.hh */,
1EBA449414B97E22003CC0EA /* ngram_query.cc */,
1EBA449514B97E22003CC0EA /* ngram_query.hh */,
1EBA449614B97E22003CC0EA /* quantize.cc */,
1EBA449714B97E22003CC0EA /* quantize.hh */,
@ -535,7 +532,6 @@
1EBA44DF14B97E22003CC0EA /* lm_exception.cc in Sources */,
1EBA44E214B97E22003CC0EA /* model_test.cc in Sources */,
1EBA44E414B97E22003CC0EA /* model.cc in Sources */,
1EBA44E614B97E22003CC0EA /* ngram_query.cc in Sources */,
1EBA44E814B97E22003CC0EA /* quantize.cc in Sources */,
1EBA44EA14B97E22003CC0EA /* read_arpa.cc in Sources */,
1EBA44ED14B97E22003CC0EA /* search_hashed.cc in Sources */,
@ -589,6 +585,7 @@
GCC_C_LANGUAGE_STANDARD = gnu99;
GCC_DYNAMIC_NO_PIC = NO;
GCC_ENABLE_OBJC_EXCEPTIONS = YES;
GCC_INPUT_FILETYPE = automatic;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
@ -601,10 +598,11 @@
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = (
"$(LIBRARY_MY_BOOST)",
../..,
/opt/local/include,
);
MACOSX_DEPLOYMENT_TARGET = 10.7;
MACOSX_DEPLOYMENT_TARGET = 10.6;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = macosx;
};
@ -625,10 +623,11 @@
GCC_WARN_ABOUT_RETURN_TYPE = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
HEADER_SEARCH_PATHS = (
"$(XCODE_MOSES_BOOST)",
../..,
/opt/local/include,
);
MACOSX_DEPLOYMENT_TARGET = 10.7;
MACOSX_DEPLOYMENT_TARGET = 10.6;
SDKROOT = macosx;
};
name = Release;

5
contrib/other-builds/moses-chart-cmd/.cproject
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<?fileVersion 4.0.0?>
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.162355801">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.162355801" moduleId="org.eclipse.cdt.core.settings" name="Debug">
@ -85,7 +87,6 @@
<listOptionValue builtIn="false" value="pthread"/>
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="bz2"/>
<listOptionValue builtIn="false" value="rt"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.128214028" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>

5
contrib/other-builds/moses-cmd/.cproject
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<?fileVersion 4.0.0?>
<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
<storageModule moduleId="org.eclipse.cdt.core.settings">
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.461114338">
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.461114338" moduleId="org.eclipse.cdt.core.settings" name="Debug">
@ -81,7 +83,6 @@
<listOptionValue builtIn="false" value="pthread"/>
<listOptionValue builtIn="false" value="z"/>
<listOptionValue builtIn="false" value="bz2"/>
<listOptionValue builtIn="false" value="rt"/>
</option>
<option id="gnu.cpp.link.option.userobjs.1542590830" name="Other objects" superClass="gnu.cpp.link.option.userobjs"/>
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.983725033" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">

4562
contrib/other-builds/moses.xcodeproj/project.pbxproj
View File

File diff suppressed because it is too large Load Diff

123
contrib/other-builds/moses/.project
View File

@ -286,16 +286,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/ConfusionNet.h</locationURI>
</link>
<link>
<name>DecodeFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.cpp</locationURI>
</link>
<link>
<name>DecodeFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/DecodeFeature.h</locationURI>
</link>
<link>
<name>DecodeGraph.cpp</name>
<type>1</type>
@ -526,36 +516,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LVoc.h</locationURI>
</link>
<link>
<name>LexicalReordering.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.cpp</locationURI>
</link>
<link>
<name>LexicalReordering.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReordering.h</locationURI>
</link>
<link>
<name>LexicalReorderingState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.cpp</locationURI>
</link>
<link>
<name>LexicalReorderingState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingState.h</locationURI>
</link>
<link>
<name>LexicalReorderingTable.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.cpp</locationURI>
</link>
<link>
<name>LexicalReorderingTable.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LexicalReorderingTable.h</locationURI>
</link>
<link>
<name>Manager.cpp</name>
<type>1</type>
@ -1076,16 +1036,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/BleuScoreFeature.h</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/ChartBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ChartBasedFeatureContext.h</locationURI>
</link>
<link>
<name>FF/ControlRecombination.cpp</name>
<type>1</type>
@ -1096,6 +1046,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ControlRecombination.h</locationURI>
</link>
<link>
<name>FF/DecodeFeature.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.cpp</locationURI>
</link>
<link>
<name>FF/DecodeFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/DecodeFeature.h</locationURI>
</link>
<link>
<name>FF/DistortionScoreProducer.cpp</name>
<type>1</type>
@ -1167,19 +1127,14 @@
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h</locationURI>
</link>
<link>
<name>FF/OSM-Feature</name>
<name>FF/LexicalReordering</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.cpp</locationURI>
</link>
<link>
<name>FF/PhraseBasedFeatureContext.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhraseBasedFeatureContext.h</locationURI>
<name>FF/OSM-Feature</name>
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>FF/PhraseBoundaryFeature.cpp</name>
@ -1226,6 +1181,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/PhrasePenalty.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatefulFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatefulFF.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatelessFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatelessFF.h</locationURI>
</link>
<link>
<name>FF/SourceWordDeletionFeature.cpp</name>
<type>1</type>
@ -1651,6 +1626,36 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/bin/lm.log</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReordering.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReordering.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReordering.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingState.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingState.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingState.h</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingTable.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.cpp</locationURI>
</link>
<link>
<name>FF/LexicalReordering/LexicalReorderingTable.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/LexicalReordering/LexicalReorderingTable.h</locationURI>
</link>
<link>
<name>FF/OSM-Feature/OpSequenceModel.cpp</name>
<type>1</type>

29
jam-files/sanity.jam
View File

@ -73,17 +73,30 @@ rule test_library ( name ) {
constant CLEANING : $(cleaning) ;
}
shared-command-line = ;
local argv = [ modules.peek : ARGV ] ;
while $(argv) {
if $(argv[1]) = "link=shared" {
shared-command-line = <link>shared ;
}
argv = $(argv[2-]) ;
}
#Determine if a library can be compiled statically.
rule auto-shared ( name : additional * ) {
additional ?= "" ;
if [ test_flags $(additional)" -static -l"$(name) ] {
return ;
if $(shared-command-line) = "<link>shared" {
return "<link>shared" ;
} else {
if $(FORCE-STATIC) {
echo "Could not statically link against lib $(name). Your build will probably fail." ;
if [ test_flags $(additional)" -static -l"$(name) ] {
return ;
} else {
return "<link>shared" ;
if $(FORCE-STATIC) {
echo "Could not statically link against lib $(name). Your build will probably fail." ;
return ;
} else {
return "<link>shared" ;
}
}
}
}
@ -133,8 +146,8 @@ rule boost-lib ( name macro : deps * ) {
flags += " -static" ;
}
if [ test_flags $(flags) : $(main) ] {
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>single $(boost-search) <name>boost_$(name)$(boost-lib-version) : <link>static : <library>$(deps) ;
lib inner_boost_$(name) : : <threading>multi $(boost-search) <name>boost_$(name)-mt$(boost-lib-version) : <link>static : <library>$(deps) ;
} else {
lib inner_boost_$(name) : : $(boost-search) <name>boost_$(name)$(boost-lib-version) : : <library>$(deps) ;
}
@ -143,7 +156,7 @@ rule boost-lib ( name macro : deps * ) {
alias boost_$(name) : inner_boost_$(name) : <link>shared ;
requirements += <define>BOOST_$(macro) ;
} else {
alias boost_$(name) : inner_boost_$(name) : : : <link>shared:<define>BOOST_$(macro) ;
alias boost_$(name) : inner_boost_$(name) : <link>static ;
}
}

1
mira/Main.cpp
View File

@ -41,7 +41,6 @@ namespace mpi = boost::mpi;
#include "moses/ChartTrellisPath.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/ThreadPool.h"
#include "moses/LexicalReordering.h"
#include "mert/BleuScorer.h"
#include "moses/FeatureVector.h"

2
misc/processLexicalTable.cpp
View File

@ -3,7 +3,7 @@
#include "moses/Timer.h"
#include "moses/InputFileStream.h"
#include "moses/LexicalReorderingTable.h"
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
using namespace Moses;

2
misc/queryLexicalTable.cpp
View File

@ -5,7 +5,7 @@
#include "moses/FactorCollection.h"
#include "moses/Timer.h"
#include "moses/InputFileStream.h"
#include "moses/LexicalReorderingTable.h"
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
using namespace Moses;

9
moses-cmd/IOWrapper.cpp
View File

@ -217,12 +217,11 @@ void OutputSurface(std::ostream &out, const Hypothesis &edge, const std::vector<
//preface surface form with UNK if marking unknowns
const Word &word = phrase.GetWord(pos);
if(markUnknown && word.IsOOV()) {
out << "UNK" << *factor;
out << "UNK" << *factor;
} else {
out << *factor;
}
else {
out << *factor;
}
for (size_t i = 1 ; i < outputFactorOrder.size() ; i++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[i]);
CHECK(factor);

2
moses/ChartHypothesis.cpp
View File

@ -162,7 +162,7 @@ void ChartHypothesis::Evaluate()
StatelessFeatureFunction::GetStatelessFeatureFunctions();
for (unsigned i = 0; i < sfs.size(); ++i) {
if (! staticData.IsFeatureFunctionIgnored( *sfs[i] )) {
sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown);
sfs[i]->EvaluateChart(*this,&m_scoreBreakdown);
}
}

1
moses/ChartHypothesis.h
View File

@ -34,6 +34,7 @@ namespace Moses
class ChartHypothesis;
class ChartManager;
class RuleCubeItem;
class FFState;
typedef std::vector<ChartHypothesis*> ChartArcList;

17
moses/ChartManager.cpp
View File

@ -52,6 +52,12 @@ ChartManager::ChartManager(InputType const& source)
,m_parser(source, m_hypoStackColl)
,m_translationOptionList(StaticData::Instance().GetRuleLimit(), source)
{
const StaticData &staticData = StaticData::Instance();
long sentenceID = source.GetTranslationId();
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
if (m_constraint) {
VERBOSE(1, "Search constraint to output: " << *m_constraint<<endl);
}
}
ChartManager::~ChartManager()
@ -126,6 +132,8 @@ void ChartManager::ProcessSentence()
void ChartManager::AddXmlChartOptions()
{
const StaticData &staticData = StaticData::Instance();
const Phrase *constraint = GetConstraint();
const std::vector <ChartTranslationOptions*> xmlChartOptionsList = m_source.GetXmlChartTranslationOptions();
IFVERBOSE(2) {
cerr << "AddXmlChartOptions " << xmlChartOptionsList.size() << endl;
@ -141,6 +149,15 @@ void ChartManager::AddXmlChartOptions()
RuleCubeItem* item = new RuleCubeItem( *opt, m_hypoStackColl );
ChartHypothesis* hypo = new ChartHypothesis(*opt, *item, *this);
if (constraint) {
Phrase hypoPhrase = hypo->GetOutputPhrase();
if (!constraint->Contains(hypoPhrase)) {
delete item;
delete hypo;
continue;
}
}
hypo->Evaluate();
const Word &targetLHS = hypo->GetTargetLHS();

3
moses/ChartManager.h
View File

@ -59,6 +59,7 @@ private:
std::auto_ptr<SentenceStats> m_sentenceStats;
clock_t m_start; /**< starting time, used for logging */
unsigned m_hypothesisId; /* For handing out hypothesis ids to ChartHypothesis */
const Phrase *m_constraint;
ChartParser m_parser;
@ -101,6 +102,8 @@ public:
return m_hypothesisId++;
}
const Phrase *GetConstraint() const
{ return m_constraint; }
};
}

12
moses/DecodeStepTranslation.cpp
View File

@ -137,7 +137,7 @@ void DecodeStepTranslation::ProcessInitialTranslation(
}
}
void DecodeStepTranslation::ProcessInitialTranslationLegacy(
void DecodeStepTranslation::ProcessInitialTranslationLEGACY(
const InputType &source
,PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit
@ -147,7 +147,7 @@ void DecodeStepTranslation::ProcessInitialTranslationLegacy(
const size_t tableLimit = phraseDictionary->GetTableLimit();
const WordsRange wordsRange(startPos, endPos);
const TargetPhraseCollectionWithSourcePhrase *phraseColl = phraseDictionary->GetTargetPhraseCollectionLegacy(source,wordsRange);
const TargetPhraseCollectionWithSourcePhrase *phraseColl = phraseDictionary->GetTargetPhraseCollectionLEGACY(source,wordsRange);
if (phraseColl != NULL) {
IFVERBOSE(3) {
@ -171,7 +171,7 @@ void DecodeStepTranslation::ProcessInitialTranslationLegacy(
const TargetPhrase &targetPhrase = **iterTargetPhrase;
const Phrase &sourcePhrase = *iterSourcePhrase;
const InputPath &inputPath = GetInputPathLegacy(targetPhrase, sourcePhrase, inputPathList);
const InputPath &inputPath = GetInputPathLEGACY(targetPhrase, sourcePhrase, inputPathList);
TranslationOption *transOpt = new TranslationOption(wordsRange, targetPhrase);
transOpt->SetInputPath(inputPath);
@ -184,7 +184,7 @@ void DecodeStepTranslation::ProcessInitialTranslationLegacy(
}
}
const InputPath &DecodeStepTranslation::GetInputPathLegacy(
const InputPath &DecodeStepTranslation::GetInputPathLEGACY(
const TargetPhrase targetPhrase,
const Phrase sourcePhrase,
const InputPathList &inputPathList) const
@ -207,7 +207,7 @@ const InputPath &DecodeStepTranslation::GetInputPathLegacy(
UTIL_THROW(util::Exception, "Input path not found");
}
void DecodeStepTranslation::ProcessLegacy(const TranslationOption &inputPartialTranslOpt
void DecodeStepTranslation::ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
@ -229,7 +229,7 @@ void DecodeStepTranslation::ProcessLegacy(const TranslationOption &inputPartialT
const size_t tableLimit = phraseDictionary->GetTableLimit();
const TargetPhraseCollectionWithSourcePhrase *phraseColl
= phraseDictionary->GetTargetPhraseCollectionLegacy(toc->GetSource(),sourceWordsRange);
= phraseDictionary->GetTargetPhraseCollectionLEGACY(toc->GetSource(),sourceWordsRange);
if (phraseColl != NULL) {

6
moses/DecodeStepTranslation.h
View File

@ -61,11 +61,11 @@ public:
, const TargetPhraseCollection *phraseColl) const;
// legacy
void ProcessInitialTranslationLegacy(const InputType &source
void ProcessInitialTranslationLEGACY(const InputType &source
, PartialTranslOptColl &outputPartialTranslOptColl
, size_t startPos, size_t endPos, bool adhereTableLimit
, const InputPathList &inputPathList) const;
void ProcessLegacy(const TranslationOption &inputPartialTranslOpt
void ProcessLEGACY(const TranslationOption &inputPartialTranslOpt
, const DecodeStep &decodeStep
, PartialTranslOptColl &outputPartialTranslOptColl
, TranslationOptionCollection *toc
@ -75,7 +75,7 @@ private:
// I'm not sure whether this actually works or not for binary phrase table.
// The source phrase only appears to contain the 1st word, therefore, this function
// only compares the 1st word
const InputPath &GetInputPathLegacy(const TargetPhrase targetPhrase,
const InputPath &GetInputPathLEGACY(const TargetPhrase targetPhrase,
const Phrase sourcePhrase,
const InputPathList &inputPathList) const;

10
moses/FF/BleuScoreFeature.h
View File

@ -118,6 +118,16 @@ public:
FFState* EvaluateChart(const ChartHypothesis& cur_hypo,
int featureID,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
bool Enabled() const {
return m_enabled;
}

24
moses/FF/ChartBasedFeatureContext.cpp
View File

@ -1,24 +0,0 @@
#include "ChartBasedFeatureContext.h"
#include "moses/ChartHypothesis.h"
#include "moses/ChartManager.h"
namespace Moses
{
ChartBasedFeatureContext::ChartBasedFeatureContext
(const ChartHypothesis* hypothesis):
m_hypothesis(hypothesis),
m_targetPhrase(hypothesis->GetCurrTargetPhrase()),
m_source(hypothesis->GetManager().GetSource())
{}
ChartBasedFeatureContext::ChartBasedFeatureContext(
const TargetPhrase& targetPhrase,
const InputType& source):
m_hypothesis(NULL),
m_targetPhrase(targetPhrase),
m_source(source)
{}
}

39
moses/FF/ChartBasedFeatureContext.h
View File

@ -1,39 +0,0 @@
#pragma once
namespace Moses
{
class ChartHypothesis;
class InputType;
class TargetPhrase;
/**
* Same as PhraseBasedFeatureContext, but for chart-based Moses.
**/
class ChartBasedFeatureContext
{
//The context either has a hypothesis (during search) or a
//TargetPhrase and source sentence (during pre-calculation)
//TODO: should the context also include some info on where the TargetPhrase
//is anchored (assuming it's lexicalised), which is available at pre-calc?
const ChartHypothesis* m_hypothesis;
const TargetPhrase& m_targetPhrase;
const InputType& m_source;
public:
ChartBasedFeatureContext(const ChartHypothesis* hypothesis);
ChartBasedFeatureContext(const TargetPhrase& targetPhrase,
const InputType& source);
const InputType& GetSource() const {
return m_source;
}
const TargetPhrase& GetTargetPhrase() const {
return m_targetPhrase;
}
};
} // namespace

10
moses/FF/ControlRecombination.h
View File

@ -35,6 +35,16 @@ public:
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
//! return the state associated with the empty hypothesis for a given sentence
virtual const FFState* EmptyHypothesisState(const InputType &input) const;

2
moses/DecodeFeature.cpp → moses/FF/DecodeFeature.cpp
View File

@ -23,7 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <iostream>
#include "DecodeFeature.h"
#include "StaticData.h"
#include "moses/StaticData.h"
using namespace std;

20
moses/DecodeFeature.h → moses/FF/DecodeFeature.h
View File

@ -24,9 +24,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <vector>
#include "FactorTypeSet.h"
#include "moses/FF/StatelessFeatureFunction.h"
#include "TypeDef.h"
#include "moses/FactorTypeSet.h"
#include "moses/TypeDef.h"
namespace Moses
{
@ -63,6 +63,22 @@ public:
bool IsUseable(const FactorMask &mask) const;
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
protected:
std::vector<FactorType> m_input;
std::vector<FactorType> m_output;

11
moses/FF/DistortionScoreProducer.h
View File

@ -40,6 +40,17 @@ public:
ScoreComponentCollection*) const {
throw std::logic_error("DistortionScoreProducer not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
}

6
moses/FF/Factory.cpp
View File

@ -10,7 +10,7 @@
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
#include "moses/TranslationModel/PhraseDictionaryDynSuffixArray.h"
#include "moses/LexicalReordering.h"
#include "moses/FF/LexicalReordering/LexicalReordering.h"
#include "moses/FF/BleuScoreFeature.h"
#include "moses/FF/TargetWordInsertionFeature.h"
@ -30,6 +30,8 @@
#include "moses/FF/PhrasePenalty.h"
#include "moses/FF/OSM-Feature/OpSequenceModel.h"
#include "moses/FF/ControlRecombination.h"
#include "moses/FF/SkeletonStatelessFF.h"
#include "moses/FF/SkeletonStatefulFF.h"
#include "moses/LM/Ken.h"
#ifdef LM_IRST
@ -138,6 +140,8 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(PhrasePenalty);
MOSES_FNAME2("UnknownWordPenalty", UnknownWordPenaltyProducer);
MOSES_FNAME(ControlRecombination);
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(SkeletonStatefulFF);
#ifdef HAVE_SYNLM
MOSES_FNAME(SyntacticLanguageModel);

9
moses/FF/FeatureFunction.h
View File

@ -4,8 +4,6 @@
#include <vector>
#include <set>
#include <string>
#include "PhraseBasedFeatureContext.h"
#include "ChartBasedFeatureContext.h"
#include "moses/TypeDef.h"
namespace Moses
@ -16,7 +14,6 @@ class TargetPhrase;
class TranslationOption;
class Hypothesis;
class ChartHypothesis;
class FFState;
class InputType;
class ScoreComponentCollection;
class WordsBitmap;
@ -102,8 +99,7 @@ public:
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const {
}
, ScoreComponentCollection &estimatedFutureScore) const = 0;
// This method is called once all the translation options are retrieved from the phrase table, and
// just before search.
@ -111,8 +107,7 @@ public:
// Currently not used by any FF. Not called by moses_chart
virtual void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const {
}
, ScoreComponentCollection &scoreBreakdown) const = 0;
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void ReadParameters();

4
moses/FF/GlobalLexicalModel.cpp
View File

@ -167,11 +167,11 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP
}
void GlobalLexicalModel::Evaluate
(const PhraseBasedFeatureContext& context,
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
accumulator->PlusEquals( this,
GetFromCacheOrScorePhrase(context.GetTargetPhrase()) );
GetFromCacheOrScorePhrase(hypo.GetCurrTargetPhrase()) );
}
bool GlobalLexicalModel::IsUseable(const FactorMask &mask) const

18
moses/FF/GlobalLexicalModel.h
View File

@ -64,20 +64,32 @@ public:
GlobalLexicalModel(const std::string &line);
virtual ~GlobalLexicalModel();
void SetParameter(const std::string& key, const std::string& value);
void InitializeForInput( Sentence const& in );
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const PhraseBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(
const ChartBasedFeatureContext& context,
const ChartHypothesis& hypo,
ScoreComponentCollection* accumulator) const {
throw std::logic_error("GlobalLexicalModel not supported in chart decoder, yet");
}
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};

10
moses/FF/GlobalLexicalModelUnlimited.h
View File

@ -90,6 +90,16 @@ public:
throw std::logic_error("GlobalLexicalModelUnlimited not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void AddFeature(ScoreComponentCollection* accumulator,
StringPiece sourceTrigger, StringPiece sourceWord, StringPiece targetTrigger,
StringPiece targetWord) const;

15
moses/FF/InputFeature.h
View File

@ -29,10 +29,23 @@ public:
return m_numRealWordCount;
}
virtual void Evaluate(const InputType &input
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
};

2
moses/LexicalReordering.cpp → moses/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -3,7 +3,7 @@
#include "moses/FF/FFState.h"
#include "LexicalReordering.h"
#include "LexicalReorderingState.h"
#include "StaticData.h"
#include "moses/StaticData.h"
using namespace std;

23
moses/LexicalReordering.h → moses/FF/LexicalReordering/LexicalReordering.h
View File

@ -3,15 +3,16 @@
#include <string>
#include <vector>
#include "Factor.h"
#include "Phrase.h"
#include "TypeDef.h"
#include "Util.h"
#include "WordsRange.h"
#include "moses/Factor.h"
#include "moses/Phrase.h"
#include "moses/TypeDef.h"
#include "moses/Util.h"
#include "moses/WordsRange.h"
#include "LexicalReorderingState.h"
#include "LexicalReorderingTable.h"
#include "moses/FF/StatefulFeatureFunction.h"
#include "util/exception.hh"
namespace Moses
@ -48,9 +49,17 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis&,
int /* featureID */,
ScoreComponentCollection*) const {
CHECK(0); // not valid for chart decoder
return NULL;
UTIL_THROW(util::Exception, "LexicalReordering is not valid for chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
private:
bool DecodeCondition(std::string s);

8
moses/LexicalReorderingState.cpp → moses/FF/LexicalReordering/LexicalReorderingState.cpp
View File

@ -4,10 +4,10 @@
#include "util/check.hh"
#include "moses/FF/FFState.h"
#include "Hypothesis.h"
#include "WordsRange.h"
#include "ReorderingStack.h"
#include "TranslationOption.h"
#include "moses/Hypothesis.h"
#include "moses/WordsRange.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "LexicalReordering.h"
#include "LexicalReorderingState.h"

10
moses/LexicalReorderingState.h → moses/FF/LexicalReordering/LexicalReorderingState.h
View File

@ -4,12 +4,12 @@
#include <vector>
#include <string>
#include "Hypothesis.h"
#include "moses/Hypothesis.h"
#include "LexicalReordering.h"
#include "WordsRange.h"
#include "WordsBitmap.h"
#include "ReorderingStack.h"
#include "TranslationOption.h"
#include "moses/WordsRange.h"
#include "moses/WordsBitmap.h"
#include "moses/ReorderingStack.h"
#include "moses/TranslationOption.h"
#include "moses/FF/FFState.h"

12
moses/LexicalReorderingTable.cpp → moses/FF/LexicalReordering/LexicalReorderingTable.cpp
View File

@ -1,15 +1,15 @@
#include "LexicalReorderingTable.h"
#include "InputFileStream.h"
#include "moses/InputFileStream.h"
//#include "LVoc.h" //need IPhrase
#include "StaticData.h"
#include "moses/StaticData.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "GenerationDictionary.h"
#include "TargetPhrase.h"
#include "TargetPhraseCollection.h"
#include "moses/GenerationDictionary.h"
#include "moses/TargetPhrase.h"
#include "moses/TargetPhraseCollection.h"
#ifndef WIN32
#include "TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#endif
namespace Moses

12
moses/LexicalReorderingTable.h → moses/FF/LexicalReordering/LexicalReorderingTable.h
View File

@ -13,12 +13,12 @@
#endif
//moses dependencies:
#include "TypeDef.h"
#include "Phrase.h"
#include "InputType.h"
#include "ConfusionNet.h"
#include "Sentence.h"
#include "PrefixTreeMap.h"
#include "moses/TypeDef.h"
#include "moses/Phrase.h"
#include "moses/InputType.h"
#include "moses/ConfusionNet.h"
#include "moses/Sentence.h"
#include "moses/PrefixTreeMap.h"
namespace Moses
{

42
moses/FF/OSM-Feature/OpSequenceModel.cpp
View File

@ -13,6 +13,9 @@ namespace Moses
OpSequenceModel::OpSequenceModel(const std::string &line)
:StatefulFeatureFunction("OpSequenceModel", 5, line )
{
sFactor = 0;
tFactor = 0;
numFeatures = 5;
ReadParameters();
}
@ -47,7 +50,7 @@ void OpSequenceModel:: Evaluate(const Phrase &source
WordsBitmap myBitmap(source.GetSize());
vector <string> mySourcePhrase;
vector <string> myTargetPhrase;
vector<float> scores(5);
vector<float> scores;
vector <int> alignments;
int startIndex = 0;
int endIndex = source.GetSize();
@ -55,28 +58,27 @@ void OpSequenceModel:: Evaluate(const Phrase &source
const AlignmentInfo &align = targetPhrase.GetAlignTerm();
AlignmentInfo::const_iterator iter;
for (iter = align.begin(); iter != align.end(); ++iter) {
alignments.push_back(iter->first);
alignments.push_back(iter->second);
}
for (int i = 0; i < targetPhrase.GetSize(); i++) {
if (targetPhrase.GetWord(i).IsOOV())
if (targetPhrase.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
myTargetPhrase.push_back("_TRANS_SLF_");
else
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(0)->GetString().as_string());
myTargetPhrase.push_back(targetPhrase.GetWord(i).GetFactor(tFactor)->GetString().as_string());
}
for (int i = 0; i < source.GetSize(); i++) {
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
}
obj.setPhrases(mySourcePhrase , myTargetPhrase);
obj.constructCepts(alignments,startIndex,endIndex-1,targetPhrase.GetSize());
obj.computeOSMFeature(startIndex,myBitmap);
obj.calculateOSMProb(*OSM);
obj.populateScores(scores);
obj.populateScores(scores,numFeatures);
estimatedFutureScore.PlusEquals(this, scores);
}
@ -96,7 +98,7 @@ FFState* OpSequenceModel::Evaluate(
osmHypothesis obj;
vector <string> mySourcePhrase;
vector <string> myTargetPhrase;
vector<float> scores(5);
vector<float> scores;
//target.GetWord(0)
@ -140,16 +142,16 @@ FFState* OpSequenceModel::Evaluate(
for (int i = startIndex; i <= endIndex; i++) {
myBitmap.SetValue(i,0); // resetting coverage of this phrase ...
mySourcePhrase.push_back(source.GetWord(i).GetFactor(0)->GetString().as_string());
mySourcePhrase.push_back(source.GetWord(i).GetFactor(sFactor)->GetString().as_string());
// cerr<<mySourcePhrase[i]<<endl;
}
for (int i = 0; i < target.GetSize(); i++) {
if (target.GetWord(i).IsOOV())
if (target.GetWord(i).IsOOV() && sFactor == 0 && tFactor == 0)
myTargetPhrase.push_back("_TRANS_SLF_");
else
myTargetPhrase.push_back(target.GetWord(i).GetFactor(0)->GetString().as_string());
myTargetPhrase.push_back(target.GetWord(i).GetFactor(tFactor)->GetString().as_string());
}
@ -161,7 +163,8 @@ FFState* OpSequenceModel::Evaluate(
obj.setPhrases(mySourcePhrase , myTargetPhrase);
obj.computeOSMFeature(startIndex,myBitmap);
obj.calculateOSMProb(*OSM);
obj.populateScores(scores);
obj.populateScores(scores,numFeatures);
//obj.print();
/*
if (bitmap.GetFirstGapPos() == NOT_FOUND)
@ -175,14 +178,7 @@ FFState* OpSequenceModel::Evaluate(
}
*/
/*
vector<float> scores(5);
scores[0] = 0.343423f;
scores[1] = 1.343423f;
scores[2] = 2.343423f;
scores[3] = 3.343423f;
scores[4] = 4.343423f;
*/
accumulator->PlusEquals(this, scores);
@ -225,7 +221,7 @@ std::vector<float> OpSequenceModel::GetFutureScores(const Phrase &source, const
iter = m_futureCost.find(pp);
//iter = m_coll.find(pp);
if (iter == m_futureCost.end()) {
vector<float> scores(5, 0);
vector<float> scores(numFeatures, 0);
scores[0] = unkOpProb;
return scores;
} else {
@ -239,8 +235,14 @@ void OpSequenceModel::SetParameter(const std::string& key, const std::string& va
if (key == "path") {
m_lmPath = value;
} else if (key == "numFeatures") {
numFeatures = Scan<int>(value);
} else if (key == "order") {
lmOrder = Scan<int>(value);
} else if (key == "sFactor") {
sFactor = Scan<int>(value);
} else if (key == "tFactor") {
tFactor = Scan<int>(value);
} else {
StatefulFeatureFunction::SetParameter(key, value);
}

17
moses/FF/OSM-Feature/OpSequenceModel.h
View File

@ -21,6 +21,9 @@ public:
int lmOrder;
float unkOpProb;
int sFactor; // Source Factor ...
int tFactor; // Target Factor ...
int numFeatures; // Number of features used ...
OpSequenceModel(const std::string &line);
@ -32,16 +35,20 @@ public:
const FFState* prev_state,
ScoreComponentCollection* accumulator) const;
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
virtual const FFState* EmptyHypothesisState(const InputType &input) const;
virtual std::string GetScoreProducerWeightShortName(unsigned idx=0) const;

7
moses/FF/OSM-Feature/osmHyp.cpp
View File

@ -105,7 +105,6 @@ void osmHypothesis :: removeReorderingOperations()
deletionCount = 0;
openGapCount = 0;
gapWidth = 0;
//cout<<"I came here"<<endl;
std::vector <std::string> tupleSequence;
@ -581,10 +580,14 @@ void osmHypothesis :: constructCepts(vector <int> & align , int startIndex , int
}
void osmHypothesis :: populateScores(vector <float> & scores)
void osmHypothesis :: populateScores(vector <float> & scores , const int numFeatures)
{
scores.clear();
scores.push_back(opProb);
if (numFeatures == 1)
return;
scores.push_back(gapWidth);
scores.push_back(gapCount);
scores.push_back(openGapCount);

2
moses/FF/OSM-Feature/osmHyp.h
View File

@ -89,7 +89,7 @@ public:
void setState(const FFState* prev_state);
osmState * saveState();
void print();
void populateScores(std::vector <float> & scores);
void populateScores(std::vector <float> & scores , const int numFeatures);
void setState(const lm::ngram::State & val) {
lmState = val;
}

33
moses/FF/PhraseBasedFeatureContext.cpp
View File

@ -1,33 +0,0 @@
#include "PhraseBasedFeatureContext.h"
#include "moses/Hypothesis.h"
#include "moses/Manager.h"
#include "moses/TranslationOption.h"
namespace Moses
{
PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) :
m_hypothesis(hypothesis),
m_translationOption(m_hypothesis->GetTranslationOption()),
m_source(m_hypothesis->GetManager().GetSource()) {}
PhraseBasedFeatureContext::PhraseBasedFeatureContext
(const TranslationOption& translationOption, const InputType& source) :
m_hypothesis(NULL),
m_translationOption(translationOption),
m_source(source)
{}
const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const
{
return m_translationOption.GetTargetPhrase();
}
const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const
{
if (!m_hypothesis) {
throw std::logic_error("Coverage vector not available during pre-calculation");
}
return m_hypothesis->GetWordsBitmap();
}
}

44
moses/FF/PhraseBasedFeatureContext.h
View File

@ -1,44 +0,0 @@
#pragma once
namespace Moses
{
class Hypothesis;
class TranslationOption;
class InputType;
class TargetPhrase;
class WordsBitmap;
/**
* Contains all that a feature function can access without affecting recombination.
* For stateless features, this is all that it can access. Currently this is not
* used for stateful features, as it would need to be retro-fitted to the LM feature.
* TODO: Expose source segmentation,lattice path.
* XXX Don't add anything to the context that would break recombination XXX
**/
class PhraseBasedFeatureContext
{
// The context either has a hypothesis (during search), or a TranslationOption and
// source sentence (during pre-calculation).
const Hypothesis* m_hypothesis;
const TranslationOption& m_translationOption;
const InputType& m_source;
public:
PhraseBasedFeatureContext(const Hypothesis* hypothesis);
PhraseBasedFeatureContext(const TranslationOption& translationOption,
const InputType& source);
const TranslationOption& GetTranslationOption() const {
return m_translationOption;
}
const InputType& GetSource() const {
return m_source;
}
const TargetPhrase& GetTargetPhrase() const; //convenience method
const WordsBitmap& GetWordsBitmap() const;
};
} // namespace

11
moses/FF/PhraseBoundaryFeature.h
View File

@ -52,6 +52,17 @@ public:
ScoreComponentCollection* ) const {
throw std::logic_error("PhraseBoundaryState not supported in chart decoder, yet");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

11
moses/FF/PhraseLengthFeature.h
View File

@ -24,11 +24,20 @@ public:
return true;
}
void EvaluateChart(const ChartBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhraseLengthFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown

10
moses/FF/PhrasePairFeature.cpp
View File

@ -106,11 +106,11 @@ void PhrasePairFeature::Load()
}
void PhrasePairFeature::Evaluate(
const PhraseBasedFeatureContext& context,
const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const TargetPhrase& target = context.GetTargetPhrase();
const Phrase& source = context.GetTranslationOption().GetInputPath().GetPhrase();
const TargetPhrase& target = hypo.GetCurrTargetPhrase();
const Phrase& source = hypo.GetTranslationOption().GetInputPath().GetPhrase();
if (m_simple) {
ostringstream namestr;
namestr << "pp_";
@ -131,7 +131,7 @@ void PhrasePairFeature::Evaluate(
accumulator->SparsePlusEquals(namestr.str(),1);
}
if (m_domainTrigger) {
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const bool use_topicid = input.GetUseTopicId();
const bool use_topicid_prob = input.GetUseTopicIdAndProb();
@ -199,7 +199,7 @@ void PhrasePairFeature::Evaluate(
}
}
if (m_sourceContext) {
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
// range over source words to get context
for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) {

14
moses/FF/PhrasePairFeature.h
View File

@ -37,14 +37,24 @@ public:
bool IsUseable(const FactorMask &mask) const;
void Evaluate(const PhraseBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartBasedFeatureContext& context,
void EvaluateChart(const ChartHypothesis& hypo,
ScoreComponentCollection*) const {
throw std::logic_error("PhrasePairFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Load();
void SetParameter(const std::string& key, const std::string& value);

13
moses/FF/PhrasePenalty.h
View File

@ -18,6 +18,19 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
};
} //namespace

8
moses/FF/SkeletonStatefulFF.cpp Normal file
View File

@ -0,0 +1,8 @@
#include "SkeletonStatefulFF.h"
namespace Moses
{
}

64
moses/FF/SkeletonStatefulFF.h Normal file
View File

@ -0,0 +1,64 @@
#pragma once
#include <string>
#include "StatefulFeatureFunction.h"
#include "FFState.h"
namespace Moses
{
class SkeletonState : public FFState
{
public:
int Compare(const FFState& other) const
{
return 0;
}
};
class SkeletonStatefulFF : public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(const std::string &line)
:StatefulFeatureFunction("StatefulFeatureFunction", line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
FFState* Evaluate(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState();
}
FFState* EvaluateChart(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState();
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const
{
return new SkeletonState();
}
};
}

7
moses/FF/SkeletonStatelessFF.cpp Normal file
View File

@ -0,0 +1,7 @@
#include "SkeletonStatelessFF.h"
namespace Moses
{
}

38
moses/FF/SkeletonStatelessFF.h Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#include <string>
#include "StatelessFeatureFunction.h"
namespace Moses
{
class SkeletonStatelessFF : public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(const std::string &line)
:StatelessFeatureFunction("SkeletonStatelessFF", line)
{}
bool IsUseable(const FactorMask &mask) const
{ return true; }
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
virtual void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
};
}

12
moses/FF/SourceWordDeletionFeature.h
View File

@ -28,10 +28,20 @@ public:
bool IsUseable(const FactorMask &mask) const;
virtual void Evaluate(const Phrase &source
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,

1
moses/FF/StatefulFeatureFunction.h
View File

@ -4,6 +4,7 @@
namespace Moses
{
class FFState;
/** base class for all stateful feature functions.
* eg. LM, distortion penalty

10
moses/FF/StatelessFeatureFunction.h
View File

@ -23,16 +23,14 @@ public:
/**
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const {
}
virtual void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const = 0;
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const {
}
virtual void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const = 0;
virtual bool IsStateless() const {
return true;

12
moses/FF/TargetBigramFeature.h
View File

@ -45,8 +45,18 @@ public:
virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */,
int /* featureID */,
ScoreComponentCollection* ) const {
abort();
throw std::logic_error("TargetBigramFeature not valid in chart decoder");
}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

11
moses/FF/TargetNgramFeature.h
View File

@ -191,6 +191,17 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
void SetParameter(const std::string& key, const std::string& value);
private:

10
moses/FF/TargetWordInsertionFeature.h
View File

@ -32,6 +32,16 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void ComputeFeatures(const Phrase &source,
const TargetPhrase& targetPhrase,

16
moses/FF/UnknownWordPenaltyProducer.h
View File

@ -22,6 +22,22 @@ public:
}
std::vector<float> DefaultWeights() const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
}

10
moses/FF/WordPenaltyProducer.h
View File

@ -22,6 +22,16 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
};

12
moses/FF/WordTranslationFeature.cpp
View File

@ -137,12 +137,12 @@ void WordTranslationFeature::Load()
}
void WordTranslationFeature::Evaluate
(const PhraseBasedFeatureContext& context,
(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{
const Sentence& input = static_cast<const Sentence&>(context.GetSource());
const TranslationOption& transOpt = context.GetTranslationOption();
const TargetPhrase& targetPhrase = context.GetTargetPhrase();
const Sentence& input = static_cast<const Sentence&>(hypo.GetInput());
const TranslationOption& transOpt = hypo.GetTranslationOption();
const TargetPhrase& targetPhrase = hypo.GetCurrTargetPhrase();
const AlignmentInfo &alignment = targetPhrase.GetAlignTerm();
// process aligned words
@ -243,7 +243,7 @@ void WordTranslationFeature::Evaluate
}
}
if (m_sourceContext) {
size_t globalSourceIndex = context.GetTranslationOption().GetStartPos() + sourceIndex;
size_t globalSourceIndex = hypo.GetTranslationOption().GetStartPos() + sourceIndex;
if (!m_domainTrigger && globalSourceIndex == 0) {
// add <s> trigger feature for source
stringstream feature;
@ -349,7 +349,7 @@ void WordTranslationFeature::Evaluate
}
void WordTranslationFeature::EvaluateChart(
const ChartBasedFeatureContext& context,
const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{
UTIL_THROW(util::Exception, "Need source phrase. Can't be arsed at the moment");

16
moses/FF/WordTranslationFeature.h
View File

@ -39,6 +39,7 @@ private:
public:
WordTranslationFeature(const std::string &line);
void SetParameter(const std::string& key, const std::string& value);
bool IsUseable(const FactorMask &mask) const;
void Load();
@ -47,12 +48,21 @@ public:
return new DummyState();
}
void Evaluate(const PhraseBasedFeatureContext& context,
void Evaluate(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const;
void EvaluateChart(const ChartBasedFeatureContext& context,
void EvaluateChart(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const;
void SetParameter(const std::string& key, const std::string& value);
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
}

7
moses/FeatureVector.h
View File

@ -264,6 +264,7 @@ public:
#endif
private:
friend void swap(FVector &first, FVector &second);
/** Internal get and set. */
const FValue& get(const FName& name) const;
@ -310,6 +311,12 @@ private:
};
inline void swap(FVector &first, FVector &second)
{
swap(first.m_features, second.m_features);
swap(first.m_coreFeatures, second.m_coreFeatures);
}
std::ostream& operator<<( std::ostream& out, const FVector& fv);
//Element-wise operations
const FVector operator+(const FVector& lhs, const FVector& rhs);

2
moses/GenerationDictionary.h
View File

@ -29,7 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "ScoreComponentCollection.h"
#include "Phrase.h"
#include "TypeDef.h"
#include "DecodeFeature.h"
#include "moses/FF/DecodeFeature.h"
namespace Moses
{

3
moses/Hypothesis.cpp
View File

@ -30,7 +30,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Hypothesis.h"
#include "Util.h"
#include "SquareMatrix.h"
#include "LexicalReordering.h"
#include "StaticData.h"
#include "InputType.h"
#include "Manager.h"
@ -263,7 +262,7 @@ void Hypothesis::EvaluateWith(const StatelessFeatureFunction& slff)
{
const StaticData &staticData = StaticData::Instance();
if (! staticData.IsFeatureFunctionIgnored( slff )) {
slff.Evaluate(PhraseBasedFeatureContext(this), &m_scoreBreakdown);
slff.Evaluate(*this, &m_scoreBreakdown);
}
}

1
moses/Hypothesis.h
View File

@ -48,7 +48,6 @@ class FFState;
class StatelessFeatureFunction;
class StatefulFeatureFunction;
class Manager;
class LexicalReordering;
typedef std::vector<Hypothesis*> ArcList;

1
moses/Jamfile
View File

@ -60,6 +60,7 @@ lib moses :
TranslationModel/CYKPlusParser/*.cpp
FF/*.cpp
FF/OSM-Feature/*.cpp
FF/LexicalReordering/*.cpp
: #exceptions
ThreadPool.cpp
SyntacticLanguageModel.cpp

20
moses/LM/Backward.cpp
View File

@ -24,7 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "lm/left.hh"
#include "lm/model.hh"
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
#include "moses/Hypothesis.h"
#include "moses/Phrase.h"
@ -39,7 +39,7 @@ namespace Moses
{
/** Constructs a new backward language model. */
template <class Model> BackwardLanguageModel<Model>::BackwardLanguageModel(const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(file,factorType,lazy)
template <class Model> BackwardLanguageModel<Model>::BackwardLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line,file,factorType,lazy)
{
//
// This space intentionally left blank
@ -288,30 +288,30 @@ template <class Model> FFState *BackwardLanguageModel<Model>::Evaluate(const Phr
}
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy)
LanguageModel *ConstructBackwardLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
{
try {
lm::ngram::ModelType model_type;
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
switch(model_type) {
case lm::ngram::PROBING:
return new BackwardLanguageModel<lm::ngram::ProbingModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy);
case lm::ngram::REST_PROBING:
return new BackwardLanguageModel<lm::ngram::RestProbingModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::RestProbingModel>(line, file, factorType, lazy);
case lm::ngram::TRIE:
return new BackwardLanguageModel<lm::ngram::TrieModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::TrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_TRIE:
return new BackwardLanguageModel<lm::ngram::QuantTrieModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::QuantTrieModel>(line, file, factorType, lazy);
case lm::ngram::ARRAY_TRIE:
return new BackwardLanguageModel<lm::ngram::ArrayTrieModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::ArrayTrieModel>(line, file, factorType, lazy);
case lm::ngram::QUANT_ARRAY_TRIE:
return new BackwardLanguageModel<lm::ngram::QuantArrayTrieModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::QuantArrayTrieModel>(line, file, factorType, lazy);
default:
std::cerr << "Unrecognized kenlm model type " << model_type << std::endl;
abort();
}
} else {
return new BackwardLanguageModel<lm::ngram::ProbingModel>(file, factorType, lazy);
return new BackwardLanguageModel<lm::ngram::ProbingModel>(line, file, factorType, lazy);
}
} catch (std::exception &e) {
std::cerr << e.what() << std::endl;

4
moses/LM/Backward.h
View File

@ -33,7 +33,7 @@ namespace Moses
{
//! This will also load. Returns a templated backward LM.
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy);
LanguageModel *ConstructBackwardLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
class FFState;
// template<typename M> class BackwardLanguageModelTest;
@ -45,7 +45,7 @@ class BackwardLanguageModelTest;
template <class Model> class BackwardLanguageModel : public LanguageModelKen<Model>
{
public:
BackwardLanguageModel(const std::string &file, FactorType factorType, bool lazy);
BackwardLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;

2
moses/LM/BackwardLMState.h
View File

@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifndef moses_BackwardLMState_h
#define moses_BackwardLMState_h
#include "moses/FFState.h"
#include "moses/FF/FFState.h"
#include "moses/LM/Backward.h"
#include "lm/state.hh"

33
moses/LM/BackwardTest.cpp
View File

@ -70,6 +70,7 @@ public:
backwardLM(
static_cast< BackwardLanguageModel<lm::ngram::ProbingModel> * >(
ConstructBackwardLM(
"LM1=1.0",
boost::unit_test::framework::master_test_suite().argv[1],
0,
false)
@ -116,9 +117,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"the",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -141,9 +144,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"the licenses",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 2 );
@ -166,9 +171,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"the licenses for",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 3 );
@ -191,9 +198,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"the licenses for most",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 4 );
@ -235,9 +244,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"the",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -261,9 +272,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"licenses",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -287,9 +300,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"for",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );
@ -313,9 +328,11 @@ public:
outputFactorOrder.push_back(0);
phrase.CreateFromString(
Input,
outputFactorOrder,
"most",
StaticData::Instance().GetFactorDelimiter());
StaticData::Instance().GetFactorDelimiter(),
NULL);
BOOST_CHECK( phrase.GetSize() == 1 );

4
moses/LM/Base.h
View File

@ -90,6 +90,10 @@ public:
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const;
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
};

13
moses/LM/IRST.cpp
View File

@ -48,19 +48,8 @@ LanguageModelIRST::LanguageModelIRST(const std::string &line)
throw runtime_error("Error: " + SPrint(threadCount) + " number of threads specified but IRST LM is not threadsafe.");
}
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
ReadParameters();
if (args[0] == "factor") {
m_factorType = Scan<FactorType>(args[1]);
} else if (args[0] == "order") {
m_nGramOrder = Scan<size_t>(args[1]);
} else if (args[0] == "path") {
m_filePath = args[1];
} else {
throw "Unknown argument " + args[0];
}
}
}
LanguageModelIRST::~LanguageModelIRST()

16
moses/LM/Implementation.cpp
View File

@ -41,6 +41,22 @@ using namespace std;
namespace Moses
{
LanguageModelImplementation::LanguageModelImplementation(const std::string& description, const std::string &line)
:LanguageModel(description, line)
{
}
void LanguageModelImplementation::SetParameter(const std::string& key, const std::string& value)
{
if (key == "order") {
m_nGramOrder = Scan<size_t>(value);
} else if (key == "path") {
m_filePath = value;
} else {
LanguageModel::SetParameter(key, value);
}
}
void LanguageModelImplementation::ShiftOrPush(std::vector<const Word*> &contextFactor, const Word &word) const
{

7
moses/LM/Implementation.h
View File

@ -61,13 +61,14 @@ protected:
Word m_sentenceStartWord, m_sentenceEndWord; //! Contains factors which represents the beging and end words for this LM.
//! Usually <s> and </s>
LanguageModelImplementation(const std::string& description, const std::string &line)
:LanguageModel(description, line) {
}
LanguageModelImplementation(const std::string& description, const std::string &line);
public:
virtual ~LanguageModelImplementation() {}
void SetParameter(const std::string& key, const std::string& value);
/* get score of n-gram. n-gram should not be bigger than m_nGramOrder
* Specific implementation can return State and len data to be used in hypothesis pruning
* \param contextFactor n-gram to be scored

6
moses/LM/Jamfile
View File

@ -75,7 +75,11 @@ obj ORLM.o : ORLM.cpp ..//headers ../TranslationModel/DynSAInclude//dynsa : : :
#Top-level LM library. If you've added a file that doesn't depend on external
#libraries, put it here.
alias LM : Base.cpp Implementation.cpp Joint.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ORLM.o
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp Implementation.cpp Joint.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ORLM.o
../../lm//kenlm ..//headers $(dependencies) ;
alias macros : : : : <define>$(lmmacros) ;
#Unit test for Backward LM
import testing ;
run BackwardTest.cpp ..//moses LM ../../lm//kenlm /top//boost_unit_test_framework : : backward.arpa ;

126
moses/LM/Ken.cpp
View File

@ -60,62 +60,62 @@ struct KenLMState : public FFState {
}
};
/*
* An implementation of single factor LM using Ken's code.
*/
template <class Model> class LanguageModelKen : public LanguageModel
{
public:
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
const FFState *EmptyHypothesisState(const InputType &/*input*/) const {
KenLMState *ret = new KenLMState();
ret->state = m_ngram->BeginSentenceState();
return ret;
}
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
void IncrementalCallback(Incremental::Manager &manager) const {
manager.LMCallback(*m_ngram, m_lmIdLookup);
}
bool IsUseable(const FactorMask &mask) const;
private:
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
for (; ; ++index, --position) {
if (index == end) return index;
if (position == -1) {
*index = m_ngram->GetVocabulary().BeginSentence();
return index + 1;
}
*index = TranslateID(hypo.GetWord(position));
}
}
boost::shared_ptr<Model> m_ngram;
std::vector<lm::WordIndex> m_lmIdLookup;
FactorType m_factorType;
const Factor *m_beginSentenceFactor;
};
///*
// * An implementation of single factor LM using Ken's code.
// */
//template <class Model> class LanguageModelKen : public LanguageModel
//{
//public:
// LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
//
// const FFState *EmptyHypothesisState(const InputType &/*input*/) const {
// KenLMState *ret = new KenLMState();
// ret->state = m_ngram->BeginSentenceState();
// return ret;
// }
//
// void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
//
// FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
//
// FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
//
// void IncrementalCallback(Incremental::Manager &manager) const {
// manager.LMCallback(*m_ngram, m_lmIdLookup);
// }
//
// bool IsUseable(const FactorMask &mask) const;
//private:
// LanguageModelKen(const LanguageModelKen<Model> &copy_from);
//
// lm::WordIndex TranslateID(const Word &word) const {
// std::size_t factor = word.GetFactor(m_factorType)->GetId();
// return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
// }
//
// // Convert last words of hypothesis into vocab ids, returning an end pointer.
// lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
// lm::WordIndex *index = indices;
// lm::WordIndex *end = indices + m_ngram->Order() - 1;
// int position = hypo.GetCurrTargetWordsRange().GetEndPos();
// for (; ; ++index, --position) {
// if (index == end) return index;
// if (position == -1) {
// *index = m_ngram->GetVocabulary().BeginSentence();
// return index + 1;
// }
// *index = TranslateID(hypo.GetWord(position));
// }
// }
//
// boost::shared_ptr<Model> m_ngram;
//
// std::vector<lm::WordIndex> m_lmIdLookup;
//
// FactorType m_factorType;
//
// const Factor *m_beginSentenceFactor;
//};
class MappingBuilder : public lm::EnumerateVocab
{
@ -137,6 +137,8 @@ private:
std::vector<lm::WordIndex> &m_mapping;
};
} // namespace
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
:LanguageModel("KENLM", line)
,m_factorType(factorType)
@ -168,6 +170,13 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageM
{
}
template <class Model> const FFState * LanguageModelKen<Model>::EmptyHypothesisState(const InputType &/*input*/) const
{
KenLMState *ret = new KenLMState();
ret->state = m_ngram->BeginSentenceState();
return ret;
}
template <class Model> void LanguageModelKen<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const
{
fullScore = 0;
@ -342,6 +351,10 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const Cha
return newState;
}
template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Incremental::Manager &manager) const {
manager.LMCallback(*m_ngram, m_lmIdLookup);
}
template <class Model>
bool LanguageModelKen<Model>::IsUseable(const FactorMask &mask) const
{
@ -349,7 +362,6 @@ bool LanguageModelKen<Model>::IsUseable(const FactorMask &mask) const
return ret;
}
} // namespace
LanguageModel *ConstructKenLM(const std::string &line)
{

63
moses/LM/Ken.h
View File

@ -23,19 +23,80 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#define moses_LanguageModelKen_h
#include <string>
#include <boost/shared_ptr.hpp>
#include "lm/word_index.hh"
#include "moses/LM/Base.h"
#include "moses/Hypothesis.h"
#include "moses/TypeDef.h"
#include "moses/Word.h"
namespace Moses
{
class LanguageModel;
//class LanguageModel;
class FFState;
LanguageModel *ConstructKenLM(const std::string &line);
//! This will also load. Returns a templated KenLM class
LanguageModel *ConstructKenLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
/*
* An implementation of single factor LM using Kenneth's code.
*/
template <class Model> class LanguageModelKen : public LanguageModel
{
public:
LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
virtual FFState *Evaluate(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const;
virtual FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const;
virtual void IncrementalCallback(Incremental::Manager &manager) const;
virtual bool IsUseable(const FactorMask &mask) const;
protected:
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
private:
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
for (; ; ++index, --position) {
if (index == end) return index;
if (position == -1) {
*index = m_ngram->GetVocabulary().BeginSentence();
return index + 1;
}
*index = TranslateID(hypo.GetWord(position));
}
}
std::vector<lm::WordIndex> m_lmIdLookup;
};
} // namespace Moses
#endif

8
moses/LM/LDHT.cpp
View File

@ -2,6 +2,13 @@
// Oliver Wilson <oliver.wilson@ed.ac.uk>
//
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND
#include "LM/Base.h"
#include "LM/LDHT.h"
#include "moses/FFState.h"
@ -411,3 +418,4 @@ float LanguageModelLDHT::calcScoreFromState(LDHTLMState* state) const
} // namespace Moses.
#endif

15
moses/LM/ParallelBackoff.cpp
View File

@ -35,6 +35,21 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/TypeDef.h"
#include "moses/Util.h"
// By default, SRILM defines a function called zopen.
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__
#define HAVE_ZOPEN
#endif
#include "FNgramSpecs.h"
#include "FNgramStats.h"
#include "FactoredVocab.h"

8
moses/LM/Rand.cpp
View File

@ -17,6 +17,13 @@ License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
// This file should be compiled only when the LM_RAND flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
#ifdef LM_RAND
#include <limits>
#include <iostream>
#include <fstream>
@ -140,3 +147,4 @@ void LanguageModelRandLM::CleanUpAfterSentenceProcessing(const InputType& source
}
#endif

30
moses/LM/SRI.cpp
View File

@ -31,6 +31,21 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "moses/Phrase.h"
#include "moses/StaticData.h"
// By default, SRILM defines a function called zopen.
//
// However, on Mac OS X (and possibly other BSDs),
// <stdio.h> already defines a zopen function.
//
// To resolve this conflict, SRILM checks to see if HAVE_ZOPEN is defined.
// If it is, SRILM will rename its zopen function as my_zopen.
//
// So, before importing any SRILM headers,
// it is important to define HAVE_ZOPEN if we are on an Apple OS:
//
#ifdef __APPLE__
#define HAVE_ZOPEN
#endif
#include "Vocab.h"
#include "Ngram.h"
@ -43,20 +58,7 @@ LanguageModelSRI::LanguageModelSRI(const std::string &line)
,m_srilmVocab(0)
,m_srilmModel(0)
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "factor") {
m_factorType = Scan<FactorType>(args[1]);
} else if (args[0] == "order") {
m_nGramOrder = Scan<size_t>(args[1]);
} else if (args[0] == "path") {
m_filePath = args[1];
} else {
throw "Unknown argument " + args[0];
}
}
ReadParameters();
}
LanguageModelSRI::~LanguageModelSRI()

9
moses/LM/SingleFactor.cpp
View File

@ -72,6 +72,15 @@ bool LanguageModelSingleFactor::IsUseable(const FactorMask &mask) const
return ret;
}
void LanguageModelSingleFactor::SetParameter(const std::string& key, const std::string& value)
{
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
} else {
LanguageModelImplementation::SetParameter(key, value);
}
}
}

1
moses/LM/SingleFactor.h
View File

@ -48,6 +48,7 @@ protected:
public:
virtual ~LanguageModelSingleFactor();
bool IsUseable(const FactorMask &mask) const;
void SetParameter(const std::string& key, const std::string& value);
const Factor *GetSentenceStart() const {
return m_sentenceStart;

1
moses/Manager.cpp
View File

@ -37,7 +37,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "TrellisPath.h"
#include "TrellisPathCollection.h"
#include "TranslationOption.h"
#include "LexicalReordering.h"
#include "TranslationOptionCollection.h"
#include "Timer.h"
#include "moses/FF/DistortionScoreProducer.h"

65
moses/PCNTools.cpp
View File

@ -2,6 +2,9 @@
#include <iostream>
#include <cstdlib>
#include "Util.h"
using namespace std;
namespace PCN
{
@ -25,6 +28,17 @@ inline void eatws(const std::string& in, int& c)
}
}
std::string getString(const std::string& in, int &c)
{
std::string ret;
eatws(in,c);
while (c < (int)in.size() && get(in,c) != ' ' && get(in,c) != ')' && get(in,c) != ',') {
ret += get(in,c++);
}
eatws(in,c);
return ret;
}
// from 'foo' return foo
std::string getEscapedString(const std::string& in, int &c)
{
@ -82,29 +96,52 @@ CNAlt getCNAlt(const std::string& in, int &c)
return CNAlt();
}
size_t cnNext = 1;
std::vector<float> probs;
probs.push_back(getFloat(in,c));
// read all tokens after the 1st
std::vector<string> toks;
toks.push_back(getString(in,c));
while (get(in,c) == ',') {
c++;
float val = getFloat(in,c);
probs.push_back(val);
string tok = getString(in,c);
toks.push_back(tok);
}
//if we read more than one prob, this was a lattice, last item was column increment
if (probs.size()>1) {
cnNext = static_cast<size_t>(probs.back());
probs.pop_back();
if (cnNext < 1) {
; //throw "bad link length"
std::cerr << "PCN/PLF parse error: bad link length at last element of cn alt block\n";
return CNAlt();
}
std::vector<float> probs;
// dense scores
size_t ind;
for (ind = 0; ind < toks.size() - 1; ++ind) {
const string &tok = toks[ind];
if (tok.find('=') == tok.npos) {
float val = Moses::Scan<float>(tok);
probs.push_back(val);
}
else {
// beginning of sparse feature
break;
}
}
// sparse features
std::map<string, float> sparseFeatures;
for (; ind < toks.size() - 1; ++ind) {
const string &tok = toks[ind];
vector<string> keyValue = Moses::Tokenize(tok, "=");
CHECK(keyValue.size() == 2);
float prob = Moses::Scan<float>(keyValue[1]);
sparseFeatures[ keyValue[0] ] = prob;
}
//last item is column increment
cnNext = Moses::Scan<size_t>(toks.back());
if (get(in,c++) != ')') {
std::cerr << "PCN/PLF parse error: expected ) at end of cn alt block\n"; // throw "expected )";
return CNAlt();
}
eatws(in,c);
return CNAlt(std::pair<std::string, std::vector<float> >(word,probs), cnNext);
return CNAlt(word, probs, sparseFeatures, cnNext);
}
// parse (('foo', 0.23), ('bar', 0.77))

23
moses/PCNTools.h
View File

@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#define moses_PCNTools
#include <vector>
#include <map>
#include <string>
#include <utility>
#include <cstdlib>
@ -33,7 +34,27 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace PCN
{
typedef std::pair<std::pair<std::string, std::vector<float> >, size_t> CNAlt;
struct CNAlt
{
CNAlt()
{}
CNAlt(const std::string &word,
const std::vector<float> &denseFeatures,
const std::map<std::string, float> &sparseFeatures,
size_t next)
:m_word(word)
,m_denseFeatures(denseFeatures)
,m_sparseFeatures(sparseFeatures)
,m_next(next)
{}
std::string m_word;
std::vector<float> m_denseFeatures;
std::map<std::string, float> m_sparseFeatures;
size_t m_next;
};
//typedef std::pair<std::pair<std::string, std::vector<float> >, size_t> CNAlt;
typedef std::vector<CNAlt> CNCol;
typedef std::vector<CNCol> CN;

20
moses/Phrase.cpp
View File

@ -358,6 +358,26 @@ void Phrase::OnlyTheseFactors(const FactorMask &factors)
}
}
void Phrase::InitStartEndWord()
{
FactorCollection &factorCollection = FactorCollection::Instance();
Word startWord(Input);
const Factor *factor = factorCollection.AddFactor(Input, 0, BOS_); // TODO - non-factored
startWord.SetFactor(0, factor);
PrependWord(startWord);
Word endWord(Input);
factor = factorCollection.AddFactor(Input, 0, EOS_); // TODO - non-factored
endWord.SetFactor(0, factor);
AddWord(endWord);
}
bool Phrase::Contains(const Phrase &sought) const
{
}
TO_STRING_BODY(Phrase);
// friend

11
moses/Phrase.h
View File

@ -63,6 +63,13 @@ public:
/** create phrase from vectors of words */
explicit Phrase(const std::vector< const Word* > &mergeWords);
/* This isn't a swap function because classes inherit from Phrase and might
* not override swap, which would be bad.
*/
void SwapWords(Phrase &other) {
swap(m_words, other.m_words);
}
/** destructor */
virtual ~Phrase();
@ -121,6 +128,8 @@ public:
bool Contains(const std::vector< std::vector<std::string> > &subPhraseVector
, const std::vector<FactorType> &inputFactor) const;
bool Contains(const Phrase &sought) const;
//! create an empty word at the end of the phrase
Word &AddWord();
//! create copy of input word at the end of the phrase
@ -141,6 +150,8 @@ public:
m_words.erase(m_words.begin() + pos);
}
void InitStartEndWord();
//! create new phrase class that is a substring of this phrase
Phrase GetSubString(const WordsRange &wordsRange) const;
Phrase GetSubString(const WordsRange &wordsRange, FactorType factorType) const;

11
moses/RuleCubeItem.cpp
View File

@ -20,6 +20,7 @@
#include "ChartCell.h"
#include "ChartCellCollection.h"
#include "ChartTranslationOptions.h"
#include "ChartManager.h"
#include "RuleCubeItem.h"
#include "RuleCubeQueue.h"
#include "WordsRange.h"
@ -77,6 +78,16 @@ void RuleCubeItem::CreateHypothesis(const ChartTranslationOptions &transOpt,
ChartManager &manager)
{
m_hypothesis = new ChartHypothesis(transOpt, *this, manager);
const Phrase *constraint = manager.GetConstraint();
if (constraint) {
Phrase hypoPhrase = m_hypothesis->GetOutputPhrase();
if (!constraint->Contains(hypoPhrase)) {
delete m_hypothesis;
m_hypothesis = NULL;
return;
}
}
m_hypothesis->Evaluate();
m_score = m_hypothesis->GetTotalScore();
}

8
moses/ScoreComponentCollection.h
View File

@ -63,8 +63,11 @@ namespace Moses
class ScoreComponentCollection
{
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
friend void swap(ScoreComponentCollection &first, ScoreComponentCollection &second);
private:
FVector m_scores;
typedef std::pair<size_t,size_t> IndexPair;
typedef std::map<const FeatureFunction*,IndexPair> ScoreIndexMap;
static ScoreIndexMap s_scoreIndexes;
@ -414,5 +417,10 @@ struct SCCPlus {
}
};
inline void swap(ScoreComponentCollection &first, ScoreComponentCollection &second)
{
swap(first.m_scores, second.m_scores);
}
}
#endif

17
moses/ScoreComponentCollectionTest.cpp
View File

@ -34,12 +34,18 @@ class MockStatelessFeatureFunction : public StatelessFeatureFunction
public:
MockStatelessFeatureFunction(const string& desc, size_t n, const string &line) :
StatelessFeatureFunction(desc,n, line) {}
virtual void Evaluate(const PhraseBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void EvaluateChart(const ChartBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void Evaluate(const TargetPhrase &targetPhrase
void Evaluate(const Hypothesis&, ScoreComponentCollection*) const {}
void EvaluateChart(const ChartHypothesis&, ScoreComponentCollection*) const {}
void Evaluate(const InputType &input
, const InputPath &inputPath
, ScoreComponentCollection &scoreBreakdown) const
{}
void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const {
}
, ScoreComponentCollection &estimatedFutureScore) const
{}
};
class MockSingleFeature : public MockStatelessFeatureFunction
@ -60,6 +66,7 @@ public:
bool IsUseable(const FactorMask &mask) const {
return true;
}
};
class MockSparseFeature : public MockStatelessFeatureFunction

15
moses/Sentence.cpp
View File

@ -194,21 +194,6 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
return 1;
}
void Sentence::InitStartEndWord()
{
FactorCollection &factorCollection = FactorCollection::Instance();
Word startWord(Input);
const Factor *factor = factorCollection.AddFactor(Input, 0, BOS_); // TODO - non-factored
startWord.SetFactor(0, factor);
PrependWord(startWord);
Word endWord(Input);
factor = factorCollection.AddFactor(Input, 0, EOS_); // TODO - non-factored
endWord.SetFactor(0, factor);
AddWord(endWord);
}
void Sentence::ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders)
{
FactorType placeholderFactor = StaticData::Instance().GetPlaceholderFactor().first;

1
moses/Sentence.h
View File

@ -58,7 +58,6 @@ protected:
NonTerminalSet m_defaultLabelSet;
void InitStartEndWord();
void ProcessPlaceholders(const std::vector< std::pair<size_t, std::string> > &placeholders);

67
moses/StaticData.cpp
View File

@ -35,7 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Util.h"
#include "FactorCollection.h"
#include "Timer.h"
#include "LexicalReordering.h"
#include "SentenceStats.h"
#include "UserMessage.h"
#include "TranslationOption.h"
@ -452,37 +451,7 @@ bool StaticData::LoadData(Parameter *parameter)
Scan<long>(m_parameter->GetParam("start-translation-id")[0]) : 0;
// Read in constraint decoding file, if provided
if(m_parameter->GetParam("constraint").size()) {
if (m_parameter->GetParam("search-algorithm").size() > 0
&& Scan<size_t>(m_parameter->GetParam("search-algorithm")[0]) != 0) {
cerr << "Can use -constraint only with stack-based search (-search-algorithm 0)" << endl;
exit(1);
}
m_constraintFileName = m_parameter->GetParam("constraint")[0];
InputFileStream constraintFile(m_constraintFileName);
std::string line;
long sentenceID = GetStartTranslationId() - 1;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");
if (vecStr.size() == 1) {
sentenceID++;
Phrase phrase(0);
phrase.CreateFromString(Output, GetOutputFactorOrder(), vecStr[0], GetFactorDelimiter(), NULL);
m_constraints.insert(make_pair(sentenceID,phrase));
} else if (vecStr.size() == 2) {
sentenceID = Scan<long>(vecStr[0]);
Phrase phrase(0);
phrase.CreateFromString(Output, GetOutputFactorOrder(), vecStr[1], GetFactorDelimiter(), NULL);
m_constraints.insert(make_pair(sentenceID,phrase));
} else {
CHECK(false);
}
}
}
ForcedDecoding();
// use of xml in input
if (m_parameter->GetParam("xml-input").size() == 0) m_xmlInputType = XmlPassThrough;
@ -1081,5 +1050,39 @@ void StaticData::OverrideFeatures()
}
void StaticData::ForcedDecoding()
{
if(m_parameter->GetParam("constraint").size()) {
bool addBeginEndWord = (m_searchAlgorithm == ChartDecoding) || (m_searchAlgorithm == ChartIncremental);
m_constraintFileName = m_parameter->GetParam("constraint")[0];
InputFileStream constraintFile(m_constraintFileName);
std::string line;
long sentenceID = GetStartTranslationId() - 1;
while (getline(constraintFile, line)) {
vector<string> vecStr = Tokenize(line, "\t");
Phrase phrase(0);
if (vecStr.size() == 1) {
sentenceID++;
phrase.CreateFromString(Output, GetOutputFactorOrder(), vecStr[0], GetFactorDelimiter(), NULL);
} else if (vecStr.size() == 2) {
sentenceID = Scan<long>(vecStr[0]);
phrase.CreateFromString(Output, GetOutputFactorOrder(), vecStr[1], GetFactorDelimiter(), NULL);
} else {
CHECK(false);
}
if (addBeginEndWord) {
phrase.InitStartEndWord();
}
m_constraints.insert(make_pair(sentenceID,phrase));
}
}
}
} // namespace

7
moses/StaticData.h
View File

@ -221,9 +221,9 @@ protected:
//! load decoding steps
bool LoadDecodeGraphs();
void ReduceTransOptCache() const;
bool m_continuePartialTranslation;
void ForcedDecoding();
bool m_continuePartialTranslation;
std::string m_binPath;
public:
@ -425,9 +425,6 @@ public:
bool IsChart() const {
return m_searchAlgorithm == ChartDecoding || m_searchAlgorithm == ChartIncremental;
}
const WordPenaltyProducer *GetWordPenaltyProducer() const {
return m_wpProducer;
}
WordPenaltyProducer *GetWordPenaltyProducer() { // for mira
return m_wpProducer;
}

9
moses/SyntacticLanguageModel.cpp
View File

@ -1,3 +1,10 @@
// This file should be compiled only when the HAVE_SYNLM flag is enabled.
//
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when HAVE_SYNLM is disabled.
//
#ifdef HAVE_SYNLM
//
#include "StaticData.h"
@ -166,3 +173,5 @@ FFState* SyntacticLanguageModel::Evaluate(const Hypothesis& cur_hypo,
}
}
#endif

46
moses/TargetPhrase.cpp
View File

@ -201,6 +201,52 @@ void TargetPhrase::Merge(const TargetPhrase &copy, const std::vector<FactorType>
m_fullScore += copy.m_fullScore;
}
void TargetPhrase::SetProperties(const StringPiece &str)
{
if (str.size() == 0) {
return;
}
vector<string> toks;
TokenizeMultiCharSeparator(toks, str.as_string(), "{{");
for (size_t i = 0; i < toks.size(); ++i) {
string &tok = toks[i];
if (tok.empty()) {
continue;
}
size_t endPos = tok.rfind("}");
tok = tok.substr(0, endPos - 1);
vector<string> keyValue = TokenizeFirstOnly(tok, " ");
CHECK(keyValue.size() == 2);
SetProperty(keyValue[0], keyValue[1]);
}
}
void TargetPhrase::GetProperty(const std::string &key, std::string &value, bool &found) const
{
std::map<std::string, std::string>::const_iterator iter;
iter = m_properties.find(key);
if (iter == m_properties.end()) {
found = false;
} else {
found = true;
value = iter->second;
}
}
void swap(TargetPhrase &first, TargetPhrase &second)
{
first.SwapWords(second);
std::swap(first.m_fullScore, second.m_fullScore);
std::swap(first.m_futureScore, second.m_futureScore);
swap(first.m_scoreBreakdown, second.m_scoreBreakdown);
std::swap(first.m_alignTerm, second.m_alignTerm);
std::swap(first.m_alignNonTerm, second.m_alignNonTerm);
std::swap(first.m_lhsTarget, second.m_lhsTarget);
}
TO_STRING_BODY(TargetPhrase);
std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp)

14
moses/TargetPhrase.h
View File

@ -22,6 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifndef moses_TargetPhrase_h
#define moses_TargetPhrase_h
#include <algorithm>
#include <vector>
#include "TypeDef.h"
#include "Phrase.h"
@ -43,14 +44,17 @@ class InputPath;
*/
class TargetPhrase: public Phrase
{
private:
friend std::ostream& operator<<(std::ostream&, const TargetPhrase&);
protected:
friend void swap(TargetPhrase &first, TargetPhrase &second);
float m_fullScore, m_futureScore;
ScoreComponentCollection m_scoreBreakdown;
const AlignmentInfo* m_alignTerm, *m_alignNonTerm;
const Word *m_lhsTarget;
std::map<std::string, std::string> m_properties;
public:
TargetPhrase();
TargetPhrase(const TargetPhrase &copy);
@ -119,11 +123,19 @@ public:
return *m_alignNonTerm;
}
void SetProperties(const StringPiece &str);
void SetProperty(const std::string &key, const std::string &value) {
m_properties[key] = value;
}
void GetProperty(const std::string &key, std::string &value, bool &found) const;
void Merge(const TargetPhrase &copy, const std::vector<FactorType>& factorVec);
TO_STRING();
};
void swap(TargetPhrase &first, TargetPhrase &second);
std::ostream& operator<<(std::ostream&, const TargetPhrase&);
/**

2
moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h
View File

@ -22,7 +22,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#ifndef moses_LexicalReorderingTableCompact_h
#define moses_LexicalReorderingTableCompact_h
#include "moses/LexicalReorderingTable.h"
#include "moses/FF/LexicalReordering/LexicalReorderingTable.h"
#include "moses/StaticData.h"
#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/GenerationDictionary.h"

Some files were not shown because too many files have changed in this diff Show More