moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-16 23:10:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:moses-smt/mosesdecoder

Browse Source
This commit is contained in:
Hieu Hoang 2017-04-27 13:48:28 +01:00
commit b199e654df
27 changed files with 227 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
contrib/other-builds/moses/.project
View File

@ -1200,6 +1200,31 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/EditOps.h</locationURI>
</link>
<link>
<name>FF/ExampleStatefulFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ExampleStatefulFF.cpp</locationURI>
</link>
<link>
<name>FF/ExampleStatefulFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ExampleStatefulFF.h</locationURI>
</link>
<link>
<name>FF/ExampleStatelessFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ExampleStatelessFF.cpp</locationURI>
</link>
<link>
<name>FF/ExampleStatelessFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ExampleStatelessFF.h</locationURI>
</link>
<link>
<name>FF/ExampleTranslationOptionListFeature.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/ExampleTranslationOptionListFeature.h</locationURI>
</link>
<link>
<name>FF/FFState.cpp</name>
<type>1</type>
@ -1420,26 +1445,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SetSourcePhrase.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatefulFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatefulFF.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatelessFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/FF/SkeletonStatelessFF.h</locationURI>
</link>
<link>
<name>FF/SoftMatchingFeature.cpp</name>
<type>1</type>
@ -1695,6 +1700,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/DALMWrapper.h</locationURI>
</link>
<link>
<name>LM/ExampleLM.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/ExampleLM.cpp</locationURI>
</link>
<link>
<name>LM/ExampleLM.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/ExampleLM.h</locationURI>
</link>
<link>
<name>LM/IRST.cpp</name>
<type>1</type>
@ -1845,16 +1860,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SingleFactor.h</locationURI>
</link>
<link>
<name>LM/SkeletonLM.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SkeletonLM.cpp</locationURI>
</link>
<link>
<name>LM/SkeletonLM.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/LM/SkeletonLM.h</locationURI>
</link>
<link>
<name>LM/backward.arpa</name>
<type>1</type>
@ -2140,6 +2145,16 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/ExamplePT.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/ExamplePT.cpp</locationURI>
</link>
<link>
<name>TranslationModel/ExamplePT.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/ExamplePT.h</locationURI>
</link>
<link>
<name>TranslationModel/PhraseDictionary.cpp</name>
<type>1</type>
@ -2290,16 +2305,6 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/SkeletonPT.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/SkeletonPT.cpp</locationURI>
</link>
<link>
<name>TranslationModel/SkeletonPT.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/SkeletonPT.h</locationURI>
</link>
<link>
<name>TranslationModel/UG</name>
<type>2</type>
@ -3135,6 +3140,16 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerCYKPlus.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerMemory.cpp</name>
<type>1</type>
@ -3165,16 +3180,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h</locationURI>
</link>
<link>
<name>TranslationModel/CYKPlusParser/CompletedRuleCollection.cpp</name>
<type>1</type>

40
contrib/other-builds/moses2/.project
View File

@ -357,6 +357,26 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/Distortion.h</locationURI>
</link>
<link>
<name>FF/ExampleStatefulFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatefulFF.cpp</locationURI>
</link>
<link>
<name>FF/ExampleStatefulFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatefulFF.h</locationURI>
</link>
<link>
<name>FF/ExampleStatelessFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatelessFF.cpp</locationURI>
</link>
<link>
<name>FF/ExampleStatelessFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/ExampleStatelessFF.h</locationURI>
</link>
<link>
<name>FF/FFState.cpp</name>
<type>1</type>
@ -427,26 +447,6 @@
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/PointerState.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/SkeletonStatefulFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatefulFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/SkeletonStatefulFF.h</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.cpp</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/SkeletonStatelessFF.cpp</locationURI>
</link>
<link>
<name>FF/SkeletonStatelessFF.h</name>
<type>1</type>
<locationURI>PARENT-3-PROJECT_LOC/moses2/FF/SkeletonStatelessFF.h</locationURI>
</link>
<link>
<name>FF/StatefulFeatureFunction.cpp</name>
<type>1</type>

20
moses/FF/SkeletonStatefulFF.cpp → moses/FF/ExampleStatefulFF.cpp
View File

@ -1,5 +1,5 @@
#include <vector>
#include "SkeletonStatefulFF.h"
#include "ExampleStatefulFF.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/Hypothesis.h"
@ -9,7 +9,7 @@ namespace Moses
{
////////////////////////////////////////////////////////////////
SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
ExampleStatefulFF::ExampleStatefulFF(const std::string &line)
:StatefulFeatureFunction(3, line)
{
ReadParameters();
@ -19,7 +19,7 @@ SkeletonStatefulFF::SkeletonStatefulFF(const std::string &line)
// An empty implementation of this function is provided by StatefulFeatureFunction.
// Unless you are actually implementing this, please remove it from your
// implementation (and the declaration in the header file to reduce code clutter.
void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source
void ExampleStatefulFF::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedScores) const
@ -28,7 +28,7 @@ void SkeletonStatefulFF::EvaluateInIsolation(const Phrase &source
// An empty implementation of this function is provided by StatefulFeatureFunction.
// Unless you are actually implementing this, please remove it from your
// implementation (and the declaration in the header file to reduce code clutter.
void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input
void ExampleStatefulFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -39,11 +39,11 @@ void SkeletonStatefulFF::EvaluateWithSourceContext(const InputType &input
// An empty implementation of this function is provided by StatefulFeatureFunction.
// Unless you are actually implementing this, please remove it from your
// implementation (and the declaration in the header file to reduce code clutter.
void SkeletonStatefulFF::EvaluateTranslationOptionListWithSourceContext
void ExampleStatefulFF::EvaluateTranslationOptionListWithSourceContext
(const InputType &input, const TranslationOptionList &translationOptionList) const
{}
FFState* SkeletonStatefulFF::EvaluateWhenApplied(
FFState* ExampleStatefulFF::EvaluateWhenApplied(
const Hypothesis& cur_hypo,
const FFState* prev_state,
ScoreComponentCollection* accumulator) const
@ -59,18 +59,18 @@ FFState* SkeletonStatefulFF::EvaluateWhenApplied(
accumulator->PlusEquals(this, "sparse-name", 2.4);
// int targetLen = cur_hypo.GetCurrTargetPhrase().GetSize(); // ??? [UG]
return new SkeletonState(0);
return new ExampleState(0);
}
FFState* SkeletonStatefulFF::EvaluateWhenApplied(
FFState* ExampleStatefulFF::EvaluateWhenApplied(
const ChartHypothesis& /* cur_hypo */,
int /* featureID - used to index the state in the previous hypotheses */,
ScoreComponentCollection* accumulator) const
{
return new SkeletonState(0);
return new ExampleState(0);
}
void SkeletonStatefulFF::SetParameter(const std::string& key, const std::string& value)
void ExampleStatefulFF::SetParameter(const std::string& key, const std::string& value)
{
if (key == "arg") {
// set value here

12
moses/FF/SkeletonStatefulFF.h → moses/FF/ExampleStatefulFF.h
View File

@ -7,11 +7,11 @@
namespace Moses
{
class SkeletonState : public FFState
class ExampleState : public FFState
{
int m_targetLen;
public:
SkeletonState(int targetLen)
ExampleState(int targetLen)
:m_targetLen(targetLen) {
}
@ -19,22 +19,22 @@ public:
return (size_t) m_targetLen;
}
virtual bool operator==(const FFState& o) const {
const SkeletonState& other = static_cast<const SkeletonState&>(o);
const ExampleState& other = static_cast<const ExampleState&>(o);
return m_targetLen == other.m_targetLen;
}
};
class SkeletonStatefulFF : public StatefulFeatureFunction
class ExampleStatefulFF : public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(const std::string &line);
ExampleStatefulFF(const std::string &line);
bool IsUseable(const FactorMask &mask) const {
return true;
}
virtual const FFState* EmptyHypothesisState(const InputType &input) const {
return new SkeletonState(0);
return new ExampleState(0);
}
// An empty implementation of this function is provided by StatefulFeatureFunction.

16
moses/FF/SkeletonStatelessFF.cpp → moses/FF/ExampleStatelessFF.cpp
View File

@ -1,5 +1,5 @@
#include <vector>
#include "SkeletonStatelessFF.h"
#include "ExampleStatelessFF.h"
#include "moses/ScoreComponentCollection.h"
#include "moses/TargetPhrase.h"
@ -7,13 +7,13 @@ using namespace std;
namespace Moses
{
SkeletonStatelessFF::SkeletonStatelessFF(const std::string &line)
ExampleStatelessFF::ExampleStatelessFF(const std::string &line)
:StatelessFeatureFunction(2, line)
{
ReadParameters();
}
void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source
void ExampleStatelessFF::EvaluateInIsolation(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedScores) const
@ -29,7 +29,7 @@ void SkeletonStatelessFF::EvaluateInIsolation(const Phrase &source
}
void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
void ExampleStatelessFF::EvaluateWithSourceContext(const InputType &input
, const InputPath &inputPath
, const TargetPhrase &targetPhrase
, const StackVec *stackVec
@ -43,20 +43,20 @@ void SkeletonStatelessFF::EvaluateWithSourceContext(const InputType &input
}
}
void SkeletonStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input
void ExampleStatelessFF::EvaluateTranslationOptionListWithSourceContext(const InputType &input
, const TranslationOptionList &translationOptionList) const
{}
void SkeletonStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
void ExampleStatelessFF::EvaluateWhenApplied(const Hypothesis& hypo,
ScoreComponentCollection* accumulator) const
{}
void SkeletonStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
void ExampleStatelessFF::EvaluateWhenApplied(const ChartHypothesis &hypo,
ScoreComponentCollection* accumulator) const
{}
void SkeletonStatelessFF::SetParameter(const std::string& key, const std::string& value)
void ExampleStatelessFF::SetParameter(const std::string& key, const std::string& value)
{
if (key == "arg") {
// set value here

4
moses/FF/SkeletonStatelessFF.h → moses/FF/ExampleStatelessFF.h
View File

@ -6,10 +6,10 @@
namespace Moses
{
class SkeletonStatelessFF : public StatelessFeatureFunction
class ExampleStatelessFF : public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(const std::string &line);
ExampleStatelessFF(const std::string &line);
bool IsUseable(const FactorMask &mask) const {
return true;

4
moses/FF/SkeletonTranslationOptionListFeature.h → moses/FF/ExampleTranslationOptionListFeature.h
View File

@ -6,10 +6,10 @@
namespace Moses
{
class SkeletonTranslationOptionListFeature : public StatelessFeatureFunction
class ExampleTranslationOptionListFeature : public StatelessFeatureFunction
{
public:
SkeletonTranslationOptionListFeature(const std::string &line)
ExampleTranslationOptionListFeature(const std::string &line)
:StatelessFeatureFunction(1, line) {
ReadParameters();
}

24
moses/FF/Factory.cpp
View File

@ -65,16 +65,16 @@
#include "SyntaxRHS.h"
#include "DeleteRules.h"
#include "moses/FF/SkeletonStatelessFF.h"
#include "moses/FF/SkeletonStatefulFF.h"
#include "moses/LM/SkeletonLM.h"
#include "moses/LM/InMemoryPerSentenceOnDemandLM.h"
#include "moses/FF/SkeletonTranslationOptionListFeature.h"
#include "moses/FF/ExampleStatelessFF.h"
#include "moses/FF/ExampleStatefulFF.h"
#include "moses/LM/ExampleLM.h"
#include "moses/FF/ExampleTranslationOptionListFeature.h"
#include "moses/LM/BilingualLM.h"
#include "moses/TranslationModel/SkeletonPT.h"
#include "moses/TranslationModel/ExamplePT.h"
#include "moses/Syntax/InputWeightFF.h"
#include "moses/Syntax/RuleTableFF.h"
#include "moses/LM/InMemoryPerSentenceOnDemandLM.h"
#include "moses/FF/EditOps.h"
#include "moses/FF/CorrectionPattern.h"
@ -297,13 +297,13 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(UnalignedWordCountFeature);
MOSES_FNAME(DeleteRules);
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(SkeletonStatefulFF);
MOSES_FNAME(SkeletonLM);
MOSES_FNAME(InMemoryPerSentenceOnDemandLM);
MOSES_FNAME(SkeletonTranslationOptionListFeature);
MOSES_FNAME(SkeletonPT);
MOSES_FNAME(ExampleStatelessFF);
MOSES_FNAME(ExampleStatefulFF);
MOSES_FNAME(ExampleLM);
MOSES_FNAME(ExampleTranslationOptionListFeature);
MOSES_FNAME(ExamplePT);
MOSES_FNAME(InMemoryPerSentenceOnDemandLM);
MOSES_FNAME(EditOps);
MOSES_FNAME(CorrectionPattern);

8
moses/LM/SkeletonLM.cpp → moses/LM/ExampleLM.cpp
View File

@ -1,12 +1,12 @@
#include "SkeletonLM.h"
#include "ExampleLM.h"
#include "moses/FactorCollection.h"
using namespace std;
namespace Moses
{
SkeletonLM::SkeletonLM(const std::string &line)
ExampleLM::ExampleLM(const std::string &line)
:LanguageModelSingleFactor(line)
{
ReadParameters();
@ -24,11 +24,11 @@ SkeletonLM::SkeletonLM(const std::string &line)
m_sentenceEndWord[m_factorType] = m_sentenceEnd;
}
SkeletonLM::~SkeletonLM()
ExampleLM::~ExampleLM()
{
}
LMResult SkeletonLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
LMResult ExampleLM::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
LMResult ret;
ret.score = contextFactor.size();

6
moses/LM/SkeletonLM.h → moses/LM/ExampleLM.h
View File

@ -7,13 +7,13 @@
namespace Moses
{
class SkeletonLM : public LanguageModelSingleFactor
class ExampleLM : public LanguageModelSingleFactor
{
protected:
public:
SkeletonLM(const std::string &line);
~SkeletonLM();
ExampleLM(const std::string &line);
~ExampleLM();
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
};

2
moses/LM/Jamfile
View File

@ -138,7 +138,7 @@ if $(with-dalm) {
#Top-level LM library. If you've added a file that doesn't depend on external
#libraries, put it here.
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp ExampleLM.cpp
../../lm//kenlm ..//headers $(dependencies) ;
alias macros : : : : <define>$(lmmacros) ;

20
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.cpp → moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.cpp
View File

@ -18,7 +18,7 @@
***********************************************************************/
#include <iostream>
#include "ChartRuleLookupManagerSkeleton.h"
#include "ChartRuleLookupManagerExample.h"
#include "DotChartInMemory.h"
#include "moses/Util.h"
@ -29,29 +29,29 @@
#include "moses/NonTerminal.h"
#include "moses/ChartCellCollection.h"
#include "moses/TranslationModel/PhraseDictionaryMemory.h"
#include "moses/TranslationModel/SkeletonPT.h"
#include "moses/TranslationModel/ExamplePT.h"
using namespace std;
namespace Moses
{
ChartRuleLookupManagerSkeleton::ChartRuleLookupManagerSkeleton(
ChartRuleLookupManagerExample::ChartRuleLookupManagerExample(
const ChartParser &parser,
const ChartCellCollectionBase &cellColl,
const SkeletonPT &skeletonPt)
const ExamplePT &skeletonPt)
: ChartRuleLookupManager(parser, cellColl)
, m_skeletonPT(skeletonPt)
{
cerr << "starting ChartRuleLookupManagerSkeleton" << endl;
cerr << "starting ChartRuleLookupManagerExample" << endl;
}
ChartRuleLookupManagerSkeleton::~ChartRuleLookupManagerSkeleton()
ChartRuleLookupManagerExample::~ChartRuleLookupManagerExample()
{
// RemoveAllInColl(m_tpColl);
}
void ChartRuleLookupManagerSkeleton::GetChartRuleCollection(
void ChartRuleLookupManagerExample::GetChartRuleCollection(
const InputPath &inputPath,
size_t last,
ChartParserCallback &outColl)
@ -74,12 +74,12 @@ void ChartRuleLookupManagerSkeleton::GetChartRuleCollection(
}
TargetPhrase *
ChartRuleLookupManagerSkeleton::
ChartRuleLookupManagerExample::
CreateTargetPhrase(const Word &sourceWord) const
{
// create a target phrase from the 1st word of the source, prefix with 'ChartManagerSkeleton:'
// create a target phrase from the 1st word of the source, prefix with 'ChartManagerExample:'
string str = sourceWord.GetFactor(0)->GetString().as_string();
str = "ChartManagerSkeleton:" + str;
str = "ChartManagerExample:" + str;
TargetPhrase *tp = new TargetPhrase(&m_skeletonPT);
Word &word = tp->AddWord();

14
moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h → moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h
View File

@ -29,16 +29,16 @@ class TargetPhraseCollection;
class ChartParserCallback;
class DottedRuleColl;
class Range;
class SkeletonPT;
class ExamplePT;
class ChartRuleLookupManagerSkeleton : public ChartRuleLookupManager
class ChartRuleLookupManagerExample : public ChartRuleLookupManager
{
public:
ChartRuleLookupManagerSkeleton(const ChartParser &parser,
const ChartCellCollectionBase &cellColl,
const SkeletonPT &skeletonPt);
ChartRuleLookupManagerExample(const ChartParser &parser,
const ChartCellCollectionBase &cellColl,
const ExamplePT &skeletonPt);
~ChartRuleLookupManagerSkeleton();
~ChartRuleLookupManagerExample();
virtual void GetChartRuleCollection(
const InputPath &inputPath,
@ -50,7 +50,7 @@ private:
StackVec m_stackVec;
std::vector<TargetPhraseCollection::shared_ptr > m_tpColl;
const SkeletonPT &m_skeletonPT;
const ExamplePT &m_skeletonPT;
};
} // namespace Moses

26
moses/TranslationModel/SkeletonPT.cpp → moses/TranslationModel/ExamplePT.cpp
View File

@ -1,29 +1,29 @@
// vim:tabstop=2
#include "SkeletonPT.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
#include "ExamplePT.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h"
using namespace std;
namespace Moses
{
SkeletonPT::SkeletonPT(const std::string &line)
ExamplePT::ExamplePT(const std::string &line)
: PhraseDictionary(line, true)
{
ReadParameters();
}
void SkeletonPT::Load(AllOptions::ptr const& opts)
void ExamplePT::Load(AllOptions::ptr const& opts)
{
m_options = opts;
SetFeaturesToApply();
}
void SkeletonPT::InitializeForInput(ttasksptr const& ttask)
void ExamplePT::InitializeForInput(ttasksptr const& ttask)
{
ReduceCache();
}
void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
void ExamplePT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
{
CacheColl &cache = GetCache();
@ -46,14 +46,14 @@ void SkeletonPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQu
}
}
TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
TargetPhrase *ExamplePT::CreateTargetPhrase(const Phrase &sourcePhrase) const
{
// create a target phrase from the 1st word of the source, prefix with 'SkeletonPT:'
// create a target phrase from the 1st word of the source, prefix with 'ExamplePT:'
assert(sourcePhrase.GetSize());
assert(m_output.size() == 1);
string str = sourcePhrase.GetWord(0).GetFactor(0)->GetString().as_string();
str = "SkeletonPT:" + str;
str = "ExamplePT:" + str;
TargetPhrase *tp = new TargetPhrase(this);
Word &word = tp->AddWord();
@ -69,17 +69,17 @@ TargetPhrase *SkeletonPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
return tp;
}
ChartRuleLookupManager* SkeletonPT::CreateRuleLookupManager(const ChartParser &parser,
ChartRuleLookupManager* ExamplePT::CreateRuleLookupManager(const ChartParser &parser,
const ChartCellCollectionBase &cellCollection,
std::size_t /*maxChartSpan*/)
{
return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this);
return new ChartRuleLookupManagerExample(parser, cellCollection, *this);
}
TO_STRING_BODY(SkeletonPT);
TO_STRING_BODY(ExamplePT);
// friend
ostream& operator<<(ostream& out, const SkeletonPT& phraseDict)
ostream& operator<<(ostream& out, const ExamplePT& phraseDict)
{
return out;
}

6
moses/TranslationModel/SkeletonPT.h → moses/TranslationModel/ExamplePT.h
View File

@ -9,12 +9,12 @@ class ChartParser;
class ChartCellCollectionBase;
class ChartRuleLookupManager;
class SkeletonPT : public PhraseDictionary
class ExamplePT : public PhraseDictionary
{
friend std::ostream& operator<<(std::ostream&, const SkeletonPT&);
friend std::ostream& operator<<(std::ostream&, const ExamplePT&);
public:
SkeletonPT(const std::string &line);
ExamplePT(const std::string &line);
void Load(AllOptions::ptr const& opts);

2
moses/TranslationModel/PhraseDictionaryMemoryPerSentence.cpp
View File

@ -1,6 +1,6 @@
// vim:tabstop=2
#include "PhraseDictionaryMemoryPerSentence.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerExample.h"
using namespace std;

1
moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp
View File

@ -1,6 +1,5 @@
// vim:tabstop=2
#include "PhraseDictionaryMemoryPerSentenceOnDemand.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
#include <sstream>
using namespace std;

1
moses/TranslationModel/PhraseDictionaryTransliteration.cpp
View File

@ -2,7 +2,6 @@
#include <cstdlib>
#include "PhraseDictionaryTransliteration.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
#include "moses/DecodeGraph.h"
#include "moses/DecodeStep.h"
#include "util/tempfile.hh"

1
moses/TranslationModel/ProbingPT.cpp
View File

@ -4,7 +4,6 @@
#include "moses/FactorCollection.h"
#include "moses/TargetPhraseCollection.h"
#include "moses/InputFileStream.h"
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
#include "probingpt/querying.h"
#include "probingpt/probing_hash_utils.h"

32
moses2/FF/SkeletonStatefulFF.cpp → moses2/FF/ExampleStatefulFF.cpp
View File

@ -1,11 +1,11 @@
/*
* SkeletonStatefulFF.cpp
* ExampleStatefulFF.cpp
*
* Created on: 27 Oct 2015
* Author: hieu
*/
#include <sstream>
#include "SkeletonStatefulFF.h"
#include "ExampleStatefulFF.h"
#include "../PhraseBased/Manager.h"
#include "../PhraseBased/Hypothesis.h"
@ -14,12 +14,12 @@ using namespace std;
namespace Moses2
{
class SkeletonState: public FFState
class ExampleState: public FFState
{
public:
int targetLen;
SkeletonState() {
ExampleState() {
// uninitialised
}
@ -27,7 +27,7 @@ public:
return (size_t) targetLen;
}
virtual bool operator==(const FFState& o) const {
const SkeletonState& other = static_cast<const SkeletonState&>(o);
const ExampleState& other = static_cast<const ExampleState&>(o);
return targetLen == other.targetLen;
}
@ -40,52 +40,52 @@ public:
};
////////////////////////////////////////////////////////////////////////////////////////
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line) :
ExampleStatefulFF::ExampleStatefulFF(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
{
ReadParameters();
}
SkeletonStatefulFF::~SkeletonStatefulFF()
ExampleStatefulFF::~ExampleStatefulFF()
{
// TODO Auto-generated destructor stub
}
FFState* SkeletonStatefulFF::BlankState(MemPool &pool, const System &sys) const
FFState* ExampleStatefulFF::BlankState(MemPool &pool, const System &sys) const
{
return new (pool.Allocate<SkeletonState>()) SkeletonState();
return new (pool.Allocate<ExampleState>()) ExampleState();
}
void SkeletonStatefulFF::EmptyHypothesisState(FFState &state,
void ExampleStatefulFF::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr, const InputType &input,
const Hypothesis &hypo) const
{
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
ExampleState &stateCast = static_cast<ExampleState&>(state);
stateCast.targetLen = 0;
}
void SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool,
void ExampleStatefulFF::EvaluateInIsolation(MemPool &pool,
const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
void ExampleStatefulFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void SkeletonStatefulFF::EvaluateWhenApplied(const ManagerBase &mgr,
void ExampleStatefulFF::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
ExampleState &stateCast = static_cast<ExampleState&>(state);
stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
}
void SkeletonStatefulFF::EvaluateWhenApplied(const SCFG::Manager &mgr,
void ExampleStatefulFF::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{

12
moses2/FF/SkeletonStatefulFF.h → moses2/FF/ExampleStatefulFF.h
View File

@ -1,23 +1,22 @@
/*
* SkeletonStatefulFF.h
* ExampleStatefulFF.h
*
* Created on: 27 Oct 2015
* Author: hieu
*/
#ifndef SKELETONSTATEFULFF_H_
#define SKELETONSTATEFULFF_H_
#pragma once
#include "StatefulFeatureFunction.h"
namespace Moses2
{
class SkeletonStatefulFF: public StatefulFeatureFunction
class ExampleStatefulFF: public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatefulFF();
ExampleStatefulFF(size_t startInd, const std::string &line);
virtual ~ExampleStatefulFF();
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
@ -45,4 +44,3 @@ public:
}
#endif /* SKELETONSTATEFULFF_H_ */

12
moses2/FF/SkeletonStatelessFF.cpp → moses2/FF/ExampleStatelessFF.cpp
View File

@ -6,31 +6,31 @@
*/
#include "../Scores.h"
#include "SkeletonStatelessFF.h"
#include "ExampleStatelessFF.h"
namespace Moses2
{
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd,
const std::string &line) :
ExampleStatelessFF::ExampleStatelessFF(size_t startInd,
const std::string &line) :
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
}
SkeletonStatelessFF::~SkeletonStatelessFF()
ExampleStatelessFF::~ExampleStatelessFF()
{
// TODO Auto-generated destructor stub
}
void SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool,
void ExampleStatelessFF::EvaluateInIsolation(MemPool &pool,
const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
void ExampleStatelessFF::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{

6
moses2/FF/SkeletonStatelessFF.h → moses2/FF/ExampleStatelessFF.h
View File

@ -12,11 +12,11 @@
namespace Moses2
{
class SkeletonStatelessFF: public StatelessFeatureFunction
class ExampleStatelessFF: public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatelessFF();
ExampleStatelessFF(size_t startInd, const std::string &line);
virtual ~ExampleStatelessFF();
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,

8
moses2/FF/FeatureRegistry.cpp
View File

@ -16,8 +16,8 @@
#include "WordPenalty.h"
#include "OSM/OpSequenceModel.h"
#include "SkeletonStatefulFF.h"
#include "SkeletonStatelessFF.h"
#include "ExampleStatefulFF.h"
#include "ExampleStatelessFF.h"
using namespace std;
@ -70,8 +70,8 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(WordPenalty);
MOSES_FNAME(OpSequenceModel);
MOSES_FNAME(SkeletonStatefulFF);
MOSES_FNAME(SkeletonStatelessFF);
MOSES_FNAME(ExampleStatefulFF);
MOSES_FNAME(ExampleStatelessFF);
}
FeatureRegistry::~FeatureRegistry()

4
moses2/Jamfile
View File

@ -47,8 +47,8 @@ alias deps : ..//z ..//boost_iostreams ..//boost_filesystem : : : $(max-factors
FF/FeatureFunctions.cpp
FF/FeatureRegistry.cpp
FF/PhrasePenalty.cpp
FF/SkeletonStatefulFF.cpp
FF/SkeletonStatelessFF.cpp
FF/ExampleStatefulFF.cpp
FF/ExampleStatelessFF.cpp
FF/StatefulFeatureFunction.cpp
FF/StatelessFeatureFunction.cpp
FF/WordPenalty.cpp

58
scripts/ems/experiment.meta
View File

@ -969,21 +969,6 @@ parse-input-devtest
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
ignore-unless: use-mira
template: $input-parser < IN > OUT
parse-relax-input
in: split-input
out: input
default-name: tuning/input.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parse-relaxer < IN > OUT
parse-relax-input-devtest
in: split-input-devtest
out: input-devtest
default-name: tuning/input.devtest.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
ignore-unless: use-mira
template: $input-parse-relaxer < IN > OUT
factorize-input
in: parsed-input
out: factorized-input
@ -1059,6 +1044,21 @@ split-input-devtest
pass-unless: input-splitter
ignore-unless: use-mira
template: $input-splitter -model IN1.$input-extension < IN > OUT
parse-relax-input
in: split-input
out: input
default-name: tuning/input.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parse-relaxer < IN > OUT
parse-relax-input-devtest
in: split-input-devtest
out: input-devtest
default-name: tuning/input.devtest.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
ignore-unless: use-mira
template: $input-parse-relaxer < IN > OUT
reference-from-sgm
in: reference-sgm input-sgm
out: raw-reference
@ -1252,20 +1252,6 @@ mock-parse-input
default-name: evaluation/input.mock-parsed
pass-unless: mock-input-parser-devtesteval
template: $mock-input-parser-devtesteval < IN > OUT
parse-input
in: mock-parsed-input
out: parsed-input
default-name: evaluation/input.parsed
pass-unless: input-parser
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parser < IN > OUT
parse-relax-input
in: split-input
out: input
default-name: evaluation/input.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parse-relaxer < IN > OUT
factorize-input
in: parsed-input
out: factorized-input
@ -1303,6 +1289,20 @@ split-input
default-name: evaluation/input.split
pass-unless: input-splitter
template: $input-splitter -model IN1.$input-extension < IN > OUT
parse-input
in: mock-parsed-input
out: parsed-input
default-name: evaluation/input.parsed
pass-unless: input-parser
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parser < IN > OUT
parse-relax-input
in: split-input
out: input
default-name: evaluation/input.parse-relaxed
pass-unless: input-parse-relaxer
pass-if: skip-parse-input-devtesteval mock-input-parser-devtesteval
template: $input-parse-relaxer < IN > OUT
filter
in: input TRAINING:sigtest-filter-phrase-translation-table TRAINING:sigtest-filter-reordering-table TRAINING:corpus-mml-prefilter=OR=TRAINING:corpus-mml-postfilter=OR=TRAINING:domains TRAINING:transliteration-table
out: filtered-dir

5
scripts/training/rdlm/train_rdlm.py
View File

@ -102,6 +102,11 @@ parser.add_argument(
parser.add_argument(
"--mmap", dest="mmap", action="store_true",
help="Use memory-mapped file (for lower memory consumption).")
parser.add_argument(
"--train-host", dest="train_host",
help="Execute nplm training on this host, via ssh")
parser.add_argument("--extra-settings", dest="extra_settings",
help="Extra settings to be passed to NPLM")
parser.set_defaults(