diff --git a/contrib/other-builds/OnDiskPt/.cproject b/contrib/other-builds/OnDiskPt/.cproject
index f551380fd..e32a5baea 100644
--- a/contrib/other-builds/OnDiskPt/.cproject
+++ b/contrib/other-builds/OnDiskPt/.cproject
@@ -11,12 +11,12 @@
-
-
+
+
@@ -72,13 +72,13 @@
-
-
+
+
diff --git a/contrib/other-builds/moses-chart-cmd/.cproject b/contrib/other-builds/moses-chart-cmd/.cproject
index 811c61c58..d03866163 100644
--- a/contrib/other-builds/moses-chart-cmd/.cproject
+++ b/contrib/other-builds/moses-chart-cmd/.cproject
@@ -33,7 +33,6 @@
diff --git a/contrib/other-builds/moses/.project b/contrib/other-builds/moses/.project
index 70fe2c0be..0b65a40e6 100644
--- a/contrib/other-builds/moses/.project
+++ b/contrib/other-builds/moses/.project
@@ -1156,6 +1156,16 @@
1
PARENT-3-PROJECT_LOC/moses/FF/InputFeature.h
+
+ FF/InternalStructStatelessFF.cpp
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/InternalStructStatelessFF.cpp
+
+
+ FF/InternalStructStatelessFF.h
+ 1
+ PARENT-3-PROJECT_LOC/moses/FF/InternalStructStatelessFF.h
+
FF/LexicalReordering
2
diff --git a/moses/FF/BleuScoreFeature.cpp b/moses/FF/BleuScoreFeature.cpp
index 452555855..71620410d 100644
--- a/moses/FF/BleuScoreFeature.cpp
+++ b/moses/FF/BleuScoreFeature.cpp
@@ -76,7 +76,7 @@ void BleuScoreState::AddNgramCountAndMatches(std::vector< size_t >& counts,
BleuScoreFeature::BleuScoreFeature(const std::string &line)
- :StatefulFeatureFunction("BleuScoreFeature",1, line),
+ :StatefulFeatureFunction(1, line),
m_enabled(true),
m_sentence_bleu(true),
m_simple_history_bleu(false),
diff --git a/moses/FF/ConstrainedDecoding.h b/moses/FF/ConstrainedDecoding.h
index 6d382abc9..f7228a86d 100644
--- a/moses/FF/ConstrainedDecoding.h
+++ b/moses/FF/ConstrainedDecoding.h
@@ -34,7 +34,7 @@ class ConstrainedDecoding : public StatefulFeatureFunction
{
public:
ConstrainedDecoding(const std::string &line)
- :StatefulFeatureFunction("ConstrainedDecoding", 1, line)
+ :StatefulFeatureFunction(1, line)
,m_maxUnknowns(0) {
m_tuneable = false;
ReadParameters();
diff --git a/moses/FF/ControlRecombination.h b/moses/FF/ControlRecombination.h
index 2ac4aa8e3..3d1d0d987 100644
--- a/moses/FF/ControlRecombination.h
+++ b/moses/FF/ControlRecombination.h
@@ -45,7 +45,7 @@ class ControlRecombination : public StatefulFeatureFunction
{
public:
ControlRecombination(const std::string &line)
- :StatefulFeatureFunction("ControlRecombination", 0, line)
+ :StatefulFeatureFunction(0, line)
,m_type(SameOutput)
{
diff --git a/moses/FF/DecodeFeature.cpp b/moses/FF/DecodeFeature.cpp
index e4e7d9960..7a8745066 100644
--- a/moses/FF/DecodeFeature.cpp
+++ b/moses/FF/DecodeFeature.cpp
@@ -29,27 +29,24 @@ using namespace std;
namespace Moses
{
-DecodeFeature::DecodeFeature( const std::string& description
- , const std::string &line)
- : StatelessFeatureFunction(description, line)
+DecodeFeature::DecodeFeature(const std::string &line)
+ : StatelessFeatureFunction(line)
{
VERBOSE(2,"DecodeFeature:" << std::endl);
}
-DecodeFeature::DecodeFeature( const std::string& description
- , size_t numScoreComponents
+DecodeFeature::DecodeFeature(size_t numScoreComponents
, const std::string &line)
- : StatelessFeatureFunction(description,numScoreComponents, line)
+ : StatelessFeatureFunction(numScoreComponents, line)
{
VERBOSE(2,"DecodeFeature: no factors yet" << std::endl);
}
-DecodeFeature::DecodeFeature(const std::string& description
- , size_t numScoreComponents
+DecodeFeature::DecodeFeature(size_t numScoreComponents
, const std::vector &input
, const std::vector &output
, const std::string &line)
- : StatelessFeatureFunction(description,numScoreComponents, line)
+ : StatelessFeatureFunction(numScoreComponents, line)
, m_input(input), m_output(output)
{
m_inputFactors = FactorMask(input);
diff --git a/moses/FF/DecodeFeature.h b/moses/FF/DecodeFeature.h
index 71fffecfa..920a0240e 100644
--- a/moses/FF/DecodeFeature.h
+++ b/moses/FF/DecodeFeature.h
@@ -38,15 +38,12 @@ class DecodeFeature : public StatelessFeatureFunction
{
public:
- DecodeFeature( const std::string& description
+ DecodeFeature(const std::string &line);
+
+ DecodeFeature(size_t numScoreComponents
, const std::string &line);
- DecodeFeature( const std::string& description
- , size_t numScoreComponents
- , const std::string &line);
-
- DecodeFeature( const std::string& description
- , size_t numScoreComponents
+ DecodeFeature(size_t numScoreComponents
, const std::vector &input
, const std::vector &output
, const std::string &line);
diff --git a/moses/FF/DistortionScoreProducer.cpp b/moses/FF/DistortionScoreProducer.cpp
index 54fb8ae95..303f35236 100644
--- a/moses/FF/DistortionScoreProducer.cpp
+++ b/moses/FF/DistortionScoreProducer.cpp
@@ -23,7 +23,7 @@ struct DistortionState_traditional : public FFState {
};
DistortionScoreProducer::DistortionScoreProducer(const std::string &line)
- : StatefulFeatureFunction("Distortion", 1, line)
+ : StatefulFeatureFunction(1, line)
{
ReadParameters();
}
diff --git a/moses/FF/ExternalFeature.h b/moses/FF/ExternalFeature.h
index 04414f0a8..f199ca99f 100644
--- a/moses/FF/ExternalFeature.h
+++ b/moses/FF/ExternalFeature.h
@@ -37,7 +37,7 @@ class ExternalFeature : public StatefulFeatureFunction
{
public:
ExternalFeature(const std::string &line)
- :StatefulFeatureFunction("ExternalFeature", line) {
+ :StatefulFeatureFunction(line) {
ReadParameters();
}
~ExternalFeature();
diff --git a/moses/FF/Factory.cpp b/moses/FF/Factory.cpp
index 9a9bcdb82..78e703304 100644
--- a/moses/FF/Factory.cpp
+++ b/moses/FF/Factory.cpp
@@ -4,7 +4,6 @@
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
#include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h"
#include "moses/TranslationModel/PhraseDictionaryMemory.h"
-#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h"
#include "moses/TranslationModel/PhraseDictionaryMultiModel.h"
#include "moses/TranslationModel/PhraseDictionaryMultiModelCounts.h"
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
@@ -38,6 +37,9 @@
#include "moses/LM/SkeletonLM.h"
#include "moses/TranslationModel/SkeletonPT.h"
+#ifdef HAVE_CMPH
+#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h"
+#endif
#ifdef PT_UG
#include "moses/TranslationModel/mmsapt.h"
#endif
@@ -144,7 +146,6 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME2("PhraseDictionaryBinary", PhraseDictionaryTreeAdaptor);
MOSES_FNAME(PhraseDictionaryOnDisk);
MOSES_FNAME(PhraseDictionaryMemory);
- MOSES_FNAME(PhraseDictionaryCompact);
MOSES_FNAME(PhraseDictionaryMultiModel);
MOSES_FNAME(PhraseDictionaryMultiModelCounts);
MOSES_FNAME(PhraseDictionaryALSuffixArray);
@@ -161,6 +162,9 @@ FeatureRegistry::FeatureRegistry()
MOSES_FNAME(SkeletonLM);
MOSES_FNAME(SkeletonPT);
+#ifdef HAVE_CMPH
+ MOSES_FNAME(PhraseDictionaryCompact);
+#endif
#ifdef PT_UG
MOSES_FNAME(Mmsapt);
#endif
@@ -183,7 +187,9 @@ FeatureRegistry::FeatureRegistry()
Add("KENLM", new KenFactory());
}
-FeatureRegistry::~FeatureRegistry() {}
+FeatureRegistry::~FeatureRegistry()
+{
+}
void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory)
{
@@ -203,4 +209,15 @@ void FeatureRegistry::Construct(const std::string &name, const std::string &line
i->second->Create(line);
}
+void FeatureRegistry::PrintFF() const
+{
+ std::cerr << "Available feature functions:" << std::endl;
+ Map::const_iterator iter;
+ for (iter = registry_.begin(); iter != registry_.end(); ++iter) {
+ const string &ffName = iter->first;
+ std::cerr << ffName << std::endl;
+ }
+
+}
+
} // namespace Moses
diff --git a/moses/FF/Factory.h b/moses/FF/Factory.h
index 4fc93d061..1c3867c09 100644
--- a/moses/FF/Factory.h
+++ b/moses/FF/Factory.h
@@ -18,6 +18,7 @@ public:
~FeatureRegistry();
void Construct(const std::string &name, const std::string &line);
+ void PrintFF() const;
private:
void Add(const std::string &name, FeatureFactory *factory);
diff --git a/moses/FF/FeatureFunction.cpp b/moses/FF/FeatureFunction.cpp
index 14a7527cf..8dede9866 100644
--- a/moses/FF/FeatureFunction.cpp
+++ b/moses/FF/FeatureFunction.cpp
@@ -30,40 +30,27 @@ FeatureFunction &FeatureFunction::FindFeatureFunction(const std::string& name)
}
FeatureFunction::
-FeatureFunction(const std::string& description,
- const std::string& line)
+FeatureFunction(const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(1)
{
- Initialize(description, line);
+ Initialize(line);
}
FeatureFunction::
-FeatureFunction(const std::string& description,
- size_t numScoreComponents,
+FeatureFunction(size_t numScoreComponents,
const std::string& line)
: m_tuneable(true)
, m_numScoreComponents(numScoreComponents)
{
- Initialize(description, line);
+ Initialize(line);
}
void
FeatureFunction::
-Initialize(const std::string& description, const std::string &line)
+Initialize(const std::string &line)
{
- ParseLine(description, line);
-
- if (m_description == "") {
- size_t index = description_counts.count(description);
-
- ostringstream dstream;
- dstream << description;
- dstream << index;
-
- description_counts.insert(description);
- m_description = dstream.str();
- }
+ ParseLine(line);
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
@@ -71,11 +58,13 @@ Initialize(const std::string& description, const std::string &line)
FeatureFunction::~FeatureFunction() {}
-void FeatureFunction::ParseLine(const std::string& description, const std::string &line)
+void FeatureFunction::ParseLine(const std::string &line)
{
vector toks = Tokenize(line);
CHECK(toks.size());
+ string nameStub = toks[0];
+
set keys;
for (size_t i = 1; i < toks.size(); ++i) {
@@ -93,6 +82,19 @@ void FeatureFunction::ParseLine(const std::string& description, const std::strin
m_args.push_back(args);
}
}
+
+ // name
+ if (m_description == "") {
+ size_t index = description_counts.count(nameStub);
+
+ ostringstream dstream;
+ dstream << nameStub;
+ dstream << index;
+
+ description_counts.insert(nameStub);
+ m_description = dstream.str();
+ }
+
}
void FeatureFunction::SetParameter(const std::string& key, const std::string& value)
diff --git a/moses/FF/FeatureFunction.h b/moses/FF/FeatureFunction.h
index 0eec06768..98a637223 100644
--- a/moses/FF/FeatureFunction.h
+++ b/moses/FF/FeatureFunction.h
@@ -36,8 +36,8 @@ protected:
//In case there's multiple producers with the same description
static std::multiset description_counts;
- void Initialize(const std::string& description, const std::string &line);
- void ParseLine(const std::string& description, const std::string &line);
+ void Initialize(const std::string &line);
+ void ParseLine(const std::string &line);
public:
static const std::vector& GetFeatureFunctions() {
@@ -45,8 +45,8 @@ public:
}
static FeatureFunction &FindFeatureFunction(const std::string& name);
- FeatureFunction(const std::string& description, const std::string &line);
- FeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
+ FeatureFunction(const std::string &line);
+ FeatureFunction(size_t numScoreComponents, const std::string &line);
virtual bool IsStateless() const = 0;
virtual ~FeatureFunction();
diff --git a/moses/FF/GlobalLexicalModel.cpp b/moses/FF/GlobalLexicalModel.cpp
index d4ce6c875..acffc76ee 100644
--- a/moses/FF/GlobalLexicalModel.cpp
+++ b/moses/FF/GlobalLexicalModel.cpp
@@ -11,7 +11,7 @@ using namespace std;
namespace Moses
{
GlobalLexicalModel::GlobalLexicalModel(const std::string &line)
- : StatelessFeatureFunction("GlobalLexicalModel",1, line)
+ : StatelessFeatureFunction(1, line)
{
std::cerr << "Creating global lexical model...\n";
ReadParameters();
diff --git a/moses/FF/GlobalLexicalModelUnlimited.cpp b/moses/FF/GlobalLexicalModelUnlimited.cpp
index f1f4b693b..ffa6c08c4 100644
--- a/moses/FF/GlobalLexicalModelUnlimited.cpp
+++ b/moses/FF/GlobalLexicalModelUnlimited.cpp
@@ -11,7 +11,7 @@ using namespace std;
namespace Moses
{
GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line)
- :StatelessFeatureFunction("GlobalLexicalModelUnlimited", 0, line)
+ :StatelessFeatureFunction(0, line)
{
CHECK(false); // TODO need to update arguments to key=value
diff --git a/moses/FF/InputFeature.cpp b/moses/FF/InputFeature.cpp
index 14efa12f0..a6ba048e5 100644
--- a/moses/FF/InputFeature.cpp
+++ b/moses/FF/InputFeature.cpp
@@ -12,7 +12,7 @@ using namespace std;
namespace Moses
{
InputFeature::InputFeature(const std::string &line)
- :StatelessFeatureFunction("InputFeature", line)
+ :StatelessFeatureFunction(line)
{
ReadParameters();
}
diff --git a/moses/FF/InternalStructStatelessFF.h b/moses/FF/InternalStructStatelessFF.h
index e97473fe1..6802479ce 100644
--- a/moses/FF/InternalStructStatelessFF.h
+++ b/moses/FF/InternalStructStatelessFF.h
@@ -10,7 +10,7 @@ class InternalStructStatelessFF : public StatelessFeatureFunction
{
public:
InternalStructStatelessFF(const std::string &line)
- :StatelessFeatureFunction("InternalStructStatelessFF", line)
+ :StatelessFeatureFunction(line)
{}
bool IsUseable(const FactorMask &mask) const
diff --git a/moses/FF/LexicalReordering/LexicalReordering.cpp b/moses/FF/LexicalReordering/LexicalReordering.cpp
index 2486714a5..c73c0324b 100644
--- a/moses/FF/LexicalReordering/LexicalReordering.cpp
+++ b/moses/FF/LexicalReordering/LexicalReordering.cpp
@@ -10,7 +10,7 @@ using namespace std;
namespace Moses
{
LexicalReordering::LexicalReordering(const std::string &line)
- : StatefulFeatureFunction("LexicalReordering", line)
+ : StatefulFeatureFunction(line)
{
std::cerr << "Initializing LexicalReordering.." << std::endl;
diff --git a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp
index c33851899..27396aa5a 100644
--- a/moses/FF/LexicalReordering/LexicalReorderingTable.cpp
+++ b/moses/FF/LexicalReordering/LexicalReorderingTable.cpp
@@ -8,7 +8,7 @@
#include "moses/TargetPhrase.h"
#include "moses/TargetPhraseCollection.h"
-#ifndef WIN32
+#ifdef HAVE_CMPH
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
#endif
@@ -52,8 +52,10 @@ void auxAppend(IPhrase& head, const IPhrase& tail)
LexicalReorderingTable* LexicalReorderingTable::LoadAvailable(const std::string& filePath, const FactorList& f_factors, const FactorList& e_factors, const FactorList& c_factors)
{
//decide use Compact or Tree or Memory table
- LexicalReorderingTable *compactLexr =
- LexicalReorderingTableCompact::CheckAndLoad(filePath + ".minlexr", f_factors, e_factors, c_factors);
+ LexicalReorderingTable *compactLexr = NULL;
+#ifdef HAVE_CMPH
+ compactLexr = LexicalReorderingTableCompact::CheckAndLoad(filePath + ".minlexr", f_factors, e_factors, c_factors);
+#endif
if(compactLexr)
return compactLexr;
if(FileExists(filePath+".binlexr.idx")) {
diff --git a/moses/FF/OSM-Feature/OpSequenceModel.cpp b/moses/FF/OSM-Feature/OpSequenceModel.cpp
index e35e31a33..553196270 100644
--- a/moses/FF/OSM-Feature/OpSequenceModel.cpp
+++ b/moses/FF/OSM-Feature/OpSequenceModel.cpp
@@ -11,7 +11,7 @@ namespace Moses
{
OpSequenceModel::OpSequenceModel(const std::string &line)
- :StatefulFeatureFunction("OpSequenceModel", 5, line )
+ :StatefulFeatureFunction(5, line )
{
sFactor = 0;
tFactor = 0;
diff --git a/moses/FF/PhraseBoundaryFeature.cpp b/moses/FF/PhraseBoundaryFeature.cpp
index a398fdc6a..d82181b76 100644
--- a/moses/FF/PhraseBoundaryFeature.cpp
+++ b/moses/FF/PhraseBoundaryFeature.cpp
@@ -18,7 +18,7 @@ int PhraseBoundaryState::Compare(const FFState& other) const
}
PhraseBoundaryFeature::PhraseBoundaryFeature(const std::string &line)
- : StatefulFeatureFunction("PhraseBoundaryFeature", 0, line)
+ : StatefulFeatureFunction(0, line)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;
ReadParameters();
diff --git a/moses/FF/PhraseLengthFeature.cpp b/moses/FF/PhraseLengthFeature.cpp
index fb69a2de3..43e0d1b2d 100644
--- a/moses/FF/PhraseLengthFeature.cpp
+++ b/moses/FF/PhraseLengthFeature.cpp
@@ -10,7 +10,7 @@ namespace Moses
using namespace std;
PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
- :StatelessFeatureFunction("PhraseLengthFeature", 0, line)
+ :StatelessFeatureFunction(0, line)
{
ReadParameters();
}
diff --git a/moses/FF/PhrasePairFeature.cpp b/moses/FF/PhrasePairFeature.cpp
index 564d131ba..e122bf6ab 100644
--- a/moses/FF/PhrasePairFeature.cpp
+++ b/moses/FF/PhrasePairFeature.cpp
@@ -15,7 +15,7 @@ namespace Moses
{
PhrasePairFeature::PhrasePairFeature(const std::string &line)
- :StatelessFeatureFunction("PhrasePairFeature", 0, line)
+ :StatelessFeatureFunction(0, line)
{
std::cerr << "Initializing PhrasePairFeature.." << std::endl;
ReadParameters();
diff --git a/moses/FF/PhrasePenalty.cpp b/moses/FF/PhrasePenalty.cpp
index 8b54939a7..b3e493707 100644
--- a/moses/FF/PhrasePenalty.cpp
+++ b/moses/FF/PhrasePenalty.cpp
@@ -5,7 +5,7 @@
namespace Moses
{
PhrasePenalty::PhrasePenalty(const std::string &line)
- : StatelessFeatureFunction("PhrasePenalty",1, line)
+ : StatelessFeatureFunction(1, line)
{
ReadParameters();
}
diff --git a/moses/FF/SkeletonStatefulFF.h b/moses/FF/SkeletonStatefulFF.h
index c3bb10e37..5b0ca0f95 100644
--- a/moses/FF/SkeletonStatefulFF.h
+++ b/moses/FF/SkeletonStatefulFF.h
@@ -22,7 +22,7 @@ class SkeletonStatefulFF : public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(const std::string &line)
- :StatefulFeatureFunction("SkeletonStatefulFF", 3, line)
+ :StatefulFeatureFunction(3, line)
{}
bool IsUseable(const FactorMask &mask) const {
diff --git a/moses/FF/SkeletonStatelessFF.h b/moses/FF/SkeletonStatelessFF.h
index f3c494a94..d17f324aa 100644
--- a/moses/FF/SkeletonStatelessFF.h
+++ b/moses/FF/SkeletonStatelessFF.h
@@ -10,7 +10,7 @@ class SkeletonStatelessFF : public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(const std::string &line)
- :StatelessFeatureFunction("SkeletonStatelessFF", 2, line)
+ :StatelessFeatureFunction(2, line)
{}
bool IsUseable(const FactorMask &mask) const {
diff --git a/moses/FF/SourceWordDeletionFeature.cpp b/moses/FF/SourceWordDeletionFeature.cpp
index b920cc619..12918a9e9 100644
--- a/moses/FF/SourceWordDeletionFeature.cpp
+++ b/moses/FF/SourceWordDeletionFeature.cpp
@@ -18,7 +18,7 @@ namespace Moses
using namespace std;
SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line)
- :StatelessFeatureFunction("SourceWordDeletionFeature", 0, line),
+ :StatelessFeatureFunction(0, line),
m_unrestricted(true)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;
diff --git a/moses/FF/StatefulFeatureFunction.cpp b/moses/FF/StatefulFeatureFunction.cpp
index 37b9c2dee..9e61ed05f 100644
--- a/moses/FF/StatefulFeatureFunction.cpp
+++ b/moses/FF/StatefulFeatureFunction.cpp
@@ -5,14 +5,14 @@ namespace Moses
std::vector StatefulFeatureFunction::m_statefulFFs;
-StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, const std::string &line)
- : FeatureFunction(description, line)
+StatefulFeatureFunction::StatefulFeatureFunction(const std::string &line)
+ : FeatureFunction(line)
{
m_statefulFFs.push_back(this);
}
-StatefulFeatureFunction::StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
- : FeatureFunction(description,numScoreComponents, line)
+StatefulFeatureFunction::StatefulFeatureFunction(size_t numScoreComponents, const std::string &line)
+ : FeatureFunction(numScoreComponents, line)
{
m_statefulFFs.push_back(this);
}
diff --git a/moses/FF/StatefulFeatureFunction.h b/moses/FF/StatefulFeatureFunction.h
index e99355ba2..75b46d827 100644
--- a/moses/FF/StatefulFeatureFunction.h
+++ b/moses/FF/StatefulFeatureFunction.h
@@ -19,8 +19,8 @@ public:
return m_statefulFFs;
}
- StatefulFeatureFunction(const std::string& description, const std::string &line);
- StatefulFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
+ StatefulFeatureFunction(const std::string &line);
+ StatefulFeatureFunction(size_t numScoreComponents, const std::string &line);
/**
* \brief This interface should be implemented.
diff --git a/moses/FF/StatelessFeatureFunction.cpp b/moses/FF/StatelessFeatureFunction.cpp
index 51b98064e..ecad23e6f 100644
--- a/moses/FF/StatelessFeatureFunction.cpp
+++ b/moses/FF/StatelessFeatureFunction.cpp
@@ -5,14 +5,14 @@ namespace Moses
std::vector StatelessFeatureFunction::m_statelessFFs;
-StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, const std::string &line)
- :FeatureFunction(description, line)
+StatelessFeatureFunction::StatelessFeatureFunction(const std::string &line)
+ :FeatureFunction(line)
{
m_statelessFFs.push_back(this);
}
-StatelessFeatureFunction::StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line)
- :FeatureFunction(description, numScoreComponents, line)
+StatelessFeatureFunction::StatelessFeatureFunction(size_t numScoreComponents, const std::string &line)
+ :FeatureFunction(numScoreComponents, line)
{
m_statelessFFs.push_back(this);
}
diff --git a/moses/FF/StatelessFeatureFunction.h b/moses/FF/StatelessFeatureFunction.h
index e2368cf58..fde740115 100644
--- a/moses/FF/StatelessFeatureFunction.h
+++ b/moses/FF/StatelessFeatureFunction.h
@@ -18,8 +18,8 @@ public:
return m_statelessFFs;
}
- StatelessFeatureFunction(const std::string& description, const std::string &line);
- StatelessFeatureFunction(const std::string& description, size_t numScoreComponents, const std::string &line);
+ StatelessFeatureFunction(const std::string &line);
+ StatelessFeatureFunction(size_t numScoreComponents, const std::string &line);
/**
* This should be implemented for features that apply to phrase-based models.
**/
diff --git a/moses/FF/TargetBigramFeature.cpp b/moses/FF/TargetBigramFeature.cpp
index 083e16ac0..337e47050 100644
--- a/moses/FF/TargetBigramFeature.cpp
+++ b/moses/FF/TargetBigramFeature.cpp
@@ -18,7 +18,7 @@ int TargetBigramState::Compare(const FFState& other) const
}
TargetBigramFeature::TargetBigramFeature(const std::string &line)
- :StatefulFeatureFunction("TargetBigramFeature", 0, line)
+ :StatefulFeatureFunction(0, line)
{
std::cerr << "Initializing target bigram feature.." << std::endl;
ReadParameters();
diff --git a/moses/FF/TargetNgramFeature.cpp b/moses/FF/TargetNgramFeature.cpp
index 8dc934ffd..a14082266 100644
--- a/moses/FF/TargetNgramFeature.cpp
+++ b/moses/FF/TargetNgramFeature.cpp
@@ -38,7 +38,7 @@ int TargetNgramState::Compare(const FFState& other) const
}
TargetNgramFeature::TargetNgramFeature(const std::string &line)
- :StatefulFeatureFunction("TargetNgramFeature", 0, line)
+ :StatefulFeatureFunction(0, line)
{
std::cerr << "Initializing target ngram feature.." << std::endl;
ReadParameters();
diff --git a/moses/FF/TargetWordInsertionFeature.cpp b/moses/FF/TargetWordInsertionFeature.cpp
index f59e75bc6..6011e9365 100644
--- a/moses/FF/TargetWordInsertionFeature.cpp
+++ b/moses/FF/TargetWordInsertionFeature.cpp
@@ -16,7 +16,7 @@ namespace Moses
using namespace std;
TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
- :StatelessFeatureFunction("TargetWordInsertionFeature", 0, line),
+ :StatelessFeatureFunction(0, line),
m_unrestricted(true)
{
std::cerr << "Initializing target word insertion feature.." << std::endl;
diff --git a/moses/FF/UnknownWordPenaltyProducer.cpp b/moses/FF/UnknownWordPenaltyProducer.cpp
index fcb91a430..edc05dd27 100644
--- a/moses/FF/UnknownWordPenaltyProducer.cpp
+++ b/moses/FF/UnknownWordPenaltyProducer.cpp
@@ -7,7 +7,7 @@ using namespace std;
namespace Moses
{
UnknownWordPenaltyProducer::UnknownWordPenaltyProducer(const std::string &line)
- : StatelessFeatureFunction("UnknownWordPenalty",1, line)
+ : StatelessFeatureFunction(1, line)
{
m_tuneable = false;
ReadParameters();
diff --git a/moses/FF/WordPenaltyProducer.cpp b/moses/FF/WordPenaltyProducer.cpp
index 8d77d68fd..ac11824f5 100644
--- a/moses/FF/WordPenaltyProducer.cpp
+++ b/moses/FF/WordPenaltyProducer.cpp
@@ -7,7 +7,7 @@ using namespace std;
namespace Moses
{
WordPenaltyProducer::WordPenaltyProducer(const std::string &line)
- : StatelessFeatureFunction("WordPenalty",1, line)
+ : StatelessFeatureFunction(1, line)
{
ReadParameters();
}
diff --git a/moses/FF/WordTranslationFeature.cpp b/moses/FF/WordTranslationFeature.cpp
index e45659cde..0a5936203 100644
--- a/moses/FF/WordTranslationFeature.cpp
+++ b/moses/FF/WordTranslationFeature.cpp
@@ -18,7 +18,7 @@ namespace Moses
{
WordTranslationFeature::WordTranslationFeature(const std::string &line)
- :StatelessFeatureFunction("WordTranslationFeature", 0, line)
+ :StatelessFeatureFunction(0, line)
,m_unrestricted(true)
,m_simple(true)
,m_sourceContext(false)
diff --git a/moses/GenerationDictionary.cpp b/moses/GenerationDictionary.cpp
index ee9dbcb06..efebd1ae2 100644
--- a/moses/GenerationDictionary.cpp
+++ b/moses/GenerationDictionary.cpp
@@ -36,7 +36,7 @@ namespace Moses
{
GenerationDictionary::GenerationDictionary(const std::string &line)
- : DecodeFeature("Generation", line)
+ : DecodeFeature(line)
{
ReadParameters();
}
diff --git a/moses/LM/Base.cpp b/moses/LM/Base.cpp
index b77d64fef..f59b5e31b 100644
--- a/moses/LM/Base.cpp
+++ b/moses/LM/Base.cpp
@@ -34,8 +34,8 @@ using namespace std;
namespace Moses
{
-LanguageModel::LanguageModel(const std::string& description, const std::string &line) :
- StatefulFeatureFunction(description, StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line )
+LanguageModel::LanguageModel(const std::string &line) :
+ StatefulFeatureFunction(StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1, line )
{
m_enableOOVFeature = StaticData::Instance().GetLMEnableOOVFeature();
}
diff --git a/moses/LM/Base.h b/moses/LM/Base.h
index 5c9616e46..fb24ba627 100644
--- a/moses/LM/Base.h
+++ b/moses/LM/Base.h
@@ -43,7 +43,7 @@ class Phrase;
class LanguageModel : public StatefulFeatureFunction
{
protected:
- LanguageModel(const std::string& description, const std::string &line);
+ LanguageModel(const std::string &line);
// This can't be in the constructor for virual function dispatch reasons
diff --git a/moses/LM/IRST.cpp b/moses/LM/IRST.cpp
index bde78e1b8..44c5d8c4c 100644
--- a/moses/LM/IRST.cpp
+++ b/moses/LM/IRST.cpp
@@ -40,7 +40,7 @@ using namespace std;
namespace Moses
{
LanguageModelIRST::LanguageModelIRST(const std::string &line)
- :LanguageModelSingleFactor("IRSTLM", line)
+ :LanguageModelSingleFactor(line)
{
const StaticData &staticData = StaticData::Instance();
int threadCount = staticData.ThreadCount();
diff --git a/moses/LM/Implementation.cpp b/moses/LM/Implementation.cpp
index 3f94fece8..267b94dd6 100644
--- a/moses/LM/Implementation.cpp
+++ b/moses/LM/Implementation.cpp
@@ -41,8 +41,8 @@ using namespace std;
namespace Moses
{
-LanguageModelImplementation::LanguageModelImplementation(const std::string& description, const std::string &line)
- :LanguageModel(description, line)
+LanguageModelImplementation::LanguageModelImplementation(const std::string &line)
+ :LanguageModel(line)
{
}
diff --git a/moses/LM/Implementation.h b/moses/LM/Implementation.h
index e6a230ad8..a39f5e42b 100644
--- a/moses/LM/Implementation.h
+++ b/moses/LM/Implementation.h
@@ -60,7 +60,7 @@ protected:
Word m_sentenceStartWord, m_sentenceEndWord; //! Contains factors which represents the beging and end words for this LM.
//! Usually and
- LanguageModelImplementation(const std::string& description, const std::string &line);
+ LanguageModelImplementation(const std::string &line);
public:
diff --git a/moses/LM/Joint.h b/moses/LM/Joint.h
index e94c29b4f..1dbdb019e 100644
--- a/moses/LM/Joint.h
+++ b/moses/LM/Joint.h
@@ -50,7 +50,7 @@ protected:
size_t m_implFactor;
public:
LanguageModelJoint(const std::string &line, LanguageModelSingleFactor *lmImpl)
- :LanguageModelMultiFactor("JointLM", line) {
+ :LanguageModelMultiFactor(line) {
m_lmImpl = lmImpl;
}
diff --git a/moses/LM/Ken.cpp b/moses/LM/Ken.cpp
index ed0278029..2bb6a16f9 100644
--- a/moses/LM/Ken.cpp
+++ b/moses/LM/Ken.cpp
@@ -140,7 +140,7 @@ private:
} // namespace
template LanguageModelKen::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, bool lazy)
- :LanguageModel("KENLM", line)
+ :LanguageModel(line)
,m_factorType(factorType)
{
lm::ngram::Config config;
@@ -161,7 +161,7 @@ template LanguageModelKen::LanguageModelKen(const std::stri
}
template LanguageModelKen::LanguageModelKen(const LanguageModelKen ©_from)
- :LanguageModel(copy_from.GetScoreProducerDescription(), copy_from.GetArgLine()),
+ :LanguageModel(copy_from.GetArgLine()),
m_ngram(copy_from.m_ngram),
// TODO: don't copy this.
m_lmIdLookup(copy_from.m_lmIdLookup),
diff --git a/moses/LM/MultiFactor.h b/moses/LM/MultiFactor.h
index 619ad94f3..df476f93f 100644
--- a/moses/LM/MultiFactor.h
+++ b/moses/LM/MultiFactor.h
@@ -41,8 +41,8 @@ class LanguageModelMultiFactor : public LanguageModelImplementation
protected:
FactorMask m_factorTypes;
- LanguageModelMultiFactor(const std::string& description, const std::string &line)
- :LanguageModelImplementation(description, line) {
+ LanguageModelMultiFactor(const std::string &line)
+ :LanguageModelImplementation(line) {
}
public:
diff --git a/moses/LM/NeuralLMWrapper.cpp b/moses/LM/NeuralLMWrapper.cpp
index 41ceded9d..467c41846 100644
--- a/moses/LM/NeuralLMWrapper.cpp
+++ b/moses/LM/NeuralLMWrapper.cpp
@@ -10,7 +10,7 @@ using namespace std;
namespace Moses
{
NeuralLMWrapper::NeuralLMWrapper(const std::string &line)
-:LanguageModelSingleFactor("NeuralLM", line)
+:LanguageModelSingleFactor(line)
{
// This space intentionally left blank
}
diff --git a/moses/LM/ORLM.h b/moses/LM/ORLM.h
index 4443b4732..dd2a675d8 100644
--- a/moses/LM/ORLM.h
+++ b/moses/LM/ORLM.h
@@ -22,7 +22,7 @@ class LanguageModelORLM : public LanguageModelSingleFactor
public:
typedef count_t T; // type for ORLM filter
LanguageModelORLM(const std::string &line)
- :LanguageModelSingleFactor("ORLM", line)
+ :LanguageModelSingleFactor(line)
,m_lm(0) {
}
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
diff --git a/moses/LM/ParallelBackoff.cpp b/moses/LM/ParallelBackoff.cpp
index 3d67e9c90..063e9c818 100644
--- a/moses/LM/ParallelBackoff.cpp
+++ b/moses/LM/ParallelBackoff.cpp
@@ -85,7 +85,7 @@ private:
public:
LanguageModelParallelBackoff(const std::string &line)
- :LanguageModelMultiFactor("ParallelBackoffLM", line) {
+ :LanguageModelMultiFactor(line) {
}
~LanguageModelParallelBackoff();
diff --git a/moses/LM/Rand.cpp b/moses/LM/Rand.cpp
index 0f11fea07..a3c53b483 100644
--- a/moses/LM/Rand.cpp
+++ b/moses/LM/Rand.cpp
@@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
// The following ifdef prevents XCode and other non-bjam build systems
// from attempting to compile this file when LM_RAND is disabled.
//
-#ifdef LM_RAND
#include
#include
@@ -44,7 +43,7 @@ namespace Moses
{
LanguageModelRandLM::LanguageModelRandLM(const std::string &line)
- :LanguageModelSingleFactor("RandLM", line)
+ :LanguageModelSingleFactor(line)
, m_lm(0)
{
}
@@ -147,4 +146,3 @@ void LanguageModelRandLM::CleanUpAfterSentenceProcessing(const InputType& source
}
-#endif
diff --git a/moses/LM/SRI.cpp b/moses/LM/SRI.cpp
index 6182bd22c..708374668 100644
--- a/moses/LM/SRI.cpp
+++ b/moses/LM/SRI.cpp
@@ -54,7 +54,7 @@ using namespace std;
namespace Moses
{
LanguageModelSRI::LanguageModelSRI(const std::string &line)
- :LanguageModelSingleFactor("SRILM", line)
+ :LanguageModelSingleFactor(line)
,m_srilmVocab(0)
,m_srilmModel(0)
{
diff --git a/moses/LM/SingleFactor.cpp b/moses/LM/SingleFactor.cpp
index 04f410c98..d2c0f990a 100644
--- a/moses/LM/SingleFactor.cpp
+++ b/moses/LM/SingleFactor.cpp
@@ -38,8 +38,8 @@ using namespace std;
namespace Moses
{
-LanguageModelSingleFactor::LanguageModelSingleFactor(const std::string& description, const std::string &line)
- :LanguageModelImplementation(description, line)
+LanguageModelSingleFactor::LanguageModelSingleFactor(const std::string &line)
+ :LanguageModelImplementation(line)
{
m_nullContextState = new PointerState(NULL);
m_beginSentenceState = new PointerState(NULL);
diff --git a/moses/LM/SingleFactor.h b/moses/LM/SingleFactor.h
index af0edd4c6..eeb5cdbef 100644
--- a/moses/LM/SingleFactor.h
+++ b/moses/LM/SingleFactor.h
@@ -43,7 +43,7 @@ protected:
FFState *m_nullContextState;
FFState *m_beginSentenceState;
- LanguageModelSingleFactor(const std::string& description, const std::string &line);
+ LanguageModelSingleFactor(const std::string &line);
public:
virtual ~LanguageModelSingleFactor();
diff --git a/moses/LM/SkeletonLM.cpp b/moses/LM/SkeletonLM.cpp
index 6bf47e108..6c11512fe 100644
--- a/moses/LM/SkeletonLM.cpp
+++ b/moses/LM/SkeletonLM.cpp
@@ -7,7 +7,7 @@ using namespace std;
namespace Moses
{
SkeletonLM::SkeletonLM(const std::string &line)
- :LanguageModelSingleFactor("SkeletonLM", line)
+ :LanguageModelSingleFactor(line)
{
ReadParameters();
diff --git a/moses/OutputCollector.h b/moses/OutputCollector.h
index 5f72433d8..22cfb46a6 100644
--- a/moses/OutputCollector.h
+++ b/moses/OutputCollector.h
@@ -62,7 +62,7 @@ public:
}
bool OutputIsCout() const {
- return (m_outStream == std::cout);
+ return (m_outStream == &std::cout);
}
/**
diff --git a/moses/Parameter.cpp b/moses/Parameter.cpp
index 902c5c610..3f34abeb9 100644
--- a/moses/Parameter.cpp
+++ b/moses/Parameter.cpp
@@ -270,6 +270,7 @@ bool Parameter::LoadParam(int argc, char* argv[])
&& (configPath = FindParam("-config", argc, argv)) == "") {
PrintCredit();
Explain();
+ PrintFF();
cerr << endl;
UserMessage::Add("No configuration file was specified. Use -config or -f");
@@ -1302,6 +1303,11 @@ void Parameter::OverwriteParam(const string ¶mName, PARAM_VEC values)
VERBOSE(2, std::endl);
}
+void Parameter::PrintFF() const
+{
+ StaticData::Instance().GetFeatureRegistry().PrintFF();
+}
+
std::set Parameter::GetWeightNames() const
{
std::set ret;
diff --git a/moses/Parameter.h b/moses/Parameter.h
index f5016fb2b..973a1f09b 100644
--- a/moses/Parameter.h
+++ b/moses/Parameter.h
@@ -62,6 +62,7 @@ protected:
void AddParam(const std::string ¶mName, const std::string &abbrevName, const std::string &description);
void PrintCredit();
+ void PrintFF() const;
void SetWeight(const std::string &name, size_t ind, float weight);
void SetWeight(const std::string &name, size_t ind, const std::vector &weights);
diff --git a/moses/ScoreComponentCollectionTest.cpp b/moses/ScoreComponentCollectionTest.cpp
index f44595aec..06f999e3d 100644
--- a/moses/ScoreComponentCollectionTest.cpp
+++ b/moses/ScoreComponentCollectionTest.cpp
@@ -32,8 +32,8 @@ BOOST_AUTO_TEST_SUITE(scc)
class MockStatelessFeatureFunction : public StatelessFeatureFunction
{
public:
- MockStatelessFeatureFunction(const string& desc, size_t n, const string &line) :
- StatelessFeatureFunction(desc,n, line) {}
+ MockStatelessFeatureFunction(size_t n, const string &line) :
+ StatelessFeatureFunction(n, line) {}
void Evaluate(const Hypothesis&, ScoreComponentCollection*) const {}
void EvaluateChart(const ChartHypothesis&, ScoreComponentCollection*) const {}
void Evaluate(const InputType &input
@@ -52,7 +52,7 @@ public:
class MockSingleFeature : public MockStatelessFeatureFunction
{
public:
- MockSingleFeature(): MockStatelessFeatureFunction("MockSingle",1, "MockSingle") {}
+ MockSingleFeature(): MockStatelessFeatureFunction(1, "MockSingle") {}
bool IsUseable(const FactorMask &mask) const {
return true;
@@ -62,7 +62,7 @@ public:
class MockMultiFeature : public MockStatelessFeatureFunction
{
public:
- MockMultiFeature(): MockStatelessFeatureFunction("MockMulti", 5, "MockMulti") {}
+ MockMultiFeature(): MockStatelessFeatureFunction(5, "MockMulti") {}
bool IsUseable(const FactorMask &mask) const {
return true;
@@ -73,7 +73,7 @@ public:
class MockSparseFeature : public MockStatelessFeatureFunction
{
public:
- MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", 0, "MockSparse") {}
+ MockSparseFeature(): MockStatelessFeatureFunction(0, "MockSparse") {}
bool IsUseable(const FactorMask &mask) const {
return true;
diff --git a/moses/StaticData.cpp b/moses/StaticData.cpp
index 4ca807888..1aad30254 100644
--- a/moses/StaticData.cpp
+++ b/moses/StaticData.cpp
@@ -24,7 +24,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "util/check.hh"
#include "TypeDef.h"
-#include "moses/FF/Factory.h"
#include "moses/FF/WordPenaltyProducer.h"
#include "moses/FF/UnknownWordPenaltyProducer.h"
#include "moses/FF/InputFeature.h"
@@ -510,7 +509,6 @@ bool StaticData::LoadData(Parameter *parameter)
map featureIndexMap;
const vector &features = m_parameter->GetParam("feature");
- FeatureRegistry registry;
for (size_t i = 0; i < features.size(); ++i) {
const string &line = Trim(features[i]);
cerr << "line=" << line << endl;
@@ -521,7 +519,7 @@ bool StaticData::LoadData(Parameter *parameter)
const string &feature = toks[0];
- registry.Construct(feature, line);
+ m_registry.Construct(feature, line);
}
OverrideFeatures();
diff --git a/moses/StaticData.h b/moses/StaticData.h
index 3e5f2921b..16136f849 100644
--- a/moses/StaticData.h
+++ b/moses/StaticData.h
@@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Parameter.h"
#include "SentenceStats.h"
#include "ScoreComponentCollection.h"
+#include "moses/FF/Factory.h"
namespace Moses
{
@@ -206,6 +207,8 @@ protected:
FactorType m_placeHolderFactor;
bool m_useLegacyPT;
+ FeatureRegistry m_registry;
+
StaticData();
void LoadChartDecodingParameters();
@@ -752,6 +755,9 @@ public:
return m_placeHolderFactor;
}
+ const FeatureRegistry &GetFeatureRegistry() const
+ { return m_registry; }
+
/** check whether we should be using the old code to support binary phrase-table.
** eventually, we'll stop support the binary phrase-table and delete this legacy code
**/
diff --git a/moses/TranslationModel/CompactPT/Jamfile b/moses/TranslationModel/CompactPT/Jamfile
index 452c0048e..f42d63f64 100644
--- a/moses/TranslationModel/CompactPT/Jamfile
+++ b/moses/TranslationModel/CompactPT/Jamfile
@@ -5,12 +5,13 @@ if $(with-cmph) {
lib cmph : : $(with-cmph)/lib $(with-cmph)/lib64 ;
includes += $(with-cmph)/include ;
current = "--with-cmph=$(with-cmph)" ;
+ fakelib CompactPT : [ glob *.cpp ] ../..//headers cmph : $(includes) $(PT-LOG) : : $(includes) ;
}
else {
alias cmph ;
+ fakelib CompactPT ;
}
path-constant PT-LOG : bin/pt.log ;
update-if-changed $(PT-LOG) $(current) ;
-fakelib CompactPT : [ glob *.cpp ] ../..//headers cmph : $(includes) $(PT-LOG) : : $(includes) ;
diff --git a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
index c60d0f944..876e334eb 100644
--- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
+++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
@@ -42,7 +42,7 @@ namespace Moses
{
PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line)
- :PhraseDictionary("PhraseDictionaryCompact", line)
+ :PhraseDictionary(line)
,m_inMemory(true)
,m_useAlignmentInfo(true)
,m_hash(10, 16)
diff --git a/moses/TranslationModel/DynSAInclude/vocab.cpp b/moses/TranslationModel/DynSAInclude/vocab.cpp
index da1443f66..dfb3aabff 100644
--- a/moses/TranslationModel/DynSAInclude/vocab.cpp
+++ b/moses/TranslationModel/DynSAInclude/vocab.cpp
@@ -118,7 +118,7 @@ bool Vocab::Load(FileHandler* vcbin, const FactorDirection& direction,
std::string line, word_str;
wordID_t id;
- void *ret = getline(*vcbin, line);
+ std::istream &ret = getline(*vcbin, line);
CHECK(ret);
std::istringstream first(line.c_str());
uint32_t vcbsize(0);
diff --git a/moses/TranslationModel/PhraseDictionary.cpp b/moses/TranslationModel/PhraseDictionary.cpp
index 6c7aeefc9..159e798a1 100644
--- a/moses/TranslationModel/PhraseDictionary.cpp
+++ b/moses/TranslationModel/PhraseDictionary.cpp
@@ -32,8 +32,8 @@ using namespace std;
namespace Moses
{
-PhraseDictionary::PhraseDictionary(const std::string &description, const std::string &line)
- :DecodeFeature(description, line)
+PhraseDictionary::PhraseDictionary(const std::string &line)
+ :DecodeFeature(line)
,m_tableLimit(20) // default
,m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE)
{
diff --git a/moses/TranslationModel/PhraseDictionary.h b/moses/TranslationModel/PhraseDictionary.h
index 671fe19b2..f8477f70e 100644
--- a/moses/TranslationModel/PhraseDictionary.h
+++ b/moses/TranslationModel/PhraseDictionary.h
@@ -59,7 +59,7 @@ class ChartParser;
class PhraseDictionary : public DecodeFeature
{
public:
- PhraseDictionary(const std::string &description, const std::string &line);
+ PhraseDictionary(const std::string &line);
virtual ~PhraseDictionary() {
}
diff --git a/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp b/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp
index f9022eeee..ac5a817bb 100644
--- a/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp
+++ b/moses/TranslationModel/PhraseDictionaryDynSuffixArray.cpp
@@ -10,7 +10,7 @@ namespace Moses
{
PhraseDictionaryDynSuffixArray::
PhraseDictionaryDynSuffixArray(const std::string &line)
- : PhraseDictionary("PhraseDictionaryDynSuffixArray", line)
+ : PhraseDictionary(line)
,m_biSA(new BilingualDynSuffixArray())
{
ReadParameters();
diff --git a/moses/TranslationModel/PhraseDictionaryMemory.cpp b/moses/TranslationModel/PhraseDictionaryMemory.cpp
index 4ad3c5e87..e7f0174d0 100644
--- a/moses/TranslationModel/PhraseDictionaryMemory.cpp
+++ b/moses/TranslationModel/PhraseDictionaryMemory.cpp
@@ -41,7 +41,7 @@ using namespace std;
namespace Moses
{
PhraseDictionaryMemory::PhraseDictionaryMemory(const std::string &line)
- : RuleTableTrie("PhraseDictionaryMemory", line)
+ : RuleTableTrie(line)
{
ReadParameters();
diff --git a/moses/TranslationModel/PhraseDictionaryMemory.h b/moses/TranslationModel/PhraseDictionaryMemory.h
index 44e57e7f4..a1a23eca3 100644
--- a/moses/TranslationModel/PhraseDictionaryMemory.h
+++ b/moses/TranslationModel/PhraseDictionaryMemory.h
@@ -39,8 +39,8 @@ class PhraseDictionaryMemory : public RuleTableTrie
friend class RuleTableLoader;
protected:
- PhraseDictionaryMemory(const std::string &description, const std::string &line)
- : RuleTableTrie(description, line) {
+ PhraseDictionaryMemory(int type, const std::string &line)
+ : RuleTableTrie(line) {
}
public:
diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp
index 923df4fe6..be3986929 100644
--- a/moses/TranslationModel/PhraseDictionaryMultiModel.cpp
+++ b/moses/TranslationModel/PhraseDictionaryMultiModel.cpp
@@ -26,7 +26,7 @@ namespace Moses
{
PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
- :PhraseDictionary("PhraseDictionaryMultiModel", line)
+ :PhraseDictionary(line)
{
ReadParameters();
@@ -43,10 +43,11 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
CHECK(m_pdStr.size() == m_multimodelweights.size() || m_pdStr.size()*numWeights == m_multimodelweights.size());
}
-PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &description, const std::string &line)
- :PhraseDictionary(description, line)
+PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(int type, const std::string &line)
+ :PhraseDictionary(line)
{
- if (description == "PhraseDictionaryMultiModelCounts") {
+ if (type == 1) {
+ // PhraseDictionaryMultiModelCounts
CHECK(m_pdStr.size() == m_multimodelweights.size() || m_pdStr.size()*4 == m_multimodelweights.size());
}
}
diff --git a/moses/TranslationModel/PhraseDictionaryMultiModel.h b/moses/TranslationModel/PhraseDictionaryMultiModel.h
index cd385895b..4d7786534 100644
--- a/moses/TranslationModel/PhraseDictionaryMultiModel.h
+++ b/moses/TranslationModel/PhraseDictionaryMultiModel.h
@@ -61,7 +61,7 @@ class PhraseDictionaryMultiModel: public PhraseDictionary
public:
PhraseDictionaryMultiModel(const std::string &line);
- PhraseDictionaryMultiModel(const std::string &description, const std::string &line);
+ PhraseDictionaryMultiModel(int type, const std::string &line);
~PhraseDictionaryMultiModel();
void Load();
virtual void CollectSufficientStatistics(const Phrase& src, std::map* allStats) const;
diff --git a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
index e33c9e8a8..6f0f85466 100644
--- a/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
+++ b/moses/TranslationModel/PhraseDictionaryMultiModelCounts.cpp
@@ -61,7 +61,7 @@ namespace Moses
{
PhraseDictionaryMultiModelCounts::PhraseDictionaryMultiModelCounts(const std::string &line)
- :PhraseDictionaryMultiModel("PhraseDictionaryMultiModelCounts", line)
+ :PhraseDictionaryMultiModel(1, line)
{
m_mode = "instance_weighting";
m_combineFunction = InstanceWeighting;
diff --git a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
index 7c5ae9a37..066fb4067 100644
--- a/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
+++ b/moses/TranslationModel/PhraseDictionaryTreeAdaptor.cpp
@@ -27,7 +27,7 @@ namespace Moses
PhraseDictionaryTreeAdaptor::
PhraseDictionaryTreeAdaptor(const std::string &line)
- : PhraseDictionary("PhraseDictionaryBinary", line)
+ : PhraseDictionary(line)
{
ReadParameters();
}
diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp
index 89356ad7a..a5625cb71 100644
--- a/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp
+++ b/moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.cpp
@@ -21,7 +21,7 @@ using namespace std;
namespace Moses
{
PhraseDictionaryALSuffixArray::PhraseDictionaryALSuffixArray(const std::string &line)
- : PhraseDictionaryMemory("PhraseDictionaryALSuffixArray", line)
+ : PhraseDictionaryMemory(1, line)
{
const StaticData &staticData = StaticData::Instance();
if (staticData.ThreadCount() > 1) {
diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp
index cfad3f830..542df66a9 100644
--- a/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp
+++ b/moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.cpp
@@ -54,7 +54,7 @@ namespace Moses
{
PhraseDictionaryFuzzyMatch::PhraseDictionaryFuzzyMatch(const std::string &line)
- :PhraseDictionary("PhraseDictionaryFuzzyMatch", line)
+ :PhraseDictionary(line)
,m_FuzzyMatchWrapper(NULL)
{
CHECK(m_args.size() == 0);
diff --git a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp
index b8fc434fb..5998e24a5 100644
--- a/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp
+++ b/moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.cpp
@@ -35,7 +35,7 @@ using namespace std;
namespace Moses
{
PhraseDictionaryOnDisk::PhraseDictionaryOnDisk(const std::string &line)
- : MyBase("PhraseDictionaryOnDisk", line)
+ : MyBase(line)
{
ReadParameters();
}
diff --git a/moses/TranslationModel/RuleTable/Trie.h b/moses/TranslationModel/RuleTable/Trie.h
index 0d363f747..c7cb7e991 100644
--- a/moses/TranslationModel/RuleTable/Trie.h
+++ b/moses/TranslationModel/RuleTable/Trie.h
@@ -40,8 +40,8 @@ class Word;
class RuleTableTrie : public PhraseDictionary
{
public:
- RuleTableTrie(const std::string &description, const std::string &line)
- : PhraseDictionary(description, line) {
+ RuleTableTrie(const std::string &line)
+ : PhraseDictionary(line) {
}
virtual ~RuleTableTrie();
diff --git a/moses/TranslationModel/RuleTable/UTrie.h b/moses/TranslationModel/RuleTable/UTrie.h
index 80a2e4b3d..8222bb632 100644
--- a/moses/TranslationModel/RuleTable/UTrie.h
+++ b/moses/TranslationModel/RuleTable/UTrie.h
@@ -46,7 +46,7 @@ class RuleTableUTrie : public RuleTableTrie
{
public:
RuleTableUTrie(const std::string &line)
- : RuleTableTrie("RuleTableUTrie", line) {
+ : RuleTableTrie(line) {
}
const UTrieNode &GetRootNode() const {
diff --git a/moses/TranslationModel/SkeletonPT.cpp b/moses/TranslationModel/SkeletonPT.cpp
index 322494d68..83496f352 100644
--- a/moses/TranslationModel/SkeletonPT.cpp
+++ b/moses/TranslationModel/SkeletonPT.cpp
@@ -7,7 +7,7 @@ using namespace std;
namespace Moses
{
SkeletonPT::SkeletonPT(const std::string &line)
- : PhraseDictionary("SkeletonPT", line)
+ : PhraseDictionary(line)
{
ReadParameters();
}
diff --git a/moses/TranslationOptionCollectionLattice.cpp b/moses/TranslationOptionCollectionLattice.cpp
index 878ab7968..a932e2a99 100644
--- a/moses/TranslationOptionCollectionLattice.cpp
+++ b/moses/TranslationOptionCollectionLattice.cpp
@@ -29,6 +29,7 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
const InputFeature *inputFeature = StaticData::Instance().GetInputFeature();
CHECK(inputFeature);
+ size_t maxPhraseLength = StaticData::Instance().GetMaxPhraseLength();
size_t size = input.GetSize();
// 1-word phrases
@@ -76,6 +77,11 @@ TranslationOptionCollectionLattice::TranslationOptionCollectionLattice(
}
size_t startPos = prevPath.GetWordsRange().GetStartPos();
+
+ if (endPos - startPos + 1 > maxPhraseLength) {
+ continue;
+ }
+
WordsRange range(startPos, endPos);
const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);
diff --git a/phrase-extract/extract-ghkm/Subgraph.cpp b/phrase-extract/extract-ghkm/Subgraph.cpp
index 6c4af8acb..0c941218d 100644
--- a/phrase-extract/extract-ghkm/Subgraph.cpp
+++ b/phrase-extract/extract-ghkm/Subgraph.cpp
@@ -17,8 +17,8 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
+#include
#include "Subgraph.h"
-
#include "Node.h"
namespace Moses
diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta
index 374038b1b..bbf1bed0d 100644
--- a/scripts/ems/experiment.meta
+++ b/scripts/ems/experiment.meta
@@ -967,7 +967,7 @@ decode
default-name: evaluation/output
qsub-script: yes
ignore-if: use-hiero
- rerun-on-change: decoder decoder-settings nbest report-segmentation report-precision-by-coverage analyze-search-graph
+ rerun-on-change: decoder decoder-settings nbest report-segmentation report-precision-by-coverage analyze-search-graph wade
error: Translation was not performed correctly
not-error: trans: No such file or directory
hiero-decode
@@ -1031,6 +1031,13 @@ lowercase-reference
pass-if: recaser
multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl
template: $output-lowercaser < IN > OUT
+wade
+ in: filtered-dir truecased-input tokenized-reference alignment system-output
+ out: wade-analysis
+ default-name: evaluation/wade-analysis
+ ignore-unless: wade
+ rerun-on-change: wade
+ template: $moses-script-dir/ems/support/run-wade.perl $wade IN IN1 IN2 IN3 IN4 OUT
nist-bleu
in: wrapped-output reference-sgm
out: nist-bleu-score
@@ -1131,6 +1138,6 @@ analysis-precision
[REPORTING] single
report
- in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model
+ in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:bolt-bleu-score EVALUATION:bolt-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage EVALUATION:analysis-prec TRAINING:biconcor-model EVALUATION:wade-analysis
out: report
default-name: evaluation/report
diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl
index 706cbe07c..777147af8 100755
--- a/scripts/ems/experiment.perl
+++ b/scripts/ems/experiment.perl
@@ -2646,6 +2646,7 @@ sub define_evaluation_decode {
my $report_segmentation = &backoff_and_get("EVALUATION:$set:report-segmentation");
my $analyze_search_graph = &backoff_and_get("EVALUATION:$set:analyze-search-graph");
my $report_precision_by_coverage = &backoff_and_get("EVALUATION:$set:report-precision-by-coverage");
+ my $use_wade = &backoff_and_get("EVALUATION:$set:wade");
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
my $word_alignment = &backoff_and_get("TRAINING:include-word-alignment-in-rules");
@@ -2666,6 +2667,9 @@ sub define_evaluation_decode {
$settings .= " -t";
}
}
+ if ($use_wade) {
+ $settings .= " -T $system_output.details";
+ }
$settings .= " -text-type \"test\"";
my $addTags = &backoff_and_get("EVALUATION:$set:add-tags");
diff --git a/scripts/ems/support/run-wade.perl b/scripts/ems/support/run-wade.perl
new file mode 100755
index 000000000..418ff8c97
--- /dev/null
+++ b/scripts/ems/support/run-wade.perl
@@ -0,0 +1,30 @@
+#!/usr/bin/perl
+
+use strict;
+use File::Temp qw/ tempfile tempdir /;
+
+if (scalar(@ARGV) != 7) {
+ print STDERR "Usage: run-wade.perl wade-script filtered-dir input reference alignment output wade-output\n";
+ exit 1;
+}
+
+my ($wade_script, $filtered_dir, $input, $reference, $alignment, $output, $wade_output) = @ARGV;
+my ($cfh, $cfile) = tempfile();
+
+#unless ($phrase_table =~ /.*\.gz/){ $phrase_table .= ".gz";}
+
+print $cfh "phrase-table=$filtered_dir/phrase-table.0-0.1.1.gz\n";
+print $cfh "fr_test=$input\n";
+print $cfh "en_test=$reference\n";
+print $cfh "fren_align_test=$alignment\n";
+print $cfh "translation_details=$output.details\n";
+
+print $cfh "candidatealign_out=$wade_output.candalign\n";
+print $cfh "annotatedalign_out=$wade_output.annotalign\n";
+print $cfh "transspans_out=$wade_output.transspans\n";
+print $cfh "wade_output=$wade_output\n";
+close $cfh;
+
+print $cfile . "\n";
+
+system("python $wade_script $cfile");