moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

get rid of FeatureFunction::unlimited. Every ff can have sparse AND dense features

Browse Source
This commit is contained in:
Hieu Hoang 2013-05-08 15:34:56 +01:00
parent 9081479f88
commit 4890e4673a
19 changed files with 19 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
moses-chart-cmd/IOWrapper.cpp
View File

@ -425,7 +425,7 @@ void IOWrapper::OutputFeatureScores( std::ostream& out, const ScoreComponentColl
bool labeledOutput = staticData.IsLabeledNBestList();
// regular features (not sparse)
if (ff->GetNumScoreComponents() != FeatureFunction::unlimited) {
if (ff->GetNumScoreComponents() != 0) {
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
lastName = ff->GetScoreProducerDescription();
out << " " << lastName << "=";

2
moses-chart-cmd/Main.cpp
View File

@ -189,7 +189,7 @@ bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source
static void PrintFeatureWeight(const FeatureFunction* ff)
{
size_t numScoreComps = ff->GetNumScoreComponents();
if (numScoreComps != FeatureFunction::unlimited) {
if (numScoreComps != 0) {
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
cout << ff->GetScoreProducerDescription() << "=";
for (size_t i = 0; i < numScoreComps; ++i) {

2
moses-cmd/IOWrapper.cpp
View File

@ -484,7 +484,7 @@ void OutputFeatureScores( std::ostream& out
bool labeledOutput = staticData.IsLabeledNBestList();
// regular features (not sparse)
if (ff->GetNumScoreComponents() != FeatureFunction::unlimited) {
if (ff->GetNumScoreComponents() != 0) {
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
lastName = ff->GetScoreProducerDescription();
out << " " << lastName << "=";

4
moses-cmd/Main.cpp
View File

@ -449,7 +449,7 @@ private:
static void PrintFeatureWeight(const FeatureFunction* ff)
{
size_t numScoreComps = ff->GetNumScoreComponents();
if (numScoreComps != FeatureFunction::unlimited) {
if (numScoreComps != 0) {
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
cout << ff->GetScoreProducerDescription() << "=";
for (size_t i = 0; i < numScoreComps; ++i) {
@ -491,7 +491,7 @@ static void ShowWeights()
size_t OutputFeatureWeightsForHypergraph(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream)
{
size_t numScoreComps = ff->GetNumScoreComponents();
if (numScoreComps != FeatureFunction::unlimited) {
if (numScoreComps != 0) {
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
if (numScoreComps > 1) {
for (size_t i = 0; i < numScoreComps; ++i) {

9
moses/FeatureFunction.cpp
View File

@ -73,7 +73,6 @@ const TargetPhrase& ChartBasedFeatureContext::GetTargetPhrase() const
}
multiset<string> FeatureFunction::description_counts;
const size_t FeatureFunction::unlimited = -1;
std::vector<FeatureFunction*> FeatureFunction::m_producers;
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
@ -97,10 +96,7 @@ FeatureFunction::FeatureFunction(const std::string& description, const std::stri
m_description = dstream.str();
}
if (m_numScoreComponents != unlimited) {
ScoreComponentCollection::RegisterScoreProducer(this);
}
m_producers.push_back(this);
}
@ -120,10 +116,7 @@ FeatureFunction::FeatureFunction(const std::string& description, size_t numScore
m_description = dstream.str();
}
if (numScoreComponents != unlimited) {
ScoreComponentCollection::RegisterScoreProducer(this);
}
ScoreComponentCollection::RegisterScoreProducer(this);
m_producers.push_back(this);
}

3
moses/FeatureFunction.h
View File

@ -99,15 +99,12 @@ public:
virtual bool IsStateless() const = 0;
virtual ~FeatureFunction();
static const size_t unlimited;
static void ResetDescriptionCounts() {
description_counts.clear();
}
//! returns the number of scores that a subclass produces.
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
//! sparse features returned unlimited
size_t GetNumScoreComponents() const {return m_numScoreComponents;}
//! returns a string description of this producer

2
moses/GlobalLexicalModelUnlimited.cpp
View File

@ -10,7 +10,7 @@ using namespace std;
namespace Moses
{
GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line)
:StatelessFeatureFunction("glm",FeatureFunction::unlimited, line)
:StatelessFeatureFunction("glm", 0, line)
{
const vector<string> modelSpec = Tokenize(line);

2
moses/Manager.cpp
View File

@ -754,7 +754,7 @@ void Manager::OutputFeatureValuesForHypergraph(const Hypothesis* hypo, std::ostr
size_t Manager::OutputFeatureWeightsForSLF(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream) const
{
size_t numScoreComps = ff->GetNumScoreComponents();
if (numScoreComps != FeatureFunction::unlimited) {
if (numScoreComps != 0) {
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
for (size_t i = 0; i < numScoreComps; ++i) {
outputSearchGraphStream << "# " << ff->GetScoreProducerDescription()

2
moses/PhraseBoundaryFeature.cpp
View File

@ -15,7 +15,7 @@ int PhraseBoundaryState::Compare(const FFState& other) const
}
PhraseBoundaryFeature::PhraseBoundaryFeature(const std::string &line)
: StatefulFeatureFunction("PhraseBoundaryFeature", FeatureFunction::unlimited, line)
: StatefulFeatureFunction("PhraseBoundaryFeature", 0, line)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;

2
moses/PhraseLengthFeature.cpp
View File

@ -9,7 +9,7 @@ namespace Moses {
using namespace std;
PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
:StatelessFeatureFunction("pl", FeatureFunction::unlimited, line)
:StatelessFeatureFunction("pl", 0, line)
{
}

2
moses/PhrasePairFeature.cpp
View File

@ -12,7 +12,7 @@ using namespace std;
namespace Moses {
PhrasePairFeature::PhrasePairFeature(const std::string &line)
:StatelessFeatureFunction("PhrasePairFeature", FeatureFunction::unlimited, line)
:StatelessFeatureFunction("PhrasePairFeature", 0, line)
{
std::cerr << "Initializing PhrasePairFeature.." << std::endl;

11
moses/ScoreComponentCollection.cpp
View File

@ -19,7 +19,6 @@ ScoreComponentCollection::ScoreComponentCollection() : m_scores(s_denseVectorSiz
void ScoreComponentCollection::RegisterScoreProducer
(const FeatureFunction* scoreProducer)
{
CHECK(scoreProducer->GetNumScoreComponents() != FeatureFunction::unlimited);
size_t start = s_denseVectorSize;
size_t end = start + scoreProducer->GetNumScoreComponents();
VERBOSE(1, "FeatureFunction: " << scoreProducer->GetScoreProducerDescription() << " start: " << start << " end: " << end << endl);
@ -27,14 +26,6 @@ void ScoreComponentCollection::RegisterScoreProducer
s_denseVectorSize = end;
}
void ScoreComponentCollection::UnregisterScoreProducer
(const FeatureFunction* scoreProducer)
{
CHECK(scoreProducer->GetNumScoreComponents() != FeatureFunction::unlimited);
ScoreIndexMap::iterator iter = s_scoreIndexes.find(scoreProducer);
CHECK(iter != s_scoreIndexes.end());
s_scoreIndexes.erase(iter);
}
float ScoreComponentCollection::GetWeightedScore() const
{
@ -62,7 +53,6 @@ void ScoreComponentCollection::MultiplyEquals(float scalar)
// Multiply all weights of this sparse producer by a given scalar
void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float scalar) {
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
std::stringstream name;
@ -74,7 +64,6 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
// Count weights belonging to this sparse producer
size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp) {
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
size_t weights = 0;
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {

9
moses/ScoreComponentCollection.h
View File

@ -106,7 +106,6 @@ public:
* be allocated space in the dense part of the feature vector.
**/
static void RegisterScoreProducer(const FeatureFunction* scoreProducer);
static void UnregisterScoreProducer(const FeatureFunction* scoreProducer);
/** Load from file */
bool Load(const std::string& filename)
@ -194,7 +193,6 @@ public:
//For features which have an unbounded number of components
void MinusEquals(const FeatureFunction*sp, const std::string& name, float score)
{
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
FName fname(sp->GetScoreProducerDescription(),name);
m_scores[fname] -= score;
}
@ -242,7 +240,6 @@ public:
//For features which have an unbounded number of components
void PlusEquals(const FeatureFunction*sp, const std::string& name, float score)
{
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
FName fname(sp->GetScoreProducerDescription(),name);
m_scores[fname] += score;
}
@ -250,7 +247,6 @@ public:
//For features which have an unbounded number of components
void PlusEquals(const FeatureFunction*sp, const StringPiece& name, float score)
{
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
FName fname(sp->GetScoreProducerDescription(),name);
m_scores[fname] += score;
}
@ -289,7 +285,6 @@ public:
//For features which have an unbounded number of components
void Assign(const FeatureFunction*sp, const std::string name, float score)
{
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
FName fname(sp->GetScoreProducerDescription(),name);
m_scores[fname] = score;
}
@ -321,7 +316,8 @@ public:
std::vector<float> GetScoresForProducer(const FeatureFunction* sp) const
{
size_t components = sp->GetNumScoreComponents();
if (components == FeatureFunction::unlimited) return std::vector<float>();
assert(components > 0);
std::vector<float> res(components);
IndexPair indexes = GetIndexes(sp);
for (size_t i = 0; i < res.size(); ++i) {
@ -386,7 +382,6 @@ public:
float GetScoreForProducer
(const FeatureFunction* sp, const std::string& name) const
{
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
FName fname(sp->GetScoreProducerDescription(),name);
return m_scores[fname];
}

2
moses/ScoreComponentCollectionTest.cpp
View File

@ -54,7 +54,7 @@ class MockMultiFeature : public MockStatelessFeatureFunction {
class MockSparseFeature : public MockStatelessFeatureFunction {
public:
MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", FeatureFunction::unlimited, "MockSparse") {}
MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", 0, "MockSparse") {}
};

2
moses/SourceWordDeletionFeature.cpp
View File

@ -15,7 +15,7 @@ namespace Moses {
using namespace std;
SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line)
:StatelessFeatureFunction("SourceWordDeletionFeature", FeatureFunction::unlimited, line),
:StatelessFeatureFunction("SourceWordDeletionFeature", 0, line),
m_unrestricted(true)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;

2
moses/TargetBigramFeature.cpp
View File

@ -15,7 +15,7 @@ int TargetBigramState::Compare(const FFState& other) const {
}
TargetBigramFeature::TargetBigramFeature(const std::string &line)
:StatefulFeatureFunction("TargetBigramFeature", FeatureFunction::unlimited, line)
:StatefulFeatureFunction("TargetBigramFeature", 0, line)
{
std::cerr << "Initializing target bigram feature.." << std::endl;

2
moses/TargetNgramFeature.cpp
View File

@ -38,7 +38,7 @@ int TargetNgramState::Compare(const FFState& other) const {
}
TargetNgramFeature::TargetNgramFeature(const std::string &line)
:StatefulFeatureFunction("TargetNgramFeature", FeatureFunction::unlimited, line)
:StatefulFeatureFunction("TargetNgramFeature", 0, line)
,m_sparseProducerWeight(1)
{
std::cerr << "Initializing target ngram feature.." << std::endl;

2
moses/TargetWordInsertionFeature.cpp
View File

@ -14,7 +14,7 @@ namespace Moses {
using namespace std;
TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
:StatelessFeatureFunction("TargetWordInsertionFeature", FeatureFunction::unlimited, line),
:StatelessFeatureFunction("TargetWordInsertionFeature", 0, line),
m_unrestricted(true)
{
std::cerr << "Initializing target word insertion feature.." << std::endl;

2
moses/WordTranslationFeature.cpp
View File

@ -15,7 +15,7 @@ namespace Moses {
using namespace std;
WordTranslationFeature::WordTranslationFeature(const std::string &line)
:StatelessFeatureFunction("WordTranslationFeature", FeatureFunction::unlimited, line)
:StatelessFeatureFunction("WordTranslationFeature", 0, line)
,m_unrestricted(true)
,m_sparseProducerWeight(1)
,m_simple(true)