mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
get rid of FeatureFunction::unlimited. Every ff can have sparse AND dense features
This commit is contained in:
parent
9081479f88
commit
4890e4673a
@ -425,7 +425,7 @@ void IOWrapper::OutputFeatureScores( std::ostream& out, const ScoreComponentColl
|
||||
bool labeledOutput = staticData.IsLabeledNBestList();
|
||||
|
||||
// regular features (not sparse)
|
||||
if (ff->GetNumScoreComponents() != FeatureFunction::unlimited) {
|
||||
if (ff->GetNumScoreComponents() != 0) {
|
||||
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
|
||||
lastName = ff->GetScoreProducerDescription();
|
||||
out << " " << lastName << "=";
|
||||
|
@ -189,7 +189,7 @@ bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source
|
||||
static void PrintFeatureWeight(const FeatureFunction* ff)
|
||||
{
|
||||
size_t numScoreComps = ff->GetNumScoreComponents();
|
||||
if (numScoreComps != FeatureFunction::unlimited) {
|
||||
if (numScoreComps != 0) {
|
||||
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
|
||||
cout << ff->GetScoreProducerDescription() << "=";
|
||||
for (size_t i = 0; i < numScoreComps; ++i) {
|
||||
|
@ -484,7 +484,7 @@ void OutputFeatureScores( std::ostream& out
|
||||
bool labeledOutput = staticData.IsLabeledNBestList();
|
||||
|
||||
// regular features (not sparse)
|
||||
if (ff->GetNumScoreComponents() != FeatureFunction::unlimited) {
|
||||
if (ff->GetNumScoreComponents() != 0) {
|
||||
if( labeledOutput && lastName != ff->GetScoreProducerDescription() ) {
|
||||
lastName = ff->GetScoreProducerDescription();
|
||||
out << " " << lastName << "=";
|
||||
|
@ -449,7 +449,7 @@ private:
|
||||
static void PrintFeatureWeight(const FeatureFunction* ff)
|
||||
{
|
||||
size_t numScoreComps = ff->GetNumScoreComponents();
|
||||
if (numScoreComps != FeatureFunction::unlimited) {
|
||||
if (numScoreComps != 0) {
|
||||
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
|
||||
cout << ff->GetScoreProducerDescription() << "=";
|
||||
for (size_t i = 0; i < numScoreComps; ++i) {
|
||||
@ -491,7 +491,7 @@ static void ShowWeights()
|
||||
size_t OutputFeatureWeightsForHypergraph(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream)
|
||||
{
|
||||
size_t numScoreComps = ff->GetNumScoreComponents();
|
||||
if (numScoreComps != FeatureFunction::unlimited) {
|
||||
if (numScoreComps != 0) {
|
||||
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
|
||||
if (numScoreComps > 1) {
|
||||
for (size_t i = 0; i < numScoreComps; ++i) {
|
||||
|
@ -73,7 +73,6 @@ const TargetPhrase& ChartBasedFeatureContext::GetTargetPhrase() const
|
||||
}
|
||||
|
||||
multiset<string> FeatureFunction::description_counts;
|
||||
const size_t FeatureFunction::unlimited = -1;
|
||||
|
||||
std::vector<FeatureFunction*> FeatureFunction::m_producers;
|
||||
std::vector<const StatelessFeatureFunction*> StatelessFeatureFunction::m_statelessFFs;
|
||||
@ -97,10 +96,7 @@ FeatureFunction::FeatureFunction(const std::string& description, const std::stri
|
||||
m_description = dstream.str();
|
||||
}
|
||||
|
||||
if (m_numScoreComponents != unlimited) {
|
||||
ScoreComponentCollection::RegisterScoreProducer(this);
|
||||
}
|
||||
|
||||
m_producers.push_back(this);
|
||||
}
|
||||
|
||||
@ -120,10 +116,7 @@ FeatureFunction::FeatureFunction(const std::string& description, size_t numScore
|
||||
m_description = dstream.str();
|
||||
}
|
||||
|
||||
if (numScoreComponents != unlimited) {
|
||||
ScoreComponentCollection::RegisterScoreProducer(this);
|
||||
}
|
||||
|
||||
ScoreComponentCollection::RegisterScoreProducer(this);
|
||||
m_producers.push_back(this);
|
||||
}
|
||||
|
||||
|
@ -99,15 +99,12 @@ public:
|
||||
virtual bool IsStateless() const = 0;
|
||||
virtual ~FeatureFunction();
|
||||
|
||||
static const size_t unlimited;
|
||||
|
||||
static void ResetDescriptionCounts() {
|
||||
description_counts.clear();
|
||||
}
|
||||
|
||||
//! returns the number of scores that a subclass produces.
|
||||
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
|
||||
//! sparse features returned unlimited
|
||||
size_t GetNumScoreComponents() const {return m_numScoreComponents;}
|
||||
|
||||
//! returns a string description of this producer
|
||||
|
@ -10,7 +10,7 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
GlobalLexicalModelUnlimited::GlobalLexicalModelUnlimited(const std::string &line)
|
||||
:StatelessFeatureFunction("glm",FeatureFunction::unlimited, line)
|
||||
:StatelessFeatureFunction("glm", 0, line)
|
||||
{
|
||||
const vector<string> modelSpec = Tokenize(line);
|
||||
|
||||
|
@ -754,7 +754,7 @@ void Manager::OutputFeatureValuesForHypergraph(const Hypothesis* hypo, std::ostr
|
||||
size_t Manager::OutputFeatureWeightsForSLF(size_t index, const FeatureFunction* ff, std::ostream &outputSearchGraphStream) const
|
||||
{
|
||||
size_t numScoreComps = ff->GetNumScoreComponents();
|
||||
if (numScoreComps != FeatureFunction::unlimited) {
|
||||
if (numScoreComps != 0) {
|
||||
vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff);
|
||||
for (size_t i = 0; i < numScoreComps; ++i) {
|
||||
outputSearchGraphStream << "# " << ff->GetScoreProducerDescription()
|
||||
|
@ -15,7 +15,7 @@ int PhraseBoundaryState::Compare(const FFState& other) const
|
||||
}
|
||||
|
||||
PhraseBoundaryFeature::PhraseBoundaryFeature(const std::string &line)
|
||||
: StatefulFeatureFunction("PhraseBoundaryFeature", FeatureFunction::unlimited, line)
|
||||
: StatefulFeatureFunction("PhraseBoundaryFeature", 0, line)
|
||||
{
|
||||
std::cerr << "Initializing source word deletion feature.." << std::endl;
|
||||
|
||||
|
@ -9,7 +9,7 @@ namespace Moses {
|
||||
using namespace std;
|
||||
|
||||
PhraseLengthFeature::PhraseLengthFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("pl", FeatureFunction::unlimited, line)
|
||||
:StatelessFeatureFunction("pl", 0, line)
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -12,7 +12,7 @@ using namespace std;
|
||||
namespace Moses {
|
||||
|
||||
PhrasePairFeature::PhrasePairFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("PhrasePairFeature", FeatureFunction::unlimited, line)
|
||||
:StatelessFeatureFunction("PhrasePairFeature", 0, line)
|
||||
{
|
||||
std::cerr << "Initializing PhrasePairFeature.." << std::endl;
|
||||
|
||||
|
@ -19,7 +19,6 @@ ScoreComponentCollection::ScoreComponentCollection() : m_scores(s_denseVectorSiz
|
||||
void ScoreComponentCollection::RegisterScoreProducer
|
||||
(const FeatureFunction* scoreProducer)
|
||||
{
|
||||
CHECK(scoreProducer->GetNumScoreComponents() != FeatureFunction::unlimited);
|
||||
size_t start = s_denseVectorSize;
|
||||
size_t end = start + scoreProducer->GetNumScoreComponents();
|
||||
VERBOSE(1, "FeatureFunction: " << scoreProducer->GetScoreProducerDescription() << " start: " << start << " end: " << end << endl);
|
||||
@ -27,14 +26,6 @@ void ScoreComponentCollection::RegisterScoreProducer
|
||||
s_denseVectorSize = end;
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::UnregisterScoreProducer
|
||||
(const FeatureFunction* scoreProducer)
|
||||
{
|
||||
CHECK(scoreProducer->GetNumScoreComponents() != FeatureFunction::unlimited);
|
||||
ScoreIndexMap::iterator iter = s_scoreIndexes.find(scoreProducer);
|
||||
CHECK(iter != s_scoreIndexes.end());
|
||||
s_scoreIndexes.erase(iter);
|
||||
}
|
||||
|
||||
float ScoreComponentCollection::GetWeightedScore() const
|
||||
{
|
||||
@ -62,7 +53,6 @@ void ScoreComponentCollection::MultiplyEquals(float scalar)
|
||||
|
||||
// Multiply all weights of this sparse producer by a given scalar
|
||||
void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float scalar) {
|
||||
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
||||
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
||||
std::stringstream name;
|
||||
@ -74,7 +64,6 @@ void ScoreComponentCollection::MultiplyEquals(const FeatureFunction* sp, float s
|
||||
|
||||
// Count weights belonging to this sparse producer
|
||||
size_t ScoreComponentCollection::GetNumberWeights(const FeatureFunction* sp) {
|
||||
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
std::string prefix = sp->GetScoreProducerDescription() + FName::SEP;
|
||||
size_t weights = 0;
|
||||
for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) {
|
||||
|
@ -106,7 +106,6 @@ public:
|
||||
* be allocated space in the dense part of the feature vector.
|
||||
**/
|
||||
static void RegisterScoreProducer(const FeatureFunction* scoreProducer);
|
||||
static void UnregisterScoreProducer(const FeatureFunction* scoreProducer);
|
||||
|
||||
/** Load from file */
|
||||
bool Load(const std::string& filename)
|
||||
@ -194,7 +193,6 @@ public:
|
||||
//For features which have an unbounded number of components
|
||||
void MinusEquals(const FeatureFunction*sp, const std::string& name, float score)
|
||||
{
|
||||
assert(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
FName fname(sp->GetScoreProducerDescription(),name);
|
||||
m_scores[fname] -= score;
|
||||
}
|
||||
@ -242,7 +240,6 @@ public:
|
||||
//For features which have an unbounded number of components
|
||||
void PlusEquals(const FeatureFunction*sp, const std::string& name, float score)
|
||||
{
|
||||
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
FName fname(sp->GetScoreProducerDescription(),name);
|
||||
m_scores[fname] += score;
|
||||
}
|
||||
@ -250,7 +247,6 @@ public:
|
||||
//For features which have an unbounded number of components
|
||||
void PlusEquals(const FeatureFunction*sp, const StringPiece& name, float score)
|
||||
{
|
||||
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
FName fname(sp->GetScoreProducerDescription(),name);
|
||||
m_scores[fname] += score;
|
||||
}
|
||||
@ -289,7 +285,6 @@ public:
|
||||
//For features which have an unbounded number of components
|
||||
void Assign(const FeatureFunction*sp, const std::string name, float score)
|
||||
{
|
||||
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
FName fname(sp->GetScoreProducerDescription(),name);
|
||||
m_scores[fname] = score;
|
||||
}
|
||||
@ -321,7 +316,8 @@ public:
|
||||
std::vector<float> GetScoresForProducer(const FeatureFunction* sp) const
|
||||
{
|
||||
size_t components = sp->GetNumScoreComponents();
|
||||
if (components == FeatureFunction::unlimited) return std::vector<float>();
|
||||
assert(components > 0);
|
||||
|
||||
std::vector<float> res(components);
|
||||
IndexPair indexes = GetIndexes(sp);
|
||||
for (size_t i = 0; i < res.size(); ++i) {
|
||||
@ -386,7 +382,6 @@ public:
|
||||
float GetScoreForProducer
|
||||
(const FeatureFunction* sp, const std::string& name) const
|
||||
{
|
||||
CHECK(sp->GetNumScoreComponents() == FeatureFunction::unlimited);
|
||||
FName fname(sp->GetScoreProducerDescription(),name);
|
||||
return m_scores[fname];
|
||||
}
|
||||
|
@ -54,7 +54,7 @@ class MockMultiFeature : public MockStatelessFeatureFunction {
|
||||
|
||||
class MockSparseFeature : public MockStatelessFeatureFunction {
|
||||
public:
|
||||
MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", FeatureFunction::unlimited, "MockSparse") {}
|
||||
MockSparseFeature(): MockStatelessFeatureFunction("MockSparse", 0, "MockSparse") {}
|
||||
};
|
||||
|
||||
|
||||
|
@ -15,7 +15,7 @@ namespace Moses {
|
||||
using namespace std;
|
||||
|
||||
SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("SourceWordDeletionFeature", FeatureFunction::unlimited, line),
|
||||
:StatelessFeatureFunction("SourceWordDeletionFeature", 0, line),
|
||||
m_unrestricted(true)
|
||||
{
|
||||
std::cerr << "Initializing source word deletion feature.." << std::endl;
|
||||
|
@ -15,7 +15,7 @@ int TargetBigramState::Compare(const FFState& other) const {
|
||||
}
|
||||
|
||||
TargetBigramFeature::TargetBigramFeature(const std::string &line)
|
||||
:StatefulFeatureFunction("TargetBigramFeature", FeatureFunction::unlimited, line)
|
||||
:StatefulFeatureFunction("TargetBigramFeature", 0, line)
|
||||
{
|
||||
std::cerr << "Initializing target bigram feature.." << std::endl;
|
||||
|
||||
|
@ -38,7 +38,7 @@ int TargetNgramState::Compare(const FFState& other) const {
|
||||
}
|
||||
|
||||
TargetNgramFeature::TargetNgramFeature(const std::string &line)
|
||||
:StatefulFeatureFunction("TargetNgramFeature", FeatureFunction::unlimited, line)
|
||||
:StatefulFeatureFunction("TargetNgramFeature", 0, line)
|
||||
,m_sparseProducerWeight(1)
|
||||
{
|
||||
std::cerr << "Initializing target ngram feature.." << std::endl;
|
||||
|
@ -14,7 +14,7 @@ namespace Moses {
|
||||
using namespace std;
|
||||
|
||||
TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("TargetWordInsertionFeature", FeatureFunction::unlimited, line),
|
||||
:StatelessFeatureFunction("TargetWordInsertionFeature", 0, line),
|
||||
m_unrestricted(true)
|
||||
{
|
||||
std::cerr << "Initializing target word insertion feature.." << std::endl;
|
||||
|
@ -15,7 +15,7 @@ namespace Moses {
|
||||
using namespace std;
|
||||
|
||||
WordTranslationFeature::WordTranslationFeature(const std::string &line)
|
||||
:StatelessFeatureFunction("WordTranslationFeature", FeatureFunction::unlimited, line)
|
||||
:StatelessFeatureFunction("WordTranslationFeature", 0, line)
|
||||
,m_unrestricted(true)
|
||||
,m_sparseProducerWeight(1)
|
||||
,m_simple(true)
|
||||
|
Loading…
Reference in New Issue
Block a user