moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Goodbye ScoreIndexManager.

Browse Source 
Compiles ok, but haven't dared to run regression yet.


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mira-mtm5@3608 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
bhaddow 2010-10-06 22:06:49 +00:00
parent 82dba81917
commit 7e72ceea22
58 changed files with 222 additions and 708 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mira/Decoder.cpp
View File

@ -175,12 +175,12 @@ namespace Mira {
}
ScoreComponentCollection MosesDecoder::getWeights() {
return StaticData::Instance().GetAllWeightsScoreComponentCollection();
return StaticData::Instance().GetAllWeights();
}
void MosesDecoder::setWeights(const ScoreComponentCollection& weights) {
cerr << "New weights: " << weights << endl;
StaticData::InstanceNonConst().SetAllWeightsScoreComponentCollection(weights);
StaticData::InstanceNonConst().SetAllWeights(weights);
}
void MosesDecoder::updateHistory(const vector<const Word*>& words) {

13
moses-chart-cmd/src/Main.cpp
View File

@ -115,19 +115,12 @@ int main(int argc, char* argv[])
IOWrapper *ioWrapper = GetIODevice(staticData);
// check on weights
vector<float> weights = staticData.GetAllWeights();
const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
TRACE_ERR("The global weight vector looks like this:");
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights);
TRACE_ERR("\n");
}
// every score must have a weight! check that here:
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
return EXIT_FAILURE;
}
if (ioWrapper == NULL)
return EXIT_FAILURE;

4
moses-chart/src/ChartHypothesis.cpp
View File

@ -274,8 +274,8 @@ void Hypothesis::CalcLMScore()
{
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
{ // get from other prev hypo. faster
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
m_lmNGram.Assign(prevHypo->m_lmNGram);
m_lmPrefix = prevHypo->m_lmPrefix;
m_lmNGram = prevHypo->m_lmNGram;
}
else
{ // calc

12
moses-cmd/src/Main.cpp
View File

@ -345,18 +345,12 @@ int main(int argc, char** argv) {
}
// check on weights
vector<float> weights = staticData.GetAllWeights();
const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
TRACE_ERR("The global weight vector looks like this:");
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights);
TRACE_ERR("\n");
}
// every score must have a weight! check that here:
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
exit(1);
}
InputType* source = NULL;

3
moses-cmd/src/TranslationAnalysis.cpp
View File

@ -110,7 +110,8 @@ namespace TranslationAnalysis {
}
}
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights());
os << translationPath.back()->GetScoreBreakdown();
os << " weighted(TODO)";
os << std::endl;
}

6
moses/src/BleuScoreFeature.cpp
View File

@ -69,11 +69,7 @@ BleuScoreFeature::BleuScoreFeature():
m_count_history(BleuScoreState::bleu_order),
m_match_history(BleuScoreState::bleu_order),
m_target_length_history(0),
m_ref_length_history(0)
{
const_cast<ScoreIndexManager&>
(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
}
m_ref_length_history(0) {}
void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs)
{

14
moses/src/DummyScoreProducers.cpp
View File

@ -37,10 +37,6 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in
NOT_FOUND);
}
DistortionScoreProducer::DistortionScoreProducer(ScoreIndexManager &scoreIndexManager)
{
scoreIndexManager.AddScoreProducer(this);
}
size_t DistortionScoreProducer::GetNumScoreComponents() const
{
@ -104,11 +100,6 @@ FFState* DistortionScoreProducer::Evaluate(
}
WordPenaltyProducer::WordPenaltyProducer(ScoreIndexManager &scoreIndexManager)
{
scoreIndexManager.AddScoreProducer(this);
}
size_t WordPenaltyProducer::GetNumScoreComponents() const
{
return 1;
@ -131,11 +122,6 @@ void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollect
out->PlusEquals(this, -static_cast<float>(tp.GetSize()));
}
UnknownWordPenaltyProducer::UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager)
{
scoreIndexManager.AddScoreProducer(this);
}
size_t UnknownWordPenaltyProducer::GetNumScoreComponents() const
{
return 1;

6
moses/src/DummyScoreProducers.h
View File

@ -14,7 +14,7 @@ class WordsRange;
*/
class DistortionScoreProducer : public StatefulFeatureFunction {
public:
DistortionScoreProducer(ScoreIndexManager &scoreIndexManager);
DistortionScoreProducer() {}
float CalculateDistortionScore(const Hypothesis& hypo,
const WordsRange &prev, const WordsRange &curr, const int FirstGapPosition) const;
@ -38,7 +38,7 @@ public:
*/
class WordPenaltyProducer : public StatelessFeatureFunction {
public:
WordPenaltyProducer(ScoreIndexManager &scoreIndexManager);
WordPenaltyProducer() {}
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;
@ -54,7 +54,7 @@ public:
/** unknown word penalty */
class UnknownWordPenaltyProducer : public StatelessFeatureFunction {
public:
UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager);
UnknownWordPenaltyProducer() {}
size_t GetNumScoreComponents() const;
std::string GetScoreProducerDescription() const;

30
moses/src/FeatureVector.cpp
View File

@ -26,8 +26,6 @@
#include <sstream>
#include <stdexcept>
#include "StaticData.h"
#include "ScoreIndexManager.h"
#include "FeatureVector.h"
using namespace std;
@ -76,8 +74,7 @@ namespace Moses {
return ! (*this == rhs);
}
FVector::FVector(size_t size, FValue defaultValue)
:m_size(size)
FVector::FVector( FValue defaultValue)
{
m_features[DEFAULT_NAME] = defaultValue;
}
@ -382,31 +379,6 @@ namespace Moses {
return product;
}
FValue FVector::inner_product(const std::vector<FValue>& rhs) const {
assert(size() == rhs.size());
const ScoreIndexManager &sim = StaticData::Instance().GetScoreIndexManager();
FValue product = 0;
if (get(DEFAULT_NAME) == 0) {
for (const_iterator i = cbegin(); i != cend(); i++) {
const FName &fname = i->first;
if (fname != DEFAULT_NAME) {
const std::string &name = fname.name();
size_t index = sim.GetFeatureIndex(name);
assert(rhs.size() > index);
FValue rv = rhs[ index ];
product += i->second * rv;
}
}
} else {
for (size_t i = 0; i < rhs.size(); i++) {
product += rhs[i] * get(sim.GetFeatureName(i));
}
}
return product;
}
const FVector operator+(const FVector& lhs, const FVector& rhs) {
return FVector(lhs) += rhs;
}

13
moses/src/FeatureVector.h
View File

@ -67,6 +67,7 @@ namespace Moses {
//const std::string& root() const {return m_root;}
size_t hash() const;
bool operator==(const FName& rhs) const ;
bool operator!=(const FName& rhs) const ;
@ -108,12 +109,8 @@ namespace Moses {
class FVector
{
public:
// do not implement
FVector();
bool operator=(const FName& rhs);
/** Empty feature vector, possibly with default value */
FVector(size_t size, FValue defaultValue = DEFAULT);
FVector(FValue defaultValue = DEFAULT);
typedef boost::unordered_map<FName,FValue,FNameHash, FNameEquals> FNVmap;
/** Iterators */
@ -142,15 +139,14 @@ namespace Moses {
/** Element access */
ProxyFVector operator[](const FName& name);
FValue operator[](const FName& name) const;
/** Size */
size_t size() const {return m_size;}
size_t size() const {return m_features.size();}
/** Equality */
bool operator== (const FVector& rhs) const;
bool operator!= (const FVector& rhs) const;
FValue inner_product(const std::vector<FValue>& rhs) const;
FValue inner_product(const FVector& rhs) const;
@ -192,7 +188,6 @@ namespace Moses {
FNVmap m_features;
size_t m_size;
#ifdef MPI_ENABLE
//serialization

7
moses/src/GenerationDictionary.cpp
View File

@ -33,13 +33,10 @@ using namespace std;
namespace Moses
{
GenerationDictionary::GenerationDictionary(size_t numFeatures, ScoreIndexManager &scoreIndexManager,
GenerationDictionary::GenerationDictionary(size_t numFeatures,
const std::vector<FactorType> &input,
const std::vector<FactorType> &output)
: Dictionary(numFeatures), DecodeFeature(input,output)
{
scoreIndexManager.AddScoreProducer(this);
}
: Dictionary(numFeatures), DecodeFeature(input,output) {}
bool GenerationDictionary::Load(const std::string &filePath, FactorDirection direction)
{

1
moses/src/GenerationDictionary.h
View File

@ -57,7 +57,6 @@ public:
*/
GenerationDictionary(
size_t numFeatures,
ScoreIndexManager &scoreIndexManager,
const std::vector<FactorType> &input,
const std::vector<FactorType> &output);
virtual ~GenerationDictionary();

6
moses/src/GlobalLexicalModel.cpp
View File

@ -9,17 +9,11 @@ using namespace std;
namespace Moses
{
GlobalLexicalModel::GlobalLexicalModel(const string &filePath,
const float weight,
const vector< FactorType >& inFactors,
const vector< FactorType >& outFactors)
{
std::cerr << "Creating global lexical model...\n";
// register as score producer
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
std::vector< float > weights;
weights.push_back( weight );
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
// load model
LoadData( filePath, inFactors, outFactors );

1
moses/src/GlobalLexicalModel.h
View File

@ -48,7 +48,6 @@ private:
public:
GlobalLexicalModel(const std::string &filePath,
const float weight,
const std::vector< FactorType >& inFactors,
const std::vector< FactorType >& outFactors);
virtual ~GlobalLexicalModel();

7
moses/src/LMList.cpp
View File

@ -95,13 +95,6 @@ void LMList::Add(LanguageModel *lm)
{
m_coll.push_back(lm);
m_maxNGramOrder = (lm->GetNGramOrder() > m_maxNGramOrder) ? lm->GetNGramOrder() : m_maxNGramOrder;
const ScoreIndexManager &scoreMgr = StaticData::Instance().GetScoreIndexManager();
size_t startInd = scoreMgr.GetBeginIndex(lm->GetScoreBookkeepingID())
,endInd = scoreMgr.GetEndIndex(lm->GetScoreBookkeepingID()) - 1;
m_minInd = min(m_minInd, startInd);
m_maxInd = max(m_maxInd, endInd);
}
}

7
moses/src/LMList.h
View File

@ -19,7 +19,6 @@ protected:
CollType m_coll;
size_t m_maxNGramOrder;
size_t m_minInd, m_maxInd;
public:
typedef CollType::iterator iterator;
@ -31,8 +30,6 @@ public:
LMList()
:m_maxNGramOrder(0)
,m_minInd(std::numeric_limits<size_t>::max())
,m_maxInd(0)
{}
void CleanUp();
~LMList();
@ -47,10 +44,6 @@ public:
size_t GetMaxNGramOrder() const
{ return m_maxNGramOrder; }
size_t GetMinIndex() const
{ return m_minInd; }
size_t GetMaxIndex() const
{ return m_maxInd; }
};

11
moses/src/LanguageModel.cpp
View File

@ -37,11 +37,8 @@ using namespace std;
namespace Moses
{
LanguageModel::LanguageModel(bool registerScore, ScoreIndexManager &scoreIndexManager)
{
if (registerScore)
scoreIndexManager.AddScoreProducer(this);
}
LanguageModel::LanguageModel() {}
LanguageModel::~LanguageModel() {}
// don't inline virtual funcs...
@ -252,9 +249,7 @@ FFState* LanguageModel::Evaluate(
}
float LanguageModel::GetWeight() const {
size_t lmIndex = StaticData::Instance().GetScoreIndexManager().
GetBeginIndex(GetScoreBookkeepingID());
return StaticData::Instance().GetAllWeights()[lmIndex];
return StaticData::Instance().GetAllWeights().GetScoreForProducer(this);
}
}

4
moses/src/LanguageModel.h
View File

@ -49,10 +49,8 @@ protected:
void ShiftOrPush(std::vector<const Word*> &contextFactor, const Word &word) const;
/** constructor to be called by inherited class
* \param registerScore whether this LM will be directly used to score sentence.
* Usually true, except where LM is a component in a composite LM, eg. LanguageModelJoint
*/
LanguageModel(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModel();
public:
/* Returned from LM implementations which points at the state used. For example, if a trigram score was requested

3
moses/src/LanguageModelDelegate.h
View File

@ -31,8 +31,7 @@ namespace Moses {
class LanguageModelDelegate: public LanguageModelSingleFactor {
public:
LanguageModelDelegate(bool registerScore, ScoreIndexManager &scoreIndexManager, LanguageModelSingleFactor* delegate) :
LanguageModelSingleFactor(registerScore, scoreIndexManager), m_delegate(delegate)
LanguageModelDelegate(LanguageModelSingleFactor* delegate) : m_delegate(delegate)
{
m_nGramOrder = m_delegate->GetNGramOrder();
m_factorType = m_delegate->GetFactorType();

32
moses/src/LanguageModelFactory.cpp
View File

@ -60,7 +60,6 @@ namespace LanguageModelFactory
, const std::vector<FactorType> &factorTypes
, size_t nGramOrder
, const std::string &languageModelFile
, ScoreIndexManager &scoreIndexManager
, int dub)
{
LanguageModel *lm = NULL;
@ -68,61 +67,52 @@ namespace LanguageModelFactory
{
case RandLM:
#ifdef LM_RAND
lm = new LanguageModelRandLM(true,
scoreIndexManager);
lm = new LanguageModelRandLM();
#endif
break;
case Remote:
#ifdef LM_REMOTE
lm = new LanguageModelRemote(true,scoreIndexManager);
lm = new LanguageModelRemote();
#endif
break;
case SRI:
#ifdef LM_SRI
lm = new LanguageModelSRI(true, scoreIndexManager);
lm = new LanguageModelSRI();
#endif
break;
case IRST:
#ifdef LM_IRST
lm = new LanguageModelIRST(true, scoreIndexManager, dub);
lm = new LanguageModelIRST(dub);
#endif
break;
case Skip:
#ifdef LM_SRI
lm = new LanguageModelSkip(new LanguageModelSRI(false, scoreIndexManager)
, true
, scoreIndexManager);
lm = new LanguageModelSkip(new LanguageModelSRI());
#elif LM_INTERNAL
lm = new LanguageModelSkip(new LanguageModelInternal(false, scoreIndexManager)
, true
, scoreIndexManager);
lm = new LanguageModelSkip(new LanguageModelInternal());
#endif
break;
case Ken:
#ifdef LM_KEN
lm = new LanguageModelKen(true, scoreIndexManager);
lm = new LanguageModelKen();
#endif
break;
case Joint:
#ifdef LM_SRI
lm = new LanguageModelJoint(new LanguageModelSRI(false, scoreIndexManager)
, true
, scoreIndexManager);
lm = new LanguageModelJoint(new LanguageModelSRI());
#elif LM_INTERNAL
lm = new LanguageModelJoint(new LanguageModelInternal(false, scoreIndexManager)
, true
, scoreIndexManager);
lm = new LanguageModelJoint(new LanguageModelInternal());
#endif
break;
case ParallelBackoff:
#ifdef LM_SRI
lm = new LanguageModelParallelBackoff(true, scoreIndexManager);
lm = new LanguageModelParallelBackoff();
#endif
break;
case Internal:
#ifdef LM_INTERNAL
lm = new LanguageModelInternal(true, scoreIndexManager);
lm = new LanguageModelInternal();
#endif
break;
}

2
moses/src/LanguageModelFactory.h
View File

@ -11,7 +11,6 @@ namespace Moses
{
class LanguageModel;
class ScoreIndexManager;
namespace LanguageModelFactory {
@ -23,7 +22,6 @@ namespace LanguageModelFactory {
, const std::vector<FactorType> &factorTypes
, size_t nGramOrder
, const std::string &languageModelFile
, ScoreIndexManager &scoreIndexManager
, int dub);
};

5
moses/src/LanguageModelIRST.cpp
View File

@ -42,9 +42,8 @@ using namespace std;
namespace Moses
{
LanguageModelIRST::LanguageModelIRST(bool registerScore, ScoreIndexManager &scoreIndexManager, int dub)
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
,m_lmtb(0),m_lmtb_dub(dub)
LanguageModelIRST::LanguageModelIRST(int dub)
:m_lmtb(0),m_lmtb_dub(dub)
{
}

2
moses/src/LanguageModelIRST.h
View File

@ -66,7 +66,7 @@ protected:
};
public:
LanguageModelIRST(bool registerScore, ScoreIndexManager &scoreIndexManager, int dub);
LanguageModelIRST(int dub);
~LanguageModelIRST();
bool Load(const std::string &filePath
, FactorType factorType

4
moses/src/LanguageModelInternal.cpp
View File

@ -10,10 +10,6 @@ using namespace std;
namespace Moses
{
LanguageModelInternal::LanguageModelInternal(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
{
}
bool LanguageModelInternal::Load(const std::string &filePath
, FactorType factorType

2
moses/src/LanguageModelInternal.h
View File

@ -26,7 +26,7 @@ protected:
float GetValue(const Factor *factor0, const Factor *factor1, const Factor *factor2, State* finalState) const;
public:
LanguageModelInternal(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelInternal() {}
bool Load(const std::string &filePath
, FactorType factorType
, size_t nGramOrder);

3
moses/src/LanguageModelJoint.h
View File

@ -48,8 +48,7 @@ protected:
size_t m_implFactor;
public:
LanguageModelJoint(LanguageModelSingleFactor *lmImpl, bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelMultiFactor(registerScore, scoreIndexManager)
LanguageModelJoint(LanguageModelSingleFactor *lmImpl)
{
m_lmImpl = lmImpl;
}

4
moses/src/LanguageModelKen.cpp
View File

@ -38,8 +38,8 @@ using namespace std;
namespace Moses
{
LanguageModelKen::LanguageModelKen(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelSingleFactor(registerScore, scoreIndexManager), m_ngram(NULL)
LanguageModelKen::LanguageModelKen()
:m_ngram(NULL)
{
}

2
moses/src/LanguageModelKen.h
View File

@ -42,7 +42,7 @@ protected:
lm::ngram::Model *m_ngram;
public:
LanguageModelKen(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelKen();
~LanguageModelKen();
bool Load(const std::string &filePath
, FactorType factorType

3
moses/src/LanguageModelMultiFactor.cpp
View File

@ -24,9 +24,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace Moses
{
LanguageModelMultiFactor::LanguageModelMultiFactor(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModel(registerScore, scoreIndexManager)
{}
std::string LanguageModelMultiFactor::GetScoreProducerDescription() const
{

2
moses/src/LanguageModelMultiFactor.h
View File

@ -39,7 +39,7 @@ class LanguageModelMultiFactor : public LanguageModel
protected:
FactorMask m_factorTypes;
LanguageModelMultiFactor(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelMultiFactor(){}
public:
virtual bool Load(const std::string &filePath

6
moses/src/LanguageModelParallelBackoff.cpp
View File

@ -35,12 +35,6 @@ using namespace std;
namespace Moses
{
LanguageModelParallelBackoff::LanguageModelParallelBackoff(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelMultiFactor(registerScore, scoreIndexManager)
{
///
}
LanguageModelParallelBackoff::~LanguageModelParallelBackoff()
{
///

2
moses/src/LanguageModelParallelBackoff.h
View File

@ -78,7 +78,7 @@ protected:
WidMatrix *widMatrix;
public:
LanguageModelParallelBackoff(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelParallelBackoff(){}
~LanguageModelParallelBackoff();

3
moses/src/LanguageModelRandLM.h
View File

@ -38,8 +38,7 @@ class Phrase;
class LanguageModelRandLM : public LanguageModelSingleFactor {
public:
LanguageModelRandLM(bool registerScore, ScoreIndexManager &scoreIndexManager)
: LanguageModelSingleFactor(registerScore, scoreIndexManager), m_lm(0) {}
LanguageModelRandLM() : m_lm(0) {}
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const;
~LanguageModelRandLM() {

5
moses/src/LanguageModelRemote.cpp
View File

@ -14,11 +14,6 @@ namespace Moses {
const Factor* LanguageModelRemote::BOS = NULL;
const Factor* LanguageModelRemote::EOS = (LanguageModelRemote::BOS + 1);
LanguageModelRemote::LanguageModelRemote(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
{
}
bool LanguageModelRemote::Load(const std::string &filePath
, FactorType factorType
, size_t nGramOrder)

2
moses/src/LanguageModelRemote.h
View File

@ -29,7 +29,7 @@ class LanguageModelRemote : public LanguageModelSingleFactor {
static const Factor* BOS;
static const Factor* EOS;
public:
LanguageModelRemote(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelRemote() {}
~LanguageModelRemote();
void ClearSentenceCache() { m_cache.tree.clear(); m_curId = 1000; }
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0, unsigned int* len = 0) const;

5
moses/src/LanguageModelSRI.cpp
View File

@ -35,9 +35,8 @@ using namespace std;
namespace Moses
{
LanguageModelSRI::LanguageModelSRI(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
, m_srilmVocab(0)
LanguageModelSRI::LanguageModelSRI()
: m_srilmVocab(0)
, m_srilmModel(0)
{
}

2
moses/src/LanguageModelSRI.h
View File

@ -50,7 +50,7 @@ protected:
VocabIndex GetLmID( const Factor *factor ) const;
public:
LanguageModelSRI(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelSRI();
~LanguageModelSRI();
bool Load(const std::string &filePath
, FactorType factorType

4
moses/src/LanguageModelSingleFactor.cpp
View File

@ -38,10 +38,6 @@ namespace Moses
// static variable init
LanguageModelSingleFactor::State LanguageModelSingleFactor::UnknownState=0;
LanguageModelSingleFactor::LanguageModelSingleFactor(bool registerScore, ScoreIndexManager &scoreIndexManager)
:LanguageModel(registerScore, scoreIndexManager)
{
}
LanguageModelSingleFactor::~LanguageModelSingleFactor() {}

2
moses/src/LanguageModelSingleFactor.h
View File

@ -38,7 +38,7 @@ protected:
const Factor *m_sentenceStart, *m_sentenceEnd;
FactorType m_factorType;
LanguageModelSingleFactor(bool registerScore, ScoreIndexManager &scoreIndexManager);
LanguageModelSingleFactor() {}
public:
static State UnknownState;

5
moses/src/LanguageModelSkip.h
View File

@ -45,10 +45,7 @@ public:
/** Constructor
* \param lmImpl SRI or IRST LM which this LM can use to load data
*/
LanguageModelSkip(LanguageModelSingleFactor *lmImpl
, bool registerScore
, ScoreIndexManager &scoreIndexManager)
: LanguageModelSingleFactor(registerScore, scoreIndexManager)
LanguageModelSkip(LanguageModelSingleFactor *lmImpl)
{
m_lmImpl = lmImpl;
}

4
moses/src/LexicalReordering.cpp
View File

@ -53,9 +53,7 @@ LexicalReordering::LexicalReordering(std::vector<FactorType>& f_factors,
exit(1);
}
// add ScoreProducer - don't do this before our object is set up
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
const_cast<StaticData&>(StaticData::Instance()).SetWeights(this, weights);
m_table = LexicalReorderingTable::LoadAvailable(filePath, m_factorsF, m_factorsE, std::vector<FactorType>());
}

2
moses/src/Makefile.am
View File

@ -78,7 +78,6 @@ libmoses_la_HEADERS = \
ReorderingConstraint.h \
ReorderingStack.h \
ScoreComponentCollection.h \
ScoreIndexManager.h \
ScoreProducer.h \
Search.h \
SearchCubePruning.h \
@ -207,7 +206,6 @@ libmoses_la_SOURCES = \
ReorderingConstraint.cpp \
ReorderingStack.cpp \
ScoreComponentCollection.cpp \
ScoreIndexManager.cpp \
ScoreProducer.cpp \
Search.cpp \
SearchCubePruning.cpp \

3
moses/src/Manager.cpp
View File

@ -564,8 +564,7 @@ void OutputSearchNode(long translationId, std::ostream &outputSearchGraphStream,
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
outputSearchGraphStream << " [ ";
StaticData::Instance().GetScoreIndexManager().PrintLabeledScores( outputSearchGraphStream, scoreBreakdown );
outputSearchGraphStream << " [ " << StaticData::Instance().GetAllWeights();
outputSearchGraphStream << " ]";
outputSearchGraphStream << " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl;

7
moses/src/OnlineCommand.cpp
View File

@ -109,11 +109,10 @@ void OnlineCommand::Execute() const
staticData.ReLoadParameter();
// check on weights
vector<float> weights = staticData.GetAllWeights();
const ScoreComponentCollection& weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
TRACE_ERR("The global weight vector looks like this:");
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
TRACE_ERR("The global weight vector looks like this: ");
TRACE_ERR(weights);
TRACE_ERR("\n");
}

2
moses/src/PhraseDictionary.cpp
View File

@ -64,8 +64,6 @@ PhraseDictionaryFeature::PhraseDictionaryFeature
m_alignmentsFile(alignmentsFile)
{
const StaticData& staticData = StaticData::Instance();
const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).AddScoreProducer(this);
//const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).InitFeatureNames();
if (implementation == Memory || implementation == SCFG || implementation == OnDisk || implementation == SuffixArray) {
m_useThreadSafePhraseDictionary = true;
} else {

2
moses/src/PhraseDictionaryMemory.h
View File

@ -63,7 +63,7 @@ public:
virtual void InitializeForInput(InputType const&)
{/* Don't do anything source specific here as this object is shared between threads.*/}
void GetChartRuleCollection(ChartTranslationOptionList &outColl
void GetChartRuleCollection(ChartTranslationOptionList &/*outColl*/
,InputType const& /*src*/
, WordsRange const& /*range*/
, bool /*adhereTableLimit*/

96
moses/src/ScoreComponentCollection.cpp
View File

@ -1,56 +1,36 @@
// $Id$
#include <vector>
#include "ScoreComponentCollection.h"
#include "StaticData.h"
using namespace std;
namespace Moses
{
ScoreComponentCollection::ScoreComponentCollection()
: m_scores(StaticData::Instance().GetTotalScoreComponents(), 0.0f)
, m_sim(&StaticData::Instance().GetScoreIndexManager())
{}
const ScoreIndexManager* ScoreComponentCollection::GetSim() {
return &(StaticData::Instance().GetScoreIndexManager());
}
float ScoreComponentCollection::GetWeightedScore() const
{
float ret = InnerProduct(StaticData::Instance().GetAllWeights());
return ret;
return m_scores.inner_product(StaticData::Instance().GetAllWeights().m_scores);
}
void ScoreComponentCollection::ZeroAllLM(const LMList& lmList)
{
for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind)
{
m_scores[m_sim->GetFeatureName(ind)] = 0;
}
for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) {
Assign(*i, 0);
}
}
void ScoreComponentCollection::PlusEqualsAllLM(const LMList& lmList, const ScoreComponentCollection& rhs)
{
for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind)
{
m_scores[m_sim->GetFeatureName(ind)] += rhs.m_scores[m_sim->GetFeatureName(ind)];
}
for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) {
PlusEquals(*i,rhs);
}
}
void ScoreComponentCollection::PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores)
{
assert(scores.size() == sp->GetNumScoreComponents());
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
for (std::vector<float>::const_iterator vi = scores.begin();
vi != scores.end(); ++vi)
{
const std::string &name = m_sim->GetFeatureName(i++);
m_scores[name] += *vi;
}
}
void ScoreComponentCollection::MultiplyEquals(float scalar)
{
@ -64,63 +44,9 @@ void ScoreComponentCollection::MultiplyEquals(const ScoreComponentCollection& rh
std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs)
{
os << "<<" << rhs.m_scores[rhs.m_sim->GetFeatureName(0)];
for (size_t i=1; i<rhs.m_scores.size(); i++)
os << ", " << rhs.m_scores[rhs.m_sim->GetFeatureName(i)];
os << "<<" << rhs.m_scores;
return os << ">>";
}
ScoreComponentCollection StaticData::GetAllWeightsScoreComponentCollection() const
{
Moses::ScoreComponentCollection ret;
const std::vector<const ScoreProducer*> &producers = m_scoreIndexManager.GetProducers();
std::vector<const ScoreProducer*>::const_iterator iter;
for (iter = producers.begin(); iter != producers.end(); ++iter)
{
const ScoreProducer *producer = *iter;
unsigned int bookId = producer->GetScoreBookkeepingID();
size_t startInd = m_scoreIndexManager.GetBeginIndex(bookId);
size_t endInd = m_scoreIndexManager.GetEndIndex(bookId);
std::vector<float> weights;
copy(m_allWeights.begin() + startInd, m_allWeights.begin() + endInd, inserter(weights,weights.end()));
ret.Assign(producer, weights);
}
return ret;
}
void StaticData::SetAllWeightsScoreComponentCollection(const ScoreComponentCollection &weightsScoreComponentCollection)
{
const std::vector<const ScoreProducer*> &producers = m_scoreIndexManager.GetProducers();
std::vector<const ScoreProducer*>::const_iterator iter;
for (iter = producers.begin(); iter != producers.end(); ++iter)
{
const ScoreProducer *producer = *iter;
unsigned int bookId = producer->GetScoreBookkeepingID();
size_t startInd = m_scoreIndexManager.GetBeginIndex(bookId);
size_t endInd = m_scoreIndexManager.GetEndIndex(bookId);
std::vector<float> weights = weightsScoreComponentCollection.GetScoresForProducer(producer);
size_t allInd = startInd;
for (size_t ind = 0; ind < weights.size(); ++ind)
{
m_allWeights[allInd] = weights[ind];
allInd++;
}
assert(allInd == endInd);
}
}
void ScoreComponentCollection::L1Normalise() {
m_scores /= m_scores.l1norm();
}

91
moses/src/ScoreComponentCollection.h
View File

@ -32,7 +32,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "LMList.h"
#include "ScoreProducer.h"
#include "ScoreIndexManager.h"
#include "FeatureVector.h"
#include "TypeDef.h"
#include "Util.h"
@ -62,11 +61,8 @@ namespace Moses
*/
class ScoreComponentCollection {
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
friend class ScoreIndexManager;
private:
const ScoreIndexManager* GetSim();
FVector m_scores;
const ScoreIndexManager* m_sim;
public:
//! Create a new score collection with all values set to 0.0
@ -75,11 +71,8 @@ public:
//! Clone a score collection
ScoreComponentCollection(const ScoreComponentCollection& rhs)
: m_scores(rhs.m_scores)
, m_sim(rhs.m_sim)
{}
inline size_t size() const { return m_scores.size(); }
float operator[](size_t x) const { return m_scores[m_sim->GetFeatureName(x)]; }
//! Set all values to 0.0
void ZeroAll()
@ -102,71 +95,58 @@ public:
m_scores -= rhs.m_scores;
}
//! Add scores from a single ScoreProducer only
//! The length of scores must be equal to the number of score components
//! produced by sp
void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores);
//! Add scores from a single ScoreProducer only
//! The length of scores must be equal to the number of score components
//! produced by sp
void PlusEquals(const ScoreProducer* sp, const ScoreComponentCollection& scores)
{
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
const size_t end = m_sim->GetEndIndex(sp->GetScoreBookkeepingID());
for (; i < end; ++i)
{
m_scores[m_sim->GetFeatureName(i)] += scores.m_scores[m_sim->GetFeatureName(i)];
}
const std::vector<FName>& names = sp->GetFeatureNames();
for (std::vector<FName>::const_iterator i = names.begin();
i != names.end(); ++i) {
m_scores[*i] += scores.m_scores[*i];
}
}
//! Add scores from a single ScoreProducer only
//! The length of scores must be equal to the number of score components
//! produced by sp
void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores)
{
const std::vector<FName>& names = sp->GetFeatureNames();
assert(names.size() == scores.size());
for (size_t i = 0; i < scores.size(); ++i) {
m_scores[names[i]] = scores[i];
}
}
//! Special version PlusEquals(ScoreProducer, vector<float>)
//! to add the score from a single ScoreProducer that produces
//! a single value
void PlusEquals(const ScoreProducer* sp, float score)
{
assert(1 == sp->GetNumScoreComponents());
const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
m_scores[m_sim->GetFeatureName(i)] += score;
m_scores[sp->GetFeatureNames()[0]] += score;
}
void Assign(const ScoreProducer* sp, const std::vector<float>& scores)
{
assert(scores.size() == sp->GetNumScoreComponents());
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
for (std::vector<float>::const_iterator vi = scores.begin();
vi != scores.end(); ++vi)
{
m_scores[m_sim->GetFeatureName(i++)] = *vi;
}
const std::vector<FName>& names = sp->GetFeatureNames();
for (size_t i = 0; i < scores.size(); ++i) {
m_scores[names[i]] = scores[i];
}
}
void Assign(const ScoreComponentCollection &copy)
{
m_scores = copy.m_scores;
}
//! Special version PlusEquals(ScoreProducer, vector<float>)
//! to add the score from a single ScoreProducer that produces
//! a single value
void Assign(const ScoreProducer* sp, float score)
{
assert(1 == sp->GetNumScoreComponents());
const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
m_scores[m_sim->GetFeatureName(i)] = score;
m_scores[sp->GetFeatureNames()[0]] = score;
}
void Assign(size_t index, float score)
{
m_scores[m_sim->GetFeatureName(index)] = score;
}
//! Used to find the weighted total of scores. rhs should contain a vector of weights
//! of the same length as the number of scores.
float InnerProduct(const std::vector<float>& rhs) const
{
return m_scores.inner_product(rhs);
}
float InnerProduct(const ScoreComponentCollection& rhs) const
{
@ -183,28 +163,20 @@ public:
//! return a vector of all the scores associated with a certain ScoreProducer
std::vector<float> GetScoresForProducer(const ScoreProducer* sp) const
{
size_t id = sp->GetScoreBookkeepingID();
const size_t begin = m_sim->GetBeginIndex(id);
const size_t end = m_sim->GetEndIndex(id);
std::vector<float> res(end-begin);
size_t j = 0;
for (size_t i = begin; i < end; i++) {
res[j++] = m_scores[m_sim->GetFeatureName(i)];
}
return res;
std::vector<float> res(sp->GetNumScoreComponents());
const std::vector<FName>& names = sp->GetFeatureNames();
for (size_t i = 0; i < names.size(); ++i) {
res[i] = m_scores[names[i]];
}
return res;
}
//! if a ScoreProducer produces a single score (for example, a language model score)
//! this will return it. If not, this method will throw
float GetScoreForProducer(const ScoreProducer* sp) const
{
size_t id = sp->GetScoreBookkeepingID();
const size_t begin = m_sim->GetBeginIndex(id);
#ifndef NDEBUG
const size_t end = m_sim->GetEndIndex(id);
assert(end-begin == 1);
#endif
return m_scores[m_sim->GetFeatureName(begin)];
assert(sp->GetNumScoreComponents() == 1);
return m_scores[sp->GetFeatureNames()[0]];
}
float GetWeightedScore() const;
@ -227,7 +199,6 @@ public:
template<class Archive>
void load(Archive &ar, const unsigned int version) {
ar >> m_scores;
m_sim = GetSim();
}

148
moses/src/ScoreIndexManager.cpp
View File

@ -1,148 +0,0 @@
// $Id$
#include <iostream>
#include <iomanip>
#include <string>
#include <cstdio>
#include <cassert>
#include "Util.h"
#include "StaticData.h"
#include "ScoreIndexManager.h"
#include "ScoreProducer.h"
#include "ScoreComponentCollection.h" // debugging
namespace Moses
{
using namespace std;
void ScoreIndexManager::AddScoreProducer(const ScoreProducer* sp)
{
// Producers must be inserted in the order they are created
const_cast<ScoreProducer*>(sp)->CreateScoreBookkeepingID();
assert(m_begins.size() == (sp->GetScoreBookkeepingID()));
m_producers.push_back(sp);
m_begins.push_back(m_last);
size_t numScoreCompsProduced = sp->GetNumScoreComponents();
assert(numScoreCompsProduced > 0);
m_last += numScoreCompsProduced;
m_ends.push_back(m_last);
m_featureNamesDirty = true;
InitFeatureNames();
/*VERBOSE(1,"Added ScoreProducer(" << sp->GetScoreBookkeepingID()
<< " " << sp->GetScoreProducerDescription()
<< ") index=" << m_begins.back() << "-" << m_ends.back()-1 << std::endl);
*/
}
void ScoreIndexManager::PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scores) const
{
std::vector<float> weights(scores.m_scores.size(), 1.0f);
PrintLabeledWeightedScores(os, scores, weights);
}
void ScoreIndexManager::PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scores, const std::vector<float>& weights) const
{
assert(m_featureShortNames.size() == weights.size());
string lastName = "";
for (size_t i = 0; i < m_featureShortNames.size(); ++i)
{
if (i>0)
{
os << " ";
}
if (lastName != m_featureShortNames[i])
{
os << m_featureShortNames[i] << ": ";
lastName = m_featureShortNames[i];
}
os << weights[i] * scores[i];
}
}
void ScoreIndexManager::InitFeatureNames() {
if (!m_featureNamesDirty)
return;
m_featureNames.clear();
m_featureIndexes.clear();
m_featureShortNames.clear();
size_t globalIndex = 0;
vector<const ScoreProducer *>::const_iterator it;
for (it = m_producers.begin(); it != m_producers.end(); ++it) {
size_t scoreCount = (*it)->GetNumScoreComponents();
for (size_t i = 0; i < scoreCount; ++i) {
ostringstream oStream;
oStream << /* (*it)->GetScoreProducerDescription() << "_" << */ globalIndex;
m_featureNames.push_back(oStream.str());
m_featureIndexes[oStream.str()] = globalIndex;
++globalIndex;
}
}
m_featureNamesDirty = false;
}
#ifdef HAVE_PROTOBUF
void ScoreIndexManager::SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const {
for (size_t i = 0; i < m_featureNames.size(); ++i) {
hg->add_feature_names(m_featureNames[i]);
}
}
#endif
void ScoreIndexManager::InitWeightVectorFromFile(const std::string& fnam, vector<float>* m_allWeights) const {
assert(m_allWeights->size() == m_featureNames.size());
ifstream in(fnam.c_str());
assert(in.good());
char buf[2000];
map<string, double> name2val;
while (!in.eof()) {
in.getline(buf, 2000);
if (strlen(buf) == 0) continue;
if (buf[0] == '#') continue;
istringstream is(buf);
string fname;
double val;
is >> fname >> val;
map<string, double>::iterator i = name2val.find(fname);
assert(i == name2val.end()); // duplicate weight name
name2val[fname] = val;
}
assert(m_allWeights->size() == m_featureNames.size());
for (size_t i = 0; i < m_featureNames.size(); ++i) {
map<string, double>::iterator iter = name2val.find(m_featureNames[i]);
if (iter == name2val.end()) {
cerr << "No weight found found for feature: " << m_featureNames[i] << endl;
abort();
}
(*m_allWeights)[i] = iter->second;
}
}
std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim)
{
for (size_t i = 0; i < sim.m_featureNames.size(); ++i) {
os << sim.m_featureNames[i] << endl;
}
os << endl;
return os;
}
const std::string &ScoreIndexManager::GetFeatureName(size_t fIndex) const
{
#ifndef NDEBUG
if (m_featureNames.size() <= fIndex) {
printf("%s: %i ScoreIndexManager::GetFeatureName(m_producers.size() = %zd, m_featureNames.size() = %zd, fIndex = %zd)\n", __FILE__, __LINE__, m_producers.size(), m_featureNames.size(), fIndex);
}
#endif
assert(m_featureNames.size() > fIndex);
return m_featureNames[fIndex];
}
//! get the index of a feature by name
size_t ScoreIndexManager::GetFeatureIndex(const std::string& fName) const
{
return m_featureIndexes.find(fName)->second;
}
}

75
moses/src/ScoreIndexManager.h
View File

@ -1,75 +0,0 @@
// $Id$
#ifndef moses_ScoreIndexManager_h
#define moses_ScoreIndexManager_h
#include <iostream>
#include <vector>
#if HAVE_CONFIG_H
#include "config.h"
#endif
#ifdef HAVE_PROTOBUF
#include "hypergraph.pb.h"
#endif
namespace Moses
{
class ScoreProducer;
class ScoreComponentCollection; // debugging only
class StatefulFeatureFunction;
class StatelessFeatureFunction;
/** Keep track of scores and score producers. Each score producer is reserved contiguous set of slots
* to put their score components. All the score components are arranged in a vector with no gaps.
* Only 1 ScoreIndexManager object should be instantiated
*/
class ScoreIndexManager
{
friend std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim);
public:
ScoreIndexManager() : m_last(0), m_featureNamesDirty(true) {}
//! new score producer to manage. Producers must be inserted in the order they are created
void AddScoreProducer(const ScoreProducer* producer);
//! starting score index for a particular score producer with scoreBookkeepingID
size_t GetBeginIndex(size_t scoreBookkeepingID) const { return m_begins[scoreBookkeepingID]; }
//! end score index for a particular score producer with scoreBookkeepingID
size_t GetEndIndex(size_t scoreBookkeepingID) const { return m_ends[scoreBookkeepingID]; }
//! sum of all score components from every score producer
size_t GetTotalNumberOfScores() const { return m_last; }
//! print unweighted scores of each ScoreManager to stream os
void PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scc) const;
//! print weighted scores of each ScoreManager to stream os
void PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scc, const std::vector<float>& weights) const;
//! get the name of a feature by index
const std::string &GetFeatureName(size_t fIndex) const;
//! get the index of a feature by name
size_t GetFeatureIndex(const std::string& fName) const;
const std::vector<const ScoreProducer*> &GetProducers() const
{ return m_producers; }
#ifdef HAVE_PROTOBUF
void SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const;
#endif
void InitWeightVectorFromFile(const std::string& fnam, std::vector<float>* m_allWeights) const;
private:
ScoreIndexManager(const ScoreIndexManager&); // don't implement
void InitFeatureNames();
std::vector<size_t> m_begins;
std::vector<size_t> m_ends;
std::vector<const ScoreProducer*> m_producers; /**< all the score producers in this run */
std::vector<std::string> m_featureNames;
std::map<std::string, size_t> m_featureIndexes;
std::vector<std::string> m_featureShortNames;
size_t m_last;
bool m_featureNamesDirty;
};
}
#endif

25
moses/src/ScoreProducer.cpp
View File

@ -1,20 +1,31 @@
// $Id$
#include <iostream>
#include <typeinfo>
#include <sstream>
#include "ScoreProducer.h"
#include "StaticData.h"
#include "ScoreIndexManager.h"
using namespace std;
namespace Moses
{
unsigned int ScoreProducer::s_globalScoreBookkeepingIdCounter(0);
ScoreProducer::~ScoreProducer() {}
ScoreProducer::ScoreProducer()
{
m_scoreBookkeepingId = UNASSIGNED;
const vector<FName>& ScoreProducer::GetFeatureNames() const {
if (m_names.size() != GetNumScoreComponents()) {
const string& desc = GetScoreProducerDescription();
if (GetNumScoreComponents() == 1) {
m_names.push_back(FName(desc));
} else {
for (size_t i = 1; i <= GetNumScoreComponents(); ++i) {
ostringstream id;
id << i;
m_names.push_back(FName(desc,id.str()));
}
}
}
return m_names;
}
}

27
moses/src/ScoreProducer.h
View File

@ -5,44 +5,35 @@
#include <string>
#include <limits>
#include <vector>
#include "FeatureVector.h"
namespace Moses
{
class Hypothesis;
class ScoreComponentCollection;
class ScoreIndexManager;
class FFState;
/** to keep track of the various things that can produce a score,
* we use this evil implementation-inheritance to give them each
* a unique, sequential (read: good for vector indices) ID
*
/*
* @note do not confuse this with a producer/consumer pattern.
* this is not a producer in that sense.
*/
class ScoreProducer
{
private:
static unsigned int s_globalScoreBookkeepingIdCounter;
unsigned int m_scoreBookkeepingId;
mutable std::vector<FName> m_names; //for features with fixed number of values
ScoreProducer(const ScoreProducer&); // don't implement
#define UNASSIGNED std::numeric_limits<unsigned int>::max()
protected:
// it would be nice to force registration here, but some Producer objects
// are constructed before they know how many scores they have
ScoreProducer();
ScoreProducer() {}
virtual ~ScoreProducer();
public:
//! contiguous id
unsigned int GetScoreBookkeepingID() const { return m_scoreBookkeepingId; }
void CreateScoreBookkeepingID() { m_scoreBookkeepingId = s_globalScoreBookkeepingIdCounter++;}
//! returns the number of scores that a subclass produces.
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
//! will cause an error if this producer does not have a fixed number
//! of scores (eg sparse features)
virtual size_t GetNumScoreComponents() const = 0;
//! returns a string description of this producer
@ -54,6 +45,8 @@ public:
//! returns the number of scores gathered from the input (0 by default)
virtual size_t GetNumInputScores() const { return 0; };
const std::vector<FName>& GetFeatureNames() const;
virtual bool IsStateless() const = 0;
};

92
moses/src/StaticData.cpp
View File

@ -300,14 +300,14 @@ bool StaticData::LoadData(Parameter *parameter)
// word penalties
for (size_t i = 0; i < m_parameter->GetParam("weight-w").size(); ++i) {
float weightWordPenalty = Scan<float>( m_parameter->GetParam("weight-w")[i] );
m_wordPenaltyProducers.push_back(new WordPenaltyProducer(m_scoreIndexManager));
m_allWeights.push_back(weightWordPenalty);
m_wordPenaltyProducers.push_back(new WordPenaltyProducer());
SetWeight(m_wordPenaltyProducers.back(), weightWordPenalty);
}
float weightUnknownWord = (m_parameter->GetParam("weight-u").size() > 0) ? Scan<float>(m_parameter->GetParam("weight-u")[0]) : 1;
m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(m_scoreIndexManager);
m_allWeights.push_back(weightUnknownWord);
m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer();
SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord);
// reordering constraints
m_maxDistortion = (m_parameter->GetParam("distortion-limit").size() > 0) ?
@ -547,8 +547,6 @@ bool StaticData::LoadData(Parameter *parameter)
}
//m_scoreIndexManager.InitFeatureNames();
return true;
}
@ -574,6 +572,16 @@ void StaticData::SetBooleanParameter( bool *parameter, string parameterName, boo
}
}
void StaticData::SetWeight(const ScoreProducer* sp, float weight)
{
m_allWeights.Assign(sp,weight);
}
void StaticData::SetWeights(const ScoreProducer* sp, const std::vector<float>& weights)
{
m_allWeights.Assign(sp,weights);
}
StaticData::~StaticData()
{
RemoveAllInColl(m_phraseDictionary);
@ -706,7 +714,8 @@ bool StaticData::LoadGlobalLexicalModel()
}
vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],",");
vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],",");
m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], weight[i], inputFactors, outputFactors ) );
m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], inputFactors, outputFactors ) );
SetWeight(m_globalLexicalModels.back(),weight[i]);
}
return true;
}
@ -718,11 +727,6 @@ bool StaticData::LoadLanguageModels()
// weights
vector<float> weightAll = Scan<float>(m_parameter->GetParam("weight-l"));
for (size_t i = 0 ; i < weightAll.size() ; i++)
{
m_allWeights.push_back(weightAll[i]);
}
// dictionary upper-bounds fo all IRST LMs
vector<int> LMdub = Scan<int>(m_parameter->GetParam("lmodel-dub"));
if (m_parameter->GetParam("lmodel-dub").size() == 0){
@ -739,7 +743,7 @@ bool StaticData::LoadLanguageModels()
{
LanguageModel* lm = NULL;
if (languageModelsLoaded.find(lmVector[i]) != languageModelsLoaded.end()) {
lm = new LanguageModelDelegate(true, m_scoreIndexManager,
lm = new LanguageModelDelegate(
static_cast<LanguageModelSingleFactor*>(languageModelsLoaded[lmVector[i]]));
} else {
vector<string> token = Tokenize(lmVector[i]);
@ -774,7 +778,6 @@ bool StaticData::LoadLanguageModels()
, factorTypes
, nGramOrder
, languageModelFile
, m_scoreIndexManager
, LMdub[i]);
if (lm == NULL)
{
@ -785,9 +788,9 @@ bool StaticData::LoadLanguageModels()
}
m_languageModel.Add(lm);
SetWeight(lm,weightAll[i]);
}
}
//m_scoreIndexManager.InitFeatureNames();
// flag indicating that language models were loaded,
// since phrase table loading requires their presence
m_fLMsLoaded = true;
@ -832,17 +835,19 @@ bool StaticData::LoadGenerationTables()
VERBOSE(1, filePath << endl);
m_generationDictionary.push_back(new GenerationDictionary(numFeatures, m_scoreIndexManager, input,output));
m_generationDictionary.push_back(new GenerationDictionary(numFeatures, input,output));
assert(m_generationDictionary.back() && "could not create GenerationDictionary");
if (!m_generationDictionary.back()->Load(filePath, Output))
{
delete m_generationDictionary.back();
return false;
}
vector<float> gdWeights;
for(size_t i = 0; i < numFeatures; i++) {
assert(currWeightNum < weight.size());
m_allWeights.push_back(weight[currWeightNum++]);
gdWeights.push_back(weight[currWeightNum++]);
}
SetWeights(m_generationDictionary.back(), gdWeights);
}
if (currWeightNum != weight.size()) {
TRACE_ERR( " [WARNING] config file has " << weight.size() << " generation weights listed, but the configuration for generation files indicates there should be " << currWeightNum << "!\n");
@ -983,27 +988,27 @@ bool StaticData::LoadPhraseTables()
assert(numScoreComponent==weight.size());
std::copy(weight.begin(),weight.end(),std::back_inserter(m_allWeights));
//This is needed for regression testing, but the phrase table
//might not really be loading here
IFVERBOSE(1)
PrintUserTime(string("Start loading PhraseTable ") + filePath);
VERBOSE(1,"filePath: " << filePath <<endl);
PhraseDictionaryFeature* pdf = new PhraseDictionaryFeature(
implementation
, numScoreComponent
, (currDict==0 ? m_numInputScores : 0)
, input
, output
, filePath
, weight
, maxTargetPhrase[index]
, targetPath, alignmentsFile);
PhraseDictionaryFeature* pdf = new PhraseDictionaryFeature(
implementation
, numScoreComponent
, (currDict==0 ? m_numInputScores : 0)
, input
, output
, filePath
, weight
, maxTargetPhrase[index]
, targetPath, alignmentsFile);
m_phraseDictionary.push_back(pdf);
m_phraseDictionary.push_back(pdf);
SetWeights(m_phraseDictionary.back(),weight);
@ -1094,8 +1099,8 @@ void StaticData::LoadPhraseBasedParameters()
}
for (size_t i = 0; i < distortionWeightCount; ++i) {
float weightDistortion = Scan<float>(distortionWeights[i]);
m_distortionScoreProducers.push_back(new DistortionScoreProducer(m_scoreIndexManager));
m_allWeights.push_back(weightDistortion);
m_distortionScoreProducers.push_back(new DistortionScoreProducer());
SetWeight(m_distortionScoreProducers.back(), weightDistortion);
}
}
@ -1205,7 +1210,7 @@ bool StaticData::LoadReferences() {
}
float bleuWeight = Scan<float>(bleuWeightStr[0]);
m_bleuScoreFeature = new BleuScoreFeature();
m_allWeights.push_back(bleuWeight);
SetWeight(m_bleuScoreFeature, bleuWeight);
vector<vector<string> > references(referenceFiles.size());
for (size_t i =0; i < referenceFiles.size(); ++i) {
@ -1245,7 +1250,7 @@ bool StaticData::LoadTargetBigramFeature()
UserMessage::Add("Wrong number of arguments for target-bigram-feature-wordlist (expected 1).");
return false;
}
m_targetBigramFeature = new TargetBigramFeature(m_scoreIndexManager);
m_targetBigramFeature = new TargetBigramFeature();
if (!m_targetBigramFeature->Load(wordFile[0])) {
UserMessage::Add("Unable to load word list from file " + wordFile[0]);
return false;
@ -1254,19 +1259,6 @@ bool StaticData::LoadTargetBigramFeature()
return true;
}
void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights)
{
const size_t id = sp->GetScoreBookkeepingID();
const size_t begin = m_scoreIndexManager.GetBeginIndex(id);
const size_t end = m_scoreIndexManager.GetEndIndex(id);
assert(end - begin == weights.size());
if (m_allWeights.size() < end)
m_allWeights.resize(end);
std::vector<float>::const_iterator weightIter = weights.begin();
for (size_t i = begin; i < end; i++)
m_allWeights[i] = *weightIter++;
}
const TranslationOptionList* StaticData::FindTransOptListInCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase) const
{
std::pair<size_t, Phrase> key(decodeGraph.GetPosition(), sourcePhrase);
@ -1365,7 +1357,7 @@ void StaticData::ReLoadParameter()
{
continue;
}
SetWeightsForScoreProducer(*iterSP, Weights);
SetWeights(*iterSP, Weights);
}
// std::cerr << "There are " << m_phraseDictionary.size() << " m_phraseDictionaryfeatures" << std::endl;
@ -1390,7 +1382,7 @@ void StaticData::ReLoadParameter()
// std::cerr << tmp_weights << std::endl;
SetWeightsForScoreProducer(&phraseDictionaryFeature, tmp_weights);
SetWeights(&phraseDictionaryFeature, tmp_weights);
}
}

64
moses/src/StaticData.h
View File

@ -35,7 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#endif
#include "TypeDef.h"
#include "ScoreIndexManager.h"
#include "FactorCollection.h"
#include "Parameter.h"
#include "LanguageModel.h"
@ -82,8 +81,7 @@ protected:
Parameter *m_parameter;
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
LMList m_languageModel;
ScoreIndexManager m_scoreIndexManager;
std::vector<float> m_allWeights;
ScoreComponentCollection m_allWeights;
std::vector<LexicalReordering*> m_reorderModels;
std::vector<GlobalLexicalModel*> m_globalLexicalModels;
std::vector<DecodeGraph*> m_decodeGraphs;
@ -206,6 +204,8 @@ protected:
StaticData();
void LoadPhraseBasedParameters();
void LoadChartDecodingParameters();
void LoadNonTerminals();
@ -369,27 +369,18 @@ public:
{
return m_translationOptionThreshold;
}
//! returns the total number of score components across all types, all factors
size_t GetTotalScoreComponents() const
{
return m_scoreIndexManager.GetTotalNumberOfScores();
}
const ScoreIndexManager& GetScoreIndexManager() const
{
return m_scoreIndexManager;
}
const TranslationSystem& GetTranslationSystem(std::string id) const {
std::map<std::string, TranslationSystem>::const_iterator iter =
m_translationSystems.find(id);
VERBOSE(2, "Looking for translation system id " << id << std::endl);
if (iter == m_translationSystems.end()) {
VERBOSE(1, "Translation system not found " << id << std::endl);
throw std::runtime_error("Unknown translation system id");
} else {
return iter->second;
}
const TranslationSystem& GetTranslationSystem(std::string id) const {
std::map<std::string, TranslationSystem>::const_iterator iter =
m_translationSystems.find(id);
VERBOSE(2, "Looking for translation system id " << id << std::endl);
if (iter == m_translationSystems.end()) {
VERBOSE(1, "Translation system not found " << id << std::endl);
throw std::runtime_error("Unknown translation system id");
} else {
return iter->second;
}
}
size_t GetVerboseLevel() const
{
return m_verboseLevel;
@ -457,16 +448,39 @@ public:
{ return m_outputWordGraph; }
//! Sets the global score vector weights for a given ScoreProducer.
void SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights);
InputTypeEnum GetInputType() const {return m_inputType;}
SearchAlgorithm GetSearchAlgorithm() const {return m_searchAlgorithm;}
size_t GetNumInputScores() const {return m_numInputScores;}
const std::vector<float>& GetAllWeights() const
const ScoreComponentCollection& GetAllWeights() const
{
return m_allWeights;
}
ScoreComponentCollection GetAllWeightsScoreComponentCollection() const;
void SetAllWeights(const ScoreComponentCollection& weights)
{
m_allWeights = weights;
}
//Weight for a single-valued feature
float GetWeight(const ScoreProducer* sp) const
{
return m_allWeights.GetScoreForProducer(sp);
}
//Weight for a single-valued feature
void SetWeight(const ScoreProducer* sp, float weight) ;
//Weights for feature with fixed number of values
std::vector<float> GetWeights(const ScoreProducer* sp) const
{
return m_allWeights.GetScoresForProducer(sp);
}
//Weights for feature with fixed number of values
void SetWeights(const ScoreProducer* sp, const std::vector<float>& weights);
bool UseAlignmentInfo() const { return m_UseAlignmentInfo;}
void UseAlignmentInfo(bool a){ m_UseAlignmentInfo=a; };

9
moses/src/TargetBigramFeature.cpp
View File

@ -9,11 +9,6 @@ namespace Moses {
using namespace std;
TargetBigramFeature::TargetBigramFeature(ScoreIndexManager &scoreIndexManager)
{
scoreIndexManager.AddScoreProducer(this);
}
bool TargetBigramFeature::Load(const std::string &filePath)
{
InputFileStream inFile(filePath);
@ -75,7 +70,9 @@ FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
map<string,size_t>::const_iterator first, second;
if ((first = m_wordMap.find(words[i-1])) != m_wordMap.end() &&
(second = m_wordMap.find(words[i])) != m_wordMap.end()) {
accumulator->Assign(first->second * second->second, 1);
cerr << "FIXME" << endl;
assert(0);
//accumulator->Assign(first->second * second->second, 1);
}
}

2
moses/src/TargetBigramFeature.h
View File

@ -13,7 +13,7 @@ namespace Moses
*/
class TargetBigramFeature : public StatefulFeatureFunction {
public:
TargetBigramFeature(ScoreIndexManager &scoreIndexManager);
TargetBigramFeature(){}
bool Load(const std::string &filePath);

20
moses/src/TargetPhrase.cpp
View File

@ -26,7 +26,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "GenerationDictionary.h"
#include "LanguageModel.h"
#include "StaticData.h"
#include "ScoreIndexManager.h"
#include "LMList.h"
#include "ScoreComponentCollection.h"
#include "Util.h"
@ -93,17 +92,8 @@ void TargetPhrase::SetScore(float score)
const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT);
const ScoreProducer* prod = system.GetPhraseDictionaries()[0];
//get the weight list
unsigned int id = prod->GetScoreBookkeepingID();
const vector<float> &allWeights = StaticData::Instance().GetAllWeights();
vector<float> weights = StaticData::Instance().GetWeights(prod);
size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id);
size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id);
vector<float> weights;
std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights));
//find out how many items are in the score vector for this producer
size_t numScores = prod->GetNumScoreComponents();
@ -125,13 +115,7 @@ void TargetPhrase::SetScore(const TranslationSystem* system, const Scores &score
const ScoreProducer* prod = system->GetPhraseDictionaries()[0];
//get the weight list
unsigned int id = prod->GetScoreBookkeepingID();
const vector<float> &allWeights = StaticData::Instance().GetAllWeights();
size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id);
size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id);
vector<float> weights;
std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights));
vector<float> weights = StaticData::Instance().GetWeights(prod);
//expand the input weight vector
assert(scoreVector.size() <= prod->GetNumScoreComponents());

12
moses/src/TranslationSystem.cpp
View File

@ -153,22 +153,16 @@ namespace Moses {
}
float TranslationSystem::GetWeightWordPenalty() const {
//const ScoreComponentCollection weights = StaticData::Instance().GetAllWeights();
size_t wpIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(m_wpProducer->GetScoreBookkeepingID());
return StaticData::Instance().GetAllWeights()[wpIndex];
return StaticData::Instance().GetWeight(m_wpProducer);
}
float TranslationSystem::GetWeightUnknownWordPenalty() const {
size_t uwpIndex = StaticData::Instance().GetScoreIndexManager().
GetBeginIndex(m_unknownWpProducer->GetScoreBookkeepingID());
return StaticData::Instance().GetAllWeights()[uwpIndex];
return StaticData::Instance().GetWeight(m_unknownWpProducer);
}
float TranslationSystem::GetWeightDistortion() const {
assert(m_distortionScoreProducer);
size_t distIndex = StaticData::Instance().GetScoreIndexManager().
GetBeginIndex(m_distortionScoreProducer->GetScoreBookkeepingID());
return StaticData::Instance().GetAllWeights()[distIndex];
return StaticData::Instance().GetWeight(m_distortionScoreProducer);
}
};