mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Goodbye ScoreIndexManager.
Compiles ok, but haven't dared to run regression yet. git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mira-mtm5@3608 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
82dba81917
commit
7e72ceea22
@ -175,12 +175,12 @@ namespace Mira {
|
||||
}
|
||||
|
||||
ScoreComponentCollection MosesDecoder::getWeights() {
|
||||
return StaticData::Instance().GetAllWeightsScoreComponentCollection();
|
||||
return StaticData::Instance().GetAllWeights();
|
||||
}
|
||||
|
||||
void MosesDecoder::setWeights(const ScoreComponentCollection& weights) {
|
||||
cerr << "New weights: " << weights << endl;
|
||||
StaticData::InstanceNonConst().SetAllWeightsScoreComponentCollection(weights);
|
||||
StaticData::InstanceNonConst().SetAllWeights(weights);
|
||||
}
|
||||
|
||||
void MosesDecoder::updateHistory(const vector<const Word*>& words) {
|
||||
|
@ -115,19 +115,12 @@ int main(int argc, char* argv[])
|
||||
IOWrapper *ioWrapper = GetIODevice(staticData);
|
||||
|
||||
// check on weights
|
||||
vector<float> weights = staticData.GetAllWeights();
|
||||
const ScoreComponentCollection& weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
|
||||
TRACE_ERR("The global weight vector looks like this:");
|
||||
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
|
||||
TRACE_ERR("The global weight vector looks like this: ");
|
||||
TRACE_ERR(weights);
|
||||
TRACE_ERR("\n");
|
||||
}
|
||||
// every score must have a weight! check that here:
|
||||
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
|
||||
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
if (ioWrapper == NULL)
|
||||
return EXIT_FAILURE;
|
||||
|
||||
|
@ -274,8 +274,8 @@ void Hypothesis::CalcLMScore()
|
||||
{
|
||||
if (targetPhrasePos == 0 && numTargetTerminals >= lmList.GetMaxNGramOrder() - 1)
|
||||
{ // get from other prev hypo. faster
|
||||
m_lmPrefix.Assign(prevHypo->m_lmPrefix);
|
||||
m_lmNGram.Assign(prevHypo->m_lmNGram);
|
||||
m_lmPrefix = prevHypo->m_lmPrefix;
|
||||
m_lmNGram = prevHypo->m_lmNGram;
|
||||
}
|
||||
else
|
||||
{ // calc
|
||||
|
@ -345,18 +345,12 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
|
||||
// check on weights
|
||||
vector<float> weights = staticData.GetAllWeights();
|
||||
const ScoreComponentCollection& weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
|
||||
TRACE_ERR("The global weight vector looks like this:");
|
||||
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
|
||||
TRACE_ERR("The global weight vector looks like this: ");
|
||||
TRACE_ERR(weights);
|
||||
TRACE_ERR("\n");
|
||||
}
|
||||
// every score must have a weight! check that here:
|
||||
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
|
||||
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
InputType* source = NULL;
|
||||
|
@ -110,7 +110,8 @@ namespace TranslationAnalysis {
|
||||
}
|
||||
}
|
||||
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
|
||||
StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights());
|
||||
os << translationPath.back()->GetScoreBreakdown();
|
||||
os << " weighted(TODO)";
|
||||
os << std::endl;
|
||||
}
|
||||
|
||||
|
@ -69,11 +69,7 @@ BleuScoreFeature::BleuScoreFeature():
|
||||
m_count_history(BleuScoreState::bleu_order),
|
||||
m_match_history(BleuScoreState::bleu_order),
|
||||
m_target_length_history(0),
|
||||
m_ref_length_history(0)
|
||||
{
|
||||
const_cast<ScoreIndexManager&>
|
||||
(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
|
||||
}
|
||||
m_ref_length_history(0) {}
|
||||
|
||||
void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs)
|
||||
{
|
||||
|
@ -37,10 +37,6 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in
|
||||
NOT_FOUND);
|
||||
}
|
||||
|
||||
DistortionScoreProducer::DistortionScoreProducer(ScoreIndexManager &scoreIndexManager)
|
||||
{
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
|
||||
size_t DistortionScoreProducer::GetNumScoreComponents() const
|
||||
{
|
||||
@ -104,11 +100,6 @@ FFState* DistortionScoreProducer::Evaluate(
|
||||
}
|
||||
|
||||
|
||||
WordPenaltyProducer::WordPenaltyProducer(ScoreIndexManager &scoreIndexManager)
|
||||
{
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
|
||||
size_t WordPenaltyProducer::GetNumScoreComponents() const
|
||||
{
|
||||
return 1;
|
||||
@ -131,11 +122,6 @@ void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollect
|
||||
out->PlusEquals(this, -static_cast<float>(tp.GetSize()));
|
||||
}
|
||||
|
||||
UnknownWordPenaltyProducer::UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager)
|
||||
{
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
|
||||
size_t UnknownWordPenaltyProducer::GetNumScoreComponents() const
|
||||
{
|
||||
return 1;
|
||||
|
@ -14,7 +14,7 @@ class WordsRange;
|
||||
*/
|
||||
class DistortionScoreProducer : public StatefulFeatureFunction {
|
||||
public:
|
||||
DistortionScoreProducer(ScoreIndexManager &scoreIndexManager);
|
||||
DistortionScoreProducer() {}
|
||||
|
||||
float CalculateDistortionScore(const Hypothesis& hypo,
|
||||
const WordsRange &prev, const WordsRange &curr, const int FirstGapPosition) const;
|
||||
@ -38,7 +38,7 @@ public:
|
||||
*/
|
||||
class WordPenaltyProducer : public StatelessFeatureFunction {
|
||||
public:
|
||||
WordPenaltyProducer(ScoreIndexManager &scoreIndexManager);
|
||||
WordPenaltyProducer() {}
|
||||
|
||||
size_t GetNumScoreComponents() const;
|
||||
std::string GetScoreProducerDescription() const;
|
||||
@ -54,7 +54,7 @@ public:
|
||||
/** unknown word penalty */
|
||||
class UnknownWordPenaltyProducer : public StatelessFeatureFunction {
|
||||
public:
|
||||
UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager);
|
||||
UnknownWordPenaltyProducer() {}
|
||||
|
||||
size_t GetNumScoreComponents() const;
|
||||
std::string GetScoreProducerDescription() const;
|
||||
|
@ -26,8 +26,6 @@
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "StaticData.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
#include "FeatureVector.h"
|
||||
|
||||
using namespace std;
|
||||
@ -76,8 +74,7 @@ namespace Moses {
|
||||
return ! (*this == rhs);
|
||||
}
|
||||
|
||||
FVector::FVector(size_t size, FValue defaultValue)
|
||||
:m_size(size)
|
||||
FVector::FVector( FValue defaultValue)
|
||||
{
|
||||
m_features[DEFAULT_NAME] = defaultValue;
|
||||
}
|
||||
@ -382,31 +379,6 @@ namespace Moses {
|
||||
return product;
|
||||
}
|
||||
|
||||
FValue FVector::inner_product(const std::vector<FValue>& rhs) const {
|
||||
assert(size() == rhs.size());
|
||||
const ScoreIndexManager &sim = StaticData::Instance().GetScoreIndexManager();
|
||||
FValue product = 0;
|
||||
if (get(DEFAULT_NAME) == 0) {
|
||||
for (const_iterator i = cbegin(); i != cend(); i++) {
|
||||
const FName &fname = i->first;
|
||||
if (fname != DEFAULT_NAME) {
|
||||
const std::string &name = fname.name();
|
||||
size_t index = sim.GetFeatureIndex(name);
|
||||
assert(rhs.size() > index);
|
||||
|
||||
FValue rv = rhs[ index ];
|
||||
product += i->second * rv;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i < rhs.size(); i++) {
|
||||
product += rhs[i] * get(sim.GetFeatureName(i));
|
||||
}
|
||||
}
|
||||
return product;
|
||||
}
|
||||
|
||||
|
||||
const FVector operator+(const FVector& lhs, const FVector& rhs) {
|
||||
return FVector(lhs) += rhs;
|
||||
}
|
||||
|
@ -67,6 +67,7 @@ namespace Moses {
|
||||
//const std::string& root() const {return m_root;}
|
||||
|
||||
size_t hash() const;
|
||||
|
||||
|
||||
bool operator==(const FName& rhs) const ;
|
||||
bool operator!=(const FName& rhs) const ;
|
||||
@ -108,12 +109,8 @@ namespace Moses {
|
||||
class FVector
|
||||
{
|
||||
public:
|
||||
// do not implement
|
||||
FVector();
|
||||
bool operator=(const FName& rhs);
|
||||
|
||||
/** Empty feature vector, possibly with default value */
|
||||
FVector(size_t size, FValue defaultValue = DEFAULT);
|
||||
FVector(FValue defaultValue = DEFAULT);
|
||||
|
||||
typedef boost::unordered_map<FName,FValue,FNameHash, FNameEquals> FNVmap;
|
||||
/** Iterators */
|
||||
@ -142,15 +139,14 @@ namespace Moses {
|
||||
/** Element access */
|
||||
ProxyFVector operator[](const FName& name);
|
||||
FValue operator[](const FName& name) const;
|
||||
|
||||
|
||||
/** Size */
|
||||
size_t size() const {return m_size;}
|
||||
size_t size() const {return m_features.size();}
|
||||
|
||||
/** Equality */
|
||||
bool operator== (const FVector& rhs) const;
|
||||
bool operator!= (const FVector& rhs) const;
|
||||
|
||||
FValue inner_product(const std::vector<FValue>& rhs) const;
|
||||
FValue inner_product(const FVector& rhs) const;
|
||||
|
||||
|
||||
@ -192,7 +188,6 @@ namespace Moses {
|
||||
|
||||
|
||||
FNVmap m_features;
|
||||
size_t m_size;
|
||||
|
||||
#ifdef MPI_ENABLE
|
||||
//serialization
|
||||
|
@ -33,13 +33,10 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
GenerationDictionary::GenerationDictionary(size_t numFeatures, ScoreIndexManager &scoreIndexManager,
|
||||
GenerationDictionary::GenerationDictionary(size_t numFeatures,
|
||||
const std::vector<FactorType> &input,
|
||||
const std::vector<FactorType> &output)
|
||||
: Dictionary(numFeatures), DecodeFeature(input,output)
|
||||
{
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
: Dictionary(numFeatures), DecodeFeature(input,output) {}
|
||||
|
||||
bool GenerationDictionary::Load(const std::string &filePath, FactorDirection direction)
|
||||
{
|
||||
|
@ -57,7 +57,6 @@ public:
|
||||
*/
|
||||
GenerationDictionary(
|
||||
size_t numFeatures,
|
||||
ScoreIndexManager &scoreIndexManager,
|
||||
const std::vector<FactorType> &input,
|
||||
const std::vector<FactorType> &output);
|
||||
virtual ~GenerationDictionary();
|
||||
|
@ -9,17 +9,11 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
GlobalLexicalModel::GlobalLexicalModel(const string &filePath,
|
||||
const float weight,
|
||||
const vector< FactorType >& inFactors,
|
||||
const vector< FactorType >& outFactors)
|
||||
{
|
||||
std::cerr << "Creating global lexical model...\n";
|
||||
|
||||
// register as score producer
|
||||
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
|
||||
std::vector< float > weights;
|
||||
weights.push_back( weight );
|
||||
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
|
||||
|
||||
// load model
|
||||
LoadData( filePath, inFactors, outFactors );
|
||||
|
@ -48,7 +48,6 @@ private:
|
||||
|
||||
public:
|
||||
GlobalLexicalModel(const std::string &filePath,
|
||||
const float weight,
|
||||
const std::vector< FactorType >& inFactors,
|
||||
const std::vector< FactorType >& outFactors);
|
||||
virtual ~GlobalLexicalModel();
|
||||
|
@ -95,13 +95,6 @@ void LMList::Add(LanguageModel *lm)
|
||||
{
|
||||
m_coll.push_back(lm);
|
||||
m_maxNGramOrder = (lm->GetNGramOrder() > m_maxNGramOrder) ? lm->GetNGramOrder() : m_maxNGramOrder;
|
||||
|
||||
const ScoreIndexManager &scoreMgr = StaticData::Instance().GetScoreIndexManager();
|
||||
size_t startInd = scoreMgr.GetBeginIndex(lm->GetScoreBookkeepingID())
|
||||
,endInd = scoreMgr.GetEndIndex(lm->GetScoreBookkeepingID()) - 1;
|
||||
|
||||
m_minInd = min(m_minInd, startInd);
|
||||
m_maxInd = max(m_maxInd, endInd);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -19,7 +19,6 @@ protected:
|
||||
CollType m_coll;
|
||||
|
||||
size_t m_maxNGramOrder;
|
||||
size_t m_minInd, m_maxInd;
|
||||
|
||||
public:
|
||||
typedef CollType::iterator iterator;
|
||||
@ -31,8 +30,6 @@ public:
|
||||
|
||||
LMList()
|
||||
:m_maxNGramOrder(0)
|
||||
,m_minInd(std::numeric_limits<size_t>::max())
|
||||
,m_maxInd(0)
|
||||
{}
|
||||
void CleanUp();
|
||||
~LMList();
|
||||
@ -47,10 +44,6 @@ public:
|
||||
|
||||
size_t GetMaxNGramOrder() const
|
||||
{ return m_maxNGramOrder; }
|
||||
size_t GetMinIndex() const
|
||||
{ return m_minInd; }
|
||||
size_t GetMaxIndex() const
|
||||
{ return m_maxInd; }
|
||||
|
||||
};
|
||||
|
||||
|
@ -37,11 +37,8 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
LanguageModel::LanguageModel(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
{
|
||||
if (registerScore)
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
LanguageModel::LanguageModel() {}
|
||||
|
||||
LanguageModel::~LanguageModel() {}
|
||||
|
||||
// don't inline virtual funcs...
|
||||
@ -252,9 +249,7 @@ FFState* LanguageModel::Evaluate(
|
||||
}
|
||||
|
||||
float LanguageModel::GetWeight() const {
|
||||
size_t lmIndex = StaticData::Instance().GetScoreIndexManager().
|
||||
GetBeginIndex(GetScoreBookkeepingID());
|
||||
return StaticData::Instance().GetAllWeights()[lmIndex];
|
||||
return StaticData::Instance().GetAllWeights().GetScoreForProducer(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -49,10 +49,8 @@ protected:
|
||||
void ShiftOrPush(std::vector<const Word*> &contextFactor, const Word &word) const;
|
||||
|
||||
/** constructor to be called by inherited class
|
||||
* \param registerScore whether this LM will be directly used to score sentence.
|
||||
* Usually true, except where LM is a component in a composite LM, eg. LanguageModelJoint
|
||||
*/
|
||||
LanguageModel(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModel();
|
||||
|
||||
public:
|
||||
/* Returned from LM implementations which points at the state used. For example, if a trigram score was requested
|
||||
|
@ -31,8 +31,7 @@ namespace Moses {
|
||||
class LanguageModelDelegate: public LanguageModelSingleFactor {
|
||||
|
||||
public:
|
||||
LanguageModelDelegate(bool registerScore, ScoreIndexManager &scoreIndexManager, LanguageModelSingleFactor* delegate) :
|
||||
LanguageModelSingleFactor(registerScore, scoreIndexManager), m_delegate(delegate)
|
||||
LanguageModelDelegate(LanguageModelSingleFactor* delegate) : m_delegate(delegate)
|
||||
{
|
||||
m_nGramOrder = m_delegate->GetNGramOrder();
|
||||
m_factorType = m_delegate->GetFactorType();
|
||||
|
@ -60,7 +60,6 @@ namespace LanguageModelFactory
|
||||
, const std::vector<FactorType> &factorTypes
|
||||
, size_t nGramOrder
|
||||
, const std::string &languageModelFile
|
||||
, ScoreIndexManager &scoreIndexManager
|
||||
, int dub)
|
||||
{
|
||||
LanguageModel *lm = NULL;
|
||||
@ -68,61 +67,52 @@ namespace LanguageModelFactory
|
||||
{
|
||||
case RandLM:
|
||||
#ifdef LM_RAND
|
||||
lm = new LanguageModelRandLM(true,
|
||||
scoreIndexManager);
|
||||
lm = new LanguageModelRandLM();
|
||||
#endif
|
||||
break;
|
||||
case Remote:
|
||||
#ifdef LM_REMOTE
|
||||
lm = new LanguageModelRemote(true,scoreIndexManager);
|
||||
lm = new LanguageModelRemote();
|
||||
#endif
|
||||
break;
|
||||
|
||||
case SRI:
|
||||
#ifdef LM_SRI
|
||||
lm = new LanguageModelSRI(true, scoreIndexManager);
|
||||
lm = new LanguageModelSRI();
|
||||
#endif
|
||||
break;
|
||||
case IRST:
|
||||
#ifdef LM_IRST
|
||||
lm = new LanguageModelIRST(true, scoreIndexManager, dub);
|
||||
lm = new LanguageModelIRST(dub);
|
||||
#endif
|
||||
break;
|
||||
case Skip:
|
||||
#ifdef LM_SRI
|
||||
lm = new LanguageModelSkip(new LanguageModelSRI(false, scoreIndexManager)
|
||||
, true
|
||||
, scoreIndexManager);
|
||||
lm = new LanguageModelSkip(new LanguageModelSRI());
|
||||
#elif LM_INTERNAL
|
||||
lm = new LanguageModelSkip(new LanguageModelInternal(false, scoreIndexManager)
|
||||
, true
|
||||
, scoreIndexManager);
|
||||
lm = new LanguageModelSkip(new LanguageModelInternal());
|
||||
#endif
|
||||
break;
|
||||
case Ken:
|
||||
#ifdef LM_KEN
|
||||
lm = new LanguageModelKen(true, scoreIndexManager);
|
||||
lm = new LanguageModelKen();
|
||||
#endif
|
||||
break;
|
||||
case Joint:
|
||||
#ifdef LM_SRI
|
||||
lm = new LanguageModelJoint(new LanguageModelSRI(false, scoreIndexManager)
|
||||
, true
|
||||
, scoreIndexManager);
|
||||
lm = new LanguageModelJoint(new LanguageModelSRI());
|
||||
#elif LM_INTERNAL
|
||||
lm = new LanguageModelJoint(new LanguageModelInternal(false, scoreIndexManager)
|
||||
, true
|
||||
, scoreIndexManager);
|
||||
lm = new LanguageModelJoint(new LanguageModelInternal());
|
||||
#endif
|
||||
break;
|
||||
case ParallelBackoff:
|
||||
#ifdef LM_SRI
|
||||
lm = new LanguageModelParallelBackoff(true, scoreIndexManager);
|
||||
lm = new LanguageModelParallelBackoff();
|
||||
#endif
|
||||
break;
|
||||
case Internal:
|
||||
#ifdef LM_INTERNAL
|
||||
lm = new LanguageModelInternal(true, scoreIndexManager);
|
||||
lm = new LanguageModelInternal();
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
@ -11,7 +11,6 @@ namespace Moses
|
||||
{
|
||||
|
||||
class LanguageModel;
|
||||
class ScoreIndexManager;
|
||||
|
||||
namespace LanguageModelFactory {
|
||||
|
||||
@ -23,7 +22,6 @@ namespace LanguageModelFactory {
|
||||
, const std::vector<FactorType> &factorTypes
|
||||
, size_t nGramOrder
|
||||
, const std::string &languageModelFile
|
||||
, ScoreIndexManager &scoreIndexManager
|
||||
, int dub);
|
||||
|
||||
};
|
||||
|
@ -42,9 +42,8 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
LanguageModelIRST::LanguageModelIRST(bool registerScore, ScoreIndexManager &scoreIndexManager, int dub)
|
||||
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
|
||||
,m_lmtb(0),m_lmtb_dub(dub)
|
||||
LanguageModelIRST::LanguageModelIRST(int dub)
|
||||
:m_lmtb(0),m_lmtb_dub(dub)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -66,7 +66,7 @@ protected:
|
||||
};
|
||||
|
||||
public:
|
||||
LanguageModelIRST(bool registerScore, ScoreIndexManager &scoreIndexManager, int dub);
|
||||
LanguageModelIRST(int dub);
|
||||
~LanguageModelIRST();
|
||||
bool Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
|
@ -10,10 +10,6 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
LanguageModelInternal::LanguageModelInternal(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
|
||||
{
|
||||
}
|
||||
|
||||
bool LanguageModelInternal::Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
|
@ -26,7 +26,7 @@ protected:
|
||||
float GetValue(const Factor *factor0, const Factor *factor1, const Factor *factor2, State* finalState) const;
|
||||
|
||||
public:
|
||||
LanguageModelInternal(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelInternal() {}
|
||||
bool Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
, size_t nGramOrder);
|
||||
|
@ -48,8 +48,7 @@ protected:
|
||||
|
||||
size_t m_implFactor;
|
||||
public:
|
||||
LanguageModelJoint(LanguageModelSingleFactor *lmImpl, bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelMultiFactor(registerScore, scoreIndexManager)
|
||||
LanguageModelJoint(LanguageModelSingleFactor *lmImpl)
|
||||
{
|
||||
m_lmImpl = lmImpl;
|
||||
}
|
||||
|
@ -38,8 +38,8 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
LanguageModelKen::LanguageModelKen(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelSingleFactor(registerScore, scoreIndexManager), m_ngram(NULL)
|
||||
LanguageModelKen::LanguageModelKen()
|
||||
:m_ngram(NULL)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -42,7 +42,7 @@ protected:
|
||||
lm::ngram::Model *m_ngram;
|
||||
|
||||
public:
|
||||
LanguageModelKen(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelKen();
|
||||
~LanguageModelKen();
|
||||
bool Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
|
@ -24,9 +24,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
LanguageModelMultiFactor::LanguageModelMultiFactor(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModel(registerScore, scoreIndexManager)
|
||||
{}
|
||||
|
||||
std::string LanguageModelMultiFactor::GetScoreProducerDescription() const
|
||||
{
|
||||
|
@ -39,7 +39,7 @@ class LanguageModelMultiFactor : public LanguageModel
|
||||
protected:
|
||||
FactorMask m_factorTypes;
|
||||
|
||||
LanguageModelMultiFactor(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelMultiFactor(){}
|
||||
|
||||
public:
|
||||
virtual bool Load(const std::string &filePath
|
||||
|
@ -35,12 +35,6 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
LanguageModelParallelBackoff::LanguageModelParallelBackoff(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelMultiFactor(registerScore, scoreIndexManager)
|
||||
{
|
||||
///
|
||||
}
|
||||
|
||||
LanguageModelParallelBackoff::~LanguageModelParallelBackoff()
|
||||
{
|
||||
///
|
||||
|
@ -78,7 +78,7 @@ protected:
|
||||
WidMatrix *widMatrix;
|
||||
|
||||
public:
|
||||
LanguageModelParallelBackoff(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelParallelBackoff(){}
|
||||
|
||||
|
||||
~LanguageModelParallelBackoff();
|
||||
|
@ -38,8 +38,7 @@ class Phrase;
|
||||
|
||||
class LanguageModelRandLM : public LanguageModelSingleFactor {
|
||||
public:
|
||||
LanguageModelRandLM(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
: LanguageModelSingleFactor(registerScore, scoreIndexManager), m_lm(0) {}
|
||||
LanguageModelRandLM() : m_lm(0) {}
|
||||
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
|
||||
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const;
|
||||
~LanguageModelRandLM() {
|
||||
|
@ -14,11 +14,6 @@ namespace Moses {
|
||||
const Factor* LanguageModelRemote::BOS = NULL;
|
||||
const Factor* LanguageModelRemote::EOS = (LanguageModelRemote::BOS + 1);
|
||||
|
||||
LanguageModelRemote::LanguageModelRemote(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
|
||||
{
|
||||
}
|
||||
|
||||
bool LanguageModelRemote::Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
, size_t nGramOrder)
|
||||
|
@ -29,7 +29,7 @@ class LanguageModelRemote : public LanguageModelSingleFactor {
|
||||
static const Factor* BOS;
|
||||
static const Factor* EOS;
|
||||
public:
|
||||
LanguageModelRemote(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelRemote() {}
|
||||
~LanguageModelRemote();
|
||||
void ClearSentenceCache() { m_cache.tree.clear(); m_curId = 1000; }
|
||||
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0, unsigned int* len = 0) const;
|
||||
|
@ -35,9 +35,8 @@ using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
LanguageModelSRI::LanguageModelSRI(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModelSingleFactor(registerScore, scoreIndexManager)
|
||||
, m_srilmVocab(0)
|
||||
LanguageModelSRI::LanguageModelSRI()
|
||||
: m_srilmVocab(0)
|
||||
, m_srilmModel(0)
|
||||
{
|
||||
}
|
||||
|
@ -50,7 +50,7 @@ protected:
|
||||
VocabIndex GetLmID( const Factor *factor ) const;
|
||||
|
||||
public:
|
||||
LanguageModelSRI(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelSRI();
|
||||
~LanguageModelSRI();
|
||||
bool Load(const std::string &filePath
|
||||
, FactorType factorType
|
||||
|
@ -38,10 +38,6 @@ namespace Moses
|
||||
// static variable init
|
||||
LanguageModelSingleFactor::State LanguageModelSingleFactor::UnknownState=0;
|
||||
|
||||
LanguageModelSingleFactor::LanguageModelSingleFactor(bool registerScore, ScoreIndexManager &scoreIndexManager)
|
||||
:LanguageModel(registerScore, scoreIndexManager)
|
||||
{
|
||||
}
|
||||
LanguageModelSingleFactor::~LanguageModelSingleFactor() {}
|
||||
|
||||
|
||||
|
@ -38,7 +38,7 @@ protected:
|
||||
const Factor *m_sentenceStart, *m_sentenceEnd;
|
||||
FactorType m_factorType;
|
||||
|
||||
LanguageModelSingleFactor(bool registerScore, ScoreIndexManager &scoreIndexManager);
|
||||
LanguageModelSingleFactor() {}
|
||||
|
||||
public:
|
||||
static State UnknownState;
|
||||
|
@ -45,10 +45,7 @@ public:
|
||||
/** Constructor
|
||||
* \param lmImpl SRI or IRST LM which this LM can use to load data
|
||||
*/
|
||||
LanguageModelSkip(LanguageModelSingleFactor *lmImpl
|
||||
, bool registerScore
|
||||
, ScoreIndexManager &scoreIndexManager)
|
||||
: LanguageModelSingleFactor(registerScore, scoreIndexManager)
|
||||
LanguageModelSkip(LanguageModelSingleFactor *lmImpl)
|
||||
{
|
||||
m_lmImpl = lmImpl;
|
||||
}
|
||||
|
@ -53,9 +53,7 @@ LexicalReordering::LexicalReordering(std::vector<FactorType>& f_factors,
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// add ScoreProducer - don't do this before our object is set up
|
||||
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
|
||||
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
|
||||
const_cast<StaticData&>(StaticData::Instance()).SetWeights(this, weights);
|
||||
|
||||
m_table = LexicalReorderingTable::LoadAvailable(filePath, m_factorsF, m_factorsE, std::vector<FactorType>());
|
||||
}
|
||||
|
@ -78,7 +78,6 @@ libmoses_la_HEADERS = \
|
||||
ReorderingConstraint.h \
|
||||
ReorderingStack.h \
|
||||
ScoreComponentCollection.h \
|
||||
ScoreIndexManager.h \
|
||||
ScoreProducer.h \
|
||||
Search.h \
|
||||
SearchCubePruning.h \
|
||||
@ -207,7 +206,6 @@ libmoses_la_SOURCES = \
|
||||
ReorderingConstraint.cpp \
|
||||
ReorderingStack.cpp \
|
||||
ScoreComponentCollection.cpp \
|
||||
ScoreIndexManager.cpp \
|
||||
ScoreProducer.cpp \
|
||||
Search.cpp \
|
||||
SearchCubePruning.cpp \
|
||||
|
@ -564,8 +564,7 @@ void OutputSearchNode(long translationId, std::ostream &outputSearchGraphStream,
|
||||
|
||||
ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown();
|
||||
scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() );
|
||||
outputSearchGraphStream << " [ ";
|
||||
StaticData::Instance().GetScoreIndexManager().PrintLabeledScores( outputSearchGraphStream, scoreBreakdown );
|
||||
outputSearchGraphStream << " [ " << StaticData::Instance().GetAllWeights();
|
||||
outputSearchGraphStream << " ]";
|
||||
|
||||
outputSearchGraphStream << " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl;
|
||||
|
@ -109,11 +109,10 @@ void OnlineCommand::Execute() const
|
||||
staticData.ReLoadParameter();
|
||||
// check on weights
|
||||
|
||||
vector<float> weights = staticData.GetAllWeights();
|
||||
const ScoreComponentCollection& weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
|
||||
TRACE_ERR("The global weight vector looks like this:");
|
||||
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
|
||||
TRACE_ERR("The global weight vector looks like this: ");
|
||||
TRACE_ERR(weights);
|
||||
TRACE_ERR("\n");
|
||||
}
|
||||
|
||||
|
@ -64,8 +64,6 @@ PhraseDictionaryFeature::PhraseDictionaryFeature
|
||||
m_alignmentsFile(alignmentsFile)
|
||||
{
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).AddScoreProducer(this);
|
||||
//const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).InitFeatureNames();
|
||||
if (implementation == Memory || implementation == SCFG || implementation == OnDisk || implementation == SuffixArray) {
|
||||
m_useThreadSafePhraseDictionary = true;
|
||||
} else {
|
||||
|
@ -63,7 +63,7 @@ public:
|
||||
virtual void InitializeForInput(InputType const&)
|
||||
{/* Don't do anything source specific here as this object is shared between threads.*/}
|
||||
|
||||
void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
void GetChartRuleCollection(ChartTranslationOptionList &/*outColl*/
|
||||
,InputType const& /*src*/
|
||||
, WordsRange const& /*range*/
|
||||
, bool /*adhereTableLimit*/
|
||||
|
@ -1,56 +1,36 @@
|
||||
// $Id$
|
||||
#include <vector>
|
||||
|
||||
#include "ScoreComponentCollection.h"
|
||||
#include "StaticData.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
ScoreComponentCollection::ScoreComponentCollection()
|
||||
: m_scores(StaticData::Instance().GetTotalScoreComponents(), 0.0f)
|
||||
, m_sim(&StaticData::Instance().GetScoreIndexManager())
|
||||
{}
|
||||
|
||||
const ScoreIndexManager* ScoreComponentCollection::GetSim() {
|
||||
return &(StaticData::Instance().GetScoreIndexManager());
|
||||
}
|
||||
|
||||
float ScoreComponentCollection::GetWeightedScore() const
|
||||
{
|
||||
float ret = InnerProduct(StaticData::Instance().GetAllWeights());
|
||||
return ret;
|
||||
return m_scores.inner_product(StaticData::Instance().GetAllWeights().m_scores);
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::ZeroAllLM(const LMList& lmList)
|
||||
{
|
||||
|
||||
for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind)
|
||||
{
|
||||
m_scores[m_sim->GetFeatureName(ind)] = 0;
|
||||
}
|
||||
for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) {
|
||||
Assign(*i, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::PlusEqualsAllLM(const LMList& lmList, const ScoreComponentCollection& rhs)
|
||||
{
|
||||
|
||||
for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind)
|
||||
{
|
||||
m_scores[m_sim->GetFeatureName(ind)] += rhs.m_scores[m_sim->GetFeatureName(ind)];
|
||||
}
|
||||
|
||||
for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) {
|
||||
PlusEquals(*i,rhs);
|
||||
}
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores)
|
||||
{
|
||||
assert(scores.size() == sp->GetNumScoreComponents());
|
||||
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||
for (std::vector<float>::const_iterator vi = scores.begin();
|
||||
vi != scores.end(); ++vi)
|
||||
{
|
||||
const std::string &name = m_sim->GetFeatureName(i++);
|
||||
m_scores[name] += *vi;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ScoreComponentCollection::MultiplyEquals(float scalar)
|
||||
{
|
||||
@ -64,63 +44,9 @@ void ScoreComponentCollection::MultiplyEquals(const ScoreComponentCollection& rh
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs)
|
||||
{
|
||||
os << "<<" << rhs.m_scores[rhs.m_sim->GetFeatureName(0)];
|
||||
for (size_t i=1; i<rhs.m_scores.size(); i++)
|
||||
os << ", " << rhs.m_scores[rhs.m_sim->GetFeatureName(i)];
|
||||
os << "<<" << rhs.m_scores;
|
||||
return os << ">>";
|
||||
}
|
||||
|
||||
ScoreComponentCollection StaticData::GetAllWeightsScoreComponentCollection() const
|
||||
{
|
||||
Moses::ScoreComponentCollection ret;
|
||||
|
||||
const std::vector<const ScoreProducer*> &producers = m_scoreIndexManager.GetProducers();
|
||||
std::vector<const ScoreProducer*>::const_iterator iter;
|
||||
|
||||
for (iter = producers.begin(); iter != producers.end(); ++iter)
|
||||
{
|
||||
const ScoreProducer *producer = *iter;
|
||||
unsigned int bookId = producer->GetScoreBookkeepingID();
|
||||
size_t startInd = m_scoreIndexManager.GetBeginIndex(bookId);
|
||||
size_t endInd = m_scoreIndexManager.GetEndIndex(bookId);
|
||||
|
||||
|
||||
std::vector<float> weights;
|
||||
copy(m_allWeights.begin() + startInd, m_allWeights.begin() + endInd, inserter(weights,weights.end()));
|
||||
|
||||
ret.Assign(producer, weights);
|
||||
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void StaticData::SetAllWeightsScoreComponentCollection(const ScoreComponentCollection &weightsScoreComponentCollection)
|
||||
{
|
||||
const std::vector<const ScoreProducer*> &producers = m_scoreIndexManager.GetProducers();
|
||||
std::vector<const ScoreProducer*>::const_iterator iter;
|
||||
|
||||
for (iter = producers.begin(); iter != producers.end(); ++iter)
|
||||
{
|
||||
const ScoreProducer *producer = *iter;
|
||||
unsigned int bookId = producer->GetScoreBookkeepingID();
|
||||
size_t startInd = m_scoreIndexManager.GetBeginIndex(bookId);
|
||||
size_t endInd = m_scoreIndexManager.GetEndIndex(bookId);
|
||||
|
||||
|
||||
std::vector<float> weights = weightsScoreComponentCollection.GetScoresForProducer(producer);
|
||||
|
||||
size_t allInd = startInd;
|
||||
for (size_t ind = 0; ind < weights.size(); ++ind)
|
||||
{
|
||||
m_allWeights[allInd] = weights[ind];
|
||||
allInd++;
|
||||
}
|
||||
assert(allInd == endInd);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::L1Normalise() {
|
||||
m_scores /= m_scores.l1norm();
|
||||
}
|
||||
|
@ -32,7 +32,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
#include "LMList.h"
|
||||
#include "ScoreProducer.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
#include "FeatureVector.h"
|
||||
#include "TypeDef.h"
|
||||
#include "Util.h"
|
||||
@ -62,11 +61,8 @@ namespace Moses
|
||||
*/
|
||||
class ScoreComponentCollection {
|
||||
friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs);
|
||||
friend class ScoreIndexManager;
|
||||
private:
|
||||
const ScoreIndexManager* GetSim();
|
||||
FVector m_scores;
|
||||
const ScoreIndexManager* m_sim;
|
||||
|
||||
public:
|
||||
//! Create a new score collection with all values set to 0.0
|
||||
@ -75,11 +71,8 @@ public:
|
||||
//! Clone a score collection
|
||||
ScoreComponentCollection(const ScoreComponentCollection& rhs)
|
||||
: m_scores(rhs.m_scores)
|
||||
, m_sim(rhs.m_sim)
|
||||
{}
|
||||
|
||||
inline size_t size() const { return m_scores.size(); }
|
||||
float operator[](size_t x) const { return m_scores[m_sim->GetFeatureName(x)]; }
|
||||
|
||||
//! Set all values to 0.0
|
||||
void ZeroAll()
|
||||
@ -102,71 +95,58 @@ public:
|
||||
m_scores -= rhs.m_scores;
|
||||
}
|
||||
|
||||
//! Add scores from a single ScoreProducer only
|
||||
//! The length of scores must be equal to the number of score components
|
||||
//! produced by sp
|
||||
void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores);
|
||||
|
||||
//! Add scores from a single ScoreProducer only
|
||||
//! The length of scores must be equal to the number of score components
|
||||
//! produced by sp
|
||||
void PlusEquals(const ScoreProducer* sp, const ScoreComponentCollection& scores)
|
||||
{
|
||||
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||
const size_t end = m_sim->GetEndIndex(sp->GetScoreBookkeepingID());
|
||||
for (; i < end; ++i)
|
||||
{
|
||||
m_scores[m_sim->GetFeatureName(i)] += scores.m_scores[m_sim->GetFeatureName(i)];
|
||||
}
|
||||
const std::vector<FName>& names = sp->GetFeatureNames();
|
||||
for (std::vector<FName>::const_iterator i = names.begin();
|
||||
i != names.end(); ++i) {
|
||||
m_scores[*i] += scores.m_scores[*i];
|
||||
}
|
||||
}
|
||||
|
||||
//! Add scores from a single ScoreProducer only
|
||||
//! The length of scores must be equal to the number of score components
|
||||
//! produced by sp
|
||||
void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores)
|
||||
{
|
||||
const std::vector<FName>& names = sp->GetFeatureNames();
|
||||
assert(names.size() == scores.size());
|
||||
for (size_t i = 0; i < scores.size(); ++i) {
|
||||
m_scores[names[i]] = scores[i];
|
||||
}
|
||||
}
|
||||
|
||||
//! Special version PlusEquals(ScoreProducer, vector<float>)
|
||||
//! to add the score from a single ScoreProducer that produces
|
||||
//! a single value
|
||||
void PlusEquals(const ScoreProducer* sp, float score)
|
||||
{
|
||||
assert(1 == sp->GetNumScoreComponents());
|
||||
const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||
m_scores[m_sim->GetFeatureName(i)] += score;
|
||||
m_scores[sp->GetFeatureNames()[0]] += score;
|
||||
}
|
||||
|
||||
void Assign(const ScoreProducer* sp, const std::vector<float>& scores)
|
||||
{
|
||||
assert(scores.size() == sp->GetNumScoreComponents());
|
||||
size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||
for (std::vector<float>::const_iterator vi = scores.begin();
|
||||
vi != scores.end(); ++vi)
|
||||
{
|
||||
m_scores[m_sim->GetFeatureName(i++)] = *vi;
|
||||
}
|
||||
const std::vector<FName>& names = sp->GetFeatureNames();
|
||||
for (size_t i = 0; i < scores.size(); ++i) {
|
||||
m_scores[names[i]] = scores[i];
|
||||
}
|
||||
}
|
||||
|
||||
void Assign(const ScoreComponentCollection ©)
|
||||
{
|
||||
m_scores = copy.m_scores;
|
||||
}
|
||||
|
||||
//! Special version PlusEquals(ScoreProducer, vector<float>)
|
||||
//! to add the score from a single ScoreProducer that produces
|
||||
//! a single value
|
||||
void Assign(const ScoreProducer* sp, float score)
|
||||
{
|
||||
assert(1 == sp->GetNumScoreComponents());
|
||||
const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID());
|
||||
m_scores[m_sim->GetFeatureName(i)] = score;
|
||||
m_scores[sp->GetFeatureNames()[0]] = score;
|
||||
}
|
||||
|
||||
void Assign(size_t index, float score)
|
||||
{
|
||||
m_scores[m_sim->GetFeatureName(index)] = score;
|
||||
}
|
||||
|
||||
//! Used to find the weighted total of scores. rhs should contain a vector of weights
|
||||
//! of the same length as the number of scores.
|
||||
float InnerProduct(const std::vector<float>& rhs) const
|
||||
{
|
||||
return m_scores.inner_product(rhs);
|
||||
}
|
||||
|
||||
float InnerProduct(const ScoreComponentCollection& rhs) const
|
||||
{
|
||||
@ -183,28 +163,20 @@ public:
|
||||
//! return a vector of all the scores associated with a certain ScoreProducer
|
||||
std::vector<float> GetScoresForProducer(const ScoreProducer* sp) const
|
||||
{
|
||||
size_t id = sp->GetScoreBookkeepingID();
|
||||
const size_t begin = m_sim->GetBeginIndex(id);
|
||||
const size_t end = m_sim->GetEndIndex(id);
|
||||
std::vector<float> res(end-begin);
|
||||
size_t j = 0;
|
||||
for (size_t i = begin; i < end; i++) {
|
||||
res[j++] = m_scores[m_sim->GetFeatureName(i)];
|
||||
}
|
||||
return res;
|
||||
std::vector<float> res(sp->GetNumScoreComponents());
|
||||
const std::vector<FName>& names = sp->GetFeatureNames();
|
||||
for (size_t i = 0; i < names.size(); ++i) {
|
||||
res[i] = m_scores[names[i]];
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
//! if a ScoreProducer produces a single score (for example, a language model score)
|
||||
//! this will return it. If not, this method will throw
|
||||
float GetScoreForProducer(const ScoreProducer* sp) const
|
||||
{
|
||||
size_t id = sp->GetScoreBookkeepingID();
|
||||
const size_t begin = m_sim->GetBeginIndex(id);
|
||||
#ifndef NDEBUG
|
||||
const size_t end = m_sim->GetEndIndex(id);
|
||||
assert(end-begin == 1);
|
||||
#endif
|
||||
return m_scores[m_sim->GetFeatureName(begin)];
|
||||
assert(sp->GetNumScoreComponents() == 1);
|
||||
return m_scores[sp->GetFeatureNames()[0]];
|
||||
}
|
||||
|
||||
float GetWeightedScore() const;
|
||||
@ -227,7 +199,6 @@ public:
|
||||
template<class Archive>
|
||||
void load(Archive &ar, const unsigned int version) {
|
||||
ar >> m_scores;
|
||||
m_sim = GetSim();
|
||||
|
||||
}
|
||||
|
||||
|
@ -1,148 +0,0 @@
|
||||
// $Id$
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <string>
|
||||
#include <cstdio>
|
||||
#include <cassert>
|
||||
#include "Util.h"
|
||||
#include "StaticData.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
#include "ScoreProducer.h"
|
||||
#include "ScoreComponentCollection.h" // debugging
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
using namespace std;
|
||||
|
||||
void ScoreIndexManager::AddScoreProducer(const ScoreProducer* sp)
|
||||
{
|
||||
// Producers must be inserted in the order they are created
|
||||
const_cast<ScoreProducer*>(sp)->CreateScoreBookkeepingID();
|
||||
assert(m_begins.size() == (sp->GetScoreBookkeepingID()));
|
||||
|
||||
m_producers.push_back(sp);
|
||||
|
||||
m_begins.push_back(m_last);
|
||||
size_t numScoreCompsProduced = sp->GetNumScoreComponents();
|
||||
assert(numScoreCompsProduced > 0);
|
||||
m_last += numScoreCompsProduced;
|
||||
m_ends.push_back(m_last);
|
||||
m_featureNamesDirty = true;
|
||||
InitFeatureNames();
|
||||
/*VERBOSE(1,"Added ScoreProducer(" << sp->GetScoreBookkeepingID()
|
||||
<< " " << sp->GetScoreProducerDescription()
|
||||
<< ") index=" << m_begins.back() << "-" << m_ends.back()-1 << std::endl);
|
||||
*/
|
||||
}
|
||||
|
||||
void ScoreIndexManager::PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scores) const
|
||||
{
|
||||
std::vector<float> weights(scores.m_scores.size(), 1.0f);
|
||||
PrintLabeledWeightedScores(os, scores, weights);
|
||||
}
|
||||
|
||||
void ScoreIndexManager::PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scores, const std::vector<float>& weights) const
|
||||
{
|
||||
assert(m_featureShortNames.size() == weights.size());
|
||||
string lastName = "";
|
||||
for (size_t i = 0; i < m_featureShortNames.size(); ++i)
|
||||
{
|
||||
if (i>0)
|
||||
{
|
||||
os << " ";
|
||||
}
|
||||
if (lastName != m_featureShortNames[i])
|
||||
{
|
||||
os << m_featureShortNames[i] << ": ";
|
||||
lastName = m_featureShortNames[i];
|
||||
}
|
||||
os << weights[i] * scores[i];
|
||||
}
|
||||
}
|
||||
|
||||
void ScoreIndexManager::InitFeatureNames() {
|
||||
if (!m_featureNamesDirty)
|
||||
return;
|
||||
m_featureNames.clear();
|
||||
m_featureIndexes.clear();
|
||||
m_featureShortNames.clear();
|
||||
size_t globalIndex = 0;
|
||||
vector<const ScoreProducer *>::const_iterator it;
|
||||
|
||||
for (it = m_producers.begin(); it != m_producers.end(); ++it) {
|
||||
size_t scoreCount = (*it)->GetNumScoreComponents();
|
||||
for (size_t i = 0; i < scoreCount; ++i) {
|
||||
ostringstream oStream;
|
||||
oStream << /* (*it)->GetScoreProducerDescription() << "_" << */ globalIndex;
|
||||
m_featureNames.push_back(oStream.str());
|
||||
m_featureIndexes[oStream.str()] = globalIndex;
|
||||
++globalIndex;
|
||||
}
|
||||
}
|
||||
m_featureNamesDirty = false;
|
||||
}
|
||||
|
||||
#ifdef HAVE_PROTOBUF
|
||||
void ScoreIndexManager::SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const {
|
||||
for (size_t i = 0; i < m_featureNames.size(); ++i) {
|
||||
hg->add_feature_names(m_featureNames[i]);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
void ScoreIndexManager::InitWeightVectorFromFile(const std::string& fnam, vector<float>* m_allWeights) const {
|
||||
assert(m_allWeights->size() == m_featureNames.size());
|
||||
ifstream in(fnam.c_str());
|
||||
assert(in.good());
|
||||
char buf[2000];
|
||||
map<string, double> name2val;
|
||||
while (!in.eof()) {
|
||||
in.getline(buf, 2000);
|
||||
if (strlen(buf) == 0) continue;
|
||||
if (buf[0] == '#') continue;
|
||||
istringstream is(buf);
|
||||
string fname;
|
||||
double val;
|
||||
is >> fname >> val;
|
||||
map<string, double>::iterator i = name2val.find(fname);
|
||||
assert(i == name2val.end()); // duplicate weight name
|
||||
name2val[fname] = val;
|
||||
}
|
||||
assert(m_allWeights->size() == m_featureNames.size());
|
||||
for (size_t i = 0; i < m_featureNames.size(); ++i) {
|
||||
map<string, double>::iterator iter = name2val.find(m_featureNames[i]);
|
||||
if (iter == name2val.end()) {
|
||||
cerr << "No weight found found for feature: " << m_featureNames[i] << endl;
|
||||
abort();
|
||||
}
|
||||
(*m_allWeights)[i] = iter->second;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim)
|
||||
{
|
||||
for (size_t i = 0; i < sim.m_featureNames.size(); ++i) {
|
||||
os << sim.m_featureNames[i] << endl;
|
||||
}
|
||||
os << endl;
|
||||
return os;
|
||||
}
|
||||
|
||||
const std::string &ScoreIndexManager::GetFeatureName(size_t fIndex) const
|
||||
{
|
||||
#ifndef NDEBUG
|
||||
if (m_featureNames.size() <= fIndex) {
|
||||
printf("%s: %i ScoreIndexManager::GetFeatureName(m_producers.size() = %zd, m_featureNames.size() = %zd, fIndex = %zd)\n", __FILE__, __LINE__, m_producers.size(), m_featureNames.size(), fIndex);
|
||||
}
|
||||
#endif
|
||||
assert(m_featureNames.size() > fIndex);
|
||||
return m_featureNames[fIndex];
|
||||
}
|
||||
//! get the index of a feature by name
|
||||
size_t ScoreIndexManager::GetFeatureIndex(const std::string& fName) const
|
||||
{
|
||||
return m_featureIndexes.find(fName)->second;
|
||||
}
|
||||
|
||||
}
|
@ -1,75 +0,0 @@
|
||||
// $Id$
|
||||
|
||||
#ifndef moses_ScoreIndexManager_h
|
||||
#define moses_ScoreIndexManager_h
|
||||
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#if HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
#ifdef HAVE_PROTOBUF
|
||||
#include "hypergraph.pb.h"
|
||||
#endif
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ScoreProducer;
|
||||
class ScoreComponentCollection; // debugging only
|
||||
class StatefulFeatureFunction;
|
||||
class StatelessFeatureFunction;
|
||||
|
||||
/** Keep track of scores and score producers. Each score producer is reserved contiguous set of slots
|
||||
* to put their score components. All the score components are arranged in a vector with no gaps.
|
||||
* Only 1 ScoreIndexManager object should be instantiated
|
||||
*/
|
||||
class ScoreIndexManager
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim);
|
||||
public:
|
||||
ScoreIndexManager() : m_last(0), m_featureNamesDirty(true) {}
|
||||
|
||||
//! new score producer to manage. Producers must be inserted in the order they are created
|
||||
void AddScoreProducer(const ScoreProducer* producer);
|
||||
|
||||
//! starting score index for a particular score producer with scoreBookkeepingID
|
||||
size_t GetBeginIndex(size_t scoreBookkeepingID) const { return m_begins[scoreBookkeepingID]; }
|
||||
//! end score index for a particular score producer with scoreBookkeepingID
|
||||
size_t GetEndIndex(size_t scoreBookkeepingID) const { return m_ends[scoreBookkeepingID]; }
|
||||
//! sum of all score components from every score producer
|
||||
size_t GetTotalNumberOfScores() const { return m_last; }
|
||||
//! print unweighted scores of each ScoreManager to stream os
|
||||
void PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scc) const;
|
||||
//! print weighted scores of each ScoreManager to stream os
|
||||
void PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scc, const std::vector<float>& weights) const;
|
||||
//! get the name of a feature by index
|
||||
const std::string &GetFeatureName(size_t fIndex) const;
|
||||
//! get the index of a feature by name
|
||||
size_t GetFeatureIndex(const std::string& fName) const;
|
||||
|
||||
const std::vector<const ScoreProducer*> &GetProducers() const
|
||||
{ return m_producers; }
|
||||
|
||||
#ifdef HAVE_PROTOBUF
|
||||
void SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const;
|
||||
#endif
|
||||
void InitWeightVectorFromFile(const std::string& fnam, std::vector<float>* m_allWeights) const;
|
||||
private:
|
||||
ScoreIndexManager(const ScoreIndexManager&); // don't implement
|
||||
void InitFeatureNames();
|
||||
|
||||
std::vector<size_t> m_begins;
|
||||
std::vector<size_t> m_ends;
|
||||
std::vector<const ScoreProducer*> m_producers; /**< all the score producers in this run */
|
||||
std::vector<std::string> m_featureNames;
|
||||
std::map<std::string, size_t> m_featureIndexes;
|
||||
std::vector<std::string> m_featureShortNames;
|
||||
size_t m_last;
|
||||
bool m_featureNamesDirty;
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -1,20 +1,31 @@
|
||||
// $Id$
|
||||
|
||||
#include <iostream>
|
||||
#include <typeinfo>
|
||||
#include <sstream>
|
||||
|
||||
#include "ScoreProducer.h"
|
||||
#include "StaticData.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
unsigned int ScoreProducer::s_globalScoreBookkeepingIdCounter(0);
|
||||
|
||||
ScoreProducer::~ScoreProducer() {}
|
||||
|
||||
ScoreProducer::ScoreProducer()
|
||||
{
|
||||
m_scoreBookkeepingId = UNASSIGNED;
|
||||
const vector<FName>& ScoreProducer::GetFeatureNames() const {
|
||||
if (m_names.size() != GetNumScoreComponents()) {
|
||||
const string& desc = GetScoreProducerDescription();
|
||||
if (GetNumScoreComponents() == 1) {
|
||||
m_names.push_back(FName(desc));
|
||||
} else {
|
||||
for (size_t i = 1; i <= GetNumScoreComponents(); ++i) {
|
||||
ostringstream id;
|
||||
id << i;
|
||||
m_names.push_back(FName(desc,id.str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return m_names;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -5,44 +5,35 @@
|
||||
|
||||
#include <string>
|
||||
#include <limits>
|
||||
#include <vector>
|
||||
|
||||
#include "FeatureVector.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class Hypothesis;
|
||||
class ScoreComponentCollection;
|
||||
class ScoreIndexManager;
|
||||
class FFState;
|
||||
|
||||
/** to keep track of the various things that can produce a score,
|
||||
* we use this evil implementation-inheritance to give them each
|
||||
* a unique, sequential (read: good for vector indices) ID
|
||||
*
|
||||
/*
|
||||
* @note do not confuse this with a producer/consumer pattern.
|
||||
* this is not a producer in that sense.
|
||||
*/
|
||||
class ScoreProducer
|
||||
{
|
||||
private:
|
||||
static unsigned int s_globalScoreBookkeepingIdCounter;
|
||||
unsigned int m_scoreBookkeepingId;
|
||||
mutable std::vector<FName> m_names; //for features with fixed number of values
|
||||
|
||||
ScoreProducer(const ScoreProducer&); // don't implement
|
||||
|
||||
#define UNASSIGNED std::numeric_limits<unsigned int>::max()
|
||||
|
||||
protected:
|
||||
// it would be nice to force registration here, but some Producer objects
|
||||
// are constructed before they know how many scores they have
|
||||
ScoreProducer();
|
||||
ScoreProducer() {}
|
||||
virtual ~ScoreProducer();
|
||||
|
||||
public:
|
||||
//! contiguous id
|
||||
unsigned int GetScoreBookkeepingID() const { return m_scoreBookkeepingId; }
|
||||
void CreateScoreBookkeepingID() { m_scoreBookkeepingId = s_globalScoreBookkeepingIdCounter++;}
|
||||
//! returns the number of scores that a subclass produces.
|
||||
//! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc
|
||||
//! will cause an error if this producer does not have a fixed number
|
||||
//! of scores (eg sparse features)
|
||||
virtual size_t GetNumScoreComponents() const = 0;
|
||||
|
||||
//! returns a string description of this producer
|
||||
@ -54,6 +45,8 @@ public:
|
||||
//! returns the number of scores gathered from the input (0 by default)
|
||||
virtual size_t GetNumInputScores() const { return 0; };
|
||||
|
||||
const std::vector<FName>& GetFeatureNames() const;
|
||||
|
||||
virtual bool IsStateless() const = 0;
|
||||
|
||||
};
|
||||
|
@ -300,14 +300,14 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
// word penalties
|
||||
for (size_t i = 0; i < m_parameter->GetParam("weight-w").size(); ++i) {
|
||||
float weightWordPenalty = Scan<float>( m_parameter->GetParam("weight-w")[i] );
|
||||
m_wordPenaltyProducers.push_back(new WordPenaltyProducer(m_scoreIndexManager));
|
||||
m_allWeights.push_back(weightWordPenalty);
|
||||
m_wordPenaltyProducers.push_back(new WordPenaltyProducer());
|
||||
SetWeight(m_wordPenaltyProducers.back(), weightWordPenalty);
|
||||
}
|
||||
|
||||
|
||||
float weightUnknownWord = (m_parameter->GetParam("weight-u").size() > 0) ? Scan<float>(m_parameter->GetParam("weight-u")[0]) : 1;
|
||||
m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(m_scoreIndexManager);
|
||||
m_allWeights.push_back(weightUnknownWord);
|
||||
m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer();
|
||||
SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord);
|
||||
|
||||
// reordering constraints
|
||||
m_maxDistortion = (m_parameter->GetParam("distortion-limit").size() > 0) ?
|
||||
@ -547,8 +547,6 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
}
|
||||
|
||||
|
||||
//m_scoreIndexManager.InitFeatureNames();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -574,6 +572,16 @@ void StaticData::SetBooleanParameter( bool *parameter, string parameterName, boo
|
||||
}
|
||||
}
|
||||
|
||||
void StaticData::SetWeight(const ScoreProducer* sp, float weight)
|
||||
{
|
||||
m_allWeights.Assign(sp,weight);
|
||||
}
|
||||
|
||||
void StaticData::SetWeights(const ScoreProducer* sp, const std::vector<float>& weights)
|
||||
{
|
||||
m_allWeights.Assign(sp,weights);
|
||||
}
|
||||
|
||||
StaticData::~StaticData()
|
||||
{
|
||||
RemoveAllInColl(m_phraseDictionary);
|
||||
@ -706,7 +714,8 @@ bool StaticData::LoadGlobalLexicalModel()
|
||||
}
|
||||
vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],",");
|
||||
vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],",");
|
||||
m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], weight[i], inputFactors, outputFactors ) );
|
||||
m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], inputFactors, outputFactors ) );
|
||||
SetWeight(m_globalLexicalModels.back(),weight[i]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@ -718,11 +727,6 @@ bool StaticData::LoadLanguageModels()
|
||||
// weights
|
||||
vector<float> weightAll = Scan<float>(m_parameter->GetParam("weight-l"));
|
||||
|
||||
for (size_t i = 0 ; i < weightAll.size() ; i++)
|
||||
{
|
||||
m_allWeights.push_back(weightAll[i]);
|
||||
}
|
||||
|
||||
// dictionary upper-bounds fo all IRST LMs
|
||||
vector<int> LMdub = Scan<int>(m_parameter->GetParam("lmodel-dub"));
|
||||
if (m_parameter->GetParam("lmodel-dub").size() == 0){
|
||||
@ -739,7 +743,7 @@ bool StaticData::LoadLanguageModels()
|
||||
{
|
||||
LanguageModel* lm = NULL;
|
||||
if (languageModelsLoaded.find(lmVector[i]) != languageModelsLoaded.end()) {
|
||||
lm = new LanguageModelDelegate(true, m_scoreIndexManager,
|
||||
lm = new LanguageModelDelegate(
|
||||
static_cast<LanguageModelSingleFactor*>(languageModelsLoaded[lmVector[i]]));
|
||||
} else {
|
||||
vector<string> token = Tokenize(lmVector[i]);
|
||||
@ -774,7 +778,6 @@ bool StaticData::LoadLanguageModels()
|
||||
, factorTypes
|
||||
, nGramOrder
|
||||
, languageModelFile
|
||||
, m_scoreIndexManager
|
||||
, LMdub[i]);
|
||||
if (lm == NULL)
|
||||
{
|
||||
@ -785,9 +788,9 @@ bool StaticData::LoadLanguageModels()
|
||||
}
|
||||
|
||||
m_languageModel.Add(lm);
|
||||
SetWeight(lm,weightAll[i]);
|
||||
}
|
||||
}
|
||||
//m_scoreIndexManager.InitFeatureNames();
|
||||
// flag indicating that language models were loaded,
|
||||
// since phrase table loading requires their presence
|
||||
m_fLMsLoaded = true;
|
||||
@ -832,17 +835,19 @@ bool StaticData::LoadGenerationTables()
|
||||
|
||||
VERBOSE(1, filePath << endl);
|
||||
|
||||
m_generationDictionary.push_back(new GenerationDictionary(numFeatures, m_scoreIndexManager, input,output));
|
||||
m_generationDictionary.push_back(new GenerationDictionary(numFeatures, input,output));
|
||||
assert(m_generationDictionary.back() && "could not create GenerationDictionary");
|
||||
if (!m_generationDictionary.back()->Load(filePath, Output))
|
||||
{
|
||||
delete m_generationDictionary.back();
|
||||
return false;
|
||||
}
|
||||
vector<float> gdWeights;
|
||||
for(size_t i = 0; i < numFeatures; i++) {
|
||||
assert(currWeightNum < weight.size());
|
||||
m_allWeights.push_back(weight[currWeightNum++]);
|
||||
gdWeights.push_back(weight[currWeightNum++]);
|
||||
}
|
||||
SetWeights(m_generationDictionary.back(), gdWeights);
|
||||
}
|
||||
if (currWeightNum != weight.size()) {
|
||||
TRACE_ERR( " [WARNING] config file has " << weight.size() << " generation weights listed, but the configuration for generation files indicates there should be " << currWeightNum << "!\n");
|
||||
@ -983,27 +988,27 @@ bool StaticData::LoadPhraseTables()
|
||||
|
||||
assert(numScoreComponent==weight.size());
|
||||
|
||||
std::copy(weight.begin(),weight.end(),std::back_inserter(m_allWeights));
|
||||
|
||||
//This is needed for regression testing, but the phrase table
|
||||
//might not really be loading here
|
||||
IFVERBOSE(1)
|
||||
PrintUserTime(string("Start loading PhraseTable ") + filePath);
|
||||
VERBOSE(1,"filePath: " << filePath <<endl);
|
||||
|
||||
PhraseDictionaryFeature* pdf = new PhraseDictionaryFeature(
|
||||
implementation
|
||||
, numScoreComponent
|
||||
, (currDict==0 ? m_numInputScores : 0)
|
||||
, input
|
||||
, output
|
||||
, filePath
|
||||
, weight
|
||||
, maxTargetPhrase[index]
|
||||
, targetPath, alignmentsFile);
|
||||
|
||||
PhraseDictionaryFeature* pdf = new PhraseDictionaryFeature(
|
||||
implementation
|
||||
, numScoreComponent
|
||||
, (currDict==0 ? m_numInputScores : 0)
|
||||
, input
|
||||
, output
|
||||
, filePath
|
||||
, weight
|
||||
, maxTargetPhrase[index]
|
||||
, targetPath, alignmentsFile);
|
||||
|
||||
m_phraseDictionary.push_back(pdf);
|
||||
|
||||
m_phraseDictionary.push_back(pdf);
|
||||
|
||||
SetWeights(m_phraseDictionary.back(),weight);
|
||||
|
||||
|
||||
|
||||
@ -1094,8 +1099,8 @@ void StaticData::LoadPhraseBasedParameters()
|
||||
}
|
||||
for (size_t i = 0; i < distortionWeightCount; ++i) {
|
||||
float weightDistortion = Scan<float>(distortionWeights[i]);
|
||||
m_distortionScoreProducers.push_back(new DistortionScoreProducer(m_scoreIndexManager));
|
||||
m_allWeights.push_back(weightDistortion);
|
||||
m_distortionScoreProducers.push_back(new DistortionScoreProducer());
|
||||
SetWeight(m_distortionScoreProducers.back(), weightDistortion);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1205,7 +1210,7 @@ bool StaticData::LoadReferences() {
|
||||
}
|
||||
float bleuWeight = Scan<float>(bleuWeightStr[0]);
|
||||
m_bleuScoreFeature = new BleuScoreFeature();
|
||||
m_allWeights.push_back(bleuWeight);
|
||||
SetWeight(m_bleuScoreFeature, bleuWeight);
|
||||
|
||||
vector<vector<string> > references(referenceFiles.size());
|
||||
for (size_t i =0; i < referenceFiles.size(); ++i) {
|
||||
@ -1245,7 +1250,7 @@ bool StaticData::LoadTargetBigramFeature()
|
||||
UserMessage::Add("Wrong number of arguments for target-bigram-feature-wordlist (expected 1).");
|
||||
return false;
|
||||
}
|
||||
m_targetBigramFeature = new TargetBigramFeature(m_scoreIndexManager);
|
||||
m_targetBigramFeature = new TargetBigramFeature();
|
||||
if (!m_targetBigramFeature->Load(wordFile[0])) {
|
||||
UserMessage::Add("Unable to load word list from file " + wordFile[0]);
|
||||
return false;
|
||||
@ -1254,19 +1259,6 @@ bool StaticData::LoadTargetBigramFeature()
|
||||
return true;
|
||||
}
|
||||
|
||||
void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights)
|
||||
{
|
||||
const size_t id = sp->GetScoreBookkeepingID();
|
||||
const size_t begin = m_scoreIndexManager.GetBeginIndex(id);
|
||||
const size_t end = m_scoreIndexManager.GetEndIndex(id);
|
||||
assert(end - begin == weights.size());
|
||||
if (m_allWeights.size() < end)
|
||||
m_allWeights.resize(end);
|
||||
std::vector<float>::const_iterator weightIter = weights.begin();
|
||||
for (size_t i = begin; i < end; i++)
|
||||
m_allWeights[i] = *weightIter++;
|
||||
}
|
||||
|
||||
const TranslationOptionList* StaticData::FindTransOptListInCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase) const
|
||||
{
|
||||
std::pair<size_t, Phrase> key(decodeGraph.GetPosition(), sourcePhrase);
|
||||
@ -1365,7 +1357,7 @@ void StaticData::ReLoadParameter()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
SetWeightsForScoreProducer(*iterSP, Weights);
|
||||
SetWeights(*iterSP, Weights);
|
||||
}
|
||||
|
||||
// std::cerr << "There are " << m_phraseDictionary.size() << " m_phraseDictionaryfeatures" << std::endl;
|
||||
@ -1390,7 +1382,7 @@ void StaticData::ReLoadParameter()
|
||||
|
||||
// std::cerr << tmp_weights << std::endl;
|
||||
|
||||
SetWeightsForScoreProducer(&phraseDictionaryFeature, tmp_weights);
|
||||
SetWeights(&phraseDictionaryFeature, tmp_weights);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -35,7 +35,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#endif
|
||||
|
||||
#include "TypeDef.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
#include "FactorCollection.h"
|
||||
#include "Parameter.h"
|
||||
#include "LanguageModel.h"
|
||||
@ -82,8 +81,7 @@ protected:
|
||||
Parameter *m_parameter;
|
||||
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
|
||||
LMList m_languageModel;
|
||||
ScoreIndexManager m_scoreIndexManager;
|
||||
std::vector<float> m_allWeights;
|
||||
ScoreComponentCollection m_allWeights;
|
||||
std::vector<LexicalReordering*> m_reorderModels;
|
||||
std::vector<GlobalLexicalModel*> m_globalLexicalModels;
|
||||
std::vector<DecodeGraph*> m_decodeGraphs;
|
||||
@ -206,6 +204,8 @@ protected:
|
||||
|
||||
StaticData();
|
||||
|
||||
|
||||
|
||||
void LoadPhraseBasedParameters();
|
||||
void LoadChartDecodingParameters();
|
||||
void LoadNonTerminals();
|
||||
@ -369,27 +369,18 @@ public:
|
||||
{
|
||||
return m_translationOptionThreshold;
|
||||
}
|
||||
//! returns the total number of score components across all types, all factors
|
||||
size_t GetTotalScoreComponents() const
|
||||
{
|
||||
return m_scoreIndexManager.GetTotalNumberOfScores();
|
||||
}
|
||||
const ScoreIndexManager& GetScoreIndexManager() const
|
||||
{
|
||||
return m_scoreIndexManager;
|
||||
}
|
||||
|
||||
const TranslationSystem& GetTranslationSystem(std::string id) const {
|
||||
std::map<std::string, TranslationSystem>::const_iterator iter =
|
||||
m_translationSystems.find(id);
|
||||
VERBOSE(2, "Looking for translation system id " << id << std::endl);
|
||||
if (iter == m_translationSystems.end()) {
|
||||
VERBOSE(1, "Translation system not found " << id << std::endl);
|
||||
throw std::runtime_error("Unknown translation system id");
|
||||
} else {
|
||||
return iter->second;
|
||||
}
|
||||
const TranslationSystem& GetTranslationSystem(std::string id) const {
|
||||
std::map<std::string, TranslationSystem>::const_iterator iter =
|
||||
m_translationSystems.find(id);
|
||||
VERBOSE(2, "Looking for translation system id " << id << std::endl);
|
||||
if (iter == m_translationSystems.end()) {
|
||||
VERBOSE(1, "Translation system not found " << id << std::endl);
|
||||
throw std::runtime_error("Unknown translation system id");
|
||||
} else {
|
||||
return iter->second;
|
||||
}
|
||||
}
|
||||
size_t GetVerboseLevel() const
|
||||
{
|
||||
return m_verboseLevel;
|
||||
@ -457,16 +448,39 @@ public:
|
||||
{ return m_outputWordGraph; }
|
||||
|
||||
//! Sets the global score vector weights for a given ScoreProducer.
|
||||
void SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights);
|
||||
InputTypeEnum GetInputType() const {return m_inputType;}
|
||||
SearchAlgorithm GetSearchAlgorithm() const {return m_searchAlgorithm;}
|
||||
size_t GetNumInputScores() const {return m_numInputScores;}
|
||||
|
||||
const std::vector<float>& GetAllWeights() const
|
||||
const ScoreComponentCollection& GetAllWeights() const
|
||||
{
|
||||
return m_allWeights;
|
||||
}
|
||||
ScoreComponentCollection GetAllWeightsScoreComponentCollection() const;
|
||||
|
||||
void SetAllWeights(const ScoreComponentCollection& weights)
|
||||
{
|
||||
m_allWeights = weights;
|
||||
}
|
||||
|
||||
//Weight for a single-valued feature
|
||||
float GetWeight(const ScoreProducer* sp) const
|
||||
{
|
||||
return m_allWeights.GetScoreForProducer(sp);
|
||||
}
|
||||
|
||||
//Weight for a single-valued feature
|
||||
void SetWeight(const ScoreProducer* sp, float weight) ;
|
||||
|
||||
|
||||
//Weights for feature with fixed number of values
|
||||
std::vector<float> GetWeights(const ScoreProducer* sp) const
|
||||
{
|
||||
return m_allWeights.GetScoresForProducer(sp);
|
||||
}
|
||||
|
||||
//Weights for feature with fixed number of values
|
||||
void SetWeights(const ScoreProducer* sp, const std::vector<float>& weights);
|
||||
|
||||
|
||||
bool UseAlignmentInfo() const { return m_UseAlignmentInfo;}
|
||||
void UseAlignmentInfo(bool a){ m_UseAlignmentInfo=a; };
|
||||
|
@ -9,11 +9,6 @@ namespace Moses {
|
||||
|
||||
using namespace std;
|
||||
|
||||
TargetBigramFeature::TargetBigramFeature(ScoreIndexManager &scoreIndexManager)
|
||||
{
|
||||
scoreIndexManager.AddScoreProducer(this);
|
||||
}
|
||||
|
||||
bool TargetBigramFeature::Load(const std::string &filePath)
|
||||
{
|
||||
InputFileStream inFile(filePath);
|
||||
@ -75,7 +70,9 @@ FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo,
|
||||
map<string,size_t>::const_iterator first, second;
|
||||
if ((first = m_wordMap.find(words[i-1])) != m_wordMap.end() &&
|
||||
(second = m_wordMap.find(words[i])) != m_wordMap.end()) {
|
||||
accumulator->Assign(first->second * second->second, 1);
|
||||
cerr << "FIXME" << endl;
|
||||
assert(0);
|
||||
//accumulator->Assign(first->second * second->second, 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -13,7 +13,7 @@ namespace Moses
|
||||
*/
|
||||
class TargetBigramFeature : public StatefulFeatureFunction {
|
||||
public:
|
||||
TargetBigramFeature(ScoreIndexManager &scoreIndexManager);
|
||||
TargetBigramFeature(){}
|
||||
|
||||
bool Load(const std::string &filePath);
|
||||
|
||||
|
@ -26,7 +26,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "GenerationDictionary.h"
|
||||
#include "LanguageModel.h"
|
||||
#include "StaticData.h"
|
||||
#include "ScoreIndexManager.h"
|
||||
#include "LMList.h"
|
||||
#include "ScoreComponentCollection.h"
|
||||
#include "Util.h"
|
||||
@ -93,17 +92,8 @@ void TargetPhrase::SetScore(float score)
|
||||
const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
const ScoreProducer* prod = system.GetPhraseDictionaries()[0];
|
||||
|
||||
//get the weight list
|
||||
unsigned int id = prod->GetScoreBookkeepingID();
|
||||
|
||||
const vector<float> &allWeights = StaticData::Instance().GetAllWeights();
|
||||
vector<float> weights = StaticData::Instance().GetWeights(prod);
|
||||
|
||||
size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id);
|
||||
size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id);
|
||||
|
||||
vector<float> weights;
|
||||
|
||||
std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights));
|
||||
|
||||
//find out how many items are in the score vector for this producer
|
||||
size_t numScores = prod->GetNumScoreComponents();
|
||||
@ -125,13 +115,7 @@ void TargetPhrase::SetScore(const TranslationSystem* system, const Scores &score
|
||||
|
||||
const ScoreProducer* prod = system->GetPhraseDictionaries()[0];
|
||||
|
||||
//get the weight list
|
||||
unsigned int id = prod->GetScoreBookkeepingID();
|
||||
const vector<float> &allWeights = StaticData::Instance().GetAllWeights();
|
||||
size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id);
|
||||
size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id);
|
||||
vector<float> weights;
|
||||
std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights));
|
||||
vector<float> weights = StaticData::Instance().GetWeights(prod);
|
||||
|
||||
//expand the input weight vector
|
||||
assert(scoreVector.size() <= prod->GetNumScoreComponents());
|
||||
|
@ -153,22 +153,16 @@ namespace Moses {
|
||||
}
|
||||
|
||||
float TranslationSystem::GetWeightWordPenalty() const {
|
||||
//const ScoreComponentCollection weights = StaticData::Instance().GetAllWeights();
|
||||
size_t wpIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(m_wpProducer->GetScoreBookkeepingID());
|
||||
return StaticData::Instance().GetAllWeights()[wpIndex];
|
||||
return StaticData::Instance().GetWeight(m_wpProducer);
|
||||
}
|
||||
|
||||
float TranslationSystem::GetWeightUnknownWordPenalty() const {
|
||||
size_t uwpIndex = StaticData::Instance().GetScoreIndexManager().
|
||||
GetBeginIndex(m_unknownWpProducer->GetScoreBookkeepingID());
|
||||
return StaticData::Instance().GetAllWeights()[uwpIndex];
|
||||
return StaticData::Instance().GetWeight(m_unknownWpProducer);
|
||||
}
|
||||
|
||||
float TranslationSystem::GetWeightDistortion() const {
|
||||
assert(m_distortionScoreProducer);
|
||||
size_t distIndex = StaticData::Instance().GetScoreIndexManager().
|
||||
GetBeginIndex(m_distortionScoreProducer->GetScoreBookkeepingID());
|
||||
return StaticData::Instance().GetAllWeights()[distIndex];
|
||||
return StaticData::Instance().GetWeight(m_distortionScoreProducer);
|
||||
}
|
||||
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user