moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Nicola Bertoldi 2014-01-14 18:01:51 +01:00
parent 433986a7ac
commit 326d3bd508
5 changed files with 62 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
moses/FF/DynamicCacheBasedLanguageModel.cpp
View File

@ -109,7 +109,6 @@ float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase&
}
}
it = m_cache.find(w);
// VERBOSE(1,"cblm::Evaluate: cheching cache for w:|" << w << "|" << std::endl);
if (it != m_cache.end()) { //found!
score = ((*it).second).second;
@ -138,7 +137,6 @@ float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase
w += tp.GetWord(endpos).GetFactor(0)->GetString().as_string();
it = m_cache.find(w);
// VERBOSE(1,"cblm::Evaluate_All_Substrings: cheching cache for w:|" << w << "|" << std::endl);
if (it != m_cache.end()) { //found!
score += ((*it).second).second;
VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
@ -197,7 +195,7 @@ void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
VERBOSE(1,"words.size():|" << words.size() << "|" << std::endl);
VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
for (size_t j=0; j<words.size(); j++) {
words[j] = Trim(words[j]);
VERBOSE(3,"CacheBasedLanguageModel::Update word[" << j << "]:"<< words[j] << " age:" << age << " decaying_score(age):" << decaying_score(age) << std::endl);
@ -211,16 +209,16 @@ void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int
void DynamicCacheBasedLanguageModel::Insert(std::string &entries)
{
if (entries != "") {
VERBOSE(1,"entries:|" << entries << "|" << std::endl);
VERBOSE(3,"entries:|" << entries << "|" << std::endl);
std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
VERBOSE(1,"elements.size() after:|" << elements.size() << "|" << std::endl);
VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
Insert(elements);
}
}
void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
{
VERBOSE(1,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
VERBOSE(3,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
Decay();
Update(ngrams,1);
// Print();
@ -229,7 +227,7 @@ void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
void DynamicCacheBasedLanguageModel::Execute(std::string command)
{
VERBOSE(1,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
VERBOSE(2,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
std::vector<std::string> commands = Tokenize(command, "||");
Execute(commands);
}
@ -244,18 +242,18 @@ void DynamicCacheBasedLanguageModel::Execute(std::vector<std::string> commands)
void DynamicCacheBasedLanguageModel::Execute_Single_Command(std::string command)
{
VERBOSE(1,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
VERBOSE(2,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
if (command == "clear") {
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
Clear();
} else if (command == "settype_wholestring") {
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
SetQueryType(CBLM_QUERY_TYPE_WHOLESTRING);
} else if (command == "settype_allsubstrings") {
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
SetQueryType(CBLM_QUERY_TYPE_ALLSUBSTRINGS);
} else {
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
}
}

2
moses/FF/WordPenaltyProducer.cpp
View File

@ -17,10 +17,8 @@ void WordPenaltyProducer::Evaluate(const Phrase &source
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{
std::cerr << "WordPenaltyProducer::Evaluate(const Phrase &source, ....) START" << std::endl;
float score = - (float) targetPhrase.GetNumTerminals();
scoreBreakdown.Assign(this, score);
std::cerr << "WordPenaltyProducer::Evaluate(const Phrase &source, ....) END" << std::endl;
}
}

19
moses/Sentence.cpp
View File

@ -125,44 +125,31 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
this->SetSpecifiesWeightSetting(false);
}
std::cerr << "Sentence:: line before DLT:|" << line << "|" << endl;
// std::cerr << "Sentence:: line before DLT:|" << line << "|" << endl;
std::vector< std::map<std::string, std::string> > dlt_meta = ProcessAndStripDLT(line);
std::cerr << "Sentence:: line after DLT:|" << line << "|" << endl;
// std::cerr << "Sentence:: line after DLT:|" << line << "|" << endl;
std::vector< std::map<std::string, std::string> >::iterator dlt_meta_it = dlt_meta.begin();
for (dlt_meta_it = dlt_meta.begin(); dlt_meta_it != dlt_meta.end(); ++dlt_meta_it) {
std::cerr << "Sentence:: processing DLT info" << endl;
DynamicCacheBasedLanguageModel* cblm = StaticData::Instance().GetDynamicCacheBasedLanguageModel();
PhraseDictionaryDynamicCacheBased* cbtm = StaticData::Instance().GetPhraseDictionaryDynamicCacheBased();
if ((*dlt_meta_it).find("cbtm") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cbtm:|" << (*dlt_meta_it)["cbtm"] << "|" << endl;
if (cbtm) cbtm->Insert((*dlt_meta_it)["cbtm"]);
if (cbtm) std::cerr << "Inserted:|" << (*dlt_meta_it)["cbtm"] << "|" << endl;
}
if ((*dlt_meta_it).find("cbtm-command") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cbtm-command:|" << (*dlt_meta_it)["cbtm-command"] << "|" << endl;
if (cbtm) cbtm->Execute((*dlt_meta_it)["cbtm-command"]);
if (cbtm) std::cerr << "Sentence:: Executed:|" << (*dlt_meta_it)["cbtm-command"] << "|" << endl;
}
if ((*dlt_meta_it).find("cbtm-file") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cbtm-file:|" << (*dlt_meta_it)["cbtm-file"] << "|" << endl;
// if (cbtm) cbtm->Load((*dlt_meta_it)["cbtm-file"]);
if (cbtm) std::cerr << "Sentence:: Loaded:|" << (*dlt_meta_it)["cbtm-file"] << "|" << endl;
if (cbtm) cbtm->Load((*dlt_meta_it)["cbtm-file"]);
}
if ((*dlt_meta_it).find("cblm") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cblm:|" << (*dlt_meta_it)["cblm"] << "|" << endl;
if (cblm) cblm->Insert((*dlt_meta_it)["cblm"]);
if (cblm) std::cerr << "Inserted:|" << (*dlt_meta_it)["cblm"] << "|" << endl;
}
if ((*dlt_meta_it).find("cblm-command") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cblm-command:|" << (*dlt_meta_it)["cblm-command"] << "|" << endl;
if (cblm) cblm->Execute((*dlt_meta_it)["cblm-command"]);
if (cblm) std::cerr << "Sentence:: Executed:|" << (*dlt_meta_it)["cblm-command"] << "|" << endl;
}
if ((*dlt_meta_it).find("cblm-file") != (*dlt_meta_it).end()) {
std::cerr << "Sentence:: cblm-file:|" << (*dlt_meta_it)["cblm-file"] << "|" << endl;
if (cblm) cblm->Load((*dlt_meta_it)["cblm-file"]);
if (cblm) std::cerr << "Sentence:: Loaded:|" << (*dlt_meta_it)["cblm-file"] << "|" << endl;
}
}

250
moses/TranslationModel/PhraseDictionaryDynamicCacheBased.cpp
View File

@ -104,37 +104,29 @@ void PhraseDictionaryDynamicCacheBased::InitializeForInput(InputType const& sour
const TargetPhraseCollection *PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const
{
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const START" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
TargetPhraseCollection* tpc = NULL;
VERBOSE(2,"source:|" << source << "|" << std::endl);
VERBOSE(3,"source:|" << source << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(source);
if(it != m_cacheTM.end())
{
VERBOSE(2,"source:|" << source << "| FOUND" << std::endl);
VERBOSE(3,"source:|" << source << "| FOUND" << std::endl);
tpc = (it->second).first;
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) TPC before:|" << *tpc << "|" << std::endl);
std::vector<const TargetPhrase*>::const_iterator it2 = tpc->begin();
while (it2 != tpc->end())
{
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) HERE I AM *(*it2) before:|" << *(*it2) << "|" << std::endl);
((TargetPhrase*) *it2)->Evaluate(source, GetFeaturesToApply());
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) HERE I AM *(*it2( after:|" << *(*it2) << "|" << std::endl);
it2++;
}
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) TPC before:|" << *tpc << "|" << std::endl);
}
if (tpc) {
tpc->NthElement(m_tableLimit); // sort the phrases for the decoder
}
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const END" << std::endl);
return tpc;
}
@ -172,13 +164,11 @@ void PhraseDictionaryDynamicCacheBased::SetScoreType(size_t type) {
void PhraseDictionaryDynamicCacheBased::SetMaxAge(unsigned int age) {
VERBOSE(2, "PhraseDictionaryCache MaxAge(unsigned int age) START" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
m_maxAge = age;
VERBOSE(2, "PhraseDictionaryCache MaxAge: " << m_maxAge << std::endl);
VERBOSE(2, "PhraseDictionaryCache MaxAge(unsigned int age) END" << std::endl);
}
@ -221,8 +211,7 @@ float PhraseDictionaryDynamicCacheBased::decaying_score(const int age)
void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent) START" << std::endl);
VERBOSE(1,"m_maxAge:|" << m_maxAge << "|" << std::endl);
VERBOSE(3,"m_maxAge:|" << m_maxAge << "|" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
@ -251,53 +240,44 @@ void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int
}
precomputedScores.push_back(sc_vec);
}
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent) END" << std::endl);
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
}
Scores PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int age)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) START" << std::endl);
VERBOSE(1,"age:|" << age << "|" << std::endl);
VERBOSE(3,"age:|" << age << "|" << std::endl);
if (age < precomputedScores.size())
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
return precomputedScores.at(age);
}
else
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
return precomputedScores.at(m_maxAge);
}
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Insert(std::string &entries)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::string &entries) START" << std::endl);
if (entries != "") {
VERBOSE(1,"entries:|" << entries << "|" << std::endl);
VERBOSE(3,"entries:|" << entries << "|" << std::endl);
std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||||");
VERBOSE(1,"elements.size() after:|" << elements.size() << "|" << std::endl);
VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
Insert(elements);
}
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::string &entries) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries) START" << std::endl);
VERBOSE(1,"entries.size():|" << entries.size() << "|" << std::endl);
VERBOSE(3,"entries.size():|" << entries.size() << "|" << std::endl);
Decay();
Update(entries, "1");
IFVERBOSE(2) Print();
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)" << std::endl);
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)" << std::endl);
std::vector<std::string> pp;
std::vector<std::string>::iterator it;
@ -305,8 +285,8 @@ void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries,
{
pp.clear();
pp = TokenizeMultiCharSeparator((*it), "|||");
VERBOSE(1,"pp[0]:|" << pp[0] << "|" << std::endl);
VERBOSE(1,"pp[1]:|" << pp[1] << "|" << std::endl);
VERBOSE(3,"pp[0]:|" << pp[0] << "|" << std::endl);
VERBOSE(3,"pp[1]:|" << pp[1] << "|" << std::endl);
Update(pp[0], pp[1], ageString);
}
@ -314,7 +294,7 @@ void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries,
void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)" << std::endl);
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)" << std::endl);
const StaticData &staticData = StaticData::Instance();
const std::string& factorDelimiter = staticData.GetFactorDelimiter();
Phrase sourcePhrase(0);
@ -324,32 +304,32 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
int age = strtod(ageString.c_str(), &err_ind_temp);
//target
targetPhrase.Clear();
VERBOSE(2, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
targetPhrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetPhraseString, factorDelimiter, NULL);
VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl);
//TODO: Would be better to reuse source phrases, but ownership has to be
//consistent across phrase table implementations
sourcePhrase.Clear();
VERBOSE(2, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
VERBOSE(3, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
sourcePhrase.CreateFromString(Input, staticData.GetInputFactorOrder(), sourcePhraseString, factorDelimiter, NULL);
VERBOSE(2, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
Update(sourcePhrase, targetPhrase, age);
}
void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
VERBOSE(2, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
VERBOSE(3, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(sp);
VERBOSE(1,"sp:|" << sp << "|" << std::endl);
VERBOSE(3,"sp:|" << sp << "|" << std::endl);
if(it!=m_cacheTM.end())
{
VERBOSE(1,"sp:|" << sp << "| FOUND" << std::endl);
VERBOSE(3,"sp:|" << sp << "| FOUND" << std::endl);
// p is found
// here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap
// and then add new entry
@ -372,7 +352,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
}
if (!found)
{
VERBOSE(1,"tp:|" << tp << "| NOT FOUND" << std::endl);
VERBOSE(3,"tp:|" << tp << "| NOT FOUND" << std::endl);
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
@ -380,14 +360,14 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
tp_pos = tpc->GetSize()-1;
ac->push_back(age);
m_entries++;
VERBOSE(1,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
VERBOSE(1,"ac size:|" << ac->size() << "|" << std::endl);
VERBOSE(1,"tp:|" << tp << "| INSERTED" << std::endl);
VERBOSE(3,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl);
VERBOSE(3,"tp:|" << tp << "| INSERTED" << std::endl);
}
}
else
{
VERBOSE(1,"sp:|" << sp << "| NOT FOUND" << std::endl);
VERBOSE(3,"sp:|" << sp << "| NOT FOUND" << std::endl);
// p is not found
// create target collection
// we have to create new target collection age pair and add new entry to target collection age pair
@ -395,7 +375,6 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
TargetPhraseCollection* tpc = new TargetPhraseCollection();
AgeCollection* ac = new AgeCollection();
m_cacheTM.insert(make_pair(sp,make_pair(tpc,ac)));
VERBOSE(1,"HERE 1" << std::endl);
//tp is not found
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
@ -404,98 +383,14 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
tpc->Add(targetPhrase.release());
ac->push_back(age);
m_entries++;
VERBOSE(1,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
VERBOSE(1,"ac size:|" << ac->size() << "|" << std::endl);
VERBOSE(1,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
VERBOSE(3,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl);
VERBOSE(3,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
}
}
/*
void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
VERBOSE(2, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(sp);
VERBOSE(1,"sp:|" << sp << "|" << std::endl);
if(it!=m_cacheTM.end())
{
VERBOSE(1,"sp:|" << sp << "| FOUND" << std::endl);
// p is found
// here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap
// and then add new entry
TargetCollectionAgePair TgtCollAgePair = it->second;
TargetPhraseCollection* tpc = TgtCollAgePair.first;
TargetAgeMap* tam = TgtCollAgePair.second;
TargetAgeMap::iterator tam_it = tam->find(tp);
if (tam_it!=tam->end())
{
VERBOSE(1,"tp:|" << tp << "| FOUND" << std::endl);
//tp is found
size_t tp_pos = ((*tam_it).second).second;
((*tam_it).second).first = age;
TargetPhrase* tp_ptr = tpc->GetTargetPhrase(tp_pos);
// tp_ptr->SetScore(m_feature,precomputedScores[age]);
// tp_ptr->GetScoreBreakdown().Assign(m_feature, precomputedScores.at(age));
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
VERBOSE(1,"tp:|" << tp << "| UPDATED" << std::endl);
}
else
{
VERBOSE(1,"tp:|" << tp << "| NOT FOUND" << std::endl);
//tp is not found
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
//Now that the source phrase is ready, we give the target phrase a copy
targetPhrase->SetSourcePhrase(sp);
// targetPhrase->SetScore(m_feature,precomputedScores[age]);
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
tpc->Add(targetPhrase.release());
size_t tp_pos = tpc->GetSize()-1;
AgePosPair app(age,tp_pos);
TargetAgePosPair taap(tp,app);
tam->insert(taap);
m_entries++;
VERBOSE(1,"tp:|" << tp << "| INSERTED" << std::endl);
}
}
else
{
VERBOSE(1,"sp:|" << sp << "| NOT FOUND" << std::endl);
// p is not found
// create target collection
// we have to create new target collection age pair and add new entry to target collection age pair
TargetPhraseCollection* tpc = new TargetPhraseCollection();
TargetAgeMap* tam = new TargetAgeMap();
m_cacheTM.insert(make_pair(sp,make_pair(tpc,tam)));
VERBOSE(1,"HERE 1" << std::endl);
//tp is not found
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
//Now that the source phrase is ready, we give the target phrase a copy
targetPhrase->SetSourcePhrase(sp);
// targetPhrase->SetScore(m_feature,precomputedScores.at(age));
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
tpc->Add(targetPhrase.release());
size_t tp_pos = tpc->GetSize()-1;
AgePosPair app(age,tp_pos);
TargetAgePosPair taap(tp,app);
tam->insert(taap);
m_entries++;
VERBOSE(1,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
}
}
*/
void PhraseDictionaryDynamicCacheBased::Decay()
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay() START" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
@ -504,18 +399,16 @@ void PhraseDictionaryDynamicCacheBased::Decay()
{
Decay((*it).first);
}
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay() END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) START" << std::endl);
VERBOSE(1,"p:|" << p << "|" << std::endl);
VERBOSE(3,"p:|" << p << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(p);
VERBOSE(1,"searching:|" << p << "|" << std::endl);
VERBOSE(3,"searching:|" << p << "|" << std::endl);
if (it != m_cacheTM.end())
{
VERBOSE(1,"found:|" << p << "|" << std::endl);
VERBOSE(3,"found:|" << p << "|" << std::endl);
//p is found
TargetCollectionAgePair TgtCollAgePair = it->second;
@ -525,26 +418,26 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
//loop in inverted order to allow a correct deletion of std::vectors tpc and ac
for (int tp_pos = tpc->GetSize() - 1 ; tp_pos >= 0; tp_pos--)
{
VERBOSE(1,"p:|" << p << "|" << std::endl);
VERBOSE(3,"p:|" << p << "|" << std::endl);
unsigned int tp_age = ac->at(tp_pos); //increase the age by 1
tp_age++; //increase the age by 1
VERBOSE(1,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
VERBOSE(3,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
TargetPhrase* tp_ptr = (TargetPhrase*) tpc->GetTargetPhrase(tp_pos);
VERBOSE(1,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
VERBOSE(3,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
if (tp_age > m_maxAge){
VERBOSE(1,"tp_age:|" << tp_age << "| TOO BIG" << std::endl);
VERBOSE(3,"tp_age:|" << tp_age << "| TOO BIG" << std::endl);
tpc->Remove(tp_pos); //delete entry in the Target Phrase Collection
ac->erase(ac->begin() + tp_pos); //delete entry in the Age Collection
m_entries--;
}
else{
VERBOSE(1,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl);
VERBOSE(3,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl);
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age));
ac->at(tp_pos) = tp_age;
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
}
}
if (tpc->GetSize() == 0)
@ -558,94 +451,40 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
else
{
//do nothing
VERBOSE(1,"p:|" << p << "| NOT FOUND" << std::endl);
VERBOSE(3,"p:|" << p << "| NOT FOUND" << std::endl);
}
//put here the removal of entries with age greater than m_maxAge
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) END" << std::endl);
}
/*
void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) START" << std::endl);
VERBOSE(1,"p:|" << p << "|" << std::endl);
cacheMap::const_iterator it = m_cacheTM.find(p);
VERBOSE(1,"searching:|" << p << "|" << std::endl);
if (it != m_cacheTM.end())
{
VERBOSE(1,"found:|" << p << "|" << std::endl);
//p is found
TargetCollectionAgePair TgtCollAgePair = it->second;
TargetPhraseCollection* tpc = TgtCollAgePair.first;
TargetAgeMap* tam = TgtCollAgePair.second;
TargetAgeMap::iterator tam_it;
for (tam_it=tam->begin(); tam_it!=tam->end();tam_it++)
{
VERBOSE(1,"p:|" << p << "|" << std::endl);
int tp_age = ((*tam_it).second).first + 1; //increase the age by 1
VERBOSE(1,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
((*tam_it).second).first = tp_age;
size_t tp_pos = ((*tam_it).second).second;
TargetPhrase* tp_ptr = tpc->GetTargetPhrase(tp_pos);
VERBOSE(1,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age));
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
}
}
else
{
//do nothing
VERBOSE(1,"p:|" << p << "| NOT FOUND" << std::endl);
}
//put here the removal of entries with age greater than m_maxAge
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) END" << std::endl);
}
*/
void PhraseDictionaryDynamicCacheBased::Execute(std::string command)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::string command) START" << std::endl);
VERBOSE(1,"command:|" << command << "|" << std::endl);
VERBOSE(2,"command:|" << command << "|" << std::endl);
std::vector<std::string> commands = Tokenize(command, "||");
Execute(commands);
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::string command) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands) START" << std::endl);
for (size_t j=0; j<commands.size(); j++) {
Execute_Single_Command(commands[j]);
}
IFVERBOSE(2) Print();
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command)
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command) START" << std::endl);
VERBOSE(1,"command:|" << command << "|" << std::endl);
if (command == "clear") {
VERBOSE(1,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "|. Cache cleared." << std::endl);
VERBOSE(2,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "|. Cache cleared." << std::endl);
Clear();
} else {
VERBOSE(1,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
VERBOSE(2,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
}
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command) END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Clear()
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Clear() START" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
#endif
@ -658,13 +497,11 @@ void PhraseDictionaryDynamicCacheBased::Clear()
}
m_cacheTM.clear();
m_entries = 0;
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Clear() END" << std::endl);
}
void PhraseDictionaryDynamicCacheBased::Print() const
{
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl);
VERBOSE(1,"entries:|" << m_entries << "|" << std::endl);
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl);
#ifdef WITH_THREADS
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
#endif
@ -677,9 +514,8 @@ void PhraseDictionaryDynamicCacheBased::Print() const
for(itr = tpc->begin(); itr != tpc->end(); itr++)
{
std::string target = (*itr)->ToString();
VERBOSE(1, source << " ||| " << target << std::endl);
std::cout << source << " ||| " << target << std::endl;
}
// TPCvector.clear();
source.clear();
}
}

12
moses/Util.cpp
View File

@ -113,7 +113,7 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
std::string lline = ToLower(line);
bool check_dlt = true;
std::cerr << "GLOBAL START" << endl;
// std::cerr << "GLOBAL START" << endl;
while (check_dlt) {
size_t start = lline.find("<dlt");
if (start == std::string::npos) {
@ -129,7 +129,7 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
}
//std::string dlt = Trim(lline.substr(start+4, close-start-4));
std::string dlt = Trim(line.substr(start+4, close-start-4));
std::cerr << "dlt:|" << dlt << "|" << endl;
// std::cerr << "dlt:|" << dlt << "|" << endl;
line.erase(start,close-start+2);
lline.erase(start,close-start+2);
@ -140,8 +140,8 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
if (dlt[i] == '=') {
std::string label = dlt.substr(0, i);
std::string val = dlt.substr(i+1);
std::cerr << "label:|" << label << "|" << endl;
std::cerr << "val:|" << val << "|" << endl;
// std::cerr << "label:|" << label << "|" << endl;
// std::cerr << "val:|" << val << "|" << endl;
if (val[0] == '"') {
val = val.substr(1);
// it admits any double quotation mark in the value of the attribute
@ -171,14 +171,14 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
dlt = Trim(dlt);
tmp_meta[label] = val;
std::cerr << "tmp_meta:|" << tmp_meta[label] << "|" << endl;
// std::cerr << "tmp_meta:|" << tmp_meta[label] << "|" << endl;
}
}
meta.push_back(tmp_meta);
}
}
std::cerr << "GLOBAL END" << endl;
// std::cerr << "GLOBAL END" << endl;
return meta;
}