mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
code cleanup
This commit is contained in:
parent
433986a7ac
commit
326d3bd508
@ -109,7 +109,6 @@ float DynamicCacheBasedLanguageModel::Evaluate_Whole_String(const TargetPhrase&
|
||||
}
|
||||
}
|
||||
it = m_cache.find(w);
|
||||
// VERBOSE(1,"cblm::Evaluate: cheching cache for w:|" << w << "|" << std::endl);
|
||||
|
||||
if (it != m_cache.end()) { //found!
|
||||
score = ((*it).second).second;
|
||||
@ -138,7 +137,6 @@ float DynamicCacheBasedLanguageModel::Evaluate_All_Substrings(const TargetPhrase
|
||||
w += tp.GetWord(endpos).GetFactor(0)->GetString().as_string();
|
||||
it = m_cache.find(w);
|
||||
|
||||
// VERBOSE(1,"cblm::Evaluate_All_Substrings: cheching cache for w:|" << w << "|" << std::endl);
|
||||
if (it != m_cache.end()) { //found!
|
||||
score += ((*it).second).second;
|
||||
VERBOSE(3,"cblm::Evaluate_All_Substrings: found w:|" << w << "| actual score:|" << ((*it).second).second << "| score:|" << score << "|" << std::endl);
|
||||
@ -197,7 +195,7 @@ void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
VERBOSE(1,"words.size():|" << words.size() << "|" << std::endl);
|
||||
VERBOSE(3,"words.size():|" << words.size() << "|" << std::endl);
|
||||
for (size_t j=0; j<words.size(); j++) {
|
||||
words[j] = Trim(words[j]);
|
||||
VERBOSE(3,"CacheBasedLanguageModel::Update word[" << j << "]:"<< words[j] << " age:" << age << " decaying_score(age):" << decaying_score(age) << std::endl);
|
||||
@ -211,16 +209,16 @@ void DynamicCacheBasedLanguageModel::Update(std::vector<std::string> words, int
|
||||
void DynamicCacheBasedLanguageModel::Insert(std::string &entries)
|
||||
{
|
||||
if (entries != "") {
|
||||
VERBOSE(1,"entries:|" << entries << "|" << std::endl);
|
||||
VERBOSE(3,"entries:|" << entries << "|" << std::endl);
|
||||
std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||");
|
||||
VERBOSE(1,"elements.size() after:|" << elements.size() << "|" << std::endl);
|
||||
VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
|
||||
Insert(elements);
|
||||
}
|
||||
}
|
||||
|
||||
void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
|
||||
{
|
||||
VERBOSE(1,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
|
||||
VERBOSE(3,"DynamicCacheBasedLanguageModel Insert ngrams.size():|" << ngrams.size() << "|" << std::endl);
|
||||
Decay();
|
||||
Update(ngrams,1);
|
||||
// Print();
|
||||
@ -229,7 +227,7 @@ void DynamicCacheBasedLanguageModel::Insert(std::vector<std::string> ngrams)
|
||||
|
||||
void DynamicCacheBasedLanguageModel::Execute(std::string command)
|
||||
{
|
||||
VERBOSE(1,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
|
||||
VERBOSE(2,"DynamicCacheBasedLanguageModel::Execute(std::string command:|" << command << "|" << std::endl);
|
||||
std::vector<std::string> commands = Tokenize(command, "||");
|
||||
Execute(commands);
|
||||
}
|
||||
@ -244,18 +242,18 @@ void DynamicCacheBasedLanguageModel::Execute(std::vector<std::string> commands)
|
||||
|
||||
void DynamicCacheBasedLanguageModel::Execute_Single_Command(std::string command)
|
||||
{
|
||||
VERBOSE(1,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
|
||||
VERBOSE(2,"CacheBasedLanguageModel::Execute_Single_Command(std::string command:|" << command << "|" << std::endl);
|
||||
if (command == "clear") {
|
||||
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
|
||||
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Cache cleared." << std::endl);
|
||||
Clear();
|
||||
} else if (command == "settype_wholestring") {
|
||||
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
|
||||
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_WHOLESTRING << " (CBLM_QUERY_TYPE_WHOLESTRING)." << std::endl);
|
||||
SetQueryType(CBLM_QUERY_TYPE_WHOLESTRING);
|
||||
} else if (command == "settype_allsubstrings") {
|
||||
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
|
||||
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "|. Query type set to " << CBLM_QUERY_TYPE_ALLSUBSTRINGS << " (CBLM_QUERY_TYPE_ALLSUBSTRINGS)." << std::endl);
|
||||
SetQueryType(CBLM_QUERY_TYPE_ALLSUBSTRINGS);
|
||||
} else {
|
||||
VERBOSE(1,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
|
||||
VERBOSE(2,"CacheBasedLanguageModel Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -17,10 +17,8 @@ void WordPenaltyProducer::Evaluate(const Phrase &source
|
||||
, ScoreComponentCollection &scoreBreakdown
|
||||
, ScoreComponentCollection &estimatedFutureScore) const
|
||||
{
|
||||
std::cerr << "WordPenaltyProducer::Evaluate(const Phrase &source, ....) START" << std::endl;
|
||||
float score = - (float) targetPhrase.GetNumTerminals();
|
||||
scoreBreakdown.Assign(this, score);
|
||||
std::cerr << "WordPenaltyProducer::Evaluate(const Phrase &source, ....) END" << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -125,44 +125,31 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
||||
this->SetSpecifiesWeightSetting(false);
|
||||
}
|
||||
|
||||
std::cerr << "Sentence:: line before DLT:|" << line << "|" << endl;
|
||||
// std::cerr << "Sentence:: line before DLT:|" << line << "|" << endl;
|
||||
std::vector< std::map<std::string, std::string> > dlt_meta = ProcessAndStripDLT(line);
|
||||
std::cerr << "Sentence:: line after DLT:|" << line << "|" << endl;
|
||||
// std::cerr << "Sentence:: line after DLT:|" << line << "|" << endl;
|
||||
|
||||
std::vector< std::map<std::string, std::string> >::iterator dlt_meta_it = dlt_meta.begin();
|
||||
for (dlt_meta_it = dlt_meta.begin(); dlt_meta_it != dlt_meta.end(); ++dlt_meta_it) {
|
||||
std::cerr << "Sentence:: processing DLT info" << endl;
|
||||
DynamicCacheBasedLanguageModel* cblm = StaticData::Instance().GetDynamicCacheBasedLanguageModel();
|
||||
PhraseDictionaryDynamicCacheBased* cbtm = StaticData::Instance().GetPhraseDictionaryDynamicCacheBased();
|
||||
if ((*dlt_meta_it).find("cbtm") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cbtm:|" << (*dlt_meta_it)["cbtm"] << "|" << endl;
|
||||
if (cbtm) cbtm->Insert((*dlt_meta_it)["cbtm"]);
|
||||
if (cbtm) std::cerr << "Inserted:|" << (*dlt_meta_it)["cbtm"] << "|" << endl;
|
||||
}
|
||||
if ((*dlt_meta_it).find("cbtm-command") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cbtm-command:|" << (*dlt_meta_it)["cbtm-command"] << "|" << endl;
|
||||
if (cbtm) cbtm->Execute((*dlt_meta_it)["cbtm-command"]);
|
||||
if (cbtm) std::cerr << "Sentence:: Executed:|" << (*dlt_meta_it)["cbtm-command"] << "|" << endl;
|
||||
}
|
||||
if ((*dlt_meta_it).find("cbtm-file") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cbtm-file:|" << (*dlt_meta_it)["cbtm-file"] << "|" << endl;
|
||||
// if (cbtm) cbtm->Load((*dlt_meta_it)["cbtm-file"]);
|
||||
if (cbtm) std::cerr << "Sentence:: Loaded:|" << (*dlt_meta_it)["cbtm-file"] << "|" << endl;
|
||||
if (cbtm) cbtm->Load((*dlt_meta_it)["cbtm-file"]);
|
||||
}
|
||||
if ((*dlt_meta_it).find("cblm") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cblm:|" << (*dlt_meta_it)["cblm"] << "|" << endl;
|
||||
if (cblm) cblm->Insert((*dlt_meta_it)["cblm"]);
|
||||
if (cblm) std::cerr << "Inserted:|" << (*dlt_meta_it)["cblm"] << "|" << endl;
|
||||
}
|
||||
if ((*dlt_meta_it).find("cblm-command") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cblm-command:|" << (*dlt_meta_it)["cblm-command"] << "|" << endl;
|
||||
if (cblm) cblm->Execute((*dlt_meta_it)["cblm-command"]);
|
||||
if (cblm) std::cerr << "Sentence:: Executed:|" << (*dlt_meta_it)["cblm-command"] << "|" << endl;
|
||||
}
|
||||
if ((*dlt_meta_it).find("cblm-file") != (*dlt_meta_it).end()) {
|
||||
std::cerr << "Sentence:: cblm-file:|" << (*dlt_meta_it)["cblm-file"] << "|" << endl;
|
||||
if (cblm) cblm->Load((*dlt_meta_it)["cblm-file"]);
|
||||
if (cblm) std::cerr << "Sentence:: Loaded:|" << (*dlt_meta_it)["cblm-file"] << "|" << endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -104,37 +104,29 @@ void PhraseDictionaryDynamicCacheBased::InitializeForInput(InputType const& sour
|
||||
|
||||
const TargetPhraseCollection *PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const
|
||||
{
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const START" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
|
||||
#endif
|
||||
TargetPhraseCollection* tpc = NULL;
|
||||
VERBOSE(2,"source:|" << source << "|" << std::endl);
|
||||
VERBOSE(3,"source:|" << source << "|" << std::endl);
|
||||
cacheMap::const_iterator it = m_cacheTM.find(source);
|
||||
if(it != m_cacheTM.end())
|
||||
{
|
||||
VERBOSE(2,"source:|" << source << "| FOUND" << std::endl);
|
||||
VERBOSE(3,"source:|" << source << "| FOUND" << std::endl);
|
||||
tpc = (it->second).first;
|
||||
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) TPC before:|" << *tpc << "|" << std::endl);
|
||||
|
||||
std::vector<const TargetPhrase*>::const_iterator it2 = tpc->begin();
|
||||
|
||||
while (it2 != tpc->end())
|
||||
{
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) HERE I AM *(*it2) before:|" << *(*it2) << "|" << std::endl);
|
||||
((TargetPhrase*) *it2)->Evaluate(source, GetFeaturesToApply());
|
||||
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) HERE I AM *(*it2( after:|" << *(*it2) << "|" << std::endl);
|
||||
it2++;
|
||||
}
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) TPC before:|" << *tpc << "|" << std::endl);
|
||||
}
|
||||
if (tpc) {
|
||||
tpc->NthElement(m_tableLimit); // sort the phrases for the decoder
|
||||
}
|
||||
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::GetTargetPhraseCollection(const Phrase &source) const END" << std::endl);
|
||||
return tpc;
|
||||
}
|
||||
|
||||
@ -172,13 +164,11 @@ void PhraseDictionaryDynamicCacheBased::SetScoreType(size_t type) {
|
||||
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::SetMaxAge(unsigned int age) {
|
||||
VERBOSE(2, "PhraseDictionaryCache MaxAge(unsigned int age) START" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
|
||||
#endif
|
||||
m_maxAge = age;
|
||||
VERBOSE(2, "PhraseDictionaryCache MaxAge: " << m_maxAge << std::endl);
|
||||
VERBOSE(2, "PhraseDictionaryCache MaxAge(unsigned int age) END" << std::endl);
|
||||
}
|
||||
|
||||
|
||||
@ -221,8 +211,7 @@ float PhraseDictionaryDynamicCacheBased::decaying_score(const int age)
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent) START" << std::endl);
|
||||
VERBOSE(1,"m_maxAge:|" << m_maxAge << "|" << std::endl);
|
||||
VERBOSE(3,"m_maxAge:|" << m_maxAge << "|" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
@ -251,53 +240,44 @@ void PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int
|
||||
}
|
||||
precomputedScores.push_back(sc_vec);
|
||||
}
|
||||
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::SetPreComputedScores(const unsigned int numScoreComponent) END" << std::endl);
|
||||
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
}
|
||||
|
||||
Scores PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int age)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) START" << std::endl);
|
||||
VERBOSE(1,"age:|" << age << "|" << std::endl);
|
||||
VERBOSE(3,"age:|" << age << "|" << std::endl);
|
||||
if (age < precomputedScores.size())
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
|
||||
return precomputedScores.at(age);
|
||||
}
|
||||
else
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
|
||||
return precomputedScores.at(m_maxAge);
|
||||
}
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::GetPreComputedScores(const unsigned int) END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Insert(std::string &entries)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::string &entries) START" << std::endl);
|
||||
if (entries != "") {
|
||||
VERBOSE(1,"entries:|" << entries << "|" << std::endl);
|
||||
VERBOSE(3,"entries:|" << entries << "|" << std::endl);
|
||||
std::vector<std::string> elements = TokenizeMultiCharSeparator(entries, "||||");
|
||||
VERBOSE(1,"elements.size() after:|" << elements.size() << "|" << std::endl);
|
||||
VERBOSE(3,"elements.size() after:|" << elements.size() << "|" << std::endl);
|
||||
Insert(elements);
|
||||
}
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::string &entries) END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries) START" << std::endl);
|
||||
VERBOSE(1,"entries.size():|" << entries.size() << "|" << std::endl);
|
||||
VERBOSE(3,"entries.size():|" << entries.size() << "|" << std::endl);
|
||||
Decay();
|
||||
Update(entries, "1");
|
||||
IFVERBOSE(2) Print();
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Insert(std::vector<std::string> entries) END" << std::endl);
|
||||
}
|
||||
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)" << std::endl);
|
||||
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries, std::string ageString)" << std::endl);
|
||||
std::vector<std::string> pp;
|
||||
|
||||
std::vector<std::string>::iterator it;
|
||||
@ -305,8 +285,8 @@ void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries,
|
||||
{
|
||||
pp.clear();
|
||||
pp = TokenizeMultiCharSeparator((*it), "|||");
|
||||
VERBOSE(1,"pp[0]:|" << pp[0] << "|" << std::endl);
|
||||
VERBOSE(1,"pp[1]:|" << pp[1] << "|" << std::endl);
|
||||
VERBOSE(3,"pp[0]:|" << pp[0] << "|" << std::endl);
|
||||
VERBOSE(3,"pp[1]:|" << pp[1] << "|" << std::endl);
|
||||
|
||||
Update(pp[0], pp[1], ageString);
|
||||
}
|
||||
@ -314,7 +294,7 @@ void PhraseDictionaryDynamicCacheBased::Update(std::vector<std::string> entries,
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)" << std::endl);
|
||||
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, std::string targetPhraseString, std::string ageString)" << std::endl);
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const std::string& factorDelimiter = staticData.GetFactorDelimiter();
|
||||
Phrase sourcePhrase(0);
|
||||
@ -324,32 +304,32 @@ void PhraseDictionaryDynamicCacheBased::Update(std::string sourcePhraseString, s
|
||||
int age = strtod(ageString.c_str(), &err_ind_temp);
|
||||
//target
|
||||
targetPhrase.Clear();
|
||||
VERBOSE(2, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
||||
VERBOSE(3, "targetPhraseString:|" << targetPhraseString << "|" << std::endl);
|
||||
targetPhrase.CreateFromString(Output, staticData.GetOutputFactorOrder(), targetPhraseString, factorDelimiter, NULL);
|
||||
VERBOSE(2, "targetPhrase:|" << targetPhrase << "|" << std::endl);
|
||||
|
||||
//TODO: Would be better to reuse source phrases, but ownership has to be
|
||||
//consistent across phrase table implementations
|
||||
sourcePhrase.Clear();
|
||||
VERBOSE(2, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
|
||||
VERBOSE(3, "sourcePhraseString:|" << sourcePhraseString << "|" << std::endl);
|
||||
sourcePhrase.CreateFromString(Input, staticData.GetInputFactorOrder(), sourcePhraseString, factorDelimiter, NULL);
|
||||
VERBOSE(2, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
||||
VERBOSE(3, "sourcePhrase:|" << sourcePhrase << "|" << std::endl);
|
||||
Update(sourcePhrase, targetPhrase, age);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
|
||||
VERBOSE(3,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
VERBOSE(2, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
|
||||
VERBOSE(3, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
|
||||
|
||||
cacheMap::const_iterator it = m_cacheTM.find(sp);
|
||||
VERBOSE(1,"sp:|" << sp << "|" << std::endl);
|
||||
VERBOSE(3,"sp:|" << sp << "|" << std::endl);
|
||||
if(it!=m_cacheTM.end())
|
||||
{
|
||||
VERBOSE(1,"sp:|" << sp << "| FOUND" << std::endl);
|
||||
VERBOSE(3,"sp:|" << sp << "| FOUND" << std::endl);
|
||||
// p is found
|
||||
// here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap
|
||||
// and then add new entry
|
||||
@ -372,7 +352,7 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
}
|
||||
if (!found)
|
||||
{
|
||||
VERBOSE(1,"tp:|" << tp << "| NOT FOUND" << std::endl);
|
||||
VERBOSE(3,"tp:|" << tp << "| NOT FOUND" << std::endl);
|
||||
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
|
||||
|
||||
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
|
||||
@ -380,14 +360,14 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
tp_pos = tpc->GetSize()-1;
|
||||
ac->push_back(age);
|
||||
m_entries++;
|
||||
VERBOSE(1,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
|
||||
VERBOSE(1,"ac size:|" << ac->size() << "|" << std::endl);
|
||||
VERBOSE(1,"tp:|" << tp << "| INSERTED" << std::endl);
|
||||
VERBOSE(3,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
|
||||
VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl);
|
||||
VERBOSE(3,"tp:|" << tp << "| INSERTED" << std::endl);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
VERBOSE(1,"sp:|" << sp << "| NOT FOUND" << std::endl);
|
||||
VERBOSE(3,"sp:|" << sp << "| NOT FOUND" << std::endl);
|
||||
// p is not found
|
||||
// create target collection
|
||||
// we have to create new target collection age pair and add new entry to target collection age pair
|
||||
@ -395,7 +375,6 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
TargetPhraseCollection* tpc = new TargetPhraseCollection();
|
||||
AgeCollection* ac = new AgeCollection();
|
||||
m_cacheTM.insert(make_pair(sp,make_pair(tpc,ac)));
|
||||
VERBOSE(1,"HERE 1" << std::endl);
|
||||
|
||||
//tp is not found
|
||||
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
|
||||
@ -404,98 +383,14 @@ void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
tpc->Add(targetPhrase.release());
|
||||
ac->push_back(age);
|
||||
m_entries++;
|
||||
VERBOSE(1,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
|
||||
VERBOSE(1,"ac size:|" << ac->size() << "|" << std::endl);
|
||||
VERBOSE(1,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
|
||||
VERBOSE(3,"tpc size:|" << tpc->GetSize() << "|" << std::endl);
|
||||
VERBOSE(3,"ac size:|" << ac->size() << "|" << std::endl);
|
||||
VERBOSE(3,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
void PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Update(Phrase sp, Phrase tp, int age)" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
VERBOSE(2, "PhraseDictionaryCache inserting sp:|" << sp << "| tp:|" << tp << "| age:|" << age << "|" << std::endl);
|
||||
|
||||
cacheMap::const_iterator it = m_cacheTM.find(sp);
|
||||
VERBOSE(1,"sp:|" << sp << "|" << std::endl);
|
||||
if(it!=m_cacheTM.end())
|
||||
{
|
||||
VERBOSE(1,"sp:|" << sp << "| FOUND" << std::endl);
|
||||
// p is found
|
||||
// here we have to remove the target phrase from targetphrasecollection and from the TargetAgeMap
|
||||
// and then add new entry
|
||||
|
||||
TargetCollectionAgePair TgtCollAgePair = it->second;
|
||||
TargetPhraseCollection* tpc = TgtCollAgePair.first;
|
||||
TargetAgeMap* tam = TgtCollAgePair.second;
|
||||
TargetAgeMap::iterator tam_it = tam->find(tp);
|
||||
if (tam_it!=tam->end())
|
||||
{
|
||||
VERBOSE(1,"tp:|" << tp << "| FOUND" << std::endl);
|
||||
//tp is found
|
||||
size_t tp_pos = ((*tam_it).second).second;
|
||||
((*tam_it).second).first = age;
|
||||
TargetPhrase* tp_ptr = tpc->GetTargetPhrase(tp_pos);
|
||||
// tp_ptr->SetScore(m_feature,precomputedScores[age]);
|
||||
// tp_ptr->GetScoreBreakdown().Assign(m_feature, precomputedScores.at(age));
|
||||
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
|
||||
VERBOSE(1,"tp:|" << tp << "| UPDATED" << std::endl);
|
||||
}
|
||||
else
|
||||
{
|
||||
VERBOSE(1,"tp:|" << tp << "| NOT FOUND" << std::endl);
|
||||
//tp is not found
|
||||
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
|
||||
//Now that the source phrase is ready, we give the target phrase a copy
|
||||
targetPhrase->SetSourcePhrase(sp);
|
||||
// targetPhrase->SetScore(m_feature,precomputedScores[age]);
|
||||
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
|
||||
tpc->Add(targetPhrase.release());
|
||||
size_t tp_pos = tpc->GetSize()-1;
|
||||
AgePosPair app(age,tp_pos);
|
||||
TargetAgePosPair taap(tp,app);
|
||||
tam->insert(taap);
|
||||
m_entries++;
|
||||
VERBOSE(1,"tp:|" << tp << "| INSERTED" << std::endl);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
VERBOSE(1,"sp:|" << sp << "| NOT FOUND" << std::endl);
|
||||
// p is not found
|
||||
// create target collection
|
||||
// we have to create new target collection age pair and add new entry to target collection age pair
|
||||
|
||||
TargetPhraseCollection* tpc = new TargetPhraseCollection();
|
||||
TargetAgeMap* tam = new TargetAgeMap();
|
||||
m_cacheTM.insert(make_pair(sp,make_pair(tpc,tam)));
|
||||
VERBOSE(1,"HERE 1" << std::endl);
|
||||
|
||||
//tp is not found
|
||||
std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(tp));
|
||||
//Now that the source phrase is ready, we give the target phrase a copy
|
||||
targetPhrase->SetSourcePhrase(sp);
|
||||
|
||||
// targetPhrase->SetScore(m_feature,precomputedScores.at(age));
|
||||
targetPhrase->GetScoreBreakdown().Assign(this, GetPreComputedScores(age));
|
||||
tpc->Add(targetPhrase.release());
|
||||
size_t tp_pos = tpc->GetSize()-1;
|
||||
AgePosPair app(age,tp_pos);
|
||||
TargetAgePosPair taap(tp,app);
|
||||
tam->insert(taap);
|
||||
m_entries++;
|
||||
VERBOSE(1,"sp:|" << sp << "| tp:|" << tp << "| INSERTED" << std::endl);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Decay()
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay() START" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
@ -504,18 +399,16 @@ void PhraseDictionaryDynamicCacheBased::Decay()
|
||||
{
|
||||
Decay((*it).first);
|
||||
}
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay() END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) START" << std::endl);
|
||||
VERBOSE(1,"p:|" << p << "|" << std::endl);
|
||||
VERBOSE(3,"p:|" << p << "|" << std::endl);
|
||||
cacheMap::const_iterator it = m_cacheTM.find(p);
|
||||
VERBOSE(1,"searching:|" << p << "|" << std::endl);
|
||||
VERBOSE(3,"searching:|" << p << "|" << std::endl);
|
||||
if (it != m_cacheTM.end())
|
||||
{
|
||||
VERBOSE(1,"found:|" << p << "|" << std::endl);
|
||||
VERBOSE(3,"found:|" << p << "|" << std::endl);
|
||||
//p is found
|
||||
|
||||
TargetCollectionAgePair TgtCollAgePair = it->second;
|
||||
@ -525,26 +418,26 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
|
||||
//loop in inverted order to allow a correct deletion of std::vectors tpc and ac
|
||||
for (int tp_pos = tpc->GetSize() - 1 ; tp_pos >= 0; tp_pos--)
|
||||
{
|
||||
VERBOSE(1,"p:|" << p << "|" << std::endl);
|
||||
VERBOSE(3,"p:|" << p << "|" << std::endl);
|
||||
unsigned int tp_age = ac->at(tp_pos); //increase the age by 1
|
||||
tp_age++; //increase the age by 1
|
||||
VERBOSE(1,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
|
||||
VERBOSE(3,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
|
||||
|
||||
TargetPhrase* tp_ptr = (TargetPhrase*) tpc->GetTargetPhrase(tp_pos);
|
||||
VERBOSE(1,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
|
||||
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
VERBOSE(3,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
|
||||
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
|
||||
if (tp_age > m_maxAge){
|
||||
VERBOSE(1,"tp_age:|" << tp_age << "| TOO BIG" << std::endl);
|
||||
VERBOSE(3,"tp_age:|" << tp_age << "| TOO BIG" << std::endl);
|
||||
tpc->Remove(tp_pos); //delete entry in the Target Phrase Collection
|
||||
ac->erase(ac->begin() + tp_pos); //delete entry in the Age Collection
|
||||
m_entries--;
|
||||
}
|
||||
else{
|
||||
VERBOSE(1,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl);
|
||||
VERBOSE(3,"tp_age:|" << tp_age << "| STILL GOOD" << std::endl);
|
||||
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age));
|
||||
ac->at(tp_pos) = tp_age;
|
||||
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
VERBOSE(3,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
}
|
||||
}
|
||||
if (tpc->GetSize() == 0)
|
||||
@ -558,94 +451,40 @@ void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
|
||||
else
|
||||
{
|
||||
//do nothing
|
||||
VERBOSE(1,"p:|" << p << "| NOT FOUND" << std::endl);
|
||||
VERBOSE(3,"p:|" << p << "| NOT FOUND" << std::endl);
|
||||
}
|
||||
|
||||
//put here the removal of entries with age greater than m_maxAge
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) END" << std::endl);
|
||||
}
|
||||
|
||||
/*
|
||||
void PhraseDictionaryDynamicCacheBased::Decay(Phrase p)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) START" << std::endl);
|
||||
VERBOSE(1,"p:|" << p << "|" << std::endl);
|
||||
cacheMap::const_iterator it = m_cacheTM.find(p);
|
||||
VERBOSE(1,"searching:|" << p << "|" << std::endl);
|
||||
if (it != m_cacheTM.end())
|
||||
{
|
||||
VERBOSE(1,"found:|" << p << "|" << std::endl);
|
||||
//p is found
|
||||
|
||||
TargetCollectionAgePair TgtCollAgePair = it->second;
|
||||
TargetPhraseCollection* tpc = TgtCollAgePair.first;
|
||||
TargetAgeMap* tam = TgtCollAgePair.second;
|
||||
|
||||
TargetAgeMap::iterator tam_it;
|
||||
for (tam_it=tam->begin(); tam_it!=tam->end();tam_it++)
|
||||
{
|
||||
VERBOSE(1,"p:|" << p << "|" << std::endl);
|
||||
int tp_age = ((*tam_it).second).first + 1; //increase the age by 1
|
||||
|
||||
VERBOSE(1,"p:|" << p << "| " << " new tp_age:|" << tp_age << "|" << std::endl);
|
||||
((*tam_it).second).first = tp_age;
|
||||
size_t tp_pos = ((*tam_it).second).second;
|
||||
|
||||
TargetPhrase* tp_ptr = tpc->GetTargetPhrase(tp_pos);
|
||||
VERBOSE(1,"p:|" << p << "| " << "tp_age:|" << tp_age << "| " << "*tp_ptr:|" << *tp_ptr << "|" << std::endl);
|
||||
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
|
||||
tp_ptr->GetScoreBreakdown().Assign(this, GetPreComputedScores(tp_age));
|
||||
VERBOSE(1,"precomputedScores.size():|" << precomputedScores.size() << "|" << std::endl);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
//do nothing
|
||||
VERBOSE(1,"p:|" << p << "| NOT FOUND" << std::endl);
|
||||
}
|
||||
|
||||
//put here the removal of entries with age greater than m_maxAge
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Decay(Phrase p) END" << std::endl);
|
||||
}
|
||||
*/
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Execute(std::string command)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::string command) START" << std::endl);
|
||||
VERBOSE(1,"command:|" << command << "|" << std::endl);
|
||||
VERBOSE(2,"command:|" << command << "|" << std::endl);
|
||||
std::vector<std::string> commands = Tokenize(command, "||");
|
||||
Execute(commands);
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::string command) END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands) START" << std::endl);
|
||||
for (size_t j=0; j<commands.size(); j++) {
|
||||
Execute_Single_Command(commands[j]);
|
||||
}
|
||||
IFVERBOSE(2) Print();
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute(std::vector<std::string> commands) END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command)
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command) START" << std::endl);
|
||||
VERBOSE(1,"command:|" << command << "|" << std::endl);
|
||||
if (command == "clear") {
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "|. Cache cleared." << std::endl);
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "|. Cache cleared." << std::endl);
|
||||
Clear();
|
||||
} else {
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased Execute command:|"<< command << "| is unknown. Skipped." << std::endl);
|
||||
}
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Execute_Single_Command(std::string command) END" << std::endl);
|
||||
}
|
||||
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Clear()
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Clear() START" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> lock(m_cacheLock);
|
||||
#endif
|
||||
@ -658,13 +497,11 @@ void PhraseDictionaryDynamicCacheBased::Clear()
|
||||
}
|
||||
m_cacheTM.clear();
|
||||
m_entries = 0;
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Clear() END" << std::endl);
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynamicCacheBased::Print() const
|
||||
{
|
||||
VERBOSE(1,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl);
|
||||
VERBOSE(1,"entries:|" << m_entries << "|" << std::endl);
|
||||
VERBOSE(2,"PhraseDictionaryDynamicCacheBased::Print()" << std::endl);
|
||||
#ifdef WITH_THREADS
|
||||
boost::shared_lock<boost::shared_mutex> read_lock(m_cacheLock);
|
||||
#endif
|
||||
@ -677,9 +514,8 @@ void PhraseDictionaryDynamicCacheBased::Print() const
|
||||
for(itr = tpc->begin(); itr != tpc->end(); itr++)
|
||||
{
|
||||
std::string target = (*itr)->ToString();
|
||||
VERBOSE(1, source << " ||| " << target << std::endl);
|
||||
std::cout << source << " ||| " << target << std::endl;
|
||||
}
|
||||
// TPCvector.clear();
|
||||
source.clear();
|
||||
}
|
||||
}
|
||||
|
@ -113,7 +113,7 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
|
||||
std::string lline = ToLower(line);
|
||||
bool check_dlt = true;
|
||||
|
||||
std::cerr << "GLOBAL START" << endl;
|
||||
// std::cerr << "GLOBAL START" << endl;
|
||||
while (check_dlt) {
|
||||
size_t start = lline.find("<dlt");
|
||||
if (start == std::string::npos) {
|
||||
@ -129,7 +129,7 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
|
||||
}
|
||||
//std::string dlt = Trim(lline.substr(start+4, close-start-4));
|
||||
std::string dlt = Trim(line.substr(start+4, close-start-4));
|
||||
std::cerr << "dlt:|" << dlt << "|" << endl;
|
||||
// std::cerr << "dlt:|" << dlt << "|" << endl;
|
||||
line.erase(start,close-start+2);
|
||||
lline.erase(start,close-start+2);
|
||||
|
||||
@ -140,8 +140,8 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
|
||||
if (dlt[i] == '=') {
|
||||
std::string label = dlt.substr(0, i);
|
||||
std::string val = dlt.substr(i+1);
|
||||
std::cerr << "label:|" << label << "|" << endl;
|
||||
std::cerr << "val:|" << val << "|" << endl;
|
||||
// std::cerr << "label:|" << label << "|" << endl;
|
||||
// std::cerr << "val:|" << val << "|" << endl;
|
||||
if (val[0] == '"') {
|
||||
val = val.substr(1);
|
||||
// it admits any double quotation mark in the value of the attribute
|
||||
@ -171,14 +171,14 @@ std::vector< std::map<std::string, std::string> > ProcessAndStripDLT(std::string
|
||||
dlt = Trim(dlt);
|
||||
|
||||
tmp_meta[label] = val;
|
||||
std::cerr << "tmp_meta:|" << tmp_meta[label] << "|" << endl;
|
||||
// std::cerr << "tmp_meta:|" << tmp_meta[label] << "|" << endl;
|
||||
}
|
||||
}
|
||||
|
||||
meta.push_back(tmp_meta);
|
||||
}
|
||||
}
|
||||
std::cerr << "GLOBAL END" << endl;
|
||||
// std::cerr << "GLOBAL END" << endl;
|
||||
return meta;
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user