moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

run beautify.perl. Consistent formatting for .h & .cpp files in Mert directory

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mert-mtm5@4167 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
machacekmatous 2011-08-29 14:08:17 +00:00
parent 8b97df9367
commit 0a44787f55
39 changed files with 3122 additions and 2863 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

357
mert/BleuScorer.cpp
View File

@ -1,199 +1,204 @@
#include "BleuScorer.h" #include "BleuScorer.h"
BleuScorer::BleuScorer(const string& config = "") : StatisticsBasedScorer("BLEU",config),_refLengthStrategy(BLEU_CLOSEST) { BleuScorer::BleuScorer(const string& config = "") : StatisticsBasedScorer("BLEU",config),_refLengthStrategy(BLEU_CLOSEST)
//configure regularisation {
static string KEY_REFLEN = "reflen"; //configure regularisation
static string REFLEN_AVERAGE = "average"; static string KEY_REFLEN = "reflen";
static string REFLEN_SHORTEST = "shortest"; static string REFLEN_AVERAGE = "average";
static string REFLEN_CLOSEST = "closest"; static string REFLEN_SHORTEST = "shortest";
static string REFLEN_CLOSEST = "closest";
string reflen = getConfig(KEY_REFLEN,REFLEN_CLOSEST); string reflen = getConfig(KEY_REFLEN,REFLEN_CLOSEST);
if (reflen == REFLEN_AVERAGE) { if (reflen == REFLEN_AVERAGE) {
_refLengthStrategy = BLEU_AVERAGE; _refLengthStrategy = BLEU_AVERAGE;
} else if (reflen == REFLEN_SHORTEST) { } else if (reflen == REFLEN_SHORTEST) {
_refLengthStrategy = BLEU_SHORTEST; _refLengthStrategy = BLEU_SHORTEST;
} else if (reflen == REFLEN_CLOSEST) { } else if (reflen == REFLEN_CLOSEST) {
_refLengthStrategy = BLEU_CLOSEST; _refLengthStrategy = BLEU_CLOSEST;
} else { } else {
throw runtime_error("Unknown reference length strategy: " + reflen); throw runtime_error("Unknown reference length strategy: " + reflen);
} }
cerr << "Using reference length strategy: " << reflen << endl; cerr << "Using reference length strategy: " << reflen << endl;
static string KEY_NGRAMS = "ngramlen"; static string KEY_NGRAMS = "ngramlen";
string ngramlen = getConfig(KEY_NGRAMS,"4"); string ngramlen = getConfig(KEY_NGRAMS,"4");
LENGTH = strtol(ngramlen.c_str(), NULL, 10); LENGTH = strtol(ngramlen.c_str(), NULL, 10);
} }
/** /**
* count the ngrams of each type, up to the given length in the input line. * count the ngrams of each type, up to the given length in the input line.
**/ **/
size_t BleuScorer::countNgrams(const string& line, counts_t& counts, unsigned int n) { size_t BleuScorer::countNgrams(const string& line, counts_t& counts, unsigned int n)
vector<int> encoded_tokens; {
//cerr << line << endl; vector<int> encoded_tokens;
encode(line,encoded_tokens); //cerr << line << endl;
//copy(encoded_tokens.begin(), encoded_tokens.end(), ostream_iterator<int>(cerr," ")); encode(line,encoded_tokens);
//cerr << endl; //copy(encoded_tokens.begin(), encoded_tokens.end(), ostream_iterator<int>(cerr," "));
for (size_t k = 1; k <= n; ++k) { //cerr << endl;
//ngram order longer than sentence - no point for (size_t k = 1; k <= n; ++k) {
if (k > encoded_tokens.size()) { //ngram order longer than sentence - no point
continue; if (k > encoded_tokens.size()) {
} continue;
for (size_t i = 0; i < encoded_tokens.size()-k+1; ++i) { }
vector<int> ngram; for (size_t i = 0; i < encoded_tokens.size()-k+1; ++i) {
for (size_t j = i; j < i+k && j < encoded_tokens.size(); ++j) { vector<int> ngram;
ngram.push_back(encoded_tokens[j]); for (size_t j = i; j < i+k && j < encoded_tokens.size(); ++j) {
} ngram.push_back(encoded_tokens[j]);
int count = 1; }
counts_it oldcount = counts.find(ngram); int count = 1;
if (oldcount != counts.end()) { counts_it oldcount = counts.find(ngram);
count = (oldcount->second) + 1; if (oldcount != counts.end()) {
} count = (oldcount->second) + 1;
//cerr << count << endl; }
counts[ngram] = count; //cerr << count << endl;
//cerr << endl; counts[ngram] = count;
} //cerr << endl;
} }
//cerr << "counted ngrams" << endl; }
//dump_counts(counts); //cerr << "counted ngrams" << endl;
return encoded_tokens.size(); //dump_counts(counts);
return encoded_tokens.size();
} }
void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles) { void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles)
//make sure reference data is clear {
_refcounts.clear(); //make sure reference data is clear
_reflengths.clear(); _refcounts.clear();
_encodings.clear(); _reflengths.clear();
_encodings.clear();
//load reference data //load reference data
for (size_t i = 0; i < referenceFiles.size(); ++i) { for (size_t i = 0; i < referenceFiles.size(); ++i) {
TRACE_ERR("Loading reference from " << referenceFiles[i] << endl); TRACE_ERR("Loading reference from " << referenceFiles[i] << endl);
ifstream refin(referenceFiles[i].c_str()); ifstream refin(referenceFiles[i].c_str());
if (!refin) { if (!refin) {
throw runtime_error("Unable to open: " + referenceFiles[i]); throw runtime_error("Unable to open: " + referenceFiles[i]);
} }
string line; string line;
size_t sid = 0; //sentence counter size_t sid = 0; //sentence counter
while (getline(refin,line)) { while (getline(refin,line)) {
//cerr << line << endl; //cerr << line << endl;
if (i == 0) { if (i == 0) {
counts_t* counts = new counts_t(); //these get leaked counts_t* counts = new counts_t(); //these get leaked
_refcounts.push_back(counts); _refcounts.push_back(counts);
vector<size_t> lengths; vector<size_t> lengths;
_reflengths.push_back(lengths); _reflengths.push_back(lengths);
} }
if (_refcounts.size() <= sid) { if (_refcounts.size() <= sid) {
throw runtime_error("File " + referenceFiles[i] + " has too many sentences"); throw runtime_error("File " + referenceFiles[i] + " has too many sentences");
} }
counts_t counts; counts_t counts;
size_t length = countNgrams(line,counts,LENGTH); size_t length = countNgrams(line,counts,LENGTH);
//for any counts larger than those already there, merge them in //for any counts larger than those already there, merge them in
for (counts_it ci = counts.begin(); ci != counts.end(); ++ci) { for (counts_it ci = counts.begin(); ci != counts.end(); ++ci) {
counts_it oldcount_it = _refcounts[sid]->find(ci->first); counts_it oldcount_it = _refcounts[sid]->find(ci->first);
int oldcount = 0; int oldcount = 0;
if (oldcount_it != _refcounts[sid]->end()) { if (oldcount_it != _refcounts[sid]->end()) {
oldcount = oldcount_it->second; oldcount = oldcount_it->second;
} }
int newcount = ci->second; int newcount = ci->second;
if (newcount > oldcount) { if (newcount > oldcount) {
_refcounts[sid]->operator[](ci->first) = newcount; _refcounts[sid]->operator[](ci->first) = newcount;
} }
} }
//add in the length //add in the length
_reflengths[sid].push_back(length); _reflengths[sid].push_back(length);
if (sid > 0 && sid % 100 == 0) { if (sid > 0 && sid % 100 == 0) {
TRACE_ERR("."); TRACE_ERR(".");
} }
++sid; ++sid;
} }
refin.close(); refin.close();
TRACE_ERR(endl); TRACE_ERR(endl);
} }
} }
void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) { void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
{
// cerr << text << endl; // cerr << text << endl;
// cerr << sid << endl; // cerr << sid << endl;
//dump_counts(*_refcounts[sid]); //dump_counts(*_refcounts[sid]);
if (sid >= _refcounts.size()) { if (sid >= _refcounts.size()) {
stringstream msg; stringstream msg;
msg << "Sentence id (" << sid << ") not found in reference set"; msg << "Sentence id (" << sid << ") not found in reference set";
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }
counts_t testcounts; counts_t testcounts;
//stats for this line //stats for this line
vector<float> stats(LENGTH*2);; vector<float> stats(LENGTH*2);;
size_t length = countNgrams(text,testcounts,LENGTH); size_t length = countNgrams(text,testcounts,LENGTH);
//dump_counts(testcounts); //dump_counts(testcounts);
if (_refLengthStrategy == BLEU_SHORTEST) { if (_refLengthStrategy == BLEU_SHORTEST) {
//cerr << reflengths.size() << " " << sid << endl; //cerr << reflengths.size() << " " << sid << endl;
int shortest = *min_element(_reflengths[sid].begin(),_reflengths[sid].end()); int shortest = *min_element(_reflengths[sid].begin(),_reflengths[sid].end());
stats.push_back(shortest); stats.push_back(shortest);
} else if (_refLengthStrategy == BLEU_AVERAGE) { } else if (_refLengthStrategy == BLEU_AVERAGE) {
int total = 0; int total = 0;
for (size_t i = 0; i < _reflengths[sid].size(); ++i) { for (size_t i = 0; i < _reflengths[sid].size(); ++i) {
total += _reflengths[sid][i]; total += _reflengths[sid][i];
} }
float mean = (float)total/_reflengths[sid].size(); float mean = (float)total/_reflengths[sid].size();
stats.push_back(mean); stats.push_back(mean);
} else if (_refLengthStrategy == BLEU_CLOSEST) { } else if (_refLengthStrategy == BLEU_CLOSEST) {
int min_diff = INT_MAX; int min_diff = INT_MAX;
int min_idx = 0; int min_idx = 0;
for (size_t i = 0; i < _reflengths[sid].size(); ++i) { for (size_t i = 0; i < _reflengths[sid].size(); ++i) {
int reflength = _reflengths[sid][i]; int reflength = _reflengths[sid][i];
if (abs(reflength-(int)length) < abs(min_diff)) { //look for the closest reference if (abs(reflength-(int)length) < abs(min_diff)) { //look for the closest reference
min_diff = reflength-length; min_diff = reflength-length;
min_idx = i; min_idx = i;
}else if (abs(reflength-(int)length) == abs(min_diff)) { // if two references has the same closest length, take the shortest } else if (abs(reflength-(int)length) == abs(min_diff)) { // if two references has the same closest length, take the shortest
if (reflength < (int)_reflengths[sid][min_idx]){ if (reflength < (int)_reflengths[sid][min_idx]) {
min_idx = i; min_idx = i;
} }
} }
} }
stats.push_back(_reflengths[sid][min_idx]); stats.push_back(_reflengths[sid][min_idx]);
} else { } else {
throw runtime_error("Unsupported reflength strategy"); throw runtime_error("Unsupported reflength strategy");
} }
//cerr << "computed length" << endl; //cerr << "computed length" << endl;
//precision on each ngram type //precision on each ngram type
for (counts_it testcounts_it = testcounts.begin(); for (counts_it testcounts_it = testcounts.begin();
testcounts_it != testcounts.end(); ++testcounts_it) { testcounts_it != testcounts.end(); ++testcounts_it) {
counts_it refcounts_it = _refcounts[sid]->find(testcounts_it->first); counts_it refcounts_it = _refcounts[sid]->find(testcounts_it->first);
int correct = 0; int correct = 0;
int guess = testcounts_it->second; int guess = testcounts_it->second;
if (refcounts_it != _refcounts[sid]->end()) { if (refcounts_it != _refcounts[sid]->end()) {
correct = min(refcounts_it->second,guess); correct = min(refcounts_it->second,guess);
} }
size_t len = testcounts_it->first.size(); size_t len = testcounts_it->first.size();
stats[len*2-2] += correct; stats[len*2-2] += correct;
stats[len*2-1] += guess; stats[len*2-1] += guess;
} }
stringstream sout; stringstream sout;
copy(stats.begin(),stats.end(),ostream_iterator<float>(sout," ")); copy(stats.begin(),stats.end(),ostream_iterator<float>(sout," "));
//TRACE_ERR(sout.str() << endl); //TRACE_ERR(sout.str() << endl);
string stats_str = sout.str(); string stats_str = sout.str();
entry.set(stats_str); entry.set(stats_str);
} }
float BleuScorer::calculateScore(const vector<float>& comps) { float BleuScorer::calculateScore(const vector<float>& comps)
//cerr << "BLEU: "; {
//copy(comps.begin(),comps.end(), ostream_iterator<int>(cerr," ")); //cerr << "BLEU: ";
float logbleu = 0.0; //copy(comps.begin(),comps.end(), ostream_iterator<int>(cerr," "));
for (int i = 0; i < LENGTH; ++i) { float logbleu = 0.0;
if (comps[2*i] == 0) { for (int i = 0; i < LENGTH; ++i) {
return 0.0; if (comps[2*i] == 0) {
} return 0.0;
logbleu += log(comps[2*i]) - log(comps[2*i+1]); }
logbleu += log(comps[2*i]) - log(comps[2*i+1]);
}
logbleu /= LENGTH; }
float brevity = 1.0 - (float)comps[LENGTH*2]/comps[1];//reflength divided by test length logbleu /= LENGTH;
if (brevity < 0.0) { float brevity = 1.0 - (float)comps[LENGTH*2]/comps[1];//reflength divided by test length
logbleu += brevity; if (brevity < 0.0) {
} logbleu += brevity;
//cerr << " " << exp(logbleu) << endl; }
return exp(logbleu); //cerr << " " << exp(logbleu) << endl;
return exp(logbleu);
} }

125
mert/BleuScorer.h
View File

@ -23,73 +23,74 @@ enum BleuReferenceLengthStrategy { BLEU_AVERAGE, BLEU_SHORTEST, BLEU_CLOSEST };
/** /**
* Bleu scoring * Bleu scoring
**/ **/
class BleuScorer: public StatisticsBasedScorer { class BleuScorer: public StatisticsBasedScorer
public: {
BleuScorer(const string& config); public:
virtual void setReferenceFiles(const vector<string>& referenceFiles); BleuScorer(const string& config);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry); virtual void setReferenceFiles(const vector<string>& referenceFiles);
int LENGTH; virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
int LENGTH;
size_t NumberOfScores() const {
//cerr << "BleuScorer: " << (2 * LENGTH + 1) << endl;
return (2 * LENGTH + 1);
};
bool useAlignment() const {
//cout << "BleuScorer::useAlignment returning false" << endl;
return false;
};
size_t NumberOfScores() const {
//cerr << "BleuScorer: " << (2 * LENGTH + 1) << endl;
protected: return (2 * LENGTH + 1);
float calculateScore(const vector<float>& comps); };
bool useAlignment() const {
private: //cout << "BleuScorer::useAlignment returning false" << endl;
//no copy return false;
BleuScorer(const BleuScorer&); };
~BleuScorer(){};
BleuScorer& operator=(const BleuScorer&);
//Used to construct the ngram map
struct CompareNgrams {
int operator() (const vector<int>& a, const vector<int>& b) {
size_t i;
size_t as = a.size();
size_t bs = b.size();
for (i = 0; i < as && i < bs; ++i) {
if (a[i] < b[i]) {
//cerr << "true" << endl;
return true;
}
if (a[i] > b[i]) {
//cerr << "false" << endl;
return false;
}
}
//entries are equal, shortest wins
return as < bs;;
}
};
typedef map<vector<int>,int,CompareNgrams> counts_t;
typedef map<vector<int>,int,CompareNgrams>::iterator counts_it;
typedef vector<counts_t*> refcounts_t;
size_t countNgrams(const string& line, counts_t& counts, unsigned int n); protected:
float calculateScore(const vector<float>& comps);
void dump_counts(counts_t& counts) { private:
for (counts_it i = counts.begin(); i != counts.end(); ++i) { //no copy
cerr << "("; BleuScorer(const BleuScorer&);
copy(i->first.begin(), i->first.end(), ostream_iterator<int>(cerr," ")); ~BleuScorer() {};
cerr << ") " << i->second << ", "; BleuScorer& operator=(const BleuScorer&);
} //Used to construct the ngram map
cerr << endl; struct CompareNgrams {
} int operator() (const vector<int>& a, const vector<int>& b) {
BleuReferenceLengthStrategy _refLengthStrategy; size_t i;
size_t as = a.size();
// data extracted from reference files size_t bs = b.size();
refcounts_t _refcounts; for (i = 0; i < as && i < bs; ++i) {
vector<vector<size_t> > _reflengths; if (a[i] < b[i]) {
//cerr << "true" << endl;
return true;
}
if (a[i] > b[i]) {
//cerr << "false" << endl;
return false;
}
}
//entries are equal, shortest wins
return as < bs;;
}
};
typedef map<vector<int>,int,CompareNgrams> counts_t;
typedef map<vector<int>,int,CompareNgrams>::iterator counts_it;
typedef vector<counts_t*> refcounts_t;
size_t countNgrams(const string& line, counts_t& counts, unsigned int n);
void dump_counts(counts_t& counts) {
for (counts_it i = counts.begin(); i != counts.end(); ++i) {
cerr << "(";
copy(i->first.begin(), i->first.end(), ostream_iterator<int>(cerr," "));
cerr << ") " << i->second << ", ";
}
cerr << endl;
}
BleuReferenceLengthStrategy _refLengthStrategy;
// data extracted from reference files
refcounts_t _refcounts;
vector<vector<size_t> > _reflengths;
}; };

159
mert/Data.cpp
View File

@ -13,114 +13,113 @@
Data::Data(Scorer& ptr): Data::Data(Scorer& ptr):
theScorer(&ptr) theScorer(&ptr)
{ {
score_type = (*theScorer).getName(); score_type = (*theScorer).getName();
TRACE_ERR("Data::score_type " << score_type << std::endl); TRACE_ERR("Data::score_type " << score_type << std::endl);
TRACE_ERR("Data::Scorer type from Scorer: " << theScorer->getName() << endl); TRACE_ERR("Data::Scorer type from Scorer: " << theScorer->getName() << endl);
featdata=new FeatureData; featdata=new FeatureData;
scoredata=new ScoreData(*theScorer); scoredata=new ScoreData(*theScorer);
}; };
void Data::loadnbest(const std::string &file) void Data::loadnbest(const std::string &file)
{ {
TRACE_ERR("loading nbest from " << file << std::endl); TRACE_ERR("loading nbest from " << file << std::endl);
FeatureStats featentry; FeatureStats featentry;
ScoreStats scoreentry; ScoreStats scoreentry;
std::string sentence_index; std::string sentence_index;
inputfilestream inp(file); // matches a stream with a file. Opens the file inputfilestream inp(file); // matches a stream with a file. Opens the file
if (!inp.good()) if (!inp.good())
throw runtime_error("Unable to open: " + file); throw runtime_error("Unable to open: " + file);
std::string substring, subsubstring, stringBuf; std::string substring, subsubstring, stringBuf;
std::string theSentence; std::string theSentence;
std::string theFeatures; std::string theFeatures;
std::string theAlignment; std::string theAlignment;
std::string::size_type loc; std::string::size_type loc;
while (getline(inp,stringBuf,'\n')){ while (getline(inp,stringBuf,'\n')) {
if (stringBuf.empty()) continue; if (stringBuf.empty()) continue;
// TRACE_ERR("stringBuf: " << stringBuf << std::endl); // TRACE_ERR("stringBuf: " << stringBuf << std::endl);
getNextPound(stringBuf, substring, "|||"); //first field getNextPound(stringBuf, substring, "|||"); //first field
sentence_index = substring; sentence_index = substring;
getNextPound(stringBuf, substring, "|||"); //second field getNextPound(stringBuf, substring, "|||"); //second field
theSentence = substring; theSentence = substring;
// adding statistics for error measures // adding statistics for error measures
featentry.reset(); featentry.reset();
scoreentry.clear(); scoreentry.clear();
getNextPound(stringBuf, substring, "|||"); //third field getNextPound(stringBuf, substring, "|||"); //third field
theFeatures = substring; theFeatures = substring;
if (stringBuf.length() > 0) { if (stringBuf.length() > 0) {
getNextPound(stringBuf, substring, "|||"); //fourth field sentence score getNextPound(stringBuf, substring, "|||"); //fourth field sentence score
if (stringBuf.length() > 0) { if (stringBuf.length() > 0) {
getNextPound(stringBuf, substring, "|||"); //fourth field only there if alignment scorer getNextPound(stringBuf, substring, "|||"); //fourth field only there if alignment scorer
theAlignment = substring; theAlignment = substring;
} }
} }
//TODO check alignment exists if scorers need it //TODO check alignment exists if scorers need it
if (!theScorer->useAlignment()) { if (!theScorer->useAlignment()) {
theScorer->prepareStats(sentence_index, theSentence, scoreentry); theScorer->prepareStats(sentence_index, theSentence, scoreentry);
} else { } else {
//an interpolated score would need both sentence and alignment //an interpolated score would need both sentence and alignment
theSentence += "|||"; theSentence += "|||";
theSentence += theAlignment; theSentence += theAlignment;
theScorer->prepareStats(sentence_index, theSentence, scoreentry); theScorer->prepareStats(sentence_index, theSentence, scoreentry);
} }
scoredata->add(scoreentry, sentence_index); scoredata->add(scoreentry, sentence_index);
if (!existsFeatureNames()){ if (!existsFeatureNames()) {
std::string stringsupport=theFeatures; std::string stringsupport=theFeatures;
// adding feature names // adding feature names
std::string features=""; std::string features="";
std::string tmpname=""; std::string tmpname="";
size_t tmpidx=0;
while (!stringsupport.empty()) {
// TRACE_ERR("Decompounding: " << substring << std::endl);
getNextPound(stringsupport, subsubstring);
// string ending with ":" are skipped, because they are the names of the features
if ((loc = subsubstring.find(":")) != subsubstring.length()-1) {
features+=tmpname+"_"+stringify(tmpidx)+" ";
tmpidx++;
} else {
tmpidx=0;
tmpname=subsubstring.substr(0,subsubstring.size() - 1);
}
}
featdata->setFeatureMap(features);
}
size_t tmpidx=0;
while (!stringsupport.empty()){
// TRACE_ERR("Decompounding: " << substring << std::endl);
getNextPound(stringsupport, subsubstring);
// string ending with ":" are skipped, because they are the names of the features
if ((loc = subsubstring.find(":")) != subsubstring.length()-1){
features+=tmpname+"_"+stringify(tmpidx)+" ";
tmpidx++;
}
else{
tmpidx=0;
tmpname=subsubstring.substr(0,subsubstring.size() - 1);
}
}
featdata->setFeatureMap(features);
}
// adding features // adding features
while (!theFeatures.empty()){ while (!theFeatures.empty()) {
// TRACE_ERR("Decompounding: " << theFeatures << std::endl); // TRACE_ERR("Decompounding: " << theFeatures << std::endl);
getNextPound(theFeatures, subsubstring); getNextPound(theFeatures, subsubstring);
// string ending with ":" are skipped, because they are the names of the features // string ending with ":" are skipped, because they are the names of the features
if ((loc = subsubstring.find(":")) != subsubstring.length()-1){ if ((loc = subsubstring.find(":")) != subsubstring.length()-1) {
featentry.add(ATOFST(subsubstring.c_str())); featentry.add(ATOFST(subsubstring.c_str()));
} }
} }
featdata->add(featentry,sentence_index); featdata->add(featentry,sentence_index);
} }
inp.close(); inp.close();
} }

93
mert/Data.h
View File

@ -24,49 +24,70 @@ class Scorer;
class Data class Data
{ {
protected: protected:
ScoreData* scoredata; ScoreData* scoredata;
FeatureData* featdata; FeatureData* featdata;
private: private:
Scorer* theScorer; Scorer* theScorer;
std::string score_type; std::string score_type;
size_t number_of_scores; //number of scores size_t number_of_scores; //number of scores
public: public:
Data(Scorer& sc); Data(Scorer& sc);
~Data(){};
inline void clear() { scoredata->clear(); featdata->clear(); }
ScoreData* getScoreData() { return scoredata; };
FeatureData* getFeatureData() { return featdata; };
inline size_t NumberOfFeatures() const{ return featdata->NumberOfFeatures(); }
inline void NumberOfFeatures(size_t v){ featdata->NumberOfFeatures(v); }
inline std::string Features() const{ return featdata->Features(); }
inline void Features(const std::string f){ featdata->Features(f); }
void loadnbest(const std::string &file); ~Data() {};
void load(const std::string &featfile,const std::string &scorefile){ inline void clear() {
featdata->load(featfile); scoredata->clear();
scoredata->load(scorefile); featdata->clear();
} }
void save(const std::string &featfile,const std::string &scorefile, bool bin=false){
if (bin) cerr << "Binary write mode is selected" << endl;
else cerr << "Binary write mode is NOT selected" << endl;
featdata->save(featfile, bin);
scoredata->save(scorefile, bin);
}
inline bool existsFeatureNames(){ return featdata->existsFeatureNames(); }; ScoreData* getScoreData() {
return scoredata;
inline std::string getFeatureName(size_t idx){ return featdata->getFeatureName(idx); }; };
inline size_t getFeatureIndex(const std::string& name){ return featdata->getFeatureIndex(name); }; FeatureData* getFeatureData() {
return featdata;
};
inline size_t NumberOfFeatures() const {
return featdata->NumberOfFeatures();
}
inline void NumberOfFeatures(size_t v) {
featdata->NumberOfFeatures(v);
}
inline std::string Features() const {
return featdata->Features();
}
inline void Features(const std::string f) {
featdata->Features(f);
}
void loadnbest(const std::string &file);
void load(const std::string &featfile,const std::string &scorefile) {
featdata->load(featfile);
scoredata->load(scorefile);
}
void save(const std::string &featfile,const std::string &scorefile, bool bin=false) {
if (bin) cerr << "Binary write mode is selected" << endl;
else cerr << "Binary write mode is NOT selected" << endl;
featdata->save(featfile, bin);
scoredata->save(scorefile, bin);
}
inline bool existsFeatureNames() {
return featdata->existsFeatureNames();
};
inline std::string getFeatureName(size_t idx) {
return featdata->getFeatureName(idx);
};
inline size_t getFeatureIndex(const std::string& name) {
return featdata->getFeatureIndex(name);
};
}; };

148
mert/FeatureArray.cpp
View File

@ -16,137 +16,137 @@ FeatureArray::FeatureArray(): idx("")
void FeatureArray::savetxt(std::ofstream& outFile) void FeatureArray::savetxt(std::ofstream& outFile)
{ {
outFile << FEATURES_TXT_BEGIN << " " << idx << " " << array_.size() outFile << FEATURES_TXT_BEGIN << " " << idx << " " << array_.size()
<< " " << number_of_features << " " << features << std::endl; << " " << number_of_features << " " << features << std::endl;
for (featarray_t::iterator i = array_.begin(); i !=array_.end(); i++){ for (featarray_t::iterator i = array_.begin(); i !=array_.end(); i++) {
i->savetxt(outFile); i->savetxt(outFile);
outFile << std::endl; outFile << std::endl;
} }
outFile << FEATURES_TXT_END << std::endl; outFile << FEATURES_TXT_END << std::endl;
} }
void FeatureArray::savebin(std::ofstream& outFile) void FeatureArray::savebin(std::ofstream& outFile)
{ {
outFile << FEATURES_BIN_BEGIN << " " << idx << " " << array_.size() outFile << FEATURES_BIN_BEGIN << " " << idx << " " << array_.size()
<< " " << number_of_features << " " << features << std::endl; << " " << number_of_features << " " << features << std::endl;
for (featarray_t::iterator i = array_.begin(); i !=array_.end(); i++) for (featarray_t::iterator i = array_.begin(); i !=array_.end(); i++)
i->savebin(outFile); i->savebin(outFile);
outFile << FEATURES_BIN_END << std::endl; outFile << FEATURES_BIN_END << std::endl;
} }
void FeatureArray::save(std::ofstream& inFile, bool bin) void FeatureArray::save(std::ofstream& inFile, bool bin)
{ {
if (size()>0) if (size()>0)
(bin)?savebin(inFile):savetxt(inFile); (bin)?savebin(inFile):savetxt(inFile);
} }
void FeatureArray::save(const std::string &file, bool bin) void FeatureArray::save(const std::string &file, bool bin)
{ {
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
save(outFile); save(outFile);
outFile.close(); outFile.close();
} }
void FeatureArray::loadbin(ifstream& inFile, size_t n) void FeatureArray::loadbin(ifstream& inFile, size_t n)
{ {
FeatureStats entry(number_of_features); FeatureStats entry(number_of_features);
for (size_t i=0 ; i < n; i++){ for (size_t i=0 ; i < n; i++) {
entry.loadbin(inFile); entry.loadbin(inFile);
add(entry); add(entry);
} }
} }
void FeatureArray::loadtxt(ifstream& inFile, size_t n) void FeatureArray::loadtxt(ifstream& inFile, size_t n)
{ {
FeatureStats entry(number_of_features); FeatureStats entry(number_of_features);
for (size_t i=0 ; i < n; i++){ for (size_t i=0 ; i < n; i++) {
entry.loadtxt(inFile); entry.loadtxt(inFile);
add(entry); add(entry);
} }
} }
void FeatureArray::load(ifstream& inFile) void FeatureArray::load(ifstream& inFile)
{ {
size_t number_of_entries=0; size_t number_of_entries=0;
bool binmode=false; bool binmode=false;
std::string substring, stringBuf; std::string substring, stringBuf;
std::string::size_type loc; std::string::size_type loc;
std::getline(inFile, stringBuf); std::getline(inFile, stringBuf);
if (!inFile.good()){ if (!inFile.good()) {
return; return;
} }
if (!stringBuf.empty()){ if (!stringBuf.empty()) {
if ((loc = stringBuf.find(FEATURES_TXT_BEGIN)) == 0){ if ((loc = stringBuf.find(FEATURES_TXT_BEGIN)) == 0) {
binmode=false; binmode=false;
}else if ((loc = stringBuf.find(FEATURES_BIN_BEGIN)) == 0){ } else if ((loc = stringBuf.find(FEATURES_BIN_BEGIN)) == 0) {
binmode=true; binmode=true;
}else{ } else {
TRACE_ERR("ERROR: FeatureArray::load(): Wrong header"); TRACE_ERR("ERROR: FeatureArray::load(): Wrong header");
return; return;
} }
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
idx = substring; idx = substring;
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
number_of_entries = atoi(substring.c_str()); number_of_entries = atoi(substring.c_str());
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
number_of_features = atoi(substring.c_str()); number_of_features = atoi(substring.c_str());
features = stringBuf; features = stringBuf;
} }
(binmode)?loadbin(inFile, number_of_entries):loadtxt(inFile, number_of_entries); (binmode)?loadbin(inFile, number_of_entries):loadtxt(inFile, number_of_entries);
std::getline(inFile, stringBuf); std::getline(inFile, stringBuf);
if (!stringBuf.empty()){ if (!stringBuf.empty()) {
if ((loc = stringBuf.find(FEATURES_TXT_END)) != 0 && (loc = stringBuf.find(FEATURES_BIN_END)) != 0){ if ((loc = stringBuf.find(FEATURES_TXT_END)) != 0 && (loc = stringBuf.find(FEATURES_BIN_END)) != 0) {
TRACE_ERR("ERROR: FeatureArray::load(): Wrong footer"); TRACE_ERR("ERROR: FeatureArray::load(): Wrong footer");
return; return;
} }
} }
} }
void FeatureArray::load(const std::string &file) void FeatureArray::load(const std::string &file)
{ {
TRACE_ERR("loading data from " << file << std::endl); TRACE_ERR("loading data from " << file << std::endl);
inputfilestream inFile(file); // matches a stream with a file. Opens the file inputfilestream inFile(file); // matches a stream with a file. Opens the file
load((ifstream&) inFile); load((ifstream&) inFile);
inFile.close(); inFile.close();
} }
void FeatureArray::merge(FeatureArray& e) void FeatureArray::merge(FeatureArray& e)
{ {
//dummy implementation //dummy implementation
for (size_t i=0; i<e.size(); i++) for (size_t i=0; i<e.size(); i++)
add(e.get(i)); add(e.get(i));
} }
bool FeatureArray::check_consistency() bool FeatureArray::check_consistency()
{ {
size_t sz = NumberOfFeatures(); size_t sz = NumberOfFeatures();
if (sz == 0) if (sz == 0)
return true; return true;
for (featarray_t::iterator i=array_.begin(); i!=array_.end(); i++) for (featarray_t::iterator i=array_.begin(); i!=array_.end(); i++)
if (i->size()!=sz) if (i->size()!=sz)
return false; return false;
return true; return true;
} }

94
mert/FeatureArray.h
View File

@ -27,47 +27,71 @@ using namespace std;
class FeatureArray class FeatureArray
{ {
protected: protected:
featarray_t array_; featarray_t array_;
size_t number_of_features; size_t number_of_features;
std::string features; std::string features;
private: private:
std::string idx; // idx to identify the utterance, it can differ from the index inside the vector std::string idx; // idx to identify the utterance, it can differ from the index inside the vector
public: public:
FeatureArray(); FeatureArray();
~FeatureArray(){};
inline void clear() { array_.clear(); }
inline std::string getIndex(){ return idx; }
inline void setIndex(const std::string & value){ idx=value; }
inline FeatureStats& get(size_t i){ return array_.at(i); } ~FeatureArray() {};
inline const FeatureStats& get(size_t i)const{ return array_.at(i); }
void add(FeatureStats e){ array_.push_back(e); }
void merge(FeatureArray& e); inline void clear() {
array_.clear();
}
inline size_t size(){ return array_.size(); } inline std::string getIndex() {
inline size_t NumberOfFeatures() const{ return number_of_features; } return idx;
inline void NumberOfFeatures(size_t v){ number_of_features = v; } }
inline std::string Features() const{ return features; } inline void setIndex(const std::string & value) {
inline void Features(const std::string f){ features = f; } idx=value;
}
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile);
void save(ofstream& outFile, bool bin=false);
void save(const std::string &file, bool bin=false);
inline void save(bool bin=false){ save("/dev/stdout",bin); }
void loadtxt(ifstream& inFile, size_t n); inline FeatureStats& get(size_t i) {
void loadbin(ifstream& inFile, size_t n); return array_.at(i);
void load(ifstream& inFile); }
void load(const std::string &file); inline const FeatureStats& get(size_t i)const {
return array_.at(i);
bool check_consistency(); }
void add(FeatureStats e) {
array_.push_back(e);
}
void merge(FeatureArray& e);
inline size_t size() {
return array_.size();
}
inline size_t NumberOfFeatures() const {
return number_of_features;
}
inline void NumberOfFeatures(size_t v) {
number_of_features = v;
}
inline std::string Features() const {
return features;
}
inline void Features(const std::string f) {
features = f;
}
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile);
void save(ofstream& outFile, bool bin=false);
void save(const std::string &file, bool bin=false);
inline void save(bool bin=false) {
save("/dev/stdout",bin);
}
void loadtxt(ifstream& inFile, size_t n);
void loadbin(ifstream& inFile, size_t n);
void load(ifstream& inFile);
void load(const std::string &file);
bool check_consistency();
}; };

154
mert/FeatureData.cpp
View File

@ -18,127 +18,127 @@ FeatureData::FeatureData() {};
void FeatureData::save(std::ofstream& outFile, bool bin) void FeatureData::save(std::ofstream& outFile, bool bin)
{ {
for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++) for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++)
i->save(outFile, bin); i->save(outFile, bin);
} }
void FeatureData::save(const std::string &file, bool bin) void FeatureData::save(const std::string &file, bool bin)
{ {
if (file.empty()) return; if (file.empty()) return;
TRACE_ERR("saving the array into " << file << std::endl); TRACE_ERR("saving the array into " << file << std::endl);
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
save(outFile, bin); save(outFile, bin);
outFile.close(); outFile.close();
} }
void FeatureData::load(ifstream& inFile) void FeatureData::load(ifstream& inFile)
{ {
FeatureArray entry; FeatureArray entry;
while (!inFile.eof()){ while (!inFile.eof()) {
if (!inFile.good()){ if (!inFile.good()) {
std::cerr << "ERROR FeatureData::load inFile.good()" << std::endl; std::cerr << "ERROR FeatureData::load inFile.good()" << std::endl;
} }
entry.clear(); entry.clear();
entry.load(inFile); entry.load(inFile);
if (entry.size() == 0) if (entry.size() == 0)
break; break;
if (size() == 0){ if (size() == 0) {
setFeatureMap(entry.Features()); setFeatureMap(entry.Features());
} }
add(entry); add(entry);
} }
} }
void FeatureData::load(const std::string &file) void FeatureData::load(const std::string &file)
{ {
TRACE_ERR("loading feature data from " << file << std::endl); TRACE_ERR("loading feature data from " << file << std::endl);
inputfilestream inFile(file); // matches a stream with a file. Opens the file inputfilestream inFile(file); // matches a stream with a file. Opens the file
if (!inFile) { if (!inFile) {
throw runtime_error("Unable to open feature file: " + file); throw runtime_error("Unable to open feature file: " + file);
} }
load((ifstream&) inFile); load((ifstream&) inFile);
inFile.close(); inFile.close();
} }
void FeatureData::add(FeatureArray& e){ void FeatureData::add(FeatureArray& e)
if (exists(e.getIndex())){ // array at position e.getIndex() already exists {
//enlarge array at position e.getIndex() if (exists(e.getIndex())) { // array at position e.getIndex() already exists
size_t pos = getIndex(e.getIndex()); //enlarge array at position e.getIndex()
array_.at(pos).merge(e); size_t pos = getIndex(e.getIndex());
} array_.at(pos).merge(e);
else{ } else {
array_.push_back(e); array_.push_back(e);
setIndex(); setIndex();
} }
} }
void FeatureData::add(FeatureStats& e, const std::string & sent_idx){ void FeatureData::add(FeatureStats& e, const std::string & sent_idx)
if (exists(sent_idx)){ // array at position e.getIndex() already exists {
//enlarge array at position e.getIndex() if (exists(sent_idx)) { // array at position e.getIndex() already exists
size_t pos = getIndex(sent_idx); //enlarge array at position e.getIndex()
// TRACE_ERR("Inserting " << e << " in array " << sent_idx << std::endl); size_t pos = getIndex(sent_idx);
array_.at(pos).add(e); // TRACE_ERR("Inserting " << e << " in array " << sent_idx << std::endl);
} array_.at(pos).add(e);
else{ } else {
// TRACE_ERR("Creating a new entry in the array and inserting " << e << std::endl); // TRACE_ERR("Creating a new entry in the array and inserting " << e << std::endl);
FeatureArray a; FeatureArray a;
a.NumberOfFeatures(number_of_features); a.NumberOfFeatures(number_of_features);
a.Features(features); a.Features(features);
a.setIndex(sent_idx); a.setIndex(sent_idx);
a.add(e); a.add(e);
add(a); add(a);
} }
} }
bool FeatureData::check_consistency() bool FeatureData::check_consistency()
{ {
if (array_.size() == 0) if (array_.size() == 0)
return true; return true;
for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++)
if (!i->check_consistency()) return false;
return true; for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++)
if (!i->check_consistency()) return false;
return true;
} }
void FeatureData::setIndex() void FeatureData::setIndex()
{ {
size_t j=0; size_t j=0;
for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++){ for (featdata_t::iterator i = array_.begin(); i !=array_.end(); i++) {
idx2arrayname_[j]=(*i).getIndex(); idx2arrayname_[j]=(*i).getIndex();
arrayname2idx_[(*i).getIndex()] = j; arrayname2idx_[(*i).getIndex()] = j;
j++; j++;
} }
} }
void FeatureData::setFeatureMap(const std::string feat) void FeatureData::setFeatureMap(const std::string feat)
{ {
number_of_features = 0; number_of_features = 0;
features=feat; features=feat;
std::string substring, stringBuf; std::string substring, stringBuf;
stringBuf=features; stringBuf=features;
while (!stringBuf.empty()){ while (!stringBuf.empty()) {
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
featname2idx_[substring]=idx2featname_.size(); featname2idx_[substring]=idx2featname_.size();
idx2featname_[idx2featname_.size()]=substring; idx2featname_[idx2featname_.size()]=substring;
number_of_features++; number_of_features++;
} }
} }

176
mert/FeatureData.h
View File

@ -20,86 +20,116 @@ using namespace std;
class FeatureData class FeatureData
{ {
protected: protected:
featdata_t array_; featdata_t array_;
idx2name idx2arrayname_; //map from index to name of array idx2name idx2arrayname_; //map from index to name of array
name2idx arrayname2idx_; //map from name to index of array name2idx arrayname2idx_; //map from name to index of array
private: private:
size_t number_of_features; size_t number_of_features;
std::string features; std::string features;
map<std::string, size_t> featname2idx_; //map from name to index of features
map<size_t, std::string> idx2featname_; //map from index to name of features
map<std::string, size_t> featname2idx_; //map from name to index of features
map<size_t, std::string> idx2featname_; //map from index to name of features
public: public:
FeatureData(); FeatureData();
~FeatureData(){};
inline void clear() { array_.clear(); }
inline FeatureArray get(const std::string& idx){ return array_.at(getIndex(idx)); }
inline FeatureArray& get(size_t idx){ return array_.at(idx); }
inline const FeatureArray& get(size_t idx) const{ return array_.at(idx); }
inline bool exists(const std::string & sent_idx){ return exists(getIndex(sent_idx)); } ~FeatureData() {};
inline bool exists(int sent_idx){ return (sent_idx>-1 && sent_idx<(int) array_.size())?true:false; }
inline FeatureStats& get(size_t i, size_t j){ return array_.at(i).get(j); } inline void clear() {
inline const FeatureStats& get(size_t i, size_t j) const { return array_.at(i).get(j); } array_.clear();
void add(FeatureArray& e);
void add(FeatureStats& e, const std::string& sent_idx);
inline size_t size(){ return array_.size(); }
inline size_t NumberOfFeatures() const{ return number_of_features; }
inline void NumberOfFeatures(size_t v){ number_of_features = v; }
inline std::string Features() const{ return features; }
inline void Features(const std::string f){ features = f; }
void save(const std::string &file, bool bin=false);
void save(ofstream& outFile, bool bin=false);
inline void save(bool bin=false){ save("/dev/stdout", bin); }
void load(ifstream& inFile);
void load(const std::string &file);
bool check_consistency();
void setIndex();
inline int getIndex(const std::string& idx){
name2idx::iterator i = arrayname2idx_.find(idx);
if (i!=arrayname2idx_.end())
return i->second;
else
return -1;
} }
inline std::string getIndex(size_t idx){ inline FeatureArray get(const std::string& idx) {
idx2name::iterator i = idx2arrayname_.find(idx); return array_.at(getIndex(idx));
if (i!=idx2arrayname_.end()) }
throw runtime_error("there is no entry at index " + idx); inline FeatureArray& get(size_t idx) {
return i->second; return array_.at(idx);
} }
inline const FeatureArray& get(size_t idx) const {
return array_.at(idx);
bool existsFeatureNames(){ return (idx2featname_.size() > 0)?true:false; }; }
std::string getFeatureName(size_t idx){ inline bool exists(const std::string & sent_idx) {
if (idx >= idx2featname_.size()) return exists(getIndex(sent_idx));
throw runtime_error("Error: you required an too big index"); }
return idx2featname_[idx]; inline bool exists(int sent_idx) {
}; return (sent_idx>-1 && sent_idx<(int) array_.size())?true:false;
}
size_t getFeatureIndex(const std::string& name){
if (featname2idx_.find(name)!=featname2idx_.end()) inline FeatureStats& get(size_t i, size_t j) {
throw runtime_error("Error: feature is unknown"); return array_.at(i).get(j);
return featname2idx_[name]; }
}; inline const FeatureStats& get(size_t i, size_t j) const {
return array_.at(i).get(j);
}
void add(FeatureArray& e);
void add(FeatureStats& e, const std::string& sent_idx);
inline size_t size() {
return array_.size();
}
inline size_t NumberOfFeatures() const {
return number_of_features;
}
inline void NumberOfFeatures(size_t v) {
number_of_features = v;
}
inline std::string Features() const {
return features;
}
inline void Features(const std::string f) {
features = f;
}
void save(const std::string &file, bool bin=false);
void save(ofstream& outFile, bool bin=false);
inline void save(bool bin=false) {
save("/dev/stdout", bin);
}
void load(ifstream& inFile);
void load(const std::string &file);
bool check_consistency();
void setIndex();
inline int getIndex(const std::string& idx) {
name2idx::iterator i = arrayname2idx_.find(idx);
if (i!=arrayname2idx_.end())
return i->second;
else
return -1;
}
inline std::string getIndex(size_t idx) {
idx2name::iterator i = idx2arrayname_.find(idx);
if (i!=idx2arrayname_.end())
throw runtime_error("there is no entry at index " + idx);
return i->second;
}
bool existsFeatureNames() {
return (idx2featname_.size() > 0)?true:false;
};
std::string getFeatureName(size_t idx) {
if (idx >= idx2featname_.size())
throw runtime_error("Error: you required an too big index");
return idx2featname_[idx];
};
size_t getFeatureIndex(const std::string& name) {
if (featname2idx_.find(name)!=featname2idx_.end())
throw runtime_error("Error: feature is unknown");
return featname2idx_[name];
};
void setFeatureMap(const std::string feat); void setFeatureMap(const std::string feat);
}; };

105
mert/FeatureStats.cpp
View File

@ -14,123 +14,124 @@
FeatureStats::FeatureStats() FeatureStats::FeatureStats()
{ {
available_ = AVAILABLE_; available_ = AVAILABLE_;
entries_ = 0; entries_ = 0;
array_ = new FeatureStatsType[available_]; array_ = new FeatureStatsType[available_];
}; };
FeatureStats::~FeatureStats() FeatureStats::~FeatureStats()
{ {
delete array_; delete array_;
}; };
FeatureStats::FeatureStats(const FeatureStats &stats) FeatureStats::FeatureStats(const FeatureStats &stats)
{ {
available_ = stats.available(); available_ = stats.available();
entries_ = stats.size(); entries_ = stats.size();
array_ = new FeatureStatsType[available_]; array_ = new FeatureStatsType[available_];
memcpy(array_,stats.getArray(),featbytes_); memcpy(array_,stats.getArray(),featbytes_);
}; };
FeatureStats::FeatureStats(const size_t size) FeatureStats::FeatureStats(const size_t size)
{ {
available_ = size; available_ = size;
entries_ = size; entries_ = size;
array_ = new FeatureStatsType[available_]; array_ = new FeatureStatsType[available_];
memset(array_,0,featbytes_); memset(array_,0,featbytes_);
}; };
FeatureStats::FeatureStats(std::string &theString) FeatureStats::FeatureStats(std::string &theString)
{ {
set(theString); set(theString);
} }
void FeatureStats::expand() void FeatureStats::expand()
{ {
available_*=2; available_*=2;
featstats_t t_ = new FeatureStatsType[available_]; featstats_t t_ = new FeatureStatsType[available_];
memcpy(t_,array_,featbytes_); memcpy(t_,array_,featbytes_);
delete array_; delete array_;
array_=t_; array_=t_;
} }
void FeatureStats::add(FeatureStatsType v) void FeatureStats::add(FeatureStatsType v)
{ {
if (isfull()) expand(); if (isfull()) expand();
array_[entries_++]=v; array_[entries_++]=v;
} }
void FeatureStats::set(std::string &theString) void FeatureStats::set(std::string &theString)
{ {
std::string substring, stringBuf; std::string substring, stringBuf;
reset(); reset();
while (!theString.empty()){ while (!theString.empty()) {
getNextPound(theString, substring); getNextPound(theString, substring);
add(ATOFST(substring.c_str())); add(ATOFST(substring.c_str()));
} }
} }
void FeatureStats::loadbin(std::ifstream& inFile) void FeatureStats::loadbin(std::ifstream& inFile)
{ {
inFile.read((char*) array_, featbytes_); inFile.read((char*) array_, featbytes_);
} }
void FeatureStats::loadtxt(std::ifstream& inFile) void FeatureStats::loadtxt(std::ifstream& inFile)
{ {
std::string theString; std::string theString;
std::getline(inFile, theString); std::getline(inFile, theString);
set(theString); set(theString);
} }
void FeatureStats::loadtxt(const std::string &file) void FeatureStats::loadtxt(const std::string &file)
{ {
// TRACE_ERR("loading the stats from " << file << std::endl); // TRACE_ERR("loading the stats from " << file << std::endl);
std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file
loadtxt(inFile); loadtxt(inFile);
} }
void FeatureStats::savetxt(const std::string &file) void FeatureStats::savetxt(const std::string &file)
{ {
// TRACE_ERR("saving the stats into " << file << std::endl); // TRACE_ERR("saving the stats into " << file << std::endl);
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
savetxt(outFile); savetxt(outFile);
} }
void FeatureStats::savetxt(std::ofstream& outFile) void FeatureStats::savetxt(std::ofstream& outFile)
{ {
// TRACE_ERR("saving the stats" << std::endl); // TRACE_ERR("saving the stats" << std::endl);
outFile << *this; outFile << *this;
} }
void FeatureStats::savebin(std::ofstream& outFile) void FeatureStats::savebin(std::ofstream& outFile)
{ {
outFile.write((char*) array_, featbytes_); outFile.write((char*) array_, featbytes_);
} }
FeatureStats& FeatureStats::operator=(const FeatureStats &stats) FeatureStats& FeatureStats::operator=(const FeatureStats &stats)
{ {
delete array_; delete array_;
available_ = stats.available(); available_ = stats.available();
entries_ = stats.size(); entries_ = stats.size();
array_ = new FeatureStatsType[available_]; array_ = new FeatureStatsType[available_];
memcpy(array_,stats.getArray(),featbytes_); memcpy(array_,stats.getArray(),featbytes_);
return *this; return *this;
} }
/**write the whole object to a stream*/ /**write the whole object to a stream*/
ostream& operator<<(ostream& o, const FeatureStats& e){ ostream& operator<<(ostream& o, const FeatureStats& e)
for (size_t i=0; i< e.size(); i++) {
o << e.get(i) << " "; for (size_t i=0; i< e.size(); i++)
return o; o << e.get(i) << " ";
return o;
} }

93
mert/FeatureStats.h
View File

@ -25,46 +25,67 @@ using namespace std;
class FeatureStats class FeatureStats
{ {
private: private:
featstats_t array_; featstats_t array_;
size_t entries_; size_t entries_;
size_t available_; size_t available_;
public: public:
FeatureStats(); FeatureStats();
FeatureStats(const size_t size); FeatureStats(const size_t size);
FeatureStats(const FeatureStats &stats); FeatureStats(const FeatureStats &stats);
FeatureStats(std::string &theString); FeatureStats(std::string &theString);
FeatureStats& operator=(const FeatureStats &stats); FeatureStats& operator=(const FeatureStats &stats);
~FeatureStats();
bool isfull(){return (entries_ < available_)?0:1; }
void expand();
void add(FeatureStatsType v);
inline void clear() { memset((void*) array_,0,featbytes_); }
inline FeatureStatsType get(size_t i){ return array_[i]; }
inline FeatureStatsType get(size_t i)const{ return array_[i]; }
inline featstats_t getArray() const { return array_; }
void set(std::string &theString); ~FeatureStats();
inline size_t bytes() const{ return featbytes_; } bool isfull() {
inline size_t size() const{ return entries_; } return (entries_ < available_)?0:1;
inline size_t available() const{ return available_; } }
void expand();
void savetxt(const std::string &file); void add(FeatureStatsType v);
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile); inline void clear() {
inline void savetxt(){ savetxt("/dev/stdout"); } memset((void*) array_,0,featbytes_);
}
void loadtxt(const std::string &file);
void loadtxt(ifstream& inFile); inline FeatureStatsType get(size_t i) {
void loadbin(ifstream& inFile); return array_[i];
}
inline FeatureStatsType get(size_t i)const {
return array_[i];
}
inline featstats_t getArray() const {
return array_;
}
void set(std::string &theString);
inline size_t bytes() const {
return featbytes_;
}
inline size_t size() const {
return entries_;
}
inline size_t available() const {
return available_;
}
void savetxt(const std::string &file);
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile);
inline void savetxt() {
savetxt("/dev/stdout");
}
void loadtxt(const std::string &file);
void loadtxt(ifstream& inFile);
void loadbin(ifstream& inFile);
inline void reset() {
entries_ = 0;
clear();
}
inline void reset(){ entries_ = 0; clear(); }
/**write the whole object to a stream*/ /**write the whole object to a stream*/
friend ostream& operator<<(ostream& o, const FeatureStats& e); friend ostream& operator<<(ostream& o, const FeatureStats& e);
}; };

331
mert/InterpolatedScorer.cpp
View File

@ -5,196 +5,201 @@
using namespace std; using namespace std;
InterpolatedScorer::InterpolatedScorer (const string& name, const string& config): Scorer(name,config) { InterpolatedScorer::InterpolatedScorer (const string& name, const string& config): Scorer(name,config)
//configure regularisation {
static string KEY_WEIGHTS = "weights"; //configure regularisation
static string KEY_TYPE = "regtype"; static string KEY_WEIGHTS = "weights";
static string KEY_WINDOW = "regwin"; static string KEY_TYPE = "regtype";
static string KEY_CASE = "case"; static string KEY_WINDOW = "regwin";
static string TYPE_NONE = "none"; static string KEY_CASE = "case";
static string TYPE_AVERAGE = "average"; static string TYPE_NONE = "none";
static string TYPE_MINIMUM = "min"; static string TYPE_AVERAGE = "average";
static string TRUE = "true"; static string TYPE_MINIMUM = "min";
static string FALSE = "false"; static string TRUE = "true";
static string FALSE = "false";
string type = getConfig(KEY_TYPE,TYPE_NONE); string type = getConfig(KEY_TYPE,TYPE_NONE);
if (type == TYPE_NONE) { if (type == TYPE_NONE) {
_regularisationStrategy = REG_NONE; _regularisationStrategy = REG_NONE;
} else if (type == TYPE_AVERAGE) { } else if (type == TYPE_AVERAGE) {
_regularisationStrategy = REG_AVERAGE; _regularisationStrategy = REG_AVERAGE;
} else if (type == TYPE_MINIMUM) { } else if (type == TYPE_MINIMUM) {
_regularisationStrategy = REG_MINIMUM; _regularisationStrategy = REG_MINIMUM;
} else { } else {
throw runtime_error("Unknown scorer regularisation strategy: " + type); throw runtime_error("Unknown scorer regularisation strategy: " + type);
} }
cerr << "Using scorer regularisation strategy: " << type << endl; cerr << "Using scorer regularisation strategy: " << type << endl;
string window = getConfig(KEY_WINDOW,"0"); string window = getConfig(KEY_WINDOW,"0");
_regularisationWindow = atoi(window.c_str()); _regularisationWindow = atoi(window.c_str());
cerr << "Using scorer regularisation window: " << _regularisationWindow << endl; cerr << "Using scorer regularisation window: " << _regularisationWindow << endl;
string preservecase = getConfig(KEY_CASE,TRUE); string preservecase = getConfig(KEY_CASE,TRUE);
if (preservecase == TRUE) { if (preservecase == TRUE) {
_preserveCase = true; _preserveCase = true;
}else if (preservecase == FALSE) { } else if (preservecase == FALSE) {
_preserveCase = false; _preserveCase = false;
} }
cerr << "Using case preservation: " << _preserveCase << endl; cerr << "Using case preservation: " << _preserveCase << endl;
// name would be: HAMMING,BLEU or similar // name would be: HAMMING,BLEU or similar
string scorers = name; string scorers = name;
while (scorers.length() > 0) { while (scorers.length() > 0) {
string scorertype = ""; string scorertype = "";
getNextPound(scorers,scorertype,","); getNextPound(scorers,scorertype,",");
ScorerFactory SF; ScorerFactory SF;
Scorer *theScorer=SF.getScorer(scorertype,config); Scorer *theScorer=SF.getScorer(scorertype,config);
_scorers.push_back(theScorer); _scorers.push_back(theScorer);
} }
if (_scorers.size() == 0) { if (_scorers.size() == 0) {
throw runtime_error("There are no scorers"); throw runtime_error("There are no scorers");
} }
cout << "Number of scorers: " << _scorers.size() << endl; cout << "Number of scorers: " << _scorers.size() << endl;
//TODO debug this //TODO debug this
string wtype = getConfig(KEY_WEIGHTS,""); string wtype = getConfig(KEY_WEIGHTS,"");
//Default weights set to uniform ie. if two weights 0.5 each //Default weights set to uniform ie. if two weights 0.5 each
//weights should add to 1 //weights should add to 1
if (wtype.length() == 0) { if (wtype.length() == 0) {
float weight = 1.0/_scorers.size() ; float weight = 1.0/_scorers.size() ;
//cout << " Default weights:" << weight << endl; //cout << " Default weights:" << weight << endl;
for (size_t i = 0; i < _scorers.size(); i ++) { for (size_t i = 0; i < _scorers.size(); i ++) {
_scorerWeights.push_back(weight); _scorerWeights.push_back(weight);
}
}else{
float tot=0;
//cout << "Defined weights:" << endl;
while (wtype.length() > 0) {
string scoreweight = "";
getNextPound(wtype,scoreweight,"+");
float weight = atof(scoreweight.c_str());
_scorerWeights.push_back(weight);
tot += weight;
//cout << " :" << weight ;
}
//cout << endl;
if (tot != float(1)) {
throw runtime_error("The interpolated scorers weights do not sum to 1");
}
} }
cout << "The weights for the interpolated scorers are: " << endl; } else {
for (vector<float>::iterator it = _scorerWeights.begin(); it < _scorerWeights.end(); it++) { float tot=0;
cout << *it << " " ; //cout << "Defined weights:" << endl;
while (wtype.length() > 0) {
string scoreweight = "";
getNextPound(wtype,scoreweight,"+");
float weight = atof(scoreweight.c_str());
_scorerWeights.push_back(weight);
tot += weight;
//cout << " :" << weight ;
} }
cout <<endl; //cout << endl;
if (tot != float(1)) {
throw runtime_error("The interpolated scorers weights do not sum to 1");
}
}
cout << "The weights for the interpolated scorers are: " << endl;
for (vector<float>::iterator it = _scorerWeights.begin(); it < _scorerWeights.end(); it++) {
cout << *it << " " ;
}
cout <<endl;
} }
void InterpolatedScorer::setScoreData(ScoreData* data) { void InterpolatedScorer::setScoreData(ScoreData* data)
size_t last = 0; {
_scoreData = data; size_t last = 0;
for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end();itsc++){ _scoreData = data;
int numScoresScorer = (*itsc)->NumberOfScores(); for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
ScoreData* newData =new ScoreData(**itsc); int numScoresScorer = (*itsc)->NumberOfScores();
for (size_t i = 0; i < data->size(); i++){ ScoreData* newData =new ScoreData(**itsc);
ScoreArray scoreArray = data->get(i); for (size_t i = 0; i < data->size(); i++) {
ScoreArray newScoreArray; ScoreArray scoreArray = data->get(i);
std::string istr; ScoreArray newScoreArray;
std::stringstream out; std::string istr;
out << i; std::stringstream out;
istr = out.str(); out << i;
size_t numNBest = scoreArray.size(); istr = out.str();
//cout << " Datasize " << data->size() << " NumNBest " << numNBest << endl ; size_t numNBest = scoreArray.size();
for (size_t j = 0; j < numNBest ; j++){ //cout << " Datasize " << data->size() << " NumNBest " << numNBest << endl ;
ScoreStats scoreStats = data->get(i, j); for (size_t j = 0; j < numNBest ; j++) {
//cout << "Scorestats " << scoreStats << " i " << i << " j " << j << endl; ScoreStats scoreStats = data->get(i, j);
ScoreStats newScoreStats; //cout << "Scorestats " << scoreStats << " i " << i << " j " << j << endl;
for (size_t k = last; k < size_t(numScoresScorer + last); k++) { ScoreStats newScoreStats;
ScoreStatsType score = scoreStats.get(k); for (size_t k = last; k < size_t(numScoresScorer + last); k++) {
newScoreStats.add(score); ScoreStatsType score = scoreStats.get(k);
} newScoreStats.add(score);
//cout << " last " << last << " NumScores " << numScoresScorer << "newScorestats " << newScoreStats << endl; }
newScoreArray.add(newScoreStats); //cout << " last " << last << " NumScores " << numScoresScorer << "newScorestats " << newScoreStats << endl;
} newScoreArray.add(newScoreStats);
newScoreArray.setIndex(istr); }
newData->add(newScoreArray); newScoreArray.setIndex(istr);
} newData->add(newScoreArray);
//newData->dump(); }
(*itsc)->setScoreData(newData); //newData->dump();
last += numScoresScorer; (*itsc)->setScoreData(newData);
} last += numScoresScorer;
}
} }
/** The interpolated scorer calls a vector of scorers and combines them with /** The interpolated scorer calls a vector of scorers and combines them with
weights **/ weights **/
void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& diffs, void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) { statscores_t& scores)
{
//cout << "*******InterpolatedScorer::score" << endl; //cout << "*******InterpolatedScorer::score" << endl;
size_t scorerNum = 0; size_t scorerNum = 0;
for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end();itsc++){ for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
int numScores = (*itsc)->NumberOfScores(); int numScores = (*itsc)->NumberOfScores();
statscores_t tscores; statscores_t tscores;
(*itsc)->score(candidates,diffs,tscores); (*itsc)->score(candidates,diffs,tscores);
size_t inc = 0; size_t inc = 0;
for (statscores_t::iterator itstatsc = tscores.begin(); itstatsc!=tscores.end();itstatsc++){ for (statscores_t::iterator itstatsc = tscores.begin(); itstatsc!=tscores.end(); itstatsc++) {
//cout << "Scores " << (*itstatsc) << endl; //cout << "Scores " << (*itstatsc) << endl;
float weight = _scorerWeights[scorerNum]; float weight = _scorerWeights[scorerNum];
if (weight == 0) { if (weight == 0) {
stringstream msg; stringstream msg;
msg << "No weights for scorer" << scorerNum ; msg << "No weights for scorer" << scorerNum ;
throw runtime_error(msg.str()); throw runtime_error(msg.str());
} }
if (scorerNum == 0) { if (scorerNum == 0) {
scores.push_back(weight * (*itstatsc)); scores.push_back(weight * (*itstatsc));
} else { } else {
scores[inc] += weight * (*itstatsc); scores[inc] += weight * (*itstatsc);
} }
//cout << "Scorer:" << scorerNum << " scoreNum:" << inc << " score: " << (*itstatsc) << " weight:" << weight << endl; //cout << "Scorer:" << scorerNum << " scoreNum:" << inc << " score: " << (*itstatsc) << " weight:" << weight << endl;
inc++; inc++;
} }
scorerNum++; scorerNum++;
} }
} }
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles) { void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
for (vector<Scorer *>::iterator itsc = _scorers.begin(); itsc!=_scorers.end();itsc++){ {
//the scorers that use alignments use the reference files in the constructor through config for (vector<Scorer *>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
(*itsc)->setReferenceFiles(referenceFiles); //the scorers that use alignments use the reference files in the constructor through config
} (*itsc)->setReferenceFiles(referenceFiles);
}
} }
// Text can be: // Text can be:
// Reference sentence ||| Reference sentence alignment information (as given by MOSES -include-alignment-in-n-best) // Reference sentence ||| Reference sentence alignment information (as given by MOSES -include-alignment-in-n-best)
// If a permutation distance scorer, send alignment info // If a permutation distance scorer, send alignment info
// Else if other scorer, remove the alignment info and then send reference as usual // Else if other scorer, remove the alignment info and then send reference as usual
void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) { void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
stringstream buff; {
string align = text; stringstream buff;
string sentence = ""; string align = text;
size_t alignmentData = text.find("|||"); string sentence = "";
//Get sentence and alignment parts size_t alignmentData = text.find("|||");
if(alignmentData != string::npos) { //Get sentence and alignment parts
getNextPound(align,sentence, "|||"); if(alignmentData != string::npos) {
} getNextPound(align,sentence, "|||");
int i=0; }
for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end();itsc++){ int i=0;
ScoreStats tempEntry; for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
if ((*itsc)->useAlignment()) { ScoreStats tempEntry;
(*itsc)->prepareStats(sid, text, tempEntry); if ((*itsc)->useAlignment()) {
} else { (*itsc)->prepareStats(sid, text, tempEntry);
(*itsc)->prepareStats(sid, sentence, tempEntry); } else {
} (*itsc)->prepareStats(sid, sentence, tempEntry);
if (i > 0) buff << " ";
buff << tempEntry;
i++;
} }
//cout << " Scores for interpolated: " << buff << endl; if (i > 0) buff << " ";
string str = buff.str(); buff << tempEntry;
entry.set(str); i++;
}
//cout << " Scores for interpolated: " << buff << endl;
string str = buff.str();
entry.set(str);
} }

73
mert/InterpolatedScorer.h
View File

@ -18,48 +18,49 @@
/** /**
* Abstract base class for scorers that include other scorers eg. * Abstract base class for scorers that include other scorers eg.
* Interpolated HAMMING and BLEU scorer **/ * Interpolated HAMMING and BLEU scorer **/
class InterpolatedScorer : public Scorer { class InterpolatedScorer : public Scorer
{
public: public:
// name would be: "HAMMING,BLEU" or similar // name would be: "HAMMING,BLEU" or similar
InterpolatedScorer(const string& name, const string& config); InterpolatedScorer(const string& name, const string& config);
~InterpolatedScorer(){}; ~InterpolatedScorer() {};
void score(const candidates_t& candidates, const diffs_t& diffs, void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores); statscores_t& scores);
void setReferenceFiles(const vector<string>& referenceFiles); void setReferenceFiles(const vector<string>& referenceFiles);
void prepareStats(size_t sid, const string& text, ScoreStats& entry); void prepareStats(size_t sid, const string& text, ScoreStats& entry);
size_t NumberOfScores() const { size_t NumberOfScores() const {
size_t sz=0; size_t sz=0;
for (vector<Scorer*>::const_iterator itsc = _scorers.begin(); itsc < _scorers.end();itsc++){ for (vector<Scorer*>::const_iterator itsc = _scorers.begin(); itsc < _scorers.end(); itsc++) {
sz += (*itsc)->NumberOfScores(); sz += (*itsc)->NumberOfScores();
} }
return sz; return sz;
}; };
bool useAlignment() const {
//cout << "InterpolatedScorer::useAlignment" << endl;
for (vector<Scorer*>::const_iterator itsc = _scorers.begin(); itsc < _scorers.end();itsc++){
if ((*itsc)->useAlignment()){
//cout <<"InterpolatedScorer::useAlignment Returning true"<<endl;
return true;
}
}
return false;
};
//calculate the actual score - this gets done in the individual scorers bool useAlignment() const {
//statscore_t calculateScore(const vector<statscore_t>& totals); //cout << "InterpolatedScorer::useAlignment" << endl;
void setScoreData(ScoreData* data); for (vector<Scorer*>::const_iterator itsc = _scorers.begin(); itsc < _scorers.end(); itsc++) {
if ((*itsc)->useAlignment()) {
//cout <<"InterpolatedScorer::useAlignment Returning true"<<endl;
return true;
}
}
return false;
};
protected: //calculate the actual score - this gets done in the individual scorers
//statscore_t calculateScore(const vector<statscore_t>& totals);
void setScoreData(ScoreData* data);
//regularisation protected:
ScorerRegularisationStrategy _regularisationStrategy;
size_t _regularisationWindow;
vector<Scorer*> _scorers; //regularisation
vector<float> _scorerWeights; ScorerRegularisationStrategy _regularisationStrategy;
size_t _regularisationWindow;
vector<Scorer*> _scorers;
vector<float> _scorerWeights;
}; };

415
mert/Optimizer.cpp
View File

@ -14,31 +14,34 @@ static const float MAX_FLOAT=numeric_limits<float>::max();
void Optimizer::SetScorer(Scorer *S){ void Optimizer::SetScorer(Scorer *S)
{
if(scorer) if(scorer)
delete scorer; delete scorer;
scorer=S; scorer=S;
} }
void Optimizer::SetFData(FeatureData *F){ void Optimizer::SetFData(FeatureData *F)
{
if(FData) if(FData)
delete FData; delete FData;
FData=F; FData=F;
}; };
Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start):scorer(NULL),FData(NULL){ Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start):scorer(NULL),FData(NULL)
{
//warning: the init vector is a full set of parameters, of dimension pdim! //warning: the init vector is a full set of parameters, of dimension pdim!
Point::pdim=Pd; Point::pdim=Pd;
assert(start.size()==Pd); assert(start.size()==Pd);
Point::dim=i2O.size(); Point::dim=i2O.size();
Point::optindices=i2O; Point::optindices=i2O;
if (Point::pdim>Point::dim){ if (Point::pdim>Point::dim) {
for (unsigned int i=0;i<Point::pdim;i++){ for (unsigned int i=0; i<Point::pdim; i++) {
unsigned int j = 0; unsigned int j = 0;
while (j<Point::dim && i!=i2O[j]) while (j<Point::dim && i!=i2O[j])
j++; j++;
if (j==Point::dim)//the index i wasnt found on optindices, it is a fixed index, we use the value of the start vector if (j==Point::dim)//the index i wasnt found on optindices, it is a fixed index, we use the value of the start vector
Point::fixedweights[i]=start[i]; Point::fixedweights[i]=start[i];
@ -46,12 +49,14 @@ Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start)
} }
}; };
Optimizer::~Optimizer(){ Optimizer::~Optimizer()
{
delete scorer; delete scorer;
delete FData; delete FData;
} }
statscore_t Optimizer::GetStatScore(const Point& param)const{ statscore_t Optimizer::GetStatScore(const Point& param)const
{
vector<unsigned> bests; vector<unsigned> bests;
Get1bests(param,bests); Get1bests(param,bests);
//copy(bests.begin(),bests.end(),ostream_iterator<unsigned>(cerr," ")); //copy(bests.begin(),bests.end(),ostream_iterator<unsigned>(cerr," "));
@ -60,23 +65,25 @@ statscore_t Optimizer::GetStatScore(const Point& param)const{
}; };
/**compute the intersection of 2 lines*/ /**compute the intersection of 2 lines*/
float intersect (float m1, float b1,float m2,float b2){ float intersect (float m1, float b1,float m2,float b2)
{
float isect = ((b2-b1)/(m1-m2)); float isect = ((b2-b1)/(m1-m2));
if (!isfinite(isect)) { if (!isfinite(isect)) {
isect = MAX_FLOAT; isect = MAX_FLOAT;
} }
return isect; return isect;
} }
map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float newt,pair<unsigned,unsigned> newdiff){ map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float newt,pair<unsigned,unsigned> newdiff)
{
map<float,diff_t>::iterator it=thresholdmap.find(newt); map<float,diff_t>::iterator it=thresholdmap.find(newt);
if(it!=thresholdmap.end()){ if(it!=thresholdmap.end()) {
//the threshold already exists!! this is very unlikely //the threshold already exists!! this is very unlikely
if(it->second.back().first==newdiff.first) if(it->second.back().first==newdiff.first)
it->second.back().second=newdiff.second;//there was already a diff for this sentence, we change the 1 best; it->second.back().second=newdiff.second;//there was already a diff for this sentence, we change the 1 best;
else else
it->second.push_back(newdiff); it->second.push_back(newdiff);
}else{ } else {
//normal case //normal case
pair< map<float,diff_t >::iterator,bool > ins=thresholdmap.insert(threshold(newt,diff_t(1,newdiff))); pair< map<float,diff_t >::iterator,bool > ins=thresholdmap.insert(threshold(newt,diff_t(1,newdiff)));
assert(ins.second);//we really inserted something assert(ins.second);//we really inserted something
@ -86,244 +93,247 @@ map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float
}; };
statscore_t Optimizer::LineOptimize(const Point& origin,const Point& direction,Point& bestpoint)const{ statscore_t Optimizer::LineOptimize(const Point& origin,const Point& direction,Point& bestpoint)const
{
// we are looking for the best Point on the line y=Origin+x*direction // we are looking for the best Point on the line y=Origin+x*direction
float min_int=0.0001; float min_int=0.0001;
//typedef pair<unsigned,unsigned> diff;//first the sentence that changes, second is the new 1best for this sentence //typedef pair<unsigned,unsigned> diff;//first the sentence that changes, second is the new 1best for this sentence
//list<threshold> thresholdlist; //list<threshold> thresholdlist;
map<float,diff_t> thresholdmap; map<float,diff_t> thresholdmap;
thresholdmap[MIN_FLOAT]=diff_t(); thresholdmap[MIN_FLOAT]=diff_t();
vector<unsigned> first1best;//the vector of nbests for x=-inf vector<unsigned> first1best;//the vector of nbests for x=-inf
for(unsigned int S=0;S<size();S++){ for(unsigned int S=0; S<size(); S++) {
map<float,diff_t >::iterator previnserted=thresholdmap.begin(); map<float,diff_t >::iterator previnserted=thresholdmap.begin();
//first we determine the translation with the best feature score for each sentence and each value of x //first we determine the translation with the best feature score for each sentence and each value of x
//cerr << "Sentence " << S << endl; //cerr << "Sentence " << S << endl;
multimap<float,unsigned> gradient; multimap<float,unsigned> gradient;
vector<float> f0; vector<float> f0;
f0.resize(FData->get(S).size()); f0.resize(FData->get(S).size());
for(unsigned j=0;j<FData->get(S).size();j++){ for(unsigned j=0; j<FData->get(S).size(); j++) {
gradient.insert(pair<float,unsigned>(direction*(FData->get(S,j)),j));//gradient of the feature function for this particular target sentence gradient.insert(pair<float,unsigned>(direction*(FData->get(S,j)),j));//gradient of the feature function for this particular target sentence
f0[j]=origin*FData->get(S,j);//compute the feature function at the origin point f0[j]=origin*FData->get(S,j);//compute the feature function at the origin point
} }
//now lets compute the 1best for each value of x //now lets compute the 1best for each value of x
// vector<pair<float,unsigned> > onebest; // vector<pair<float,unsigned> > onebest;
multimap<float,unsigned>::iterator gradientit=gradient.begin(); multimap<float,unsigned>::iterator gradientit=gradient.begin();
multimap<float,unsigned>::iterator highest_f0=gradient.begin(); multimap<float,unsigned>::iterator highest_f0=gradient.begin();
float smallest=gradientit->first;//smallest gradient float smallest=gradientit->first;//smallest gradient
//several candidates can have the lowest slope (eg for word penalty where the gradient is an integer ) //several candidates can have the lowest slope (eg for word penalty where the gradient is an integer )
gradientit++; gradientit++;
while(gradientit!=gradient.end()&&gradientit->first==smallest){ while(gradientit!=gradient.end()&&gradientit->first==smallest) {
// cerr<<"ni"<<gradientit->second<<endl;; // cerr<<"ni"<<gradientit->second<<endl;;
//cerr<<"fos"<<f0[gradientit->second]<<" "<<f0[index]<<" "<<index<<endl; //cerr<<"fos"<<f0[gradientit->second]<<" "<<f0[index]<<" "<<index<<endl;
if(f0[gradientit->second]>f0[highest_f0->second]) if(f0[gradientit->second]>f0[highest_f0->second])
highest_f0=gradientit;//the highest line is the one with he highest f0 highest_f0=gradientit;//the highest line is the one with he highest f0
gradientit++; gradientit++;
} }
gradientit = highest_f0; gradientit = highest_f0;
first1best.push_back(highest_f0->second); first1best.push_back(highest_f0->second);
//now we look for the intersections points indicating a change of 1 best //now we look for the intersections points indicating a change of 1 best
//we use the fact that the function is convex, which means that the gradient can only go up //we use the fact that the function is convex, which means that the gradient can only go up
while(gradientit!=gradient.end()){ while(gradientit!=gradient.end()) {
map<float,unsigned>::iterator leftmost=gradientit; map<float,unsigned>::iterator leftmost=gradientit;
float m=gradientit->first; float m=gradientit->first;
float b=f0[gradientit->second]; float b=f0[gradientit->second];
multimap<float,unsigned>::iterator gradientit2=gradientit; multimap<float,unsigned>::iterator gradientit2=gradientit;
gradientit2++; gradientit2++;
float leftmostx=MAX_FLOAT; float leftmostx=MAX_FLOAT;
for(;gradientit2!=gradient.end();gradientit2++){ for(; gradientit2!=gradient.end(); gradientit2++) {
//cerr<<"--"<<d++<<' '<<gradientit2->first<<' '<<gradientit2->second<<endl; //cerr<<"--"<<d++<<' '<<gradientit2->first<<' '<<gradientit2->second<<endl;
//look for all candidate with a gradient bigger than the current one and find the one with the leftmost intersection //look for all candidate with a gradient bigger than the current one and find the one with the leftmost intersection
float curintersect; float curintersect;
if(m!=gradientit2->first){ if(m!=gradientit2->first) {
curintersect=intersect(m,b,gradientit2->first,f0[gradientit2->second]); curintersect=intersect(m,b,gradientit2->first,f0[gradientit2->second]);
//cerr << "curintersect: " << curintersect << " leftmostx: " << leftmostx << endl; //cerr << "curintersect: " << curintersect << " leftmostx: " << leftmostx << endl;
if(curintersect<=leftmostx){ if(curintersect<=leftmostx) {
//we have found an intersection to the left of the leftmost we had so far. //we have found an intersection to the left of the leftmost we had so far.
//we might have curintersect==leftmostx for example is 2 candidates are the same //we might have curintersect==leftmostx for example is 2 candidates are the same
//in that case its better its better to update leftmost to gradientit2 to avoid some recomputing later //in that case its better its better to update leftmost to gradientit2 to avoid some recomputing later
leftmostx=curintersect; leftmostx=curintersect;
leftmost=gradientit2;//this is the new reference leftmost=gradientit2;//this is the new reference
} }
} }
} }
if (leftmost == gradientit) { if (leftmost == gradientit) {
//we didn't find any more intersections //we didn't find any more intersections
//the rightmost bestindex is the one with the highest slope. //the rightmost bestindex is the one with the highest slope.
assert(abs(leftmost->first-gradient.rbegin()->first)<0.0001);//they should be egal but there might be assert(abs(leftmost->first-gradient.rbegin()->first)<0.0001);//they should be egal but there might be
//a small difference due to rounding error //a small difference due to rounding error
break; break;
} }
//we have found the next intersection! //we have found the next intersection!
pair<unsigned,unsigned> newd(S,leftmost->second);//new onebest for Sentence S is leftmost->second pair<unsigned,unsigned> newd(S,leftmost->second);//new onebest for Sentence S is leftmost->second
if(leftmostx-previnserted->first<min_int){ if(leftmostx-previnserted->first<min_int) {
/* Require that the intersection Point be at least min_int /* Require that the intersection Point be at least min_int
to the right of the previous one(for this sentence). If not, we replace the to the right of the previous one(for this sentence). If not, we replace the
previous intersection Point with this one. Yes, it can even previous intersection Point with this one. Yes, it can even
happen that the new intersection Point is slightly to the happen that the new intersection Point is slightly to the
left of the old one, because of numerical imprecision. left of the old one, because of numerical imprecision.
we do not check that we are to the right of the penultimate point also. it this happen the 1best the inteval will be wrong we do not check that we are to the right of the penultimate point also. it this happen the 1best the inteval will be wrong
we are going to replace previnsert by the new one because we do not want to keep we are going to replace previnsert by the new one because we do not want to keep
2 very close threshold: if the minima is there it could be an artifact 2 very close threshold: if the minima is there it could be an artifact
*/ */
map<float,diff_t>::iterator tit=thresholdmap.find(leftmostx); map<float,diff_t>::iterator tit=thresholdmap.find(leftmostx);
if(tit==previnserted){ if(tit==previnserted) {
//the threshold is the same as before can happen if 2 candidates are the same for example //the threshold is the same as before can happen if 2 candidates are the same for example
assert(previnserted->second.back().first==newd.first); assert(previnserted->second.back().first==newd.first);
previnserted->second.back()=newd;//just replace the 1 best fors sentence S previnserted->second.back()=newd;//just replace the 1 best fors sentence S
//previnsert doesnt change //previnsert doesnt change
}else{ } else {
if(tit==thresholdmap.end()){ if(tit==thresholdmap.end()) {
thresholdmap[leftmostx]=previnserted->second;//We keep the diffs at previnsert thresholdmap[leftmostx]=previnserted->second;//We keep the diffs at previnsert
thresholdmap.erase(previnserted);//erase old previnsert thresholdmap.erase(previnserted);//erase old previnsert
previnserted=thresholdmap.find(leftmostx);//point previnsert to the new threshold previnserted=thresholdmap.find(leftmostx);//point previnsert to the new threshold
previnserted->second.back()=newd;//we update the diff for sentence S previnserted->second.back()=newd;//we update the diff for sentence S
}else{//threshold already exists but is not the previous one. } else { //threshold already exists but is not the previous one.
//we append the diffs in previnsert to tit before destroying previnsert //we append the diffs in previnsert to tit before destroying previnsert
tit->second.insert(tit->second.end(),previnserted->second.begin(),previnserted->second.end()); tit->second.insert(tit->second.end(),previnserted->second.begin(),previnserted->second.end());
assert(tit->second.back().first==newd.first); assert(tit->second.back().first==newd.first);
tit->second.back()=newd;//change diff for sentence S tit->second.back()=newd;//change diff for sentence S
thresholdmap.erase(previnserted);//erase old previnsert thresholdmap.erase(previnserted);//erase old previnsert
previnserted=tit;//point previnsert to the new threshold previnserted=tit;//point previnsert to the new threshold
} }
} }
assert(previnserted != thresholdmap.end()); assert(previnserted != thresholdmap.end());
}else{//normal insertion process } else { //normal insertion process
previnserted=AddThreshold(thresholdmap,leftmostx,newd); previnserted=AddThreshold(thresholdmap,leftmostx,newd);
} }
gradientit=leftmost; gradientit=leftmost;
} //while(gradientit!=gradient.end()){ } //while(gradientit!=gradient.end()){
} //loop on S } //loop on S
//now the thresholdlist is up to date: //now the thresholdlist is up to date:
//it contains a list of all the parameter_ts where the function changed its value, along with the nbest list for the interval after each threshold //it contains a list of all the parameter_ts where the function changed its value, along with the nbest list for the interval after each threshold
map<float,diff_t >::iterator thrit; map<float,diff_t >::iterator thrit;
if(verboselevel()>6){ if(verboselevel()>6) {
cerr << "Thresholds:(" <<thresholdmap.size()<<")"<< endl; cerr << "Thresholds:(" <<thresholdmap.size()<<")"<< endl;
for (thrit = thresholdmap.begin();thrit!=thresholdmap.end();thrit++){ for (thrit = thresholdmap.begin(); thrit!=thresholdmap.end(); thrit++) {
cerr << "x: " << thrit->first << " diffs"; cerr << "x: " << thrit->first << " diffs";
for (size_t j = 0; j < thrit->second.size(); ++j) { for (size_t j = 0; j < thrit->second.size(); ++j) {
cerr << " " <<thrit->second[j].first << "," << thrit->second[j].second; cerr << " " <<thrit->second[j].first << "," << thrit->second[j].second;
} }
cerr << endl; cerr << endl;
} }
} }
//last thing to do is compute the Stat score (ie BLEU) and find the minimum //last thing to do is compute the Stat score (ie BLEU) and find the minimum
thrit=thresholdmap.begin(); thrit=thresholdmap.begin();
++thrit;//first diff corrrespond to MIN_FLOAT and first1best ++thrit;//first diff corrrespond to MIN_FLOAT and first1best
diffs_t diffs; diffs_t diffs;
for(;thrit!=thresholdmap.end();thrit++) for(; thrit!=thresholdmap.end(); thrit++)
diffs.push_back(thrit->second); diffs.push_back(thrit->second);
vector<statscore_t> scores=GetIncStatScore(first1best,diffs); vector<statscore_t> scores=GetIncStatScore(first1best,diffs);
thrit=thresholdmap.begin(); thrit=thresholdmap.begin();
statscore_t bestscore=MIN_FLOAT; statscore_t bestscore=MIN_FLOAT;
float bestx=MIN_FLOAT; float bestx=MIN_FLOAT;
assert(scores.size()==thresholdmap.size());//we skipped the first el of thresholdlist but GetIncStatScore return 1 more for first1best assert(scores.size()==thresholdmap.size());//we skipped the first el of thresholdlist but GetIncStatScore return 1 more for first1best
for(unsigned int sc=0;sc!=scores.size();sc++){ for(unsigned int sc=0; sc!=scores.size(); sc++) {
//cerr << "x=" << thrit->first << " => " << scores[sc] << endl; //cerr << "x=" << thrit->first << " => " << scores[sc] << endl;
if (scores[sc] > bestscore) { if (scores[sc] > bestscore) {
//This is the score for the interval [lit2->first, (lit2+1)->first] //This is the score for the interval [lit2->first, (lit2+1)->first]
//unless we're at the last score, when it's the score //unless we're at the last score, when it's the score
//for the interval [lit2->first,+inf] //for the interval [lit2->first,+inf]
bestscore = scores[sc]; bestscore = scores[sc];
//if we're not in [-inf,x1] or [xn,+inf] then just take the value //if we're not in [-inf,x1] or [xn,+inf] then just take the value
//if x which splits the interval in half. For the rightmost interval, //if x which splits the interval in half. For the rightmost interval,
//take x to be the last interval boundary + 0.1, and for the leftmost //take x to be the last interval boundary + 0.1, and for the leftmost
//interval, take x to be the first interval boundary - 1000. //interval, take x to be the first interval boundary - 1000.
//These values are taken from cmert. //These values are taken from cmert.
float leftx = thrit->first; float leftx = thrit->first;
if (thrit == thresholdmap.begin()) { if (thrit == thresholdmap.begin()) {
leftx = MIN_FLOAT; leftx = MIN_FLOAT;
} }
++thrit; ++thrit;
float rightx = MAX_FLOAT; float rightx = MAX_FLOAT;
if (thrit != thresholdmap.end()) { if (thrit != thresholdmap.end()) {
rightx = thrit->first; rightx = thrit->first;
} }
--thrit; --thrit;
//cerr << "leftx: " << leftx << " rightx: " << rightx << endl; //cerr << "leftx: " << leftx << " rightx: " << rightx << endl;
if (leftx == MIN_FLOAT) { if (leftx == MIN_FLOAT) {
bestx = rightx-1000; bestx = rightx-1000;
} else if (rightx == MAX_FLOAT) { } else if (rightx == MAX_FLOAT) {
bestx = leftx+0.1; bestx = leftx+0.1;
} else { } else {
bestx = 0.5 * (rightx + leftx); bestx = 0.5 * (rightx + leftx);
} }
//cerr << "x = " << "set new bestx to: " << bestx << endl; //cerr << "x = " << "set new bestx to: " << bestx << endl;
} }
++thrit; ++thrit;
} }
if(abs(bestx)<0.00015){ if(abs(bestx)<0.00015) {
bestx=0.0;//the origin of the line is the best point!we put it back at 0 so we do not propagate rounding erros bestx=0.0;//the origin of the line is the best point!we put it back at 0 so we do not propagate rounding erros
//finally! we manage to extract the best score; //finally! we manage to extract the best score;
//now we convert bestx (position on the line) to a point! //now we convert bestx (position on the line) to a point!
if(verboselevel()>4) if(verboselevel()>4)
cerr<<"best point on line at origin"<<endl; cerr<<"best point on line at origin"<<endl;
} }
if(verboselevel()>3){ if(verboselevel()>3) {
// cerr<<"end Lineopt, bestx="<<bestx<<endl; // cerr<<"end Lineopt, bestx="<<bestx<<endl;
} }
bestpoint=direction*bestx+origin; bestpoint=direction*bestx+origin;
bestpoint.score=bestscore; bestpoint.score=bestscore;
return bestscore; return bestscore;
}; };
void Optimizer::Get1bests(const Point& P,vector<unsigned>& bests)const{ void Optimizer::Get1bests(const Point& P,vector<unsigned>& bests)const
{
assert(FData); assert(FData);
bests.clear(); bests.clear();
bests.resize(size()); bests.resize(size());
for(unsigned i=0;i<size();i++){ for(unsigned i=0; i<size(); i++) {
float bestfs=MIN_FLOAT; float bestfs=MIN_FLOAT;
unsigned idx=0; unsigned idx=0;
unsigned j; unsigned j;
for(j=0;j<FData->get(i).size();j++){ for(j=0; j<FData->get(i).size(); j++) {
float curfs=P*FData->get(i,j); float curfs=P*FData->get(i,j);
if(curfs>bestfs){ if(curfs>bestfs) {
bestfs=curfs; bestfs=curfs;
idx=j; idx=j;
} }
} }
bests[i]=idx; bests[i]=idx;
} }
} }
statscore_t Optimizer::Run(Point& P)const{ statscore_t Optimizer::Run(Point& P)const
if(!FData){ {
if(!FData) {
cerr<<"error trying to optimize without Features loaded"<<endl; cerr<<"error trying to optimize without Features loaded"<<endl;
exit(2); exit(2);
} }
if(!scorer){ if(!scorer) {
cerr<<"error trying to optimize without a Scorer loaded"<<endl; cerr<<"error trying to optimize without a Scorer loaded"<<endl;
exit(2); exit(2);
} }
if (scorer->getReferenceSize()!=FData->size()){ if (scorer->getReferenceSize()!=FData->size()) {
cerr<<"error size mismatch between FeatureData and Scorer"<<endl; cerr<<"error size mismatch between FeatureData and Scorer"<<endl;
exit(2); exit(2);
} }
statscore_t score=GetStatScore(P); statscore_t score=GetStatScore(P);
P.score=score; P.score=score;
if(verboselevel()>2) if(verboselevel()>2)
cerr<<"Starting point: "<< P << " => "<< P.score << endl; cerr<<"Starting point: "<< P << " => "<< P.score << endl;
statscore_t s=TrueRun(P); statscore_t s=TrueRun(P);
P.score=s;//just in case its not done in TrueRun P.score=s;//just in case its not done in TrueRun
@ -331,9 +341,10 @@ statscore_t Optimizer::Run(Point& P)const{
cerr<<"Ending point: "<< P <<" => "<< s << endl; cerr<<"Ending point: "<< P <<" => "<< s << endl;
return s; return s;
} }
vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<vector <pair<unsigned,unsigned> > > thediffs)const{
vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<vector <pair<unsigned,unsigned> > > thediffs)const
{
assert(scorer); assert(scorer);
vector<statscore_t> theres; vector<statscore_t> theres;
@ -347,61 +358,62 @@ vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<
//---------------- code for the powell optimizer //---------------- code for the powell optimizer
float SimpleOptimizer::eps=0.0001; float SimpleOptimizer::eps=0.0001;
statscore_t SimpleOptimizer::TrueRun(Point& P)const{ statscore_t SimpleOptimizer::TrueRun(Point& P)const
{
statscore_t prevscore=0; statscore_t prevscore=0;
statscore_t bestscore=MIN_FLOAT; statscore_t bestscore=MIN_FLOAT;
Point best; Point best;
//If P is already defined and provides a score //If P is already defined and provides a score
//we must improve over this score //we must improve over this score
if(P.score>bestscore){ if(P.score>bestscore) {
bestscore=P.score; bestscore=P.score;
best=P; best=P;
} }
int nrun=0; int nrun=0;
do{ do {
++nrun; ++nrun;
if(verboselevel()>2&&nrun>1) if(verboselevel()>2&&nrun>1)
cerr<<"last diff="<<bestscore-prevscore<<" nrun "<<nrun<<endl; cerr<<"last diff="<<bestscore-prevscore<<" nrun "<<nrun<<endl;
prevscore=bestscore; prevscore=bestscore;
Point linebest; Point linebest;
for(unsigned int d=0;d<Point::getdim();d++){ for(unsigned int d=0; d<Point::getdim(); d++) {
if(verboselevel()>4){ if(verboselevel()>4) {
// cerr<<"minimizing along direction "<<d<<endl; // cerr<<"minimizing along direction "<<d<<endl;
cerr<<"starting point: " << P << " => " << prevscore << endl; cerr<<"starting point: " << P << " => " << prevscore << endl;
} }
Point direction; Point direction;
for(unsigned int i=0;i<Point::getdim();i++) for(unsigned int i=0; i<Point::getdim(); i++)
direction[i]; direction[i];
direction[d]=1.0; direction[d]=1.0;
statscore_t curscore=LineOptimize(P,direction,linebest);//find the minimum on the line statscore_t curscore=LineOptimize(P,direction,linebest);//find the minimum on the line
if(verboselevel()>5){ if(verboselevel()>5) {
cerr<<"direction: "<< d << " => " << curscore << endl; cerr<<"direction: "<< d << " => " << curscore << endl;
cerr<<"\tending point: "<< linebest << " => " << curscore << endl; cerr<<"\tending point: "<< linebest << " => " << curscore << endl;
} }
if(curscore>bestscore){ if(curscore>bestscore) {
bestscore=curscore; bestscore=curscore;
best=linebest; best=linebest;
if(verboselevel()>3){ if(verboselevel()>3) {
cerr<<"new best dir:"<<d<<" ("<<nrun<<")"<<endl; cerr<<"new best dir:"<<d<<" ("<<nrun<<")"<<endl;
cerr<<"new best Point "<<best<< " => " <<curscore<<endl; cerr<<"new best Point "<<best<< " => " <<curscore<<endl;
} }
} }
} }
P=best;//update the current vector with the best point on all line tested P=best;//update the current vector with the best point on all line tested
if(verboselevel()>3) if(verboselevel()>3)
cerr<<nrun<<"\t"<<P<<endl; cerr<<nrun<<"\t"<<P<<endl;
}while(bestscore-prevscore>eps); } while(bestscore-prevscore>eps);
if(verboselevel()>2){ if(verboselevel()>2) {
cerr<<"end Powell Algo, nrun="<<nrun<<endl; cerr<<"end Powell Algo, nrun="<<nrun<<endl;
cerr<<"last diff="<<bestscore-prevscore<<endl; cerr<<"last diff="<<bestscore-prevscore<<endl;
cerr<<"\t"<<P<<endl; cerr<<"\t"<<P<<endl;
} }
return bestscore; return bestscore;
} }
@ -409,58 +421,63 @@ statscore_t SimpleOptimizer::TrueRun(Point& P)const{
/**RandomOptimizer to use as beaseline and test.\n /**RandomOptimizer to use as beaseline and test.\n
Just return a random point*/ Just return a random point*/
statscore_t RandomOptimizer::TrueRun(Point& P)const{ statscore_t RandomOptimizer::TrueRun(Point& P)const
{
vector<parameter_t> min(Point::getdim()); vector<parameter_t> min(Point::getdim());
vector<parameter_t> max(Point::getdim()); vector<parameter_t> max(Point::getdim());
for(unsigned int d=0;d<Point::getdim();d++){ for(unsigned int d=0; d<Point::getdim(); d++) {
min[d]=0.0; min[d]=0.0;
max[d]=1.0; max[d]=1.0;
} }
P.Randomize(min,max); P.Randomize(min,max);
statscore_t score=GetStatScore(P); statscore_t score=GetStatScore(P);
P.score=score; P.score=score;
return score; return score;
} }
//-------------------------------------- //--------------------------------------
vector<string> OptimizerFactory::typenames; vector<string> OptimizerFactory::typenames;
void OptimizerFactory::SetTypeNames(){ void OptimizerFactory::SetTypeNames()
if(typenames.empty()){ {
if(typenames.empty()) {
typenames.resize(NOPTIMIZER); typenames.resize(NOPTIMIZER);
typenames[POWELL]="powell"; typenames[POWELL]="powell";
typenames[RANDOM]="random"; typenames[RANDOM]="random";
//add new type there //add new type there
} }
} }
vector<string> OptimizerFactory::GetTypeNames(){ vector<string> OptimizerFactory::GetTypeNames()
{
if(typenames.empty()) if(typenames.empty())
SetTypeNames(); SetTypeNames();
return typenames; return typenames;
} }
OptimizerFactory::OptType OptimizerFactory::GetOType(string type){ OptimizerFactory::OptType OptimizerFactory::GetOType(string type)
{
unsigned int thetype; unsigned int thetype;
if(typenames.empty()) if(typenames.empty())
SetTypeNames(); SetTypeNames();
for(thetype=0;thetype<typenames.size();thetype++) for(thetype=0; thetype<typenames.size(); thetype++)
if(typenames[thetype]==type) if(typenames[thetype]==type)
break; break;
return((OptType)thetype); return((OptType)thetype);
}; };
Optimizer* OptimizerFactory::BuildOptimizer(unsigned dim,vector<unsigned> i2o,vector<parameter_t> start,string type){ Optimizer* OptimizerFactory::BuildOptimizer(unsigned dim,vector<unsigned> i2o,vector<parameter_t> start,string type)
{
OptType T=GetOType(type); OptType T=GetOType(type);
if(T==NOPTIMIZER){ if(T==NOPTIMIZER) {
cerr<<"Error: unknown Optimizer type "<<type<<endl; cerr<<"Error: unknown Optimizer type "<<type<<endl;
cerr<<"Known Algorithm are:"<<endl; cerr<<"Known Algorithm are:"<<endl;
unsigned int thetype; unsigned int thetype;
for(thetype=0;thetype<typenames.size();thetype++) for(thetype=0; thetype<typenames.size(); thetype++)
cerr<<typenames[thetype]<<endl; cerr<<typenames[thetype]<<endl;
throw ("unknown Optimizer Type"); throw ("unknown Optimizer Type");
} }
switch((OptType)T){ switch((OptType)T) {
case POWELL: case POWELL:
return new SimpleOptimizer(dim,i2o,start); return new SimpleOptimizer(dim,i2o,start);
break; break;
@ -469,6 +486,6 @@ Optimizer* OptimizerFactory::BuildOptimizer(unsigned dim,vector<unsigned> i2o,ve
break; break;
default: default:
cerr<<"Error: unknown optimizer"<<type<<endl; cerr<<"Error: unknown optimizer"<<type<<endl;
return NULL; return NULL;
} }
} }

46
mert/Optimizer.h
View File

@ -15,61 +15,69 @@ typedef float featurescore;
using namespace std; using namespace std;
/**abstract virtual class*/ /**abstract virtual class*/
class Optimizer{ class Optimizer
protected: {
Scorer * scorer; //no accessor for them only child can use them protected:
FeatureData * FData;//no accessor for them only child can use them Scorer * scorer; //no accessor for them only child can use them
public: FeatureData * FData;//no accessor for them only child can use them
public:
Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start); Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start);
void SetScorer(Scorer *S); void SetScorer(Scorer *S);
void SetFData(FeatureData *F); void SetFData(FeatureData *F);
virtual ~Optimizer(); virtual ~Optimizer();
unsigned size()const{return (FData?FData->size():0);} unsigned size()const {
return (FData?FData->size():0);
}
/**Generic wrapper around TrueRun to check a few things. Non virtual*/ /**Generic wrapper around TrueRun to check a few things. Non virtual*/
statscore_t Run(Point&)const; statscore_t Run(Point&)const;
/**main function that perform an optimization*/ /**main function that perform an optimization*/
virtual statscore_t TrueRun(Point&)const=0; virtual statscore_t TrueRun(Point&)const=0;
/**given a set of lambdas, get the nbest for each sentence*/ /**given a set of lambdas, get the nbest for each sentence*/
void Get1bests(const Point& param,vector<unsigned>& bests)const; void Get1bests(const Point& param,vector<unsigned>& bests)const;
/**given a set of nbests, get the Statistical score*/ /**given a set of nbests, get the Statistical score*/
statscore_t GetStatScore(const vector<unsigned>& nbests)const{return scorer->score(nbests);}; statscore_t GetStatScore(const vector<unsigned>& nbests)const {
return scorer->score(nbests);
};
/**given a set of lambdas, get the total statistical score*/ /**given a set of lambdas, get the total statistical score*/
statscore_t GetStatScore(const Point& param)const; statscore_t GetStatScore(const Point& param)const;
vector<statscore_t > GetIncStatScore(vector<unsigned> ref,vector<vector <pair<unsigned,unsigned> > >)const; vector<statscore_t > GetIncStatScore(vector<unsigned> ref,vector<vector <pair<unsigned,unsigned> > >)const;
statscore_t LineOptimize(const Point& start,const Point& direction,Point& best)const;//Get the optimal Lambda and the best score in a particular direction from a given Point statscore_t LineOptimize(const Point& start,const Point& direction,Point& best)const;//Get the optimal Lambda and the best score in a particular direction from a given Point
}; };
/**default basic optimizer*/ /**default basic optimizer*/
class SimpleOptimizer: public Optimizer{ class SimpleOptimizer: public Optimizer
{
private: private:
static float eps; static float eps;
public: public:
SimpleOptimizer(unsigned dim,vector<unsigned> i2O,vector<parameter_t> start):Optimizer(dim,i2O,start){}; SimpleOptimizer(unsigned dim,vector<unsigned> i2O,vector<parameter_t> start):Optimizer(dim,i2O,start) {};
virtual statscore_t TrueRun(Point&)const; virtual statscore_t TrueRun(Point&)const;
}; };
class RandomOptimizer: public Optimizer{ class RandomOptimizer: public Optimizer
{
public: public:
RandomOptimizer(unsigned dim,vector<unsigned> i2O,vector<parameter_t> start):Optimizer(dim,i2O,start){}; RandomOptimizer(unsigned dim,vector<unsigned> i2O,vector<parameter_t> start):Optimizer(dim,i2O,start) {};
virtual statscore_t TrueRun(Point&)const; virtual statscore_t TrueRun(Point&)const;
}; };
class OptimizerFactory{ class OptimizerFactory
public: {
public:
// unsigned dim; // unsigned dim;
//Point Start; //Point Start;
static vector<string> GetTypeNames(); static vector<string> GetTypeNames();
static Optimizer* BuildOptimizer(unsigned dim,vector<unsigned>tooptimize,vector<parameter_t> start,string type); static Optimizer* BuildOptimizer(unsigned dim,vector<unsigned>tooptimize,vector<parameter_t> start,string type);
private: private:
enum OptType{POWELL=0,RANDOM,NOPTIMIZER};//Add new optimizer here BEFORE NOPTIMZER enum OptType {POWELL=0,RANDOM,NOPTIMIZER}; //Add new optimizer here BEFORE NOPTIMZER
static OptType GetOType(string); static OptType GetOType(string);
static vector<string> typenames; static vector<string> typenames;
static void SetTypeNames(); static void SetTypeNames();
}; };

119
mert/PerScorer.cpp
View File

@ -1,69 +1,72 @@
#include "PerScorer.h" #include "PerScorer.h"
void PerScorer::setReferenceFiles(const vector<string>& referenceFiles) { void PerScorer::setReferenceFiles(const vector<string>& referenceFiles)
// for each line in the reference file, create a multiset of the {
// word ids // for each line in the reference file, create a multiset of the
if (referenceFiles.size() != 1) { // word ids
throw runtime_error("PER only supports a single reference"); if (referenceFiles.size() != 1) {
throw runtime_error("PER only supports a single reference");
}
_reftokens.clear();
_reflengths.clear();
ifstream in(referenceFiles[0].c_str());
if (!in) {
throw runtime_error("Unable to open " + referenceFiles[0]);
}
string line;
int sid = 0;
while (getline(in,line)) {
vector<int> tokens;
encode(line,tokens);
_reftokens.push_back(multiset<int>());
for (size_t i = 0; i < tokens.size(); ++i) {
_reftokens.back().insert(tokens[i]);
} }
_reftokens.clear(); _reflengths.push_back(tokens.size());
_reflengths.clear(); if (sid > 0 && sid % 100 == 0) {
ifstream in(referenceFiles[0].c_str()); TRACE_ERR(".");
if (!in) {
throw runtime_error("Unable to open " + referenceFiles[0]);
} }
string line; ++sid;
int sid = 0; }
while (getline(in,line)) { TRACE_ERR(endl);
vector<int> tokens;
encode(line,tokens);
_reftokens.push_back(multiset<int>());
for (size_t i = 0; i < tokens.size(); ++i) {
_reftokens.back().insert(tokens[i]);
}
_reflengths.push_back(tokens.size());
if (sid > 0 && sid % 100 == 0) {
TRACE_ERR(".");
}
++sid;
}
TRACE_ERR(endl);
} }
void PerScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) { void PerScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
if (sid >= _reflengths.size()) { {
stringstream msg; if (sid >= _reflengths.size()) {
msg << "Sentence id (" << sid << ") not found in reference set"; stringstream msg;
throw runtime_error(msg.str()); msg << "Sentence id (" << sid << ") not found in reference set";
} throw runtime_error(msg.str());
//calculate correct, output_length and ref_length for }
//the line and store it in entry //calculate correct, output_length and ref_length for
vector<int> testtokens; //the line and store it in entry
encode(text,testtokens); vector<int> testtokens;
multiset<int> testtokens_all(testtokens.begin(),testtokens.end()); encode(text,testtokens);
set<int> testtokens_unique(testtokens.begin(),testtokens.end()); multiset<int> testtokens_all(testtokens.begin(),testtokens.end());
int correct = 0; set<int> testtokens_unique(testtokens.begin(),testtokens.end());
for (set<int>::iterator i = testtokens_unique.begin(); int correct = 0;
i != testtokens_unique.end(); ++i) { for (set<int>::iterator i = testtokens_unique.begin();
int token = *i; i != testtokens_unique.end(); ++i) {
correct += min(_reftokens[sid].count(token), testtokens_all.count(token)); int token = *i;
} correct += min(_reftokens[sid].count(token), testtokens_all.count(token));
}
ostringstream stats;
stats << correct << " " << testtokens.size() << " " << _reflengths[sid] << " " ; ostringstream stats;
string stats_str = stats.str(); stats << correct << " " << testtokens.size() << " " << _reflengths[sid] << " " ;
entry.set(stats_str); string stats_str = stats.str();
entry.set(stats_str);
} }
float PerScorer::calculateScore(const vector<float>& comps) { float PerScorer::calculateScore(const vector<float>& comps)
float denom = comps[2]; {
float num = comps[0] - max(float(0),comps[1]-comps[2]); float denom = comps[2];
if (denom == 0) { float num = comps[0] - max(float(0),comps[1]-comps[2]);
//shouldn't happen! if (denom == 0) {
return 0.0; //shouldn't happen!
} else { return 0.0;
return num/denom; } else {
} return num/denom;
}
} }

60
mert/PerScorer.h
View File

@ -22,34 +22,40 @@ using namespace std;
* as 1 - (correct - max(0,output_length - ref_length)) / ref_length * as 1 - (correct - max(0,output_length - ref_length)) / ref_length
* In fact, we ignore the " 1 - " so that it can be maximised. * In fact, we ignore the " 1 - " so that it can be maximised.
**/ **/
class PerScorer: public StatisticsBasedScorer { class PerScorer: public StatisticsBasedScorer
public: {
PerScorer(const string& config = "") : StatisticsBasedScorer("PER",config) {} public:
virtual void setReferenceFiles(const vector<string>& referenceFiles); PerScorer(const string& config = "") : StatisticsBasedScorer("PER",config) {}
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry); virtual void setReferenceFiles(const vector<string>& referenceFiles);
virtual void prepareStats(size_t sid, const string& text, ScoreStats& entry);
virtual void whoami() {
cerr << "I AM PerScorer" << std::endl;
}
size_t NumberOfScores() const { cerr << "PerScorer: 3" << endl; return 3; };
bool useAlignment() const {return false;};
virtual void whoami() {
protected: cerr << "I AM PerScorer" << std::endl;
}
virtual float calculateScore(const vector<float>& comps) ;
size_t NumberOfScores() const {
private: cerr << "PerScorer: 3" << endl;
return 3;
//no copy };
PerScorer(const PerScorer&); bool useAlignment() const {
~PerScorer(){}; return false;
PerScorer& operator=(const PerScorer&); };
// data extracted from reference files
vector<size_t> _reflengths; protected:
vector<multiset<int> > _reftokens;
virtual float calculateScore(const vector<float>& comps) ;
private:
//no copy
PerScorer(const PerScorer&);
~PerScorer() {};
PerScorer& operator=(const PerScorer&);
// data extracted from reference files
vector<size_t> _reflengths;
vector<multiset<int> > _reftokens;
}; };
#endif //__PERSCORER_H #endif //__PERSCORER_H

447
mert/Permutation.cpp
View File

@ -16,27 +16,26 @@ using namespace std;
Permutation::Permutation(const string &alignment, const int sourceLength, const int targetLength ) Permutation::Permutation(const string &alignment, const int sourceLength, const int targetLength )
{ {
if (sourceLength > 0) if (sourceLength > 0) {
{ set(alignment, sourceLength);
set(alignment, sourceLength); }
} m_targetLength = targetLength;
m_targetLength = targetLength;
} }
size_t Permutation::getLength() const size_t Permutation::getLength() const
{ {
return int(m_array.size()); return int(m_array.size());
} }
void Permutation::dump() const void Permutation::dump() const
{ {
int j=0; int j=0;
for (vector<int>::const_iterator i = m_array.begin(); i !=m_array.end(); i++){ for (vector<int>::const_iterator i = m_array.begin(); i !=m_array.end(); i++) {
cout << "("; cout << "(";
cout << j << ":" << *i ; cout << j << ":" << *i ;
cout << "), "; cout << "), ";
j++; j++;
} }
cout << endl; cout << endl;
} }
@ -49,286 +48,272 @@ void Permutation::dump() const
void Permutation::set(const string & alignment,const int sourceLength) void Permutation::set(const string & alignment,const int sourceLength)
{ {
//cout << "******** Permutation::set :" << alignment << ": len : " << sourceLength <<endl; //cout << "******** Permutation::set :" << alignment << ": len : " << sourceLength <<endl;
if(sourceLength <= 0) if(sourceLength <= 0) {
{ //not found
//not found cerr << "Source sentence length not positive:"<< sourceLength << endl;
cerr << "Source sentence length not positive:"<< sourceLength << endl; exit(0);
exit(0); }
if (alignment.length() <= 0) {
//alignment empty - could happen but not good
cerr << "Alignment string empty:"<< alignment << endl;
}
//Tokenise on whitespace
string buf; // Have a buffer string
stringstream ss(alignment); // Insert the string into a stream
vector<string> tokens; // Create vector to hold our words
while (ss >> buf)
tokens.push_back(buf);
vector<int> tempPerm(sourceLength, -1);
//Set tempPerm to have one target position per source position
for (size_t i=0; i<tokens.size(); i++) {
string temp = tokens[i];
int posDelimeter = temp.find("-");
if(posDelimeter == int(string::npos)) {
cerr << "Delimiter not found - :"<< tokens[i] << endl;
exit(1);
} }
int sourcePos = atoi((temp.substr(0, posDelimeter)).c_str());
if (alignment.length() <= 0) int targetPos = atoi((temp.substr(posDelimeter+1)).c_str());
{ //cout << "SP:" << sourcePos << " TP:" << targetPos << endl;
//alignment empty - could happen but not good if (sourcePos > sourceLength) {
cerr << "Alignment string empty:"<< alignment << endl; cerr << "Source sentence length:" << sourceLength << " is smaller than alignment source position:" << sourcePos << endl;
exit(1);
} }
//If have multiple target pos aligned to one source,
//Tokenise on whitespace // then ignore all but first alignment
string buf; // Have a buffer string if (tempPerm[sourcePos] == -1 || tempPerm[sourcePos] > targetPos) {
stringstream ss(alignment); // Insert the string into a stream tempPerm[sourcePos] = targetPos;
vector<string> tokens; // Create vector to hold our words
while (ss >> buf)
tokens.push_back(buf);
vector<int> tempPerm(sourceLength, -1);
//Set tempPerm to have one target position per source position
for (size_t i=0; i<tokens.size(); i++) {
string temp = tokens[i];
int posDelimeter = temp.find("-");
if(posDelimeter == int(string::npos)) {
cerr << "Delimiter not found - :"<< tokens[i] << endl;
exit(1);
}
int sourcePos = atoi((temp.substr(0, posDelimeter)).c_str());
int targetPos = atoi((temp.substr(posDelimeter+1)).c_str());
//cout << "SP:" << sourcePos << " TP:" << targetPos << endl;
if (sourcePos > sourceLength) {
cerr << "Source sentence length:" << sourceLength << " is smaller than alignment source position:" << sourcePos << endl;
exit(1);
}
//If have multiple target pos aligned to one source,
// then ignore all but first alignment
if (tempPerm[sourcePos] == -1 || tempPerm[sourcePos] > targetPos)
{
tempPerm[sourcePos] = targetPos;
}
} }
}
//TODO //TODO
//Set final permutation in m_array //Set final permutation in m_array
//Take care of: source - null //Take care of: source - null
// multiple_source - one target // multiple_source - one target
// unaligned target // unaligned target
// Input: 1-9 2-1 4-3 4-4 5-6 6-6 7-6 8-8 // Input: 1-9 2-1 4-3 4-4 5-6 6-6 7-6 8-8
// Convert source: 1 2 3 4 5 6 7 8 // Convert source: 1 2 3 4 5 6 7 8
// target: 9 1 -1 3 6 6 6 8 -> 8 1 2 3 4 5 6 7 // target: 9 1 -1 3 6 6 6 8 -> 8 1 2 3 4 5 6 7
// 1st step: Add null aligned source to previous alignment // 1st step: Add null aligned source to previous alignment
// target: 9 1 -1 3 6 6 6 8 -> 9 1 1 3 6 6 6 8 // target: 9 1 -1 3 6 6 6 8 -> 9 1 1 3 6 6 6 8
int last=0; int last=0;
m_array.assign(sourceLength, -1); m_array.assign(sourceLength, -1);
//get a searcheable index //get a searcheable index
multimap<int, int> invMap; multimap<int, int> invMap;
multimap<int, int>::iterator it; multimap<int, int>::iterator it;
//cout << " SourceP -> TargetP " << endl; //cout << " SourceP -> TargetP " << endl;
for (size_t i=0; i<tempPerm.size(); i++) for (size_t i=0; i<tempPerm.size(); i++) {
{ if (tempPerm[i] == -1) {
if (tempPerm[i] == -1) { tempPerm[i] = last;
tempPerm[i] = last; } else {
} else { last = tempPerm[i];
last = tempPerm[i];
}
//cout << i << " -> " << tempPerm[i] << endl;
//Key is target pos, value is source pos
invMap.insert(pair<int,int>(tempPerm[i],int(i)));
} }
//cout << i << " -> " << tempPerm[i] << endl;
//Key is target pos, value is source pos
invMap.insert(pair<int,int>(tempPerm[i],int(i)));
}
// 2nd step: Get target into index of multimap and sort // 2nd step: Get target into index of multimap and sort
// Convert source: 1 2 3 4 5 6 7 8 // Convert source: 1 2 3 4 5 6 7 8
// target: 9 1 0 3 6 6 6 8 -> 0 1 3 6 6 6 8 9 // target: 9 1 0 3 6 6 6 8 -> 0 1 3 6 6 6 8 9
// source: 3 2 4 5 6 7 8 1 // source: 3 2 4 5 6 7 8 1
int i=0; int i=0;
//cout << " TargetP => SourceP : TargetIndex " << endl; //cout << " TargetP => SourceP : TargetIndex " << endl;
for ( it=invMap.begin() ; it != invMap.end(); it++ ) for ( it=invMap.begin() ; it != invMap.end(); it++ ) {
{ //cout << (*it).first << " => " << (*it).second << " : " << i << endl;
//cout << (*it).first << " => " << (*it).second << " : " << i << endl; //find source position
//find source position m_array[(*it).second] = i;
m_array[(*it).second] = i; i++;
i++; }
}
bool ok = checkValidPermutation(m_array); bool ok = checkValidPermutation(m_array);
//dump(); //dump();
if (!ok) { if (!ok) {
throw runtime_error(" Created invalid permutation"); throw runtime_error(" Created invalid permutation");
} }
} }
//Static //Static
vector<int> Permutation::invert(const vector<int> & inVector) vector<int> Permutation::invert(const vector<int> & inVector)
{ {
vector<int> outVector(inVector.size()); vector<int> outVector(inVector.size());
for (size_t i=0; i<inVector.size(); i++){ for (size_t i=0; i<inVector.size(); i++) {
outVector[inVector[i]] = int(i); outVector[inVector[i]] = int(i);
} }
return outVector; return outVector;
} }
//Static //Static
//Permutations start at 0 //Permutations start at 0
bool Permutation::checkValidPermutation(vector<int> const & inVector) bool Permutation::checkValidPermutation(vector<int> const & inVector)
{ {
vector<int> test(inVector.size(),-1); vector<int> test(inVector.size(),-1);
for (size_t i=0; i< inVector.size(); i++){ for (size_t i=0; i< inVector.size(); i++) {
//No multiple entries of same value allowed //No multiple entries of same value allowed
if (test[inVector[i]] > -1){ if (test[inVector[i]] > -1) {
cerr << "Permutation error: multiple entries of same value\n" << endl; cerr << "Permutation error: multiple entries of same value\n" << endl;
return false; return false;
}
test[inVector[i]] ++;
} }
for (size_t i=0; i<inVector.size(); i++){ test[inVector[i]] ++;
//No holes allowed }
if (test[inVector[i]] == -1) { for (size_t i=0; i<inVector.size(); i++) {
cerr << "Permutation error: missing values\n" << endl; //No holes allowed
return false; if (test[inVector[i]] == -1) {
} cerr << "Permutation error: missing values\n" << endl;
return false;
} }
return true; }
return true;
} }
//TODO default to HAMMING //TODO default to HAMMING
//Note: it returns the distance that is not normalised //Note: it returns the distance that is not normalised
float Permutation::distance(const Permutation &permCompare, const distanceMetric_t &type) const float Permutation::distance(const Permutation &permCompare, const distanceMetric_t &type) const
{ {
float score=0; float score=0;
//cout << "*****Permutation::distance" <<endl;
//cout << "Ref:" << endl;
//dump();
//cout << "Comp:" << endl;
//permCompare.dump();
if (type == HAMMING_DISTANCE) { //cout << "*****Permutation::distance" <<endl;
score = calculateHamming(permCompare); //cout << "Ref:" << endl;
} else if (type == KENDALL_DISTANCE) { //dump();
score = calculateKendall(permCompare); //cout << "Comp:" << endl;
} else { //permCompare.dump();
throw runtime_error("Distance type not valid");
}
float brevityPenalty = 1.0 - (float) permCompare.getTargetLength()/getTargetLength() ;//reflength divided by trans length
if (brevityPenalty < 0.0) {
score = score * exp(brevityPenalty);
}
//cout << "Distance type:" << type << endl; if (type == HAMMING_DISTANCE) {
//cout << "Score: "<< score << endl; score = calculateHamming(permCompare);
return score; } else if (type == KENDALL_DISTANCE) {
score = calculateKendall(permCompare);
} else {
throw runtime_error("Distance type not valid");
}
float brevityPenalty = 1.0 - (float) permCompare.getTargetLength()/getTargetLength() ;//reflength divided by trans length
if (brevityPenalty < 0.0) {
score = score * exp(brevityPenalty);
}
//cout << "Distance type:" << type << endl;
//cout << "Score: "<< score << endl;
return score;
} }
float Permutation::calculateHamming(const Permutation & compare) const float Permutation::calculateHamming(const Permutation & compare) const
{ {
float score=0; float score=0;
vector<int> compareArray = compare.getArray(); vector<int> compareArray = compare.getArray();
if (getLength() != compare.getLength()) { if (getLength() != compare.getLength()) {
cerr << "1stperm: " << getLength() << " 2ndperm: " << compare.getLength() << endl; cerr << "1stperm: " << getLength() << " 2ndperm: " << compare.getLength() << endl;
throw runtime_error("Length of permutations not equal"); throw runtime_error("Length of permutations not equal");
}
if (getLength() == 0) {
cerr << "Empty permutation" << endl;
return 0;
}
for (size_t i=0; i<getLength(); i++) {
if (m_array[i] != compareArray[i]) {
score++;
} }
if (getLength() == 0) {
cerr << "Empty permutation" << endl;
return 0;
}
for (size_t i=0; i<getLength(); i++)
{
if (m_array[i] != compareArray[i])
{
score++;
}
} }
score = 1 - (score / getLength()); score = 1 - (score / getLength());
return score; return score;
} }
float Permutation::calculateKendall(const Permutation & compare) const float Permutation::calculateKendall(const Permutation & compare) const
{ {
float score=0; float score=0;
vector<int> compareArray = compare.getArray(); vector<int> compareArray = compare.getArray();
if (getLength() != compare.getLength()) { if (getLength() != compare.getLength()) {
cerr << "1stperm: " << getLength() << " 2ndperm: " << compare.getLength() << endl; cerr << "1stperm: " << getLength() << " 2ndperm: " << compare.getLength() << endl;
throw runtime_error("Length of permutations not equal"); throw runtime_error("Length of permutations not equal");
}
if (getLength() == 0) {
cerr << "Empty permutation" << endl;
return 0;
}
for (size_t i=0; i<getLength(); i++) {
for (size_t j=0; j<getLength(); j++) {
if ((m_array[i] < m_array[j]) && (compareArray[i] > compareArray[j])) {
score++;
}
} }
if (getLength() == 0) { }
cerr << "Empty permutation" << endl; score = (score / ((getLength()*getLength() - getLength()) /2 ) );
return 0; //Adjusted Kendall's tau correlates better with human judgements
} score = sqrt (score);
for (size_t i=0; i<getLength(); i++) score = 1 - score;
{
for (size_t j=0; j<getLength(); j++)
{
if ((m_array[i] < m_array[j]) && (compareArray[i] > compareArray[j]))
{
score++;
}
}
}
score = (score / ((getLength()*getLength() - getLength()) /2 ) );
//Adjusted Kendall's tau correlates better with human judgements
score = sqrt (score);
score = 1 - score;
return score; return score;
} }
vector<int> Permutation::getArray() const vector<int> Permutation::getArray() const
{ {
vector<int> ret = m_array; vector<int> ret = m_array;
return ret; return ret;
} }
//Static //Static
//This function is called with test which is //This function is called with test which is
// the 5th field in moses nbest output when called with -include-alignment-in-n-best // the 5th field in moses nbest output when called with -include-alignment-in-n-best
//eg. 0=0 1-2=1-2 3=3 4=4 5=5 6=6 7-9=7-8 10=9 11-13=10-11 (source-target) //eg. 0=0 1-2=1-2 3=3 4=4 5=5 6=6 7-9=7-8 10=9 11-13=10-11 (source-target)
string Permutation::convertMosesToStandard(string const & alignment) string Permutation::convertMosesToStandard(string const & alignment)
{ {
if (alignment.length() == 0) if (alignment.length() == 0) {
{
cerr << "Alignment input string empty" << endl; cerr << "Alignment input string empty" << endl;
} }
string working = alignment; string working = alignment;
string out; string out;
stringstream oss; stringstream oss;
while (working.length() > 0) while (working.length() > 0) {
{ string align;
string align; getNextPound(working,align," ");
getNextPound(working,align," ");
//If found an alignment //If found an alignment
if (align.length() > 0) if (align.length() > 0) {
{ size_t posDelimeter = align.find("=");
size_t posDelimeter = align.find("="); if(posDelimeter== string::npos) {
if(posDelimeter== string::npos) cerr << "Delimiter not found = :"<< align << endl;
{ exit(0);
cerr << "Delimiter not found = :"<< align << endl; }
exit(0); int firstSourcePos,lastSourcePos,firstTargetPos,lastTargetPos;
} string sourcePoss = align.substr(0, posDelimeter);
int firstSourcePos,lastSourcePos,firstTargetPos,lastTargetPos; string targetPoss = align.substr(posDelimeter+1);
string sourcePoss = align.substr(0, posDelimeter); posDelimeter = sourcePoss.find("-");
string targetPoss = align.substr(posDelimeter+1); if(posDelimeter < string::npos) {
posDelimeter = sourcePoss.find("-"); firstSourcePos = atoi((sourcePoss.substr(0, posDelimeter)).c_str());
if(posDelimeter < string::npos) { lastSourcePos = atoi((sourcePoss.substr(posDelimeter+1)).c_str());
firstSourcePos = atoi((sourcePoss.substr(0, posDelimeter)).c_str()); } else {
lastSourcePos = atoi((sourcePoss.substr(posDelimeter+1)).c_str()); firstSourcePos = atoi(sourcePoss.c_str());
} else { lastSourcePos = firstSourcePos;
firstSourcePos = atoi(sourcePoss.c_str()); }
lastSourcePos = firstSourcePos; posDelimeter = targetPoss.find("-");
} if(posDelimeter < string::npos) {
posDelimeter = targetPoss.find("-"); firstTargetPos = atoi((targetPoss.substr(0, posDelimeter)).c_str());
if(posDelimeter < string::npos) { lastTargetPos = atoi((targetPoss.substr(posDelimeter+1)).c_str());
firstTargetPos = atoi((targetPoss.substr(0, posDelimeter)).c_str()); } else {
lastTargetPos = atoi((targetPoss.substr(posDelimeter+1)).c_str()); firstTargetPos = atoi(targetPoss.c_str());
} else { lastTargetPos = firstTargetPos;
firstTargetPos = atoi(targetPoss.c_str()); }
lastTargetPos = firstTargetPos; for (int i = firstSourcePos; i <= lastSourcePos; i++) {
} for (int j = firstTargetPos; j <= lastTargetPos; j++) {
for (int i = firstSourcePos; i <= lastSourcePos; i++) { oss << i << "-" << j << " ";
for (int j = firstTargetPos; j <= lastTargetPos; j++) { }
oss << i << "-" << j << " "; }
}
}
} //else case where two spaces ? } //else case where two spaces ?
} }
out = oss.str(); out = oss.str();
//cout << "ConverttoStandard: " << out << endl; //cout << "ConverttoStandard: " << out << endl;
return out; return out;
} }

56
mert/Permutation.h
View File

@ -20,41 +20,45 @@
class Permutation class Permutation
{ {
public: public:
//Can be HAMMING_DISTANCE or KENDALLS_DISTANCE //Can be HAMMING_DISTANCE or KENDALLS_DISTANCE
Permutation(const std::string &alignment = std::string(), const int sourceLength = 0, const int targetLength = 0 ); Permutation(const std::string &alignment = std::string(), const int sourceLength = 0, const int targetLength = 0 );
~Permutation(){}; ~Permutation() {};
inline void clear() { m_array.clear(); } inline void clear() {
inline size_t size(){ return m_array.size(); } m_array.clear();
}
inline size_t size() {
return m_array.size();
}
void set(const std::string &alignment,const int sourceLength); void set(const std::string &alignment,const int sourceLength);
float distance(const Permutation &permCompare, const distanceMetric_t &strategy = HAMMING_DISTANCE) const; float distance(const Permutation &permCompare, const distanceMetric_t &strategy = HAMMING_DISTANCE) const;
//Const //Const
void dump() const; void dump() const;
size_t getLength() const; size_t getLength() const;
vector<int> getArray() const; vector<int> getArray() const;
int getTargetLength() const { int getTargetLength() const {
return m_targetLength; return m_targetLength;
} }
//Static //Static
static std::string convertMosesToStandard(std::string const & alignment); static std::string convertMosesToStandard(std::string const & alignment);
static vector<int> invert(vector<int> const & inVector); static vector<int> invert(vector<int> const & inVector);
static bool checkValidPermutation(vector<int> const & inVector); static bool checkValidPermutation(vector<int> const & inVector);
protected: protected:
vector<int> m_array; vector<int> m_array;
int m_targetLength; int m_targetLength;
float calculateHamming(const Permutation & compare) const; float calculateHamming(const Permutation & compare) const;
float calculateKendall(const Permutation & compare) const; float calculateKendall(const Permutation & compare) const;
private: private:
}; };

363
mert/PermutationScorer.cpp
View File

@ -4,215 +4,212 @@ using namespace std;
const int PermutationScorer::SCORE_PRECISION = 5; const int PermutationScorer::SCORE_PRECISION = 5;
PermutationScorer::PermutationScorer(const string &distanceMetric, const string &config) PermutationScorer::PermutationScorer(const string &distanceMetric, const string &config)
:SentenceLevelScorer(distanceMetric,config) :SentenceLevelScorer(distanceMetric,config)
{ {
//configure regularisation //configure regularisation
static string KEY_REFCHOICE = "refchoice"; static string KEY_REFCHOICE = "refchoice";
static string REFCHOICE_AVERAGE = "average"; static string REFCHOICE_AVERAGE = "average";
static string REFCHOICE_CLOSEST = "closest"; static string REFCHOICE_CLOSEST = "closest";
string refchoice = getConfig(KEY_REFCHOICE,REFCHOICE_CLOSEST);
if (refchoice == REFCHOICE_AVERAGE) {
m_refChoiceStrategy = REFERENCE_CHOICE_AVERAGE;
} else if (refchoice == REFCHOICE_CLOSEST) {
m_refChoiceStrategy = REFERENCE_CHOICE_CLOSEST;
} else {
throw runtime_error("Unknown reference choice strategy: " + refchoice);
}
cerr << "Using reference choice strategy: " << refchoice << endl;
if (distanceMetric.compare("HAMMING") == 0) { string refchoice = getConfig(KEY_REFCHOICE,REFCHOICE_CLOSEST);
m_distanceMetric = HAMMING_DISTANCE; if (refchoice == REFCHOICE_AVERAGE) {
} else if (distanceMetric.compare("KENDALL") == 0) { m_refChoiceStrategy = REFERENCE_CHOICE_AVERAGE;
m_distanceMetric = KENDALL_DISTANCE; } else if (refchoice == REFCHOICE_CLOSEST) {
} m_refChoiceStrategy = REFERENCE_CHOICE_CLOSEST;
cerr << "Using permutation distance metric: " << distanceMetric << endl; } else {
throw runtime_error("Unknown reference choice strategy: " + refchoice);
}
cerr << "Using reference choice strategy: " << refchoice << endl;
//Get reference alignments from scconfig refalign option if (distanceMetric.compare("HAMMING") == 0) {
static string KEY_ALIGNMENT_FILES = "refalign"; m_distanceMetric = HAMMING_DISTANCE;
string refalign = getConfig(KEY_ALIGNMENT_FILES,""); } else if (distanceMetric.compare("KENDALL") == 0) {
//cout << refalign << endl; m_distanceMetric = KENDALL_DISTANCE;
if (refalign.length() > 0){ }
string substring; cerr << "Using permutation distance metric: " << distanceMetric << endl;
while (!refalign.empty()){
getNextPound(refalign, substring, "+");
m_referenceAlignments.push_back(substring);
}
}
//Get length of source sentences read in from scconfig source option //Get reference alignments from scconfig refalign option
// this is essential for extractor but unneccesary for mert executable static string KEY_ALIGNMENT_FILES = "refalign";
static string KEY_SOURCE_FILE = "source"; string refalign = getConfig(KEY_ALIGNMENT_FILES,"");
string sourceFile = getConfig(KEY_SOURCE_FILE,""); //cout << refalign << endl;
if (sourceFile.length() > 0) { if (refalign.length() > 0) {
cerr << "Loading source sentence lengths from " << sourceFile << endl; string substring;
ifstream sourcein(sourceFile.c_str()); while (!refalign.empty()) {
if (!sourcein) { getNextPound(refalign, substring, "+");
throw runtime_error("Unable to open: " + sourceFile); m_referenceAlignments.push_back(substring);
}
string line;
while (getline(sourcein,line)) {
size_t wordNumber = 0;
string word;
while(!line.empty()){
getNextPound(line, word, " ");
wordNumber++;
}
m_sourceLengths.push_back(wordNumber);
}
sourcein.close();
} }
}
//Get length of source sentences read in from scconfig source option
// this is essential for extractor but unneccesary for mert executable
static string KEY_SOURCE_FILE = "source";
string sourceFile = getConfig(KEY_SOURCE_FILE,"");
if (sourceFile.length() > 0) {
cerr << "Loading source sentence lengths from " << sourceFile << endl;
ifstream sourcein(sourceFile.c_str());
if (!sourcein) {
throw runtime_error("Unable to open: " + sourceFile);
}
string line;
while (getline(sourcein,line)) {
size_t wordNumber = 0;
string word;
while(!line.empty()) {
getNextPound(line, word, " ");
wordNumber++;
}
m_sourceLengths.push_back(wordNumber);
}
sourcein.close();
}
} }
void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles) { void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles)
cout << "*******setReferenceFiles" << endl; {
//make sure reference data is clear cout << "*******setReferenceFiles" << endl;
m_referencePerms.clear(); //make sure reference data is clear
m_referencePerms.clear();
vector< vector< int> > targetLengths; vector< vector< int> > targetLengths;
//Just getting target length from reference text file //Just getting target length from reference text file
for (size_t i = 0; i < referenceFiles.size(); ++i) for (size_t i = 0; i < referenceFiles.size(); ++i) {
{ vector <int> lengths;
vector <int> lengths; cout << "Loading reference from " << referenceFiles[i] << endl;
cout << "Loading reference from " << referenceFiles[i] << endl; ifstream refin(referenceFiles[i].c_str());
ifstream refin(referenceFiles[i].c_str()); if (!refin) {
if (!refin) cerr << "Unable to open: " << referenceFiles[i] << endl;
{ throw runtime_error("Unable to open alignment file");
cerr << "Unable to open: " << referenceFiles[i] << endl;
throw runtime_error("Unable to open alignment file");
}
string line;
while (getline(refin,line))
{
int count = getNumberWords(line);
lengths.push_back(count);
}
targetLengths.push_back(lengths);
} }
string line;
while (getline(refin,line)) {
int count = getNumberWords(line);
lengths.push_back(count);
}
targetLengths.push_back(lengths);
}
//load reference data //load reference data
//NOTE ignoring normal reference file, only using previously saved alignment reference files //NOTE ignoring normal reference file, only using previously saved alignment reference files
for (size_t i = 0; i < m_referenceAlignments.size(); ++i) for (size_t i = 0; i < m_referenceAlignments.size(); ++i) {
{ vector<Permutation> referencePerms;
vector<Permutation> referencePerms; cout << "Loading reference from " << m_referenceAlignments[i] << endl;
cout << "Loading reference from " << m_referenceAlignments[i] << endl; ifstream refin(m_referenceAlignments[i].c_str());
ifstream refin(m_referenceAlignments[i].c_str()); if (!refin) {
if (!refin) cerr << "Unable to open: " << m_referenceAlignments[i] << endl;
{ throw runtime_error("Unable to open alignment file");
cerr << "Unable to open: " << m_referenceAlignments[i] << endl;
throw runtime_error("Unable to open alignment file");
}
string line;
size_t sid = 0; //sentence counter
while (getline(refin,line))
{
//cout << line << endl;
//Line needs to be of the format: 0-0 1-1 1-2 etc source-target
Permutation perm(line, m_sourceLengths[sid],targetLengths[i][sid]);
//perm.dump();
referencePerms.push_back(perm);
//check the source sentence length is the same for previous file
if (perm.getLength() != m_sourceLengths[sid])
{
cerr << "Permutation Length: " << perm.getLength() << endl;
cerr << "Source length: " << m_sourceLengths[sid] << " for sid " << sid << endl;
throw runtime_error("Source sentence lengths not the same: ");
}
sid++;
}
m_referencePerms.push_back(referencePerms);
} }
string line;
size_t sid = 0; //sentence counter
while (getline(refin,line)) {
//cout << line << endl;
//Line needs to be of the format: 0-0 1-1 1-2 etc source-target
Permutation perm(line, m_sourceLengths[sid],targetLengths[i][sid]);
//perm.dump();
referencePerms.push_back(perm);
//check the source sentence length is the same for previous file
if (perm.getLength() != m_sourceLengths[sid]) {
cerr << "Permutation Length: " << perm.getLength() << endl;
cerr << "Source length: " << m_sourceLengths[sid] << " for sid " << sid << endl;
throw runtime_error("Source sentence lengths not the same: ");
}
sid++;
}
m_referencePerms.push_back(referencePerms);
}
} }
int PermutationScorer::getNumberWords (const string& text) const { int PermutationScorer::getNumberWords (const string& text) const
int count = 0; {
string line = trimStr(text); int count = 0;
if (line.length()>0) { string line = trimStr(text);
int pos = line.find(" "); if (line.length()>0) {
while (pos!=int(string::npos)){ int pos = line.find(" ");
count++; while (pos!=int(string::npos)) {
pos = line.find(" ",pos+1); count++;
} pos = line.find(" ",pos+1);
count++;
} }
return count; count++;
}
return count;
} }
void PermutationScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) { void PermutationScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
//cout << "*******prepareStats" ; {
//cout << text << endl; //cout << "*******prepareStats" ;
//cout << sid << endl; //cout << text << endl;
//cout << "Reference0align:" << endl; //cout << sid << endl;
//m_referencePerms[0][sid].dump(); //cout << "Reference0align:" << endl;
//m_referencePerms[0][sid].dump();
string sentence = ""; string sentence = "";
string align = text; string align = text;
size_t alignmentData = text.find("|||"); size_t alignmentData = text.find("|||");
//Get sentence and alignment parts //Get sentence and alignment parts
if(alignmentData != string::npos) { if(alignmentData != string::npos) {
getNextPound(align,sentence, "|||"); getNextPound(align,sentence, "|||");
} else { } else {
align = text; align = text;
}
int translationLength = getNumberWords(sentence);
//A vector of Permutations for each sentence
vector< vector<Permutation> > nBestPerms;
float distanceValue;
//need to create permutations for each nbest line
string standardFormat = Permutation::convertMosesToStandard(align);
Permutation perm(standardFormat, m_sourceLengths[sid],translationLength);
//perm.dump();
if (m_refChoiceStrategy == REFERENCE_CHOICE_AVERAGE) {
float total = 0;
for (size_t i = 0; i < m_referencePerms.size(); ++i) {
float dist = perm.distance(m_referencePerms[i][sid], m_distanceMetric);
total += dist;
//cout << "Ref number: " << i << " distance: " << dist << endl;
} }
int translationLength = getNumberWords(sentence); float mean = (float)total/m_referencePerms.size();
//cout << "MultRef strategy AVERAGE: total " << total << " mean " << mean << " number " << m_referencePerms.size() << endl;
distanceValue = mean;
} else if (m_refChoiceStrategy == REFERENCE_CHOICE_CLOSEST) {
float max_val = 0;
for (size_t i = 0; i < m_referencePerms.size(); ++i) {
//A vector of Permutations for each sentence //look for the closest reference
vector< vector<Permutation> > nBestPerms; float value = perm.distance(m_referencePerms[i][sid], m_distanceMetric);
float distanceValue; //cout << "Ref number: " << i << " distance: " << value << endl;
if (value > max_val) {
//need to create permutations for each nbest line max_val = value;
string standardFormat = Permutation::convertMosesToStandard(align); }
Permutation perm(standardFormat, m_sourceLengths[sid],translationLength);
//perm.dump();
if (m_refChoiceStrategy == REFERENCE_CHOICE_AVERAGE) {
float total = 0;
for (size_t i = 0; i < m_referencePerms.size(); ++i) {
float dist = perm.distance(m_referencePerms[i][sid], m_distanceMetric);
total += dist;
//cout << "Ref number: " << i << " distance: " << dist << endl;
}
float mean = (float)total/m_referencePerms.size();
//cout << "MultRef strategy AVERAGE: total " << total << " mean " << mean << " number " << m_referencePerms.size() << endl;
distanceValue = mean;
} else if (m_refChoiceStrategy == REFERENCE_CHOICE_CLOSEST) {
float max_val = 0;
for (size_t i = 0; i < m_referencePerms.size(); ++i) {
//look for the closest reference
float value = perm.distance(m_referencePerms[i][sid], m_distanceMetric);
//cout << "Ref number: " << i << " distance: " << value << endl;
if (value > max_val) {
max_val = value;
}
}
distanceValue = max_val;
//cout << "MultRef strategy CLOSEST: max_val " << distanceValue << endl;
} else {
throw runtime_error("Unsupported reflength strategy");
} }
distanceValue = max_val;
//cout << "MultRef strategy CLOSEST: max_val " << distanceValue << endl;
} else {
throw runtime_error("Unsupported reflength strategy");
}
//SCOREROUT eg: 0.04546 //SCOREROUT eg: 0.04546
ostringstream tempStream; ostringstream tempStream;
tempStream.precision(SCORE_PRECISION); tempStream.precision(SCORE_PRECISION);
tempStream << distanceValue; tempStream << distanceValue;
string str = tempStream.str(); string str = tempStream.str();
entry.set(str); entry.set(str);
//cout << tempStream.str(); //cout << tempStream.str();
} }
//Will just be final score //Will just be final score
statscore_t PermutationScorer::calculateScore(const vector<statscore_t>& comps) { statscore_t PermutationScorer::calculateScore(const vector<statscore_t>& comps)
//cerr << "*******PermutationScorer::calculateScore" ; {
//cerr << " " << comps[0] << endl; //cerr << "*******PermutationScorer::calculateScore" ;
return comps[0]; //cerr << " " << comps[0] << endl;
return comps[0];
} }

68
mert/PermutationScorer.h
View File

@ -17,44 +17,44 @@
#include "Permutation.h" #include "Permutation.h"
/** /**
* Permutation * Permutation
**/ **/
class PermutationScorer: public SentenceLevelScorer class PermutationScorer: public SentenceLevelScorer
{ {
public: public:
PermutationScorer(const string &distanceMetric = "HAMMING", PermutationScorer(const string &distanceMetric = "HAMMING",
const string &config = string()); const string &config = string());
void setReferenceFiles(const vector<string>& referenceFiles); void setReferenceFiles(const vector<string>& referenceFiles);
void prepareStats(size_t sid, const string& text, ScoreStats& entry); void prepareStats(size_t sid, const string& text, ScoreStats& entry);
static const int SCORE_PRECISION; static const int SCORE_PRECISION;
size_t NumberOfScores() const {
//cerr << "PermutationScorer number of scores: 1" << endl;
return 1;
};
bool useAlignment() const {
//cout << "PermutationScorer::useAlignment returning true" << endl;
return true;
};
protected:
statscore_t calculateScore(const vector<statscore_t>& scores);
PermutationScorer(const PermutationScorer&);
~PermutationScorer(){};
PermutationScorer& operator=(const PermutationScorer&);
int getNumberWords (const string & line) const;
distanceMetricReferenceChoice_t m_refChoiceStrategy; size_t NumberOfScores() const {
distanceMetric_t m_distanceMetric; //cerr << "PermutationScorer number of scores: 1" << endl;
return 1;
// data extracted from reference files };
// A vector of permutations for each reference file bool useAlignment() const {
vector< vector<Permutation> > m_referencePerms; //cout << "PermutationScorer::useAlignment returning true" << endl;
vector<size_t> m_sourceLengths; return true;
vector<string> m_referenceAlignments; };
private: protected:
statscore_t calculateScore(const vector<statscore_t>& scores);
PermutationScorer(const PermutationScorer&);
~PermutationScorer() {};
PermutationScorer& operator=(const PermutationScorer&);
int getNumberWords (const string & line) const;
distanceMetricReferenceChoice_t m_refChoiceStrategy;
distanceMetric_t m_distanceMetric;
// data extracted from reference files
// A vector of permutations for each reference file
vector< vector<Permutation> > m_referencePerms;
vector<size_t> m_sourceLengths;
vector<string> m_referenceAlignments;
private:
}; };
//TODO need to read in floats for scores - necessary for selecting mean reference strategy and for BLEU? //TODO need to read in floats for scores - necessary for selecting mean reference strategy and for BLEU?

69
mert/Point.cpp
View File

@ -10,22 +10,24 @@ vector<unsigned> Point::optindices;
unsigned Point::dim=0; unsigned Point::dim=0;
map<unsigned,statscore_t> Point::fixedweights; map<unsigned,statscore_t> Point::fixedweights;
unsigned Point::pdim=0; unsigned Point::pdim=0;
unsigned Point::ncall=0; unsigned Point::ncall=0;
void Point::Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max){ void Point::Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max)
{
assert(min.size()==Point::dim); assert(min.size()==Point::dim);
assert(max.size()==Point::dim); assert(max.size()==Point::dim);
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
operator[](i)= min[i] + (float)random()/(float)RAND_MAX * (float)(max[i]-min[i]); operator[](i)= min[i] + (float)random()/(float)RAND_MAX * (float)(max[i]-min[i]);
} }
void Point::NormalizeL2(){ void Point::NormalizeL2()
{
parameter_t norm=0.0; parameter_t norm=0.0;
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
norm+= operator[](i)*operator[](i); norm+= operator[](i)*operator[](i);
if(norm!=0.0){ if(norm!=0.0) {
norm=sqrt(norm); norm=sqrt(norm);
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
operator[](i)/=norm; operator[](i)/=norm;
@ -33,22 +35,24 @@ void Point::NormalizeL2(){
} }
void Point::NormalizeL1(){ void Point::NormalizeL1()
{
parameter_t norm=0.0; parameter_t norm=0.0;
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
norm+= abs(operator[](i)); norm+= abs(operator[](i));
if(norm!=0.0){ if(norm!=0.0) {
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
operator[](i)/=norm; operator[](i)/=norm;
} }
} }
//Can initialize from a vector of dim or pdim //Can initialize from a vector of dim or pdim
Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim){ Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim)
if(init.size()==dim){ {
if(init.size()==dim) {
for (unsigned int i=0; i<Point::dim; i++) for (unsigned int i=0; i<Point::dim; i++)
operator[](i)=init[i]; operator[](i)=init[i];
}else{ } else {
assert(init.size()==pdim); assert(init.size()==pdim);
for (unsigned int i=0; i<Point::dim; i++) for (unsigned int i=0; i<Point::dim; i++)
operator[](i)=init[optindices[i]]; operator[](i)=init[optindices[i]];
@ -56,59 +60,64 @@ Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim){
}; };
double Point::operator*(const FeatureStats& F)const{ double Point::operator*(const FeatureStats& F)const
{
ncall++;//to track performance ncall++;//to track performance
double prod=0.0; double prod=0.0;
if(OptimizeAll()) if(OptimizeAll())
for (unsigned i=0; i<size(); i++) for (unsigned i=0; i<size(); i++)
prod+= operator[](i)*F.get(i); prod+= operator[](i)*F.get(i);
else{ else {
for (unsigned i=0; i<size(); i++) for (unsigned i=0; i<size(); i++)
prod+= operator[](i)*F.get(optindices[i]); prod+= operator[](i)*F.get(optindices[i]);
for(map<unsigned,float >::iterator it=fixedweights.begin();it!=fixedweights.end();it++) for(map<unsigned,float >::iterator it=fixedweights.begin(); it!=fixedweights.end(); it++)
prod+=it->second*F.get(it->first); prod+=it->second*F.get(it->first);
} }
return prod; return prod;
} }
Point Point::operator+(const Point& p2)const{ Point Point::operator+(const Point& p2)const
{
assert(p2.size()==size()); assert(p2.size()==size());
Point Res(*this); Point Res(*this);
for(unsigned i=0;i<size();i++) for(unsigned i=0; i<size(); i++)
Res[i]+=p2[i]; Res[i]+=p2[i];
Res.score=numeric_limits<statscore_t>::max(); Res.score=numeric_limits<statscore_t>::max();
return Res; return Res;
}; };
Point Point::operator*(float l)const{ Point Point::operator*(float l)const
{
Point Res(*this); Point Res(*this);
for(unsigned i=0;i<size();i++) for(unsigned i=0; i<size(); i++)
Res[i]*=l; Res[i]*=l;
Res.score=numeric_limits<statscore_t>::max(); Res.score=numeric_limits<statscore_t>::max();
return Res; return Res;
}; };
ostream& operator<<(ostream& o,const Point& P){ ostream& operator<<(ostream& o,const Point& P)
vector<parameter_t> w=P.GetAllWeights(); {
vector<parameter_t> w=P.GetAllWeights();
// o << "[" << Point::pdim << "] "; // o << "[" << Point::pdim << "] ";
for(unsigned int i=0;i<Point::pdim;i++) for(unsigned int i=0; i<Point::pdim; i++)
o << w[i] << " "; o << w[i] << " ";
// o << "=> " << P.GetScore(); // o << "=> " << P.GetScore();
return o; return o;
}; };
vector<parameter_t> Point::GetAllWeights()const{ vector<parameter_t> Point::GetAllWeights()const
{
vector<parameter_t> w; vector<parameter_t> w;
if(OptimizeAll()){ if(OptimizeAll()) {
w=*this; w=*this;
}else{ } else {
w.resize(pdim); w.resize(pdim);
for (unsigned int i=0; i<size(); i++) for (unsigned int i=0; i<size(); i++)
w[optindices[i]]=operator[](i); w[optindices[i]]=operator[](i);
for(map<unsigned,float >::iterator it=fixedweights.begin();it!=fixedweights.end();it++) for(map<unsigned,float >::iterator it=fixedweights.begin(); it!=fixedweights.end(); it++)
w[it->first]=it->second; w[it->first]=it->second;
} }
return w; return w;
}; };

29
mert/Point.h
View File

@ -10,9 +10,10 @@ class Optimizer;
/**class that handle the subset of the Feature weight on which we run the optimization*/ /**class that handle the subset of the Feature weight on which we run the optimization*/
class Point:public vector<parameter_t>{ class Point:public vector<parameter_t>
{
friend class Optimizer; friend class Optimizer;
private: private:
/**The indices over which we optimize*/ /**The indices over which we optimize*/
static vector<unsigned int> optindices; static vector<unsigned int> optindices;
/**dimension of optindices and of the parent vector*/ /**dimension of optindices and of the parent vector*/
@ -22,12 +23,18 @@ class Point:public vector<parameter_t>{
/**total size of the parameter space; we have pdim=FixedWeight.size()+optinidices.size()*/ /**total size of the parameter space; we have pdim=FixedWeight.size()+optinidices.size()*/
static unsigned int pdim; static unsigned int pdim;
static unsigned int ncall; static unsigned int ncall;
public: public:
static unsigned int getdim(){return dim;} static unsigned int getdim() {
static unsigned int getpdim(){return pdim;} return dim;
static bool OptimizeAll(){return fixedweights.empty();}; }
static unsigned int getpdim() {
return pdim;
}
static bool OptimizeAll() {
return fixedweights.empty();
};
statscore_t score; statscore_t score;
Point():vector<parameter_t>(dim){}; Point():vector<parameter_t>(dim) {};
Point(const vector<parameter_t>& init); Point(const vector<parameter_t>& init);
void Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max); void Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max);
@ -36,12 +43,16 @@ class Point:public vector<parameter_t>{
Point operator*(float)const; Point operator*(float)const;
/**write the Whole featureweight to a stream (ie pdim float)*/ /**write the Whole featureweight to a stream (ie pdim float)*/
friend ostream& operator<<(ostream& o,const Point& P); friend ostream& operator<<(ostream& o,const Point& P);
void Normalize(){ NormalizeL2(); }; void Normalize() {
NormalizeL2();
};
void NormalizeL2(); void NormalizeL2();
void NormalizeL1(); void NormalizeL1();
/**return a vector of size pdim where all weights have been put(including fixed ones)*/ /**return a vector of size pdim where all weights have been put(including fixed ones)*/
vector<parameter_t> GetAllWeights()const; vector<parameter_t> GetAllWeights()const;
statscore_t GetScore()const { return score; }; statscore_t GetScore()const {
return score;
};
}; };
#endif #endif

164
mert/ScoreArray.cpp
View File

@ -15,134 +15,134 @@ ScoreArray::ScoreArray(): idx("")
void ScoreArray::savetxt(std::ofstream& outFile, const std::string& sctype) void ScoreArray::savetxt(std::ofstream& outFile, const std::string& sctype)
{ {
outFile << SCORES_TXT_BEGIN << " " << idx << " " << array_.size() outFile << SCORES_TXT_BEGIN << " " << idx << " " << array_.size()
<< " " << number_of_scores << " " << sctype << std::endl; << " " << number_of_scores << " " << sctype << std::endl;
for (scorearray_t::iterator i = array_.begin(); i !=array_.end(); i++){ for (scorearray_t::iterator i = array_.begin(); i !=array_.end(); i++) {
i->savetxt(outFile); i->savetxt(outFile);
outFile << std::endl; outFile << std::endl;
} }
outFile << SCORES_TXT_END << std::endl; outFile << SCORES_TXT_END << std::endl;
} }
void ScoreArray::savebin(std::ofstream& outFile, const std::string& sctype) void ScoreArray::savebin(std::ofstream& outFile, const std::string& sctype)
{ {
outFile << SCORES_BIN_BEGIN << " " << idx << " " << array_.size() outFile << SCORES_BIN_BEGIN << " " << idx << " " << array_.size()
<< " " << number_of_scores << " " << sctype << std::endl; << " " << number_of_scores << " " << sctype << std::endl;
for (scorearray_t::iterator i = array_.begin(); i !=array_.end(); i++) for (scorearray_t::iterator i = array_.begin(); i !=array_.end(); i++)
i->savebin(outFile); i->savebin(outFile);
outFile << SCORES_BIN_END << std::endl; outFile << SCORES_BIN_END << std::endl;
} }
void ScoreArray::save(std::ofstream& inFile, const std::string& sctype, bool bin) void ScoreArray::save(std::ofstream& inFile, const std::string& sctype, bool bin)
{ {
if (size()>0) if (size()>0)
(bin)?savebin(inFile, sctype):savetxt(inFile, sctype); (bin)?savebin(inFile, sctype):savetxt(inFile, sctype);
} }
void ScoreArray::save(const std::string &file, const std::string& sctype, bool bin) void ScoreArray::save(const std::string &file, const std::string& sctype, bool bin)
{ {
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
save(outFile, sctype, bin); save(outFile, sctype, bin);
outFile.close(); outFile.close();
} }
void ScoreArray::loadbin(ifstream& inFile, size_t n) void ScoreArray::loadbin(ifstream& inFile, size_t n)
{ {
ScoreStats entry(number_of_scores); ScoreStats entry(number_of_scores);
for (size_t i=0 ; i < n; i++){ for (size_t i=0 ; i < n; i++) {
entry.loadbin(inFile); entry.loadbin(inFile);
add(entry); add(entry);
} }
} }
void ScoreArray::loadtxt(ifstream& inFile, size_t n) void ScoreArray::loadtxt(ifstream& inFile, size_t n)
{ {
ScoreStats entry(number_of_scores); ScoreStats entry(number_of_scores);
for (size_t i=0 ; i < n; i++){ for (size_t i=0 ; i < n; i++) {
entry.loadtxt(inFile); entry.loadtxt(inFile);
add(entry); add(entry);
} }
} }
void ScoreArray::load(ifstream& inFile) void ScoreArray::load(ifstream& inFile)
{ {
size_t number_of_entries=0; size_t number_of_entries=0;
bool binmode=false; bool binmode=false;
std::string substring, stringBuf; std::string substring, stringBuf;
std::string::size_type loc; std::string::size_type loc;
std::getline(inFile, stringBuf); std::getline(inFile, stringBuf);
if (!inFile.good()){ if (!inFile.good()) {
return; return;
} }
if (!stringBuf.empty()){ if (!stringBuf.empty()) {
if ((loc = stringBuf.find(SCORES_TXT_BEGIN)) == 0){ if ((loc = stringBuf.find(SCORES_TXT_BEGIN)) == 0) {
binmode=false; binmode=false;
}else if ((loc = stringBuf.find(SCORES_BIN_BEGIN)) == 0){ } else if ((loc = stringBuf.find(SCORES_BIN_BEGIN)) == 0) {
binmode=true; binmode=true;
}else{ } else {
TRACE_ERR("ERROR: ScoreArray::load(): Wrong header"); TRACE_ERR("ERROR: ScoreArray::load(): Wrong header");
return; return;
} }
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
idx = substring; idx = substring;
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
number_of_entries = atoi(substring.c_str()); number_of_entries = atoi(substring.c_str());
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
number_of_scores = atoi(substring.c_str()); number_of_scores = atoi(substring.c_str());
getNextPound(stringBuf, substring); getNextPound(stringBuf, substring);
score_type = substring; score_type = substring;
} }
(binmode)?loadbin(inFile, number_of_entries):loadtxt(inFile, number_of_entries); (binmode)?loadbin(inFile, number_of_entries):loadtxt(inFile, number_of_entries);
std::getline(inFile, stringBuf); std::getline(inFile, stringBuf);
if (!stringBuf.empty()){ if (!stringBuf.empty()) {
if ((loc = stringBuf.find(SCORES_TXT_END)) != 0 && (loc = stringBuf.find(SCORES_BIN_END)) != 0){ if ((loc = stringBuf.find(SCORES_TXT_END)) != 0 && (loc = stringBuf.find(SCORES_BIN_END)) != 0) {
TRACE_ERR("ERROR: ScoreArray::load(): Wrong footer"); TRACE_ERR("ERROR: ScoreArray::load(): Wrong footer");
return; return;
} }
} }
} }
void ScoreArray::load(const std::string &file) void ScoreArray::load(const std::string &file)
{ {
TRACE_ERR("loading data from " << file << std::endl); TRACE_ERR("loading data from " << file << std::endl);
inputfilestream inFile(file); // matches a stream with a file. Opens the file inputfilestream inFile(file); // matches a stream with a file. Opens the file
load((ifstream&) inFile); load((ifstream&) inFile);
inFile.close(); inFile.close();
} }
void ScoreArray::merge(ScoreArray& e) void ScoreArray::merge(ScoreArray& e)
{ {
//dummy implementation //dummy implementation
for (size_t i=0; i<e.size(); i++) for (size_t i=0; i<e.size(); i++)
add(e.get(i)); add(e.get(i));
} }
bool ScoreArray::check_consistency() bool ScoreArray::check_consistency()
{ {
size_t sz = NumberOfScores(); size_t sz = NumberOfScores();
if (sz == 0) if (sz == 0)
return true; return true;
for (scorearray_t::iterator i=array_.begin(); i!=array_.end(); i++) for (scorearray_t::iterator i=array_.begin(); i!=array_.end(); i++)
if (i->size()!=sz) if (i->size()!=sz)
return false; return false;
return true; return true;
} }

100
mert/ScoreArray.h
View File

@ -27,52 +27,76 @@ using namespace std;
class ScoreArray class ScoreArray
{ {
protected: protected:
scorearray_t array_; scorearray_t array_;
std::string score_type; std::string score_type;
size_t number_of_scores; size_t number_of_scores;
private: private:
std::string idx; // idx to identify the utterance, it can differ from the index inside the vector std::string idx; // idx to identify the utterance, it can differ from the index inside the vector
public: public:
ScoreArray(); ScoreArray();
~ScoreArray(){}; ~ScoreArray() {};
inline void clear() { array_.clear(); } inline void clear() {
array_.clear();
inline std::string getIndex(){ return idx; } }
inline void setIndex(const std::string& value){ idx=value; }
inline std::string getIndex() {
return idx;
}
inline void setIndex(const std::string& value) {
idx=value;
}
// inline ScoreStats get(size_t i){ return array_.at(i); } // inline ScoreStats get(size_t i){ return array_.at(i); }
inline ScoreStats& get(size_t i){ return array_.at(i); }
inline const ScoreStats& get(size_t i)const{ return array_.at(i); }
void add(const ScoreStats& e){ array_.push_back(e); } inline ScoreStats& get(size_t i) {
return array_.at(i);
}
inline const ScoreStats& get(size_t i)const {
return array_.at(i);
}
void merge(ScoreArray& e); void add(const ScoreStats& e) {
array_.push_back(e);
}
inline std::string name() const{ return score_type; }; void merge(ScoreArray& e);
inline void name(std::string &sctype){ score_type = sctype; };
inline size_t size(){ return array_.size(); } inline std::string name() const {
inline size_t NumberOfScores() const{ return number_of_scores; } return score_type;
inline void NumberOfScores(size_t v){ number_of_scores = v; } };
inline void name(std::string &sctype) {
void savetxt(ofstream& outFile, const std::string& sctype); score_type = sctype;
void savebin(ofstream& outFile, const std::string& sctype); };
void save(ofstream& outFile, const std::string& sctype, bool bin=false);
void save(const std::string &file, const std::string& sctype, bool bin=false); inline size_t size() {
inline void save(const std::string& sctype, bool bin=false){ save("/dev/stdout", sctype, bin); } return array_.size();
}
void loadtxt(ifstream& inFile, size_t n); inline size_t NumberOfScores() const {
void loadbin(ifstream& inFile, size_t n); return number_of_scores;
void load(ifstream& inFile); }
void load(const std::string &file); inline void NumberOfScores(size_t v) {
number_of_scores = v;
bool check_consistency(); }
void savetxt(ofstream& outFile, const std::string& sctype);
void savebin(ofstream& outFile, const std::string& sctype);
void save(ofstream& outFile, const std::string& sctype, bool bin=false);
void save(const std::string &file, const std::string& sctype, bool bin=false);
inline void save(const std::string& sctype, bool bin=false) {
save("/dev/stdout", sctype, bin);
}
void loadtxt(ifstream& inFile, size_t n);
void loadbin(ifstream& inFile, size_t n);
void load(ifstream& inFile);
void load(const std::string &file);
bool check_consistency();
}; };

176
mert/ScoreData.cpp
View File

@ -13,138 +13,138 @@
ScoreData::ScoreData(Scorer& ptr): ScoreData::ScoreData(Scorer& ptr):
theScorer(&ptr) theScorer(&ptr)
{ {
score_type = theScorer->getName(); score_type = theScorer->getName();
//theScorer->setScoreData(this);//this is not dangerous: we dont use the this pointer in SetScoreData //theScorer->setScoreData(this);//this is not dangerous: we dont use the this pointer in SetScoreData
number_of_scores = theScorer->NumberOfScores(); number_of_scores = theScorer->NumberOfScores();
TRACE_ERR("ScoreData: number_of_scores: " << number_of_scores << std::endl); TRACE_ERR("ScoreData: number_of_scores: " << number_of_scores << std::endl);
}; };
void ScoreData::dump() void ScoreData::dump()
{ {
for (vector<ScoreArray>::iterator it = array_.begin(); it !=array_.end(); it++){ for (vector<ScoreArray>::iterator it = array_.begin(); it !=array_.end(); it++) {
cout << "scorearray: " << endl; cout << "scorearray: " << endl;
for (size_t i = 0; i < (*it).size(); i++) { for (size_t i = 0; i < (*it).size(); i++) {
ScoreStats scoreStats = (*it).get(i); ScoreStats scoreStats = (*it).get(i);
cout << "scorestats: " ; cout << "scorestats: " ;
for (size_t j = 0; j < scoreStats.size(); j ++ ){ for (size_t j = 0; j < scoreStats.size(); j ++ ) {
ScoreStatsType scoreStatsType = scoreStats.get(j); ScoreStatsType scoreStatsType = scoreStats.get(j);
cout << scoreStatsType << " " ; cout << scoreStatsType << " " ;
} }
cout << endl; cout << endl;
} }
} }
} }
void ScoreData::save(std::ofstream& outFile, bool bin) void ScoreData::save(std::ofstream& outFile, bool bin)
{ {
for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++){ for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++) {
i->save(outFile, score_type, bin); i->save(outFile, score_type, bin);
} }
} }
void ScoreData::save(const std::string &file, bool bin) void ScoreData::save(const std::string &file, bool bin)
{ {
if (file.empty()) return; if (file.empty()) return;
TRACE_ERR("saving the array into " << file << std::endl); TRACE_ERR("saving the array into " << file << std::endl);
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
ScoreStats entry; ScoreStats entry;
save(outFile, bin); save(outFile, bin);
outFile.close(); outFile.close();
} }
void ScoreData::load(ifstream& inFile) void ScoreData::load(ifstream& inFile)
{ {
ScoreArray entry; ScoreArray entry;
while (!inFile.eof()){ while (!inFile.eof()) {
if (!inFile.good()){
std::cerr << "ERROR ScoreData::load inFile.good()" << std::endl;
}
entry.clear();
entry.load(inFile);
if (entry.size() == 0){ if (!inFile.good()) {
break; std::cerr << "ERROR ScoreData::load inFile.good()" << std::endl;
} }
add(entry);
} entry.clear();
theScorer->setScoreData(this); entry.load(inFile);
if (entry.size() == 0) {
break;
}
add(entry);
}
theScorer->setScoreData(this);
} }
void ScoreData::load(const std::string &file) void ScoreData::load(const std::string &file)
{ {
TRACE_ERR("loading score data from " << file << std::endl); TRACE_ERR("loading score data from " << file << std::endl);
inputfilestream inFile(file); // matches a stream with a file. Opens the file inputfilestream inFile(file); // matches a stream with a file. Opens the file
if (!inFile) { if (!inFile) {
throw runtime_error("Unable to open score file: " + file); throw runtime_error("Unable to open score file: " + file);
} }
load((ifstream&) inFile); load((ifstream&) inFile);
inFile.close(); inFile.close();
} }
void ScoreData::add(ScoreArray& e){ void ScoreData::add(ScoreArray& e)
if (exists(e.getIndex())){ // array at position e.getIndex() already exists {
//enlarge array at position e.getIndex() if (exists(e.getIndex())) { // array at position e.getIndex() already exists
size_t pos = getIndex(e.getIndex()); //enlarge array at position e.getIndex()
array_.at(pos).merge(e); size_t pos = getIndex(e.getIndex());
} array_.at(pos).merge(e);
else{ } else {
array_.push_back(e); array_.push_back(e);
setIndex(); setIndex();
} }
} }
void ScoreData::add(const ScoreStats& e, const std::string& sent_idx){ void ScoreData::add(const ScoreStats& e, const std::string& sent_idx)
if (exists(sent_idx)){ // array at position e.getIndex() already exists {
//enlarge array at position e.getIndex() if (exists(sent_idx)) { // array at position e.getIndex() already exists
size_t pos = getIndex(sent_idx); //enlarge array at position e.getIndex()
// TRACE_ERR("Inserting in array " << sent_idx << std::endl); size_t pos = getIndex(sent_idx);
array_.at(pos).add(e); // TRACE_ERR("Inserting in array " << sent_idx << std::endl);
// TRACE_ERR("size: " << size() << " -> " << a.size() << std::endl); array_.at(pos).add(e);
} // TRACE_ERR("size: " << size() << " -> " << a.size() << std::endl);
else{ } else {
// TRACE_ERR("Creating a new entry in the array" << std::endl); // TRACE_ERR("Creating a new entry in the array" << std::endl);
ScoreArray a; ScoreArray a;
a.NumberOfScores(number_of_scores); a.NumberOfScores(number_of_scores);
a.add(e); a.add(e);
a.setIndex(sent_idx); a.setIndex(sent_idx);
add(a); add(a);
// TRACE_ERR("size: " << size() << " -> " << a.size() << std::endl); // TRACE_ERR("size: " << size() << " -> " << a.size() << std::endl);
} }
} }
bool ScoreData::check_consistency() bool ScoreData::check_consistency()
{ {
if (array_.size() == 0) if (array_.size() == 0)
return true; return true;
for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++) for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++)
if (!i->check_consistency()) return false; if (!i->check_consistency()) return false;
return true; return true;
} }
void ScoreData::setIndex() void ScoreData::setIndex()
{ {
size_t j=0; size_t j=0;
for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++){ for (scoredata_t::iterator i = array_.begin(); i !=array_.end(); i++) {
idx2arrayname_[j]=i->getIndex(); idx2arrayname_[j]=i->getIndex();
arrayname2idx_[i->getIndex()]=j; arrayname2idx_[i->getIndex()]=j;
j++; j++;
} }
} }

134
mert/ScoreData.h
View File

@ -23,66 +23,92 @@ class Scorer;
class ScoreData class ScoreData
{ {
protected: protected:
scoredata_t array_; scoredata_t array_;
idx2name idx2arrayname_; //map from index to name of array idx2name idx2arrayname_; //map from index to name of array
name2idx arrayname2idx_; //map from name to index of array name2idx arrayname2idx_; //map from name to index of array
private: private:
Scorer* theScorer; Scorer* theScorer;
std::string score_type; std::string score_type;
size_t number_of_scores; size_t number_of_scores;
public: public:
ScoreData(Scorer& sc); ScoreData(Scorer& sc);
~ScoreData(){};
inline void clear() { array_.clear(); }
inline ScoreArray get(const std::string& idx){ return array_.at(getIndex(idx)); }
inline ScoreArray& get(size_t idx){ return array_.at(idx); }
inline const ScoreArray& get(size_t idx) const { return array_.at(idx); }
inline bool exists(const std::string & sent_idx){ return exists(getIndex(sent_idx)); }
inline bool exists(int sent_idx){ return (sent_idx>-1 && sent_idx<(int)array_.size())?true:false; }
inline ScoreStats& get(size_t i, size_t j){ return array_.at(i).get(j); }
inline const ScoreStats& get(size_t i, size_t j) const { return array_.at(i).get(j); }
inline std::string name(){ return score_type; };
inline std::string name(std::string &sctype){ return score_type = sctype; };
void add(ScoreArray& e); ~ScoreData() {};
void add(const ScoreStats& e, const std::string& sent_idx);
inline size_t NumberOfScores(){ return number_of_scores; }
inline size_t size(){ return array_.size(); }
void save(const std::string &file, bool bin=false);
void save(ofstream& outFile, bool bin=false);
inline void save(bool bin=false){ save("/dev/stdout", bin); }
void load(ifstream& inFile); inline void clear() {
void load(const std::string &file); array_.clear();
bool check_consistency();
void setIndex();
inline int getIndex(const std::string& idx){
name2idx::iterator i = arrayname2idx_.find(idx);
if (i!=arrayname2idx_.end())
return i->second;
else
return -1;
} }
inline std::string getIndex(size_t idx){
idx2name::iterator i = idx2arrayname_.find(idx);
if (i!=idx2arrayname_.end())
throw runtime_error("there is no entry at index " + idx);
return i->second;
}
void dump(); inline ScoreArray get(const std::string& idx) {
return array_.at(getIndex(idx));
}
inline ScoreArray& get(size_t idx) {
return array_.at(idx);
}
inline const ScoreArray& get(size_t idx) const {
return array_.at(idx);
}
inline bool exists(const std::string & sent_idx) {
return exists(getIndex(sent_idx));
}
inline bool exists(int sent_idx) {
return (sent_idx>-1 && sent_idx<(int)array_.size())?true:false;
}
inline ScoreStats& get(size_t i, size_t j) {
return array_.at(i).get(j);
}
inline const ScoreStats& get(size_t i, size_t j) const {
return array_.at(i).get(j);
}
inline std::string name() {
return score_type;
};
inline std::string name(std::string &sctype) {
return score_type = sctype;
};
void add(ScoreArray& e);
void add(const ScoreStats& e, const std::string& sent_idx);
inline size_t NumberOfScores() {
return number_of_scores;
}
inline size_t size() {
return array_.size();
}
void save(const std::string &file, bool bin=false);
void save(ofstream& outFile, bool bin=false);
inline void save(bool bin=false) {
save("/dev/stdout", bin);
}
void load(ifstream& inFile);
void load(const std::string &file);
bool check_consistency();
void setIndex();
inline int getIndex(const std::string& idx) {
name2idx::iterator i = arrayname2idx_.find(idx);
if (i!=arrayname2idx_.end())
return i->second;
else
return -1;
}
inline std::string getIndex(size_t idx) {
idx2name::iterator i = idx2arrayname_.find(idx);
if (i!=idx2arrayname_.end())
throw runtime_error("there is no entry at index " + idx);
return i->second;
}
void dump();
}; };

103
mert/ScoreStats.cpp
View File

@ -14,123 +14,124 @@
ScoreStats::ScoreStats() ScoreStats::ScoreStats()
{ {
available_ = AVAILABLE_; available_ = AVAILABLE_;
entries_ = 0; entries_ = 0;
array_ = new ScoreStatsType[available_]; array_ = new ScoreStatsType[available_];
}; };
ScoreStats::~ScoreStats() ScoreStats::~ScoreStats()
{ {
delete array_; delete array_;
}; };
ScoreStats::ScoreStats(const ScoreStats &stats) ScoreStats::ScoreStats(const ScoreStats &stats)
{ {
available_ = stats.available(); available_ = stats.available();
entries_ = stats.size(); entries_ = stats.size();
array_ = new ScoreStatsType[available_]; array_ = new ScoreStatsType[available_];
memcpy(array_,stats.getArray(),scorebytes_); memcpy(array_,stats.getArray(),scorebytes_);
}; };
ScoreStats::ScoreStats(const size_t size) ScoreStats::ScoreStats(const size_t size)
{ {
available_ = size; available_ = size;
entries_ = size; entries_ = size;
array_ = new ScoreStatsType[available_]; array_ = new ScoreStatsType[available_];
memset(array_,0,scorebytes_); memset(array_,0,scorebytes_);
}; };
ScoreStats::ScoreStats(std::string &theString) ScoreStats::ScoreStats(std::string &theString)
{ {
set(theString); set(theString);
} }
void ScoreStats::expand() void ScoreStats::expand()
{ {
available_*=2; available_*=2;
scorestats_t t_ = new ScoreStatsType[available_]; scorestats_t t_ = new ScoreStatsType[available_];
memcpy(t_,array_,scorebytes_); memcpy(t_,array_,scorebytes_);
delete array_; delete array_;
array_=t_; array_=t_;
} }
void ScoreStats::add(ScoreStatsType v) void ScoreStats::add(ScoreStatsType v)
{ {
if (isfull()) expand(); if (isfull()) expand();
array_[entries_++]=v; array_[entries_++]=v;
} }
void ScoreStats::set(std::string &theString) void ScoreStats::set(std::string &theString)
{ {
std::string substring, stringBuf; std::string substring, stringBuf;
reset(); reset();
while (!theString.empty()){ while (!theString.empty()) {
getNextPound(theString, substring); getNextPound(theString, substring);
add(ATOSST(substring.c_str())); add(ATOSST(substring.c_str()));
} }
} }
void ScoreStats::loadbin(std::ifstream& inFile) void ScoreStats::loadbin(std::ifstream& inFile)
{ {
inFile.read((char*) array_, scorebytes_); inFile.read((char*) array_, scorebytes_);
} }
void ScoreStats::loadtxt(std::ifstream& inFile) void ScoreStats::loadtxt(std::ifstream& inFile)
{ {
std::string theString; std::string theString;
std::getline(inFile, theString); std::getline(inFile, theString);
set(theString); set(theString);
} }
void ScoreStats::loadtxt(const std::string &file) void ScoreStats::loadtxt(const std::string &file)
{ {
// TRACE_ERR("loading the stats from " << file << std::endl); // TRACE_ERR("loading the stats from " << file << std::endl);
std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file
loadtxt(inFile); loadtxt(inFile);
} }
void ScoreStats::savetxt(const std::string &file) void ScoreStats::savetxt(const std::string &file)
{ {
// TRACE_ERR("saving the stats into " << file << std::endl); // TRACE_ERR("saving the stats into " << file << std::endl);
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
savetxt(outFile); savetxt(outFile);
} }
void ScoreStats::savetxt(std::ofstream& outFile) void ScoreStats::savetxt(std::ofstream& outFile)
{ {
outFile << *this; outFile << *this;
} }
void ScoreStats::savebin(std::ofstream& outFile) void ScoreStats::savebin(std::ofstream& outFile)
{ {
outFile.write((char*) array_, scorebytes_); outFile.write((char*) array_, scorebytes_);
} }
ScoreStats& ScoreStats::operator=(const ScoreStats &stats) ScoreStats& ScoreStats::operator=(const ScoreStats &stats)
{ {
delete array_; delete array_;
available_ = stats.available(); available_ = stats.available();
entries_ = stats.size(); entries_ = stats.size();
array_ = new ScoreStatsType[available_]; array_ = new ScoreStatsType[available_];
memcpy(array_,stats.getArray(),scorebytes_); memcpy(array_,stats.getArray(),scorebytes_);
return *this; return *this;
} }
/**write the whole object to a stream*/ /**write the whole object to a stream*/
ostream& operator<<(ostream& o, const ScoreStats& e){ ostream& operator<<(ostream& o, const ScoreStats& e)
for (size_t i=0; i< e.size(); i++) {
o << e.get(i) << " "; for (size_t i=0; i< e.size(); i++)
return o; o << e.get(i) << " ";
return o;
} }

101
mert/ScoreStats.h
View File

@ -26,51 +26,72 @@ using namespace std;
class ScoreStats class ScoreStats
{ {
private: private:
scorestats_t array_; scorestats_t array_;
size_t entries_; size_t entries_;
size_t available_; size_t available_;
public: public:
ScoreStats(); ScoreStats();
ScoreStats(const size_t size); ScoreStats(const size_t size);
ScoreStats(const ScoreStats &stats); ScoreStats(const ScoreStats &stats);
ScoreStats(std::string &theString); ScoreStats(std::string &theString);
ScoreStats& operator=(const ScoreStats &stats); ScoreStats& operator=(const ScoreStats &stats);
~ScoreStats();
bool isfull(){return (entries_ < available_)?0:1; }
void expand();
void add(ScoreStatsType v);
inline void clear() { memset((void*) array_,0,scorebytes_); }
inline ScoreStatsType get(size_t i){ return array_[i]; }
inline ScoreStatsType get(size_t i)const{ return array_[i]; }
inline scorestats_t getArray() const { return array_; }
void set(std::string &theString);
inline size_t bytes() const{ return scorebytes_; } ~ScoreStats();
inline size_t size() const{ return entries_; }
inline size_t available() const{ return available_; }
void savetxt(const std::string &file);
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile);
inline void savetxt(){ savetxt("/dev/stdout"); }
bool isfull() {
return (entries_ < available_)?0:1;
void loadtxt(const std::string &file); }
void loadtxt(ifstream& inFile); void expand();
void loadbin(ifstream& inFile); void add(ScoreStatsType v);
inline void reset(){ entries_ = 0; clear(); }
/**write the whole object to a stream*/ inline void clear() {
friend ostream& operator<<(ostream& o, const ScoreStats& e); memset((void*) array_,0,scorebytes_);
}
inline ScoreStatsType get(size_t i) {
return array_[i];
}
inline ScoreStatsType get(size_t i)const {
return array_[i];
}
inline scorestats_t getArray() const {
return array_;
}
void set(std::string &theString);
inline size_t bytes() const {
return scorebytes_;
}
inline size_t size() const {
return entries_;
}
inline size_t available() const {
return available_;
}
void savetxt(const std::string &file);
void savetxt(ofstream& outFile);
void savebin(ofstream& outFile);
inline void savetxt() {
savetxt("/dev/stdout");
}
void loadtxt(const std::string &file);
void loadtxt(ifstream& inFile);
void loadbin(ifstream& inFile);
inline void reset() {
entries_ = 0;
clear();
}
/**write the whole object to a stream*/
friend ostream& operator<<(ostream& o, const ScoreStats& e);
}; };

298
mert/Scorer.cpp
View File

@ -1,105 +1,108 @@
#include "Scorer.h" #include "Scorer.h"
//regularisation strategies //regularisation strategies
static float score_min(const statscores_t& scores, size_t start, size_t end) { static float score_min(const statscores_t& scores, size_t start, size_t end)
float min = numeric_limits<float>::max(); {
for (size_t i = start; i < end; ++i) { float min = numeric_limits<float>::max();
if (scores[i] < min) { for (size_t i = start; i < end; ++i) {
min = scores[i]; if (scores[i] < min) {
} min = scores[i];
} }
return min; }
return min;
} }
static float score_average(const statscores_t& scores, size_t start, size_t end) { static float score_average(const statscores_t& scores, size_t start, size_t end)
if ((end - start) < 1) { {
//shouldn't happen if ((end - start) < 1) {
return 0; //shouldn't happen
} return 0;
float total = 0; }
for (size_t j = start; j < end; ++j) { float total = 0;
total += scores[j]; for (size_t j = start; j < end; ++j) {
} total += scores[j];
}
return total / (end - start); return total / (end - start);
} }
void StatisticsBasedScorer::score(const candidates_t& candidates, const diffs_t& diffs, void StatisticsBasedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) { statscores_t& scores)
//cout << "*******StatisticsBasedScorer::score" << endl; {
if (!_scoreData) { //cout << "*******StatisticsBasedScorer::score" << endl;
throw runtime_error("Score data not loaded"); if (!_scoreData) {
} throw runtime_error("Score data not loaded");
//calculate the score for the candidates }
if (_scoreData->size() == 0) { //calculate the score for the candidates
throw runtime_error("Score data is empty"); if (_scoreData->size() == 0) {
throw runtime_error("Score data is empty");
}
if (candidates.size() == 0) {
throw runtime_error("No candidates supplied");
}
int numCounts = _scoreData->get(0,candidates[0]).size();
vector<float> totals(numCounts);
for (size_t i = 0; i < candidates.size(); ++i) {
//cout << " i " << i << " candidates[i] " << candidates[i] << endl;
ScoreStats stats = _scoreData->get(i,candidates[i]);
if (stats.size() != totals.size()) {
stringstream msg;
msg << "Statistics for (" << "," << candidates[i] << ") have incorrect "
<< "number of fields. Found: " << stats.size() << " Expected: "
<< totals.size();
throw runtime_error(msg.str());
} }
if (candidates.size() == 0) { for (size_t k = 0; k < totals.size(); ++k) {
throw runtime_error("No candidates supplied"); totals[k] += stats.get(k);
} }
int numCounts = _scoreData->get(0,candidates[0]).size(); }
vector<float> totals(numCounts); scores.push_back(calculateScore(totals));
for (size_t i = 0; i < candidates.size(); ++i) {
//cout << " i " << i << " candidates[i] " << candidates[i] << endl;
ScoreStats stats = _scoreData->get(i,candidates[i]);
if (stats.size() != totals.size()) {
stringstream msg;
msg << "Statistics for (" << "," << candidates[i] << ") have incorrect "
<< "number of fields. Found: " << stats.size() << " Expected: "
<< totals.size();
throw runtime_error(msg.str());
}
for (size_t k = 0; k < totals.size(); ++k) {
totals[k] += stats.get(k);
}
}
scores.push_back(calculateScore(totals));
candidates_t last_candidates(candidates); candidates_t last_candidates(candidates);
//apply each of the diffs, and get new scores //apply each of the diffs, and get new scores
for (size_t i = 0; i < diffs.size(); ++i) { for (size_t i = 0; i < diffs.size(); ++i) {
for (size_t j = 0; j < diffs[i].size(); ++j) { for (size_t j = 0; j < diffs[i].size(); ++j) {
size_t sid = diffs[i][j].first; size_t sid = diffs[i][j].first;
size_t nid = diffs[i][j].second; size_t nid = diffs[i][j].second;
//cout << "STSC:sid = " << sid << endl; //cout << "STSC:sid = " << sid << endl;
//cout << "STSC:nid = " << nid << endl; //cout << "STSC:nid = " << nid << endl;
size_t last_nid = last_candidates[sid]; size_t last_nid = last_candidates[sid];
//cout << "STSC:oid = " << last_nid << endl; //cout << "STSC:oid = " << last_nid << endl;
for (size_t k = 0; k < totals.size(); ++k) { for (size_t k = 0; k < totals.size(); ++k) {
float diff = _scoreData->get(sid,nid).get(k) float diff = _scoreData->get(sid,nid).get(k)
- _scoreData->get(sid,last_nid).get(k); - _scoreData->get(sid,last_nid).get(k);
totals[k] += diff; totals[k] += diff;
//cout << "STSC:nid = " << _scoreData->get(sid,nid).get(k) << endl; //cout << "STSC:nid = " << _scoreData->get(sid,nid).get(k) << endl;
//cout << "STSC:oid = " << _scoreData->get(sid,last_nid).get(k) << endl; //cout << "STSC:oid = " << _scoreData->get(sid,last_nid).get(k) << endl;
//cout << "STSC:diff = " << diff << endl; //cout << "STSC:diff = " << diff << endl;
//cout << "STSC:totals = " << totals[k] << endl; //cout << "STSC:totals = " << totals[k] << endl;
} }
last_candidates[sid] = nid; last_candidates[sid] = nid;
}
scores.push_back(calculateScore(totals));
} }
scores.push_back(calculateScore(totals));
}
//regularisation. This can either be none, or the min or average as described in //regularisation. This can either be none, or the min or average as described in
//Cer, Jurafsky and Manning at WMT08 //Cer, Jurafsky and Manning at WMT08
if (_regularisationStrategy == REG_NONE || _regularisationWindow <= 0) { if (_regularisationStrategy == REG_NONE || _regularisationWindow <= 0) {
//no regularisation //no regularisation
return; return;
} }
//window size specifies the +/- in each direction //window size specifies the +/- in each direction
statscores_t raw_scores(scores);//copy scores statscores_t raw_scores(scores);//copy scores
for (size_t i = 0; i < scores.size(); ++i) { for (size_t i = 0; i < scores.size(); ++i) {
size_t start = 0; size_t start = 0;
if (i >= _regularisationWindow) { if (i >= _regularisationWindow) {
start = i - _regularisationWindow; start = i - _regularisationWindow;
}
size_t end = min(scores.size(), i + _regularisationWindow+1);
if (_regularisationStrategy == REG_AVERAGE) {
scores[i] = score_average(raw_scores,start,end);
} else {
scores[i] = score_min(raw_scores,start,end);
}
} }
size_t end = min(scores.size(), i + _regularisationWindow+1);
if (_regularisationStrategy == REG_AVERAGE) {
scores[i] = score_average(raw_scores,start,end);
} else {
scores[i] = score_min(raw_scores,start,end);
}
}
} }
@ -110,89 +113,90 @@ void StatisticsBasedScorer::score(const candidates_t& candidates, const diffs_t
/** The sentence level scores have already been calculated, just need to average them /** The sentence level scores have already been calculated, just need to average them
and include the differences. Allows scores which are floats **/ and include the differences. Allows scores which are floats **/
void SentenceLevelScorer::score(const candidates_t& candidates, const diffs_t& diffs, void SentenceLevelScorer::score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) { statscores_t& scores)
//cout << "*******SentenceLevelScorer::score" << endl; {
if (!_scoreData) { //cout << "*******SentenceLevelScorer::score" << endl;
throw runtime_error("Score data not loaded"); if (!_scoreData) {
throw runtime_error("Score data not loaded");
}
//calculate the score for the candidates
if (_scoreData->size() == 0) {
throw runtime_error("Score data is empty");
}
if (candidates.size() == 0) {
throw runtime_error("No candidates supplied");
}
int numCounts = _scoreData->get(0,candidates[0]).size();
vector<float> totals(numCounts);
for (size_t i = 0; i < candidates.size(); ++i) {
//cout << " i " << i << " candi " << candidates[i] ;
ScoreStats stats = _scoreData->get(i,candidates[i]);
if (stats.size() != totals.size()) {
stringstream msg;
msg << "Statistics for (" << "," << candidates[i] << ") have incorrect "
<< "number of fields. Found: " << stats.size() << " Expected: "
<< totals.size();
throw runtime_error(msg.str());
} }
//calculate the score for the candidates //Add up scores for all sentences, would normally be just one score
if (_scoreData->size() == 0) {
throw runtime_error("Score data is empty");
}
if (candidates.size() == 0) {
throw runtime_error("No candidates supplied");
}
int numCounts = _scoreData->get(0,candidates[0]).size();
vector<float> totals(numCounts);
for (size_t i = 0; i < candidates.size(); ++i) {
//cout << " i " << i << " candi " << candidates[i] ;
ScoreStats stats = _scoreData->get(i,candidates[i]);
if (stats.size() != totals.size()) {
stringstream msg;
msg << "Statistics for (" << "," << candidates[i] << ") have incorrect "
<< "number of fields. Found: " << stats.size() << " Expected: "
<< totals.size();
throw runtime_error(msg.str());
}
//Add up scores for all sentences, would normally be just one score
for (size_t k = 0; k < totals.size(); ++k) {
totals[k] += stats.get(k);
//cout << " stats " << stats.get(k) ;
}
//cout << endl;
}
//take average
for (size_t k = 0; k < totals.size(); ++k) { for (size_t k = 0; k < totals.size(); ++k) {
totals[k] += stats.get(k);
//cout << " stats " << stats.get(k) ;
}
//cout << endl;
}
//take average
for (size_t k = 0; k < totals.size(); ++k) {
//cout << "totals = " << totals[k] << endl; //cout << "totals = " << totals[k] << endl;
//cout << "cand = " << candidates.size() << endl; //cout << "cand = " << candidates.size() << endl;
totals[k] /= candidates.size(); totals[k] /= candidates.size();
//cout << "finaltotals = " << totals[k] << endl; //cout << "finaltotals = " << totals[k] << endl;
} }
scores.push_back(calculateScore(totals)); scores.push_back(calculateScore(totals));
candidates_t last_candidates(candidates); candidates_t last_candidates(candidates);
//apply each of the diffs, and get new scores //apply each of the diffs, and get new scores
for (size_t i = 0; i < diffs.size(); ++i) { for (size_t i = 0; i < diffs.size(); ++i) {
for (size_t j = 0; j < diffs[i].size(); ++j) { for (size_t j = 0; j < diffs[i].size(); ++j) {
size_t sid = diffs[i][j].first; size_t sid = diffs[i][j].first;
size_t nid = diffs[i][j].second; size_t nid = diffs[i][j].second;
//cout << "sid = " << sid << endl; //cout << "sid = " << sid << endl;
//cout << "nid = " << nid << endl; //cout << "nid = " << nid << endl;
size_t last_nid = last_candidates[sid]; size_t last_nid = last_candidates[sid];
for (size_t k = 0; k < totals.size(); ++k) { for (size_t k = 0; k < totals.size(); ++k) {
float diff = _scoreData->get(sid,nid).get(k) float diff = _scoreData->get(sid,nid).get(k)
- _scoreData->get(sid,last_nid).get(k); - _scoreData->get(sid,last_nid).get(k);
//cout << "diff = " << diff << endl; //cout << "diff = " << diff << endl;
totals[k] += diff/candidates.size(); totals[k] += diff/candidates.size();
//cout << "totals = " << totals[k] << endl; //cout << "totals = " << totals[k] << endl;
} }
last_candidates[sid] = nid; last_candidates[sid] = nid;
}
scores.push_back(calculateScore(totals));
} }
scores.push_back(calculateScore(totals));
}
//regularisation. This can either be none, or the min or average as described in //regularisation. This can either be none, or the min or average as described in
//Cer, Jurafsky and Manning at WMT08 //Cer, Jurafsky and Manning at WMT08
if (_regularisationStrategy == REG_NONE || _regularisationWindow <= 0) { if (_regularisationStrategy == REG_NONE || _regularisationWindow <= 0) {
//no regularisation //no regularisation
return; return;
} }
//window size specifies the +/- in each direction //window size specifies the +/- in each direction
statscores_t raw_scores(scores);//copy scores statscores_t raw_scores(scores);//copy scores
for (size_t i = 0; i < scores.size(); ++i) { for (size_t i = 0; i < scores.size(); ++i) {
size_t start = 0; size_t start = 0;
if (i >= _regularisationWindow) { if (i >= _regularisationWindow) {
start = i - _regularisationWindow; start = i - _regularisationWindow;
}
size_t end = min(scores.size(), i + _regularisationWindow+1);
if (_regularisationStrategy == REG_AVERAGE) {
scores[i] = score_average(raw_scores,start,end);
} else {
scores[i] = score_min(raw_scores,start,end);
}
} }
size_t end = min(scores.size(), i + _regularisationWindow+1);
if (_regularisationStrategy == REG_AVERAGE) {
scores[i] = score_average(raw_scores,start,end);
} else {
scores[i] = score_min(raw_scores,start,end);
}
}
} }

469
mert/Scorer.h
View File

@ -23,173 +23,180 @@ class ScoreStats;
/** /**
* Superclass of all scorers and dummy implementation. In order to add a new * Superclass of all scorers and dummy implementation. In order to add a new
* scorer it should be sufficient to override prepareStats(), setReferenceFiles() * scorer it should be sufficient to override prepareStats(), setReferenceFiles()
* and score() (or calculateScore()). * and score() (or calculateScore()).
**/ **/
class Scorer { class Scorer
private: {
string _name; private:
string _name;
public:
Scorer(const string& name, const string& config): _name(name), _scoreData(0), _preserveCase(true){
cerr << "Scorer config string: " << config << endl;
size_t start = 0;
while (start < config.size()) {
size_t end = config.find(",",start);
if (end == string::npos) {
end = config.size();
}
string nv = config.substr(start,end-start);
size_t split = nv.find(":");
if (split == string::npos) {
throw runtime_error("Missing colon when processing scorer config: " + config);
}
string name = nv.substr(0,split);
string value = nv.substr(split+1,nv.size()-split-1);
cerr << "name: " << name << " value: " << value << endl;
_config[name] = value;
start = end+1;
}
}; public:
virtual ~Scorer(){};
Scorer(const string& name, const string& config): _name(name), _scoreData(0), _preserveCase(true) {
/** cerr << "Scorer config string: " << config << endl;
* returns the number of statistics needed for the computation of the score size_t start = 0;
**/ while (start < config.size()) {
virtual size_t NumberOfScores() const { cerr << "Scorer: 0" << endl; return 0; }; size_t end = config.find(",",start);
if (end == string::npos) {
/** end = config.size();
* set the reference files. This must be called before prepareStats.
**/
virtual void setReferenceFiles(const vector<string>& referenceFiles) {
//do nothing
}
/**
* Process the given guessed text, corresponding to the given reference sindex
* and add the appropriate statistics to the entry.
**/
virtual void prepareStats(size_t sindex, const string& text, ScoreStats& entry)
{}
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry)
{
// cerr << sindex << endl;
this->prepareStats((size_t) atoi(sindex.c_str()), text, entry);
//cerr << text << std::endl;
}
/**
* Score using each of the candidate index, then go through the diffs
* applying each in turn, and calculating a new score each time.
**/
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) {
//dummy impl
if (!_scoreData) {
throw runtime_error("score data not loaded");
}
scores.push_back(0);
for (size_t i = 0; i < diffs.size(); ++i) {
scores.push_back(0);
}
}
/**
* Calculate the score of the sentences corresponding to the list of candidate
* indices. Each index indicates the 1-best choice from the n-best list.
**/
float score(const candidates_t& candidates) {
diffs_t diffs;
statscores_t scores;
score(candidates, diffs, scores);
return scores[0];
}
const string& getName() const {return _name;}
size_t getReferenceSize() {
if (_scoreData) {
return _scoreData->size();
}
return 0;
}
/**
* Set the score data, prior to scoring.
**/
virtual void setScoreData(ScoreData* data) {
_scoreData = data;
}
/**
* The scorer returns if it uses the reference alignment data
* for permutation distance scores
**/
virtual bool useAlignment() const {
//cout << "Scorer::useAlignment returning false " << endl;
return false;
};
//calculate the actual score
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;};
protected:
typedef map<string,int> encodings_t;
typedef map<string,int>::iterator encodings_it;
ScoreData* _scoreData;
encodings_t _encodings;
bool _preserveCase;
/**
* Value of config variable. If not provided, return default.
**/
string getConfig(const string& key, const string& def="") {
map<string,string>::iterator i = _config.find(key);
if (i == _config.end()) {
return def;
} else {
return i->second;
}
} }
string nv = config.substr(start,end-start);
size_t split = nv.find(":");
/** if (split == string::npos) {
* Tokenise line and encode. throw runtime_error("Missing colon when processing scorer config: " + config);
* Note: We assume that all tokens are separated by single spaces }
**/ string name = nv.substr(0,split);
void encode(const string& line, vector<int>& encoded) { string value = nv.substr(split+1,nv.size()-split-1);
//cerr << line << endl; cerr << "name: " << name << " value: " << value << endl;
istringstream in (line); _config[name] = value;
string token; start = end+1;
while (in >> token) {
if (!_preserveCase) {
for (string::iterator i = token.begin(); i != token.end(); ++i) {
*i = tolower(*i);
}
}
encodings_it encoding = _encodings.find(token);
int encoded_token;
if (encoding == _encodings.end()) {
encoded_token = (int)_encodings.size();
_encodings[token] = encoded_token;
//cerr << encoded_token << "(n) ";
} else {
encoded_token = encoding->second;
//cerr << encoded_token << " ";
}
encoded.push_back(encoded_token);
}
//cerr << endl;
} }
private: };
map<string,string> _config; virtual ~Scorer() {};
/**
* returns the number of statistics needed for the computation of the score
**/
virtual size_t NumberOfScores() const {
cerr << "Scorer: 0" << endl;
return 0;
};
/**
* set the reference files. This must be called before prepareStats.
**/
virtual void setReferenceFiles(const vector<string>& referenceFiles) {
//do nothing
}
/**
* Process the given guessed text, corresponding to the given reference sindex
* and add the appropriate statistics to the entry.
**/
virtual void prepareStats(size_t sindex, const string& text, ScoreStats& entry)
{}
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry) {
// cerr << sindex << endl;
this->prepareStats((size_t) atoi(sindex.c_str()), text, entry);
//cerr << text << std::endl;
}
/**
* Score using each of the candidate index, then go through the diffs
* applying each in turn, and calculating a new score each time.
**/
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores) {
//dummy impl
if (!_scoreData) {
throw runtime_error("score data not loaded");
}
scores.push_back(0);
for (size_t i = 0; i < diffs.size(); ++i) {
scores.push_back(0);
}
}
/**
* Calculate the score of the sentences corresponding to the list of candidate
* indices. Each index indicates the 1-best choice from the n-best list.
**/
float score(const candidates_t& candidates) {
diffs_t diffs;
statscores_t scores;
score(candidates, diffs, scores);
return scores[0];
}
const string& getName() const {
return _name;
}
size_t getReferenceSize() {
if (_scoreData) {
return _scoreData->size();
}
return 0;
}
/**
* Set the score data, prior to scoring.
**/
virtual void setScoreData(ScoreData* data) {
_scoreData = data;
}
/**
* The scorer returns if it uses the reference alignment data
* for permutation distance scores
**/
virtual bool useAlignment() const {
//cout << "Scorer::useAlignment returning false " << endl;
return false;
};
//calculate the actual score
virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
return 0;
};
protected:
typedef map<string,int> encodings_t;
typedef map<string,int>::iterator encodings_it;
ScoreData* _scoreData;
encodings_t _encodings;
bool _preserveCase;
/**
* Value of config variable. If not provided, return default.
**/
string getConfig(const string& key, const string& def="") {
map<string,string>::iterator i = _config.find(key);
if (i == _config.end()) {
return def;
} else {
return i->second;
}
}
/**
* Tokenise line and encode.
* Note: We assume that all tokens are separated by single spaces
**/
void encode(const string& line, vector<int>& encoded) {
//cerr << line << endl;
istringstream in (line);
string token;
while (in >> token) {
if (!_preserveCase) {
for (string::iterator i = token.begin(); i != token.end(); ++i) {
*i = tolower(*i);
}
}
encodings_it encoding = _encodings.find(token);
int encoded_token;
if (encoding == _encodings.end()) {
encoded_token = (int)_encodings.size();
_encodings[token] = encoded_token;
//cerr << encoded_token << "(n) ";
} else {
encoded_token = encoding->second;
//cerr << encoded_token << " ";
}
encoded.push_back(encoded_token);
}
//cerr << endl;
}
private:
map<string,string> _config;
}; };
@ -197,11 +204,12 @@ class Scorer {
/** /**
* Abstract base class for scorers that work by adding statistics across all * Abstract base class for scorers that work by adding statistics across all
* outout sentences, then apply some formula, e.g. bleu, per. **/ * outout sentences, then apply some formula, e.g. bleu, per. **/
class StatisticsBasedScorer : public Scorer { class StatisticsBasedScorer : public Scorer
{
public: public:
StatisticsBasedScorer(const string& name, const string& config): Scorer(name,config) { StatisticsBasedScorer(const string& name, const string& config): Scorer(name,config) {
//configure regularisation //configure regularisation
static string KEY_TYPE = "regtype"; static string KEY_TYPE = "regtype";
@ -212,105 +220,110 @@ class StatisticsBasedScorer : public Scorer {
static string TYPE_MINIMUM = "min"; static string TYPE_MINIMUM = "min";
static string TRUE = "true"; static string TRUE = "true";
static string FALSE = "false"; static string FALSE = "false";
string type = getConfig(KEY_TYPE,TYPE_NONE); string type = getConfig(KEY_TYPE,TYPE_NONE);
if (type == TYPE_NONE) { if (type == TYPE_NONE) {
_regularisationStrategy = REG_NONE; _regularisationStrategy = REG_NONE;
} else if (type == TYPE_AVERAGE) { } else if (type == TYPE_AVERAGE) {
_regularisationStrategy = REG_AVERAGE; _regularisationStrategy = REG_AVERAGE;
} else if (type == TYPE_MINIMUM) { } else if (type == TYPE_MINIMUM) {
_regularisationStrategy = REG_MINIMUM; _regularisationStrategy = REG_MINIMUM;
} else { } else {
throw runtime_error("Unknown scorer regularisation strategy: " + type); throw runtime_error("Unknown scorer regularisation strategy: " + type);
} }
cerr << "Using scorer regularisation strategy: " << type << endl; cerr << "Using scorer regularisation strategy: " << type << endl;
string window = getConfig(KEY_WINDOW,"0"); string window = getConfig(KEY_WINDOW,"0");
_regularisationWindow = atoi(window.c_str()); _regularisationWindow = atoi(window.c_str());
cerr << "Using scorer regularisation window: " << _regularisationWindow << endl; cerr << "Using scorer regularisation window: " << _regularisationWindow << endl;
string preservecase = getConfig(KEY_CASE,TRUE); string preservecase = getConfig(KEY_CASE,TRUE);
if (preservecase == TRUE) { if (preservecase == TRUE) {
_preserveCase = true; _preserveCase = true;
}else if (preservecase == FALSE) { } else if (preservecase == FALSE) {
_preserveCase = false; _preserveCase = false;
} }
cerr << "Using case preservation: " << _preserveCase << endl; cerr << "Using case preservation: " << _preserveCase << endl;
} }
~StatisticsBasedScorer(){}; ~StatisticsBasedScorer() {};
virtual void score(const candidates_t& candidates, const diffs_t& diffs, virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores); statscores_t& scores);
//calculate the actual score //calculate the actual score
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;}; virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
return 0;
};
protected: protected:
//regularisation //regularisation
ScorerRegularisationStrategy _regularisationStrategy; ScorerRegularisationStrategy _regularisationStrategy;
size_t _regularisationWindow; size_t _regularisationWindow;
}; };
/** /**
* Abstract base class for scorers that work by using sentence level * Abstract base class for scorers that work by using sentence level
* statistics eg. permutation distance metrics **/ * statistics eg. permutation distance metrics **/
class SentenceLevelScorer : public Scorer { class SentenceLevelScorer : public Scorer
{
public: public:
SentenceLevelScorer(const string& name, const string& config): Scorer(name,config) { SentenceLevelScorer(const string& name, const string& config): Scorer(name,config) {
//configure regularisation //configure regularisation
static string KEY_TYPE = "regtype"; static string KEY_TYPE = "regtype";
static string KEY_WINDOW = "regwin"; static string KEY_WINDOW = "regwin";
static string KEY_CASE = "case"; static string KEY_CASE = "case";
static string TYPE_NONE = "none"; static string TYPE_NONE = "none";
static string TYPE_AVERAGE = "average"; static string TYPE_AVERAGE = "average";
static string TYPE_MINIMUM = "min"; static string TYPE_MINIMUM = "min";
static string TRUE = "true"; static string TRUE = "true";
static string FALSE = "false"; static string FALSE = "false";
string type = getConfig(KEY_TYPE,TYPE_NONE);
if (type == TYPE_NONE) {
_regularisationStrategy = REG_NONE;
} else if (type == TYPE_AVERAGE) {
_regularisationStrategy = REG_AVERAGE;
} else if (type == TYPE_MINIMUM) {
_regularisationStrategy = REG_MINIMUM;
} else {
throw runtime_error("Unknown scorer regularisation strategy: " + type);
}
cerr << "Using scorer regularisation strategy: " << type << endl;
string window = getConfig(KEY_WINDOW,"0"); string type = getConfig(KEY_TYPE,TYPE_NONE);
_regularisationWindow = atoi(window.c_str()); if (type == TYPE_NONE) {
cerr << "Using scorer regularisation window: " << _regularisationWindow << endl; _regularisationStrategy = REG_NONE;
} else if (type == TYPE_AVERAGE) {
string preservecase = getConfig(KEY_CASE,TRUE); _regularisationStrategy = REG_AVERAGE;
if (preservecase == TRUE) { } else if (type == TYPE_MINIMUM) {
_preserveCase = true; _regularisationStrategy = REG_MINIMUM;
}else if (preservecase == FALSE) { } else {
_preserveCase = false; throw runtime_error("Unknown scorer regularisation strategy: " + type);
} }
cerr << "Using case preservation: " << _preserveCase << endl; cerr << "Using scorer regularisation strategy: " << type << endl;
string window = getConfig(KEY_WINDOW,"0");
_regularisationWindow = atoi(window.c_str());
cerr << "Using scorer regularisation window: " << _regularisationWindow << endl;
string preservecase = getConfig(KEY_CASE,TRUE);
if (preservecase == TRUE) {
_preserveCase = true;
} else if (preservecase == FALSE) {
_preserveCase = false;
}
cerr << "Using case preservation: " << _preserveCase << endl;
} }
~SentenceLevelScorer(){}; ~SentenceLevelScorer() {};
virtual void score(const candidates_t& candidates, const diffs_t& diffs, virtual void score(const candidates_t& candidates, const diffs_t& diffs,
statscores_t& scores); statscores_t& scores);
//calculate the actual score //calculate the actual score
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;}; virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
return 0;
};
protected: protected:
//regularisation //regularisation
ScorerRegularisationStrategy _regularisationStrategy; ScorerRegularisationStrategy _regularisationStrategy;
size_t _regularisationWindow; size_t _regularisationWindow;
}; };

71
mert/ScorerFactory.h
View File

@ -19,43 +19,44 @@
using namespace std; using namespace std;
class ScorerFactory { class ScorerFactory
{
public: public:
vector<string> getTypes() { vector<string> getTypes() {
vector<string> types; vector<string> types;
types.push_back(string("BLEU1")); types.push_back(string("BLEU1"));
types.push_back(string("BLEU")); types.push_back(string("BLEU"));
types.push_back(string("PER")); types.push_back(string("PER"));
types.push_back(string("HAMMING")); types.push_back(string("HAMMING"));
types.push_back(string("KENDALL")); types.push_back(string("KENDALL"));
return types; return types;
}
Scorer* getScorer(const string& type, const string& config = "") {
size_t scorerTypes = type.find(",");
if(scorerTypes == string::npos) {
if (type == "BLEU1") {
string conf;
if (config.length() > 0) {
conf = config + ",ngramlen:1";
} else {
conf = config + "ngramlen:1";
} }
return (BleuScorer*) new BleuScorer(conf);
Scorer* getScorer(const string& type, const string& config = "") { } else if (type == "BLEU") {
size_t scorerTypes = type.find(","); return (BleuScorer*) new BleuScorer(config);
if(scorerTypes == string::npos) { } else if (type == "PER") {
if (type == "BLEU1") { return (PerScorer*) new PerScorer(config);
string conf; } else if ((type == "HAMMING") || (type == "KENDALL")) {
if (config.length() > 0) { return (PermutationScorer*) new PermutationScorer(type, config);
conf = config + ",ngramlen:1"; } else {
} else { throw runtime_error("Unknown scorer type: " + type);
conf = config + "ngramlen:1"; }
} } else {
return (BleuScorer*) new BleuScorer(conf); return (InterpolatedScorer*) new InterpolatedScorer(type, config);
} else if (type == "BLEU") { }
return (BleuScorer*) new BleuScorer(config); }
} else if (type == "PER") {
return (PerScorer*) new PerScorer(config);
} else if ((type == "HAMMING") || (type == "KENDALL")) {
return (PermutationScorer*) new PermutationScorer(type, config);
} else {
throw runtime_error("Unknown scorer type: " + type);
}
} else {
return (InterpolatedScorer*) new InterpolatedScorer(type, config);
}
}
}; };
#endif //__SCORER_FACTORY_H #endif //__SCORER_FACTORY_H

6
mert/Timer.cpp
View File

@ -12,8 +12,8 @@
*/ */
double Timer::elapsed_time() double Timer::elapsed_time()
{ {
time_t now; time_t now;
time(&now); time(&now);
return difftime(now, start_time); return difftime(now, start_time);
} }
@ -36,7 +36,7 @@ double Timer::get_elapsed_time()
void Timer::start(const char* msg) void Timer::start(const char* msg)
{ {
// Print an optional message, something like "Starting timer t"; // Print an optional message, something like "Starting timer t";
if (msg) TRACE_ERR( msg << std::endl); if (msg) TRACE_ERR( msg << std::endl);
// Return immediately if the timer is already running // Return immediately if the timer is already running
if (running) return; if (running) return;

8
mert/Timer.h
View File

@ -8,16 +8,16 @@
class Timer class Timer
{ {
friend std::ostream& operator<<(std::ostream& os, Timer& t); friend std::ostream& operator<<(std::ostream& os, Timer& t);
private: private:
bool running; bool running;
time_t start_time; time_t start_time;
//TODO in seconds? //TODO in seconds?
double elapsed_time(); double elapsed_time();
public: public:
/*** /***
* 'running' is initially false. A timer needs to be explicitly started * 'running' is initially false. A timer needs to be explicitly started
* using 'start' or 'restart' * using 'start' or 'restart'

85
mert/Util.cpp
View File

@ -1,7 +1,7 @@
/* /*
* Util.cpp * Util.cpp
* met - Minimum Error Training * met - Minimum Error Training
* *
* Created by Nicola Bertoldi on 13/05/08. * Created by Nicola Bertoldi on 13/05/08.
* *
*/ */
@ -18,47 +18,47 @@ Timer g_timer;
int verbose=0; int verbose=0;
int verboselevel(){ int verboselevel()
{
return verbose; return verbose;
} }
int setverboselevel(int v){ int setverboselevel(int v)
{
verbose=v; verbose=v;
return verbose; return verbose;
} }
int getNextPound(std::string &theString, std::string &substring, const std::string delimiter) int getNextPound(std::string &theString, std::string &substring, const std::string delimiter)
{ {
unsigned int pos = 0; unsigned int pos = 0;
//skip all occurrences of delimiter //skip all occurrences of delimiter
while ( pos == 0 ) while ( pos == 0 ) {
{ if ((pos = theString.find(delimiter)) != std::string::npos) {
if ((pos = theString.find(delimiter)) != std::string::npos){ substring.assign(theString, 0, pos);
substring.assign(theString, 0, pos); theString.erase(0,pos + delimiter.size());
theString.erase(0,pos + delimiter.size()); } else {
} substring.assign(theString);
else{ theString.assign("");
substring.assign(theString); }
theString.assign(""); }
} return (pos);
}
return (pos);
}; };
inputfilestream::inputfilestream(const std::string &filePath) inputfilestream::inputfilestream(const std::string &filePath)
: std::istream(0), : std::istream(0),
m_streambuf(0) m_streambuf(0)
{ {
//check if file is readable //check if file is readable
std::filebuf* fb = new std::filebuf(); std::filebuf* fb = new std::filebuf();
_good=(fb->open(filePath.c_str(), std::ios::in)!=NULL); _good=(fb->open(filePath.c_str(), std::ios::in)!=NULL);
if (filePath.size() > 3 && if (filePath.size() > 3 &&
filePath.substr(filePath.size() - 3, 3) == ".gz") filePath.substr(filePath.size() - 3, 3) == ".gz") {
{ fb->close();
fb->close(); delete fb; delete fb;
m_streambuf = new gzfilebuf(filePath.c_str()); m_streambuf = new gzfilebuf(filePath.c_str());
} else { } else {
m_streambuf = fb; m_streambuf = fb;
} }
@ -67,7 +67,8 @@ m_streambuf(0)
inputfilestream::~inputfilestream() inputfilestream::~inputfilestream()
{ {
delete m_streambuf; m_streambuf = 0; delete m_streambuf;
m_streambuf = 0;
} }
void inputfilestream::close() void inputfilestream::close()
@ -75,16 +76,15 @@ void inputfilestream::close()
} }
outputfilestream::outputfilestream(const std::string &filePath) outputfilestream::outputfilestream(const std::string &filePath)
: std::ostream(0), : std::ostream(0),
m_streambuf(0) m_streambuf(0)
{ {
//check if file is readable //check if file is readable
std::filebuf* fb = new std::filebuf(); std::filebuf* fb = new std::filebuf();
_good=(fb->open(filePath.c_str(), std::ios::out)!=NULL); _good=(fb->open(filePath.c_str(), std::ios::out)!=NULL);
if (filePath.size() > 3 && filePath.substr(filePath.size() - 3, 3) == ".gz") if (filePath.size() > 3 && filePath.substr(filePath.size() - 3, 3) == ".gz") {
{ throw runtime_error("Output to a zipped file not supported!");
throw runtime_error("Output to a zipped file not supported!");
} else { } else {
m_streambuf = fb; m_streambuf = fb;
} }
@ -93,7 +93,8 @@ m_streambuf(0)
outputfilestream::~outputfilestream() outputfilestream::~outputfilestream()
{ {
delete m_streambuf; m_streambuf = 0; delete m_streambuf;
m_streambuf = 0;
} }
void outputfilestream::close() void outputfilestream::close()
@ -103,10 +104,14 @@ void outputfilestream::close()
int swapbytes(char *p, int sz, int n) int swapbytes(char *p, int sz, int n)
{ {
char c, *l, *h; char c, *l, *h;
if((n<1) || (sz<2)) return 0; if((n<1) || (sz<2)) return 0;
for(; n--; p+=sz) for(h=(l=p)+sz; --h>l; l++) { c=*h; *h=*l; *l=c; } for(; n--; p+=sz) for(h=(l=p)+sz; --h>l; l++) {
return 0; c=*h;
*h=*l;
*l=c;
}
return 0;
}; };
@ -116,12 +121,12 @@ void ResetUserTime()
}; };
void PrintUserTime(const std::string &message) void PrintUserTime(const std::string &message)
{ {
g_timer.check(message.c_str()); g_timer.check(message.c_str());
} }
double GetUserTime() double GetUserTime()
{ {
return g_timer.get_elapsed_time(); return g_timer.get_elapsed_time();
} }

58
mert/Util.h
View File

@ -51,45 +51,49 @@ int getNextPound(std::string &theString, std::string &substring, const std::stri
template<typename T> template<typename T>
inline T Scan(const std::string &input) inline T Scan(const std::string &input)
{ {
std::stringstream stream(input); std::stringstream stream(input);
T ret; T ret;
stream >> ret; stream >> ret;
return ret; return ret;
}; };
class inputfilestream : public std::istream class inputfilestream : public std::istream
{ {
protected: protected:
std::streambuf *m_streambuf; std::streambuf *m_streambuf;
bool _good; bool _good;
public: public:
inputfilestream(const std::string &filePath); inputfilestream(const std::string &filePath);
~inputfilestream(); ~inputfilestream();
bool good(){return _good;} bool good() {
void close(); return _good;
}
void close();
}; };
class outputfilestream : public std::ostream class outputfilestream : public std::ostream
{ {
protected: protected:
std::streambuf *m_streambuf; std::streambuf *m_streambuf;
bool _good; bool _good;
public: public:
outputfilestream(const std::string &filePath); outputfilestream(const std::string &filePath);
~outputfilestream(); ~outputfilestream();
bool good(){return _good;} bool good() {
void close(); return _good;
}
void close();
}; };
template<typename T> template<typename T>
inline std::string stringify(T x) inline std::string stringify(T x)
{ {
std::ostringstream o; std::ostringstream o;
if (!(o << x)) if (!(o << x))
throw std::runtime_error("stringify(template<typename T>)"); throw std::runtime_error("stringify(template<typename T>)");
return o.str(); return o.str();
} }
// Utilities to measure decoding time // Utilities to measure decoding time
@ -99,11 +103,11 @@ double GetUserTime();
inline std::string trimStr(const std::string& Src, const std::string& c = " \r\n") inline std::string trimStr(const std::string& Src, const std::string& c = " \r\n")
{ {
unsigned int p2 = Src.find_last_not_of(c); unsigned int p2 = Src.find_last_not_of(c);
if (p2 == std::string::npos) return std::string(); if (p2 == std::string::npos) return std::string();
unsigned int p1 = Src.find_first_not_of(c); unsigned int p1 = Src.find_first_not_of(c);
if (p1 == std::string::npos) p1 = 0; if (p1 == std::string::npos) p1 = 0;
return Src.substr(p1, (p2-p1)+1); return Src.substr(p1, (p2-p1)+1);
} }

305
mert/extractor.cpp
View File

@ -18,7 +18,8 @@
using namespace std; using namespace std;
void usage() { void usage()
{
cerr<<"usage: extractor [options])"<<endl; cerr<<"usage: extractor [options])"<<endl;
cerr<<"[--sctype|-s] the scorer type (default BLEU), possibly comma separated list of interpolated types"<<endl; cerr<<"[--sctype|-s] the scorer type (default BLEU), possibly comma separated list of interpolated types"<<endl;
cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl; cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl;
@ -28,7 +29,7 @@ void usage() {
cerr<<"[--nbest|-n] the nbest file"<<endl; cerr<<"[--nbest|-n] the nbest file"<<endl;
cerr<<"[--scfile|-S] the scorer data output file"<<endl; cerr<<"[--scfile|-S] the scorer data output file"<<endl;
cerr<<"[--ffile|-F] the feature data output file"<<endl; cerr<<"[--ffile|-F] the feature data output file"<<endl;
cerr<<"[--prev-ffile|-E] comma separated list of previous feature data" <<endl; cerr<<"[--prev-ffile|-E] comma separated list of previous feature data" <<endl;
cerr<<"[--prev-scfile|-R] comma separated list of previous scorer data"<<endl; cerr<<"[--prev-scfile|-R] comma separated list of previous scorer data"<<endl;
cerr<<"[-v] verbose level"<<endl; cerr<<"[-v] verbose level"<<endl;
cerr<<"[--help|-h] print this message and exit"<<endl; cerr<<"[--help|-h] print this message and exit"<<endl;
@ -36,185 +37,185 @@ cerr<<"[--prev-ffile|-E] comma separated list of previous feature data" <<endl;
} }
static struct option long_options[] = static struct option long_options[] = {
{ {"sctype",required_argument,0,'s'},
{"sctype",required_argument,0,'s'}, {"scconfig",required_argument,0,'c'},
{"scconfig",required_argument,0,'c'}, {"reference",required_argument,0,'r'},
{"reference",required_argument,0,'r'}, {"binary",no_argument,0,'b'},
{"binary",no_argument,0,'b'}, {"nbest",required_argument,0,'n'},
{"nbest",required_argument,0,'n'}, {"scfile",required_argument,0,'S'},
{"scfile",required_argument,0,'S'}, {"ffile",required_argument,0,'F'},
{"ffile",required_argument,0,'F'}, {"prev-scfile",required_argument,0,'R'},
{"prev-scfile",required_argument,0,'R'}, {"prev-ffile",required_argument,0,'E'},
{"prev-ffile",required_argument,0,'E'}, {"verbose",required_argument,0,'v'},
{"verbose",required_argument,0,'v'}, {"help",no_argument,0,'h'},
{"help",no_argument,0,'h'}, {0, 0, 0, 0}
{0, 0, 0, 0} };
};
int option_index; int option_index;
int main(int argc, char** argv) { int main(int argc, char** argv)
{
ResetUserTime();
ResetUserTime();
/*
Timer timer; /*
timer.start("Starting..."); Timer timer;
*/ timer.start("Starting...");
*/
//defaults
string scorerType("BLEU"); //defaults
string scorerConfig(""); string scorerType("BLEU");
string referenceFile(""); string scorerConfig("");
string nbestFile(""); string referenceFile("");
string scoreDataFile("statscore.data"); string nbestFile("");
string featureDataFile("features.data"); string scoreDataFile("statscore.data");
string prevScoreDataFile(""); string featureDataFile("features.data");
string prevFeatureDataFile(""); string prevScoreDataFile("");
bool binmode = false; string prevFeatureDataFile("");
int verbosity = 0; bool binmode = false;
int c; int verbosity = 0;
while ((c=getopt_long (argc,argv, "s:w:r:a:n:S:F:R:E:v:hb", long_options, &option_index)) != -1) { int c;
switch(c) { while ((c=getopt_long (argc,argv, "s:w:r:a:n:S:F:R:E:v:hb", long_options, &option_index)) != -1) {
case 's': switch(c) {
scorerType = string(optarg); case 's':
break; scorerType = string(optarg);
case 'c': break;
scorerConfig = string(optarg); case 'c':
break; scorerConfig = string(optarg);
case 'r': break;
referenceFile = string(optarg); case 'r':
break; referenceFile = string(optarg);
case 'b': break;
binmode = true; case 'b':
break; binmode = true;
case 'n': break;
nbestFile = string(optarg); case 'n':
break; nbestFile = string(optarg);
case 'S': break;
scoreDataFile = string(optarg); case 'S':
break; scoreDataFile = string(optarg);
case 'F': break;
featureDataFile = string(optarg); case 'F':
break; featureDataFile = string(optarg);
case 'E': break;
prevFeatureDataFile = string(optarg); case 'E':
break; prevFeatureDataFile = string(optarg);
case 'R': break;
prevScoreDataFile = string(optarg); case 'R':
break; prevScoreDataFile = string(optarg);
case 'v': break;
verbosity = atoi(optarg); case 'v':
break; verbosity = atoi(optarg);
default: break;
usage(); default:
} usage();
} }
try { }
try {
//check whether score statistics file is specified //check whether score statistics file is specified
if (scoreDataFile.length() == 0){ if (scoreDataFile.length() == 0) {
throw runtime_error("Error: output score statistics file is not specified"); throw runtime_error("Error: output score statistics file is not specified");
} }
//check wheter feature file is specified //check wheter feature file is specified
if (featureDataFile.length() == 0){ if (featureDataFile.length() == 0) {
throw runtime_error("Error: output feature file is not specified"); throw runtime_error("Error: output feature file is not specified");
} }
//check whether reference file is specified when nbest is specified //check whether reference file is specified when nbest is specified
if ((nbestFile.length() > 0 && referenceFile.length() == 0)){ if ((nbestFile.length() > 0 && referenceFile.length() == 0)) {
throw runtime_error("Error: reference file is not specified; you can not score the nbest"); throw runtime_error("Error: reference file is not specified; you can not score the nbest");
} }
vector<string> nbestFiles; vector<string> nbestFiles;
if (nbestFile.length() > 0){ if (nbestFile.length() > 0) {
std::string substring; std::string substring;
while (!nbestFile.empty()){ while (!nbestFile.empty()) {
getNextPound(nbestFile, substring, ","); getNextPound(nbestFile, substring, ",");
nbestFiles.push_back(substring); nbestFiles.push_back(substring);
} }
} }
vector<string> referenceFiles; vector<string> referenceFiles;
if (referenceFile.length() > 0){ if (referenceFile.length() > 0) {
std::string substring; std::string substring;
while (!referenceFile.empty()){ while (!referenceFile.empty()) {
getNextPound(referenceFile, substring, ","); getNextPound(referenceFile, substring, ",");
referenceFiles.push_back(substring); referenceFiles.push_back(substring);
} }
} }
vector<string> prevScoreDataFiles; vector<string> prevScoreDataFiles;
if (prevScoreDataFile.length() > 0){ if (prevScoreDataFile.length() > 0) {
std::string substring; std::string substring;
while (!prevScoreDataFile.empty()){ while (!prevScoreDataFile.empty()) {
getNextPound(prevScoreDataFile, substring, ","); getNextPound(prevScoreDataFile, substring, ",");
prevScoreDataFiles.push_back(substring); prevScoreDataFiles.push_back(substring);
} }
} }
vector<string> prevFeatureDataFiles; vector<string> prevFeatureDataFiles;
if (prevFeatureDataFile.length() > 0){ if (prevFeatureDataFile.length() > 0) {
std::string substring; std::string substring;
while (!prevFeatureDataFile.empty()){ while (!prevFeatureDataFile.empty()) {
getNextPound(prevFeatureDataFile, substring, ","); getNextPound(prevFeatureDataFile, substring, ",");
prevFeatureDataFiles.push_back(substring); prevFeatureDataFiles.push_back(substring);
} }
} }
if (prevScoreDataFiles.size() != prevFeatureDataFiles.size()){ if (prevScoreDataFiles.size() != prevFeatureDataFiles.size()) {
throw runtime_error("Error: there is a different number of previous score and feature files"); throw runtime_error("Error: there is a different number of previous score and feature files");
} }
if (binmode) cerr << "Binary write mode is selected" << endl;
else cerr << "Binary write mode is NOT selected" << endl;
//TODO is comma separated list? split and create a scorer with multiple parts
TRACE_ERR("Scorer type: " << scorerType << endl);
ScorerFactory sfactory;
Scorer* scorer = sfactory.getScorer(scorerType,scorerConfig);
//load references
if (referenceFiles.size() > 0)
scorer->setReferenceFiles(referenceFiles);
PrintUserTime("References loaded"); if (binmode) cerr << "Binary write mode is selected" << endl;
else cerr << "Binary write mode is NOT selected" << endl;
Data data(*scorer);
//load old data
for (size_t i=0;i < prevScoreDataFiles.size(); i++){
data.load(prevFeatureDataFiles.at(i), prevScoreDataFiles.at(i));
}
PrintUserTime("Previous data loaded");
//computing score statistics of each nbest file
for (size_t i=0;i < nbestFiles.size(); i++){
data.loadnbest(nbestFiles.at(i));
}
PrintUserTime("Nbest entries loaded and scored"); //TODO is comma separated list? split and create a scorer with multiple parts
TRACE_ERR("Scorer type: " << scorerType << endl);
if (binmode) ScorerFactory sfactory;
cerr << "Binary write mode is selected" << endl; Scorer* scorer = sfactory.getScorer(scorerType,scorerConfig);
else
cerr << "Binary write mode is NOT selected" << endl; //load references
if (referenceFiles.size() > 0)
data.save(featureDataFile, scoreDataFile, binmode); scorer->setReferenceFiles(referenceFiles);
PrintUserTime("Stopping...");
/* PrintUserTime("References loaded");
timer.stop("Stopping...");
*/ Data data(*scorer);
return EXIT_SUCCESS; //load old data
} catch (const exception& e) { for (size_t i=0; i < prevScoreDataFiles.size(); i++) {
cerr << "Exception: " << e.what() << endl; data.load(prevFeatureDataFiles.at(i), prevScoreDataFiles.at(i));
return EXIT_FAILURE;
} }
PrintUserTime("Previous data loaded");
//computing score statistics of each nbest file
for (size_t i=0; i < nbestFiles.size(); i++) {
data.loadnbest(nbestFiles.at(i));
}
PrintUserTime("Nbest entries loaded and scored");
if (binmode)
cerr << "Binary write mode is selected" << endl;
else
cerr << "Binary write mode is NOT selected" << endl;
data.save(featureDataFile, scoreDataFile, binmode);
PrintUserTime("Stopping...");
/*
timer.stop("Stopping...");
*/
return EXIT_SUCCESS;
} catch (const exception& e) {
cerr << "Exception: " << e.what() << endl;
return EXIT_FAILURE;
}
} }

74
mert/gzfilebuf.h
View File

@ -4,66 +4,70 @@
#include <streambuf> #include <streambuf>
#include <zlib.h> #include <zlib.h>
class gzfilebuf : public std::streambuf { class gzfilebuf : public std::streambuf
{
public: public:
gzfilebuf(const char *filename) gzfilebuf(const char *filename) {
{ _gzf = gzopen(filename, "rb"); _gzf = gzopen(filename, "rb");
setg (_buff+sizeof(int), // beginning of putback area setg (_buff+sizeof(int), // beginning of putback area
_buff+sizeof(int), // read position _buff+sizeof(int), // read position
_buff+sizeof(int)); // end position _buff+sizeof(int)); // end position
} }
~gzfilebuf() { gzclose(_gzf); } ~gzfilebuf() {
gzclose(_gzf);
}
protected: protected:
virtual int_type overflow (int_type c) { virtual int_type overflow (int_type c) {
throw; throw;
} }
// write multiple characters // write multiple characters
virtual virtual
std::streamsize xsputn (const char* s, std::streamsize xsputn (const char* s,
std::streamsize num) { std::streamsize num) {
throw; throw;
} }
virtual std::streampos seekpos ( std::streampos sp, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ){ throw; virtual std::streampos seekpos ( std::streampos sp, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ) {
throw;
} }
//read one character //read one character
virtual int_type underflow () { virtual int_type underflow () {
// is read position before end of _buff? // is read position before end of _buff?
if (gptr() < egptr()) { if (gptr() < egptr()) {
return traits_type::to_int_type(*gptr()); return traits_type::to_int_type(*gptr());
} }
/* process size of putback area /* process size of putback area
* - use number of characters read * - use number of characters read
* - but at most four * - but at most four
*/ */
unsigned int numPutback = gptr() - eback(); unsigned int numPutback = gptr() - eback();
if (numPutback > sizeof(int)) { if (numPutback > sizeof(int)) {
numPutback = sizeof(int); numPutback = sizeof(int);
} }
/* copy up to four characters previously read into /* copy up to four characters previously read into
* the putback _buff (area of first four characters) * the putback _buff (area of first four characters)
*/ */
std::memmove (_buff+(sizeof(int)-numPutback), gptr()-numPutback, std::memmove (_buff+(sizeof(int)-numPutback), gptr()-numPutback,
numPutback); numPutback);
// read new characters // read new characters
int num = gzread(_gzf, _buff+sizeof(int), _buffsize-sizeof(int)); int num = gzread(_gzf, _buff+sizeof(int), _buffsize-sizeof(int));
if (num <= 0) { if (num <= 0) {
// ERROR or EOF // ERROR or EOF
return EOF; return EOF;
} }
// reset _buff pointers // reset _buff pointers
setg (_buff+(sizeof(int)-numPutback), // beginning of putback area setg (_buff+(sizeof(int)-numPutback), // beginning of putback area
_buff+sizeof(int), // read position _buff+sizeof(int), // read position
_buff+sizeof(int)+num); // end of buffer _buff+sizeof(int)+num); // end of buffer
// return next character // return next character
return traits_type::to_int_type(*gptr()); return traits_type::to_int_type(*gptr());
} }
std::streamsize xsgetn (char* s, std::streamsize xsgetn (char* s,

172
mert/mert.cpp
View File

@ -28,7 +28,8 @@ float min_interval = 1e-3;
using namespace std; using namespace std;
void usage(void) { void usage(void)
{
cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl; cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl;
cerr<<"[-n retry ntimes (default 1)]"<<endl; cerr<<"[-n retry ntimes (default 1)]"<<endl;
cerr<<"[-o\tthe indexes to optimize(default all)]"<<endl; cerr<<"[-o\tthe indexes to optimize(default all)]"<<endl;
@ -44,34 +45,34 @@ void usage(void) {
exit(1); exit(1);
} }
static struct option long_options[] = static struct option long_options[] = {
{ {"pdim", 1, 0, 'd'},
{"pdim", 1, 0, 'd'}, {"ntry",1,0,'n'},
{"ntry",1,0,'n'}, {"rseed",required_argument,0,'r'},
{"rseed",required_argument,0,'r'}, {"optimize",1,0,'o'},
{"optimize",1,0,'o'}, {"type",1,0,'t'},
{"type",1,0,'t'}, {"sctype",1,0,'s'},
{"sctype",1,0,'s'}, {"scconfig",required_argument,0,'c'},
{"scconfig",required_argument,0,'c'}, {"scfile",1,0,'S'},
{"scfile",1,0,'S'}, {"ffile",1,0,'F'},
{"ffile",1,0,'F'}, {"ifile",1,0,'i'},
{"ifile",1,0,'i'}, {"verbose",1,0,'v'},
{"verbose",1,0,'v'}, {"help",no_argument,0,'h'},
{"help",no_argument,0,'h'}, {0, 0, 0, 0}
{0, 0, 0, 0} };
};
int option_index; int option_index;
int main (int argc, char **argv) { int main (int argc, char **argv)
{
ResetUserTime();
ResetUserTime();
/*
Timer timer; /*
timer.start("Starting..."); Timer timer;
*/ timer.start("Starting...");
*/
int c,pdim,i; int c,pdim,i;
pdim=-1; pdim=-1;
int ntry=1; int ntry=1;
@ -132,23 +133,23 @@ int main (int argc, char **argv) {
usage(); usage();
if (hasSeed) { if (hasSeed) {
cerr << "Seeding random numbers with " << seed << endl; cerr << "Seeding random numbers with " << seed << endl;
srandom(seed); srandom(seed);
} else { } else {
cerr << "Seeding random numbers with system clock " << endl; cerr << "Seeding random numbers with system clock " << endl;
srandom(time(NULL)); srandom(time(NULL));
} }
ifstream opt(initfile.c_str()); ifstream opt(initfile.c_str());
if(opt.fail()){ if(opt.fail()) {
cerr<<"could not open initfile: " << initfile << endl; cerr<<"could not open initfile: " << initfile << endl;
exit(3); exit(3);
} }
start.resize(pdim);//to do:read from file start.resize(pdim);//to do:read from file
int j; int j;
for( j=0;j<pdim&&!opt.fail();j++) for( j=0; j<pdim&&!opt.fail(); j++)
opt>>start[j]; opt>>start[j];
if(j<pdim){ if(j<pdim) {
cerr<<"error could not initialize start point with " << initfile << endl; cerr<<"error could not initialize start point with " << initfile << endl;
exit(3); exit(3);
} }
@ -156,24 +157,24 @@ int main (int argc, char **argv) {
opt.close(); opt.close();
vector<string> ScoreDataFiles; vector<string> ScoreDataFiles;
if (scorerfile.length() > 0){ if (scorerfile.length() > 0) {
std::string substring; std::string substring;
while (!scorerfile.empty()){ while (!scorerfile.empty()) {
getNextPound(scorerfile, substring, ","); getNextPound(scorerfile, substring, ",");
ScoreDataFiles.push_back(substring); ScoreDataFiles.push_back(substring);
} }
} }
vector<string> FeatureDataFiles; vector<string> FeatureDataFiles;
if (featurefile.length() > 0){ if (featurefile.length() > 0) {
std::string substring; std::string substring;
while (!featurefile.empty()){ while (!featurefile.empty()) {
getNextPound(featurefile, substring, ","); getNextPound(featurefile, substring, ",");
FeatureDataFiles.push_back(substring); FeatureDataFiles.push_back(substring);
} }
} }
if (ScoreDataFiles.size() != FeatureDataFiles.size()){ if (ScoreDataFiles.size() != FeatureDataFiles.size()) {
throw runtime_error("Error: there is a different number of previous score and feature files"); throw runtime_error("Error: there is a different number of previous score and feature files");
} }
@ -183,32 +184,37 @@ int main (int argc, char **argv) {
//load data //load data
Data D(*TheScorer); Data D(*TheScorer);
for (size_t i=0;i < ScoreDataFiles.size(); i++){ for (size_t i=0; i < ScoreDataFiles.size(); i++) {
cerr<<"Loading Data from: "<< ScoreDataFiles.at(i) << " and " << FeatureDataFiles.at(i) << endl; cerr<<"Loading Data from: "<< ScoreDataFiles.at(i) << " and " << FeatureDataFiles.at(i) << endl;
D.load(FeatureDataFiles.at(i), ScoreDataFiles.at(i)); D.load(FeatureDataFiles.at(i), ScoreDataFiles.at(i));
} }
PrintUserTime("Data loaded"); PrintUserTime("Data loaded");
if (tooptimizestr.length() > 0){ if (tooptimizestr.length() > 0) {
cerr << "Weights to optimize: " << tooptimizestr << endl; cerr << "Weights to optimize: " << tooptimizestr << endl;
//parse string to get weights to optimize //parse string to get weights to optimize
//and set them as active //and set them as active
std::string substring; std::string substring;
int index; int index;
while (!tooptimizestr.empty()){ while (!tooptimizestr.empty()) {
getNextPound(tooptimizestr, substring, ","); getNextPound(tooptimizestr, substring, ",");
index = D.getFeatureIndex(substring); index = D.getFeatureIndex(substring);
cerr << "FeatNameIndex:" << index << " to insert" << endl; cerr << "FeatNameIndex:" << index << " to insert" << endl;
//index = strtol(substring.c_str(), NULL, 10); //index = strtol(substring.c_str(), NULL, 10);
if (index >= 0 && index < pdim){ tooptimize.push_back(index); } if (index >= 0 && index < pdim) {
else{ cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl; } tooptimize.push_back(index);
} else {
cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl;
}
} }
}else{ } else {
//set all weights as active //set all weights as active
tooptimize.resize(pdim);//We'll optimize on everything tooptimize.resize(pdim);//We'll optimize on everything
for(int i=0;i<pdim;i++){ tooptimize[i]=1; } for(int i=0; i<pdim; i++) {
tooptimize[i]=1;
}
} }
Optimizer *O=OptimizerFactory::BuildOptimizer(pdim,tooptimize,start,type); Optimizer *O=OptimizerFactory::BuildOptimizer(pdim,tooptimize,start,type);
@ -216,51 +222,51 @@ int main (int argc, char **argv) {
O->SetFData(D.getFeatureData()); O->SetFData(D.getFeatureData());
Point P(start);//Generate from the full feature set. Warning: must be done after Optimizer initialization Point P(start);//Generate from the full feature set. Warning: must be done after Optimizer initialization
statscore_t best=O->Run(P); statscore_t best=O->Run(P);
Point bestP=P; Point bestP=P;
statscore_t mean=best; statscore_t mean=best;
statscore_t var=best*best; statscore_t var=best*best;
stringstream oss; stringstream oss;
oss << "Try number 1"; oss << "Try number 1";
PrintUserTime(oss.str()); PrintUserTime(oss.str());
vector<parameter_t> min(Point::getdim()); vector<parameter_t> min(Point::getdim());
vector<parameter_t> max(Point::getdim()); vector<parameter_t> max(Point::getdim());
for(unsigned int d=0;d<Point::getdim();d++){ for(unsigned int d=0; d<Point::getdim(); d++) {
min[d]=0.0; min[d]=0.0;
max[d]=1.0; max[d]=1.0;
} }
//note: those mins and max are the bound for the starting points of the algorithm, not strict bound on the result! //note: those mins and max are the bound for the starting points of the algorithm, not strict bound on the result!
for(int i=1;i<ntry;i++){
P.Randomize(min,max);
statscore_t score=O->Run(P);
if(score>best){
best=score;
bestP=P;
}
mean+=score;
var+=(score*score);
oss.str("");
oss << "Try number " << (i+1);
PrintUserTime(oss.str());
}
mean/=(float)ntry;
var/=(float)ntry;
var=sqrt(abs(var-mean*mean));
if (verboselevel()>1)
cerr<<"best score: "<< best << " variance of the score (for "<<ntry<<" try): "<<var<<endl;
//L1-Normalization of the best Point
bestP.NormalizeL1(); for(int i=1; i<ntry; i++) {
P.Randomize(min,max);
cerr << "Best point: " << bestP << " => " << best << endl; statscore_t score=O->Run(P);
ofstream res("weights.txt"); if(score>best) {
res<<bestP<<endl; best=score;
bestP=P;
PrintUserTime("Stopping..."); }
mean+=score;
var+=(score*score);
oss.str("");
oss << "Try number " << (i+1);
PrintUserTime(oss.str());
}
mean/=(float)ntry;
var/=(float)ntry;
var=sqrt(abs(var-mean*mean));
if (verboselevel()>1)
cerr<<"best score: "<< best << " variance of the score (for "<<ntry<<" try): "<<var<<endl;
//L1-Normalization of the best Point
bestP.NormalizeL1();
cerr << "Best point: " << bestP << " => " << best << endl;
ofstream res("weights.txt");
res<<bestP<<endl;
PrintUserTime("Stopping...");
} }

81
mert/test_scorer.cpp
View File

@ -8,52 +8,53 @@
using namespace std; using namespace std;
int main(int argc, char** argv) { int main(int argc, char** argv)
cout << "Testing the scorer" << endl; {
//BleuScorer bs("test-scorer-data/cppstats.feats.opt");; cout << "Testing the scorer" << endl;
vector<string> references; //BleuScorer bs("test-scorer-data/cppstats.feats.opt");;
references.push_back("test_scorer_data/reference.txt"); vector<string> references;
//bs.prepare(references, "test-scorer-data/nbest.out"); references.push_back("test_scorer_data/reference.txt");
Scorer* scorer = new BleuScorer();; //bs.prepare(references, "test-scorer-data/nbest.out");
scorer->setReferenceFiles(references); Scorer* scorer = new BleuScorer();;
Data d(*scorer); scorer->setReferenceFiles(references);
d.loadnbest("test_scorer_data/nbest.out"); Data d(*scorer);
//sd.savetxt(); d.loadnbest("test_scorer_data/nbest.out");
//sd.savetxt();
//calculate two bleu scores, nbest and a diff //calculate two bleu scores, nbest and a diff
ScoreData* sd=d.getScoreData(); ScoreData* sd=d.getScoreData();
scorer->setScoreData(sd); scorer->setScoreData(sd);
candidates_t candidates(sd->size());; candidates_t candidates(sd->size());;
for (size_t i = 0; i < sd->size(); ++i) { for (size_t i = 0; i < sd->size(); ++i) {
sd->get(i,0).savetxt("/dev/stdout"); sd->get(i,0).savetxt("/dev/stdout");
} }
diffs_t diffs; diffs_t diffs;
diff_t diff; diff_t diff;
diff.push_back(make_pair(1,2)); diff.push_back(make_pair(1,2));
diff.push_back(make_pair(7,8)); diff.push_back(make_pair(7,8));
diffs.push_back(diff); diffs.push_back(diff);
statscores_t scores;
scorer->score(candidates,diffs,scores);
cout << "Bleus: " << scores[0] << " " << scores[1] << endl; statscores_t scores;
scorer->score(candidates,diffs,scores);
//try the per cout << "Bleus: " << scores[0] << " " << scores[1] << endl;
scorer = new PerScorer();
Data pd(*scorer);
scorer->setReferenceFiles(references);
pd.loadnbest("test_scorer_data/nbest.out"); //try the per
//sd.savetxt(); scorer = new PerScorer();
Data pd(*scorer);
scorer->setReferenceFiles(references);
ScoreData* psd=pd.getScoreData(); pd.loadnbest("test_scorer_data/nbest.out");
scorer->setScoreData(psd); //sd.savetxt();
for (size_t i = 0; i < psd->size(); ++i) {
psd->get(i,0).savetxt("/dev/stdout"); ScoreData* psd=pd.getScoreData();
} scorer->setScoreData(psd);
for (size_t i = 0; i < psd->size(); ++i) {
psd->get(i,0).savetxt("/dev/stdout");
}
cout << "PER: " << scorer->score(candidates) << endl; cout << "PER: " << scorer->score(candidates) << endl;
} }