mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
run beautify.perl. Consistent formatting for .h & .cpp files in Mert directory
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/mert-mtm5@4167 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
machacekmatous
parent
8b97df9367
commit
0a44787f55
@ -1,6 +1,7 @@
|
||||
#include "BleuScorer.h"
|
||||
|
||||
BleuScorer::BleuScorer(const string& config = "") : StatisticsBasedScorer("BLEU",config),_refLengthStrategy(BLEU_CLOSEST) {
|
||||
BleuScorer::BleuScorer(const string& config = "") : StatisticsBasedScorer("BLEU",config),_refLengthStrategy(BLEU_CLOSEST)
|
||||
{
|
||||
//configure regularisation
|
||||
static string KEY_REFLEN = "reflen";
|
||||
static string REFLEN_AVERAGE = "average";
|
||||
@ -30,7 +31,8 @@ BleuScorer::BleuScorer(const string& config = "") : StatisticsBasedScorer("BLEU"
|
||||
/**
|
||||
* count the ngrams of each type, up to the given length in the input line.
|
||||
**/
|
||||
size_t BleuScorer::countNgrams(const string& line, counts_t& counts, unsigned int n) {
|
||||
size_t BleuScorer::countNgrams(const string& line, counts_t& counts, unsigned int n)
|
||||
{
|
||||
vector<int> encoded_tokens;
|
||||
//cerr << line << endl;
|
||||
encode(line,encoded_tokens);
|
||||
@ -61,7 +63,8 @@ size_t BleuScorer::countNgrams(const string& line, counts_t& counts, unsigned in
|
||||
return encoded_tokens.size();
|
||||
}
|
||||
|
||||
void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
{
|
||||
//make sure reference data is clear
|
||||
_refcounts.clear();
|
||||
_reflengths.clear();
|
||||
@ -114,7 +117,8 @@ void BleuScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
}
|
||||
|
||||
|
||||
void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) {
|
||||
void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
{
|
||||
// cerr << text << endl;
|
||||
// cerr << sid << endl;
|
||||
//dump_counts(*_refcounts[sid]);
|
||||
@ -178,7 +182,8 @@ void BleuScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
entry.set(stats_str);
|
||||
}
|
||||
|
||||
float BleuScorer::calculateScore(const vector<float>& comps) {
|
||||
float BleuScorer::calculateScore(const vector<float>& comps)
|
||||
{
|
||||
//cerr << "BLEU: ";
|
||||
//copy(comps.begin(),comps.end(), ostream_iterator<int>(cerr," "));
|
||||
float logbleu = 0.0;
|
||||
|
||||
@ -23,7 +23,8 @@ enum BleuReferenceLengthStrategy { BLEU_AVERAGE, BLEU_SHORTEST, BLEU_CLOSEST };
|
||||
/**
|
||||
* Bleu scoring
|
||||
**/
|
||||
class BleuScorer: public StatisticsBasedScorer {
|
||||
class BleuScorer: public StatisticsBasedScorer
|
||||
{
|
||||
public:
|
||||
BleuScorer(const string& config);
|
||||
virtual void setReferenceFiles(const vector<string>& referenceFiles);
|
||||
|
||||
@ -98,8 +98,7 @@ void Data::loadnbest(const std::string &file)
|
||||
if ((loc = subsubstring.find(":")) != subsubstring.length()-1) {
|
||||
features+=tmpname+"_"+stringify(tmpidx)+" ";
|
||||
tmpidx++;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
tmpidx=0;
|
||||
tmpname=subsubstring.substr(0,subsubstring.size() - 1);
|
||||
}
|
||||
|
||||
41
mert/Data.h
41
mert/Data.h
@ -37,15 +37,30 @@ public:
|
||||
|
||||
~Data() {};
|
||||
|
||||
inline void clear() { scoredata->clear(); featdata->clear(); }
|
||||
inline void clear() {
|
||||
scoredata->clear();
|
||||
featdata->clear();
|
||||
}
|
||||
|
||||
ScoreData* getScoreData() { return scoredata; };
|
||||
FeatureData* getFeatureData() { return featdata; };
|
||||
ScoreData* getScoreData() {
|
||||
return scoredata;
|
||||
};
|
||||
FeatureData* getFeatureData() {
|
||||
return featdata;
|
||||
};
|
||||
|
||||
inline size_t NumberOfFeatures() const{ return featdata->NumberOfFeatures(); }
|
||||
inline void NumberOfFeatures(size_t v){ featdata->NumberOfFeatures(v); }
|
||||
inline std::string Features() const{ return featdata->Features(); }
|
||||
inline void Features(const std::string f){ featdata->Features(f); }
|
||||
inline size_t NumberOfFeatures() const {
|
||||
return featdata->NumberOfFeatures();
|
||||
}
|
||||
inline void NumberOfFeatures(size_t v) {
|
||||
featdata->NumberOfFeatures(v);
|
||||
}
|
||||
inline std::string Features() const {
|
||||
return featdata->Features();
|
||||
}
|
||||
inline void Features(const std::string f) {
|
||||
featdata->Features(f);
|
||||
}
|
||||
|
||||
void loadnbest(const std::string &file);
|
||||
|
||||
@ -63,10 +78,16 @@ public:
|
||||
scoredata->save(scorefile, bin);
|
||||
}
|
||||
|
||||
inline bool existsFeatureNames(){ return featdata->existsFeatureNames(); };
|
||||
inline bool existsFeatureNames() {
|
||||
return featdata->existsFeatureNames();
|
||||
};
|
||||
|
||||
inline std::string getFeatureName(size_t idx){ return featdata->getFeatureName(idx); };
|
||||
inline size_t getFeatureIndex(const std::string& name){ return featdata->getFeatureIndex(name); };
|
||||
inline std::string getFeatureName(size_t idx) {
|
||||
return featdata->getFeatureName(idx);
|
||||
};
|
||||
inline size_t getFeatureIndex(const std::string& name) {
|
||||
return featdata->getFeatureIndex(name);
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -39,28 +39,52 @@ public:
|
||||
|
||||
~FeatureArray() {};
|
||||
|
||||
inline void clear() { array_.clear(); }
|
||||
inline void clear() {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
inline std::string getIndex(){ return idx; }
|
||||
inline void setIndex(const std::string & value){ idx=value; }
|
||||
inline std::string getIndex() {
|
||||
return idx;
|
||||
}
|
||||
inline void setIndex(const std::string & value) {
|
||||
idx=value;
|
||||
}
|
||||
|
||||
inline FeatureStats& get(size_t i){ return array_.at(i); }
|
||||
inline const FeatureStats& get(size_t i)const{ return array_.at(i); }
|
||||
void add(FeatureStats e){ array_.push_back(e); }
|
||||
inline FeatureStats& get(size_t i) {
|
||||
return array_.at(i);
|
||||
}
|
||||
inline const FeatureStats& get(size_t i)const {
|
||||
return array_.at(i);
|
||||
}
|
||||
void add(FeatureStats e) {
|
||||
array_.push_back(e);
|
||||
}
|
||||
|
||||
void merge(FeatureArray& e);
|
||||
|
||||
inline size_t size(){ return array_.size(); }
|
||||
inline size_t NumberOfFeatures() const{ return number_of_features; }
|
||||
inline void NumberOfFeatures(size_t v){ number_of_features = v; }
|
||||
inline std::string Features() const{ return features; }
|
||||
inline void Features(const std::string f){ features = f; }
|
||||
inline size_t size() {
|
||||
return array_.size();
|
||||
}
|
||||
inline size_t NumberOfFeatures() const {
|
||||
return number_of_features;
|
||||
}
|
||||
inline void NumberOfFeatures(size_t v) {
|
||||
number_of_features = v;
|
||||
}
|
||||
inline std::string Features() const {
|
||||
return features;
|
||||
}
|
||||
inline void Features(const std::string f) {
|
||||
features = f;
|
||||
}
|
||||
|
||||
void savetxt(ofstream& outFile);
|
||||
void savebin(ofstream& outFile);
|
||||
void save(ofstream& outFile, bool bin=false);
|
||||
void save(const std::string &file, bool bin=false);
|
||||
inline void save(bool bin=false){ save("/dev/stdout",bin); }
|
||||
inline void save(bool bin=false) {
|
||||
save("/dev/stdout",bin);
|
||||
}
|
||||
|
||||
void loadtxt(ifstream& inFile, size_t n);
|
||||
void loadbin(ifstream& inFile, size_t n);
|
||||
|
||||
@ -74,26 +74,26 @@ void FeatureData::load(const std::string &file)
|
||||
inFile.close();
|
||||
}
|
||||
|
||||
void FeatureData::add(FeatureArray& e){
|
||||
void FeatureData::add(FeatureArray& e)
|
||||
{
|
||||
if (exists(e.getIndex())) { // array at position e.getIndex() already exists
|
||||
//enlarge array at position e.getIndex()
|
||||
size_t pos = getIndex(e.getIndex());
|
||||
array_.at(pos).merge(e);
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
array_.push_back(e);
|
||||
setIndex();
|
||||
}
|
||||
}
|
||||
|
||||
void FeatureData::add(FeatureStats& e, const std::string & sent_idx){
|
||||
void FeatureData::add(FeatureStats& e, const std::string & sent_idx)
|
||||
{
|
||||
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
||||
//enlarge array at position e.getIndex()
|
||||
size_t pos = getIndex(sent_idx);
|
||||
// TRACE_ERR("Inserting " << e << " in array " << sent_idx << std::endl);
|
||||
array_.at(pos).add(e);
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
// TRACE_ERR("Creating a new entry in the array and inserting " << e << std::endl);
|
||||
FeatureArray a;
|
||||
a.NumberOfFeatures(number_of_features);
|
||||
|
||||
@ -39,30 +39,58 @@ public:
|
||||
|
||||
~FeatureData() {};
|
||||
|
||||
inline void clear() { array_.clear(); }
|
||||
inline void clear() {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
inline FeatureArray get(const std::string& idx){ return array_.at(getIndex(idx)); }
|
||||
inline FeatureArray& get(size_t idx){ return array_.at(idx); }
|
||||
inline const FeatureArray& get(size_t idx) const{ return array_.at(idx); }
|
||||
inline FeatureArray get(const std::string& idx) {
|
||||
return array_.at(getIndex(idx));
|
||||
}
|
||||
inline FeatureArray& get(size_t idx) {
|
||||
return array_.at(idx);
|
||||
}
|
||||
inline const FeatureArray& get(size_t idx) const {
|
||||
return array_.at(idx);
|
||||
}
|
||||
|
||||
inline bool exists(const std::string & sent_idx){ return exists(getIndex(sent_idx)); }
|
||||
inline bool exists(int sent_idx){ return (sent_idx>-1 && sent_idx<(int) array_.size())?true:false; }
|
||||
inline bool exists(const std::string & sent_idx) {
|
||||
return exists(getIndex(sent_idx));
|
||||
}
|
||||
inline bool exists(int sent_idx) {
|
||||
return (sent_idx>-1 && sent_idx<(int) array_.size())?true:false;
|
||||
}
|
||||
|
||||
inline FeatureStats& get(size_t i, size_t j){ return array_.at(i).get(j); }
|
||||
inline const FeatureStats& get(size_t i, size_t j) const { return array_.at(i).get(j); }
|
||||
inline FeatureStats& get(size_t i, size_t j) {
|
||||
return array_.at(i).get(j);
|
||||
}
|
||||
inline const FeatureStats& get(size_t i, size_t j) const {
|
||||
return array_.at(i).get(j);
|
||||
}
|
||||
|
||||
void add(FeatureArray& e);
|
||||
void add(FeatureStats& e, const std::string& sent_idx);
|
||||
|
||||
inline size_t size(){ return array_.size(); }
|
||||
inline size_t NumberOfFeatures() const{ return number_of_features; }
|
||||
inline void NumberOfFeatures(size_t v){ number_of_features = v; }
|
||||
inline std::string Features() const{ return features; }
|
||||
inline void Features(const std::string f){ features = f; }
|
||||
inline size_t size() {
|
||||
return array_.size();
|
||||
}
|
||||
inline size_t NumberOfFeatures() const {
|
||||
return number_of_features;
|
||||
}
|
||||
inline void NumberOfFeatures(size_t v) {
|
||||
number_of_features = v;
|
||||
}
|
||||
inline std::string Features() const {
|
||||
return features;
|
||||
}
|
||||
inline void Features(const std::string f) {
|
||||
features = f;
|
||||
}
|
||||
|
||||
void save(const std::string &file, bool bin=false);
|
||||
void save(ofstream& outFile, bool bin=false);
|
||||
inline void save(bool bin=false){ save("/dev/stdout", bin); }
|
||||
inline void save(bool bin=false) {
|
||||
save("/dev/stdout", bin);
|
||||
}
|
||||
|
||||
void load(ifstream& inFile);
|
||||
void load(const std::string &file);
|
||||
@ -86,7 +114,9 @@ public:
|
||||
}
|
||||
|
||||
|
||||
bool existsFeatureNames(){ return (idx2featname_.size() > 0)?true:false; };
|
||||
bool existsFeatureNames() {
|
||||
return (idx2featname_.size() > 0)?true:false;
|
||||
};
|
||||
|
||||
std::string getFeatureName(size_t idx) {
|
||||
if (idx >= idx2featname_.size())
|
||||
|
||||
@ -129,7 +129,8 @@ FeatureStats& FeatureStats::operator=(const FeatureStats &stats)
|
||||
|
||||
|
||||
/**write the whole object to a stream*/
|
||||
ostream& operator<<(ostream& o, const FeatureStats& e){
|
||||
ostream& operator<<(ostream& o, const FeatureStats& e)
|
||||
{
|
||||
for (size_t i=0; i< e.size(); i++)
|
||||
o << e.get(i) << " ";
|
||||
return o;
|
||||
|
||||
@ -38,32 +38,53 @@ public:
|
||||
|
||||
~FeatureStats();
|
||||
|
||||
bool isfull(){return (entries_ < available_)?0:1; }
|
||||
bool isfull() {
|
||||
return (entries_ < available_)?0:1;
|
||||
}
|
||||
void expand();
|
||||
void add(FeatureStatsType v);
|
||||
|
||||
inline void clear() { memset((void*) array_,0,featbytes_); }
|
||||
inline void clear() {
|
||||
memset((void*) array_,0,featbytes_);
|
||||
}
|
||||
|
||||
inline FeatureStatsType get(size_t i){ return array_[i]; }
|
||||
inline FeatureStatsType get(size_t i)const{ return array_[i]; }
|
||||
inline featstats_t getArray() const { return array_; }
|
||||
inline FeatureStatsType get(size_t i) {
|
||||
return array_[i];
|
||||
}
|
||||
inline FeatureStatsType get(size_t i)const {
|
||||
return array_[i];
|
||||
}
|
||||
inline featstats_t getArray() const {
|
||||
return array_;
|
||||
}
|
||||
|
||||
void set(std::string &theString);
|
||||
|
||||
inline size_t bytes() const{ return featbytes_; }
|
||||
inline size_t size() const{ return entries_; }
|
||||
inline size_t available() const{ return available_; }
|
||||
inline size_t bytes() const {
|
||||
return featbytes_;
|
||||
}
|
||||
inline size_t size() const {
|
||||
return entries_;
|
||||
}
|
||||
inline size_t available() const {
|
||||
return available_;
|
||||
}
|
||||
|
||||
void savetxt(const std::string &file);
|
||||
void savetxt(ofstream& outFile);
|
||||
void savebin(ofstream& outFile);
|
||||
inline void savetxt(){ savetxt("/dev/stdout"); }
|
||||
inline void savetxt() {
|
||||
savetxt("/dev/stdout");
|
||||
}
|
||||
|
||||
void loadtxt(const std::string &file);
|
||||
void loadtxt(ifstream& inFile);
|
||||
void loadbin(ifstream& inFile);
|
||||
|
||||
inline void reset(){ entries_ = 0; clear(); }
|
||||
inline void reset() {
|
||||
entries_ = 0;
|
||||
clear();
|
||||
}
|
||||
|
||||
/**write the whole object to a stream*/
|
||||
friend ostream& operator<<(ostream& o, const FeatureStats& e);
|
||||
|
||||
@ -5,7 +5,8 @@
|
||||
using namespace std;
|
||||
|
||||
|
||||
InterpolatedScorer::InterpolatedScorer (const string& name, const string& config): Scorer(name,config) {
|
||||
InterpolatedScorer::InterpolatedScorer (const string& name, const string& config): Scorer(name,config)
|
||||
{
|
||||
//configure regularisation
|
||||
static string KEY_WEIGHTS = "weights";
|
||||
static string KEY_TYPE = "regtype";
|
||||
@ -91,7 +92,8 @@ InterpolatedScorer::InterpolatedScorer (const string& name, const string& config
|
||||
|
||||
}
|
||||
|
||||
void InterpolatedScorer::setScoreData(ScoreData* data) {
|
||||
void InterpolatedScorer::setScoreData(ScoreData* data)
|
||||
{
|
||||
size_t last = 0;
|
||||
_scoreData = data;
|
||||
for (vector<Scorer*>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
|
||||
@ -130,7 +132,8 @@ void InterpolatedScorer::setScoreData(ScoreData* data) {
|
||||
/** The interpolated scorer calls a vector of scorers and combines them with
|
||||
weights **/
|
||||
void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
|
||||
statscores_t& scores) {
|
||||
statscores_t& scores)
|
||||
{
|
||||
|
||||
//cout << "*******InterpolatedScorer::score" << endl;
|
||||
size_t scorerNum = 0;
|
||||
@ -161,7 +164,8 @@ void InterpolatedScorer::score(const candidates_t& candidates, const diffs_t& d
|
||||
|
||||
}
|
||||
|
||||
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
{
|
||||
for (vector<Scorer *>::iterator itsc = _scorers.begin(); itsc!=_scorers.end(); itsc++) {
|
||||
//the scorers that use alignments use the reference files in the constructor through config
|
||||
(*itsc)->setReferenceFiles(referenceFiles);
|
||||
@ -172,7 +176,8 @@ void InterpolatedScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
// Reference sentence ||| Reference sentence alignment information (as given by MOSES -include-alignment-in-n-best)
|
||||
// If a permutation distance scorer, send alignment info
|
||||
// Else if other scorer, remove the alignment info and then send reference as usual
|
||||
void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) {
|
||||
void InterpolatedScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
{
|
||||
stringstream buff;
|
||||
string align = text;
|
||||
string sentence = "";
|
||||
|
||||
@ -18,7 +18,8 @@
|
||||
/**
|
||||
* Abstract base class for scorers that include other scorers eg.
|
||||
* Interpolated HAMMING and BLEU scorer **/
|
||||
class InterpolatedScorer : public Scorer {
|
||||
class InterpolatedScorer : public Scorer
|
||||
{
|
||||
|
||||
public:
|
||||
// name would be: "HAMMING,BLEU" or similar
|
||||
|
||||
@ -14,19 +14,22 @@ static const float MAX_FLOAT=numeric_limits<float>::max();
|
||||
|
||||
|
||||
|
||||
void Optimizer::SetScorer(Scorer *S){
|
||||
void Optimizer::SetScorer(Scorer *S)
|
||||
{
|
||||
if(scorer)
|
||||
delete scorer;
|
||||
scorer=S;
|
||||
}
|
||||
|
||||
void Optimizer::SetFData(FeatureData *F){
|
||||
void Optimizer::SetFData(FeatureData *F)
|
||||
{
|
||||
if(FData)
|
||||
delete FData;
|
||||
FData=F;
|
||||
};
|
||||
|
||||
Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start):scorer(NULL),FData(NULL){
|
||||
Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start):scorer(NULL),FData(NULL)
|
||||
{
|
||||
//warning: the init vector is a full set of parameters, of dimension pdim!
|
||||
|
||||
Point::pdim=Pd;
|
||||
@ -46,12 +49,14 @@ Optimizer::Optimizer(unsigned Pd,vector<unsigned> i2O,vector<parameter_t> start)
|
||||
}
|
||||
};
|
||||
|
||||
Optimizer::~Optimizer(){
|
||||
Optimizer::~Optimizer()
|
||||
{
|
||||
delete scorer;
|
||||
delete FData;
|
||||
}
|
||||
|
||||
statscore_t Optimizer::GetStatScore(const Point& param)const{
|
||||
statscore_t Optimizer::GetStatScore(const Point& param)const
|
||||
{
|
||||
vector<unsigned> bests;
|
||||
Get1bests(param,bests);
|
||||
//copy(bests.begin(),bests.end(),ostream_iterator<unsigned>(cerr," "));
|
||||
@ -60,7 +65,8 @@ statscore_t Optimizer::GetStatScore(const Point& param)const{
|
||||
};
|
||||
|
||||
/**compute the intersection of 2 lines*/
|
||||
float intersect (float m1, float b1,float m2,float b2){
|
||||
float intersect (float m1, float b1,float m2,float b2)
|
||||
{
|
||||
float isect = ((b2-b1)/(m1-m2));
|
||||
if (!isfinite(isect)) {
|
||||
isect = MAX_FLOAT;
|
||||
@ -68,7 +74,8 @@ float intersect (float m1, float b1,float m2,float b2){
|
||||
return isect;
|
||||
}
|
||||
|
||||
map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float newt,pair<unsigned,unsigned> newdiff){
|
||||
map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float newt,pair<unsigned,unsigned> newdiff)
|
||||
{
|
||||
map<float,diff_t>::iterator it=thresholdmap.find(newt);
|
||||
if(it!=thresholdmap.end()) {
|
||||
//the threshold already exists!! this is very unlikely
|
||||
@ -86,7 +93,8 @@ map<float,diff_t >::iterator AddThreshold(map<float,diff_t >& thresholdmap,float
|
||||
};
|
||||
|
||||
|
||||
statscore_t Optimizer::LineOptimize(const Point& origin,const Point& direction,Point& bestpoint)const{
|
||||
statscore_t Optimizer::LineOptimize(const Point& origin,const Point& direction,Point& bestpoint)const
|
||||
{
|
||||
|
||||
// we are looking for the best Point on the line y=Origin+x*direction
|
||||
float min_int=0.0001;
|
||||
@ -285,7 +293,8 @@ statscore_t Optimizer::LineOptimize(const Point& origin,const Point& direction,P
|
||||
};
|
||||
|
||||
|
||||
void Optimizer::Get1bests(const Point& P,vector<unsigned>& bests)const{
|
||||
void Optimizer::Get1bests(const Point& P,vector<unsigned>& bests)const
|
||||
{
|
||||
assert(FData);
|
||||
bests.clear();
|
||||
bests.resize(size());
|
||||
@ -306,7 +315,8 @@ void Optimizer::Get1bests(const Point& P,vector<unsigned>& bests)const{
|
||||
|
||||
}
|
||||
|
||||
statscore_t Optimizer::Run(Point& P)const{
|
||||
statscore_t Optimizer::Run(Point& P)const
|
||||
{
|
||||
if(!FData) {
|
||||
cerr<<"error trying to optimize without Features loaded"<<endl;
|
||||
exit(2);
|
||||
@ -333,7 +343,8 @@ statscore_t Optimizer::Run(Point& P)const{
|
||||
}
|
||||
|
||||
|
||||
vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<vector <pair<unsigned,unsigned> > > thediffs)const{
|
||||
vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<vector <pair<unsigned,unsigned> > > thediffs)const
|
||||
{
|
||||
assert(scorer);
|
||||
|
||||
vector<statscore_t> theres;
|
||||
@ -347,7 +358,8 @@ vector<statscore_t> Optimizer::GetIncStatScore(vector<unsigned> thefirst,vector<
|
||||
|
||||
//---------------- code for the powell optimizer
|
||||
float SimpleOptimizer::eps=0.0001;
|
||||
statscore_t SimpleOptimizer::TrueRun(Point& P)const{
|
||||
statscore_t SimpleOptimizer::TrueRun(Point& P)const
|
||||
{
|
||||
|
||||
statscore_t prevscore=0;
|
||||
statscore_t bestscore=MIN_FLOAT;
|
||||
@ -409,7 +421,8 @@ statscore_t SimpleOptimizer::TrueRun(Point& P)const{
|
||||
/**RandomOptimizer to use as beaseline and test.\n
|
||||
Just return a random point*/
|
||||
|
||||
statscore_t RandomOptimizer::TrueRun(Point& P)const{
|
||||
statscore_t RandomOptimizer::TrueRun(Point& P)const
|
||||
{
|
||||
vector<parameter_t> min(Point::getdim());
|
||||
vector<parameter_t> max(Point::getdim());
|
||||
for(unsigned int d=0; d<Point::getdim(); d++) {
|
||||
@ -424,7 +437,8 @@ statscore_t RandomOptimizer::TrueRun(Point& P)const{
|
||||
//--------------------------------------
|
||||
vector<string> OptimizerFactory::typenames;
|
||||
|
||||
void OptimizerFactory::SetTypeNames(){
|
||||
void OptimizerFactory::SetTypeNames()
|
||||
{
|
||||
if(typenames.empty()) {
|
||||
typenames.resize(NOPTIMIZER);
|
||||
typenames[POWELL]="powell";
|
||||
@ -432,13 +446,15 @@ void OptimizerFactory::SetTypeNames(){
|
||||
//add new type there
|
||||
}
|
||||
}
|
||||
vector<string> OptimizerFactory::GetTypeNames(){
|
||||
vector<string> OptimizerFactory::GetTypeNames()
|
||||
{
|
||||
if(typenames.empty())
|
||||
SetTypeNames();
|
||||
return typenames;
|
||||
}
|
||||
|
||||
OptimizerFactory::OptType OptimizerFactory::GetOType(string type){
|
||||
OptimizerFactory::OptType OptimizerFactory::GetOType(string type)
|
||||
{
|
||||
unsigned int thetype;
|
||||
if(typenames.empty())
|
||||
SetTypeNames();
|
||||
@ -448,7 +464,8 @@ OptimizerFactory::OptType OptimizerFactory::GetOType(string type){
|
||||
return((OptType)thetype);
|
||||
};
|
||||
|
||||
Optimizer* OptimizerFactory::BuildOptimizer(unsigned dim,vector<unsigned> i2o,vector<parameter_t> start,string type){
|
||||
Optimizer* OptimizerFactory::BuildOptimizer(unsigned dim,vector<unsigned> i2o,vector<parameter_t> start,string type)
|
||||
{
|
||||
|
||||
OptType T=GetOType(type);
|
||||
if(T==NOPTIMIZER) {
|
||||
|
||||
@ -15,7 +15,8 @@ typedef float featurescore;
|
||||
|
||||
using namespace std;
|
||||
/**abstract virtual class*/
|
||||
class Optimizer{
|
||||
class Optimizer
|
||||
{
|
||||
protected:
|
||||
Scorer * scorer; //no accessor for them only child can use them
|
||||
FeatureData * FData;//no accessor for them only child can use them
|
||||
@ -25,7 +26,9 @@ class Optimizer{
|
||||
void SetFData(FeatureData *F);
|
||||
virtual ~Optimizer();
|
||||
|
||||
unsigned size()const{return (FData?FData->size():0);}
|
||||
unsigned size()const {
|
||||
return (FData?FData->size():0);
|
||||
}
|
||||
/**Generic wrapper around TrueRun to check a few things. Non virtual*/
|
||||
statscore_t Run(Point&)const;
|
||||
/**main function that perform an optimization*/
|
||||
@ -33,7 +36,9 @@ class Optimizer{
|
||||
/**given a set of lambdas, get the nbest for each sentence*/
|
||||
void Get1bests(const Point& param,vector<unsigned>& bests)const;
|
||||
/**given a set of nbests, get the Statistical score*/
|
||||
statscore_t GetStatScore(const vector<unsigned>& nbests)const{return scorer->score(nbests);};
|
||||
statscore_t GetStatScore(const vector<unsigned>& nbests)const {
|
||||
return scorer->score(nbests);
|
||||
};
|
||||
/**given a set of lambdas, get the total statistical score*/
|
||||
statscore_t GetStatScore(const Point& param)const;
|
||||
vector<statscore_t > GetIncStatScore(vector<unsigned> ref,vector<vector <pair<unsigned,unsigned> > >)const;
|
||||
@ -42,7 +47,8 @@ class Optimizer{
|
||||
|
||||
|
||||
/**default basic optimizer*/
|
||||
class SimpleOptimizer: public Optimizer{
|
||||
class SimpleOptimizer: public Optimizer
|
||||
{
|
||||
private:
|
||||
static float eps;
|
||||
public:
|
||||
@ -50,7 +56,8 @@ public:
|
||||
virtual statscore_t TrueRun(Point&)const;
|
||||
};
|
||||
|
||||
class RandomOptimizer: public Optimizer{
|
||||
class RandomOptimizer: public Optimizer
|
||||
{
|
||||
public:
|
||||
RandomOptimizer(unsigned dim,vector<unsigned> i2O,vector<parameter_t> start):Optimizer(dim,i2O,start) {};
|
||||
virtual statscore_t TrueRun(Point&)const;
|
||||
@ -58,7 +65,8 @@ public:
|
||||
|
||||
|
||||
|
||||
class OptimizerFactory{
|
||||
class OptimizerFactory
|
||||
{
|
||||
public:
|
||||
// unsigned dim;
|
||||
//Point Start;
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
#include "PerScorer.h"
|
||||
|
||||
|
||||
void PerScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
void PerScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
{
|
||||
// for each line in the reference file, create a multiset of the
|
||||
// word ids
|
||||
if (referenceFiles.size() != 1) {
|
||||
@ -32,7 +33,8 @@ void PerScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
|
||||
}
|
||||
|
||||
void PerScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) {
|
||||
void PerScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
{
|
||||
if (sid >= _reflengths.size()) {
|
||||
stringstream msg;
|
||||
msg << "Sentence id (" << sid << ") not found in reference set";
|
||||
@ -57,7 +59,8 @@ void PerScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
entry.set(stats_str);
|
||||
}
|
||||
|
||||
float PerScorer::calculateScore(const vector<float>& comps) {
|
||||
float PerScorer::calculateScore(const vector<float>& comps)
|
||||
{
|
||||
float denom = comps[2];
|
||||
float num = comps[0] - max(float(0),comps[1]-comps[2]);
|
||||
if (denom == 0) {
|
||||
|
||||
@ -22,7 +22,8 @@ using namespace std;
|
||||
* as 1 - (correct - max(0,output_length - ref_length)) / ref_length
|
||||
* In fact, we ignore the " 1 - " so that it can be maximised.
|
||||
**/
|
||||
class PerScorer: public StatisticsBasedScorer {
|
||||
class PerScorer: public StatisticsBasedScorer
|
||||
{
|
||||
public:
|
||||
PerScorer(const string& config = "") : StatisticsBasedScorer("PER",config) {}
|
||||
virtual void setReferenceFiles(const vector<string>& referenceFiles);
|
||||
@ -32,8 +33,13 @@ class PerScorer: public StatisticsBasedScorer {
|
||||
cerr << "I AM PerScorer" << std::endl;
|
||||
}
|
||||
|
||||
size_t NumberOfScores() const { cerr << "PerScorer: 3" << endl; return 3; };
|
||||
bool useAlignment() const {return false;};
|
||||
size_t NumberOfScores() const {
|
||||
cerr << "PerScorer: 3" << endl;
|
||||
return 3;
|
||||
};
|
||||
bool useAlignment() const {
|
||||
return false;
|
||||
};
|
||||
|
||||
|
||||
protected:
|
||||
|
||||
@ -16,8 +16,7 @@ using namespace std;
|
||||
|
||||
Permutation::Permutation(const string &alignment, const int sourceLength, const int targetLength )
|
||||
{
|
||||
if (sourceLength > 0)
|
||||
{
|
||||
if (sourceLength > 0) {
|
||||
set(alignment, sourceLength);
|
||||
}
|
||||
m_targetLength = targetLength;
|
||||
@ -51,15 +50,13 @@ void Permutation::set(const string & alignment,const int sourceLength)
|
||||
|
||||
//cout << "******** Permutation::set :" << alignment << ": len : " << sourceLength <<endl;
|
||||
|
||||
if(sourceLength <= 0)
|
||||
{
|
||||
if(sourceLength <= 0) {
|
||||
//not found
|
||||
cerr << "Source sentence length not positive:"<< sourceLength << endl;
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (alignment.length() <= 0)
|
||||
{
|
||||
if (alignment.length() <= 0) {
|
||||
//alignment empty - could happen but not good
|
||||
cerr << "Alignment string empty:"<< alignment << endl;
|
||||
}
|
||||
@ -89,8 +86,7 @@ void Permutation::set(const string & alignment,const int sourceLength)
|
||||
}
|
||||
//If have multiple target pos aligned to one source,
|
||||
// then ignore all but first alignment
|
||||
if (tempPerm[sourcePos] == -1 || tempPerm[sourcePos] > targetPos)
|
||||
{
|
||||
if (tempPerm[sourcePos] == -1 || tempPerm[sourcePos] > targetPos) {
|
||||
tempPerm[sourcePos] = targetPos;
|
||||
}
|
||||
}
|
||||
@ -112,8 +108,7 @@ void Permutation::set(const string & alignment,const int sourceLength)
|
||||
multimap<int, int> invMap;
|
||||
multimap<int, int>::iterator it;
|
||||
//cout << " SourceP -> TargetP " << endl;
|
||||
for (size_t i=0; i<tempPerm.size(); i++)
|
||||
{
|
||||
for (size_t i=0; i<tempPerm.size(); i++) {
|
||||
if (tempPerm[i] == -1) {
|
||||
tempPerm[i] = last;
|
||||
} else {
|
||||
@ -132,8 +127,7 @@ void Permutation::set(const string & alignment,const int sourceLength)
|
||||
// source: 3 2 4 5 6 7 8 1
|
||||
int i=0;
|
||||
//cout << " TargetP => SourceP : TargetIndex " << endl;
|
||||
for ( it=invMap.begin() ; it != invMap.end(); it++ )
|
||||
{
|
||||
for ( it=invMap.begin() ; it != invMap.end(); it++ ) {
|
||||
//cout << (*it).first << " => " << (*it).second << " : " << i << endl;
|
||||
//find source position
|
||||
m_array[(*it).second] = i;
|
||||
@ -224,10 +218,8 @@ float Permutation::calculateHamming(const Permutation & compare) const
|
||||
cerr << "Empty permutation" << endl;
|
||||
return 0;
|
||||
}
|
||||
for (size_t i=0; i<getLength(); i++)
|
||||
{
|
||||
if (m_array[i] != compareArray[i])
|
||||
{
|
||||
for (size_t i=0; i<getLength(); i++) {
|
||||
if (m_array[i] != compareArray[i]) {
|
||||
score++;
|
||||
}
|
||||
|
||||
@ -248,12 +240,9 @@ float Permutation::calculateKendall(const Permutation & compare) const
|
||||
cerr << "Empty permutation" << endl;
|
||||
return 0;
|
||||
}
|
||||
for (size_t i=0; i<getLength(); i++)
|
||||
{
|
||||
for (size_t j=0; j<getLength(); j++)
|
||||
{
|
||||
if ((m_array[i] < m_array[j]) && (compareArray[i] > compareArray[j]))
|
||||
{
|
||||
for (size_t i=0; i<getLength(); i++) {
|
||||
for (size_t j=0; j<getLength(); j++) {
|
||||
if ((m_array[i] < m_array[j]) && (compareArray[i] > compareArray[j])) {
|
||||
score++;
|
||||
}
|
||||
}
|
||||
@ -278,25 +267,21 @@ vector<int> Permutation::getArray() const
|
||||
//eg. 0=0 1-2=1-2 3=3 4=4 5=5 6=6 7-9=7-8 10=9 11-13=10-11 (source-target)
|
||||
string Permutation::convertMosesToStandard(string const & alignment)
|
||||
{
|
||||
if (alignment.length() == 0)
|
||||
{
|
||||
if (alignment.length() == 0) {
|
||||
cerr << "Alignment input string empty" << endl;
|
||||
}
|
||||
string working = alignment;
|
||||
string out;
|
||||
|
||||
stringstream oss;
|
||||
while (working.length() > 0)
|
||||
{
|
||||
while (working.length() > 0) {
|
||||
string align;
|
||||
getNextPound(working,align," ");
|
||||
|
||||
//If found an alignment
|
||||
if (align.length() > 0)
|
||||
{
|
||||
if (align.length() > 0) {
|
||||
size_t posDelimeter = align.find("=");
|
||||
if(posDelimeter== string::npos)
|
||||
{
|
||||
if(posDelimeter== string::npos) {
|
||||
cerr << "Delimiter not found = :"<< align << endl;
|
||||
exit(0);
|
||||
}
|
||||
|
||||
@ -27,8 +27,12 @@ public:
|
||||
|
||||
~Permutation() {};
|
||||
|
||||
inline void clear() { m_array.clear(); }
|
||||
inline size_t size(){ return m_array.size(); }
|
||||
inline void clear() {
|
||||
m_array.clear();
|
||||
}
|
||||
inline size_t size() {
|
||||
return m_array.size();
|
||||
}
|
||||
|
||||
|
||||
void set(const std::string &alignment,const int sourceLength);
|
||||
|
||||
@ -66,26 +66,24 @@ PermutationScorer::PermutationScorer(const string &distanceMetric, const string
|
||||
}
|
||||
}
|
||||
|
||||
void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles) {
|
||||
void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
{
|
||||
cout << "*******setReferenceFiles" << endl;
|
||||
//make sure reference data is clear
|
||||
m_referencePerms.clear();
|
||||
|
||||
vector< vector< int> > targetLengths;
|
||||
//Just getting target length from reference text file
|
||||
for (size_t i = 0; i < referenceFiles.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < referenceFiles.size(); ++i) {
|
||||
vector <int> lengths;
|
||||
cout << "Loading reference from " << referenceFiles[i] << endl;
|
||||
ifstream refin(referenceFiles[i].c_str());
|
||||
if (!refin)
|
||||
{
|
||||
if (!refin) {
|
||||
cerr << "Unable to open: " << referenceFiles[i] << endl;
|
||||
throw runtime_error("Unable to open alignment file");
|
||||
}
|
||||
string line;
|
||||
while (getline(refin,line))
|
||||
{
|
||||
while (getline(refin,line)) {
|
||||
int count = getNumberWords(line);
|
||||
lengths.push_back(count);
|
||||
}
|
||||
@ -94,20 +92,17 @@ void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
|
||||
//load reference data
|
||||
//NOTE ignoring normal reference file, only using previously saved alignment reference files
|
||||
for (size_t i = 0; i < m_referenceAlignments.size(); ++i)
|
||||
{
|
||||
for (size_t i = 0; i < m_referenceAlignments.size(); ++i) {
|
||||
vector<Permutation> referencePerms;
|
||||
cout << "Loading reference from " << m_referenceAlignments[i] << endl;
|
||||
ifstream refin(m_referenceAlignments[i].c_str());
|
||||
if (!refin)
|
||||
{
|
||||
if (!refin) {
|
||||
cerr << "Unable to open: " << m_referenceAlignments[i] << endl;
|
||||
throw runtime_error("Unable to open alignment file");
|
||||
}
|
||||
string line;
|
||||
size_t sid = 0; //sentence counter
|
||||
while (getline(refin,line))
|
||||
{
|
||||
while (getline(refin,line)) {
|
||||
//cout << line << endl;
|
||||
|
||||
//Line needs to be of the format: 0-0 1-1 1-2 etc source-target
|
||||
@ -115,8 +110,7 @@ void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
//perm.dump();
|
||||
referencePerms.push_back(perm);
|
||||
//check the source sentence length is the same for previous file
|
||||
if (perm.getLength() != m_sourceLengths[sid])
|
||||
{
|
||||
if (perm.getLength() != m_sourceLengths[sid]) {
|
||||
cerr << "Permutation Length: " << perm.getLength() << endl;
|
||||
cerr << "Source length: " << m_sourceLengths[sid] << " for sid " << sid << endl;
|
||||
throw runtime_error("Source sentence lengths not the same: ");
|
||||
@ -128,7 +122,8 @@ void PermutationScorer::setReferenceFiles(const vector<string>& referenceFiles)
|
||||
}
|
||||
}
|
||||
|
||||
int PermutationScorer::getNumberWords (const string& text) const {
|
||||
int PermutationScorer::getNumberWords (const string& text) const
|
||||
{
|
||||
int count = 0;
|
||||
string line = trimStr(text);
|
||||
if (line.length()>0) {
|
||||
@ -143,7 +138,8 @@ int PermutationScorer::getNumberWords (const string& text) const {
|
||||
}
|
||||
|
||||
|
||||
void PermutationScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry) {
|
||||
void PermutationScorer::prepareStats(size_t sid, const string& text, ScoreStats& entry)
|
||||
{
|
||||
//cout << "*******prepareStats" ;
|
||||
//cout << text << endl;
|
||||
//cout << sid << endl;
|
||||
@ -210,7 +206,8 @@ void PermutationScorer::prepareStats(size_t sid, const string& text, ScoreStats&
|
||||
}
|
||||
|
||||
//Will just be final score
|
||||
statscore_t PermutationScorer::calculateScore(const vector<statscore_t>& comps) {
|
||||
statscore_t PermutationScorer::calculateScore(const vector<statscore_t>& comps)
|
||||
{
|
||||
//cerr << "*******PermutationScorer::calculateScore" ;
|
||||
//cerr << " " << comps[0] << endl;
|
||||
return comps[0];
|
||||
|
||||
@ -14,14 +14,16 @@ map<unsigned,statscore_t> Point::fixedweights;
|
||||
unsigned Point::pdim=0;
|
||||
unsigned Point::ncall=0;
|
||||
|
||||
void Point::Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max){
|
||||
void Point::Randomize(const vector<parameter_t>& min,const vector<parameter_t>& max)
|
||||
{
|
||||
assert(min.size()==Point::dim);
|
||||
assert(max.size()==Point::dim);
|
||||
for (unsigned int i=0; i<size(); i++)
|
||||
operator[](i)= min[i] + (float)random()/(float)RAND_MAX * (float)(max[i]-min[i]);
|
||||
}
|
||||
|
||||
void Point::NormalizeL2(){
|
||||
void Point::NormalizeL2()
|
||||
{
|
||||
parameter_t norm=0.0;
|
||||
for (unsigned int i=0; i<size(); i++)
|
||||
norm+= operator[](i)*operator[](i);
|
||||
@ -33,7 +35,8 @@ void Point::NormalizeL2(){
|
||||
}
|
||||
|
||||
|
||||
void Point::NormalizeL1(){
|
||||
void Point::NormalizeL1()
|
||||
{
|
||||
parameter_t norm=0.0;
|
||||
for (unsigned int i=0; i<size(); i++)
|
||||
norm+= abs(operator[](i));
|
||||
@ -44,7 +47,8 @@ void Point::NormalizeL1(){
|
||||
}
|
||||
|
||||
//Can initialize from a vector of dim or pdim
|
||||
Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim){
|
||||
Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim)
|
||||
{
|
||||
if(init.size()==dim) {
|
||||
for (unsigned int i=0; i<Point::dim; i++)
|
||||
operator[](i)=init[i];
|
||||
@ -56,7 +60,8 @@ Point::Point(const vector<parameter_t>& init):vector<parameter_t>(Point::dim){
|
||||
};
|
||||
|
||||
|
||||
double Point::operator*(const FeatureStats& F)const{
|
||||
double Point::operator*(const FeatureStats& F)const
|
||||
{
|
||||
ncall++;//to track performance
|
||||
double prod=0.0;
|
||||
if(OptimizeAll())
|
||||
@ -70,7 +75,8 @@ double Point::operator*(const FeatureStats& F)const{
|
||||
}
|
||||
return prod;
|
||||
}
|
||||
Point Point::operator+(const Point& p2)const{
|
||||
Point Point::operator+(const Point& p2)const
|
||||
{
|
||||
assert(p2.size()==size());
|
||||
Point Res(*this);
|
||||
for(unsigned i=0; i<size(); i++)
|
||||
@ -79,7 +85,8 @@ Point Point::operator+(const Point& p2)const{
|
||||
return Res;
|
||||
};
|
||||
|
||||
Point Point::operator*(float l)const{
|
||||
Point Point::operator*(float l)const
|
||||
{
|
||||
Point Res(*this);
|
||||
for(unsigned i=0; i<size(); i++)
|
||||
Res[i]*=l;
|
||||
@ -87,7 +94,8 @@ Point Point::operator*(float l)const{
|
||||
return Res;
|
||||
};
|
||||
|
||||
ostream& operator<<(ostream& o,const Point& P){
|
||||
ostream& operator<<(ostream& o,const Point& P)
|
||||
{
|
||||
vector<parameter_t> w=P.GetAllWeights();
|
||||
// o << "[" << Point::pdim << "] ";
|
||||
for(unsigned int i=0; i<Point::pdim; i++)
|
||||
@ -96,7 +104,8 @@ Point Point::operator*(float l)const{
|
||||
return o;
|
||||
};
|
||||
|
||||
vector<parameter_t> Point::GetAllWeights()const{
|
||||
vector<parameter_t> Point::GetAllWeights()const
|
||||
{
|
||||
vector<parameter_t> w;
|
||||
if(OptimizeAll()) {
|
||||
w=*this;
|
||||
|
||||
23
mert/Point.h
23
mert/Point.h
@ -10,7 +10,8 @@ class Optimizer;
|
||||
|
||||
/**class that handle the subset of the Feature weight on which we run the optimization*/
|
||||
|
||||
class Point:public vector<parameter_t>{
|
||||
class Point:public vector<parameter_t>
|
||||
{
|
||||
friend class Optimizer;
|
||||
private:
|
||||
/**The indices over which we optimize*/
|
||||
@ -23,9 +24,15 @@ class Point:public vector<parameter_t>{
|
||||
static unsigned int pdim;
|
||||
static unsigned int ncall;
|
||||
public:
|
||||
static unsigned int getdim(){return dim;}
|
||||
static unsigned int getpdim(){return pdim;}
|
||||
static bool OptimizeAll(){return fixedweights.empty();};
|
||||
static unsigned int getdim() {
|
||||
return dim;
|
||||
}
|
||||
static unsigned int getpdim() {
|
||||
return pdim;
|
||||
}
|
||||
static bool OptimizeAll() {
|
||||
return fixedweights.empty();
|
||||
};
|
||||
statscore_t score;
|
||||
Point():vector<parameter_t>(dim) {};
|
||||
Point(const vector<parameter_t>& init);
|
||||
@ -36,12 +43,16 @@ class Point:public vector<parameter_t>{
|
||||
Point operator*(float)const;
|
||||
/**write the Whole featureweight to a stream (ie pdim float)*/
|
||||
friend ostream& operator<<(ostream& o,const Point& P);
|
||||
void Normalize(){ NormalizeL2(); };
|
||||
void Normalize() {
|
||||
NormalizeL2();
|
||||
};
|
||||
void NormalizeL2();
|
||||
void NormalizeL1();
|
||||
/**return a vector of size pdim where all weights have been put(including fixed ones)*/
|
||||
vector<parameter_t> GetAllWeights()const;
|
||||
statscore_t GetScore()const { return score; };
|
||||
statscore_t GetScore()const {
|
||||
return score;
|
||||
};
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
@ -40,32 +40,56 @@ public:
|
||||
|
||||
~ScoreArray() {};
|
||||
|
||||
inline void clear() { array_.clear(); }
|
||||
inline void clear() {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
inline std::string getIndex(){ return idx; }
|
||||
inline void setIndex(const std::string& value){ idx=value; }
|
||||
inline std::string getIndex() {
|
||||
return idx;
|
||||
}
|
||||
inline void setIndex(const std::string& value) {
|
||||
idx=value;
|
||||
}
|
||||
|
||||
// inline ScoreStats get(size_t i){ return array_.at(i); }
|
||||
|
||||
inline ScoreStats& get(size_t i){ return array_.at(i); }
|
||||
inline const ScoreStats& get(size_t i)const{ return array_.at(i); }
|
||||
inline ScoreStats& get(size_t i) {
|
||||
return array_.at(i);
|
||||
}
|
||||
inline const ScoreStats& get(size_t i)const {
|
||||
return array_.at(i);
|
||||
}
|
||||
|
||||
void add(const ScoreStats& e){ array_.push_back(e); }
|
||||
void add(const ScoreStats& e) {
|
||||
array_.push_back(e);
|
||||
}
|
||||
|
||||
void merge(ScoreArray& e);
|
||||
|
||||
inline std::string name() const{ return score_type; };
|
||||
inline void name(std::string &sctype){ score_type = sctype; };
|
||||
inline std::string name() const {
|
||||
return score_type;
|
||||
};
|
||||
inline void name(std::string &sctype) {
|
||||
score_type = sctype;
|
||||
};
|
||||
|
||||
inline size_t size(){ return array_.size(); }
|
||||
inline size_t NumberOfScores() const{ return number_of_scores; }
|
||||
inline void NumberOfScores(size_t v){ number_of_scores = v; }
|
||||
inline size_t size() {
|
||||
return array_.size();
|
||||
}
|
||||
inline size_t NumberOfScores() const {
|
||||
return number_of_scores;
|
||||
}
|
||||
inline void NumberOfScores(size_t v) {
|
||||
number_of_scores = v;
|
||||
}
|
||||
|
||||
void savetxt(ofstream& outFile, const std::string& sctype);
|
||||
void savebin(ofstream& outFile, const std::string& sctype);
|
||||
void save(ofstream& outFile, const std::string& sctype, bool bin=false);
|
||||
void save(const std::string &file, const std::string& sctype, bool bin=false);
|
||||
inline void save(const std::string& sctype, bool bin=false){ save("/dev/stdout", sctype, bin); }
|
||||
inline void save(const std::string& sctype, bool bin=false) {
|
||||
save("/dev/stdout", sctype, bin);
|
||||
}
|
||||
|
||||
void loadtxt(ifstream& inFile, size_t n);
|
||||
void loadbin(ifstream& inFile, size_t n);
|
||||
|
||||
@ -96,27 +96,27 @@ void ScoreData::load(const std::string &file)
|
||||
}
|
||||
|
||||
|
||||
void ScoreData::add(ScoreArray& e){
|
||||
void ScoreData::add(ScoreArray& e)
|
||||
{
|
||||
if (exists(e.getIndex())) { // array at position e.getIndex() already exists
|
||||
//enlarge array at position e.getIndex()
|
||||
size_t pos = getIndex(e.getIndex());
|
||||
array_.at(pos).merge(e);
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
array_.push_back(e);
|
||||
setIndex();
|
||||
}
|
||||
}
|
||||
|
||||
void ScoreData::add(const ScoreStats& e, const std::string& sent_idx){
|
||||
void ScoreData::add(const ScoreStats& e, const std::string& sent_idx)
|
||||
{
|
||||
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
||||
//enlarge array at position e.getIndex()
|
||||
size_t pos = getIndex(sent_idx);
|
||||
// TRACE_ERR("Inserting in array " << sent_idx << std::endl);
|
||||
array_.at(pos).add(e);
|
||||
// TRACE_ERR("size: " << size() << " -> " << a.size() << std::endl);
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
// TRACE_ERR("Creating a new entry in the array" << std::endl);
|
||||
ScoreArray a;
|
||||
a.NumberOfScores(number_of_scores);
|
||||
|
||||
@ -37,30 +37,56 @@ public:
|
||||
|
||||
~ScoreData() {};
|
||||
|
||||
inline void clear() { array_.clear(); }
|
||||
inline void clear() {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
inline ScoreArray get(const std::string& idx){ return array_.at(getIndex(idx)); }
|
||||
inline ScoreArray& get(size_t idx){ return array_.at(idx); }
|
||||
inline const ScoreArray& get(size_t idx) const { return array_.at(idx); }
|
||||
inline ScoreArray get(const std::string& idx) {
|
||||
return array_.at(getIndex(idx));
|
||||
}
|
||||
inline ScoreArray& get(size_t idx) {
|
||||
return array_.at(idx);
|
||||
}
|
||||
inline const ScoreArray& get(size_t idx) const {
|
||||
return array_.at(idx);
|
||||
}
|
||||
|
||||
inline bool exists(const std::string & sent_idx){ return exists(getIndex(sent_idx)); }
|
||||
inline bool exists(int sent_idx){ return (sent_idx>-1 && sent_idx<(int)array_.size())?true:false; }
|
||||
inline bool exists(const std::string & sent_idx) {
|
||||
return exists(getIndex(sent_idx));
|
||||
}
|
||||
inline bool exists(int sent_idx) {
|
||||
return (sent_idx>-1 && sent_idx<(int)array_.size())?true:false;
|
||||
}
|
||||
|
||||
inline ScoreStats& get(size_t i, size_t j){ return array_.at(i).get(j); }
|
||||
inline const ScoreStats& get(size_t i, size_t j) const { return array_.at(i).get(j); }
|
||||
inline ScoreStats& get(size_t i, size_t j) {
|
||||
return array_.at(i).get(j);
|
||||
}
|
||||
inline const ScoreStats& get(size_t i, size_t j) const {
|
||||
return array_.at(i).get(j);
|
||||
}
|
||||
|
||||
inline std::string name(){ return score_type; };
|
||||
inline std::string name(std::string &sctype){ return score_type = sctype; };
|
||||
inline std::string name() {
|
||||
return score_type;
|
||||
};
|
||||
inline std::string name(std::string &sctype) {
|
||||
return score_type = sctype;
|
||||
};
|
||||
|
||||
void add(ScoreArray& e);
|
||||
void add(const ScoreStats& e, const std::string& sent_idx);
|
||||
|
||||
inline size_t NumberOfScores(){ return number_of_scores; }
|
||||
inline size_t size(){ return array_.size(); }
|
||||
inline size_t NumberOfScores() {
|
||||
return number_of_scores;
|
||||
}
|
||||
inline size_t size() {
|
||||
return array_.size();
|
||||
}
|
||||
|
||||
void save(const std::string &file, bool bin=false);
|
||||
void save(ofstream& outFile, bool bin=false);
|
||||
inline void save(bool bin=false){ save("/dev/stdout", bin); }
|
||||
inline void save(bool bin=false) {
|
||||
save("/dev/stdout", bin);
|
||||
}
|
||||
|
||||
void load(ifstream& inFile);
|
||||
void load(const std::string &file);
|
||||
|
||||
@ -128,7 +128,8 @@ ScoreStats& ScoreStats::operator=(const ScoreStats &stats)
|
||||
|
||||
|
||||
/**write the whole object to a stream*/
|
||||
ostream& operator<<(ostream& o, const ScoreStats& e){
|
||||
ostream& operator<<(ostream& o, const ScoreStats& e)
|
||||
{
|
||||
for (size_t i=0; i< e.size(); i++)
|
||||
o << e.get(i) << " ";
|
||||
return o;
|
||||
|
||||
@ -39,26 +39,44 @@ public:
|
||||
|
||||
~ScoreStats();
|
||||
|
||||
bool isfull(){return (entries_ < available_)?0:1; }
|
||||
bool isfull() {
|
||||
return (entries_ < available_)?0:1;
|
||||
}
|
||||
void expand();
|
||||
void add(ScoreStatsType v);
|
||||
|
||||
inline void clear() { memset((void*) array_,0,scorebytes_); }
|
||||
inline void clear() {
|
||||
memset((void*) array_,0,scorebytes_);
|
||||
}
|
||||
|
||||
inline ScoreStatsType get(size_t i){ return array_[i]; }
|
||||
inline ScoreStatsType get(size_t i)const{ return array_[i]; }
|
||||
inline scorestats_t getArray() const { return array_; }
|
||||
inline ScoreStatsType get(size_t i) {
|
||||
return array_[i];
|
||||
}
|
||||
inline ScoreStatsType get(size_t i)const {
|
||||
return array_[i];
|
||||
}
|
||||
inline scorestats_t getArray() const {
|
||||
return array_;
|
||||
}
|
||||
|
||||
void set(std::string &theString);
|
||||
|
||||
inline size_t bytes() const{ return scorebytes_; }
|
||||
inline size_t size() const{ return entries_; }
|
||||
inline size_t available() const{ return available_; }
|
||||
inline size_t bytes() const {
|
||||
return scorebytes_;
|
||||
}
|
||||
inline size_t size() const {
|
||||
return entries_;
|
||||
}
|
||||
inline size_t available() const {
|
||||
return available_;
|
||||
}
|
||||
|
||||
void savetxt(const std::string &file);
|
||||
void savetxt(ofstream& outFile);
|
||||
void savebin(ofstream& outFile);
|
||||
inline void savetxt(){ savetxt("/dev/stdout"); }
|
||||
inline void savetxt() {
|
||||
savetxt("/dev/stdout");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -67,7 +85,10 @@ public:
|
||||
void loadbin(ifstream& inFile);
|
||||
|
||||
|
||||
inline void reset(){ entries_ = 0; clear(); }
|
||||
inline void reset() {
|
||||
entries_ = 0;
|
||||
clear();
|
||||
}
|
||||
|
||||
/**write the whole object to a stream*/
|
||||
friend ostream& operator<<(ostream& o, const ScoreStats& e);
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
#include "Scorer.h"
|
||||
|
||||
//regularisation strategies
|
||||
static float score_min(const statscores_t& scores, size_t start, size_t end) {
|
||||
static float score_min(const statscores_t& scores, size_t start, size_t end)
|
||||
{
|
||||
float min = numeric_limits<float>::max();
|
||||
for (size_t i = start; i < end; ++i) {
|
||||
if (scores[i] < min) {
|
||||
@ -11,7 +12,8 @@ static float score_min(const statscores_t& scores, size_t start, size_t end) {
|
||||
return min;
|
||||
}
|
||||
|
||||
static float score_average(const statscores_t& scores, size_t start, size_t end) {
|
||||
static float score_average(const statscores_t& scores, size_t start, size_t end)
|
||||
{
|
||||
if ((end - start) < 1) {
|
||||
//shouldn't happen
|
||||
return 0;
|
||||
@ -25,7 +27,8 @@ static float score_average(const statscores_t& scores, size_t start, size_t end)
|
||||
}
|
||||
|
||||
void StatisticsBasedScorer::score(const candidates_t& candidates, const diffs_t& diffs,
|
||||
statscores_t& scores) {
|
||||
statscores_t& scores)
|
||||
{
|
||||
//cout << "*******StatisticsBasedScorer::score" << endl;
|
||||
if (!_scoreData) {
|
||||
throw runtime_error("Score data not loaded");
|
||||
@ -110,7 +113,8 @@ void StatisticsBasedScorer::score(const candidates_t& candidates, const diffs_t
|
||||
/** The sentence level scores have already been calculated, just need to average them
|
||||
and include the differences. Allows scores which are floats **/
|
||||
void SentenceLevelScorer::score(const candidates_t& candidates, const diffs_t& diffs,
|
||||
statscores_t& scores) {
|
||||
statscores_t& scores)
|
||||
{
|
||||
//cout << "*******SentenceLevelScorer::score" << endl;
|
||||
if (!_scoreData) {
|
||||
throw runtime_error("Score data not loaded");
|
||||
|
||||
@ -25,7 +25,8 @@ class ScoreStats;
|
||||
* scorer it should be sufficient to override prepareStats(), setReferenceFiles()
|
||||
* and score() (or calculateScore()).
|
||||
**/
|
||||
class Scorer {
|
||||
class Scorer
|
||||
{
|
||||
private:
|
||||
string _name;
|
||||
|
||||
@ -58,7 +59,10 @@ class Scorer {
|
||||
/**
|
||||
* returns the number of statistics needed for the computation of the score
|
||||
**/
|
||||
virtual size_t NumberOfScores() const { cerr << "Scorer: 0" << endl; return 0; };
|
||||
virtual size_t NumberOfScores() const {
|
||||
cerr << "Scorer: 0" << endl;
|
||||
return 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* set the reference files. This must be called before prepareStats.
|
||||
@ -74,8 +78,7 @@ class Scorer {
|
||||
virtual void prepareStats(size_t sindex, const string& text, ScoreStats& entry)
|
||||
{}
|
||||
|
||||
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry)
|
||||
{
|
||||
virtual void prepareStats(const string& sindex, const string& text, ScoreStats& entry) {
|
||||
|
||||
// cerr << sindex << endl;
|
||||
this->prepareStats((size_t) atoi(sindex.c_str()), text, entry);
|
||||
@ -110,7 +113,9 @@ class Scorer {
|
||||
return scores[0];
|
||||
}
|
||||
|
||||
const string& getName() const {return _name;}
|
||||
const string& getName() const {
|
||||
return _name;
|
||||
}
|
||||
|
||||
size_t getReferenceSize() {
|
||||
if (_scoreData) {
|
||||
@ -135,7 +140,9 @@ class Scorer {
|
||||
return false;
|
||||
};
|
||||
//calculate the actual score
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;};
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
|
||||
return 0;
|
||||
};
|
||||
|
||||
protected:
|
||||
typedef map<string,int> encodings_t;
|
||||
@ -199,7 +206,8 @@ class Scorer {
|
||||
/**
|
||||
* Abstract base class for scorers that work by adding statistics across all
|
||||
* outout sentences, then apply some formula, e.g. bleu, per. **/
|
||||
class StatisticsBasedScorer : public Scorer {
|
||||
class StatisticsBasedScorer : public Scorer
|
||||
{
|
||||
|
||||
public:
|
||||
StatisticsBasedScorer(const string& name, const string& config): Scorer(name,config) {
|
||||
@ -244,7 +252,9 @@ class StatisticsBasedScorer : public Scorer {
|
||||
virtual void score(const candidates_t& candidates, const diffs_t& diffs,
|
||||
statscores_t& scores);
|
||||
//calculate the actual score
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;};
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
|
||||
return 0;
|
||||
};
|
||||
|
||||
protected:
|
||||
|
||||
@ -257,7 +267,8 @@ class StatisticsBasedScorer : public Scorer {
|
||||
/**
|
||||
* Abstract base class for scorers that work by using sentence level
|
||||
* statistics eg. permutation distance metrics **/
|
||||
class SentenceLevelScorer : public Scorer {
|
||||
class SentenceLevelScorer : public Scorer
|
||||
{
|
||||
|
||||
public:
|
||||
SentenceLevelScorer(const string& name, const string& config): Scorer(name,config) {
|
||||
@ -302,7 +313,9 @@ class SentenceLevelScorer : public Scorer {
|
||||
statscores_t& scores);
|
||||
|
||||
//calculate the actual score
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals){return 0;};
|
||||
virtual statscore_t calculateScore(const vector<statscore_t>& totals) {
|
||||
return 0;
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
@ -19,7 +19,8 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
class ScorerFactory {
|
||||
class ScorerFactory
|
||||
{
|
||||
|
||||
public:
|
||||
vector<string> getTypes() {
|
||||
|
||||
@ -18,11 +18,13 @@ Timer g_timer;
|
||||
|
||||
int verbose=0;
|
||||
|
||||
int verboselevel(){
|
||||
int verboselevel()
|
||||
{
|
||||
return verbose;
|
||||
}
|
||||
|
||||
int setverboselevel(int v){
|
||||
int setverboselevel(int v)
|
||||
{
|
||||
verbose=v;
|
||||
return verbose;
|
||||
}
|
||||
@ -32,13 +34,11 @@ int getNextPound(std::string &theString, std::string &substring, const std::stri
|
||||
unsigned int pos = 0;
|
||||
|
||||
//skip all occurrences of delimiter
|
||||
while ( pos == 0 )
|
||||
{
|
||||
while ( pos == 0 ) {
|
||||
if ((pos = theString.find(delimiter)) != std::string::npos) {
|
||||
substring.assign(theString, 0, pos);
|
||||
theString.erase(0,pos + delimiter.size());
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
substring.assign(theString);
|
||||
theString.assign("");
|
||||
}
|
||||
@ -55,9 +55,9 @@ m_streambuf(0)
|
||||
_good=(fb->open(filePath.c_str(), std::ios::in)!=NULL);
|
||||
|
||||
if (filePath.size() > 3 &&
|
||||
filePath.substr(filePath.size() - 3, 3) == ".gz")
|
||||
{
|
||||
fb->close(); delete fb;
|
||||
filePath.substr(filePath.size() - 3, 3) == ".gz") {
|
||||
fb->close();
|
||||
delete fb;
|
||||
m_streambuf = new gzfilebuf(filePath.c_str());
|
||||
} else {
|
||||
m_streambuf = fb;
|
||||
@ -67,7 +67,8 @@ m_streambuf(0)
|
||||
|
||||
inputfilestream::~inputfilestream()
|
||||
{
|
||||
delete m_streambuf; m_streambuf = 0;
|
||||
delete m_streambuf;
|
||||
m_streambuf = 0;
|
||||
}
|
||||
|
||||
void inputfilestream::close()
|
||||
@ -82,8 +83,7 @@ m_streambuf(0)
|
||||
std::filebuf* fb = new std::filebuf();
|
||||
_good=(fb->open(filePath.c_str(), std::ios::out)!=NULL);
|
||||
|
||||
if (filePath.size() > 3 && filePath.substr(filePath.size() - 3, 3) == ".gz")
|
||||
{
|
||||
if (filePath.size() > 3 && filePath.substr(filePath.size() - 3, 3) == ".gz") {
|
||||
throw runtime_error("Output to a zipped file not supported!");
|
||||
} else {
|
||||
m_streambuf = fb;
|
||||
@ -93,7 +93,8 @@ m_streambuf(0)
|
||||
|
||||
outputfilestream::~outputfilestream()
|
||||
{
|
||||
delete m_streambuf; m_streambuf = 0;
|
||||
delete m_streambuf;
|
||||
m_streambuf = 0;
|
||||
}
|
||||
|
||||
void outputfilestream::close()
|
||||
@ -105,7 +106,11 @@ int swapbytes(char *p, int sz, int n)
|
||||
char c, *l, *h;
|
||||
|
||||
if((n<1) || (sz<2)) return 0;
|
||||
for(; n--; p+=sz) for(h=(l=p)+sz; --h>l; l++) { c=*h; *h=*l; *l=c; }
|
||||
for(; n--; p+=sz) for(h=(l=p)+sz; --h>l; l++) {
|
||||
c=*h;
|
||||
*h=*l;
|
||||
*l=c;
|
||||
}
|
||||
return 0;
|
||||
|
||||
};
|
||||
|
||||
@ -66,7 +66,9 @@ public:
|
||||
|
||||
inputfilestream(const std::string &filePath);
|
||||
~inputfilestream();
|
||||
bool good(){return _good;}
|
||||
bool good() {
|
||||
return _good;
|
||||
}
|
||||
void close();
|
||||
};
|
||||
|
||||
@ -79,7 +81,9 @@ public:
|
||||
|
||||
outputfilestream(const std::string &filePath);
|
||||
~outputfilestream();
|
||||
bool good(){return _good;}
|
||||
bool good() {
|
||||
return _good;
|
||||
}
|
||||
void close();
|
||||
};
|
||||
|
||||
|
||||
@ -18,7 +18,8 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
void usage() {
|
||||
void usage()
|
||||
{
|
||||
cerr<<"usage: extractor [options])"<<endl;
|
||||
cerr<<"[--sctype|-s] the scorer type (default BLEU), possibly comma separated list of interpolated types"<<endl;
|
||||
cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl;
|
||||
@ -36,8 +37,7 @@ cerr<<"[--prev-ffile|-E] comma separated list of previous feature data" <<endl;
|
||||
}
|
||||
|
||||
|
||||
static struct option long_options[] =
|
||||
{
|
||||
static struct option long_options[] = {
|
||||
{"sctype",required_argument,0,'s'},
|
||||
{"scconfig",required_argument,0,'c'},
|
||||
{"reference",required_argument,0,'r'},
|
||||
@ -53,7 +53,8 @@ static struct option long_options[] =
|
||||
};
|
||||
int option_index;
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
|
||||
|
||||
ResetUserTime();
|
||||
|
||||
@ -4,15 +4,18 @@
|
||||
#include <streambuf>
|
||||
#include <zlib.h>
|
||||
|
||||
class gzfilebuf : public std::streambuf {
|
||||
class gzfilebuf : public std::streambuf
|
||||
{
|
||||
public:
|
||||
gzfilebuf(const char *filename)
|
||||
{ _gzf = gzopen(filename, "rb");
|
||||
gzfilebuf(const char *filename) {
|
||||
_gzf = gzopen(filename, "rb");
|
||||
setg (_buff+sizeof(int), // beginning of putback area
|
||||
_buff+sizeof(int), // read position
|
||||
_buff+sizeof(int)); // end position
|
||||
}
|
||||
~gzfilebuf() { gzclose(_gzf); }
|
||||
~gzfilebuf() {
|
||||
gzclose(_gzf);
|
||||
}
|
||||
protected:
|
||||
virtual int_type overflow (int_type c) {
|
||||
throw;
|
||||
@ -25,7 +28,8 @@ protected:
|
||||
throw;
|
||||
}
|
||||
|
||||
virtual std::streampos seekpos ( std::streampos sp, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ){ throw;
|
||||
virtual std::streampos seekpos ( std::streampos sp, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out ) {
|
||||
throw;
|
||||
}
|
||||
|
||||
//read one character
|
||||
|
||||
@ -28,7 +28,8 @@ float min_interval = 1e-3;
|
||||
|
||||
using namespace std;
|
||||
|
||||
void usage(void) {
|
||||
void usage(void)
|
||||
{
|
||||
cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl;
|
||||
cerr<<"[-n retry ntimes (default 1)]"<<endl;
|
||||
cerr<<"[-o\tthe indexes to optimize(default all)]"<<endl;
|
||||
@ -44,8 +45,7 @@ void usage(void) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static struct option long_options[] =
|
||||
{
|
||||
static struct option long_options[] = {
|
||||
{"pdim", 1, 0, 'd'},
|
||||
{"ntry",1,0,'n'},
|
||||
{"rseed",required_argument,0,'r'},
|
||||
@ -62,7 +62,8 @@ static struct option long_options[] =
|
||||
};
|
||||
int option_index;
|
||||
|
||||
int main (int argc, char **argv) {
|
||||
int main (int argc, char **argv)
|
||||
{
|
||||
|
||||
|
||||
ResetUserTime();
|
||||
@ -202,13 +203,18 @@ int main (int argc, char **argv) {
|
||||
index = D.getFeatureIndex(substring);
|
||||
cerr << "FeatNameIndex:" << index << " to insert" << endl;
|
||||
//index = strtol(substring.c_str(), NULL, 10);
|
||||
if (index >= 0 && index < pdim){ tooptimize.push_back(index); }
|
||||
else{ cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl; }
|
||||
if (index >= 0 && index < pdim) {
|
||||
tooptimize.push_back(index);
|
||||
} else {
|
||||
cerr << "Index " << index << " is out of bounds. Allowed indexes are [0," << (pdim-1) << "]." << endl;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
//set all weights as active
|
||||
tooptimize.resize(pdim);//We'll optimize on everything
|
||||
for(int i=0;i<pdim;i++){ tooptimize[i]=1; }
|
||||
for(int i=0; i<pdim; i++) {
|
||||
tooptimize[i]=1;
|
||||
}
|
||||
}
|
||||
|
||||
Optimizer *O=OptimizerFactory::BuildOptimizer(pdim,tooptimize,start,type);
|
||||
|
||||
@ -8,7 +8,8 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
cout << "Testing the scorer" << endl;
|
||||
//BleuScorer bs("test-scorer-data/cppstats.feats.opt");;
|
||||
vector<string> references;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user