moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 22:14:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

mert/evaluator should now compute confidence interval correctly

Browse Source
This commit is contained in:
Matous Machacek 2012-01-24 21:25:15 +01:00
parent c8cde15137
commit 5254e7917b
1 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
mert/evaluator.cpp
View File

@ -25,6 +25,8 @@ string int2string(int n);
bool moreFiles = false; bool moreFiles = false;
bool moreScorers = false; bool moreScorers = false;
float alpha = 0.05;
void usage() void usage()
{ {
cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate cand1[,cand2[,cand3...]] "<<endl; cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate cand1[,cand2[,cand3...]] "<<endl;
@ -187,14 +189,21 @@ void evaluate(const string& candFile)
} }
float avg = average(scores); float avg = average(scores);
float dev = stdDeviation(scores, avg);
sort(scores.begin(), scores.end());
int lbIdx = scores.size() * (alpha / 2);
int rbIdx = scores.size() * (1 - alpha / 2);
float lb = scores[lbIdx];
float rb = scores[rbIdx];
if (moreFiles) cout << candFile << "\t"; if (moreFiles) cout << candFile << "\t";
if (moreScorers) cout << scorer->getName() << "\t"; if (moreScorers) cout << scorer->getName() << "\t";
cout.setf(ios::fixed,ios::floatfield); cout.setf(ios::fixed,ios::floatfield);
cout.precision(4); cout.precision(4);
cout << avg << " [+-" << dev << "]"<< endl; cout << avg << "\t[" << lb << "," << rb << "]"<< endl;
} }
else else
{ {
@ -234,12 +243,3 @@ float average(const vector<float>& list)
return sum / list.size(); return sum / list.size();
} }
float stdDeviation(const vector<float>& list, float avg)
{
vector<float> tmp;
for (vector<float>::const_iterator it = list.begin(); it != list.end(); ++it)
tmp.push_back(pow(*it - avg, 2));
return sqrt(average(tmp));
}