mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-16 23:10:31 +03:00
480 lines
12 KiB
C++
480 lines
12 KiB
C++
#pragma once
|
|
|
|
#include <cmath>
|
|
#include <string>
|
|
#include <vector>
|
|
#include <set>
|
|
#include <map>
|
|
#include <queue>
|
|
#include <iostream>
|
|
#include <fstream>
|
|
#include <iterator>
|
|
#include <algorithm>
|
|
#include <limits>
|
|
#include <sstream>
|
|
#include <boost/algorithm/string.hpp>
|
|
|
|
|
|
|
|
namespace MosesTuning
|
|
{
|
|
|
|
namespace M2
|
|
{
|
|
|
|
typedef std::vector<float> Stats;
|
|
|
|
typedef std::vector<std::string> Sentence;
|
|
|
|
std::ostream& operator<<(std::ostream& o, Sentence s);
|
|
|
|
const std::string ToLower(const std::string& str);
|
|
|
|
struct Annot {
|
|
size_t i;
|
|
size_t j;
|
|
|
|
std::string type;
|
|
std::string edit;
|
|
|
|
size_t annotator;
|
|
|
|
bool operator<(Annot a) const {
|
|
return i < a.i || (i == a.i && j < a.j)
|
|
|| (i == a.i && j == a.j && annotator < a.annotator)
|
|
|| (i == a.i && j == a.j && annotator == a.annotator && transform(edit) < transform(a.edit));
|
|
}
|
|
|
|
bool operator==(Annot a) const {
|
|
return (!(*this < a) && !(a < *this));
|
|
}
|
|
|
|
static std::string transform(const std::string& e);
|
|
|
|
static bool lowercase;
|
|
};
|
|
|
|
typedef std::set<Annot> Annots;
|
|
typedef std::set<size_t> Users;
|
|
|
|
struct Unit {
|
|
Sentence first;
|
|
Annots second;
|
|
Users third;
|
|
};
|
|
|
|
typedef std::vector<Unit> M2File;
|
|
|
|
struct Edit {
|
|
Edit(float c = 1.0, size_t ch = 0, size_t unch = 1, std::string e = "")
|
|
: cost(c), changed(ch), unchanged(unch), edit(e) {}
|
|
|
|
float cost;
|
|
size_t changed;
|
|
size_t unchanged;
|
|
std::string edit;
|
|
};
|
|
|
|
Edit operator+(Edit& e1, Edit& e2);
|
|
|
|
struct Vertex {
|
|
Vertex(size_t a = 0, size_t b = 0) : i(a), j(b) {}
|
|
|
|
bool operator<(const Vertex &v) const {
|
|
return i < v.i || (i == v.i && j < v.j);
|
|
}
|
|
|
|
bool operator==(const Vertex &v) const {
|
|
return i == v.i && j == v.j;
|
|
}
|
|
|
|
size_t i;
|
|
size_t j;
|
|
};
|
|
|
|
struct Edge {
|
|
Edge(Vertex vv = Vertex(), Vertex uu = Vertex(), Edit editt = Edit())
|
|
: v(vv), u(uu), edit(editt) {}
|
|
|
|
bool operator<(const Edge &e) const {
|
|
return v < e.v || (v == e.v && u < e.u);
|
|
}
|
|
|
|
Vertex v;
|
|
Vertex u;
|
|
Edit edit;
|
|
};
|
|
|
|
Edge operator+(Edge e1, Edge e2);
|
|
|
|
typedef std::vector<size_t> Row;
|
|
typedef std::vector<Row> Matrix;
|
|
|
|
struct Info {
|
|
Info(Vertex vv = Vertex(), Edit editt = Edit())
|
|
: v(vv), edit(editt) {}
|
|
|
|
bool operator<(const Info &i) const {
|
|
return v < i.v;
|
|
}
|
|
|
|
Vertex v;
|
|
Edit edit;
|
|
};
|
|
|
|
typedef std::set<Info> Track;
|
|
typedef std::vector<Track> TrackRow;
|
|
typedef std::vector<TrackRow> TrackMatrix;
|
|
|
|
typedef std::set<Vertex> Vertices;
|
|
typedef std::set<Edge> Edges;
|
|
|
|
class M2
|
|
{
|
|
private:
|
|
M2File m_m2;
|
|
|
|
size_t m_max_unchanged;
|
|
float m_beta;
|
|
bool m_lowercase;
|
|
bool m_verbose;
|
|
|
|
public:
|
|
M2() : m_max_unchanged(2), m_beta(0.5), m_lowercase(true), m_verbose(false) { }
|
|
M2(size_t max_unchanged, float beta, bool truecase, bool verbose = false)
|
|
: m_max_unchanged(max_unchanged), m_beta(beta), m_lowercase(!truecase), m_verbose(verbose) {
|
|
if(!m_lowercase) {
|
|
Annot::lowercase = false;
|
|
}
|
|
}
|
|
|
|
float Beta() {
|
|
return m_beta;
|
|
}
|
|
|
|
void ReadM2(const std::string& filename) {
|
|
std::ifstream m2file(filename.c_str());
|
|
std::string line;
|
|
|
|
Unit unit;
|
|
bool first = true;
|
|
|
|
while(std::getline(m2file, line)) {
|
|
if(line.size() > 2) {
|
|
if(line.substr(0, 2) == "S ") {
|
|
if(!first) {
|
|
if(unit.third.empty())
|
|
unit.third.insert(0);
|
|
m_m2.push_back(unit);
|
|
}
|
|
first = false;
|
|
|
|
unit.first = Sentence();
|
|
unit.second = Annots();
|
|
|
|
std::string sentenceLine = line.substr(2);
|
|
boost::split(unit.first, sentenceLine, boost::is_any_of(" "), boost::token_compress_on);
|
|
}
|
|
if(line.substr(0, 2) == "A ") {
|
|
std::string annotLine = line.substr(2);
|
|
|
|
std::vector<std::string> annot;
|
|
boost::iter_split(annot, annotLine, boost::algorithm::first_finder("|||"));
|
|
|
|
if(annot[1] != "noop") {
|
|
Annot a;
|
|
std::stringstream rangeStr(annot[0]);
|
|
rangeStr >> a.i >> a.j;
|
|
a.type = annot[1];
|
|
a.edit = annot[2];
|
|
|
|
std::stringstream annotStr(annot[5]);
|
|
annotStr >> a.annotator;
|
|
|
|
unit.third.insert(a.annotator);
|
|
unit.second.insert(a);
|
|
} else {
|
|
std::stringstream annotStr(annot[5]);
|
|
size_t annotator;
|
|
annotStr >> annotator;
|
|
unit.third.insert(annotator);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if(unit.third.empty())
|
|
unit.third.insert(0);
|
|
m_m2.push_back(unit);
|
|
}
|
|
|
|
size_t LevenshteinMatrix(const Sentence &s1, const Sentence &s2, Matrix &d, TrackMatrix &bt) {
|
|
size_t n = s1.size();
|
|
size_t m = s2.size();
|
|
|
|
if (n == 0)
|
|
return m;
|
|
if (m == 0)
|
|
return n;
|
|
|
|
d.resize(n + 1, Row(m + 1, 0));
|
|
bt.resize(n + 1, TrackRow(m + 1));
|
|
|
|
for(size_t i = 0; i <= n; ++i) {
|
|
d[i][0] = i;
|
|
if(i > 0)
|
|
bt[i][0].insert(Info(Vertex(i - 1, 0), Edit(1, 1, 0, "")));
|
|
}
|
|
for(size_t j = 0; j <= m; ++j) {
|
|
d[0][j] = j;
|
|
if(j > 0)
|
|
bt[0][j].insert(Info(Vertex(0, j - 1), Edit(1, 1, 0, s2[j - 1])));
|
|
}
|
|
|
|
int cost;
|
|
for(size_t i = 1; i <= n; ++i) {
|
|
for(size_t j = 1; j <= m; ++j) {
|
|
if(Annot::transform(s1[i-1]) == Annot::transform(s2[j-1]))
|
|
cost = 0;
|
|
else
|
|
cost = 2;
|
|
|
|
size_t left = d[i][j - 1] + 1;
|
|
size_t down = d[i - 1][j] + 1;
|
|
size_t diag = d[i - 1][j - 1] + cost;
|
|
|
|
d[i][j] = std::min(left, std::min(down, diag));
|
|
|
|
if(d[i][j] == left)
|
|
bt[i][j].insert(Info(Vertex(i, j - 1), Edit(1, 1, 0, s2[j - 1])));
|
|
if(d[i][j] == down)
|
|
bt[i][j].insert(Info(Vertex(i - 1, j), Edit(1, 1, 0, "")));
|
|
if(d[i][j] == diag)
|
|
bt[i][j].insert(Info(Vertex(i - 1, j - 1), cost ? Edit(1, 1, 0, s2[j - 1]) : Edit(1, 0, 1, s2[j - 1]) ));
|
|
}
|
|
}
|
|
return d[n][m];
|
|
}
|
|
|
|
|
|
void BuildGraph(const TrackMatrix &bt, Vertices &V, Edges &E) {
|
|
Vertex start(bt.size() - 1, bt[0].size() - 1);
|
|
|
|
std::queue<Vertex> Q;
|
|
Q.push(start);
|
|
while(!Q.empty()) {
|
|
Vertex v = Q.front();
|
|
Q.pop();
|
|
if(V.count(v) > 0)
|
|
continue;
|
|
V.insert(v);
|
|
for(Track::iterator it = bt[v.i][v.j].begin();
|
|
it != bt[v.i][v.j].end(); ++it) {
|
|
Edge e(it->v, v, it->edit);
|
|
E.insert(e);
|
|
if(V.count(e.v) == 0)
|
|
Q.push(e.v);
|
|
}
|
|
}
|
|
|
|
Edges newE;
|
|
do {
|
|
newE.clear();
|
|
for(Edges::iterator it1 = E.begin(); it1 != E.end(); ++it1) {
|
|
for(Edges::iterator it2 = E.begin(); it2 != E.end(); ++it2) {
|
|
if(it1->u == it2->v) {
|
|
Edge e = *it1 + *it2;
|
|
if(e.edit.changed > 0 &&
|
|
e.edit.unchanged <= m_max_unchanged &&
|
|
E.count(e) == 0)
|
|
newE.insert(e);
|
|
}
|
|
}
|
|
}
|
|
E.insert(newE.begin(), newE.end());
|
|
} while(newE.size() > 0);
|
|
}
|
|
|
|
void AddWeights(Edges &E, const Unit &u, size_t aid) {
|
|
for(Edges::iterator it1 = E.begin(); it1 != E.end(); ++it1) {
|
|
if(it1->edit.changed > 0) {
|
|
const_cast<float&>(it1->edit.cost) += 0.001;
|
|
for(Annots::iterator it2 = u.second.begin(); it2 != u.second.end(); ++it2) {
|
|
// if matches an annotator
|
|
if(it1->v.i == it2->i && it1->u.i == it2->j
|
|
&& Annot::transform(it1->edit.edit) == Annot::transform(it2->edit)
|
|
&& it2->annotator == aid) {
|
|
int newWeight = -(m_max_unchanged + 1) * E.size();
|
|
const_cast<float&>(it1->edit.cost) = newWeight;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void BellmanFord(Vertices &V, Edges &E) {
|
|
Vertex source(0, 0);
|
|
std::map<Vertex, float> distance;
|
|
std::map<Vertex, Vertex> predecessor;
|
|
|
|
for(Vertices::iterator it = V.begin(); it != V.end(); ++it) {
|
|
if(*it == source)
|
|
distance[*it] = 0;
|
|
else {
|
|
distance[*it] = std::numeric_limits<float>::infinity();
|
|
}
|
|
}
|
|
|
|
for(size_t i = 1; i < V.size(); ++i) {
|
|
for(Edges::iterator it = E.begin(); it != E.end(); ++it) {
|
|
if(distance[it->v] + it->edit.cost < distance[it->u]) {
|
|
distance[it->u] = distance[it->v] + it->edit.cost;
|
|
predecessor[it->u] = it->v;
|
|
}
|
|
}
|
|
}
|
|
|
|
Edges newE;
|
|
|
|
Vertex v = *V.rbegin();
|
|
while(true) {
|
|
//std::cout << predecessor[v] << " -> " << v << std::endl;
|
|
Edges::iterator it = E.find(Edge(predecessor[v], v));
|
|
if(it != E.end()) {
|
|
Edge f = *it;
|
|
//std::cout << f << std::endl;
|
|
newE.insert(f);
|
|
|
|
v = predecessor[v];
|
|
if(v == source)
|
|
break;
|
|
} else {
|
|
std::cout << "Error" << std::endl;
|
|
break;
|
|
}
|
|
}
|
|
E.clear();
|
|
E.insert(newE.begin(), newE.end());
|
|
}
|
|
|
|
void AddStats(const std::vector<Edges> &Es, const Unit &u, Stats &stats, size_t line) {
|
|
|
|
std::map<size_t, Stats> statsPerAnnotator;
|
|
for(std::set<size_t>::iterator it = u.third.begin();
|
|
it != u.third.end(); ++it) {
|
|
statsPerAnnotator[*it] = Stats(4, 0);
|
|
}
|
|
|
|
for(Annots::iterator it = u.second.begin(); it != u.second.end(); it++)
|
|
statsPerAnnotator[it->annotator][2]++;
|
|
|
|
for(std::set<size_t>::iterator ait = u.third.begin();
|
|
ait != u.third.end(); ++ait) {
|
|
for(Edges::iterator eit = Es[*ait].begin(); eit != Es[*ait].end(); ++eit) {
|
|
if(eit->edit.changed > 0) {
|
|
statsPerAnnotator[*ait][1]++;
|
|
Annot f;
|
|
f.i = eit->v.i;
|
|
f.j = eit->u.i;
|
|
f.annotator = *ait;
|
|
f.edit = eit->edit.edit;
|
|
for(Annots::iterator fit = u.second.begin(); fit != u.second.end(); fit++) {
|
|
if(f == *fit)
|
|
statsPerAnnotator[*ait][0]++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
size_t bestAnnot = 0;
|
|
float bestF = -1;
|
|
for(std::set<size_t>::iterator it = u.third.begin();
|
|
it != u.third.end(); ++it) {
|
|
Stats localStats = stats;
|
|
localStats[0] += statsPerAnnotator[*it][0];
|
|
localStats[1] += statsPerAnnotator[*it][1];
|
|
localStats[2] += statsPerAnnotator[*it][2];
|
|
if(m_verbose)
|
|
std::cerr << *it << " : " << localStats[0] << " " << localStats[1] << " " << localStats[2] << std::endl;
|
|
float f = FScore(localStats);
|
|
if(m_verbose)
|
|
std::cerr << f << std::endl;
|
|
if(f > bestF) {
|
|
bestF = f;
|
|
bestAnnot = *it;
|
|
}
|
|
}
|
|
if(m_verbose)
|
|
std::cerr << ">> Chosen Annotator for line " << line + 1 << " : " << bestAnnot << std::endl;
|
|
stats[0] += statsPerAnnotator[bestAnnot][0];
|
|
stats[1] += statsPerAnnotator[bestAnnot][1];
|
|
stats[2] += statsPerAnnotator[bestAnnot][2];
|
|
}
|
|
|
|
void SufStats(const std::string &sStr, size_t i, Stats &stats) {
|
|
std::string temp = sStr;
|
|
|
|
Sentence s;
|
|
boost::split(s, temp, boost::is_any_of(" "), boost::token_compress_on);
|
|
|
|
Unit &unit = m_m2[i];
|
|
|
|
Matrix d;
|
|
TrackMatrix bt;
|
|
size_t distance = LevenshteinMatrix(unit.first, s, d, bt);
|
|
|
|
std::vector<Vertices> Vs(unit.third.size());
|
|
std::vector<Edges> Es(unit.third.size());
|
|
|
|
if(distance > unit.first.size()) {
|
|
std::cerr << "Levenshtein distance is greater than source size." << std::endl;
|
|
stats[0] = 0;
|
|
stats[1] = distance;
|
|
stats[2] = 0;
|
|
stats[3] = unit.first.size();
|
|
return;
|
|
} else if(distance > 0) {
|
|
for(size_t j = 0; j < unit.third.size(); j++) {
|
|
BuildGraph(bt, Vs[j], Es[j]);
|
|
AddWeights(Es[j], unit, j);
|
|
BellmanFord(Vs[j], Es[j]);
|
|
}
|
|
}
|
|
AddStats(Es, unit, stats, i);
|
|
stats[3] = unit.first.size();
|
|
}
|
|
|
|
|
|
float FScore(const Stats& stats) {
|
|
float p = 1.0;
|
|
if(stats[1] != 0)
|
|
p = (float)stats[0] / (float)stats[1];
|
|
|
|
float r = 1.0;
|
|
if(stats[2] != 0)
|
|
r = (float)stats[0] / (float)stats[2];
|
|
|
|
float denom = (m_beta * m_beta * p + r);
|
|
float f = 0.0;
|
|
if(denom != 0)
|
|
f = ((1 + m_beta * m_beta) * p * r) / denom;
|
|
return f;
|
|
}
|
|
|
|
void FScore(const Stats& stats, float &p, float &r, float &f) {
|
|
p = 1.0;
|
|
if(stats[1] != 0)
|
|
p = (float)stats[0] / (float)stats[1];
|
|
|
|
r = 1.0;
|
|
if(stats[2] != 0)
|
|
r = (float)stats[0] / (float)stats[2];
|
|
|
|
float denom = (m_beta * m_beta * p + r);
|
|
f = 0.0;
|
|
if(denom != 0)
|
|
f = ((1 + m_beta * m_beta) * p * r) / denom;
|
|
}
|
|
};
|
|
|
|
}
|
|
|
|
} |