mosesdecoder/misc/TransliterationMining.cpp
2013-12-16 18:19:44 +00:00

649 lines
15 KiB
C++

/*
########################################################################################
Transliteration Mining - A Program to Extract Transliteration Pairs from
a bilingual word list
Source Contributor: Nadir Durrani
########################################################################################
*/
#include <cstdlib>
#include <map>
#include <set>
#include <cmath>
#include <fstream>
#include <iostream>
#include <vector>
#include <sstream>
using namespace std;
double initTransitionProb;
double LAMBDA;
double addLogProbs(double A , double B) // this function adds probabilities ...
{
if (A == B)
return (A + log10(2.0));
if (A > B)
{
if (A - B > 6) // A is a lot bigger ...
return A;
else
return (A + log10(1+pow(10,(B-A))));
}
else // B > A
{
if (B - A > 6)
return B;
else
return (B + log10(1+pow(10,(A-B))));
}
}
class NodeStructure
{
public:
NodeStructure(){};
NodeStructure(vector <string> & s , vector <string> & t);
double getPosterior(){return PPR;}
void computeFwdBckProbs(map <string , double> & gammas, map <string, double> & alignmentCounts);
void computeNonTransliterationProb (map <string , double> & sourceUnigrams , map <string , double> & targetUnigrams);
void print();
vector <string> source;
vector <string> target;
~NodeStructure(){};
private:
double NTR; // Non-transliteration probability of a sentence pair ...
double PPR; // Posterior Probability ...
double ALPHA;
double BETA;
void computeGammaForEdges(map < pair <int , int> , double > & parents, map < pair <int , int> , double > & children , map <string, double> & transitionProbs , map <string, double> & alignmentCounts);
double computeFwdProbs(pair <int , int> & ST, map <string , double> & gammas, map < pair <int , int> , double > & parents);
double FwdProb (pair <int , int> & TS, map <string , double> & gammas, map < pair <int , int> , double > & parents);
double BckProb (pair <int , int> & TS, map <string , double> & gammas, map < pair <int , int> , double > & chidren);
double computeBckProbs(pair <int , int> & ST, map <string , double> & gammas, map < pair <int , int> , double > & children);
void getIncomingEdges (pair <int , int> & ST , vector < pair < int , int> > & incomingEdges);
void getOutgoingEdges (pair <int , int> & ST , vector < pair < int , int> > & outgoingEdges);
double getTransitionProb(map <string, double> & transitionProbs , pair <int,int> & edge);
void updateAlignmentCount(map <string, double> & transitionProbs, map <string, double> & alignmentCounts , pair <int,int> & edge , double alpha , double beta);
void computePosteriorProb();
double scaleGamma(double g);
void getEdge (pair <int , int> & v1 , pair <int , int> & v2 , pair <int , int> & v3);
};
void NodeStructure :: print()
{
for (int i = 0; i < source.size(); i++)
cout<<source[i];
cout<<"\t";
for (int i = 0; i < target.size(); i++)
cout<<target[i];
cout<<"\t"<<pow(10,PPR)<<endl;
}
NodeStructure :: NodeStructure(vector <string> & s , vector <string> & t)
{
source = s;
target = t;
}
void NodeStructure :: getEdge (pair <int , int> & v1 , pair <int , int> & v2 , pair <int , int> & v3)
{
if (v2.first - v1.first == 0)
v3.first = -1;
else
v3.first = v2.first;
if (v2.second - v1.second == 0)
v3.second = -1;
else
v3.second = v2.second;
}
void NodeStructure :: computeGammaForEdges(map < pair <int , int> , double > & parents, map < pair <int , int> , double > & children , map <string, double> & transitionProbs , map <string, double> & alignmentCounts)
{
vector < pair < int , int> > incomingEdges;
map < pair <int , int> , double > :: iterator cIter;
map < pair <int , int> , double > :: iterator pIter;
pair <int , int> ST = make_pair (-1,-1);
pair <int , int> edge;
children.erase(ST);
double tProb;
double alpha;
double beta;
for (cIter = children.begin(); cIter != children.end(); cIter++)
{
ST = cIter->first;
getIncomingEdges (ST , incomingEdges);
beta = cIter->second;
for (int i = 0; i< incomingEdges.size(); i++)
{
pIter = parents.find(incomingEdges[i]);
alpha = pIter->second;
getEdge (incomingEdges[i] , ST , edge);
updateAlignmentCount(transitionProbs, alignmentCounts , edge , alpha , beta);
}
}
}
void NodeStructure :: computeNonTransliterationProb (map <string , double> & sourceUnigrams , map <string , double> & targetUnigrams)
{
NTR = 0.0;
for (int i = 0; i < source.size(); i++)
{
NTR += sourceUnigrams[source[i]];
}
for (int i = 0; i < target.size(); i++)
{
NTR += targetUnigrams[target[i]];
}
}
double NodeStructure :: scaleGamma(double g)
{
double translit = log10 (1 - pow (10, PPR));
return g + translit;
}
void NodeStructure :: computePosteriorProb()
{
double LAMBDA2 = log10(1 - pow(10, LAMBDA));
double transliterate = LAMBDA2 + ALPHA; // Transliteration Prob ...
double translate = LAMBDA + NTR; // Translation Prob ...
double trans = transliterate - translate;
//cout<<LAMBDA<<" "<<LAMBDA2<<endl;
//cout<<transliterate<<" "<<translate<<" "<<trans<<endl;
//cout<<pow(10 , trans)<<endl;
double prob = 1/(1+ pow(10 , trans));
PPR = log10(prob);
//cout<<"Posterior Prob "<<PPR<<endl;
}
void NodeStructure :: computeFwdBckProbs(map <string , double> & gammas , map <string, double> & alignmentCounts)
{
pair <int , int> START = make_pair (source.size()-1 , target.size()-1);
pair <int , int> END = make_pair (-1 , -1);
map < pair <int , int> , double > parents;
parents[make_pair(-1,-1)] = 0.0;
map < pair <int , int> , double > children;
children[make_pair(source.size()-1,target.size()-1)] = 0.0;
ALPHA = computeFwdProbs(START , gammas, parents);
BETA = computeBckProbs(END , gammas, children);
computePosteriorProb();
//cout<<"Alpha "<<ALPHA<<" Beta "<<BETA<<endl;
computeGammaForEdges(parents , children , gammas , alignmentCounts);
}
void NodeStructure :: getIncomingEdges (pair <int , int> & ST , vector < pair < int , int> > & incomingEdges)
{
incomingEdges.clear();
if (ST.first == -1) // Source is NULL ..
{
incomingEdges.push_back(make_pair(ST.first , ST.second-1));
}
else if (ST.second == -1) // Target is NULL ...
{
incomingEdges.push_back(make_pair(ST.first-1 , ST.second));
}
else
{
incomingEdges.push_back(make_pair(ST.first , ST.second-1));
incomingEdges.push_back(make_pair(ST.first-1 , ST.second));
incomingEdges.push_back(make_pair(ST.first-1 , ST.second-1));
}
}
void NodeStructure :: getOutgoingEdges (pair <int , int> & ST , vector < pair < int , int> > & outgoingEdges)
{
if (ST.first == source.size()-1) // Source is END ..
{
outgoingEdges.push_back(make_pair(ST.first , ST.second+1));
}
else if (ST.second == target.size()-1) // Target is END ...
{
outgoingEdges.push_back(make_pair(ST.first+1 , ST.second));
}
else
{
outgoingEdges.push_back(make_pair(ST.first , ST.second+1));
outgoingEdges.push_back(make_pair(ST.first+1 , ST.second));
outgoingEdges.push_back(make_pair(ST.first+1 , ST.second+1));
}
}
void NodeStructure :: updateAlignmentCount(map <string, double> & transitionProbs, map <string, double> & alignmentCounts , pair <int,int> & edge , double alpha , double beta)
{
double tProb;
double tgamma;
double gamma;
map <string , double> :: iterator aCounts;
string query;
if (edge.first == -1)
query = "NULL";
else
query = source[edge.first];
query += "-";
if (edge.second == -1)
query += "NULL";
else
query += target[edge.second];
//cout<<" Query "<<query<<endl;
if (transitionProbs.size() == 0)
tProb = initTransitionProb;
else
tProb = transitionProbs[query];
tgamma = alpha + tProb + beta - ALPHA;
gamma = scaleGamma(tgamma);
//cout<<alpha<<" "<<beta<<" "<<gamma<<endl;
//cout<<tProb<<" "<<ALPHA<<endl;
aCounts = alignmentCounts.find(query);
if (aCounts == alignmentCounts.end())
{
alignmentCounts[query] = gamma;
}
else
{
double temp = aCounts->second;
aCounts->second = addLogProbs(temp , gamma);
}
}
double NodeStructure :: getTransitionProb(map <string, double> & transitionProbs , pair <int,int> & edge)
{
if (transitionProbs.size() == 0)
return initTransitionProb;
string query;
if (edge.first == -1)
query = "NULL";
else
query = source[edge.first];
query += "-";
if (edge.second == -1)
query += "NULL";
else
query += target[edge.second];
//cout<<" Query "<<query<<endl;
return transitionProbs[query];
}
double NodeStructure :: FwdProb (pair <int , int> & TS, map <string , double> & gammas, map < pair <int , int> , double > & parents)
{
double thisAlpha;
double alpha = -2000;
vector < pair < int , int> > incomingEdges;
pair <int , int> edge;
getIncomingEdges (TS , incomingEdges);
for (int k = 0; k < incomingEdges.size(); k++)
{
thisAlpha = parents[incomingEdges[k]];
getEdge (incomingEdges[k], TS , edge);
thisAlpha += getTransitionProb(gammas , edge); // Get Transition Prob ...
double temp = alpha;
alpha = addLogProbs(temp , thisAlpha); // Sum of all parents * transition prob ..
// cout<<temp<<"+"<<thisAlpha<<"="<<alpha<<endl;
}
return alpha;
}
double NodeStructure :: computeFwdProbs(pair <int , int> & ST, map <string , double> & gammas, map < pair <int , int> , double > & parents)
{
pair <int , int> TS;
double alpha;
for (int i = 0; i < source.size(); i++)
{
TS = make_pair (i , -1);
alpha = FwdProb (TS, gammas, parents);
parents[TS] = alpha;
}
for (int i = 0; i < target.size(); i++)
{
TS = make_pair (-1 , i);
alpha = FwdProb (TS, gammas, parents);
parents[TS] = alpha;
}
for (int i = 0; i < source.size(); i++)
{
for (int j = 0; j < target.size(); j++)
{
TS = make_pair (i , j);
alpha = FwdProb (TS, gammas, parents);
parents[TS] = alpha;
}
}
return parents[ST];
}
double NodeStructure :: BckProb (pair <int , int> & TS, map <string , double> & gammas, map < pair <int , int> , double > & children)
{
double thisBeta;
double beta = -2000;
vector < pair < int , int> > outgoingEdges;
pair <int , int> edge;
getOutgoingEdges (TS , outgoingEdges);
for (int k = 0; k < outgoingEdges.size(); k++)
{
thisBeta = children[outgoingEdges[k]];
getEdge (TS , outgoingEdges[k], edge);
thisBeta += getTransitionProb(gammas , edge); // Get Transition Prob ...
double temp = beta;
beta = addLogProbs(temp , thisBeta); // Sum of all parents * transition prob ..
// cout<<temp<<"+"<<thisAlpha<<"="<<alpha<<endl;
}
return beta;
}
double NodeStructure :: computeBckProbs(pair <int , int> & ST, map <string , double> & gammas, map < pair <int , int> , double > & children)
{
pair <int , int> TS;
double beta;
for (int i = source.size()-2; i >= -1; i--)
{
TS = make_pair (i , target.size()-1);
beta = BckProb (TS, gammas, children);
children[TS] = beta;
}
for (int i = target.size()-2; i >=-1; i--)
{
TS = make_pair (source.size()-1 , i);
beta = BckProb (TS, gammas, children);
children[TS] = beta;
}
for (int i = source.size()-2 ; i >= -1 ; i--)
{
for (int j = target.size()-2 ; j >= -1; j--)
{
TS = make_pair (i , j);
beta = BckProb (TS, gammas, children);
children[TS] = beta;
}
}
return children[ST];
}
void loadInput(const char * fileName, vector <string> & input)
{
/* This function loads a file into a vector of strings */
ifstream sr (fileName);
string line;
if(sr.is_open())
{
while(getline(sr , line ))
{
input.push_back(line);
}
sr.close();
}
else
{
cout<<"Unable to read "<<fileName<<endl;
exit(1);
}
}
void printGammas(map <string, double> & alignmentCounts)
{
map <string , double> :: iterator aCounts;
for (aCounts = alignmentCounts.begin(); aCounts != alignmentCounts.end(); aCounts++)
{
cout<<aCounts->first<<" "<<aCounts->second<<endl;
}
}
void getWords(string s, vector <string> & currInput)
{
/* This function splits a string into vector of strings using space character as a delimiter */
istringstream iss(s);
currInput.clear();
do
{
string sub;
iss >> sub;
currInput.push_back(sub);
} while (iss);
currInput.pop_back();
}
double getInitTransitionProb(int sourceToken, int targetToken)
{
double prod = sourceToken * targetToken;
return log10(1/prod);
}
void runIteration(map <int , NodeStructure> & graph , map <string , double> & gammas , int size)
{
map <string, double> alignmentCounts;
map <int , NodeStructure> :: iterator i;
map <string , double> :: iterator aCounts;
double sum = -2000.0;
double tPPR = -2000.0;
for (i = graph.begin(); i != graph.end(); i++)
{
i->second.computeFwdBckProbs(gammas , alignmentCounts);
double temp = tPPR;
tPPR = addLogProbs(graph[i->first].getPosterior() , temp);
}
for (aCounts = alignmentCounts.begin(); aCounts != alignmentCounts.end(); aCounts++)
{
double temp = sum;
sum = addLogProbs(aCounts->second, temp);
}
for (aCounts = alignmentCounts.begin(); aCounts != alignmentCounts.end(); aCounts++) // Normalizing ...
{
aCounts->second = aCounts->second - sum;
}
gammas.clear();
gammas = alignmentCounts;
LAMBDA = tPPR - log10(size);
}
void setNTRProbabilities(map <int , NodeStructure> & graph , map <string , double> & sourceTypes , map <string , double > & targetTypes, double sourceTokens, double targetTokens)
{
map <string , double> :: iterator i;
map <int , NodeStructure> :: iterator j;
for (i = sourceTypes.begin(); i!= sourceTypes.end(); i++)
{
i->second = log10(i->second/sourceTokens);
}
for (i = targetTypes.begin(); i!= targetTypes.end(); i++)
{
i->second = log10(i->second/targetTokens);
}
for (j = graph.begin(); j != graph.end(); j++)
{
j->second.computeNonTransliterationProb(sourceTypes , targetTypes);
}
}
void printPosterior(map <int , NodeStructure> & graph)
{
map <int , NodeStructure> :: iterator i;
for (i = graph.begin(); i != graph.end(); i++)
graph[i->first].print();
}
int main(int argc, char * argv[])
{
vector <string> input;
vector <string> source;
vector <string> target;
map <string , double> sourceTypes;
map <string , double> targetTypes;
set < vector <string> > tgt;
set < vector <string> > src;
double sourceTokens = 0;
double targetTokens = 0;
map <int , NodeStructure> graph;
map <string , double> gammas;
loadInput(argv[1],input);
cerr<<"Constructing Graph "<<endl;
for(int i=0; i<input.size(); i+=2)
{
//cerr<<input[i]<<endl;
//cerr<<input[i+1]<<endl;
getWords(input[i],source);
getWords(input[i+1],target);
if (src.find(source) == src.end())
{
for (int j = 0; j< source.size(); j++)
sourceTypes[source[j]]++;
src.insert(source);
sourceTokens += source.size();
}
if (tgt.find(target) == tgt.end())
{
for (int j = 0; j< target.size(); j++)
targetTypes[target[j]]++;
tgt.insert(target);
targetTokens += target.size();
}
NodeStructure obj (source,target);
graph[i] = obj;
}
setNTRProbabilities(graph, sourceTypes, targetTypes, sourceTokens, targetTokens);
initTransitionProb = getInitTransitionProb(sourceTypes.size()+1, targetTypes.size()+1);
LAMBDA = log10(0.5);
for (int i = 0; i< 10; i++)
{
cerr<<"Computing Probs : iteration "<<i+1<<endl;
runIteration(graph , gammas , input.size()/2);
}
printPosterior(graph);
cerr<<"Finished..."<<endl;
return 0;
}