2011-07-01 09:40:46 +04:00
|
|
|
#include <iostream>
|
|
|
|
#include <fstream>
|
|
|
|
#include <cassert>
|
|
|
|
#include <vector>
|
|
|
|
#include "extract-lex.h"
|
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
float COUNT_INCR = 1;
|
|
|
|
|
2011-07-01 09:40:46 +04:00
|
|
|
int main(int argc, char* argv[])
|
|
|
|
{
|
|
|
|
cerr << "Starting...\n";
|
|
|
|
|
|
|
|
char* &filePathTarget = argv[1];
|
|
|
|
char* &filePathSource = argv[2];
|
|
|
|
char* &filePathAlign = argv[3];
|
|
|
|
char* &filePathLexS2T = argv[4];
|
|
|
|
char* &filePathLexT2S = argv[5];
|
|
|
|
|
|
|
|
ifstream streamTarget;
|
|
|
|
ifstream streamSource;
|
|
|
|
ifstream streamAlign;
|
|
|
|
streamTarget.open(filePathTarget);
|
|
|
|
streamSource.open(filePathSource);
|
|
|
|
streamAlign.open(filePathAlign);
|
|
|
|
|
|
|
|
ofstream streamLexS2T;
|
|
|
|
ofstream streamLexT2S;
|
|
|
|
streamLexS2T.open(filePathLexS2T);
|
|
|
|
streamLexT2S.open(filePathLexT2S);
|
|
|
|
|
|
|
|
ExtractLex extractSingleton;
|
|
|
|
|
2011-07-01 14:33:04 +04:00
|
|
|
size_t lineCount = 0;
|
2011-07-01 09:40:46 +04:00
|
|
|
string lineTarget, lineSource, lineAlign;
|
|
|
|
while (getline(streamTarget, lineTarget))
|
|
|
|
{
|
2011-07-01 14:33:04 +04:00
|
|
|
if (lineCount % 10000 == 0)
|
|
|
|
cerr << lineCount << " ";
|
|
|
|
|
2011-07-01 09:40:46 +04:00
|
|
|
istream &isSource = getline(streamSource, lineSource);
|
|
|
|
assert(isSource);
|
|
|
|
istream &isAlign = getline(streamAlign, lineAlign);
|
|
|
|
assert(isAlign);
|
|
|
|
|
|
|
|
vector<string> toksTarget, toksSource, toksAlign;
|
|
|
|
Tokenize(toksTarget, lineTarget);
|
|
|
|
Tokenize(toksSource, lineSource);
|
|
|
|
Tokenize(toksAlign, lineAlign);
|
|
|
|
|
2011-07-01 14:33:04 +04:00
|
|
|
/*
|
2011-07-01 09:40:46 +04:00
|
|
|
cerr << endl
|
|
|
|
<< toksTarget.size() << " " << lineTarget << endl
|
|
|
|
<< toksSource.size() << " " << lineSource << endl
|
|
|
|
<< toksAlign.size() << " " << lineAlign << endl;
|
2011-07-01 14:33:04 +04:00
|
|
|
*/
|
2011-07-01 09:40:46 +04:00
|
|
|
|
|
|
|
extractSingleton.Process(toksTarget, toksSource, toksAlign);
|
2011-07-01 14:33:04 +04:00
|
|
|
|
|
|
|
++lineCount;
|
2011-07-01 09:40:46 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
extractSingleton.Output(streamLexS2T, streamLexT2S);
|
|
|
|
|
|
|
|
streamLexS2T.close();
|
|
|
|
streamLexT2S.close();
|
|
|
|
|
2011-07-01 14:33:04 +04:00
|
|
|
cerr << "\nFinished\n";
|
2011-07-01 09:40:46 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
const std::string *Vocab::GetOrAdd(const std::string &word)
|
|
|
|
{
|
|
|
|
const string *ret = &(*m_coll.insert(word).first);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ExtractLex::Process(vector<string> &toksTarget, vector<string> &toksSource, vector<string> &toksAlign)
|
|
|
|
{
|
|
|
|
vector<string>::const_iterator iterAlign;
|
|
|
|
for (iterAlign = toksAlign.begin(); iterAlign != toksAlign.end(); ++iterAlign)
|
|
|
|
{
|
|
|
|
const string &alignTok = *iterAlign;
|
|
|
|
|
|
|
|
vector<size_t> alignPos;
|
|
|
|
Tokenize(alignPos, alignTok, "-");
|
|
|
|
assert(alignPos.size() == 2);
|
|
|
|
assert(alignPos[0] < toksSource.size());
|
|
|
|
assert(alignPos[1] < toksTarget.size());
|
|
|
|
|
|
|
|
const string &tmpSource = toksSource[ alignPos[0] ];
|
|
|
|
const string &tmpTarget = toksTarget[ alignPos[1] ];
|
|
|
|
|
|
|
|
const string *source = m_vocab.GetOrAdd(tmpSource);
|
|
|
|
const string *target = m_vocab.GetOrAdd(tmpTarget);
|
|
|
|
|
|
|
|
Process(target, source);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
void ExtractLex::Process(const std::string *target, const std::string *source)
|
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
WordCount &wcS2T = m_collS2T[source];
|
|
|
|
WordCount &wcT2S = m_collT2S[target];
|
2011-07-01 09:40:46 +04:00
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
wcS2T.AddCount(COUNT_INCR);
|
|
|
|
wcT2S.AddCount(COUNT_INCR);
|
|
|
|
|
|
|
|
Process(wcS2T, target);
|
|
|
|
Process(wcT2S, source);
|
2011-07-01 09:40:46 +04:00
|
|
|
}
|
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
void ExtractLex::Process(WordCount &wcIn, const std::string *out)
|
2011-07-01 09:40:46 +04:00
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
std::map<const std::string*, WordCount> &collOut = wcIn.GetColl();
|
|
|
|
WordCount &wcOut = collOut[out];
|
|
|
|
wcOut.AddCount(COUNT_INCR);
|
2011-07-01 09:40:46 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void ExtractLex::Output(std::ofstream &streamLexS2T, std::ofstream &streamLexT2S)
|
|
|
|
{
|
|
|
|
Output(m_collS2T, streamLexS2T);
|
|
|
|
Output(m_collT2S, streamLexT2S);
|
|
|
|
}
|
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
void ExtractLex::Output(const std::map<const std::string*, WordCount> &coll, std::ofstream &outStream)
|
2011-07-01 09:40:46 +04:00
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
std::map<const std::string*, WordCount>::const_iterator iterOuter;
|
2011-07-01 09:40:46 +04:00
|
|
|
for (iterOuter = coll.begin(); iterOuter != coll.end(); ++iterOuter)
|
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
const string &inStr = *iterOuter->first;
|
|
|
|
const WordCount &inWC = iterOuter->second;
|
|
|
|
|
|
|
|
const std::map<const std::string*, WordCount> &outColl = inWC.GetColl();
|
2011-07-01 09:40:46 +04:00
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
std::map<const std::string*, WordCount>::const_iterator iterInner;
|
2011-07-01 09:40:46 +04:00
|
|
|
for (iterInner = outColl.begin(); iterInner != outColl.end(); ++iterInner)
|
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
const string &outStr = *iterInner->first;
|
|
|
|
const WordCount &outWC = iterInner->second;
|
|
|
|
|
|
|
|
float prob = outWC.GetCount() / inWC.GetCount();
|
2011-07-01 14:33:04 +04:00
|
|
|
outStream << inStr << " " << outStr << " " << prob << endl;
|
2011-07-01 09:40:46 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::ostream& operator<<(std::ostream &out, const WordCount &obj)
|
|
|
|
{
|
2011-07-01 14:22:55 +04:00
|
|
|
out << "(" << obj.GetCount() << ")";
|
2011-07-01 09:40:46 +04:00
|
|
|
return out;
|
|
|
|
}
|
|
|
|
|
2011-07-01 14:22:55 +04:00
|
|
|
void WordCount::AddCount(float incr)
|
2011-07-01 09:40:46 +04:00
|
|
|
{
|
|
|
|
m_count += incr;
|
|
|
|
}
|
|
|
|
|
|
|
|
|