mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-27 03:49:57 +03:00
173 lines
4.4 KiB
C++
173 lines
4.4 KiB
C++
|
#include <iostream>
|
||
|
#include <fstream>
|
||
|
#include <cassert>
|
||
|
#include <vector>
|
||
|
#include "extract-lex.h"
|
||
|
|
||
|
using namespace std;
|
||
|
|
||
|
int main(int argc, char* argv[])
|
||
|
{
|
||
|
cerr << "Starting...\n";
|
||
|
|
||
|
char* &filePathTarget = argv[1];
|
||
|
char* &filePathSource = argv[2];
|
||
|
char* &filePathAlign = argv[3];
|
||
|
char* &filePathLexS2T = argv[4];
|
||
|
char* &filePathLexT2S = argv[5];
|
||
|
|
||
|
ifstream streamTarget;
|
||
|
ifstream streamSource;
|
||
|
ifstream streamAlign;
|
||
|
streamTarget.open(filePathTarget);
|
||
|
streamSource.open(filePathSource);
|
||
|
streamAlign.open(filePathAlign);
|
||
|
|
||
|
ofstream streamLexS2T;
|
||
|
ofstream streamLexT2S;
|
||
|
streamLexS2T.open(filePathLexS2T);
|
||
|
streamLexT2S.open(filePathLexT2S);
|
||
|
|
||
|
ExtractLex extractSingleton;
|
||
|
|
||
|
string lineTarget, lineSource, lineAlign;
|
||
|
while (getline(streamTarget, lineTarget))
|
||
|
{
|
||
|
istream &isSource = getline(streamSource, lineSource);
|
||
|
assert(isSource);
|
||
|
istream &isAlign = getline(streamAlign, lineAlign);
|
||
|
assert(isAlign);
|
||
|
|
||
|
vector<string> toksTarget, toksSource, toksAlign;
|
||
|
Tokenize(toksTarget, lineTarget);
|
||
|
Tokenize(toksSource, lineSource);
|
||
|
Tokenize(toksAlign, lineAlign);
|
||
|
|
||
|
cerr << endl
|
||
|
<< toksTarget.size() << " " << lineTarget << endl
|
||
|
<< toksSource.size() << " " << lineSource << endl
|
||
|
<< toksAlign.size() << " " << lineAlign << endl;
|
||
|
|
||
|
extractSingleton.Process(toksTarget, toksSource, toksAlign);
|
||
|
|
||
|
}
|
||
|
|
||
|
extractSingleton.Output(streamLexS2T, streamLexT2S);
|
||
|
|
||
|
streamLexS2T.close();
|
||
|
streamLexT2S.close();
|
||
|
|
||
|
cerr << "Finished\n";
|
||
|
}
|
||
|
|
||
|
const std::string *Vocab::GetOrAdd(const std::string &word)
|
||
|
{
|
||
|
const string *ret = &(*m_coll.insert(word).first);
|
||
|
return ret;
|
||
|
}
|
||
|
|
||
|
void ExtractLex::Process(vector<string> &toksTarget, vector<string> &toksSource, vector<string> &toksAlign)
|
||
|
{
|
||
|
vector<string>::const_iterator iterAlign;
|
||
|
for (iterAlign = toksAlign.begin(); iterAlign != toksAlign.end(); ++iterAlign)
|
||
|
{
|
||
|
const string &alignTok = *iterAlign;
|
||
|
|
||
|
vector<size_t> alignPos;
|
||
|
Tokenize(alignPos, alignTok, "-");
|
||
|
assert(alignPos.size() == 2);
|
||
|
assert(alignPos[0] < toksSource.size());
|
||
|
assert(alignPos[1] < toksTarget.size());
|
||
|
|
||
|
const string &tmpSource = toksSource[ alignPos[0] ];
|
||
|
const string &tmpTarget = toksTarget[ alignPos[1] ];
|
||
|
|
||
|
const string *source = m_vocab.GetOrAdd(tmpSource);
|
||
|
const string *target = m_vocab.GetOrAdd(tmpTarget);
|
||
|
|
||
|
Process(target, source);
|
||
|
|
||
|
}
|
||
|
|
||
|
}
|
||
|
|
||
|
float COUNT_INCR = 1;
|
||
|
|
||
|
void ExtractLex::Process(const std::string *target, const std::string *source)
|
||
|
{
|
||
|
WordCount tmpWCTarget(target, COUNT_INCR);
|
||
|
WordCount tmpWCSource(source, COUNT_INCR);
|
||
|
|
||
|
Process(tmpWCSource, tmpWCTarget, m_collS2T);
|
||
|
Process(tmpWCTarget, tmpWCSource, m_collT2S);
|
||
|
}
|
||
|
|
||
|
void ExtractLex::Process(const WordCount &in, const WordCount &out, std::map<WordCount, WordCountColl> &coll)
|
||
|
{
|
||
|
std::map<WordCount, WordCountColl>::iterator iterMap;
|
||
|
// s2t
|
||
|
WordCountColl *wcColl = NULL;
|
||
|
iterMap = coll.find(in);
|
||
|
if (iterMap == coll.end())
|
||
|
{
|
||
|
wcColl = &coll[in];
|
||
|
}
|
||
|
else
|
||
|
{
|
||
|
const WordCount &wcIn = iterMap->first;
|
||
|
|
||
|
//cerr << wcIn << endl;
|
||
|
wcIn.AddCount(COUNT_INCR);
|
||
|
//cerr << wcIn << endl;
|
||
|
|
||
|
wcColl = &iterMap->second;
|
||
|
}
|
||
|
|
||
|
assert(in.GetCount() == COUNT_INCR);
|
||
|
assert(out.GetCount() == COUNT_INCR);
|
||
|
assert(wcColl);
|
||
|
|
||
|
pair<WordCountColl::iterator, bool> iterSet = wcColl->insert(out);
|
||
|
const WordCount &outWC = *iterSet.first;
|
||
|
outWC.AddCount(COUNT_INCR);
|
||
|
}
|
||
|
|
||
|
void ExtractLex::Output(std::ofstream &streamLexS2T, std::ofstream &streamLexT2S)
|
||
|
{
|
||
|
Output(m_collS2T, streamLexS2T);
|
||
|
Output(m_collT2S, streamLexT2S);
|
||
|
}
|
||
|
|
||
|
void ExtractLex::Output(const std::map<WordCount, WordCountColl> &coll, std::ofstream &outStream)
|
||
|
{
|
||
|
std::map<WordCount, WordCountColl>::const_iterator iterOuter;
|
||
|
for (iterOuter = coll.begin(); iterOuter != coll.end(); ++iterOuter)
|
||
|
{
|
||
|
const WordCount &in = iterOuter->first;
|
||
|
const WordCountColl &outColl = iterOuter->second;
|
||
|
|
||
|
WordCountColl::const_iterator iterInner;
|
||
|
for (iterInner = outColl.begin(); iterInner != outColl.end(); ++iterInner)
|
||
|
{
|
||
|
const WordCount &out = *iterInner;
|
||
|
outStream << in.GetString() << " " << out.GetString()
|
||
|
<< " " << in.GetCount() << " " << out.GetCount()
|
||
|
<< endl;
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
std::ostream& operator<<(std::ostream &out, const WordCount &obj)
|
||
|
{
|
||
|
out << obj.GetString() << "(" << obj.GetCount() << ")";
|
||
|
return out;
|
||
|
}
|
||
|
|
||
|
void WordCount::AddCount(float incr) const
|
||
|
{
|
||
|
m_count += incr;
|
||
|
cerr << *this << endl;
|
||
|
}
|
||
|
|
||
|
|