moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-20 15:48:05 +03:00
Code Issues Packages Projects Releases Wiki Activity
b1c702afab
mosesdecoder/contrib/eppex/phrase-extract.cpp
Barry Haddow ea49a01a80 Move from scripts to contrib
2011-11-23 15:31:16 +00:00

806 lines
30 KiB
C++
Raw Blame History

/**
* Common lossy counting phrase extraction functionality implementation.
*
* Note: The bulk of this unit is based on Philipp Koehn's code from
* phrase-extract/extract.cpp.
*
* (C) Moses: http://www.statmt.org/moses/
* (C) Ceslav Przywara, UFAL MFF UK, 2011
*
* $Id$
*/
#include <iostream>
#include <iomanip>
#include <sstream>
#include "phrase-extract.h"
#include "ISS.h"
// I'm using my own version of SafeGetline (without "using namespace std;"):
#include "SafeGetline.h"
#define LINE_MAX_LENGTH 60000
//////// Helping functions ////////
// For sorted output.
typedef std::pair<indexed_phrases_pair_t, PhrasePairsLossyCounter::frequency_t> output_pair_t;
typedef std::vector<output_pair_t> output_vector_t;
class PhraseComp {
/** @var If true, sort by target phrase first. */
bool _inverted;
bool compareAlignments(const indexed_phrases_pair_t& a, const indexed_phrases_pair_t& b);
int comparePhrases(const indexed_phrases_pair_t::phrase_t& a, const indexed_phrases_pair_t::phrase_t& b);
public:
PhraseComp(bool inverted): _inverted(inverted) {}
bool operator()(const output_pair_t& a, const output_pair_t& b);
};
void processSortedOutput(OutputProcessor& processor);
void processUnsortedOutput(OutputProcessor& processor);
void flushPhrasePair(OutputProcessor& processor, const indexed_phrases_pair_t& indexedPhrasePair, PhrasePairsLossyCounter::frequency_t frequency, int mode);
//////// Define variables declared as extern in the header /////////////////////
bool allModelsOutputFlag = false;
bool wordModel = false; // IBM word model.
REO_MODEL_TYPE wordType = REO_MSD;
bool phraseModel = false; // Std phrase-based model.
REO_MODEL_TYPE phraseType = REO_MSD;
bool hierModel = false; // Hierarchical model.
REO_MODEL_TYPE hierType = REO_MSD;
int maxPhraseLength = 0; // Eg. 7
bool translationFlag = true; // Generate extract and extract.inv
bool orientationFlag = false; // Ordering info needed?
bool sortedOutput = false; // Sort output?
LossyCountersVector lossyCounters;
#ifdef GET_COUNTS_ONLY
std::vector<size_t> phrasePairsCounters;
#endif
//////// Internal module variables /////////////////////////////////////////////
IndexedStringsStorage<word_index_t> strings;
IndexedStringsStorage<orientation_info_index_t> orientations;
//////// Untouched Philipp Koehn's code :) /////////////////////////////////////
REO_POS getOrientWordModel(SentenceAlignment & sentence, REO_MODEL_TYPE modelType,
bool connectedLeftTop, bool connectedRightTop,
int startF, int endF, int startE, int endE, int countF, int zero, int unit,
bool (*ge)(int, int), bool (*lt)(int, int) )
{
if( connectedLeftTop && !connectedRightTop)
return LEFT;
if(modelType == REO_MONO)
return UNKNOWN;
if (!connectedLeftTop && connectedRightTop)
return RIGHT;
if(modelType == REO_MSD)
return UNKNOWN;
for(int indexF=startF-2*unit; (*ge)(indexF, zero) && !connectedLeftTop; indexF=indexF-unit)
connectedLeftTop = isAligned(sentence, indexF, startE-unit);
for(int indexF=endF+2*unit; (*lt)(indexF,countF) && !connectedRightTop; indexF=indexF+unit)
connectedRightTop = isAligned(sentence, indexF, startE-unit);
if(connectedLeftTop && !connectedRightTop)
return DRIGHT;
else if(!connectedLeftTop && connectedRightTop)
return DLEFT;
return UNKNOWN;
}
// to be called with countF-1 instead of countF
REO_POS getOrientPhraseModel (SentenceAlignment & sentence, REO_MODEL_TYPE modelType,
bool connectedLeftTop, bool connectedRightTop,
int startF, int endF, int startE, int endE, int countF, int zero, int unit,
bool (*ge)(int, int), bool (*lt)(int, int),
const HSentenceVertices & inBottomRight, const HSentenceVertices & inBottomLeft)
{
HSentenceVertices::const_iterator it;
if((connectedLeftTop && !connectedRightTop) ||
//(startE == 0 && startF == 0) ||
//(startE == sentence.target.size()-1 && startF == sentence.source.size()-1) ||
((it = inBottomRight.find(startE - unit)) != inBottomRight.end() &&
it->second.find(startF-unit) != it->second.end()))
return LEFT;
if(modelType == REO_MONO)
return UNKNOWN;
if((!connectedLeftTop && connectedRightTop) ||
((it = inBottomLeft.find(startE - unit)) != inBottomLeft.end() && it->second.find(endF + unit) != it->second.end()))
return RIGHT;
if(modelType == REO_MSD)
return UNKNOWN;
connectedLeftTop = false;
for(int indexF=startF-2*unit; (*ge)(indexF, zero) && !connectedLeftTop; indexF=indexF-unit)
if(connectedLeftTop = (it = inBottomRight.find(startE - unit)) != inBottomRight.end() &&
it->second.find(indexF) != it->second.end())
return DRIGHT;
connectedRightTop = false;
for(int indexF=endF+2*unit; (*lt)(indexF, countF) && !connectedRightTop; indexF=indexF+unit)
if(connectedRightTop = (it = inBottomLeft.find(startE - unit)) != inBottomRight.end() &&
it->second.find(indexF) != it->second.end())
return DLEFT;
return UNKNOWN;
}
// to be called with countF-1 instead of countF
REO_POS getOrientHierModel (SentenceAlignment & sentence, REO_MODEL_TYPE modelType,
bool connectedLeftTop, bool connectedRightTop,
int startF, int endF, int startE, int endE, int countF, int zero, int unit,
bool (*ge)(int, int), bool (*lt)(int, int),
const HSentenceVertices & inBottomRight, const HSentenceVertices & inBottomLeft,
const HSentenceVertices & outBottomRight, const HSentenceVertices & outBottomLeft,
REO_POS phraseOrient)
{
HSentenceVertices::const_iterator it;
if(phraseOrient == LEFT ||
(connectedLeftTop && !connectedRightTop) ||
// (startE == 0 && startF == 0) ||
//(startE == sentence.target.size()-1 && startF == sentence.source.size()-1) ||
((it = inBottomRight.find(startE - unit)) != inBottomRight.end() &&
it->second.find(startF-unit) != it->second.end()) ||
((it = outBottomRight.find(startE - unit)) != outBottomRight.end() &&
it->second.find(startF-unit) != it->second.end()))
return LEFT;
if(modelType == REO_MONO)
return UNKNOWN;
if(phraseOrient == RIGHT ||
(!connectedLeftTop && connectedRightTop) ||
((it = inBottomLeft.find(startE - unit)) != inBottomLeft.end() &&
it->second.find(endF + unit) != it->second.end()) ||
((it = outBottomLeft.find(startE - unit)) != outBottomLeft.end() &&
it->second.find(endF + unit) != it->second.end()))
return RIGHT;
if(modelType == REO_MSD)
return UNKNOWN;
if(phraseOrient != UNKNOWN)
return phraseOrient;
connectedLeftTop = false;
for(int indexF=startF-2*unit; (*ge)(indexF, zero) && !connectedLeftTop; indexF=indexF-unit) {
if((connectedLeftTop = (it = inBottomRight.find(startE - unit)) != inBottomRight.end() &&
it->second.find(indexF) != it->second.end()) ||
(connectedLeftTop = (it = outBottomRight.find(startE - unit)) != outBottomRight.end() &&
it->second.find(indexF) != it->second.end()))
return DRIGHT;
}
connectedRightTop = false;
for(int indexF=endF+2*unit; (*lt)(indexF, countF) && !connectedRightTop; indexF=indexF+unit) {
if((connectedRightTop = (it = inBottomLeft.find(startE - unit)) != inBottomRight.end() &&
it->second.find(indexF) != it->second.end()) ||
(connectedRightTop = (it = outBottomLeft.find(startE - unit)) != outBottomRight.end() &&
it->second.find(indexF) != it->second.end()))
return DLEFT;
}
return UNKNOWN;
}
void insertVertex( HSentenceVertices & corners, int x, int y )
{
std::set<int> tmp;
tmp.insert(x);
std::pair< HSentenceVertices::iterator, bool > ret = corners.insert( std::pair<int, std::set<int> > (y, tmp) );
if(ret.second == false) {
ret.first->second.insert(x);
}
}
void insertPhraseVertices(
HSentenceVertices & topLeft,
HSentenceVertices & topRight,
HSentenceVertices & bottomLeft,
HSentenceVertices & bottomRight,
int startF, int startE, int endF, int endE)
{
insertVertex(topLeft, startF, startE);
insertVertex(topRight, endF, startE);
insertVertex(bottomLeft, startF, endE);
insertVertex(bottomRight, endF, endE);
}
std::string getOrientString(REO_POS orient, REO_MODEL_TYPE modelType)
{
switch(orient) {
case LEFT:
return "mono";
break;
case RIGHT:
return "swap";
break;
case DRIGHT:
return "dright";
break;
case DLEFT:
return "dleft";
break;
case UNKNOWN:
switch(modelType) {
case REO_MONO:
return "nomono";
break;
case REO_MSD:
return "other";
break;
case REO_MSLR:
return "dright";
break;
}
break;
}
}
bool ge(int first, int second)
{
return first >= second;
}
bool le(int first, int second)
{
return first <= second;
}
bool lt(int first, int second)
{
return first < second;
}
bool isAligned ( SentenceAlignment &sentence, int fi, int ei )
{
if (ei == -1 && fi == -1)
return true;
if (ei <= -1 || fi <= -1)
return false;
if (ei == sentence.target.size() && fi == sentence.source.size())
return true;
if (ei >= sentence.target.size() || fi >= sentence.source.size())
return false;
for(int i=0; i<sentence.alignedToT[ei].size(); i++)
if (sentence.alignedToT[ei][i] == fi)
return true;
return false;
}
//////// END OF untouched Philipp Koehn's code :) //////////////////////////////
/////// Slightly modified Philipp Koehn's code :) //////////////////////////////
void extract(SentenceAlignment &sentence) {
int countE = sentence.target.size();
int countF = sentence.source.size();
HPhraseVector inboundPhrases;
HSentenceVertices inTopLeft;
HSentenceVertices inTopRight;
HSentenceVertices inBottomLeft;
HSentenceVertices inBottomRight;
HSentenceVertices outTopLeft;
HSentenceVertices outTopRight;
HSentenceVertices outBottomLeft;
HSentenceVertices outBottomRight;
HSentenceVertices::const_iterator it;
bool relaxLimit = hierModel;
bool buildExtraStructure = phraseModel || hierModel;
// check alignments for target phrase startE...endE
// loop over extracted phrases which are compatible with the word-alignments
for (int startE = 0; startE < countE; startE++) {
for (
int endE = startE;
((endE < countE) && (relaxLimit || (endE < (startE + maxPhraseLength))));
endE++
) {
int minF = 9999;
int maxF = -1;
std::vector< int > usedF = sentence.alignedCountS;
for (int ei = startE; ei <= endE; ei++) {
for (int i = 0; i < sentence.alignedToT[ei].size(); i++) {
int fi = sentence.alignedToT[ei][i];
if (fi < minF) {
minF = fi;
}
if (fi > maxF) {
maxF = fi;
}
usedF[ fi ]--;
}
}
if (maxF >= 0 && // aligned to any source words at all
(relaxLimit || maxF-minF < maxPhraseLength)) { // source phrase within limits
// check if source words are aligned to out of bound target words
bool out_of_bounds = false;
for (int fi=minF; fi<=maxF && !out_of_bounds; fi++) {
if (usedF[fi]>0) {
// cout << "ouf of bounds: " << fi << "\n";
out_of_bounds = true;
}
}
// cout << "doing if for ( " << minF << "-" << maxF << ", " << startE << "," << endE << ")\n";
if (!out_of_bounds) {
// start point of source phrase may retreat over unaligned
for (int startF=minF;
(startF>=0 &&
(relaxLimit || startF>maxF-maxPhraseLength) && // within length limit
(startF==minF || sentence.alignedCountS[startF]==0)); // unaligned
startF--
)
// end point of source phrase may advance over unaligned
for (int endF=maxF;
(endF<countF &&
(relaxLimit || endF<startF+maxPhraseLength) && // within length limit
(endF==maxF || sentence.alignedCountS[endF]==0)); // unaligned
endF++
) { // at this point we have extracted a phrase
if (buildExtraStructure) { // phrase || hier
if (endE-startE < maxPhraseLength && endF-startF < maxPhraseLength) { // within limit
inboundPhrases.push_back(
HPhrase(HPhraseVertex(startF,startE), HPhraseVertex(endF,endE))
);
insertPhraseVertices(
inTopLeft, inTopRight, inBottomLeft, inBottomRight,
startF, startE, endF, endE
);
} else {
insertPhraseVertices(
outTopLeft, outTopRight, outBottomLeft, outBottomRight,
startF, startE, endF, endE
);
}
} else {
std::string orientationInfo = "";
if (orientationFlag && wordModel) { // Added orientationFlag check.
REO_POS wordPrevOrient, wordNextOrient;
bool connectedLeftTopP = isAligned( sentence, startF-1, startE-1 );
bool connectedRightTopP = isAligned( sentence, endF+1, startE-1 );
bool connectedLeftTopN = isAligned( sentence, endF+1, endE+1 );
bool connectedRightTopN = isAligned( sentence, startF-1, endE+1 );
wordPrevOrient = getOrientWordModel(sentence, wordType, connectedLeftTopP, connectedRightTopP, startF, endF, startE, endE, countF, 0, 1, &ge, &lt);
wordNextOrient = getOrientWordModel(sentence, wordType, connectedLeftTopN, connectedRightTopN, endF, startF, endE, startE, 0, countF, -1, &lt, &ge);
orientationInfo += getOrientString(wordPrevOrient, wordType) + " " + getOrientString(wordNextOrient, wordType);
}
addPhrase(sentence, startE, endE, startF, endF, orientationInfo);
}
}
}
}
}
} // end of main for loop
if (buildExtraStructure) { // phrase || hier
std::string orientationInfo = "";
REO_POS wordPrevOrient, wordNextOrient, phrasePrevOrient, phraseNextOrient, hierPrevOrient, hierNextOrient;
for (int i = 0; i < inboundPhrases.size(); i++) {
int startF = inboundPhrases[i].first.first;
int startE = inboundPhrases[i].first.second;
int endF = inboundPhrases[i].second.first;
int endE = inboundPhrases[i].second.second;
if ( orientationFlag ) { // Added orientationFlag check.
bool connectedLeftTopP = isAligned( sentence, startF-1, startE-1 );
bool connectedRightTopP = isAligned( sentence, endF+1, startE-1 );
bool connectedLeftTopN = isAligned( sentence, endF+1, endE+1 );
bool connectedRightTopN = isAligned( sentence, startF-1, endE+1 );
if (wordModel) {
wordPrevOrient = getOrientWordModel(sentence, wordType,
connectedLeftTopP, connectedRightTopP,
startF, endF, startE, endE, countF, 0, 1,
&ge, &lt);
wordNextOrient = getOrientWordModel(sentence, wordType,
connectedLeftTopN, connectedRightTopN,
endF, startF, endE, startE, 0, countF, -1,
&lt, &ge);
}
if (phraseModel) {
phrasePrevOrient = getOrientPhraseModel(sentence, phraseType,
connectedLeftTopP, connectedRightTopP,
startF, endF, startE, endE, countF-1, 0, 1, &ge, &lt, inBottomRight, inBottomLeft);
phraseNextOrient = getOrientPhraseModel(sentence, phraseType,
connectedLeftTopN, connectedRightTopN,
endF, startF, endE, startE, 0, countF-1, -1, &lt, &ge, inBottomLeft, inBottomRight);
} else {
phrasePrevOrient = phraseNextOrient = UNKNOWN;
}
if(hierModel) {
hierPrevOrient = getOrientHierModel(sentence, hierType,
connectedLeftTopP, connectedRightTopP,
startF, endF, startE, endE, countF-1, 0, 1, &ge, &lt, inBottomRight, inBottomLeft, outBottomRight, outBottomLeft, phrasePrevOrient);
hierNextOrient = getOrientHierModel(sentence, hierType,
connectedLeftTopN, connectedRightTopN,
endF, startF, endE, startE, 0, countF-1, -1, &lt, &ge, inBottomLeft, inBottomRight, outBottomLeft, outBottomRight, phraseNextOrient);
}
orientationInfo = ((wordModel)? getOrientString(wordPrevOrient, wordType) + " " + getOrientString(wordNextOrient, wordType) : "") + " | " +
((phraseModel)? getOrientString(phrasePrevOrient, phraseType) + " " + getOrientString(phraseNextOrient, phraseType) : "") + " | " +
((hierModel)? getOrientString(hierPrevOrient, hierType) + " " + getOrientString(hierNextOrient, hierType) : "");
}
addPhrase(sentence, startE, endE, startF, endF, orientationInfo);
} // end of for loop through inbound phrases
} // end if buildExtraStructure
} // end of extract()
/**
* @param sentence
* @param startE
* @param endE
* @param startF
* @param endF
* @param orientationInfo
*/
void addPhrase(SentenceAlignment &sentence, int startE, int endE, int startF, int endF, std::string &orientationInfo) {
#ifdef GET_COUNTS_ONLY
// Just get the length of phrase pair (which is now defined as maximum of the two).
phrasePairsCounters[std::max(endF - startF, endE - startE) + 1] += 1; // Don't forget +1 (span is inclusive)!
#else
alignment_t alignment;
// alignment
for (int ei = startE; ei <= endE; ++ei) {
for (int i = 0; i < sentence.alignedToT[ei].size(); ++i) {
int fi = sentence.alignedToT[ei][i];
alignment.push_back(alignment_t::value_type(fi-startF, ei-startE));
}
}
indexed_phrases_pair_t::phrase_t srcPhraseIndices, tgtPhraseIndices;
// source phrase
for (int fi = startF; fi <= endF; ++fi) {
srcPhraseIndices.push_back(strings.put(sentence.source[fi].c_str()));
}
// target phrase
for (int ei = startE; ei <= endE; ++ei) {
tgtPhraseIndices.push_back(strings.put(sentence.target[ei].c_str()));
}
// TODO: Allow for switching between min and max here.
size_t idx = std::max(srcPhraseIndices.size(), tgtPhraseIndices.size());
// Add phrase pair.
lossyCounters[idx]->lossyCounter.add(indexed_phrases_pair_t(srcPhraseIndices, tgtPhraseIndices, orientations.put(orientationInfo.c_str()), alignment));
//
if ( lossyCounters[idx]->lossyCounter.aboutToPrune() ) {
// Next addition will lead to pruning, inform:
std::cerr << 'P' << idx << std::flush;
}
#endif
} // end of addPhrase()
/////// Lossy Counting related code ////////////////////////////////////////////
void readInput(std::istream& eFile, std::istream& fFile, std::istream& aFile) {
// Note: moved out of the loop.
char englishString[LINE_MAX_LENGTH];
char foreignString[LINE_MAX_LENGTH];
char alignmentString[LINE_MAX_LENGTH];
int i = 0;
while(true) {
// Report progress?
if (++i%10000 == 0) std::cerr << "." << std::flush;
SAFE_GETLINE(eFile, englishString, LINE_MAX_LENGTH, '\n', __FILE__);
if (eFile.eof()) break;
SAFE_GETLINE(fFile, foreignString, LINE_MAX_LENGTH, '\n', __FILE__);
SAFE_GETLINE(aFile, alignmentString, LINE_MAX_LENGTH, '\n', __FILE__);
SentenceAlignment sentence;
if (sentence.create(englishString, foreignString, alignmentString, i)) {
extract(sentence);
}
}
}
void processOutput(OutputProcessor& processor) {
if ( sortedOutput ) {
processSortedOutput(processor);
}
else {
processUnsortedOutput(processor);
}
}
bool PhraseComp::operator()(const output_pair_t& a, const output_pair_t& b) {
int cmp = _inverted ? comparePhrases(a.first.tgtPhrase(), b.first.tgtPhrase()) : comparePhrases(a.first.srcPhrase(), b.first.srcPhrase());
if ( cmp == 0 ) {
// First part of pairs matches, compare the second part.
cmp = _inverted ? comparePhrases(a.first.srcPhrase(), b.first.srcPhrase()) : comparePhrases(a.first.tgtPhrase(), b.first.tgtPhrase());
if ( cmp == 0 ) {
// Also second part matches, compare alignments.
return compareAlignments(a.first, b.first);
}
else {
return cmp < 0;
}
}
else {
return cmp < 0;
}
}
bool PhraseComp::compareAlignments(const indexed_phrases_pair_t& a, const indexed_phrases_pair_t& b) {
size_t aSize = a.alignmentLength();
size_t bSize = b.alignmentLength();
size_t min = std::min(aSize, bSize);
const indexed_phrases_pair_t::alignment_point_t * aAlignment = a.alignmentData();
const indexed_phrases_pair_t::alignment_point_t * bAlignment = b.alignmentData();
int cmp = 0;
for ( size_t i = 0; i < min; ++i ) {
// Important: alignments have to be eventually inverted as well!
if ( _inverted ) {
// Inverted = compare TGT phrase alignment points first.
cmp = memcmp(aAlignment + i*2 + 1, bAlignment + i*2 + 1, sizeof(indexed_phrases_pair_t::alignment_point_t));
}
else{
// NOT inverted = compare SRC phrase alignment points first.
cmp = memcmp(aAlignment+ i*2, bAlignment + i*2, sizeof(indexed_phrases_pair_t::alignment_point_t));
}
if ( cmp == 0 ) {
if ( _inverted ) {
// Inverted = compare SRC phrase alignment points second.
cmp = memcmp(aAlignment + i*2, bAlignment + i*2, sizeof(indexed_phrases_pair_t::alignment_point_t));
}
else{
// NOT inverted = compare TGT phrase alignment points second.
cmp = memcmp(aAlignment + i*2 + 1, bAlignment + i*2 + 1, sizeof(indexed_phrases_pair_t::alignment_point_t));
}
if ( cmp != 0 ) {
return cmp < 0;
} // Otherwise continue looping.
}
else {
return cmp < 0;
}
}
// Note: LC_ALL=C GNU sort treats shorter item as lesser than longer one.
return (cmp == 0) ? (aSize < bSize) : (cmp < 0);
}
int PhraseComp::comparePhrases(const indexed_phrases_pair_t::phrase_t& a, const indexed_phrases_pair_t::phrase_t& b) {
size_t aSize = a.size();
size_t bSize = b.size();
size_t min = std::min(aSize, bSize);
int cmp = 0;
for ( size_t i = 0; i < min; ++i ) {
cmp = strcmp(strings.get(a[i]), strings.get(b[i]));
if ( cmp != 0 ) {
return cmp;
}
}
if ( aSize == bSize ) {
return 0;
}
if ( aSize < bSize ) {
return strcmp("|||", strings.get(b[min]));
}
else {
return strcmp(strings.get(a[min]), "|||");
}
}
void processSortedOutput(OutputProcessor& processor) {
output_vector_t output;
LossyCountersVector::value_type current = NULL, prev = NULL;
for ( size_t i = 1; i < lossyCounters.size(); ++i ) { // Intentionally skip 0.
current = lossyCounters[i];
if ( current != prev ) {
PhrasePairsLossyCounter& lossyCounter = current->lossyCounter;
for ( PhrasePairsLossyCounter::erasing_iterator phraseIter = lossyCounter.beginErase(); phraseIter != lossyCounter.endErase(); ++phraseIter ) {
// Store and...
output.push_back(std::make_pair(phraseIter.item(), phraseIter.frequency()));
// ...update counters.
current->outputMass += phraseIter.frequency();
current->outputSize += 1;
}
//
prev = current;
//delete current;
}
}
// Sort by source phrase.
std::sort(output.begin(), output.end(), PhraseComp(false));
// Print.
for ( output_vector_t::const_iterator iter = output.begin(); iter != output.end(); ++iter ) {
flushPhrasePair(processor, iter->first, iter->second, 1);
}
// Sort by target phrase.
std::sort(output.begin(), output.end(), PhraseComp(true));
// Print.
for ( output_vector_t::const_iterator iter = output.begin(); iter != output.end(); ++iter ) {
flushPhrasePair(processor, iter->first, iter->second, -1);
}
}
void processUnsortedOutput(OutputProcessor& processor) {
LossyCountersVector::value_type current = NULL, prev = NULL;
for ( size_t i = 1; i < lossyCounters.size(); ++i ) { // Intentionally skip 0.
current = lossyCounters[i];
if ( current != prev ) {
const PhrasePairsLossyCounter& lossyCounter = current->lossyCounter;
for ( PhrasePairsLossyCounter::const_iterator phraseIter = lossyCounter.begin(); phraseIter != lossyCounter.end(); ++phraseIter ) {
// Flush and...
flushPhrasePair(processor, phraseIter.item(), phraseIter.frequency(), 0);
// ...update counters.
current->outputMass += phraseIter.frequency();
current->outputSize += 1;
}
//
prev = current;
}
}
}
void flushPhrasePair(OutputProcessor& processor, const indexed_phrases_pair_t& indexedPhrasePair, PhrasePairsLossyCounter::frequency_t frequency, int mode = 0) {
const indexed_phrases_pair_t::phrase_t srcPhraseIndices = indexedPhrasePair.srcPhrase();
const indexed_phrases_pair_t::phrase_t tgtPhraseIndices = indexedPhrasePair.tgtPhrase();
std::string srcPhrase, tgtPhrase;
for ( indexed_phrases_pair_t::phrase_t::const_iterator indexIter = srcPhraseIndices.begin(); indexIter != srcPhraseIndices.end(); ++indexIter ) {
srcPhrase += std::string(strings.get(*indexIter)) + " ";
}
srcPhrase.resize(srcPhrase.size() - 1); // Trim the trailing " "
for ( indexed_phrases_pair_t::phrase_t::const_iterator indexIter = tgtPhraseIndices.begin(); indexIter != tgtPhraseIndices.end(); ++indexIter ) {
tgtPhrase += std::string(strings.get(*indexIter)) + " ";
}
tgtPhrase.resize(tgtPhrase.size() - 1); // Trim the trailing " "
// Actual processing is done via call to functor:
processor(srcPhrase, tgtPhrase, orientations.get(indexedPhrasePair.orientationInfo()), indexedPhrasePair.alignment(), frequency, mode);
}
void printStats(void) {
// Total counters.
size_t outputMass = 0, outputSize = 0, N = 0;
const std::string hline = "####################################################################################################################";
std::cerr << "Lossy Counting Phrase Extraction statistics:" << std::endl;
// Print header: | 3 | 15 | 15 | 15 | 7 | 10 | 10 | 10 |
std::cerr
<< hline << std::endl
<< "# length # unique out # total out # total in (N) # out/in (%) # pos. thr. # neg. thr. # max. err. #" << std::endl
<< hline << std::endl;
LossyCountersVector::value_type current = NULL, prev = NULL;
size_t from = 1, to = 1;
for ( size_t i = 1; i <= lossyCounters.size(); ++i ) { // Intentionally skip 0, intentionally increment till == size().
current = (i < lossyCounters.size()) ? lossyCounters[i] : NULL;
if ( (current == NULL) || ((current != prev) && (prev != NULL)) ) {
// Time to print.
to = i-1;
// Increment overall stats.
outputMass += prev->outputMass;
outputSize += prev->outputSize;
N += prev->lossyCounter.count();
// Print.
if ( from == to ) {
std::cerr << "# " << std::setw(6) << to << " # ";
}
else {
std::stringstream strStr;
strStr << from << "-" << to;
std::cerr << "# " << std::setw(6) << strStr.str() << " # ";
}
// Print the rest of record.
std::cerr
<< std::setw(15) << prev->outputSize << " # "
<< std::setw(15) << prev->outputMass << " # "
<< std::setw(15) << prev->lossyCounter.count() << " # "
<< std::setw(10) << std::setprecision(4) << (static_cast<double>(prev->outputMass) / static_cast<double>(prev->lossyCounter.count())) * 100 << " # "
<< std::setw(10) << prev->lossyCounter.threshold(true) << " # "
<< std::setw(10) << prev->lossyCounter.threshold() << " # "
<< std::setw(10) << prev->lossyCounter.maxError() << " #"
<< std::endl << hline << std::endl;
from = i;
}
prev = current;
}
// Print summary:
std::cerr
<< "# TOTAL # "
<< std::setw(15) << outputSize << " # "
<< std::setw(15) << outputMass << " # "
<< std::setw(15) << N << " # "
<< std::setw(10) << std::setprecision(4) << (static_cast<double>(outputMass) / static_cast<double>(N)) * 100 << " #"
<< std::endl
<< "#############################################################################" << std::endl;
}
Reference in New Issue View Git Blame Copy Permalink