moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity
cb7191e85a
mosesdecoder/phrase-extract/score-main.cpp

1311 lines
50 KiB
C++
Raw Blame History

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2009 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <sstream>
#include <assert.h>
#include <cstdlib>
#include <cstring>
#include <map>
#include <set>
#include <vector>
#include <algorithm>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/unordered_map.hpp>
#include "ScoreFeature.h"
#include "tables-core.h"
#include "ExtractionPhrasePair.h"
#include "score.h"
#include "InputFileStream.h"
#include "OutputFileStream.h"
#include "moses/Util.h"
using namespace boost::algorithm;
using namespace MosesTraining;
namespace MosesTraining
{
LexicalTable lexTable;
bool inverseFlag = false;
bool hierarchicalFlag = false;
bool pcfgFlag = false;
bool phraseOrientationFlag = false;
bool treeFragmentsFlag = false;
bool partsOfSpeechFlag = false;
bool sourceSyntaxLabelsFlag = false;
bool sourceSyntaxLabelCountsLHSFlag = false;
bool targetSyntacticPreferencesFlag = false;
bool unpairedExtractFormatFlag = false;
bool conditionOnTargetLhsFlag = false;
bool wordAlignmentFlag = true;
bool goodTuringFlag = false;
bool kneserNeyFlag = false;
bool logProbFlag = false;
int negLogProb = 1;
#define COC_MAX 10
bool lexFlag = true;
bool unalignedFlag = false;
bool unalignedFWFlag = false;
bool crossedNonTerm = false;
bool spanLength = false;
bool ruleLength = false;
bool nonTermContext = false;
bool nonTermContextTarget = false;
bool targetConstituentBoundariesFlag = false;
int countOfCounts[COC_MAX+1];
int totalDistinct = 0;
float minCount = 0;
float minCountHierarchical = 0;
bool phraseOrientationPriorsFlag = false;
boost::unordered_map<std::string,float> sourceLHSCounts;
boost::unordered_map<std::string, boost::unordered_map<std::string,float>* > targetLHSAndSourceLHSJointCounts;
std::set<std::string> sourceLabelSet;
std::map<std::string,size_t> sourceLabels;
std::vector<std::string> sourceLabelsByIndex;
std::set<std::string> partsOfSpeechSet;
boost::unordered_map<std::string,float> targetSyntacticPreferencesLHSCounts;
boost::unordered_map<std::string, boost::unordered_map<std::string,float>* > ruleTargetLHSAndTargetSyntacticPreferencesLHSJointCounts;
std::set<std::string> targetSyntacticPreferencesLabelSet;
std::map<std::string,size_t> targetSyntacticPreferencesLabels;
std::vector<std::string> targetSyntacticPreferencesLabelsByIndex;
std::vector<float> orientationClassPriorsL2R(4,0); // mono swap dleft dright
std::vector<float> orientationClassPriorsR2L(4,0); // mono swap dleft dright
Vocabulary vcbT;
Vocabulary vcbS;
} // namespace
void processLine( std::string line,
int lineID, bool includeSentenceIdFlag, int &sentenceId,
PHRASE *phraseSource, PHRASE *phraseTarget, ALIGNMENT *targetToSourceAlignment,
std::string &additionalPropertiesString,
float &count, float &pcfgSum );
void writeCountOfCounts( const std::string &fileNameCountOfCounts );
void writeLeftHandSideLabelCounts( const boost::unordered_map<std::string,float> &countsLabelLHS,
const boost::unordered_map<std::string, boost::unordered_map<std::string,float>* > &jointCountsLabelLHS,
const std::string &fileNameLeftHandSideSourceLabelCounts,
const std::string &fileNameLeftHandSideTargetSourceLabelCounts );
void writeLabelSet( const std::set<std::string> &labelSet, const std::string &fileName );
void processPhrasePairs( std::vector< ExtractionPhrasePair* > &phrasePairsWithSameSource, std::ostream &phraseTableFile,
const ScoreFeatureManager& featureManager, const MaybeLog& maybeLogProb );
void outputPhrasePair(const ExtractionPhrasePair &phrasePair, float, int, std::ostream &phraseTableFile, const ScoreFeatureManager &featureManager, const MaybeLog &maybeLog );
double computeLexicalTranslation( const PHRASE *phraseSource, const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource );
double computeUnalignedPenalty( const ALIGNMENT *alignmentTargetToSource );
std::set<std::string> functionWordList;
void loadOrientationPriors(const std::string &fileNamePhraseOrientationPriors, std::vector<float> &orientationClassPriorsL2R, std::vector<float> &orientationClassPriorsR2L);
void loadFunctionWords( const std::string &fileNameFunctionWords );
double computeUnalignedFWPenalty( const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource );
int calcCrossedNonTerm( const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource );
void printSourcePhrase( const PHRASE *phraseSource, const PHRASE *phraseTarget, const ALIGNMENT *targetToSourceAlignment, std::ostream &out );
void printTargetPhrase( const PHRASE *phraseSource, const PHRASE *phraseTarget, const ALIGNMENT *targetToSourceAlignment, std::ostream &out );
void invertAlignment( const PHRASE *phraseSource, const PHRASE *phraseTarget, const ALIGNMENT *inTargetToSourceAlignment, ALIGNMENT *outSourceToTargetAlignment );
size_t NumNonTerminal(const PHRASE *phraseSource);
int main(int argc, char* argv[])
{
std::cerr << "Score v2.1 -- "
<< "scoring methods for extracted rules" << std::endl;
ScoreFeatureManager featureManager;
if (argc < 4) {
std::cerr <<
"syntax: score extract lex phrase-table "
"[--Inverse] "
"[--Hierarchical] "
"[--LogProb] "
"[--NegLogProb] "
"[--NoLex] "
"[--GoodTuring] "
"[--KneserNey] "
"[--NoWordAlignment] "
"[--UnalignedPenalty] "
"[--UnalignedFunctionWordPenalty function-word-file] "
"[--MinCountHierarchical count] "
"[--PartsOfSpeech] "
"[--PCFG] "
"[--TreeFragments] "
"[--SourceLabels] "
"[--SourceLabelCountsLHS] "
"[--TargetSyntacticPreferences] "
"[--UnpairedExtractFormat] "
"[--ConditionOnTargetLHS] "
"[--CrossedNonTerm]"
<< std::endl;
std::cerr << featureManager.usage() << std::endl;
exit(1);
}
std::string fileNameExtract = argv[1];
std::string fileNameLex = argv[2];
std::string fileNamePhraseTable = argv[3];
std::string fileNameSourceLabelSet;
std::string fileNamePartsOfSpeechSet;
std::string fileNameCountOfCounts;
std::string fileNameFunctionWords;
std::string fileNameLeftHandSideSourceLabelCounts;
std::string fileNameLeftHandSideTargetSourceLabelCounts;
std::string fileNameTargetSyntacticPreferencesLabelSet;
std::string fileNameLeftHandSideTargetSyntacticPreferencesLabelCounts;
std::string fileNameLeftHandSideRuleTargetTargetSyntacticPreferencesLabelCounts;
std::string fileNamePhraseOrientationPriors;
// All unknown args are passed to feature manager.
std::vector<std::string> featureArgs;
for(int i=4; i<argc; i++) {
if (strcmp(argv[i],"inverse") == 0 || strcmp(argv[i],"--Inverse") == 0) {
inverseFlag = true;
std::cerr << "using inverse mode" << std::endl;
} else if (strcmp(argv[i],"--Hierarchical") == 0) {
hierarchicalFlag = true;
std::cerr << "processing hierarchical rules" << std::endl;
} else if (strcmp(argv[i],"--PCFG") == 0) {
pcfgFlag = true;
std::cerr << "including PCFG scores" << std::endl;
} else if (strcmp(argv[i],"--PhraseOrientation") == 0) {
phraseOrientationFlag = true;
std::cerr << "including phrase orientation information" << std::endl;
} else if (strcmp(argv[i],"--TreeFragments") == 0) {
treeFragmentsFlag = true;
std::cerr << "including tree fragment information from syntactic parse" << std::endl;
} else if (strcmp(argv[i],"--PartsOfSpeech") == 0) {
partsOfSpeechFlag = true;
std::cerr << "including parts-of-speech information from syntactic parse" << std::endl;
fileNamePartsOfSpeechSet = std::string(fileNamePhraseTable) + ".partsOfSpeech";
std::cerr << "writing parts-of-speech set to file " << fileNamePartsOfSpeechSet << std::endl;
} else if (strcmp(argv[i],"--SourceLabels") == 0) {
sourceSyntaxLabelsFlag = true;
std::cerr << "including source label information" << std::endl;
fileNameSourceLabelSet = std::string(fileNamePhraseTable) + ".syntaxLabels.src";
std::cerr << "writing source syntax label set to file " << fileNameSourceLabelSet << std::endl;
} else if (strcmp(argv[i],"--SourceLabelCountsLHS") == 0) {
sourceSyntaxLabelCountsLHSFlag = true;
fileNameLeftHandSideSourceLabelCounts = std::string(fileNamePhraseTable) + ".src.lhs";
fileNameLeftHandSideTargetSourceLabelCounts = std::string(fileNamePhraseTable) + ".tgt-src.lhs";
std::cerr << "counting left-hand side source labels and writing them to files " << fileNameLeftHandSideSourceLabelCounts << " and " << fileNameLeftHandSideTargetSourceLabelCounts << std::endl;
} else if (strcmp(argv[i],"--TargetSyntacticPreferences") == 0) {
targetSyntacticPreferencesFlag = true;
std::cerr << "including target syntactic preferences information" << std::endl;
fileNameTargetSyntacticPreferencesLabelSet = std::string(fileNamePhraseTable) + ".syntaxLabels.tgtpref";
std::cerr << "writing target syntactic preferences label set to file " << fileNameTargetSyntacticPreferencesLabelSet << std::endl;
fileNameLeftHandSideTargetSyntacticPreferencesLabelCounts = std::string(fileNamePhraseTable) + ".tgtpref.lhs";
fileNameLeftHandSideRuleTargetTargetSyntacticPreferencesLabelCounts = std::string(fileNamePhraseTable) + ".tgt-tgtpref.lhs";
std::cerr << "counting left-hand side target syntactic preferences labels and writing them to files "
<< fileNameLeftHandSideTargetSyntacticPreferencesLabelCounts
<< " and "
<< fileNameLeftHandSideRuleTargetTargetSyntacticPreferencesLabelCounts
<< std::endl;
} else if (strcmp(argv[i],"--UnpairedExtractFormat") == 0) {
unpairedExtractFormatFlag = true;
std::cerr << "processing unpaired extract format" << std::endl;
} else if (strcmp(argv[i],"--ConditionOnTargetLHS") == 0) {
conditionOnTargetLhsFlag = true;
std::cerr << "processing unpaired extract format" << std::endl;
} else if (strcmp(argv[i],"--NoWordAlignment") == 0) {
wordAlignmentFlag = false;
std::cerr << "omitting word alignment" << std::endl;
} else if (strcmp(argv[i],"--NoLex") == 0) {
lexFlag = false;
std::cerr << "not computing lexical translation score" << std::endl;
} else if (strcmp(argv[i],"--GoodTuring") == 0) {
goodTuringFlag = true;
fileNameCountOfCounts = std::string(fileNamePhraseTable) + ".coc";
std::cerr << "adjusting phrase translation probabilities with Good Turing discounting" << std::endl;
} else if (strcmp(argv[i],"--KneserNey") == 0) {
kneserNeyFlag = true;
fileNameCountOfCounts = std::string(fileNamePhraseTable) + ".coc";
std::cerr << "adjusting phrase translation probabilities with Kneser Ney discounting" << std::endl;
} else if (strcmp(argv[i],"--UnalignedPenalty") == 0) {
unalignedFlag = true;
std::cerr << "using unaligned word penalty" << std::endl;
} else if (strcmp(argv[i],"--UnalignedFunctionWordPenalty") == 0) {
unalignedFWFlag = true;
if (i+1==argc) {
std::cerr << "ERROR: specify function words file for unaligned function word penalty!" << std::endl;
exit(1);
}
fileNameFunctionWords = argv[++i];
std::cerr << "using unaligned function word penalty with function words from " << fileNameFunctionWords << std::endl;
} else if (strcmp(argv[i],"--LogProb") == 0) {
logProbFlag = true;
std::cerr << "using log-probabilities" << std::endl;
} else if (strcmp(argv[i],"--NegLogProb") == 0) {
logProbFlag = true;
negLogProb = -1;
std::cerr << "using negative log-probabilities" << std::endl;
} else if (strcmp(argv[i],"--MinCount") == 0) {
minCount = std::atof( argv[++i] );
std::cerr << "dropping all phrase pairs occurring less than " << minCount << " times" << std::endl;
minCount -= 0.00001; // account for rounding
} else if (strcmp(argv[i],"--MinCountHierarchical") == 0) {
minCountHierarchical = std::atof( argv[++i] );
std::cerr << "dropping all hierarchical phrase pairs occurring less than " << minCountHierarchical << " times" << std::endl;
minCountHierarchical -= 0.00001; // account for rounding
} else if (strcmp(argv[i],"--CrossedNonTerm") == 0) {
crossedNonTerm = true;
std::cerr << "crossed non-term reordering feature" << std::endl;
} else if (strcmp(argv[i],"--PhraseOrientationPriors") == 0) {
phraseOrientationPriorsFlag = true;
if (i+1==argc) {
std::cerr << "ERROR: specify priors file for phrase orientation!" << std::endl;
exit(1);
}
fileNamePhraseOrientationPriors = argv[++i];
std::cerr << "smoothing phrase orientation with priors from " << fileNamePhraseOrientationPriors << std::endl;
} else if (strcmp(argv[i],"--SpanLength") == 0) {
spanLength = true;
std::cerr << "span length feature" << std::endl;
} else if (strcmp(argv[i],"--RuleLength") == 0) {
ruleLength = true;
std::cerr << "rule length feature" << std::endl;
} else if (strcmp(argv[i],"--NonTermContext") == 0) {
nonTermContext = true;
std::cerr << "non-term context" << std::endl;
} else if (strcmp(argv[i],"--NonTermContextTarget") == 0) {
nonTermContextTarget = true;
std::cerr << "non-term context (target)" << std::endl;
} else if (strcmp(argv[i],"--TargetConstituentBoundaries") == 0) {
targetConstituentBoundariesFlag = true;
std::cerr << "including target constituent boundaries information" << std::endl;
} else {
featureArgs.push_back(argv[i]);
++i;
for (; i < argc && strncmp(argv[i], "--", 2); ++i) {
featureArgs.push_back(argv[i]);
}
if (i != argc) --i; //roll back, since we found another -- argument
}
}
MaybeLog maybeLogProb(logProbFlag, negLogProb);
// configure extra features
if (!inverseFlag) {
featureManager.configure(featureArgs);
}
// lexical translation table
if (lexFlag) {
lexTable.load( fileNameLex );
}
// function word list
if (unalignedFWFlag) {
loadFunctionWords( fileNameFunctionWords );
}
// compute count of counts for Good Turing discounting
if (goodTuringFlag || kneserNeyFlag) {
for(int i=1; i<=COC_MAX; i++) countOfCounts[i] = 0;
}
if (phraseOrientationPriorsFlag) {
loadOrientationPriors(fileNamePhraseOrientationPriors,orientationClassPriorsL2R,orientationClassPriorsR2L);
}
// sorted phrase extraction file
Moses::InputFileStream extractFile(fileNameExtract);
if (extractFile.fail()) {
std::cerr << "ERROR: could not open extract file " << fileNameExtract << std::endl;
exit(1);
}
// output file: phrase translation table
std::ostream *phraseTableFile;
if (fileNamePhraseTable == "-") {
phraseTableFile = &std::cout;
} else {
Moses::OutputFileStream *outputFile = new Moses::OutputFileStream();
bool success = outputFile->Open(fileNamePhraseTable);
if (!success) {
std::cerr << "ERROR: could not open file phrase table file "
<< fileNamePhraseTable << std::endl;
exit(1);
}
phraseTableFile = outputFile;
}
// loop through all extracted phrase translations
std::string line, lastLine;
ExtractionPhrasePair *phrasePair = NULL;
std::vector< ExtractionPhrasePair* > phrasePairsWithSameSource;
std::vector< ExtractionPhrasePair* > phrasePairsWithSameSourceAndTarget; // required for hierarchical rules only, as non-terminal alignments might make the phrases incompatible
int tmpSentenceId;
PHRASE *tmpPhraseSource, *tmpPhraseTarget;
ALIGNMENT *tmpTargetToSourceAlignment;
std::string tmpAdditionalPropertiesString;
float tmpCount=0.0f, tmpPcfgSum=0.0f;
int i=0;
if ( getline(extractFile, line) ) {
++i;
tmpPhraseSource = new PHRASE();
tmpPhraseTarget = new PHRASE();
tmpTargetToSourceAlignment = new ALIGNMENT();
processLine( std::string(line),
i, featureManager.includeSentenceId(), tmpSentenceId,
tmpPhraseSource, tmpPhraseTarget, tmpTargetToSourceAlignment,
tmpAdditionalPropertiesString,
tmpCount, tmpPcfgSum);
phrasePair = new ExtractionPhrasePair( tmpPhraseSource, tmpPhraseTarget,
tmpTargetToSourceAlignment,
tmpCount, tmpPcfgSum );
phrasePair->AddProperties( tmpAdditionalPropertiesString, tmpCount );
featureManager.addPropertiesToPhrasePair( *phrasePair, tmpCount, tmpSentenceId );
phrasePairsWithSameSource.push_back( phrasePair );
if ( hierarchicalFlag ) {
phrasePairsWithSameSourceAndTarget.push_back( phrasePair );
}
lastLine = line;
}
while ( getline(extractFile, line) ) {
// Print progress dots to stderr.
if ( ++i % 100000 == 0 ) {
std::cerr << "." << std::flush;
}
// identical to last line? just add count
if (line == lastLine) {
phrasePair->IncrementPrevious(tmpCount,tmpPcfgSum);
continue;
} else {
lastLine = line;
}
tmpPhraseSource = new PHRASE();
tmpPhraseTarget = new PHRASE();
tmpTargetToSourceAlignment = new ALIGNMENT();
tmpAdditionalPropertiesString.clear();
processLine( std::string(line),
i, featureManager.includeSentenceId(), tmpSentenceId,
tmpPhraseSource, tmpPhraseTarget, tmpTargetToSourceAlignment,
tmpAdditionalPropertiesString,
tmpCount, tmpPcfgSum);
bool matchesPrevious = false;
bool sourceMatch = true;
bool targetMatch = true;
bool alignmentMatch = true; // be careful with these,
// ExtractionPhrasePair::Matches() checks them in order and does not continue with the others
// once the first of them has been found to have to be set to false
if ( hierarchicalFlag ) {
for ( std::vector< ExtractionPhrasePair* >::const_iterator iter = phrasePairsWithSameSourceAndTarget.begin();
iter != phrasePairsWithSameSourceAndTarget.end(); ++iter ) {
if ( (*iter)->Matches( tmpPhraseSource, tmpPhraseTarget, tmpTargetToSourceAlignment,
sourceMatch, targetMatch, alignmentMatch ) ) {
matchesPrevious = true;
phrasePair = (*iter);
break;
}
}
} else {
if ( phrasePair->Matches( tmpPhraseSource, tmpPhraseTarget, tmpTargetToSourceAlignment,
sourceMatch, targetMatch, alignmentMatch ) ) {
matchesPrevious = true;
}
}
if ( matchesPrevious ) {
delete tmpPhraseSource;
delete tmpPhraseTarget;
if ( !phrasePair->Add( tmpTargetToSourceAlignment,
tmpCount, tmpPcfgSum ) ) {
delete tmpTargetToSourceAlignment;
}
phrasePair->AddProperties( tmpAdditionalPropertiesString, tmpCount );
featureManager.addPropertiesToPhrasePair( *phrasePair, tmpCount, tmpSentenceId );
} else {
if ( !phrasePairsWithSameSource.empty() &&
!sourceMatch ) {
processPhrasePairs( phrasePairsWithSameSource, *phraseTableFile, featureManager, maybeLogProb );
for ( std::vector< ExtractionPhrasePair* >::const_iterator iter=phrasePairsWithSameSource.begin();
iter!=phrasePairsWithSameSource.end(); ++iter) {
delete *iter;
}
phrasePairsWithSameSource.clear();
if ( hierarchicalFlag ) {
phrasePairsWithSameSourceAndTarget.clear();
}
}
if ( hierarchicalFlag ) {
if ( !phrasePairsWithSameSourceAndTarget.empty() &&
!targetMatch ) {
phrasePairsWithSameSourceAndTarget.clear();
}
}
phrasePair = new ExtractionPhrasePair( tmpPhraseSource, tmpPhraseTarget,
tmpTargetToSourceAlignment,
tmpCount, tmpPcfgSum );
phrasePair->AddProperties( tmpAdditionalPropertiesString, tmpCount );
featureManager.addPropertiesToPhrasePair( *phrasePair, tmpCount, tmpSentenceId );
phrasePairsWithSameSource.push_back(phrasePair);
if ( hierarchicalFlag ) {
phrasePairsWithSameSourceAndTarget.push_back(phrasePair);
}
}
}
// We've been printing progress dots to stderr. End the line.
std::cerr << std::endl;
processPhrasePairs( phrasePairsWithSameSource, *phraseTableFile, featureManager, maybeLogProb );
for ( std::vector< ExtractionPhrasePair* >::const_iterator iter=phrasePairsWithSameSource.begin();
iter!=phrasePairsWithSameSource.end(); ++iter) {
delete *iter;
}
phrasePairsWithSameSource.clear();
phraseTableFile->flush();
if (phraseTableFile != &std::cout) {
delete phraseTableFile;
}
// output count of count statistics
if (goodTuringFlag || kneserNeyFlag) {
writeCountOfCounts( fileNameCountOfCounts );
}
// source syntax labels
if (sourceSyntaxLabelsFlag && !inverseFlag) {
writeLabelSet( sourceLabelSet, fileNameSourceLabelSet );
}
if (sourceSyntaxLabelsFlag && sourceSyntaxLabelCountsLHSFlag && !inverseFlag) {
writeLeftHandSideLabelCounts( sourceLHSCounts,
targetLHSAndSourceLHSJointCounts,
fileNameLeftHandSideSourceLabelCounts,
fileNameLeftHandSideTargetSourceLabelCounts );
}
// parts-of-speech
if (partsOfSpeechFlag && !inverseFlag) {
writeLabelSet( partsOfSpeechSet, fileNamePartsOfSpeechSet );
}
// target syntactic preferences labels
if (targetSyntacticPreferencesFlag && !inverseFlag) {
writeLabelSet( targetSyntacticPreferencesLabelSet, fileNameTargetSyntacticPreferencesLabelSet );
writeLeftHandSideLabelCounts( targetSyntacticPreferencesLHSCounts,
ruleTargetLHSAndTargetSyntacticPreferencesLHSJointCounts,
fileNameLeftHandSideTargetSyntacticPreferencesLabelCounts,
fileNameLeftHandSideRuleTargetTargetSyntacticPreferencesLabelCounts );
}
}
void processLine( std::string line,
int lineID, bool includeSentenceIdFlag, int &sentenceId,
PHRASE *phraseSource, PHRASE *phraseTarget, ALIGNMENT *targetToSourceAlignment,
std::string &additionalPropertiesString,
float &count, float &pcfgSum )
{
size_t foundAdditionalProperties = line.rfind("|||");
foundAdditionalProperties = line.find("{{",foundAdditionalProperties);
if (foundAdditionalProperties != std::string::npos) {
additionalPropertiesString = line.substr(foundAdditionalProperties);
line = line.substr(0,foundAdditionalProperties);
} else {
additionalPropertiesString.clear();
}
phraseSource->clear();
phraseTarget->clear();
targetToSourceAlignment->clear();
std::vector<std::string> token;
Moses::Tokenize( token, line );
int item = 1;
for ( size_t j=0; j<token.size(); ++j ) {
if (token[j] == "|||") {
++item;
} else if (item == 1) { // source phrase
phraseSource->push_back( vcbS.storeIfNew( token[j] ) );
} else if (item == 2) { // target phrase
phraseTarget->push_back( vcbT.storeIfNew( token[j] ) );
} else if (item == 3) { // alignment
int s,t;
sscanf(token[j].c_str(), "%d-%d", &s, &t);
if ((size_t)t >= phraseTarget->size() || (size_t)s >= phraseSource->size()) {
std::cerr << "WARNING: phrase pair " << lineID
<< " has alignment point (" << s << ", " << t << ")"
<< " out of bounds (" << phraseSource->size() << ", " << phraseTarget->size() << ")"
<< std::endl;
} else {
// first alignment point? -> initialize
if ( targetToSourceAlignment->size() == 0 ) {
size_t numberOfTargetSymbols = (hierarchicalFlag ? phraseTarget->size()-1 : phraseTarget->size());
targetToSourceAlignment->resize(numberOfTargetSymbols);
}
// add alignment point
targetToSourceAlignment->at(t).insert(s);
}
} else if (includeSentenceIdFlag && item == 4) { // optional sentence id
sscanf(token[j].c_str(), "%d", &sentenceId);
} else if (item + (includeSentenceIdFlag?-1:0) == 4) { // count
sscanf(token[j].c_str(), "%f", &count);
} else if (item + (includeSentenceIdFlag?-1:0) == 5) { // target syntax PCFG score
float pcfgScore = std::atof( token[j].c_str() );
pcfgSum = pcfgScore * count;
}
}
if ( targetToSourceAlignment->size() == 0 ) {
size_t numberOfTargetSymbols = (hierarchicalFlag ? phraseTarget->size()-1 : phraseTarget->size());
targetToSourceAlignment->resize(numberOfTargetSymbols);
}
if (item + (includeSentenceIdFlag?-1:0) == 3) {
count = 1.0;
}
if (item < 3 || item > (includeSentenceIdFlag?7:6)) {
std::cerr << "ERROR: faulty line " << lineID << ": " << line << std::endl;
}
}
void writeCountOfCounts( const std::string &fileNameCountOfCounts )
{
// open file
Moses::OutputFileStream countOfCountsFile;
bool success = countOfCountsFile.Open(fileNameCountOfCounts);
if (!success) {
std::cerr << "ERROR: could not open count-of-counts file "
<< fileNameCountOfCounts << std::endl;
return;
}
// Kneser-Ney needs the total number of phrase pairs
countOfCountsFile << totalDistinct << std::endl;
// write out counts
for(int i=1; i<=COC_MAX; i++) {
countOfCountsFile << countOfCounts[ i ] << std::endl;
}
countOfCountsFile.Close();
}
void writeLeftHandSideLabelCounts( const boost::unordered_map<std::string,float> &countsLabelLHS,
const boost::unordered_map<std::string, boost::unordered_map<std::string,float>* > &jointCountsLabelLHS,
const std::string &fileNameLeftHandSideSourceLabelCounts,
const std::string &fileNameLeftHandSideTargetSourceLabelCounts )
{
// open file
Moses::OutputFileStream leftHandSideSourceLabelCounts;
bool success = leftHandSideSourceLabelCounts.Open(fileNameLeftHandSideSourceLabelCounts);
if (!success) {
std::cerr << "ERROR: could not open left-hand side label counts file "
<< fileNameLeftHandSideSourceLabelCounts << std::endl;
return;
}
// write source left-hand side counts
for (boost::unordered_map<std::string,float>::const_iterator iter=sourceLHSCounts.begin();
iter!=sourceLHSCounts.end(); ++iter) {
leftHandSideSourceLabelCounts << iter->first << " " << iter->second << std::endl;
}
leftHandSideSourceLabelCounts.Close();
// open file
Moses::OutputFileStream leftHandSideTargetSourceLabelCounts;
success = leftHandSideTargetSourceLabelCounts.Open(fileNameLeftHandSideTargetSourceLabelCounts);
if (!success) {
std::cerr << "ERROR: could not open left-hand side label joint counts file "
<< fileNameLeftHandSideTargetSourceLabelCounts << std::endl;
return;
}
// write source left-hand side / target left-hand side joint counts
for (boost::unordered_map<std::string, boost::unordered_map<std::string,float>* >::const_iterator iter=targetLHSAndSourceLHSJointCounts.begin();
iter!=targetLHSAndSourceLHSJointCounts.end(); ++iter) {
for (boost::unordered_map<std::string,float>::const_iterator iter2=(iter->second)->begin();
iter2!=(iter->second)->end(); ++iter2) {
leftHandSideTargetSourceLabelCounts << iter->first << " "<< iter2->first << " " << iter2->second << std::endl;
}
}
leftHandSideTargetSourceLabelCounts.Close();
}
void writeLabelSet( const std::set<std::string> &labelSet, const std::string &fileName )
{
// open file
Moses::OutputFileStream out;
bool success = out.Open(fileName);
if (!success) {
std::cerr << "ERROR: could not open file "
<< fileName << " for writing" << std::endl;
return;
}
for (std::set<std::string>::const_iterator iter=labelSet.begin();
iter!=labelSet.end(); ++iter) {
out << *iter << std::endl;
}
out.Close();
}
void processPhrasePairs( std::vector< ExtractionPhrasePair* > &phrasePairsWithSameSource, std::ostream &phraseTableFile,
const ScoreFeatureManager& featureManager, const MaybeLog& maybeLogProb )
{
if (phrasePairsWithSameSource.size() == 0) {
return;
}
float totalSource = 0;
//std::cerr << "phrasePairs.size() = " << phrasePairs.size() << std::endl;
// loop through phrase pairs
for ( std::vector< ExtractionPhrasePair* >::const_iterator iter=phrasePairsWithSameSource.begin();
iter!=phrasePairsWithSameSource.end(); ++iter) {
// add to total count
totalSource += (*iter)->GetCount();
}
// output the distinct phrase pairs, one at a time
for ( std::vector< ExtractionPhrasePair* >::const_iterator iter=phrasePairsWithSameSource.begin();
iter!=phrasePairsWithSameSource.end(); ++iter) {
// add to total count
outputPhrasePair( **iter, totalSource, phrasePairsWithSameSource.size(), phraseTableFile, featureManager, maybeLogProb );
}
}
void outputPhrasePair(const ExtractionPhrasePair &phrasePair,
float totalCount, int distinctCount,
std::ostream &phraseTableFile,
const ScoreFeatureManager& featureManager,
const MaybeLog& maybeLogProb )
{
assert(phrasePair.IsValid());
const ALIGNMENT *bestAlignmentT2S = phrasePair.FindBestAlignmentTargetToSource();
float count = phrasePair.GetCount();
std::map< std::string, float > domainCount;
// collect count of count statistics
if (goodTuringFlag || kneserNeyFlag) {
totalDistinct++;
int countInt = count + 0.99999;
if ((countInt <= COC_MAX) &&
(countInt > 0))
countOfCounts[ countInt ]++;
}
// output phrases
const PHRASE *phraseSource = phrasePair.GetSource();
const PHRASE *phraseTarget = phrasePair.GetTarget();
// do not output if count below threshold
if (count < minCount) {
return;
}
// do not output if hierarchical and count below threshold
if (hierarchicalFlag && count < minCountHierarchical) {
for(size_t j=0; j<phraseSource->size()-1; ++j) {
if (isNonTerminal(vcbS.getWord( phraseSource->at(j) )))
return;
}
}
// compute PCFG score
float pcfgScore = 0;
if (pcfgFlag && !inverseFlag) {
pcfgScore = phrasePair.GetPcfgScore() / count;
}
// source phrase (unless inverse)
if (!inverseFlag) {
printSourcePhrase(phraseSource, phraseTarget, bestAlignmentT2S, phraseTableFile);
phraseTableFile << " ||| ";
}
// target phrase
printTargetPhrase(phraseSource, phraseTarget, bestAlignmentT2S, phraseTableFile);
phraseTableFile << " ||| ";
// source phrase (if inverse)
if (inverseFlag) {
printSourcePhrase(phraseSource, phraseTarget, bestAlignmentT2S, phraseTableFile);
phraseTableFile << " ||| ";
}
// alignment
if ( hierarchicalFlag ) {
// always output alignment if hiero style
assert(phraseTarget->size() == bestAlignmentT2S->size()+1);
std::vector<std::string> alignment;
for ( size_t j = 0; j < phraseTarget->size() - 1; ++j ) {
if ( isNonTerminal(vcbT.getWord( phraseTarget->at(j) ))) {
if ( bestAlignmentT2S->at(j).size() != 1 ) {
std::cerr << "Error: unequal numbers of non-terminals. Make sure the text does not contain words in square brackets (like [xxx])." << std::endl;
phraseTableFile.flush();
assert(bestAlignmentT2S->at(j).size() == 1);
}
size_t sourcePos = *(bestAlignmentT2S->at(j).begin());
//phraseTableFile << sourcePos << "-" << j << " ";
std::stringstream point;
point << sourcePos << "-" << j;
alignment.push_back(point.str());
} else {
for ( std::set<size_t>::iterator setIter = (bestAlignmentT2S->at(j)).begin();
setIter != (bestAlignmentT2S->at(j)).end(); ++setIter ) {
size_t sourcePos = *setIter;
std::stringstream point;
point << sourcePos << "-" << j;
alignment.push_back(point.str());
}
}
}
// now print all alignments, sorted by source index
sort(alignment.begin(), alignment.end());
for (size_t i = 0; i < alignment.size(); ++i) {
phraseTableFile << alignment[i] << " ";
}
} else if ( !inverseFlag && wordAlignmentFlag) {
// alignment info in pb model
for (size_t j = 0; j < bestAlignmentT2S->size(); ++j) {
for ( std::set<size_t>::iterator setIter = (bestAlignmentT2S->at(j)).begin();
setIter != (bestAlignmentT2S->at(j)).end(); ++setIter ) {
size_t sourcePos = *setIter;
phraseTableFile << sourcePos << "-" << j << " ";
}
}
}
phraseTableFile << " ||| ";
// lexical translation probability
if (lexFlag) {
double lexScore = computeLexicalTranslation( phraseSource, phraseTarget, bestAlignmentT2S );
phraseTableFile << maybeLogProb( lexScore );
}
// unaligned word penalty
if (unalignedFlag) {
double penalty = computeUnalignedPenalty( bestAlignmentT2S );
phraseTableFile << " " << maybeLogProb( penalty );
}
// unaligned function word penalty
if (unalignedFWFlag) {
double penalty = computeUnalignedFWPenalty( phraseTarget, bestAlignmentT2S );
phraseTableFile << " " << maybeLogProb( penalty );
}
if (crossedNonTerm && !inverseFlag) {
phraseTableFile << " " << calcCrossedNonTerm( phraseTarget, bestAlignmentT2S );
}
// target-side PCFG score
if (pcfgFlag && !inverseFlag) {
phraseTableFile << " " << maybeLogProb( pcfgScore );
}
// extra features
ScoreFeatureContext context(phrasePair, maybeLogProb);
std::vector<float> extraDense;
std::map<std::string,float> extraSparse;
featureManager.addFeatures(context, extraDense, extraSparse);
for (size_t i = 0; i < extraDense.size(); ++i) {
phraseTableFile << " " << extraDense[i];
}
for (std::map<std::string,float>::const_iterator i = extraSparse.begin();
i != extraSparse.end(); ++i) {
phraseTableFile << " " << i->first << " " << i->second;
}
// counts
phraseTableFile << " ||| " << totalCount << " " << count;
if (kneserNeyFlag)
phraseTableFile << " " << distinctCount;
phraseTableFile << " |||";
// tree fragments
if (treeFragmentsFlag && !inverseFlag) {
const std::string *bestTreeFragment = phrasePair.FindBestPropertyValue("Tree");
if (bestTreeFragment) {
phraseTableFile << " {{Tree " << *bestTreeFragment << "}}";
}
}
// parts-of-speech
if (partsOfSpeechFlag && !inverseFlag) {
phrasePair.UpdateVocabularyFromValueTokens("POS", partsOfSpeechSet);
const std::string *bestPartOfSpeech = phrasePair.FindBestPropertyValue("POS");
if (bestPartOfSpeech) {
phraseTableFile << " {{POS " << *bestPartOfSpeech << "}}";
}
}
// syntax labels
if ((sourceSyntaxLabelsFlag || targetSyntacticPreferencesFlag) && !inverseFlag) {
unsigned nNTs = 1;
for(size_t j=0; j<phraseSource->size()-1; ++j) {
if (isNonTerminal(vcbS.getWord( phraseSource->at(j) )))
++nNTs;
}
// source syntax labels
if (sourceSyntaxLabelsFlag) {
std::string sourceLabelCounts;
sourceLabelCounts = phrasePair.CollectAllLabelsSeparateLHSAndRHS("SourceLabels",
sourceLabelSet,
sourceLHSCounts,
targetLHSAndSourceLHSJointCounts,
vcbT);
if ( !sourceLabelCounts.empty() ) {
phraseTableFile << " {{SourceLabels "
<< phraseSource->size() // for convenience: number of symbols in this rule (incl. left hand side NT)
<< " "
<< count // rule count
<< sourceLabelCounts
<< "}}";
}
}
// target syntactic preferences labels
if (targetSyntacticPreferencesFlag) {
std::string targetSyntacticPreferencesLabelCounts;
targetSyntacticPreferencesLabelCounts = phrasePair.CollectAllLabelsSeparateLHSAndRHS("TargetPreferences",
targetSyntacticPreferencesLabelSet,
targetSyntacticPreferencesLHSCounts,
ruleTargetLHSAndTargetSyntacticPreferencesLHSJointCounts,
vcbT);
if (!targetSyntacticPreferencesLabelCounts.empty()) {
phraseTableFile << " {{TargetPreferences "
<< nNTs // for convenience: number of non-terminal symbols in this rule (incl. left hand side NT)
<< " "
<< count // rule count
<< targetSyntacticPreferencesLabelCounts
<< "}}";
}
}
}
// phrase orientation
if (phraseOrientationFlag && !inverseFlag) {
phraseTableFile << " {{Orientation ";
phrasePair.CollectAllPhraseOrientations("Orientation",orientationClassPriorsL2R,orientationClassPriorsR2L,0.5,phraseTableFile);
phraseTableFile << "}}";
}
if (spanLength && !inverseFlag) {
std::string propValue = phrasePair.CollectAllPropertyValues("SpanLength");
if (!propValue.empty()) {
phraseTableFile << " {{SpanLength " << propValue << "}}";
}
}
if (ruleLength && !inverseFlag) {
std::string propValue = phrasePair.CollectAllPropertyValues("RuleLength");
if (!propValue.empty()) {
phraseTableFile << " {{RuleLength " << propValue << "}}";
}
}
if (nonTermContext && !inverseFlag) {
std::string propValue = phrasePair.CollectAllPropertyValues("NonTermContext");
if (!propValue.empty() && propValue.size() < 50000) {
size_t nNTs = NumNonTerminal(phraseSource);
phraseTableFile << " {{NonTermContext " << nNTs << " " << propValue << "}}";
}
}
if (nonTermContextTarget && !inverseFlag) {
std::string propValue = phrasePair.CollectAllPropertyValues("NonTermContextTarget");
if (!propValue.empty() && propValue.size() < 50000) {
size_t nNTs = NumNonTerminal(phraseSource);
phraseTableFile << " {{NonTermContextTarget " << nNTs << " " << propValue << "}}";
}
}
// target constituent boundaries
if (targetConstituentBoundariesFlag && !inverseFlag) {
const std::string targetConstituentBoundariesLeftValues = phrasePair.CollectAllPropertyValues("TargetConstituentBoundariesLeft");
if (!targetConstituentBoundariesLeftValues.empty()) {
phraseTableFile << " {{TargetConstituentBoundariesLeft " << targetConstituentBoundariesLeftValues << "}}";
}
const std::string targetConstituentBoundariesRightAdjacentValues = phrasePair.CollectAllPropertyValues("TargetConstituentBoundariesRightAdjacent");
if (!targetConstituentBoundariesRightAdjacentValues.empty()) {
phraseTableFile << " {{TargetConstituentBoundariesRightAdjacent " << targetConstituentBoundariesRightAdjacentValues << "}}";
}
}
phraseTableFile << std::endl;
}
size_t NumNonTerminal(const PHRASE *phraseSource)
{
size_t nNTs = 0;
for(size_t j=0; j<phraseSource->size()-1; ++j) {
if (isNonTerminal(vcbS.getWord( phraseSource->at(j) )))
++nNTs;
}
return nNTs;
}
void loadOrientationPriors(const std::string &fileNamePhraseOrientationPriors,
std::vector<float> &orientationClassPriorsL2R,
std::vector<float> &orientationClassPriorsR2L)
{
assert(orientationClassPriorsL2R.size()==4 && orientationClassPriorsR2L.size()==4); // mono swap dleft dright
std::cerr << "Loading phrase orientation priors from " << fileNamePhraseOrientationPriors;
Moses::InputFileStream inFile(fileNamePhraseOrientationPriors);
if (inFile.fail()) {
std::cerr << " - ERROR: could not open file" << std::endl;
exit(1);
}
std::string line;
size_t linesRead = 0;
float l2rSum = 0;
float r2lSum = 0;
while (getline(inFile, line)) {
std::istringstream tokenizer(line);
std::string key;
tokenizer >> key;
bool l2rFlag = false;
bool r2lFlag = false;
if (starts_with(key, "L2R_")) {
l2rFlag = true;
}
if (starts_with(key, "R2L_")) {
r2lFlag = true;
}
if (!l2rFlag && !r2lFlag) {
std::cerr << " - ERROR: malformed line in orientation priors file" << std::endl;
}
key.erase(0,4);
int orientationClassId = -1;
if (!key.compare("mono")) {
orientationClassId = 0;
}
if (!key.compare("swap")) {
orientationClassId = 1;
}
if (!key.compare("dleft")) {
orientationClassId = 2;
}
if (!key.compare("dright")) {
orientationClassId = 3;
}
if (orientationClassId == -1) {
std::cerr << " - ERROR: malformed line in orientation priors file" << std::endl;
}
float count;
tokenizer >> count;
if (l2rFlag) {
orientationClassPriorsL2R[orientationClassId] += count;
l2rSum += count;
}
if (r2lFlag) {
orientationClassPriorsR2L[orientationClassId] += count;
r2lSum += count;
}
++linesRead;
}
// normalization: return prior probabilities, not counts
if (l2rSum != 0) {
for (std::vector<float>::iterator orientationClassPriorsL2RIt = orientationClassPriorsL2R.begin();
orientationClassPriorsL2RIt != orientationClassPriorsL2R.end(); ++orientationClassPriorsL2RIt) {
*orientationClassPriorsL2RIt /= l2rSum;
}
}
if (r2lSum != 0) {
for (std::vector<float>::iterator orientationClassPriorsR2LIt = orientationClassPriorsR2L.begin();
orientationClassPriorsR2LIt != orientationClassPriorsR2L.end(); ++orientationClassPriorsR2LIt) {
*orientationClassPriorsR2LIt /= r2lSum;
}
}
std::cerr << " - read " << linesRead << " lines from orientation priors file" << std::endl;
inFile.Close();
}
bool calcCrossedNonTerm( size_t targetPos, size_t sourcePos, const ALIGNMENT *alignmentTargetToSource )
{
for (size_t currTarget = 0; currTarget < alignmentTargetToSource->size(); ++currTarget) {
if (currTarget == targetPos) {
// skip
} else {
const std::set<size_t> &sourceSet = alignmentTargetToSource->at(currTarget);
for (std::set<size_t>::const_iterator iter = sourceSet.begin();
iter != sourceSet.end(); ++iter) {
size_t currSource = *iter;
if ((currTarget < targetPos && currSource > sourcePos)
|| (currTarget > targetPos && currSource < sourcePos)
) {
return true;
}
}
}
}
return false;
}
int calcCrossedNonTerm( const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource )
{
assert(phraseTarget->size() >= alignmentTargetToSource->size() );
for (size_t targetPos = 0; targetPos < alignmentTargetToSource->size(); ++targetPos) {
if ( isNonTerminal(vcbT.getWord( phraseTarget->at(targetPos) ))) {
const std::set<size_t> &alignmentPoints = alignmentTargetToSource->at(targetPos);
assert( alignmentPoints.size() == 1 );
size_t sourcePos = *alignmentPoints.begin();
bool ret = calcCrossedNonTerm(targetPos, sourcePos, alignmentTargetToSource);
if (ret)
return 1;
}
}
return 0;
}
double computeUnalignedPenalty( const ALIGNMENT *alignmentTargetToSource )
{
// unaligned word counter
double unaligned = 1.0;
// only checking target words - source words are caught when computing inverse
for(size_t ti=0; ti<alignmentTargetToSource->size(); ++ti) {
const std::set< size_t > & srcIndices = alignmentTargetToSource->at(ti);
if (srcIndices.empty()) {
unaligned *= 2.718;
}
}
return unaligned;
}
double computeUnalignedFWPenalty( const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource )
{
// unaligned word counter
double unaligned = 1.0;
// only checking target words - source words are caught when computing inverse
for(size_t ti=0; ti<alignmentTargetToSource->size(); ++ti) {
const std::set< size_t > & srcIndices = alignmentTargetToSource->at(ti);
if (srcIndices.empty() && functionWordList.find( vcbT.getWord( phraseTarget->at(ti) ) ) != functionWordList.end()) {
unaligned *= 2.718;
}
}
return unaligned;
}
void loadFunctionWords( const std::string &fileName )
{
std::cerr << "Loading function word list from " << fileName;
Moses::InputFileStream inFile(fileName);
if (inFile.fail()) {
std::cerr << " - ERROR: could not open file" << std::endl;
exit(1);
}
std::string line;
while(getline(inFile, line)) {
std::vector<std::string> token;
Moses::Tokenize( token, line );
if (token.size() > 0)
functionWordList.insert( token[0] );
}
std::cerr << " - read " << functionWordList.size() << " function words" << std::endl;
inFile.Close();
}
double computeLexicalTranslation( const PHRASE *phraseSource, const PHRASE *phraseTarget, const ALIGNMENT *alignmentTargetToSource )
{
// lexical translation probability
double lexScore = 1.0;
int null = vcbS.getWordID("NULL");
// all target words have to be explained
for(size_t ti=0; ti<alignmentTargetToSource->size(); ti++) {
const std::set< size_t > & srcIndices = alignmentTargetToSource->at(ti);
if (srcIndices.empty()) {
// explain unaligned word by NULL
lexScore *= lexTable.permissiveLookup( null, phraseTarget->at(ti) );
} else {
// go through all the aligned words to compute average
double thisWordScore = 0;
for (std::set< size_t >::const_iterator p(srcIndices.begin()); p != srcIndices.end(); ++p) {
thisWordScore += lexTable.permissiveLookup( phraseSource->at(*p), phraseTarget->at(ti) );
}
lexScore *= thisWordScore / (double)srcIndices.size();
}
}
return lexScore;
}
void LexicalTable::load( const std::string &fileName )
{
std::cerr << "Loading lexical translation table from " << fileName;
Moses::InputFileStream inFile(fileName);
if (inFile.fail()) {
std::cerr << " - ERROR: could not open file" << std::endl;
exit(1);
}
std::string line;
int i=0;
while(getline(inFile, line)) {
i++;
if (i%100000 == 0) std::cerr << "." << std::flush;
std::vector<std::string> token;
Moses::Tokenize( token, line );
if (token.size() != 3) {
std::cerr << "line " << i << " in " << fileName
<< " has wrong number of tokens, skipping:" << std::endl
<< token.size() << " " << token[0] << " " << line << std::endl;
continue;
}
double prob = std::atof( token[2].c_str() );
WORD_ID wordT = vcbT.storeIfNew( token[0] );
WORD_ID wordS = vcbS.storeIfNew( token[1] );
ltable[ wordS ][ wordT ] = prob;
}
std::cerr << std::endl;
}
void printSourcePhrase(const PHRASE *phraseSource, const PHRASE *phraseTarget,
const ALIGNMENT *targetToSourceAlignment, std::ostream &out)
{
// get corresponding target non-terminal and output pair
ALIGNMENT *sourceToTargetAlignment = new ALIGNMENT();
invertAlignment(phraseSource, phraseTarget, targetToSourceAlignment, sourceToTargetAlignment);
// output source symbols, except root, in rule table format
for (std::size_t i = 0; i < phraseSource->size()-1; ++i) {
const std::string &word = vcbS.getWord(phraseSource->at(i));
if (!unpairedExtractFormatFlag || !isNonTerminal(word)) {
out << word << " ";
continue;
}
const std::set<std::size_t> &alignmentPoints = sourceToTargetAlignment->at(i);
assert(alignmentPoints.size() == 1);
size_t j = *(alignmentPoints.begin());
if (inverseFlag) {
out << vcbT.getWord(phraseTarget->at(j)) << word << " ";
} else {
out << word << vcbT.getWord(phraseTarget->at(j)) << " ";
}
}
// output source root symbol
if (conditionOnTargetLhsFlag && !inverseFlag) {
out << "[X]";
} else {
out << vcbS.getWord(phraseSource->back());
}
delete sourceToTargetAlignment;
}
void printTargetPhrase(const PHRASE *phraseSource, const PHRASE *phraseTarget,
const ALIGNMENT *targetToSourceAlignment, std::ostream &out)
{
// output target symbols, except root, in rule table format
for (std::size_t i = 0; i < phraseTarget->size()-1; ++i) {
const std::string &word = vcbT.getWord(phraseTarget->at(i));
if (!unpairedExtractFormatFlag || !isNonTerminal(word)) {
out << word << " ";
continue;
}
// get corresponding source non-terminal and output pair
std::set<std::size_t> alignmentPoints = targetToSourceAlignment->at(i);
assert(alignmentPoints.size() == 1);
int j = *(alignmentPoints.begin());
if (inverseFlag) {
out << word << vcbS.getWord(phraseSource->at(j)) << " ";
} else {
out << vcbS.getWord(phraseSource->at(j)) << word << " ";
}
}
// output target root symbol
if (conditionOnTargetLhsFlag) {
if (inverseFlag) {
out << "[X]";
} else {
out << vcbS.getWord(phraseSource->back());
}
} else {
out << vcbT.getWord(phraseTarget->back());
}
}
void invertAlignment(const PHRASE *phraseSource, const PHRASE *phraseTarget,
const ALIGNMENT *inTargetToSourceAlignment, ALIGNMENT *outSourceToTargetAlignment)
{
// typedef std::vector< std::set<size_t> > ALIGNMENT;
outSourceToTargetAlignment->clear();
size_t numberOfSourceSymbols = (hierarchicalFlag ? phraseSource->size()-1 : phraseSource->size());
outSourceToTargetAlignment->resize(numberOfSourceSymbols);
// add alignment point
for (size_t targetPosition = 0; targetPosition < inTargetToSourceAlignment->size(); ++targetPosition) {
for ( std::set<size_t>::iterator setIter = (inTargetToSourceAlignment->at(targetPosition)).begin();
setIter != (inTargetToSourceAlignment->at(targetPosition)).end(); ++setIter ) {
size_t sourcePosition = *setIter;
outSourceToTargetAlignment->at(sourcePosition).insert(targetPosition);
}
}
}
Reference in New Issue View Git Blame Copy Permalink