moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-26 11:28:48 +03:00
Code Issues Packages Projects Releases Wiki Activity

run beautify.perl. Consistent formatting for .h & .cpp files

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3897 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972 2011-02-24 12:39:29 +00:00
parent 508d89eda8
commit 67dd80fb7b
11 changed files with 1515 additions and 1533 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

924
moses-cmd/src/IOWrapper.cpp
View File

File diff suppressed because it is too large Load Diff

134
moses-cmd/src/IOWrapper.h
View File

@ -5,28 +5,28 @@ Moses - factored phrase-based language decoder
Copyright (c) 2006 University of Edinburgh
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***********************************************************************/
@ -55,78 +55,76 @@ POSSIBILITY OF SUCH DAMAGE.
class IOWrapper
{
protected:
long m_translationId;
long m_translationId;
const std::vector<Moses::FactorType> &m_inputFactorOrder;
const std::vector<Moses::FactorType> &m_outputFactorOrder;
const Moses::FactorMask &m_inputFactorUsed;
std::string m_inputFilePath;
Moses::InputFileStream *m_inputFile;
std::istream *m_inputStream;
std::ostream *m_nBestStream
,*m_outputWordGraphStream,*m_outputSearchGraphStream;
const std::vector<Moses::FactorType> &m_inputFactorOrder;
const std::vector<Moses::FactorType> &m_outputFactorOrder;
const Moses::FactorMask &m_inputFactorUsed;
std::string m_inputFilePath;
Moses::InputFileStream *m_inputFile;
std::istream *m_inputStream;
std::ostream *m_nBestStream
,*m_outputWordGraphStream,*m_outputSearchGraphStream;
std::ostream *m_detailedTranslationReportingStream;
std::ofstream *m_alignmentOutputStream;
bool m_surpressSingleBestOutput;
void Initialization(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath);
std::ofstream *m_alignmentOutputStream;
bool m_surpressSingleBestOutput;
void Initialization(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath);
public:
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath);
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath);
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &infilePath);
~IOWrapper();
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
, const std::vector<Moses::FactorType> &outputFactorOrder
, const Moses::FactorMask &inputFactorUsed
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &infilePath);
~IOWrapper();
Moses::InputType* GetInput(Moses::InputType *inputType);
Moses::InputType* GetInput(Moses::InputType *inputType);
void OutputBestHypo(const Moses::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
void OutputLatticeMBRNBestList(const std::vector<LatticeMBRSolution>& solutions,long translationId);
void Backtrack(const Moses::Hypothesis *hypo);
void Backtrack(const Moses::Hypothesis *hypo);
void ResetTranslationId() { m_translationId = 0; }
void ResetTranslationId() {
m_translationId = 0;
}
std::ofstream *GetAlignmentOutputStream()
{
return m_alignmentOutputStream;
}
std::ofstream *GetAlignmentOutputStream() {
return m_alignmentOutputStream;
}
std::ostream &GetOutputWordGraphStream()
{
return *m_outputWordGraphStream;
}
std::ostream &GetOutputSearchGraphStream()
{
return *m_outputSearchGraphStream;
}
std::ostream &GetOutputWordGraphStream() {
return *m_outputWordGraphStream;
}
std::ostream &GetOutputSearchGraphStream() {
return *m_outputSearchGraphStream;
}
std::ostream &GetDetailedTranslationReportingStream()
{
assert (m_detailedTranslationReportingStream);
std::ostream &GetDetailedTranslationReportingStream() {
assert (m_detailedTranslationReportingStream);
return *m_detailedTranslationReportingStream;
}
}
};
IOWrapper *GetIODevice(const Moses::StaticData &staticData);
bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source);
void OutputSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector<Moses::FactorType> &outputFactorOrder ,bool reportSegmentation, bool reportAllFactors);
void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, const std::vector<Moses::FactorType>&,
const TranslationSystem* system, long translationId);
const TranslationSystem* system, long translationId);
void OutputLatticeMBRNBest(std::ostream& out, const std::vector<LatticeMBRSolution>& solutions,long translationId);
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,
bool reportSegmentation, bool reportAllFactors, std::ostream& out);
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,
bool reportSegmentation, bool reportAllFactors, std::ostream& out);
void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,bool reportSegmentation, bool reportAllFactors, std::ostream &out);
void OutputInput(std::ostream& os, const Hypothesis* hypo);
void OutputAlignment(OutputCollector* collector, size_t lineNo, const Hypothesis *hypo);

647
moses-cmd/src/LatticeMBR.cpp
View File

@ -16,32 +16,28 @@ using namespace std;
size_t bleu_order = 4;
float UNKNGRAMLOGPROB = -20;
void GetOutputWords(const TrellisPath &path, vector <Word> &translation){
const std::vector<const Hypothesis *> &edges = path.GetEdges();
// print the surface factor of the translation
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
{
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++)
{
translation.push_back(phrase.GetWord(pos));
}
}
void GetOutputWords(const TrellisPath &path, vector <Word> &translation)
{
const std::vector<const Hypothesis *> &edges = path.GetEdges();
// print the surface factor of the translation
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
translation.push_back(phrase.GetWord(pos));
}
}
}
void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngrams)
{
for (int k = 0; k < (int)bleu_order; k++)
{
for(int i =0; i < max((int)sentence.size()-k,0); i++)
{
for (int k = 0; k < (int)bleu_order; k++) {
for(int i =0; i < max((int)sentence.size()-k,0); i++) {
Phrase ngram(Output);
for ( int j = i; j<= i+k; j++)
{
for ( int j = i; j<= i+k; j++) {
ngram.AddWord(sentence[j]);
}
++allngrams[ngram];
@ -51,86 +47,90 @@ void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngr
void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score) {
set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
if (ngramIter == m_ngrams.end()) {
ngramIter = m_ngrams.insert(ngram).first;
}
map<const Phrase*,float>& ngramScores = m_scores[node];
map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
if (scoreIter == ngramScores.end()) {
ngramScores[&(*ngramIter)] = score;
} else {
ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
}
void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score)
{
set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
if (ngramIter == m_ngrams.end()) {
ngramIter = m_ngrams.insert(ngram).first;
}
map<const Phrase*,float>& ngramScores = m_scores[node];
map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
if (scoreIter == ngramScores.end()) {
ngramScores[&(*ngramIter)] = score;
} else {
ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
}
}
NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node) {
return m_scores[node].begin();
NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node)
{
return m_scores[node].begin();
}
NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node) {
return m_scores[node].end();
NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node)
{
return m_scores[node].end();
}
LatticeMBRSolution::LatticeMBRSolution(const TrellisPath& path, bool isMap) :
m_score(0.0f){
const std::vector<const Hypothesis *> &edges = path.GetEdges();
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
{
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++)
{
m_words.push_back(phrase.GetWord(pos));
}
}
if (isMap) {
m_mapScore = path.GetTotalScore();
} else {
m_mapScore = 0;
m_score(0.0f)
{
const std::vector<const Hypothesis *> &edges = path.GetEdges();
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
m_words.push_back(phrase.GetWord(pos));
}
}
if (isMap) {
m_mapScore = path.GetTotalScore();
} else {
m_mapScore = 0;
}
}
void LatticeMBRSolution::CalcScore(map<Phrase, float>& finalNgramScores, const vector<float>& thetas, float mapWeight) {
m_ngramScores.assign(thetas.size()-1, -10000);
map < Phrase, int > counts;
extract_ngrams(m_words,counts);
//Now score this translation
m_score = thetas[0] * m_words.size();
void LatticeMBRSolution::CalcScore(map<Phrase, float>& finalNgramScores, const vector<float>& thetas, float mapWeight)
{
m_ngramScores.assign(thetas.size()-1, -10000);
//Calculate the ngramScores, working in log space at first
for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) {
float ngramPosterior = UNKNGRAMLOGPROB;
map<Phrase,float>::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first);
if (ngramPosteriorIt != finalNgramScores.end()) {
ngramPosterior = ngramPosteriorIt->second;
}
size_t ngramSize = ngrams->first.GetSize();
m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]);
}
//convert from log to probability and create weighted sum
for (size_t i = 0; i < m_ngramScores.size(); ++i) {
m_ngramScores[i] = exp(m_ngramScores[i]);
m_score += thetas[i+1] * m_ngramScores[i];
}
map < Phrase, int > counts;
extract_ngrams(m_words,counts);
//The map score
m_score += m_mapScore*mapWeight;
//Now score this translation
m_score = thetas[0] * m_words.size();
//Calculate the ngramScores, working in log space at first
for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) {
float ngramPosterior = UNKNGRAMLOGPROB;
map<Phrase,float>::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first);
if (ngramPosteriorIt != finalNgramScores.end()) {
ngramPosterior = ngramPosteriorIt->second;
}
size_t ngramSize = ngrams->first.GetSize();
m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]);
}
//convert from log to probability and create weighted sum
for (size_t i = 0; i < m_ngramScores.size(); ++i) {
m_ngramScores[i] = exp(m_ngramScores[i]);
m_score += thetas[i+1] * m_ngramScores[i];
}
//The map score
m_score += m_mapScore*mapWeight;
}
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale) {
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale)
{
//Need hyp 0 in connectedHyp - Find empty hypothesis
VERBOSE(2,"Pruning lattice to edge density " << edgeDensity << endl);
const Hypothesis* emptyHyp = connectedHyp.at(0);
@ -138,42 +138,42 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
emptyHyp = emptyHyp->GetPrevHypo();
}
connectedHyp.push_back(emptyHyp); //Add it to list of hyps
//Need hyp 0's outgoing Hyps
for (size_t i = 0; i < connectedHyp.size(); ++i) {
if (connectedHyp[i]->GetId() > 0 && connectedHyp[i]->GetPrevHypo()->GetId() == 0)
outgoingHyps[emptyHyp].insert(connectedHyp[i]);
}
//sort hyps based on estimated scores - do so by copying to multimap
multimap<float, const Hypothesis*> sortHypsByVal;
for (size_t i =0; i < estimatedScores.size(); ++i) {
sortHypsByVal.insert(make_pair<float, const Hypothesis*>(estimatedScores[i], connectedHyp[i]));
}
multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end();
float bestScore = it->first;
//store best score as score of hyp 0
sortHypsByVal.insert(make_pair<float, const Hypothesis*>(bestScore, emptyHyp));
IFVERBOSE(3) {
for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
const Hypothesis* currHyp = it->second;
cerr << "Hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl;
}
}
}
set <const Hypothesis*> survivingHyps; //store hyps that make the cut in this
VERBOSE(2, "BEST HYPO TARGET LENGTH : " << bestHypo->GetSize() << endl)
size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs
size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs
size_t numEdgesCreated = 0;
VERBOSE(2, "Target edge count: " << numEdgesTotal << endl);
float prevScore = -999999;
//now iterate over multimap
for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
float currEstimatedScore = it->first;
@ -181,13 +181,13 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
if (numEdgesCreated >= numEdgesTotal && prevScore > currEstimatedScore) //if this hyp has equal estimated score to previous, include its edges too
break;
prevScore = currEstimatedScore;
VERBOSE(3, "Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
VERBOSE(3, "Considering hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl)
survivingHyps.insert(currHyp); //CurrHyp made the cut
// is its best predecessor already included ?
if (survivingHyps.find(currHyp->GetPrevHypo()) != survivingHyps.end()) { //yes, then add an edge
vector <Edge>& edges = incomingEdges[currHyp];
@ -195,7 +195,7 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
edges.push_back(winningEdge);
++numEdgesCreated;
}
//let's try the arcs too
const ArcList *arcList = currHyp->GetArcList();
if (arcList != NULL) {
@ -204,10 +204,10 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
const Hypothesis *loserHypo = *iterArcList;
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
if (survivingHyps.find(loserPrevHypo) != survivingHyps.end()) { //found it, add edge
double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
Edge losingEdge(loserPrevHypo, currHyp, arcScore*scale, loserHypo->GetTargetPhrase());
vector <Edge>& edges = incomingEdges[currHyp];
edges.push_back(losingEdge);
edges.push_back(losingEdge);
++numEdgesCreated;
}
}
@ -215,15 +215,15 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
//Now if a successor node has already been visited, add an edge connecting the two
map < const Hypothesis*, set < const Hypothesis* > >::const_iterator outgoingIt = outgoingHyps.find(currHyp);
if (outgoingIt != outgoingHyps.end()) {//currHyp does have successors
const set<const Hypothesis*> & outHyps = outgoingIt->second; //the successors
for (set<const Hypothesis*>::const_iterator outHypIts = outHyps.begin(); outHypIts != outHyps.end(); ++outHypIts) {
const Hypothesis* succHyp = *outHypIts;
if (survivingHyps.find(succHyp) == survivingHyps.end()) //Have we encountered the successor yet?
continue; //No, move on to next
//Curr Hyp can be : a) the best predecessor of succ b) or an arc attached to succ
if (succHyp->GetPrevHypo() == currHyp) { //best predecessor
vector <Edge>& succEdges = incomingEdges[succHyp];
@ -232,7 +232,7 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
survivingHyps.insert(succHyp);
++numEdgesCreated;
}
//now, let's find an arc
const ArcList *arcList = succHyp->GetArcList();
if (arcList != NULL) {
@ -242,9 +242,9 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
if (loserPrevHypo == currHyp) { //found it
vector <Edge>& succEdges = incomingEdges[succHyp];
double arcScore = loserHypo->GetScore() - currHyp->GetScore();
double arcScore = loserHypo->GetScore() - currHyp->GetScore();
Edge losingEdge(currHyp, succHyp,scale* arcScore, loserHypo->GetTargetPhrase());
succEdges.push_back(losingEdge);
succEdges.push_back(losingEdge);
++numEdgesCreated;
}
}
@ -252,14 +252,14 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
}
}
}
connectedHyp.clear();
for (set <const Hypothesis*>::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) {
connectedHyp.push_back(*it);
}
VERBOSE(2, "Done! Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
IFVERBOSE(3) {
cerr << "Surviving hyps: " ;
for (set <const Hypothesis*>::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) {
@ -267,15 +267,16 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
}
cerr << endl;
}
}
void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges,
map<Phrase, float>& finalNgramScores, bool posteriors) {
void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges,
map<Phrase, float>& finalNgramScores, bool posteriors)
{
sort(connectedHyp.begin(),connectedHyp.end(),ascendingCoverageCmp); //sort by increasing source word cov
/*cerr << "Lattice:" << endl;
for (Lattice::const_iterator i = connectedHyp.begin(); i != connectedHyp.end(); ++i) {
const Hypothesis* h = *i;
@ -285,50 +286,49 @@ void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector
cerr << edges[e];
}
}*/
map<const Hypothesis*, float> forwardScore;
forwardScore[connectedHyp[0]] = 0.0f; //forward score of hyp 0 is 1 (or 0 in logprob space)
set< const Hypothesis *> finalHyps; //store completed hyps
NgramScores ngramScores;//ngram scores for each hyp
NgramScores ngramScores;//ngram scores for each hyp
for (size_t i = 1; i < connectedHyp.size(); ++i) {
const Hypothesis* currHyp = connectedHyp[i];
if (currHyp->GetWordsBitmap().IsComplete()) {
finalHyps.insert(currHyp);
}
VERBOSE(3, "Processing hyp: " << currHyp->GetId() << ", num words cov= " << currHyp->GetWordsBitmap().GetNumWordsCovered() << endl)
vector <Edge> & edges = incomingEdges[currHyp];
for (size_t e = 0; e < edges.size(); ++e) {
const Edge& edge = edges[e];
if (forwardScore.find(currHyp) == forwardScore.end()) {
forwardScore[currHyp] = forwardScore[edge.GetTailNode()] + edge.GetScore();
VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] = fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
}
else {
} else {
forwardScore[currHyp] = log_sum(forwardScore[currHyp], forwardScore[edge.GetTailNode()] + edge.GetScore());
VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] += fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
}
}
//Process ngrams now
for (size_t j =0 ; j < edges.size(); ++j) {
Edge& edge = edges[j];
const NgramHistory & incomingPhrases = edge.GetNgrams(incomingEdges);
//let's first score ngrams introduced by this edge
for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) {
for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) {
const Phrase& ngram = it->first;
const PathCounts& pathCounts = it->second;
VERBOSE(4, "Calculating score for: " << it->first << endl)
for (PathCounts::const_iterator pathCountIt = pathCounts.begin(); pathCountIt != pathCounts.end(); ++pathCountIt) {
//Score of an n-gram is forward score of head node of leftmost edge + all edge scores
const Path& path = pathCountIt->first;
//cerr << "path count for " << ngram << " is " << pathCountIt->second << endl;
float score = forwardScore[path[0]->GetTailNode()];
float score = forwardScore[path[0]->GetTailNode()];
for (size_t i = 0; i < path.size(); ++i) {
score += path[i]->GetScore();
}
@ -340,88 +340,86 @@ void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector
}
}
}
//Now score ngrams that are just being propagated from the history
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
it != ngramScores.nodeEnd(edge.GetTailNode()); ++it) {
const Phrase & currNgram = *(it->first);
float currNgramScore = it->second;
VERBOSE(4, "Calculating score for: " << currNgram << endl)
// For posteriors, don't double count ngrams
if (!posteriors || incomingPhrases.find(currNgram) == incomingPhrases.end()) {
float score = edge.GetScore() + currNgramScore;
ngramScores.addScore(currHyp,currNgram,score);
}
}
}
}
float Z = 9999999; //the total score of the lattice
//Done - Print out ngram posteriors for final hyps
for (set< const Hypothesis *>::iterator finalHyp = finalHyps.begin(); finalHyp != finalHyps.end(); ++finalHyp) {
const Hypothesis* hyp = *finalHyp;
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(hyp); it != ngramScores.nodeEnd(hyp); ++it) {
const Phrase& ngram = *(it->first);
if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
finalNgramScores[ngram] = it->second;
}
else {
finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]);
const Phrase& ngram = *(it->first);
if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
finalNgramScores[ngram] = it->second;
} else {
finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]);
}
}
if (Z == 9999999) {
Z = forwardScore[hyp];
}
else {
} else {
Z = log_sum(Z, forwardScore[hyp]);
}
}
//Z *= scale; //scale the score
for (map<Phrase, float>::iterator finalScoresIt = finalNgramScores.begin(); finalScoresIt != finalNgramScores.end(); ++finalScoresIt) {
finalScoresIt->second = finalScoresIt->second - Z;
IFVERBOSE(2) {
VERBOSE(2,finalScoresIt->first << " [" << finalScoresIt->second << "]" << endl);
}
}
}
const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & incomingEdges) {
const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & incomingEdges)
{
if (m_ngrams.size() > 0)
return m_ngrams;
const Phrase& currPhrase = GetWords();
//Extract the n-grams local to this edge
for (size_t start = 0; start < currPhrase.GetSize(); ++start) {
for (size_t end = start; end < start + bleu_order; ++end) {
if (end < currPhrase.GetSize()){
if (end < currPhrase.GetSize()) {
Phrase edgeNgram(Output);
for (size_t index = start; index <= end; ++index) {
edgeNgram.AddWord(currPhrase.GetWord(index));
edgeNgram.AddWord(currPhrase.GetWord(index));
}
//cout << "Inserting Phrase : " << edgeNgram << endl;
vector<const Edge*> edgeHistory;
edgeHistory.push_back(this);
storeNgramHistory(edgeNgram, edgeHistory);
}
else {
} else {
break;
}
}
}
map<const Hypothesis*, vector<Edge> >::iterator it = incomingEdges.find(m_tailNode);
if (it != incomingEdges.end()) { //node has incoming edges
vector<Edge> & inEdges = it->second;
for (vector<Edge>::iterator edge = inEdges.begin(); edge != inEdges.end(); ++edge) {//add the ngrams straddling prev and curr edge
const NgramHistory & edgeIncomingNgrams = edge->GetNgrams(incomingEdges);
for (NgramHistory::const_iterator edgeInNgramHist = edgeIncomingNgrams.begin(); edgeInNgramHist != edgeIncomingNgrams.end(); ++edgeInNgramHist) {
@ -432,39 +430,40 @@ const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & inco
IFVERBOSE(3) {
cerr << "Edge: "<< *edge <<endl;
cerr << "edgeWords: " << edgeWords << endl;
cerr << "edgeInNgram: " << edgeIncomingNgram << endl;
cerr << "edgeInNgram: " << edgeIncomingNgram << endl;
}
Phrase edgeSuffix(Output);
Phrase ngramSuffix(Output);
GetPhraseSuffix(edgeWords,back,edgeSuffix);
GetPhraseSuffix(edgeIncomingNgram,back,ngramSuffix);
if (ngramSuffix == edgeSuffix) { //we've got the suffix of previous edge
size_t edgeInNgramSize = edgeIncomingNgram.GetSize();
for (size_t i = 0; i < GetWordsSize() && i + edgeInNgramSize < bleu_order ; ++i){
for (size_t i = 0; i < GetWordsSize() && i + edgeInNgramSize < bleu_order ; ++i) {
Phrase newNgram(edgeIncomingNgram);
for (size_t j = 0; j <= i ; ++j){
for (size_t j = 0; j <= i ; ++j) {
newNgram.AddWord(GetWords().GetWord(j));
}
VERBOSE(3, "Inserting New Phrase : " << newNgram << endl)
for (PathCounts::const_iterator pathIt = edgeIncomingNgramPaths.begin(); pathIt != edgeIncomingNgramPaths.end(); ++pathIt) {
Path newNgramPath = pathIt->first;
newNgramPath.push_back(this);
storeNgramHistory(newNgram, newNgramPath, pathIt->second);
storeNgramHistory(newNgram, newNgramPath, pathIt->second);
}
}
}
}
}
}
}
return m_ngrams;
}
//Add the last lastN words of origPhrase to targetPhrase
void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const {
void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const
{
size_t origSize = origPhrase.GetSize();
size_t startIndex = origSize - lastN;
for (size_t index = startIndex; index < origPhrase.GetSize(); ++index) {
@ -472,7 +471,8 @@ void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targ
}
}
bool Edge::operator< (const Edge& compare ) const {
bool Edge::operator< (const Edge& compare ) const
{
if (m_headNode->GetId() < compare.m_headNode->GetId())
return true;
if (compare.m_headNode->GetId() < m_headNode->GetId())
@ -484,178 +484,179 @@ bool Edge::operator< (const Edge& compare ) const {
return GetScore() < compare.GetScore();
}
ostream& operator<< (ostream& out, const Edge& edge) {
ostream& operator<< (ostream& out, const Edge& edge)
{
out << "Head: " << edge.m_headNode->GetId() << ", Tail: " << edge.m_tailNode->GetId() << ", Score: " << edge.m_score << ", Phrase: " << edge.m_targetPhrase << endl;
return out;
}
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b) {
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b)
{
return a->GetWordsBitmap().GetNumWordsCovered() < b->GetWordsBitmap().GetNumWordsCovered();
}
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList,
vector<LatticeMBRSolution>& solutions, size_t n)
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList,
vector<LatticeMBRSolution>& solutions, size_t n)
{
const StaticData& staticData = StaticData::Instance();
std::map < int, bool > connected;
std::vector< const Hypothesis *> connectedList;
map<Phrase, float> ngramPosteriors;
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
map<const Hypothesis*, vector<Edge> > incomingEdges;
vector< float> estimatedScores;
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
vector<float> mbrThetas = staticData.GetLatticeMBRThetas();
float p = staticData.GetLatticeMBRPrecision();
float r = staticData.GetLatticeMBRPRatio();
float mapWeight = staticData.GetLatticeMBRMapWeight();
if (mbrThetas.size() == 0) { //thetas not specified on the command line, use p and r instead
mbrThetas.push_back(-1); //Theta 0
mbrThetas.push_back(1/(bleu_order*p));
for (size_t i = 2; i <= bleu_order; ++i){
mbrThetas.push_back(mbrThetas[i-1] / r);
}
const StaticData& staticData = StaticData::Instance();
std::map < int, bool > connected;
std::vector< const Hypothesis *> connectedList;
map<Phrase, float> ngramPosteriors;
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
map<const Hypothesis*, vector<Edge> > incomingEdges;
vector< float> estimatedScores;
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
vector<float> mbrThetas = staticData.GetLatticeMBRThetas();
float p = staticData.GetLatticeMBRPrecision();
float r = staticData.GetLatticeMBRPRatio();
float mapWeight = staticData.GetLatticeMBRMapWeight();
if (mbrThetas.size() == 0) { //thetas not specified on the command line, use p and r instead
mbrThetas.push_back(-1); //Theta 0
mbrThetas.push_back(1/(bleu_order*p));
for (size_t i = 2; i <= bleu_order; ++i) {
mbrThetas.push_back(mbrThetas[i-1] / r);
}
IFVERBOSE(2) {
VERBOSE(2,"Thetas: ");
for (size_t i = 0; i < mbrThetas.size(); ++i) {
VERBOSE(2,mbrThetas[i] << " ");
}
VERBOSE(2,endl);
}
IFVERBOSE(2) {
VERBOSE(2,"Thetas: ");
for (size_t i = 0; i < mbrThetas.size(); ++i) {
VERBOSE(2,mbrThetas[i] << " ");
}
TrellisPathList::const_iterator iter;
size_t ctr = 0;
LatticeMBRSolutionComparator comparator;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr)
{
const TrellisPath &path = **iter;
solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin()));
solutions.back().CalcScore(ngramPosteriors,mbrThetas,mapWeight);
sort(solutions.begin(), solutions.end(), comparator);
while (solutions.size() > n) {
solutions.pop_back();
}
VERBOSE(2,endl);
}
TrellisPathList::const_iterator iter;
size_t ctr = 0;
LatticeMBRSolutionComparator comparator;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr) {
const TrellisPath &path = **iter;
solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin()));
solutions.back().CalcScore(ngramPosteriors,mbrThetas,mapWeight);
sort(solutions.begin(), solutions.end(), comparator);
while (solutions.size() > n) {
solutions.pop_back();
}
VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl);
}
VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl);
}
vector<Word> doLatticeMBR(Manager& manager, TrellisPathList& nBestList) {
vector<Word> doLatticeMBR(Manager& manager, TrellisPathList& nBestList)
{
vector<LatticeMBRSolution> solutions;
getLatticeMBRNBest(manager, nBestList, solutions,1);
return solutions.at(0).GetWords();
vector<LatticeMBRSolution> solutions;
getLatticeMBRNBest(manager, nBestList, solutions,1);
return solutions.at(0).GetWords();
}
const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList) {
static const int BLEU_ORDER = 4;
static const float SMOOTH = 1;
//calculate the ngram expectations
const StaticData& staticData = StaticData::Instance();
std::map < int, bool > connected;
std::vector< const Hypothesis *> connectedList;
map<Phrase, float> ngramExpectations;
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
map<const Hypothesis*, vector<Edge> > incomingEdges;
vector< float> estimatedScores;
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
//expected length is sum of expected unigram counts
//cerr << "Thread " << pthread_self() << " Ngram expectations size: " << ngramExpectations.size() << endl;
float ref_length = 0.0f;
for (map<Phrase,float>::const_iterator ref_iter = ngramExpectations.begin();
ref_iter != ngramExpectations.end(); ++ref_iter) {
//cerr << "Ngram: " << ref_iter->first << " score: " <<
// ref_iter->second << endl;
if (ref_iter->first.GetSize() == 1) {
ref_length += exp(ref_iter->second);
// cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl;
}
}
VERBOSE(2,"REF Length: " << ref_length << endl);
const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList)
{
static const int BLEU_ORDER = 4;
static const float SMOOTH = 1;
//use the ngram expectations to rescore the nbest list.
TrellisPathList::const_iterator iter;
TrellisPathList::const_iterator best = nBestList.end();
float bestScore = -100000;
//cerr << "nbest list size: " << nBestList.GetSize() << endl;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
{
const TrellisPath &path = **iter;
vector<Word> words;
map<Phrase,int> ngrams;
GetOutputWords(path,words);
/*for (size_t i = 0; i < words.size(); ++i) {
cerr << words[i].GetFactor(0)->GetString() << " ";
}
cerr << endl;
*/
extract_ngrams(words,ngrams);
vector<float> comps(2*BLEU_ORDER+1);
float logbleu = 0.0;
float brevity = 0.0;
int hyp_length = words.size();
for (int i = 0; i < BLEU_ORDER; ++i) {
comps[2*i] = 0.0;
comps[2*i+1] = max(hyp_length-i,0);
}
for (map<Phrase,int>::const_iterator hyp_iter = ngrams.begin();
hyp_iter != ngrams.end(); ++hyp_iter) {
map<Phrase,float>::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first);
if (ref_iter != ngramExpectations.end()) {
comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second));
}
}
comps[comps.size()-1] = ref_length;
/*for (size_t i = 0; i < comps.size(); ++i) {
cerr << comps[i] << " ";
}
cerr << endl;
*/
float score = 0.0f;
if (comps[0] != 0) {
for (int i=0; i<BLEU_ORDER; i++)
{
if ( i > 0 ) {
logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
} else {
logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
}
}
logbleu /= BLEU_ORDER;
brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
if (brevity < 0.0) {
logbleu += brevity;
}
score = exp(logbleu);
}
//cerr << "score: " << score << " bestScore: " << bestScore << endl;
if (score > bestScore) {
bestScore = score;
best = iter;
VERBOSE(2,"NEW BEST: " << score << endl);
//for (size_t i = 0; i < comps.size(); ++i) {
// cerr << comps[i] << " ";
//}
//cerr << endl;
}
//calculate the ngram expectations
const StaticData& staticData = StaticData::Instance();
std::map < int, bool > connected;
std::vector< const Hypothesis *> connectedList;
map<Phrase, float> ngramExpectations;
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
map<const Hypothesis*, vector<Edge> > incomingEdges;
vector< float> estimatedScores;
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
//expected length is sum of expected unigram counts
//cerr << "Thread " << pthread_self() << " Ngram expectations size: " << ngramExpectations.size() << endl;
float ref_length = 0.0f;
for (map<Phrase,float>::const_iterator ref_iter = ngramExpectations.begin();
ref_iter != ngramExpectations.end(); ++ref_iter) {
//cerr << "Ngram: " << ref_iter->first << " score: " <<
// ref_iter->second << endl;
if (ref_iter->first.GetSize() == 1) {
ref_length += exp(ref_iter->second);
// cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl;
}
assert (best != nBestList.end());
return **best;
//vector<Word> bestWords;
//GetOutputWords(**best,bestWords);
//return bestWords;
}
VERBOSE(2,"REF Length: " << ref_length << endl);
//use the ngram expectations to rescore the nbest list.
TrellisPathList::const_iterator iter;
TrellisPathList::const_iterator best = nBestList.end();
float bestScore = -100000;
//cerr << "nbest list size: " << nBestList.GetSize() << endl;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
const TrellisPath &path = **iter;
vector<Word> words;
map<Phrase,int> ngrams;
GetOutputWords(path,words);
/*for (size_t i = 0; i < words.size(); ++i) {
cerr << words[i].GetFactor(0)->GetString() << " ";
}
cerr << endl;
*/
extract_ngrams(words,ngrams);
vector<float> comps(2*BLEU_ORDER+1);
float logbleu = 0.0;
float brevity = 0.0;
int hyp_length = words.size();
for (int i = 0; i < BLEU_ORDER; ++i) {
comps[2*i] = 0.0;
comps[2*i+1] = max(hyp_length-i,0);
}
for (map<Phrase,int>::const_iterator hyp_iter = ngrams.begin();
hyp_iter != ngrams.end(); ++hyp_iter) {
map<Phrase,float>::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first);
if (ref_iter != ngramExpectations.end()) {
comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second));
}
}
comps[comps.size()-1] = ref_length;
/*for (size_t i = 0; i < comps.size(); ++i) {
cerr << comps[i] << " ";
}
cerr << endl;
*/
float score = 0.0f;
if (comps[0] != 0) {
for (int i=0; i<BLEU_ORDER; i++) {
if ( i > 0 ) {
logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
} else {
logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
}
}
logbleu /= BLEU_ORDER;
brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
if (brevity < 0.0) {
logbleu += brevity;
}
score = exp(logbleu);
}
//cerr << "score: " << score << " bestScore: " << bestScore << endl;
if (score > bestScore) {
bestScore = score;
best = iter;
VERBOSE(2,"NEW BEST: " << score << endl);
//for (size_t i = 0; i < comps.size(); ++i) {
// cerr << comps[i] << " ";
//}
//cerr << endl;
}
}
assert (best != nBestList.end());
return **best;
//vector<Word> bestWords;
//GetOutputWords(**best,bestWords);
//return bestWords;
}

149
moses-cmd/src/LatticeMBR.h
View File

@ -34,111 +34,122 @@ T log_sum (T log_a, T log_b)
class Edge;
typedef std::vector< const Hypothesis *> Lattice;
typedef std::vector<const Edge*> Path;
typedef std::vector<const Edge*> Path;
typedef std::map<Path, size_t> PathCounts;
typedef std::map<Phrase, PathCounts > NgramHistory;
class Edge {
const Hypothesis* m_tailNode;
const Hypothesis* m_headNode;
float m_score;
TargetPhrase m_targetPhrase;
NgramHistory m_ngrams;
public:
Edge(const Hypothesis* from, const Hypothesis* to, float score, const TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) {
//cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl;
}
const Hypothesis* GetHeadNode() const {
return m_headNode;
class Edge
{
const Hypothesis* m_tailNode;
const Hypothesis* m_headNode;
float m_score;
TargetPhrase m_targetPhrase;
NgramHistory m_ngrams;
public:
Edge(const Hypothesis* from, const Hypothesis* to, float score, const TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) {
//cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl;
}
const Hypothesis* GetHeadNode() const {
return m_headNode;
}
const Hypothesis* GetTailNode() const {
return m_tailNode;
}
float GetScore() const {
return m_score;
}
size_t GetWordsSize() const {
return m_targetPhrase.GetSize();
}
const Phrase& GetWords() const {
return m_targetPhrase;
}
friend std::ostream& operator<< (std::ostream& out, const Edge& edge);
const NgramHistory& GetNgrams( std::map<const Hypothesis*, std::vector<Edge> > & incomingEdges) ;
bool operator < (const Edge & compare) const;
void GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const;
void storeNgramHistory(const Phrase& phrase, Path & path, size_t count = 1){
m_ngrams[phrase][path]+= count;
}
friend std::ostream& operator<< (std::ostream& out, const Edge& edge);
const NgramHistory& GetNgrams( std::map<const Hypothesis*, std::vector<Edge> > & incomingEdges) ;
bool operator < (const Edge & compare) const;
void GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const;
void storeNgramHistory(const Phrase& phrase, Path & path, size_t count = 1) {
m_ngrams[phrase][path]+= count;
}
};
/**
* Data structure to hold the ngram scores as we traverse the lattice. Maps (hypo,ngram) to score
*/
class NgramScores {
public:
NgramScores() {}
/** logsum this score to the existing score */
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
/** Iterate through ngrams for selected node */
typedef std::map<const Phrase*, float>::const_iterator NodeScoreIterator;
NodeScoreIterator nodeBegin(const Hypothesis* node);
NodeScoreIterator nodeEnd(const Hypothesis* node);
private:
std::set<Phrase> m_ngrams;
std::map<const Hypothesis*, std::map<const Phrase*, float> > m_scores;
class NgramScores
{
public:
NgramScores() {}
/** logsum this score to the existing score */
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
/** Iterate through ngrams for selected node */
typedef std::map<const Phrase*, float>::const_iterator NodeScoreIterator;
NodeScoreIterator nodeBegin(const Hypothesis* node);
NodeScoreIterator nodeEnd(const Hypothesis* node);
private:
std::set<Phrase> m_ngrams;
std::map<const Hypothesis*, std::map<const Phrase*, float> > m_scores;
};
/** Holds a lattice mbr solution, and its scores */
class LatticeMBRSolution {
public:
/** Read the words from the path */
LatticeMBRSolution(const TrellisPath& path, bool isMap);
const std::vector<float>& GetNgramScores() const {return m_ngramScores;}
const std::vector<Word>& GetWords() const {return m_words;}
float GetMapScore() const {return m_mapScore;}
float GetScore() const {return m_score;}
/** Initialise ngram scores */
void CalcScore(std::map<Phrase, float>& finalNgramScores, const std::vector<float>& thetas, float mapWeight);
private:
std::vector<Word> m_words;
float m_mapScore;
std::vector<float> m_ngramScores;
float m_score;
class LatticeMBRSolution
{
public:
/** Read the words from the path */
LatticeMBRSolution(const TrellisPath& path, bool isMap);
const std::vector<float>& GetNgramScores() const {
return m_ngramScores;
}
const std::vector<Word>& GetWords() const {
return m_words;
}
float GetMapScore() const {
return m_mapScore;
}
float GetScore() const {
return m_score;
}
/** Initialise ngram scores */
void CalcScore(std::map<Phrase, float>& finalNgramScores, const std::vector<float>& thetas, float mapWeight);
private:
std::vector<Word> m_words;
float m_mapScore;
std::vector<float> m_ngramScores;
float m_score;
};
struct LatticeMBRSolutionComparator {
bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) {
return a.GetScore() > b.GetScore();
}
bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) {
return a.GetScore() > b.GetScore();
}
};
void pruneLatticeFB(Lattice & connectedHyp, std::map < const Hypothesis*, std::set <const Hypothesis* > > & outgoingHyps, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges,
void pruneLatticeFB(Lattice & connectedHyp, std::map < const Hypothesis*, std::set <const Hypothesis* > > & outgoingHyps, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges,
const std::vector< float> & estimatedScores, const Hypothesis*, size_t edgeDensity,float scale);
//Use the ngram scores to rerank the nbest list, return at most n solutions
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList, std::vector<LatticeMBRSolution>& solutions, size_t n);
//calculate expectated ngram counts, clipping at 1 (ie calculating posteriors) if posteriors==true.
void calcNgramExpectations(Lattice & connectedHyp, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges, std::map<Phrase,
float>& finalNgramScores, bool posteriors);
float>& finalNgramScores, bool posteriors);
void GetOutputFactors(const TrellisPath &path, std::vector <Word> &translation);
void extract_ngrams(const std::vector<Word >& sentence, std::map < Phrase, int > & allngrams);
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b);

278
moses-cmd/src/LatticeMBRGrid.cpp
View File

@ -33,7 +33,7 @@ POSSIBILITY OF SUCH DAMAGE.
* Lattice MBR grid search. Enables a grid search through the four parameters (p,r,scale and prune) used in lattice MBR.
See 'Lattice Minimum Bayes-Risk Decoding for Statistical Machine Translation by Tromble, Kumar, Och and Macherey,
EMNLP 2008 for details of the parameters.
The grid search is controlled by specifying comma separated lists for the lmbr parameters (-lmbr-p, -lmbr-r,
-lmbr-pruning-factor and -mbr-scale). All other parameters are passed through to moses. If any of the lattice mbr
parameters are missing, then they are set to their default values. Output is of the form:
@ -58,148 +58,150 @@ using namespace Moses;
//keys
enum gridkey {lmbr_p,lmbr_r,lmbr_prune,lmbr_scale};
class Grid {
public:
/** Add a parameter with key, command line argument, and default value */
void addParam(gridkey key, const string& arg, float defaultValue) {
m_args[arg] = key;
assert(m_grid.find(key) == m_grid.end());
m_grid[key].push_back(defaultValue);
}
/** Parse the arguments, removing those that define the grid and returning a copy of the rest */
void parseArgs(int& argc, char**& argv) {
char** newargv = new char*[argc+1]; //Space to add mbr parameter
int newargc = 0;
for (int i = 0; i < argc; ++i) {
bool consumed = false;
for (map<string,gridkey>::const_iterator argi = m_args.begin(); argi != m_args.end(); ++argi) {
if (!strcmp(argv[i], argi->first.c_str())) {
++i;
if (i >= argc) {
cerr << "Error: missing parameter for " << argi->first << endl;
throw runtime_error("Missing parameter");
} else {
string value = argv[i];
gridkey key = argi->second;
if (m_grid[key].size() != 1) {
throw runtime_error("Duplicate grid argument");
}
m_grid[key].clear();
char delim = ',';
string::size_type lastpos = value.find_first_not_of(delim);
string::size_type pos = value.find_first_of(delim,lastpos);
while (string::npos != pos || string::npos != lastpos) {
float param = atof(value.substr(lastpos, pos-lastpos).c_str());
if (!param) {
cerr << "Error: Illegal grid parameter for " << argi->first << endl;
throw runtime_error("Illegal grid parameter");
}
m_grid[key].push_back(param);
lastpos = value.find_first_not_of(delim,pos);
pos = value.find_first_of(delim,lastpos);
}
consumed = true;
}
if (consumed) break;
}
}
if (!consumed) {
newargv[newargc] = new char[strlen(argv[i]) + 1];
strcpy(newargv[newargc],argv[i]);
++newargc;
}
class Grid
{
public:
/** Add a parameter with key, command line argument, and default value */
void addParam(gridkey key, const string& arg, float defaultValue) {
m_args[arg] = key;
assert(m_grid.find(key) == m_grid.end());
m_grid[key].push_back(defaultValue);
}
/** Parse the arguments, removing those that define the grid and returning a copy of the rest */
void parseArgs(int& argc, char**& argv) {
char** newargv = new char*[argc+1]; //Space to add mbr parameter
int newargc = 0;
for (int i = 0; i < argc; ++i) {
bool consumed = false;
for (map<string,gridkey>::const_iterator argi = m_args.begin(); argi != m_args.end(); ++argi) {
if (!strcmp(argv[i], argi->first.c_str())) {
++i;
if (i >= argc) {
cerr << "Error: missing parameter for " << argi->first << endl;
throw runtime_error("Missing parameter");
} else {
string value = argv[i];
gridkey key = argi->second;
if (m_grid[key].size() != 1) {
throw runtime_error("Duplicate grid argument");
}
argc = newargc;
argv = newargv;
m_grid[key].clear();
char delim = ',';
string::size_type lastpos = value.find_first_not_of(delim);
string::size_type pos = value.find_first_of(delim,lastpos);
while (string::npos != pos || string::npos != lastpos) {
float param = atof(value.substr(lastpos, pos-lastpos).c_str());
if (!param) {
cerr << "Error: Illegal grid parameter for " << argi->first << endl;
throw runtime_error("Illegal grid parameter");
}
m_grid[key].push_back(param);
lastpos = value.find_first_not_of(delim,pos);
pos = value.find_first_of(delim,lastpos);
}
consumed = true;
}
if (consumed) break;
}
/** Get the grid for a particular key.*/
const vector<float>& getGrid(gridkey key) const {
map<gridkey,vector<float> >::const_iterator iter = m_grid.find(key);
assert (iter != m_grid.end());
return iter->second;
}
private:
map<gridkey,vector<float> > m_grid;
map<string,gridkey> m_args;
}
if (!consumed) {
newargv[newargc] = new char[strlen(argv[i]) + 1];
strcpy(newargv[newargc],argv[i]);
++newargc;
}
}
argc = newargc;
argv = newargv;
}
/** Get the grid for a particular key.*/
const vector<float>& getGrid(gridkey key) const {
map<gridkey,vector<float> >::const_iterator iter = m_grid.find(key);
assert (iter != m_grid.end());
return iter->second;
}
private:
map<gridkey,vector<float> > m_grid;
map<string,gridkey> m_args;
};
int main(int argc, char* argv[]) {
cerr << "Lattice MBR Grid search" << endl;
Grid grid;
grid.addParam(lmbr_p, "-lmbr-p", 0.5);
grid.addParam(lmbr_r, "-lmbr-r", 0.5);
grid.addParam(lmbr_prune, "-lmbr-pruning-factor",30.0);
grid.addParam(lmbr_scale, "-mbr-scale",1.0);
grid.parseArgs(argc,argv);
Parameter* params = new Parameter();
if (!params->LoadParam(argc,argv)) {
params->Explain();
exit(1);
}
if (!StaticData::LoadDataStatic(params)) {
exit(1);
}
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
staticData.SetUseLatticeMBR(true);
IOWrapper* ioWrapper = GetIODevice(staticData);
int main(int argc, char* argv[])
{
cerr << "Lattice MBR Grid search" << endl;
if (!ioWrapper) {
throw runtime_error("Failed to initialise IOWrapper");
}
size_t nBestSize = staticData.GetMBRSize();
if (nBestSize <= 0){
throw new runtime_error("Non-positive size specified for n-best list");
}
size_t lineCount = 0;
InputType* source = NULL;
const vector<float>& pgrid = grid.getGrid(lmbr_p);
const vector<float>& rgrid = grid.getGrid(lmbr_r);
const vector<float>& prune_grid = grid.getGrid(lmbr_prune);
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
++lineCount;
Sentence sentence(Input);
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
Manager manager(*source,staticData.GetSearchAlgorithm(), &system);
manager.ProcessSentence();
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
//grid search
for (vector<float>::const_iterator pi = pgrid.begin(); pi != pgrid.end(); ++pi) {
float p = *pi;
staticData.SetLatticeMBRPrecision(p);
for (vector<float>::const_iterator ri = rgrid.begin(); ri != rgrid.end(); ++ri) {
float r = *ri;
staticData.SetLatticeMBRPRatio(r);
for (vector<float>::const_iterator prune_i = prune_grid.begin(); prune_i != prune_grid.end(); ++prune_i) {
size_t prune = (size_t)(*prune_i);
staticData.SetLatticeMBRPruningFactor(prune);
for (vector<float>::const_iterator scale_i = scale_grid.begin(); scale_i != scale_grid.end(); ++scale_i) {
float scale = *scale_i;
staticData.SetMBRScale(scale);
cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| ";
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),cout);
}
}
}
Grid grid;
grid.addParam(lmbr_p, "-lmbr-p", 0.5);
grid.addParam(lmbr_r, "-lmbr-r", 0.5);
grid.addParam(lmbr_prune, "-lmbr-pruning-factor",30.0);
grid.addParam(lmbr_scale, "-mbr-scale",1.0);
grid.parseArgs(argc,argv);
Parameter* params = new Parameter();
if (!params->LoadParam(argc,argv)) {
params->Explain();
exit(1);
}
if (!StaticData::LoadDataStatic(params)) {
exit(1);
}
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
staticData.SetUseLatticeMBR(true);
IOWrapper* ioWrapper = GetIODevice(staticData);
if (!ioWrapper) {
throw runtime_error("Failed to initialise IOWrapper");
}
size_t nBestSize = staticData.GetMBRSize();
if (nBestSize <= 0) {
throw new runtime_error("Non-positive size specified for n-best list");
}
size_t lineCount = 0;
InputType* source = NULL;
const vector<float>& pgrid = grid.getGrid(lmbr_p);
const vector<float>& rgrid = grid.getGrid(lmbr_r);
const vector<float>& prune_grid = grid.getGrid(lmbr_prune);
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
++lineCount;
Sentence sentence(Input);
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
Manager manager(*source,staticData.GetSearchAlgorithm(), &system);
manager.ProcessSentence();
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
//grid search
for (vector<float>::const_iterator pi = pgrid.begin(); pi != pgrid.end(); ++pi) {
float p = *pi;
staticData.SetLatticeMBRPrecision(p);
for (vector<float>::const_iterator ri = rgrid.begin(); ri != rgrid.end(); ++ri) {
float r = *ri;
staticData.SetLatticeMBRPRatio(r);
for (vector<float>::const_iterator prune_i = prune_grid.begin(); prune_i != prune_grid.end(); ++prune_i) {
size_t prune = (size_t)(*prune_i);
staticData.SetLatticeMBRPruningFactor(prune);
for (vector<float>::const_iterator scale_i = scale_grid.begin(); scale_i != scale_grid.end(); ++scale_i) {
float scale = *scale_i;
staticData.SetMBRScale(scale);
cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| ";
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),cout);
}
}
}
}
}
}

587
moses-cmd/src/Main.cpp
View File

@ -53,206 +53,211 @@ using namespace Moses;
static const size_t PRECISION = 3;
/** Enforce rounding */
void fix(std::ostream& stream, size_t size) {
stream.setf(std::ios::fixed);
stream.precision(size);
void fix(std::ostream& stream, size_t size)
{
stream.setf(std::ios::fixed);
stream.precision(size);
}
/**
* Translates a sentence.
**/
class TranslationTask : public Task {
class TranslationTask : public Task
{
public:
public:
TranslationTask(size_t lineNumber,
InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
OutputCollector* detailedTranslationCollector,
OutputCollector* alignmentInfoCollector ) :
m_source(source), m_lineNumber(lineNumber),
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
m_detailedTranslationCollector(detailedTranslationCollector),
m_alignmentInfoCollector(alignmentInfoCollector) {}
TranslationTask(size_t lineNumber,
InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
OutputCollector* detailedTranslationCollector,
OutputCollector* alignmentInfoCollector ) :
m_source(source), m_lineNumber(lineNumber),
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
m_detailedTranslationCollector(detailedTranslationCollector),
m_alignmentInfoCollector(alignmentInfoCollector) {}
void Run()
{
void Run() {
#ifdef BOOST_HAS_PTHREADS
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl);
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl);
#endif
const StaticData &staticData = StaticData::Instance();
Sentence sentence(Input);
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
Manager manager(*m_source,staticData.GetSearchAlgorithm(), &system);
manager.ProcessSentence();
//Word Graph
if (m_wordGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.GetWordGraph(m_lineNumber, out);
m_wordGraphCollector->Write(m_lineNumber, out.str());
}
//Search Graph
if (m_searchGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraph(m_lineNumber, out);
m_searchGraphCollector->Write(m_lineNumber, out.str());
const StaticData &staticData = StaticData::Instance();
Sentence sentence(Input);
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
Manager manager(*m_source,staticData.GetSearchAlgorithm(), &system);
manager.ProcessSentence();
//Word Graph
if (m_wordGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.GetWordGraph(m_lineNumber, out);
m_wordGraphCollector->Write(m_lineNumber, out.str());
}
//Search Graph
if (m_searchGraphCollector) {
ostringstream out;
fix(out,PRECISION);
manager.OutputSearchGraph(m_lineNumber, out);
m_searchGraphCollector->Write(m_lineNumber, out.str());
#ifdef HAVE_PROTOBUF
if (staticData.GetOutputSearchGraphPB()) {
ostringstream sfn;
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
string fn = sfn.str();
VERBOSE(2, "Writing search graph to " << fn << endl);
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
manager.SerializeSearchGraphPB(m_lineNumber, output);
}
if (staticData.GetOutputSearchGraphPB()) {
ostringstream sfn;
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
string fn = sfn.str();
VERBOSE(2, "Writing search graph to " << fn << endl);
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
manager.SerializeSearchGraphPB(m_lineNumber, output);
}
#endif
}
if (m_outputCollector) {
ostringstream out;
ostringstream debug;
fix(debug,PRECISION);
//All derivations - send them to debug stream
if (staticData.PrintAllDerivations()) {
manager.PrintAllDerivations(m_lineNumber, debug);
}
//Best hypothesis
const Hypothesis* bestHypo = NULL;
if (!staticData.UseMBR()) {
bestHypo = manager.GetBestHypothesis();
if (bestHypo) {
if (staticData.IsPathRecoveryEnabled()) {
OutputInput(out, bestHypo);
out << "||| ";
}
OutputSurface(
out,
bestHypo,
staticData.GetOutputFactorOrder(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors());
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
}
out << endl;
}
else
{
size_t nBestSize = staticData.GetMBRSize();
if (nBestSize <= 0)
{
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
exit(1);
}
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
IFVERBOSE(2) { PrintUserTime("calculated n-best list for (L)MBR decoding"); }
if (staticData.UseLatticeMBR())
{
if (m_nbestCollector)
{
//lattice mbr nbest
vector<LatticeMBRSolution> solutions;
size_t n = min(nBestSize, staticData.GetNBestSize());
getLatticeMBRNBest(manager,nBestList,solutions,n);
ostringstream out;
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
m_nbestCollector->Write(m_lineNumber, out.str());
}
else
{
//Lattice MBR decoding
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
IFVERBOSE(2) { PrintUserTime("finished Lattice MBR decoding"); }
}
}
else if (staticData.UseConsensusDecoding()) {
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
OutputBestHypo(conBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
IFVERBOSE(2) { PrintUserTime("finished Consensus decoding"); }
}
else
{
//MBR decoding
const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
IFVERBOSE(2) { PrintUserTime("finished MBR decoding"); }
}
}
m_outputCollector->Write(m_lineNumber,out.str(),debug.str());
}
if (m_nbestCollector && !staticData.UseLatticeMBR()) {
TrellisPathList nBestList;
ostringstream out;
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
OutputNBest(out,nBestList, staticData.GetOutputFactorOrder(), manager.GetTranslationSystem(), m_lineNumber);
m_nbestCollector->Write(m_lineNumber, out.str());
}
//detailed translation reporting
if (m_detailedTranslationCollector) {
ostringstream out;
fix(out,PRECISION);
TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
}
}
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
manager.CalcDecoderStatistics();
if (m_outputCollector) {
ostringstream out;
ostringstream debug;
fix(debug,PRECISION);
//All derivations - send them to debug stream
if (staticData.PrintAllDerivations()) {
manager.PrintAllDerivations(m_lineNumber, debug);
}
//Best hypothesis
const Hypothesis* bestHypo = NULL;
if (!staticData.UseMBR()) {
bestHypo = manager.GetBestHypothesis();
if (bestHypo) {
if (staticData.IsPathRecoveryEnabled()) {
OutputInput(out, bestHypo);
out << "||| ";
}
OutputSurface(
out,
bestHypo,
staticData.GetOutputFactorOrder(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors());
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
IFVERBOSE(1) {
debug << "BEST TRANSLATION: " << *bestHypo << endl;
}
}
out << endl;
} else {
size_t nBestSize = staticData.GetMBRSize();
if (nBestSize <= 0) {
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
exit(1);
}
TrellisPathList nBestList;
manager.CalcNBest(nBestSize, nBestList,true);
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
IFVERBOSE(2) {
PrintUserTime("calculated n-best list for (L)MBR decoding");
}
~TranslationTask() {delete m_source;}
if (staticData.UseLatticeMBR()) {
if (m_nbestCollector) {
//lattice mbr nbest
vector<LatticeMBRSolution> solutions;
size_t n = min(nBestSize, staticData.GetNBestSize());
getLatticeMBRNBest(manager,nBestList,solutions,n);
ostringstream out;
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
m_nbestCollector->Write(m_lineNumber, out.str());
} else {
//Lattice MBR decoding
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
IFVERBOSE(2) {
PrintUserTime("finished Lattice MBR decoding");
}
}
} else if (staticData.UseConsensusDecoding()) {
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
OutputBestHypo(conBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished Consensus decoding");
}
} else {
//MBR decoding
const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList);
OutputBestHypo(mbrBestHypo, m_lineNumber,
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors(),out);
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
IFVERBOSE(2) {
PrintUserTime("finished MBR decoding");
}
}
}
m_outputCollector->Write(m_lineNumber,out.str(),debug.str());
}
if (m_nbestCollector && !staticData.UseLatticeMBR()) {
TrellisPathList nBestList;
ostringstream out;
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
OutputNBest(out,nBestList, staticData.GetOutputFactorOrder(), manager.GetTranslationSystem(), m_lineNumber);
m_nbestCollector->Write(m_lineNumber, out.str());
}
//detailed translation reporting
if (m_detailedTranslationCollector) {
ostringstream out;
fix(out,PRECISION);
TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
}
IFVERBOSE(2) {
PrintUserTime("Sentence Decoding Time:");
}
manager.CalcDecoderStatistics();
}
~TranslationTask() {
delete m_source;
}
private:
InputType* m_source;
size_t m_lineNumber;
OutputCollector* m_outputCollector;
OutputCollector* m_nbestCollector;
OutputCollector* m_wordGraphCollector;
OutputCollector* m_searchGraphCollector;
OutputCollector* m_detailedTranslationCollector;
OutputCollector* m_alignmentInfoCollector;
std::ofstream *m_alignmentStream;
private:
InputType* m_source;
size_t m_lineNumber;
OutputCollector* m_outputCollector;
OutputCollector* m_nbestCollector;
OutputCollector* m_wordGraphCollector;
OutputCollector* m_searchGraphCollector;
OutputCollector* m_detailedTranslationCollector;
OutputCollector* m_alignmentInfoCollector;
std::ofstream *m_alignmentStream;
};
static void PrintFeatureWeight(const FeatureFunction* ff) {
static void PrintFeatureWeight(const FeatureFunction* ff)
{
size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID());
size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID());
for (size_t i = weightStart; i < weightEnd; ++i) {
cout << ff->GetScoreProducerDescription() << " " << ff->GetScoreProducerWeightShortName() << " "
<< StaticData::Instance().GetAllWeights()[i] << endl;
cout << ff->GetScoreProducerDescription() << " " << ff->GetScoreProducerWeightShortName() << " "
<< StaticData::Instance().GetAllWeights()[i] << endl;
}
}
static void ShowWeights() {
static void ShowWeights()
{
fix(cout,6);
const StaticData& staticData = StaticData::Instance();
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
@ -274,147 +279,149 @@ static void ShowWeights() {
}
}
int main(int argc, char** argv) {
int main(int argc, char** argv)
{
#ifdef HAVE_PROTOBUF
GOOGLE_PROTOBUF_VERIFY_VERSION;
GOOGLE_PROTOBUF_VERIFY_VERSION;
#endif
IFVERBOSE(1)
{
TRACE_ERR("command: ");
for(int i=0;i<argc;++i) TRACE_ERR(argv[i]<<" ");
TRACE_ERR(endl);
}
IFVERBOSE(1) {
TRACE_ERR("command: ");
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
TRACE_ERR(endl);
}
fix(cout,PRECISION);
fix(cerr,PRECISION);
fix(cout,PRECISION);
fix(cerr,PRECISION);
Parameter* params = new Parameter();
if (!params->LoadParam(argc,argv)) {
params->Explain();
exit(1);
}
//create threadpool, if necessary
int threadcount = (params->GetParam("threads").size() > 0) ?
Scan<size_t>(params->GetParam("threads")[0]) : 1;
Parameter* params = new Parameter();
if (!params->LoadParam(argc,argv)) {
params->Explain();
exit(1);
}
//create threadpool, if necessary
int threadcount = (params->GetParam("threads").size() > 0) ?
Scan<size_t>(params->GetParam("threads")[0]) : 1;
#ifdef WITH_THREADS
if (threadcount < 1) {
cerr << "Error: Need to specify a positive number of threads" << endl;
exit(1);
}
ThreadPool pool(threadcount);
if (threadcount < 1) {
cerr << "Error: Need to specify a positive number of threads" << endl;
exit(1);
}
ThreadPool pool(threadcount);
#else
if (threadcount > 1) {
cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
exit(1);
}
if (threadcount > 1) {
cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
exit(1);
}
#endif
if (!StaticData::LoadDataStatic(params)) {
exit(1);
}
if (params->isParamSpecified("show-weights")) {
ShowWeights();
exit(0);
}
const StaticData& staticData = StaticData::Instance();
// set up read/writing class
IOWrapper* ioWrapper = GetIODevice(staticData);
if (!StaticData::LoadDataStatic(params)) {
exit(1);
}
if (!ioWrapper) {
cerr << "Error; Failed to create IO object" << endl;
exit(1);
}
// check on weights
vector<float> weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
TRACE_ERR("The global weight vector looks like this:");
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
TRACE_ERR("\n");
}
// every score must have a weight! check that here:
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
exit(1);
}
if (params->isParamSpecified("show-weights")) {
ShowWeights();
exit(0);
}
InputType* source = NULL;
size_t lineCount = 0;
auto_ptr<OutputCollector> outputCollector;//for translations
auto_ptr<OutputCollector> nbestCollector;
auto_ptr<ofstream> nbestOut;
size_t nbestSize = staticData.GetNBestSize();
string nbestFile = staticData.GetNBestFilePath();
if (nbestSize) {
if (nbestFile == "-" || nbestFile == "/dev/stdout") {
//nbest to stdout, no 1-best
nbestCollector.reset(new OutputCollector());
} else {
//nbest to file, 1-best to stdout
nbestOut.reset(new ofstream(nbestFile.c_str()));
assert(nbestOut->good());
nbestCollector.reset(new OutputCollector(nbestOut.get()));
outputCollector.reset(new OutputCollector());
}
const StaticData& staticData = StaticData::Instance();
// set up read/writing class
IOWrapper* ioWrapper = GetIODevice(staticData);
if (!ioWrapper) {
cerr << "Error; Failed to create IO object" << endl;
exit(1);
}
// check on weights
vector<float> weights = staticData.GetAllWeights();
IFVERBOSE(2) {
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
TRACE_ERR("The global weight vector looks like this:");
for (size_t j=0; j<weights.size(); j++) {
TRACE_ERR(" " << weights[j]);
}
TRACE_ERR("\n");
}
// every score must have a weight! check that here:
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
exit(1);
}
InputType* source = NULL;
size_t lineCount = 0;
auto_ptr<OutputCollector> outputCollector;//for translations
auto_ptr<OutputCollector> nbestCollector;
auto_ptr<ofstream> nbestOut;
size_t nbestSize = staticData.GetNBestSize();
string nbestFile = staticData.GetNBestFilePath();
if (nbestSize) {
if (nbestFile == "-" || nbestFile == "/dev/stdout") {
//nbest to stdout, no 1-best
nbestCollector.reset(new OutputCollector());
} else {
outputCollector.reset(new OutputCollector());
//nbest to file, 1-best to stdout
nbestOut.reset(new ofstream(nbestFile.c_str()));
assert(nbestOut->good());
nbestCollector.reset(new OutputCollector(nbestOut.get()));
outputCollector.reset(new OutputCollector());
}
auto_ptr<OutputCollector> wordGraphCollector;
if (staticData.GetOutputWordGraph()) {
wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream())));
} else {
outputCollector.reset(new OutputCollector());
}
auto_ptr<OutputCollector> wordGraphCollector;
if (staticData.GetOutputWordGraph()) {
wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream())));
}
auto_ptr<OutputCollector> searchGraphCollector;
if (staticData.GetOutputSearchGraph()) {
searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream())));
}
auto_ptr<OutputCollector> detailedTranslationCollector;
if (staticData.IsDetailedTranslationReportingEnabled()) {
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
}
auto_ptr<OutputCollector> alignmentInfoCollector;
if (!staticData.GetAlignmentOutputFile().empty()) {
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
}
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
IFVERBOSE(1) {
ResetUserTime();
}
auto_ptr<OutputCollector> searchGraphCollector;
if (staticData.GetOutputSearchGraph()) {
searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream())));
}
auto_ptr<OutputCollector> detailedTranslationCollector;
if (staticData.IsDetailedTranslationReportingEnabled()) {
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
}
auto_ptr<OutputCollector> alignmentInfoCollector;
if (!staticData.GetAlignmentOutputFile().empty()) {
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
}
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
IFVERBOSE(1) {
ResetUserTime();
}
TranslationTask* task =
new TranslationTask(lineCount,source, outputCollector.get(),
nbestCollector.get(), wordGraphCollector.get(),
searchGraphCollector.get(),
detailedTranslationCollector.get(),
alignmentInfoCollector.get() );
TranslationTask* task =
new TranslationTask(lineCount,source, outputCollector.get(),
nbestCollector.get(), wordGraphCollector.get(),
searchGraphCollector.get(),
detailedTranslationCollector.get(),
alignmentInfoCollector.get() );
#ifdef WITH_THREADS
pool.Submit(task);
pool.Submit(task);
#else
task->Run();
task->Run();
#endif
source = NULL; //make sure it doesn't get deleted
++lineCount;
}
source = NULL; //make sure it doesn't get deleted
++lineCount;
}
#ifdef WITH_THREADS
pool.Stop(true); //flush remaining jobs
pool.Stop(true); //flush remaining jobs
#endif
#ifndef EXIT_RETURN
//This avoids that destructors are called (it can take a long time)
exit(EXIT_SUCCESS);
//This avoids that destructors are called (it can take a long time)
exit(EXIT_SUCCESS);
#else
return EXIT_SUCCESS;
return EXIT_SUCCESS;
#endif
}

32
moses-cmd/src/Main.h
View File

@ -5,28 +5,28 @@ Moses - factored phrase-based language decoder
Copyright (c) 2006 University of Edinburgh
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***********************************************************************/

152
moses-cmd/src/TranslationAnalysis.cpp
View File

@ -9,11 +9,12 @@
using namespace Moses;
namespace TranslationAnalysis {
void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, const Hypothesis* hypo)
namespace TranslationAnalysis
{
os << std::endl << "TRANSLATION HYPOTHESIS DETAILS:" << std::endl;
void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, const Hypothesis* hypo)
{
os << std::endl << "TRANSLATION HYPOTHESIS DETAILS:" << std::endl;
std::vector<const Hypothesis*> translationPath;
while (hypo) {
translationPath.push_back(hypo);
@ -24,83 +25,90 @@ namespace TranslationAnalysis {
std::vector<std::string> droppedWords;
std::vector<const Hypothesis*>::iterator tpi = translationPath.begin();
++tpi; // skip initial translation state
std::vector<std::string> sourceMap;
std::vector<std::string> targetMap;
std::vector<unsigned int> lmAcc(0);
size_t lmCalls = 0;
bool doLMStats = ((*tpi)->GetLMStats() != 0);
if (doLMStats)
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
std::vector<std::string> sourceMap;
std::vector<std::string> targetMap;
std::vector<unsigned int> lmAcc(0);
size_t lmCalls = 0;
bool doLMStats = ((*tpi)->GetLMStats() != 0);
if (doLMStats)
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
for (; tpi != translationPath.end(); ++tpi) {
std::ostringstream sms;
std::ostringstream tms;
std::ostringstream sms;
std::ostringstream tms;
std::string target = (*tpi)->GetTargetPhraseStringRep();
std::string source = (*tpi)->GetSourcePhraseStringRep();
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
WordsRange swr = (*tpi)->GetCurrSourceWordsRange();
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
WordsRange swr = (*tpi)->GetCurrSourceWordsRange();
const AlignmentInfo &alignmentInfo = (*tpi)->GetCurrTargetPhrase().GetAlignmentInfo();
// language model backoff stats,
if (doLMStats) {
std::vector<std::vector<unsigned int> >& lmstats = *(*tpi)->GetLMStats();
std::vector<std::vector<unsigned int> >::iterator i = lmstats.begin();
std::vector<unsigned int>::iterator acc = lmAcc.begin();
const AlignmentInfo &alignmentInfo = (*tpi)->GetCurrTargetPhrase().GetAlignmentInfo();
for (; i != lmstats.end(); ++i, ++acc) {
std::vector<unsigned int>::iterator j = i->begin();
lmCalls += i->size();
for (; j != i->end(); ++j) {
(*acc) += *j;
}
}
}
bool epsilon = false;
// language model backoff stats,
if (doLMStats) {
std::vector<std::vector<unsigned int> >& lmstats = *(*tpi)->GetLMStats();
std::vector<std::vector<unsigned int> >::iterator i = lmstats.begin();
std::vector<unsigned int>::iterator acc = lmAcc.begin();
for (; i != lmstats.end(); ++i, ++acc) {
std::vector<unsigned int>::iterator j = i->begin();
lmCalls += i->size();
for (; j != i->end(); ++j) {
(*acc) += *j;
}
}
}
bool epsilon = false;
if (target == "") {
target="<EPSILON>";
epsilon = true;
epsilon = true;
droppedWords.push_back(source);
}
os << " SOURCE: " << swr << " " << source << std::endl
<< " TRANSLATED AS: " << target << std::endl
<< " WORD ALIGNED: " << alignmentInfo << std::endl;
size_t twr_i = twr.GetStartPos();
size_t swr_i = swr.GetStartPos();
if (!epsilon) { sms << twr_i; }
if (epsilon) { tms << "del(" << swr_i << ")"; } else { tms << swr_i; }
swr_i++; twr_i++;
for (; twr_i <= twr.GetEndPos() && twr.GetEndPos() != NOT_FOUND; twr_i++) {
sms << '-' << twr_i;
}
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
tms << '-' << swr_i;
}
if (!epsilon) targetMap.push_back(sms.str());
sourceMap.push_back(tms.str());
<< " TRANSLATED AS: " << target << std::endl
<< " WORD ALIGNED: " << alignmentInfo << std::endl;
size_t twr_i = twr.GetStartPos();
size_t swr_i = swr.GetStartPos();
if (!epsilon) {
sms << twr_i;
}
if (epsilon) {
tms << "del(" << swr_i << ")";
} else {
tms << swr_i;
}
swr_i++;
twr_i++;
for (; twr_i <= twr.GetEndPos() && twr.GetEndPos() != NOT_FOUND; twr_i++) {
sms << '-' << twr_i;
}
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
tms << '-' << swr_i;
}
if (!epsilon) targetMap.push_back(sms.str());
sourceMap.push_back(tms.str());
}
std::vector<std::string>::iterator si = sourceMap.begin();
std::vector<std::string>::iterator ti = targetMap.begin();
os << std::endl << "SOURCE/TARGET SPANS:";
os << std::endl << " SOURCE:";
for (; si != sourceMap.end(); ++si) {
os << " " << *si;
}
os << std::endl << " TARGET:";
for (; ti != targetMap.end(); ++ti) {
os << " " << *ti;
}
os << std::endl << std::endl;
if (doLMStats && lmCalls > 0) {
std::vector<unsigned int>::iterator acc = lmAcc.begin();
const LMList& lmlist = system->GetLanguageModels();
LMList::const_iterator i = lmlist.begin();
for (; acc != lmAcc.end(); ++acc, ++i) {
char buf[256];
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
}
}
std::vector<std::string>::iterator si = sourceMap.begin();
std::vector<std::string>::iterator ti = targetMap.begin();
os << std::endl << "SOURCE/TARGET SPANS:";
os << std::endl << " SOURCE:";
for (; si != sourceMap.end(); ++si) {
os << " " << *si;
}
os << std::endl << " TARGET:";
for (; ti != targetMap.end(); ++ti) {
os << " " << *ti;
}
os << std::endl << std::endl;
if (doLMStats && lmCalls > 0) {
std::vector<unsigned int>::iterator acc = lmAcc.begin();
const LMList& lmlist = system->GetLanguageModels();
LMList::const_iterator i = lmlist.begin();
for (; acc != lmAcc.end(); ++acc, ++i) {
char buf[256];
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
}
}
if (droppedWords.size() > 0) {
std::vector<std::string>::iterator dwi = droppedWords.begin();
@ -109,9 +117,9 @@ namespace TranslationAnalysis {
os << "\tdropped=" << *dwi << std::endl;
}
}
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights());
os << std::endl;
os << std::endl;
}
}

2
moses-cmd/src/TranslationAnalysis.h
View File

@ -18,7 +18,7 @@ namespace TranslationAnalysis
* print details about the translation represented in hypothesis to
* os. Included information: phrase alignment, words dropped, scores
*/
void PrintTranslationAnalysis(const Moses::TranslationSystem* system, std::ostream &os, const Moses::Hypothesis* hypo);
void PrintTranslationAnalysis(const Moses::TranslationSystem* system, std::ostream &os, const Moses::Hypothesis* hypo);
}

3
moses-cmd/src/checkplf.cpp
View File

@ -4,7 +4,8 @@
using namespace std;
int main() {
int main()
{
cerr << "Reading PLF from STDIN...\n";
string line;
int lc = 0;

140
moses-cmd/src/mbr.cpp
View File

@ -18,11 +18,11 @@ using namespace std ;
using namespace Moses;
/* Input :
1. a sorted n-best list, with duplicates filtered out in the following format
/* Input :
1. a sorted n-best list, with duplicates filtered out in the following format
0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432
2. a weight vector
2. a weight vector
3. bleu order ( default = 4)
4. scaling factor to weigh the weight vector (default = 1.0)
@ -38,12 +38,9 @@ float min_interval = 1e-4;
void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const Factor* >, int > & allngrams)
{
vector< const Factor* > ngram;
for (int k = 0; k < BLEU_ORDER; k++)
{
for(int i =0; i < max((int)sentence.size()-k,0); i++)
{
for ( int j = i; j<= i+k; j++)
{
for (int k = 0; k < BLEU_ORDER; k++) {
for(int i =0; i < max((int)sentence.size()-k,0); i++) {
for ( int j = i; j<= i+k; j++) {
ngram.push_back(sentence[j]);
}
++allngrams[ngram];
@ -52,15 +49,15 @@ void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const
}
}
float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats ) {
float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats )
{
int comps_n = 2*BLEU_ORDER+1;
vector<int> comps(comps_n);
float logbleu = 0.0, brevity;
int hyp_length = sents[hyp].size();
for (int i =0; i<BLEU_ORDER;i++)
{
for (int i =0; i<BLEU_ORDER; i++) {
comps[2*i] = 0;
comps[2*i+1] = max(hyp_length-i,0);
}
@ -69,18 +66,15 @@ float calculate_score(const vector< vector<const Factor*> > & sents, int ref, in
map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ;
for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin();
it != hyp_ngrams.end(); it++)
{
it != hyp_ngrams.end(); it++) {
map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first);
if(ref_it != ref_ngrams.end())
{
if(ref_it != ref_ngrams.end()) {
comps[2* (it->first.size()-1)] += min(ref_it->second,it->second);
}
}
comps[comps_n-1] = sents[ref].size();
for (int i=0; i<BLEU_ORDER; i++)
{
for (int i=0; i<BLEU_ORDER; i++) {
if (comps[0] == 0)
return 0.0;
if ( i > 0 )
@ -95,7 +89,8 @@ float calculate_score(const vector< vector<const Factor*> > & sents, int ref, in
return exp(logbleu);
}
const TrellisPath doMBR(const TrellisPathList& nBestList){
const TrellisPath doMBR(const TrellisPathList& nBestList)
{
float marginal = 0;
vector<float> joint_prob_vec;
@ -104,19 +99,17 @@ const TrellisPath doMBR(const TrellisPathList& nBestList){
vector< map < vector <const Factor *>, int > > ngram_stats;
TrellisPathList::const_iterator iter;
// get max score to prevent underflow
float maxScore = -1e20;
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
{
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
const TrellisPath &path = **iter;
float score = StaticData::Instance().GetMBRScale()
* path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights());
* path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights());
if (maxScore < score) maxScore = score;
}
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
{
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
const TrellisPath &path = **iter;
joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights()) - maxScore);
marginal += joint_prob;
@ -125,62 +118,61 @@ const TrellisPath doMBR(const TrellisPathList& nBestList){
// get words in translation
vector<const Factor*> translation;
GetOutputFactors(path, translation);
// collect n-gram counts
map < vector < const Factor *>, int > counts;
extract_ngrams(translation,counts);
ngram_stats.push_back(counts);
translations.push_back(translation);
}
}
vector<float> mbr_loss;
float bleu, weightedLoss;
float weightedLossCumul = 0;
float minMBRLoss = 1000000;
int minMBRLossIdx = -1;
/* Main MBR computation done here */
iter = nBestList.begin();
for (unsigned int i = 0; i < nBestList.GetSize(); i++){
weightedLossCumul = 0;
for (unsigned int j = 0; j < nBestList.GetSize(); j++){
if ( i != j) {
bleu = calculate_score(translations, j, i,ngram_stats );
weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal);
weightedLossCumul += weightedLoss;
if (weightedLossCumul > minMBRLoss)
break;
}
}
if (weightedLossCumul < minMBRLoss){
minMBRLoss = weightedLossCumul;
minMBRLossIdx = i;
}
iter++;
}
/* Find sentence that minimises Bayes Risk under 1- BLEU loss */
return nBestList.at(minMBRLossIdx);
//return translations[minMBRLossIdx];
vector<float> mbr_loss;
float bleu, weightedLoss;
float weightedLossCumul = 0;
float minMBRLoss = 1000000;
int minMBRLossIdx = -1;
/* Main MBR computation done here */
iter = nBestList.begin();
for (unsigned int i = 0; i < nBestList.GetSize(); i++) {
weightedLossCumul = 0;
for (unsigned int j = 0; j < nBestList.GetSize(); j++) {
if ( i != j) {
bleu = calculate_score(translations, j, i,ngram_stats );
weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal);
weightedLossCumul += weightedLoss;
if (weightedLossCumul > minMBRLoss)
break;
}
}
if (weightedLossCumul < minMBRLoss) {
minMBRLoss = weightedLossCumul;
minMBRLossIdx = i;
}
iter++;
}
/* Find sentence that minimises Bayes Risk under 1- BLEU loss */
return nBestList.at(minMBRLossIdx);
//return translations[minMBRLossIdx];
}
void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation){
const std::vector<const Hypothesis *> &edges = path.GetEdges();
const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
assert (outputFactorOrder.size() == 1);
void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation)
{
const std::vector<const Hypothesis *> &edges = path.GetEdges();
const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
assert (outputFactorOrder.size() == 1);
// print the surface factor of the translation
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
{
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++)
{
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
translation.push_back(factor);
}
}
// print the surface factor of the translation
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
const Hypothesis &edge = *edges[currEdge];
const Phrase &phrase = edge.GetCurrTargetPhrase();
size_t size = phrase.GetSize();
for (size_t pos = 0 ; pos < size ; pos++) {
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
translation.push_back(factor);
}
}
}