2010-02-03 13:23:32 +03:00
|
|
|
/*
|
|
|
|
* LatticeMBR.h
|
|
|
|
* moses-cmd
|
|
|
|
*
|
|
|
|
* Created by Abhishek Arun on 26/01/2010.
|
|
|
|
* Copyright 2010 __MyCompanyName__. All rights reserved.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2010-02-24 14:15:44 +03:00
|
|
|
#ifndef moses_cmd_LatticeMBR_h
|
|
|
|
#define moses_cmd_LatticeMBR_h
|
2010-02-03 13:23:32 +03:00
|
|
|
|
|
|
|
#include <map>
|
|
|
|
#include <vector>
|
|
|
|
#include <set>
|
|
|
|
#include "Hypothesis.h"
|
2010-02-17 20:25:56 +03:00
|
|
|
#include "Manager.h"
|
2010-02-03 13:23:32 +03:00
|
|
|
#include "TrellisPathList.h"
|
|
|
|
|
|
|
|
using namespace Moses;
|
|
|
|
|
|
|
|
template<class T>
|
|
|
|
T log_sum (T log_a, T log_b)
|
|
|
|
{
|
|
|
|
T v;
|
|
|
|
if (log_a < log_b) {
|
|
|
|
v = log_b+log ( 1 + exp ( log_a-log_b ));
|
|
|
|
} else {
|
|
|
|
v = log_a+log ( 1 + exp ( log_b-log_a ));
|
|
|
|
}
|
|
|
|
return ( v );
|
|
|
|
}
|
|
|
|
|
|
|
|
class Edge;
|
|
|
|
|
|
|
|
typedef std::vector< const Hypothesis *> Lattice;
|
2011-02-24 15:39:29 +03:00
|
|
|
typedef std::vector<const Edge*> Path;
|
2010-04-12 14:04:35 +04:00
|
|
|
typedef std::map<Path, size_t> PathCounts;
|
|
|
|
typedef std::map<Phrase, PathCounts > NgramHistory;
|
2010-02-03 13:23:32 +03:00
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
class Edge
|
|
|
|
{
|
|
|
|
const Hypothesis* m_tailNode;
|
|
|
|
const Hypothesis* m_headNode;
|
|
|
|
float m_score;
|
|
|
|
TargetPhrase m_targetPhrase;
|
|
|
|
NgramHistory m_ngrams;
|
|
|
|
|
|
|
|
public:
|
|
|
|
Edge(const Hypothesis* from, const Hypothesis* to, float score, const TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) {
|
|
|
|
//cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl;
|
|
|
|
}
|
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
const Hypothesis* GetHeadNode() const {
|
2011-02-24 15:39:29 +03:00
|
|
|
return m_headNode;
|
2010-02-03 13:23:32 +03:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
const Hypothesis* GetTailNode() const {
|
|
|
|
return m_tailNode;
|
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
float GetScore() const {
|
|
|
|
return m_score;
|
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
size_t GetWordsSize() const {
|
|
|
|
return m_targetPhrase.GetSize();
|
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
const Phrase& GetWords() const {
|
|
|
|
return m_targetPhrase;
|
2011-02-24 15:39:29 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
friend std::ostream& operator<< (std::ostream& out, const Edge& edge);
|
|
|
|
|
2010-04-12 14:04:35 +04:00
|
|
|
const NgramHistory& GetNgrams( std::map<const Hypothesis*, std::vector<Edge> > & incomingEdges) ;
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
bool operator < (const Edge & compare) const;
|
2011-02-24 15:39:29 +03:00
|
|
|
|
|
|
|
void GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const;
|
|
|
|
|
|
|
|
void storeNgramHistory(const Phrase& phrase, Path & path, size_t count = 1) {
|
|
|
|
m_ngrams[phrase][path]+= count;
|
2010-02-03 13:23:32 +03:00
|
|
|
}
|
2011-02-24 15:39:29 +03:00
|
|
|
|
2010-02-03 13:23:32 +03:00
|
|
|
};
|
|
|
|
|
2010-02-17 17:14:44 +03:00
|
|
|
/**
|
|
|
|
* Data structure to hold the ngram scores as we traverse the lattice. Maps (hypo,ngram) to score
|
|
|
|
*/
|
2011-02-24 15:39:29 +03:00
|
|
|
class NgramScores
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
NgramScores() {}
|
|
|
|
|
|
|
|
/** logsum this score to the existing score */
|
|
|
|
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
|
|
|
|
|
|
|
|
/** Iterate through ngrams for selected node */
|
|
|
|
typedef std::map<const Phrase*, float>::const_iterator NodeScoreIterator;
|
|
|
|
NodeScoreIterator nodeBegin(const Hypothesis* node);
|
|
|
|
NodeScoreIterator nodeEnd(const Hypothesis* node);
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::set<Phrase> m_ngrams;
|
|
|
|
std::map<const Hypothesis*, std::map<const Phrase*, float> > m_scores;
|
2010-02-17 17:14:44 +03:00
|
|
|
};
|
|
|
|
|
2010-03-16 14:34:50 +03:00
|
|
|
|
|
|
|
/** Holds a lattice mbr solution, and its scores */
|
2011-02-24 15:39:29 +03:00
|
|
|
class LatticeMBRSolution
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
/** Read the words from the path */
|
|
|
|
LatticeMBRSolution(const TrellisPath& path, bool isMap);
|
|
|
|
const std::vector<float>& GetNgramScores() const {
|
|
|
|
return m_ngramScores;
|
|
|
|
}
|
|
|
|
const std::vector<Word>& GetWords() const {
|
|
|
|
return m_words;
|
|
|
|
}
|
|
|
|
float GetMapScore() const {
|
|
|
|
return m_mapScore;
|
|
|
|
}
|
|
|
|
float GetScore() const {
|
|
|
|
return m_score;
|
|
|
|
}
|
|
|
|
|
|
|
|
/** Initialise ngram scores */
|
|
|
|
void CalcScore(std::map<Phrase, float>& finalNgramScores, const std::vector<float>& thetas, float mapWeight);
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::vector<Word> m_words;
|
|
|
|
float m_mapScore;
|
|
|
|
std::vector<float> m_ngramScores;
|
|
|
|
float m_score;
|
2010-03-16 14:34:50 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
struct LatticeMBRSolutionComparator {
|
2011-02-24 15:39:29 +03:00
|
|
|
bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) {
|
|
|
|
return a.GetScore() > b.GetScore();
|
|
|
|
}
|
2010-03-16 14:34:50 +03:00
|
|
|
};
|
|
|
|
|
2011-02-24 15:39:29 +03:00
|
|
|
void pruneLatticeFB(Lattice & connectedHyp, std::map < const Hypothesis*, std::set <const Hypothesis* > > & outgoingHyps, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges,
|
2010-05-25 12:55:13 +04:00
|
|
|
const std::vector< float> & estimatedScores, const Hypothesis*, size_t edgeDensity,float scale);
|
2010-02-03 13:23:32 +03:00
|
|
|
|
2010-03-16 14:34:50 +03:00
|
|
|
//Use the ngram scores to rerank the nbest list, return at most n solutions
|
2010-04-12 14:04:35 +04:00
|
|
|
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList, std::vector<LatticeMBRSolution>& solutions, size_t n);
|
2010-04-07 19:47:58 +04:00
|
|
|
//calculate expectated ngram counts, clipping at 1 (ie calculating posteriors) if posteriors==true.
|
2010-05-25 12:55:13 +04:00
|
|
|
void calcNgramExpectations(Lattice & connectedHyp, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges, std::map<Phrase,
|
2011-02-24 15:39:29 +03:00
|
|
|
float>& finalNgramScores, bool posteriors);
|
2010-04-12 14:04:35 +04:00
|
|
|
void GetOutputFactors(const TrellisPath &path, std::vector <Word> &translation);
|
|
|
|
void extract_ngrams(const std::vector<Word >& sentence, std::map < Phrase, int > & allngrams);
|
2010-02-03 13:23:32 +03:00
|
|
|
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b);
|
2010-04-12 14:04:35 +04:00
|
|
|
std::vector<Word> doLatticeMBR(Manager& manager, TrellisPathList& nBestList);
|
2010-05-11 01:18:47 +04:00
|
|
|
const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList);
|
|
|
|
//std::vector<Word> doConsensusDecoding(Manager& manager, TrellisPathList& nBestList);
|
2010-02-24 14:15:44 +03:00
|
|
|
#endif
|