moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

reduce memory usage in lattice mbr

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@2906 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
bhaddow 2010-02-17 14:14:44 +00:00
parent b3a8411ae1
commit 7f45cd12f9
3 changed files with 66 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
moses-cmd/src/LatticeMBR.cpp
View File

@ -12,7 +12,7 @@
#include <algorithm>
#include <set>
int bleu_order = 4;
size_t bleu_order = 4;
float UNKNGRAMLOGPROB = -20;
void GetOutputWords(const TrellisPath &path, vector <Word> &translation){
const std::vector<const Hypothesis *> &edges = path.GetEdges();
@ -33,7 +33,7 @@ void GetOutputWords(const TrellisPath &path, vector <Word> &translation){
void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngrams)
{
for (int k = 0; k < bleu_order; k++)
for (int k = 0; k < (int)bleu_order; k++)
{
for(int i =0; i < max((int)sentence.size()-k,0); i++)
{
@ -48,6 +48,31 @@ void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngr
}
void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score) {
set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
if (ngramIter == m_ngrams.end()) {
ngramIter = m_ngrams.insert(ngram).first;
}
map<const Phrase*,float>& ngramScores = m_scores[node];
map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
if (scoreIter == ngramScores.end()) {
ngramScores[&(*ngramIter)] = score;
} else {
ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
}
}
NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node) {
return m_scores[node].begin();
}
NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node) {
return m_scores[node].end();
}
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
const vector< float> & estimatedScores, size_t edgeDensity) {
@ -186,10 +211,10 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
}
}
vector<Word> calcMBRSol(Lattice & connectedHyp, map<Phrase, float>& finalNgramScores, const vector<float> & thetas, float p, float r) {
/*vector<Word> calcMBRSol(Lattice & connectedHyp, map<Phrase, float>& finalNgramScores, const vector<float> & thetas, float p, float r) {
vector<Word> bestHyp;
return bestHyp;
}
}*/
void calcNgramPosteriors(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges, float scale, map<Phrase, float>& finalNgramScores) {
@ -200,7 +225,7 @@ void calcNgramPosteriors(Lattice & connectedHyp, map<const Hypothesis*, vector<E
forwardScore[connectedHyp[0]] = 0.0f; //forward score of hyp 0 is 1 (or 0 in logprob space)
set< const Hypothesis *> finalHyps; //store completed hyps
map<const Hypothesis*, map<Phrase,float> > ngramScores;//ngram scores for each hyp
NgramScores ngramScores;//ngram scores for each hyp
for (size_t i = 1; i < connectedHyp.size(); ++i) {
const Hypothesis* currHyp = connectedHyp[i];
@ -241,28 +266,20 @@ void calcNgramPosteriors(Lattice & connectedHyp, map<const Hypothesis*, vector<E
for (size_t i = 0; i < path.size(); ++i) {
score += path[i]->GetScore();
}
map<Phrase, float>& currHypNgramScores = ngramScores[currHyp];
if (currHypNgramScores.find(ngram) == currHypNgramScores.end())
currHypNgramScores[ngram] = score;
else
currHypNgramScores[ngram] = log_sum(currHypNgramScores[ngram], score);
ngramScores.addScore(currHyp,ngram,score);
}
}
//Now score ngrams that are just being propagated from the history
map<Phrase, float>& thisEdgeNgramScores = ngramScores[edge.GetTailNode()];
for (map<Phrase, float>::iterator it = thisEdgeNgramScores.begin(); it != thisEdgeNgramScores.end(); ++it) {
const Phrase & currNgram = it->first;
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
it != ngramScores.nodeEnd(edge.GetTailNode()); ++it) {
const Phrase & currNgram = *(it->first);
float currNgramScore = it->second;
VERBOSE(4, "Calculating score for: " << currNgram << endl)
if (incomingPhrases.find(currNgram) == incomingPhrases.end()) {
map<Phrase, float>& currHypNgramScores = ngramScores[currHyp];
float score = edge.GetScore() + currNgramScore;
if (currHypNgramScores.find(currNgram) == currHypNgramScores.end())
currHypNgramScores[currNgram] = score;
else
currHypNgramScores[currNgram] = log_sum(currHypNgramScores[currNgram], score);
ngramScores.addScore(currHyp,currNgram,score);
}
}
@ -274,14 +291,14 @@ void calcNgramPosteriors(Lattice & connectedHyp, map<const Hypothesis*, vector<E
//Done - Print out ngram posteriors for final hyps
for (set< const Hypothesis *>::iterator finalHyp = finalHyps.begin(); finalHyp != finalHyps.end(); ++finalHyp) {
const Hypothesis* hyp = *finalHyp;
const map<Phrase,float> & ngramPosteriors = ngramScores[hyp];
for (map<Phrase, float>::const_iterator it = ngramPosteriors.begin(); it != ngramPosteriors.end(); ++it) {
if (finalNgramScores.find(it->first) == finalNgramScores.end()) {
finalNgramScores[it->first] = it->second;
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(hyp); it != ngramScores.nodeEnd(hyp); ++it) {
const Phrase& ngram = *(it->first);
if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
finalNgramScores[ngram] = it->second;
}
else {
finalNgramScores[it->first] = log_sum(it->second, finalNgramScores[it->first]);
finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]);
}
}
@ -409,7 +426,7 @@ vector<Word> calcMBRSol(const TrellisPathList& nBestList, map<Phrase, float>& f
vector<float> mbrThetas = thetas;
if (thetas.size() == 0) { //thetas not specified on the command line, use p and r instead
mbrThetas.push_back(-1); //Theta 0
mbrThetas.push_back(1/(4*p));
mbrThetas.push_back(1/(bleu_order*p));
for (size_t i = 2; i <= bleu_order; ++i){
mbrThetas.push_back(mbrThetas[i-1] / r);
}
@ -464,8 +481,8 @@ vector<Word> calcMBRSol(const TrellisPathList& nBestList, map<Phrase, float>& f
if (mbrScore > argmaxScore){
argmaxScore = mbrScore;
IFVERBOSE(2) {
cout << "HYP " << ctr << "IS NEW BEST: ";
for (int i = 0; i < translation.size(); ++i)
cout << "HYP " << ctr << " IS NEW BEST: ";
for (size_t i = 0; i < translation.size(); ++i)
cout << translation[i] ;
cout << "[" << argmaxScore << "]" << endl;
}

20
moses-cmd/src/LatticeMBR.h
View File

@ -83,6 +83,26 @@ class Edge {
};
/**
* Data structure to hold the ngram scores as we traverse the lattice. Maps (hypo,ngram) to score
*/
class NgramScores {
public:
NgramScores() {}
/** logsum this score to the existing score */
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
/** Iterate through ngrams for selected node */
typedef map<const Phrase*, float>::const_iterator NodeScoreIterator;
NodeScoreIterator nodeBegin(const Hypothesis* node);
NodeScoreIterator nodeEnd(const Hypothesis* node);
private:
set<Phrase> m_ngrams;
map<const Hypothesis*, map<const Phrase*, float> > m_scores;
};
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
const vector< float> & estimatedScores, size_t edgeDensity);

4
moses-cmd/src/Main.cpp
View File

@ -195,7 +195,9 @@ int main(int argc, char* argv[])
}
}
else {
mbrBestHypo = calcMBRSol(connectedList, ngramPosteriors, staticData.GetLatticeMBRThetas(), staticData.GetLatticeMBRPrecision(), staticData.GetLatticeMBRPRatio());
cerr << "Using Lattice for Hypothesis set not yet implemented" << endl;
return EXIT_FAILURE;
//mbrBestHypo = calcMBRSol(connectedList, ngramPosteriors, staticData.GetLatticeMBRThetas(), staticData.GetLatticeMBRPrecision(), staticData.GetLatticeMBRPRatio());
}
ioWrapper->OutputBestHypo(mbrBestHypo, source->GetTranslationId(), staticData.GetReportSegmentation(),