mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-26 11:28:48 +03:00
run beautify.perl. Consistent formatting for .h & .cpp files
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3897 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
508d89eda8
commit
67dd80fb7b
File diff suppressed because it is too large
Load Diff
@ -5,28 +5,28 @@ Moses - factored phrase-based language decoder
|
||||
Copyright (c) 2006 University of Edinburgh
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
* Neither the name of the University of Edinburgh nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
* Neither the name of the University of Edinburgh nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
||||
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
||||
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
||||
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
||||
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
***********************************************************************/
|
||||
|
||||
@ -55,78 +55,76 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||
class IOWrapper
|
||||
{
|
||||
protected:
|
||||
long m_translationId;
|
||||
long m_translationId;
|
||||
|
||||
const std::vector<Moses::FactorType> &m_inputFactorOrder;
|
||||
const std::vector<Moses::FactorType> &m_outputFactorOrder;
|
||||
const Moses::FactorMask &m_inputFactorUsed;
|
||||
std::string m_inputFilePath;
|
||||
Moses::InputFileStream *m_inputFile;
|
||||
std::istream *m_inputStream;
|
||||
std::ostream *m_nBestStream
|
||||
,*m_outputWordGraphStream,*m_outputSearchGraphStream;
|
||||
const std::vector<Moses::FactorType> &m_inputFactorOrder;
|
||||
const std::vector<Moses::FactorType> &m_outputFactorOrder;
|
||||
const Moses::FactorMask &m_inputFactorUsed;
|
||||
std::string m_inputFilePath;
|
||||
Moses::InputFileStream *m_inputFile;
|
||||
std::istream *m_inputStream;
|
||||
std::ostream *m_nBestStream
|
||||
,*m_outputWordGraphStream,*m_outputSearchGraphStream;
|
||||
std::ostream *m_detailedTranslationReportingStream;
|
||||
std::ofstream *m_alignmentOutputStream;
|
||||
bool m_surpressSingleBestOutput;
|
||||
|
||||
void Initialization(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath);
|
||||
std::ofstream *m_alignmentOutputStream;
|
||||
bool m_surpressSingleBestOutput;
|
||||
|
||||
void Initialization(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath);
|
||||
|
||||
public:
|
||||
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath);
|
||||
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath);
|
||||
|
||||
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath
|
||||
, const std::string &infilePath);
|
||||
~IOWrapper();
|
||||
IOWrapper(const std::vector<Moses::FactorType> &inputFactorOrder
|
||||
, const std::vector<Moses::FactorType> &outputFactorOrder
|
||||
, const Moses::FactorMask &inputFactorUsed
|
||||
, size_t nBestSize
|
||||
, const std::string &nBestFilePath
|
||||
, const std::string &infilePath);
|
||||
~IOWrapper();
|
||||
|
||||
Moses::InputType* GetInput(Moses::InputType *inputType);
|
||||
|
||||
Moses::InputType* GetInput(Moses::InputType *inputType);
|
||||
|
||||
void OutputBestHypo(const Moses::Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
|
||||
void OutputLatticeMBRNBestList(const std::vector<LatticeMBRSolution>& solutions,long translationId);
|
||||
void Backtrack(const Moses::Hypothesis *hypo);
|
||||
void Backtrack(const Moses::Hypothesis *hypo);
|
||||
|
||||
void ResetTranslationId() { m_translationId = 0; }
|
||||
void ResetTranslationId() {
|
||||
m_translationId = 0;
|
||||
}
|
||||
|
||||
std::ofstream *GetAlignmentOutputStream()
|
||||
{
|
||||
return m_alignmentOutputStream;
|
||||
}
|
||||
std::ofstream *GetAlignmentOutputStream() {
|
||||
return m_alignmentOutputStream;
|
||||
}
|
||||
|
||||
std::ostream &GetOutputWordGraphStream()
|
||||
{
|
||||
return *m_outputWordGraphStream;
|
||||
}
|
||||
std::ostream &GetOutputSearchGraphStream()
|
||||
{
|
||||
return *m_outputSearchGraphStream;
|
||||
}
|
||||
std::ostream &GetOutputWordGraphStream() {
|
||||
return *m_outputWordGraphStream;
|
||||
}
|
||||
std::ostream &GetOutputSearchGraphStream() {
|
||||
return *m_outputSearchGraphStream;
|
||||
}
|
||||
|
||||
std::ostream &GetDetailedTranslationReportingStream()
|
||||
{
|
||||
assert (m_detailedTranslationReportingStream);
|
||||
std::ostream &GetDetailedTranslationReportingStream() {
|
||||
assert (m_detailedTranslationReportingStream);
|
||||
return *m_detailedTranslationReportingStream;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
IOWrapper *GetIODevice(const Moses::StaticData &staticData);
|
||||
bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::InputType*& source);
|
||||
void OutputSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector<Moses::FactorType> &outputFactorOrder ,bool reportSegmentation, bool reportAllFactors);
|
||||
void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, const std::vector<Moses::FactorType>&,
|
||||
const TranslationSystem* system, long translationId);
|
||||
const TranslationSystem* system, long translationId);
|
||||
void OutputLatticeMBRNBest(std::ostream& out, const std::vector<LatticeMBRSolution>& solutions,long translationId);
|
||||
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,
|
||||
bool reportSegmentation, bool reportAllFactors, std::ostream& out);
|
||||
void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/,
|
||||
bool reportSegmentation, bool reportAllFactors, std::ostream& out);
|
||||
void OutputBestHypo(const Moses::TrellisPath &path, long /*translationId*/,bool reportSegmentation, bool reportAllFactors, std::ostream &out);
|
||||
void OutputInput(std::ostream& os, const Hypothesis* hypo);
|
||||
void OutputAlignment(OutputCollector* collector, size_t lineNo, const Hypothesis *hypo);
|
||||
|
@ -16,32 +16,28 @@ using namespace std;
|
||||
|
||||
size_t bleu_order = 4;
|
||||
float UNKNGRAMLOGPROB = -20;
|
||||
void GetOutputWords(const TrellisPath &path, vector <Word> &translation){
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
|
||||
// print the surface factor of the translation
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
|
||||
{
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++)
|
||||
{
|
||||
translation.push_back(phrase.GetWord(pos));
|
||||
}
|
||||
}
|
||||
void GetOutputWords(const TrellisPath &path, vector <Word> &translation)
|
||||
{
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
|
||||
// print the surface factor of the translation
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++) {
|
||||
translation.push_back(phrase.GetWord(pos));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngrams)
|
||||
{
|
||||
for (int k = 0; k < (int)bleu_order; k++)
|
||||
{
|
||||
for(int i =0; i < max((int)sentence.size()-k,0); i++)
|
||||
{
|
||||
for (int k = 0; k < (int)bleu_order; k++) {
|
||||
for(int i =0; i < max((int)sentence.size()-k,0); i++) {
|
||||
Phrase ngram(Output);
|
||||
for ( int j = i; j<= i+k; j++)
|
||||
{
|
||||
for ( int j = i; j<= i+k; j++) {
|
||||
ngram.AddWord(sentence[j]);
|
||||
}
|
||||
++allngrams[ngram];
|
||||
@ -51,86 +47,90 @@ void extract_ngrams(const vector<Word >& sentence, map < Phrase, int > & allngr
|
||||
|
||||
|
||||
|
||||
void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score) {
|
||||
set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
|
||||
if (ngramIter == m_ngrams.end()) {
|
||||
ngramIter = m_ngrams.insert(ngram).first;
|
||||
}
|
||||
map<const Phrase*,float>& ngramScores = m_scores[node];
|
||||
map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
|
||||
if (scoreIter == ngramScores.end()) {
|
||||
ngramScores[&(*ngramIter)] = score;
|
||||
} else {
|
||||
ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
|
||||
}
|
||||
void NgramScores::addScore(const Hypothesis* node, const Phrase& ngram, float score)
|
||||
{
|
||||
set<Phrase>::const_iterator ngramIter = m_ngrams.find(ngram);
|
||||
if (ngramIter == m_ngrams.end()) {
|
||||
ngramIter = m_ngrams.insert(ngram).first;
|
||||
}
|
||||
map<const Phrase*,float>& ngramScores = m_scores[node];
|
||||
map<const Phrase*,float>::iterator scoreIter = ngramScores.find(&(*ngramIter));
|
||||
if (scoreIter == ngramScores.end()) {
|
||||
ngramScores[&(*ngramIter)] = score;
|
||||
} else {
|
||||
ngramScores[&(*ngramIter)] = log_sum(score,scoreIter->second);
|
||||
}
|
||||
}
|
||||
|
||||
NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node) {
|
||||
return m_scores[node].begin();
|
||||
NgramScores::NodeScoreIterator NgramScores::nodeBegin(const Hypothesis* node)
|
||||
{
|
||||
return m_scores[node].begin();
|
||||
}
|
||||
|
||||
|
||||
NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node) {
|
||||
return m_scores[node].end();
|
||||
NgramScores::NodeScoreIterator NgramScores::nodeEnd(const Hypothesis* node)
|
||||
{
|
||||
return m_scores[node].end();
|
||||
}
|
||||
|
||||
LatticeMBRSolution::LatticeMBRSolution(const TrellisPath& path, bool isMap) :
|
||||
m_score(0.0f){
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
|
||||
{
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++)
|
||||
{
|
||||
m_words.push_back(phrase.GetWord(pos));
|
||||
}
|
||||
}
|
||||
if (isMap) {
|
||||
m_mapScore = path.GetTotalScore();
|
||||
} else {
|
||||
m_mapScore = 0;
|
||||
m_score(0.0f)
|
||||
{
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++) {
|
||||
m_words.push_back(phrase.GetWord(pos));
|
||||
}
|
||||
}
|
||||
if (isMap) {
|
||||
m_mapScore = path.GetTotalScore();
|
||||
} else {
|
||||
m_mapScore = 0;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LatticeMBRSolution::CalcScore(map<Phrase, float>& finalNgramScores, const vector<float>& thetas, float mapWeight) {
|
||||
m_ngramScores.assign(thetas.size()-1, -10000);
|
||||
|
||||
map < Phrase, int > counts;
|
||||
extract_ngrams(m_words,counts);
|
||||
|
||||
//Now score this translation
|
||||
m_score = thetas[0] * m_words.size();
|
||||
void LatticeMBRSolution::CalcScore(map<Phrase, float>& finalNgramScores, const vector<float>& thetas, float mapWeight)
|
||||
{
|
||||
m_ngramScores.assign(thetas.size()-1, -10000);
|
||||
|
||||
//Calculate the ngramScores, working in log space at first
|
||||
for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) {
|
||||
float ngramPosterior = UNKNGRAMLOGPROB;
|
||||
map<Phrase,float>::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first);
|
||||
if (ngramPosteriorIt != finalNgramScores.end()) {
|
||||
ngramPosterior = ngramPosteriorIt->second;
|
||||
}
|
||||
size_t ngramSize = ngrams->first.GetSize();
|
||||
m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]);
|
||||
}
|
||||
|
||||
//convert from log to probability and create weighted sum
|
||||
for (size_t i = 0; i < m_ngramScores.size(); ++i) {
|
||||
m_ngramScores[i] = exp(m_ngramScores[i]);
|
||||
m_score += thetas[i+1] * m_ngramScores[i];
|
||||
}
|
||||
map < Phrase, int > counts;
|
||||
extract_ngrams(m_words,counts);
|
||||
|
||||
|
||||
//The map score
|
||||
m_score += m_mapScore*mapWeight;
|
||||
//Now score this translation
|
||||
m_score = thetas[0] * m_words.size();
|
||||
|
||||
//Calculate the ngramScores, working in log space at first
|
||||
for (map < Phrase, int >::iterator ngrams = counts.begin(); ngrams != counts.end(); ++ngrams) {
|
||||
float ngramPosterior = UNKNGRAMLOGPROB;
|
||||
map<Phrase,float>::const_iterator ngramPosteriorIt = finalNgramScores.find(ngrams->first);
|
||||
if (ngramPosteriorIt != finalNgramScores.end()) {
|
||||
ngramPosterior = ngramPosteriorIt->second;
|
||||
}
|
||||
size_t ngramSize = ngrams->first.GetSize();
|
||||
m_ngramScores[ngramSize-1] = log_sum(log((float)ngrams->second) + ngramPosterior,m_ngramScores[ngramSize-1]);
|
||||
}
|
||||
|
||||
//convert from log to probability and create weighted sum
|
||||
for (size_t i = 0; i < m_ngramScores.size(); ++i) {
|
||||
m_ngramScores[i] = exp(m_ngramScores[i]);
|
||||
m_score += thetas[i+1] * m_ngramScores[i];
|
||||
}
|
||||
|
||||
|
||||
//The map score
|
||||
m_score += m_mapScore*mapWeight;
|
||||
}
|
||||
|
||||
|
||||
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
|
||||
const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale) {
|
||||
|
||||
void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const Hypothesis* > > & outgoingHyps, map<const Hypothesis*, vector<Edge> >& incomingEdges,
|
||||
const vector< float> & estimatedScores, const Hypothesis* bestHypo, size_t edgeDensity, float scale)
|
||||
{
|
||||
|
||||
//Need hyp 0 in connectedHyp - Find empty hypothesis
|
||||
VERBOSE(2,"Pruning lattice to edge density " << edgeDensity << endl);
|
||||
const Hypothesis* emptyHyp = connectedHyp.at(0);
|
||||
@ -138,42 +138,42 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
emptyHyp = emptyHyp->GetPrevHypo();
|
||||
}
|
||||
connectedHyp.push_back(emptyHyp); //Add it to list of hyps
|
||||
|
||||
|
||||
//Need hyp 0's outgoing Hyps
|
||||
for (size_t i = 0; i < connectedHyp.size(); ++i) {
|
||||
if (connectedHyp[i]->GetId() > 0 && connectedHyp[i]->GetPrevHypo()->GetId() == 0)
|
||||
outgoingHyps[emptyHyp].insert(connectedHyp[i]);
|
||||
}
|
||||
|
||||
|
||||
//sort hyps based on estimated scores - do so by copying to multimap
|
||||
multimap<float, const Hypothesis*> sortHypsByVal;
|
||||
for (size_t i =0; i < estimatedScores.size(); ++i) {
|
||||
sortHypsByVal.insert(make_pair<float, const Hypothesis*>(estimatedScores[i], connectedHyp[i]));
|
||||
}
|
||||
|
||||
|
||||
multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end();
|
||||
float bestScore = it->first;
|
||||
//store best score as score of hyp 0
|
||||
sortHypsByVal.insert(make_pair<float, const Hypothesis*>(bestScore, emptyHyp));
|
||||
|
||||
|
||||
|
||||
|
||||
IFVERBOSE(3) {
|
||||
for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
|
||||
const Hypothesis* currHyp = it->second;
|
||||
cerr << "Hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
set <const Hypothesis*> survivingHyps; //store hyps that make the cut in this
|
||||
|
||||
|
||||
VERBOSE(2, "BEST HYPO TARGET LENGTH : " << bestHypo->GetSize() << endl)
|
||||
size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs
|
||||
size_t numEdgesTotal = edgeDensity * bestHypo->GetSize(); //as per Shankar, aim for (density * target length of MAP solution) arcs
|
||||
size_t numEdgesCreated = 0;
|
||||
VERBOSE(2, "Target edge count: " << numEdgesTotal << endl);
|
||||
|
||||
float prevScore = -999999;
|
||||
|
||||
|
||||
//now iterate over multimap
|
||||
for (multimap<float, const Hypothesis*>::const_iterator it = --sortHypsByVal.end(); it != --sortHypsByVal.begin(); --it) {
|
||||
float currEstimatedScore = it->first;
|
||||
@ -181,13 +181,13 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
|
||||
if (numEdgesCreated >= numEdgesTotal && prevScore > currEstimatedScore) //if this hyp has equal estimated score to previous, include its edges too
|
||||
break;
|
||||
|
||||
|
||||
prevScore = currEstimatedScore;
|
||||
VERBOSE(3, "Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
|
||||
VERBOSE(3, "Considering hyp " << currHyp->GetId() << ", estimated score: " << it->first << endl)
|
||||
|
||||
|
||||
survivingHyps.insert(currHyp); //CurrHyp made the cut
|
||||
|
||||
|
||||
// is its best predecessor already included ?
|
||||
if (survivingHyps.find(currHyp->GetPrevHypo()) != survivingHyps.end()) { //yes, then add an edge
|
||||
vector <Edge>& edges = incomingEdges[currHyp];
|
||||
@ -195,7 +195,7 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
edges.push_back(winningEdge);
|
||||
++numEdgesCreated;
|
||||
}
|
||||
|
||||
|
||||
//let's try the arcs too
|
||||
const ArcList *arcList = currHyp->GetArcList();
|
||||
if (arcList != NULL) {
|
||||
@ -204,10 +204,10 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
const Hypothesis *loserHypo = *iterArcList;
|
||||
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
|
||||
if (survivingHyps.find(loserPrevHypo) != survivingHyps.end()) { //found it, add edge
|
||||
double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
|
||||
double arcScore = loserHypo->GetScore() - loserPrevHypo->GetScore();
|
||||
Edge losingEdge(loserPrevHypo, currHyp, arcScore*scale, loserHypo->GetTargetPhrase());
|
||||
vector <Edge>& edges = incomingEdges[currHyp];
|
||||
edges.push_back(losingEdge);
|
||||
edges.push_back(losingEdge);
|
||||
++numEdgesCreated;
|
||||
}
|
||||
}
|
||||
@ -215,15 +215,15 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
|
||||
//Now if a successor node has already been visited, add an edge connecting the two
|
||||
map < const Hypothesis*, set < const Hypothesis* > >::const_iterator outgoingIt = outgoingHyps.find(currHyp);
|
||||
|
||||
|
||||
if (outgoingIt != outgoingHyps.end()) {//currHyp does have successors
|
||||
const set<const Hypothesis*> & outHyps = outgoingIt->second; //the successors
|
||||
for (set<const Hypothesis*>::const_iterator outHypIts = outHyps.begin(); outHypIts != outHyps.end(); ++outHypIts) {
|
||||
const Hypothesis* succHyp = *outHypIts;
|
||||
|
||||
|
||||
if (survivingHyps.find(succHyp) == survivingHyps.end()) //Have we encountered the successor yet?
|
||||
continue; //No, move on to next
|
||||
|
||||
|
||||
//Curr Hyp can be : a) the best predecessor of succ b) or an arc attached to succ
|
||||
if (succHyp->GetPrevHypo() == currHyp) { //best predecessor
|
||||
vector <Edge>& succEdges = incomingEdges[succHyp];
|
||||
@ -232,7 +232,7 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
survivingHyps.insert(succHyp);
|
||||
++numEdgesCreated;
|
||||
}
|
||||
|
||||
|
||||
//now, let's find an arc
|
||||
const ArcList *arcList = succHyp->GetArcList();
|
||||
if (arcList != NULL) {
|
||||
@ -242,9 +242,9 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
const Hypothesis* loserPrevHypo = loserHypo->GetPrevHypo();
|
||||
if (loserPrevHypo == currHyp) { //found it
|
||||
vector <Edge>& succEdges = incomingEdges[succHyp];
|
||||
double arcScore = loserHypo->GetScore() - currHyp->GetScore();
|
||||
double arcScore = loserHypo->GetScore() - currHyp->GetScore();
|
||||
Edge losingEdge(currHyp, succHyp,scale* arcScore, loserHypo->GetTargetPhrase());
|
||||
succEdges.push_back(losingEdge);
|
||||
succEdges.push_back(losingEdge);
|
||||
++numEdgesCreated;
|
||||
}
|
||||
}
|
||||
@ -252,14 +252,14 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
connectedHyp.clear();
|
||||
for (set <const Hypothesis*>::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) {
|
||||
connectedHyp.push_back(*it);
|
||||
}
|
||||
|
||||
|
||||
VERBOSE(2, "Done! Num edges created : "<< numEdgesCreated << ", numEdges wanted " << numEdgesTotal << endl)
|
||||
|
||||
|
||||
IFVERBOSE(3) {
|
||||
cerr << "Surviving hyps: " ;
|
||||
for (set <const Hypothesis*>::iterator it = survivingHyps.begin(); it != survivingHyps.end(); ++it) {
|
||||
@ -267,15 +267,16 @@ void pruneLatticeFB(Lattice & connectedHyp, map < const Hypothesis*, set <const
|
||||
}
|
||||
cerr << endl;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges,
|
||||
map<Phrase, float>& finalNgramScores, bool posteriors) {
|
||||
|
||||
|
||||
void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector<Edge> >& incomingEdges,
|
||||
map<Phrase, float>& finalNgramScores, bool posteriors)
|
||||
{
|
||||
|
||||
sort(connectedHyp.begin(),connectedHyp.end(),ascendingCoverageCmp); //sort by increasing source word cov
|
||||
|
||||
|
||||
/*cerr << "Lattice:" << endl;
|
||||
for (Lattice::const_iterator i = connectedHyp.begin(); i != connectedHyp.end(); ++i) {
|
||||
const Hypothesis* h = *i;
|
||||
@ -285,50 +286,49 @@ void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector
|
||||
cerr << edges[e];
|
||||
}
|
||||
}*/
|
||||
|
||||
|
||||
map<const Hypothesis*, float> forwardScore;
|
||||
forwardScore[connectedHyp[0]] = 0.0f; //forward score of hyp 0 is 1 (or 0 in logprob space)
|
||||
set< const Hypothesis *> finalHyps; //store completed hyps
|
||||
|
||||
NgramScores ngramScores;//ngram scores for each hyp
|
||||
|
||||
|
||||
NgramScores ngramScores;//ngram scores for each hyp
|
||||
|
||||
for (size_t i = 1; i < connectedHyp.size(); ++i) {
|
||||
const Hypothesis* currHyp = connectedHyp[i];
|
||||
if (currHyp->GetWordsBitmap().IsComplete()) {
|
||||
finalHyps.insert(currHyp);
|
||||
}
|
||||
|
||||
|
||||
VERBOSE(3, "Processing hyp: " << currHyp->GetId() << ", num words cov= " << currHyp->GetWordsBitmap().GetNumWordsCovered() << endl)
|
||||
|
||||
|
||||
vector <Edge> & edges = incomingEdges[currHyp];
|
||||
for (size_t e = 0; e < edges.size(); ++e) {
|
||||
const Edge& edge = edges[e];
|
||||
if (forwardScore.find(currHyp) == forwardScore.end()) {
|
||||
forwardScore[currHyp] = forwardScore[edge.GetTailNode()] + edge.GetScore();
|
||||
VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] = fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
forwardScore[currHyp] = log_sum(forwardScore[currHyp], forwardScore[edge.GetTailNode()] + edge.GetScore());
|
||||
VERBOSE(3, "Fwd score["<<currHyp->GetId()<<"] += fwdScore["<<edge.GetTailNode()->GetId() << "] + edge Score: " << edge.GetScore() << endl)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//Process ngrams now
|
||||
for (size_t j =0 ; j < edges.size(); ++j) {
|
||||
Edge& edge = edges[j];
|
||||
const NgramHistory & incomingPhrases = edge.GetNgrams(incomingEdges);
|
||||
|
||||
|
||||
//let's first score ngrams introduced by this edge
|
||||
for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) {
|
||||
for (NgramHistory::const_iterator it = incomingPhrases.begin(); it != incomingPhrases.end(); ++it) {
|
||||
const Phrase& ngram = it->first;
|
||||
const PathCounts& pathCounts = it->second;
|
||||
VERBOSE(4, "Calculating score for: " << it->first << endl)
|
||||
|
||||
|
||||
for (PathCounts::const_iterator pathCountIt = pathCounts.begin(); pathCountIt != pathCounts.end(); ++pathCountIt) {
|
||||
//Score of an n-gram is forward score of head node of leftmost edge + all edge scores
|
||||
const Path& path = pathCountIt->first;
|
||||
//cerr << "path count for " << ngram << " is " << pathCountIt->second << endl;
|
||||
float score = forwardScore[path[0]->GetTailNode()];
|
||||
float score = forwardScore[path[0]->GetTailNode()];
|
||||
for (size_t i = 0; i < path.size(); ++i) {
|
||||
score += path[i]->GetScore();
|
||||
}
|
||||
@ -340,88 +340,86 @@ void calcNgramExpectations(Lattice & connectedHyp, map<const Hypothesis*, vector
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//Now score ngrams that are just being propagated from the history
|
||||
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
|
||||
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(edge.GetTailNode());
|
||||
it != ngramScores.nodeEnd(edge.GetTailNode()); ++it) {
|
||||
const Phrase & currNgram = *(it->first);
|
||||
float currNgramScore = it->second;
|
||||
VERBOSE(4, "Calculating score for: " << currNgram << endl)
|
||||
|
||||
|
||||
// For posteriors, don't double count ngrams
|
||||
if (!posteriors || incomingPhrases.find(currNgram) == incomingPhrases.end()) {
|
||||
float score = edge.GetScore() + currNgramScore;
|
||||
ngramScores.addScore(currHyp,currNgram,score);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
float Z = 9999999; //the total score of the lattice
|
||||
|
||||
|
||||
//Done - Print out ngram posteriors for final hyps
|
||||
for (set< const Hypothesis *>::iterator finalHyp = finalHyps.begin(); finalHyp != finalHyps.end(); ++finalHyp) {
|
||||
const Hypothesis* hyp = *finalHyp;
|
||||
|
||||
|
||||
for (NgramScores::NodeScoreIterator it = ngramScores.nodeBegin(hyp); it != ngramScores.nodeEnd(hyp); ++it) {
|
||||
const Phrase& ngram = *(it->first);
|
||||
if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
|
||||
finalNgramScores[ngram] = it->second;
|
||||
}
|
||||
else {
|
||||
finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]);
|
||||
const Phrase& ngram = *(it->first);
|
||||
if (finalNgramScores.find(ngram) == finalNgramScores.end()) {
|
||||
finalNgramScores[ngram] = it->second;
|
||||
} else {
|
||||
finalNgramScores[ngram] = log_sum(it->second, finalNgramScores[ngram]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (Z == 9999999) {
|
||||
Z = forwardScore[hyp];
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
Z = log_sum(Z, forwardScore[hyp]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//Z *= scale; //scale the score
|
||||
|
||||
|
||||
for (map<Phrase, float>::iterator finalScoresIt = finalNgramScores.begin(); finalScoresIt != finalNgramScores.end(); ++finalScoresIt) {
|
||||
finalScoresIt->second = finalScoresIt->second - Z;
|
||||
IFVERBOSE(2) {
|
||||
VERBOSE(2,finalScoresIt->first << " [" << finalScoresIt->second << "]" << endl);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & incomingEdges) {
|
||||
|
||||
const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & incomingEdges)
|
||||
{
|
||||
|
||||
if (m_ngrams.size() > 0)
|
||||
return m_ngrams;
|
||||
|
||||
|
||||
const Phrase& currPhrase = GetWords();
|
||||
//Extract the n-grams local to this edge
|
||||
for (size_t start = 0; start < currPhrase.GetSize(); ++start) {
|
||||
for (size_t end = start; end < start + bleu_order; ++end) {
|
||||
if (end < currPhrase.GetSize()){
|
||||
if (end < currPhrase.GetSize()) {
|
||||
Phrase edgeNgram(Output);
|
||||
for (size_t index = start; index <= end; ++index) {
|
||||
edgeNgram.AddWord(currPhrase.GetWord(index));
|
||||
edgeNgram.AddWord(currPhrase.GetWord(index));
|
||||
}
|
||||
//cout << "Inserting Phrase : " << edgeNgram << endl;
|
||||
vector<const Edge*> edgeHistory;
|
||||
edgeHistory.push_back(this);
|
||||
storeNgramHistory(edgeNgram, edgeHistory);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
map<const Hypothesis*, vector<Edge> >::iterator it = incomingEdges.find(m_tailNode);
|
||||
if (it != incomingEdges.end()) { //node has incoming edges
|
||||
vector<Edge> & inEdges = it->second;
|
||||
|
||||
|
||||
for (vector<Edge>::iterator edge = inEdges.begin(); edge != inEdges.end(); ++edge) {//add the ngrams straddling prev and curr edge
|
||||
const NgramHistory & edgeIncomingNgrams = edge->GetNgrams(incomingEdges);
|
||||
for (NgramHistory::const_iterator edgeInNgramHist = edgeIncomingNgrams.begin(); edgeInNgramHist != edgeIncomingNgrams.end(); ++edgeInNgramHist) {
|
||||
@ -432,39 +430,40 @@ const NgramHistory& Edge::GetNgrams(map<const Hypothesis*, vector<Edge> > & inco
|
||||
IFVERBOSE(3) {
|
||||
cerr << "Edge: "<< *edge <<endl;
|
||||
cerr << "edgeWords: " << edgeWords << endl;
|
||||
cerr << "edgeInNgram: " << edgeIncomingNgram << endl;
|
||||
cerr << "edgeInNgram: " << edgeIncomingNgram << endl;
|
||||
}
|
||||
|
||||
|
||||
Phrase edgeSuffix(Output);
|
||||
Phrase ngramSuffix(Output);
|
||||
GetPhraseSuffix(edgeWords,back,edgeSuffix);
|
||||
GetPhraseSuffix(edgeIncomingNgram,back,ngramSuffix);
|
||||
|
||||
|
||||
if (ngramSuffix == edgeSuffix) { //we've got the suffix of previous edge
|
||||
size_t edgeInNgramSize = edgeIncomingNgram.GetSize();
|
||||
|
||||
for (size_t i = 0; i < GetWordsSize() && i + edgeInNgramSize < bleu_order ; ++i){
|
||||
|
||||
for (size_t i = 0; i < GetWordsSize() && i + edgeInNgramSize < bleu_order ; ++i) {
|
||||
Phrase newNgram(edgeIncomingNgram);
|
||||
for (size_t j = 0; j <= i ; ++j){
|
||||
for (size_t j = 0; j <= i ; ++j) {
|
||||
newNgram.AddWord(GetWords().GetWord(j));
|
||||
}
|
||||
VERBOSE(3, "Inserting New Phrase : " << newNgram << endl)
|
||||
|
||||
|
||||
for (PathCounts::const_iterator pathIt = edgeIncomingNgramPaths.begin(); pathIt != edgeIncomingNgramPaths.end(); ++pathIt) {
|
||||
Path newNgramPath = pathIt->first;
|
||||
newNgramPath.push_back(this);
|
||||
storeNgramHistory(newNgram, newNgramPath, pathIt->second);
|
||||
storeNgramHistory(newNgram, newNgramPath, pathIt->second);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return m_ngrams;
|
||||
}
|
||||
|
||||
//Add the last lastN words of origPhrase to targetPhrase
|
||||
void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const {
|
||||
void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const
|
||||
{
|
||||
size_t origSize = origPhrase.GetSize();
|
||||
size_t startIndex = origSize - lastN;
|
||||
for (size_t index = startIndex; index < origPhrase.GetSize(); ++index) {
|
||||
@ -472,7 +471,8 @@ void Edge::GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targ
|
||||
}
|
||||
}
|
||||
|
||||
bool Edge::operator< (const Edge& compare ) const {
|
||||
bool Edge::operator< (const Edge& compare ) const
|
||||
{
|
||||
if (m_headNode->GetId() < compare.m_headNode->GetId())
|
||||
return true;
|
||||
if (compare.m_headNode->GetId() < m_headNode->GetId())
|
||||
@ -484,178 +484,179 @@ bool Edge::operator< (const Edge& compare ) const {
|
||||
return GetScore() < compare.GetScore();
|
||||
}
|
||||
|
||||
ostream& operator<< (ostream& out, const Edge& edge) {
|
||||
ostream& operator<< (ostream& out, const Edge& edge)
|
||||
{
|
||||
out << "Head: " << edge.m_headNode->GetId() << ", Tail: " << edge.m_tailNode->GetId() << ", Score: " << edge.m_score << ", Phrase: " << edge.m_targetPhrase << endl;
|
||||
return out;
|
||||
}
|
||||
|
||||
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b) {
|
||||
|
||||
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b)
|
||||
{
|
||||
return a->GetWordsBitmap().GetNumWordsCovered() < b->GetWordsBitmap().GetNumWordsCovered();
|
||||
}
|
||||
|
||||
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList,
|
||||
vector<LatticeMBRSolution>& solutions, size_t n)
|
||||
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList,
|
||||
vector<LatticeMBRSolution>& solutions, size_t n)
|
||||
{
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
std::map < int, bool > connected;
|
||||
std::vector< const Hypothesis *> connectedList;
|
||||
map<Phrase, float> ngramPosteriors;
|
||||
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
|
||||
map<const Hypothesis*, vector<Edge> > incomingEdges;
|
||||
vector< float> estimatedScores;
|
||||
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
|
||||
|
||||
vector<float> mbrThetas = staticData.GetLatticeMBRThetas();
|
||||
float p = staticData.GetLatticeMBRPrecision();
|
||||
float r = staticData.GetLatticeMBRPRatio();
|
||||
float mapWeight = staticData.GetLatticeMBRMapWeight();
|
||||
if (mbrThetas.size() == 0) { //thetas not specified on the command line, use p and r instead
|
||||
mbrThetas.push_back(-1); //Theta 0
|
||||
mbrThetas.push_back(1/(bleu_order*p));
|
||||
for (size_t i = 2; i <= bleu_order; ++i){
|
||||
mbrThetas.push_back(mbrThetas[i-1] / r);
|
||||
}
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
std::map < int, bool > connected;
|
||||
std::vector< const Hypothesis *> connectedList;
|
||||
map<Phrase, float> ngramPosteriors;
|
||||
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
|
||||
map<const Hypothesis*, vector<Edge> > incomingEdges;
|
||||
vector< float> estimatedScores;
|
||||
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramPosteriors,true);
|
||||
|
||||
vector<float> mbrThetas = staticData.GetLatticeMBRThetas();
|
||||
float p = staticData.GetLatticeMBRPrecision();
|
||||
float r = staticData.GetLatticeMBRPRatio();
|
||||
float mapWeight = staticData.GetLatticeMBRMapWeight();
|
||||
if (mbrThetas.size() == 0) { //thetas not specified on the command line, use p and r instead
|
||||
mbrThetas.push_back(-1); //Theta 0
|
||||
mbrThetas.push_back(1/(bleu_order*p));
|
||||
for (size_t i = 2; i <= bleu_order; ++i) {
|
||||
mbrThetas.push_back(mbrThetas[i-1] / r);
|
||||
}
|
||||
IFVERBOSE(2) {
|
||||
VERBOSE(2,"Thetas: ");
|
||||
for (size_t i = 0; i < mbrThetas.size(); ++i) {
|
||||
VERBOSE(2,mbrThetas[i] << " ");
|
||||
}
|
||||
VERBOSE(2,endl);
|
||||
}
|
||||
IFVERBOSE(2) {
|
||||
VERBOSE(2,"Thetas: ");
|
||||
for (size_t i = 0; i < mbrThetas.size(); ++i) {
|
||||
VERBOSE(2,mbrThetas[i] << " ");
|
||||
}
|
||||
TrellisPathList::const_iterator iter;
|
||||
size_t ctr = 0;
|
||||
LatticeMBRSolutionComparator comparator;
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr)
|
||||
{
|
||||
const TrellisPath &path = **iter;
|
||||
solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin()));
|
||||
solutions.back().CalcScore(ngramPosteriors,mbrThetas,mapWeight);
|
||||
sort(solutions.begin(), solutions.end(), comparator);
|
||||
while (solutions.size() > n) {
|
||||
solutions.pop_back();
|
||||
}
|
||||
VERBOSE(2,endl);
|
||||
}
|
||||
TrellisPathList::const_iterator iter;
|
||||
size_t ctr = 0;
|
||||
LatticeMBRSolutionComparator comparator;
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter, ++ctr) {
|
||||
const TrellisPath &path = **iter;
|
||||
solutions.push_back(LatticeMBRSolution(path,iter==nBestList.begin()));
|
||||
solutions.back().CalcScore(ngramPosteriors,mbrThetas,mapWeight);
|
||||
sort(solutions.begin(), solutions.end(), comparator);
|
||||
while (solutions.size() > n) {
|
||||
solutions.pop_back();
|
||||
}
|
||||
VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl);
|
||||
}
|
||||
VERBOSE(2,"LMBR Score: " << solutions[0].GetScore() << endl);
|
||||
}
|
||||
|
||||
vector<Word> doLatticeMBR(Manager& manager, TrellisPathList& nBestList) {
|
||||
vector<Word> doLatticeMBR(Manager& manager, TrellisPathList& nBestList)
|
||||
{
|
||||
|
||||
vector<LatticeMBRSolution> solutions;
|
||||
getLatticeMBRNBest(manager, nBestList, solutions,1);
|
||||
return solutions.at(0).GetWords();
|
||||
vector<LatticeMBRSolution> solutions;
|
||||
getLatticeMBRNBest(manager, nBestList, solutions,1);
|
||||
return solutions.at(0).GetWords();
|
||||
}
|
||||
|
||||
const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList) {
|
||||
static const int BLEU_ORDER = 4;
|
||||
static const float SMOOTH = 1;
|
||||
|
||||
//calculate the ngram expectations
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
std::map < int, bool > connected;
|
||||
std::vector< const Hypothesis *> connectedList;
|
||||
map<Phrase, float> ngramExpectations;
|
||||
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
|
||||
map<const Hypothesis*, vector<Edge> > incomingEdges;
|
||||
vector< float> estimatedScores;
|
||||
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
|
||||
|
||||
//expected length is sum of expected unigram counts
|
||||
//cerr << "Thread " << pthread_self() << " Ngram expectations size: " << ngramExpectations.size() << endl;
|
||||
float ref_length = 0.0f;
|
||||
for (map<Phrase,float>::const_iterator ref_iter = ngramExpectations.begin();
|
||||
ref_iter != ngramExpectations.end(); ++ref_iter) {
|
||||
//cerr << "Ngram: " << ref_iter->first << " score: " <<
|
||||
// ref_iter->second << endl;
|
||||
if (ref_iter->first.GetSize() == 1) {
|
||||
ref_length += exp(ref_iter->second);
|
||||
// cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl;
|
||||
}
|
||||
}
|
||||
|
||||
VERBOSE(2,"REF Length: " << ref_length << endl);
|
||||
const TrellisPath doConsensusDecoding(Manager& manager, TrellisPathList& nBestList)
|
||||
{
|
||||
static const int BLEU_ORDER = 4;
|
||||
static const float SMOOTH = 1;
|
||||
|
||||
//use the ngram expectations to rescore the nbest list.
|
||||
TrellisPathList::const_iterator iter;
|
||||
TrellisPathList::const_iterator best = nBestList.end();
|
||||
float bestScore = -100000;
|
||||
//cerr << "nbest list size: " << nBestList.GetSize() << endl;
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
|
||||
{
|
||||
const TrellisPath &path = **iter;
|
||||
vector<Word> words;
|
||||
map<Phrase,int> ngrams;
|
||||
GetOutputWords(path,words);
|
||||
/*for (size_t i = 0; i < words.size(); ++i) {
|
||||
cerr << words[i].GetFactor(0)->GetString() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
extract_ngrams(words,ngrams);
|
||||
|
||||
vector<float> comps(2*BLEU_ORDER+1);
|
||||
float logbleu = 0.0;
|
||||
float brevity = 0.0;
|
||||
int hyp_length = words.size();
|
||||
for (int i = 0; i < BLEU_ORDER; ++i) {
|
||||
comps[2*i] = 0.0;
|
||||
comps[2*i+1] = max(hyp_length-i,0);
|
||||
}
|
||||
|
||||
for (map<Phrase,int>::const_iterator hyp_iter = ngrams.begin();
|
||||
hyp_iter != ngrams.end(); ++hyp_iter) {
|
||||
map<Phrase,float>::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first);
|
||||
if (ref_iter != ngramExpectations.end()) {
|
||||
comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second));
|
||||
}
|
||||
|
||||
}
|
||||
comps[comps.size()-1] = ref_length;
|
||||
/*for (size_t i = 0; i < comps.size(); ++i) {
|
||||
cerr << comps[i] << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
|
||||
float score = 0.0f;
|
||||
if (comps[0] != 0) {
|
||||
for (int i=0; i<BLEU_ORDER; i++)
|
||||
{
|
||||
if ( i > 0 ) {
|
||||
logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
|
||||
} else {
|
||||
logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
|
||||
}
|
||||
}
|
||||
logbleu /= BLEU_ORDER;
|
||||
brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
|
||||
if (brevity < 0.0) {
|
||||
logbleu += brevity;
|
||||
}
|
||||
score = exp(logbleu);
|
||||
}
|
||||
|
||||
//cerr << "score: " << score << " bestScore: " << bestScore << endl;
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
best = iter;
|
||||
VERBOSE(2,"NEW BEST: " << score << endl);
|
||||
//for (size_t i = 0; i < comps.size(); ++i) {
|
||||
// cerr << comps[i] << " ";
|
||||
//}
|
||||
//cerr << endl;
|
||||
}
|
||||
//calculate the ngram expectations
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
std::map < int, bool > connected;
|
||||
std::vector< const Hypothesis *> connectedList;
|
||||
map<Phrase, float> ngramExpectations;
|
||||
std::map < const Hypothesis*, set <const Hypothesis*> > outgoingHyps;
|
||||
map<const Hypothesis*, vector<Edge> > incomingEdges;
|
||||
vector< float> estimatedScores;
|
||||
manager.GetForwardBackwardSearchGraph(&connected, &connectedList, &outgoingHyps, &estimatedScores);
|
||||
pruneLatticeFB(connectedList, outgoingHyps, incomingEdges, estimatedScores, manager.GetBestHypothesis(), staticData.GetLatticeMBRPruningFactor(),staticData.GetMBRScale());
|
||||
calcNgramExpectations(connectedList, incomingEdges, ngramExpectations,false);
|
||||
|
||||
//expected length is sum of expected unigram counts
|
||||
//cerr << "Thread " << pthread_self() << " Ngram expectations size: " << ngramExpectations.size() << endl;
|
||||
float ref_length = 0.0f;
|
||||
for (map<Phrase,float>::const_iterator ref_iter = ngramExpectations.begin();
|
||||
ref_iter != ngramExpectations.end(); ++ref_iter) {
|
||||
//cerr << "Ngram: " << ref_iter->first << " score: " <<
|
||||
// ref_iter->second << endl;
|
||||
if (ref_iter->first.GetSize() == 1) {
|
||||
ref_length += exp(ref_iter->second);
|
||||
// cerr << "Expected for " << ref_iter->first << " is " << exp(ref_iter->second) << endl;
|
||||
}
|
||||
|
||||
assert (best != nBestList.end());
|
||||
return **best;
|
||||
//vector<Word> bestWords;
|
||||
//GetOutputWords(**best,bestWords);
|
||||
//return bestWords;
|
||||
}
|
||||
|
||||
VERBOSE(2,"REF Length: " << ref_length << endl);
|
||||
|
||||
//use the ngram expectations to rescore the nbest list.
|
||||
TrellisPathList::const_iterator iter;
|
||||
TrellisPathList::const_iterator best = nBestList.end();
|
||||
float bestScore = -100000;
|
||||
//cerr << "nbest list size: " << nBestList.GetSize() << endl;
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
|
||||
const TrellisPath &path = **iter;
|
||||
vector<Word> words;
|
||||
map<Phrase,int> ngrams;
|
||||
GetOutputWords(path,words);
|
||||
/*for (size_t i = 0; i < words.size(); ++i) {
|
||||
cerr << words[i].GetFactor(0)->GetString() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
extract_ngrams(words,ngrams);
|
||||
|
||||
vector<float> comps(2*BLEU_ORDER+1);
|
||||
float logbleu = 0.0;
|
||||
float brevity = 0.0;
|
||||
int hyp_length = words.size();
|
||||
for (int i = 0; i < BLEU_ORDER; ++i) {
|
||||
comps[2*i] = 0.0;
|
||||
comps[2*i+1] = max(hyp_length-i,0);
|
||||
}
|
||||
|
||||
for (map<Phrase,int>::const_iterator hyp_iter = ngrams.begin();
|
||||
hyp_iter != ngrams.end(); ++hyp_iter) {
|
||||
map<Phrase,float>::const_iterator ref_iter = ngramExpectations.find(hyp_iter->first);
|
||||
if (ref_iter != ngramExpectations.end()) {
|
||||
comps[2*(hyp_iter->first.GetSize()-1)] += min(exp(ref_iter->second), (float)(hyp_iter->second));
|
||||
}
|
||||
|
||||
}
|
||||
comps[comps.size()-1] = ref_length;
|
||||
/*for (size_t i = 0; i < comps.size(); ++i) {
|
||||
cerr << comps[i] << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
|
||||
float score = 0.0f;
|
||||
if (comps[0] != 0) {
|
||||
for (int i=0; i<BLEU_ORDER; i++) {
|
||||
if ( i > 0 ) {
|
||||
logbleu += log((float)comps[2*i]+SMOOTH)-log((float)comps[2*i+1]+SMOOTH);
|
||||
} else {
|
||||
logbleu += log((float)comps[2*i])-log((float)comps[2*i+1]);
|
||||
}
|
||||
}
|
||||
logbleu /= BLEU_ORDER;
|
||||
brevity = 1.0-(float)comps[comps.size()-1]/comps[1]; // comps[comps_n-1] is the ref length, comps[1] is the test length
|
||||
if (brevity < 0.0) {
|
||||
logbleu += brevity;
|
||||
}
|
||||
score = exp(logbleu);
|
||||
}
|
||||
|
||||
//cerr << "score: " << score << " bestScore: " << bestScore << endl;
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
best = iter;
|
||||
VERBOSE(2,"NEW BEST: " << score << endl);
|
||||
//for (size_t i = 0; i < comps.size(); ++i) {
|
||||
// cerr << comps[i] << " ";
|
||||
//}
|
||||
//cerr << endl;
|
||||
}
|
||||
}
|
||||
|
||||
assert (best != nBestList.end());
|
||||
return **best;
|
||||
//vector<Word> bestWords;
|
||||
//GetOutputWords(**best,bestWords);
|
||||
//return bestWords;
|
||||
}
|
||||
|
||||
|
||||
|
@ -34,111 +34,122 @@ T log_sum (T log_a, T log_b)
|
||||
class Edge;
|
||||
|
||||
typedef std::vector< const Hypothesis *> Lattice;
|
||||
typedef std::vector<const Edge*> Path;
|
||||
typedef std::vector<const Edge*> Path;
|
||||
typedef std::map<Path, size_t> PathCounts;
|
||||
typedef std::map<Phrase, PathCounts > NgramHistory;
|
||||
|
||||
class Edge {
|
||||
const Hypothesis* m_tailNode;
|
||||
const Hypothesis* m_headNode;
|
||||
float m_score;
|
||||
TargetPhrase m_targetPhrase;
|
||||
NgramHistory m_ngrams;
|
||||
|
||||
public:
|
||||
Edge(const Hypothesis* from, const Hypothesis* to, float score, const TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) {
|
||||
//cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl;
|
||||
}
|
||||
|
||||
const Hypothesis* GetHeadNode() const {
|
||||
return m_headNode;
|
||||
class Edge
|
||||
{
|
||||
const Hypothesis* m_tailNode;
|
||||
const Hypothesis* m_headNode;
|
||||
float m_score;
|
||||
TargetPhrase m_targetPhrase;
|
||||
NgramHistory m_ngrams;
|
||||
|
||||
public:
|
||||
Edge(const Hypothesis* from, const Hypothesis* to, float score, const TargetPhrase& targetPhrase) : m_tailNode(from), m_headNode(to), m_score(score), m_targetPhrase(targetPhrase) {
|
||||
//cout << "Creating new edge from Node " << from->GetId() << ", to Node : " << to->GetId() << ", score: " << score << " phrase: " << targetPhrase << endl;
|
||||
}
|
||||
|
||||
|
||||
const Hypothesis* GetHeadNode() const {
|
||||
return m_headNode;
|
||||
}
|
||||
|
||||
const Hypothesis* GetTailNode() const {
|
||||
return m_tailNode;
|
||||
}
|
||||
|
||||
|
||||
float GetScore() const {
|
||||
return m_score;
|
||||
}
|
||||
|
||||
|
||||
size_t GetWordsSize() const {
|
||||
return m_targetPhrase.GetSize();
|
||||
}
|
||||
|
||||
|
||||
const Phrase& GetWords() const {
|
||||
return m_targetPhrase;
|
||||
}
|
||||
|
||||
friend std::ostream& operator<< (std::ostream& out, const Edge& edge);
|
||||
|
||||
const NgramHistory& GetNgrams( std::map<const Hypothesis*, std::vector<Edge> > & incomingEdges) ;
|
||||
|
||||
bool operator < (const Edge & compare) const;
|
||||
|
||||
void GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const;
|
||||
|
||||
void storeNgramHistory(const Phrase& phrase, Path & path, size_t count = 1){
|
||||
m_ngrams[phrase][path]+= count;
|
||||
}
|
||||
|
||||
|
||||
friend std::ostream& operator<< (std::ostream& out, const Edge& edge);
|
||||
|
||||
const NgramHistory& GetNgrams( std::map<const Hypothesis*, std::vector<Edge> > & incomingEdges) ;
|
||||
|
||||
bool operator < (const Edge & compare) const;
|
||||
|
||||
void GetPhraseSuffix(const Phrase& origPhrase, size_t lastN, Phrase& targetPhrase) const;
|
||||
|
||||
void storeNgramHistory(const Phrase& phrase, Path & path, size_t count = 1) {
|
||||
m_ngrams[phrase][path]+= count;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* Data structure to hold the ngram scores as we traverse the lattice. Maps (hypo,ngram) to score
|
||||
*/
|
||||
class NgramScores {
|
||||
public:
|
||||
NgramScores() {}
|
||||
|
||||
/** logsum this score to the existing score */
|
||||
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
|
||||
|
||||
/** Iterate through ngrams for selected node */
|
||||
typedef std::map<const Phrase*, float>::const_iterator NodeScoreIterator;
|
||||
NodeScoreIterator nodeBegin(const Hypothesis* node);
|
||||
NodeScoreIterator nodeEnd(const Hypothesis* node);
|
||||
|
||||
private:
|
||||
std::set<Phrase> m_ngrams;
|
||||
std::map<const Hypothesis*, std::map<const Phrase*, float> > m_scores;
|
||||
class NgramScores
|
||||
{
|
||||
public:
|
||||
NgramScores() {}
|
||||
|
||||
/** logsum this score to the existing score */
|
||||
void addScore(const Hypothesis* node, const Phrase& ngram, float score);
|
||||
|
||||
/** Iterate through ngrams for selected node */
|
||||
typedef std::map<const Phrase*, float>::const_iterator NodeScoreIterator;
|
||||
NodeScoreIterator nodeBegin(const Hypothesis* node);
|
||||
NodeScoreIterator nodeEnd(const Hypothesis* node);
|
||||
|
||||
private:
|
||||
std::set<Phrase> m_ngrams;
|
||||
std::map<const Hypothesis*, std::map<const Phrase*, float> > m_scores;
|
||||
};
|
||||
|
||||
|
||||
/** Holds a lattice mbr solution, and its scores */
|
||||
class LatticeMBRSolution {
|
||||
public:
|
||||
/** Read the words from the path */
|
||||
LatticeMBRSolution(const TrellisPath& path, bool isMap);
|
||||
const std::vector<float>& GetNgramScores() const {return m_ngramScores;}
|
||||
const std::vector<Word>& GetWords() const {return m_words;}
|
||||
float GetMapScore() const {return m_mapScore;}
|
||||
float GetScore() const {return m_score;}
|
||||
|
||||
/** Initialise ngram scores */
|
||||
void CalcScore(std::map<Phrase, float>& finalNgramScores, const std::vector<float>& thetas, float mapWeight);
|
||||
|
||||
private:
|
||||
std::vector<Word> m_words;
|
||||
float m_mapScore;
|
||||
std::vector<float> m_ngramScores;
|
||||
float m_score;
|
||||
class LatticeMBRSolution
|
||||
{
|
||||
public:
|
||||
/** Read the words from the path */
|
||||
LatticeMBRSolution(const TrellisPath& path, bool isMap);
|
||||
const std::vector<float>& GetNgramScores() const {
|
||||
return m_ngramScores;
|
||||
}
|
||||
const std::vector<Word>& GetWords() const {
|
||||
return m_words;
|
||||
}
|
||||
float GetMapScore() const {
|
||||
return m_mapScore;
|
||||
}
|
||||
float GetScore() const {
|
||||
return m_score;
|
||||
}
|
||||
|
||||
/** Initialise ngram scores */
|
||||
void CalcScore(std::map<Phrase, float>& finalNgramScores, const std::vector<float>& thetas, float mapWeight);
|
||||
|
||||
private:
|
||||
std::vector<Word> m_words;
|
||||
float m_mapScore;
|
||||
std::vector<float> m_ngramScores;
|
||||
float m_score;
|
||||
};
|
||||
|
||||
struct LatticeMBRSolutionComparator {
|
||||
bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) {
|
||||
return a.GetScore() > b.GetScore();
|
||||
}
|
||||
bool operator()(const LatticeMBRSolution& a, const LatticeMBRSolution& b) {
|
||||
return a.GetScore() > b.GetScore();
|
||||
}
|
||||
};
|
||||
|
||||
void pruneLatticeFB(Lattice & connectedHyp, std::map < const Hypothesis*, std::set <const Hypothesis* > > & outgoingHyps, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges,
|
||||
void pruneLatticeFB(Lattice & connectedHyp, std::map < const Hypothesis*, std::set <const Hypothesis* > > & outgoingHyps, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges,
|
||||
const std::vector< float> & estimatedScores, const Hypothesis*, size_t edgeDensity,float scale);
|
||||
|
||||
//Use the ngram scores to rerank the nbest list, return at most n solutions
|
||||
void getLatticeMBRNBest(Manager& manager, TrellisPathList& nBestList, std::vector<LatticeMBRSolution>& solutions, size_t n);
|
||||
//calculate expectated ngram counts, clipping at 1 (ie calculating posteriors) if posteriors==true.
|
||||
void calcNgramExpectations(Lattice & connectedHyp, std::map<const Hypothesis*, std::vector<Edge> >& incomingEdges, std::map<Phrase,
|
||||
float>& finalNgramScores, bool posteriors);
|
||||
float>& finalNgramScores, bool posteriors);
|
||||
void GetOutputFactors(const TrellisPath &path, std::vector <Word> &translation);
|
||||
void extract_ngrams(const std::vector<Word >& sentence, std::map < Phrase, int > & allngrams);
|
||||
bool ascendingCoverageCmp(const Hypothesis* a, const Hypothesis* b);
|
||||
|
@ -33,7 +33,7 @@ POSSIBILITY OF SUCH DAMAGE.
|
||||
* Lattice MBR grid search. Enables a grid search through the four parameters (p,r,scale and prune) used in lattice MBR.
|
||||
See 'Lattice Minimum Bayes-Risk Decoding for Statistical Machine Translation by Tromble, Kumar, Och and Macherey,
|
||||
EMNLP 2008 for details of the parameters.
|
||||
|
||||
|
||||
The grid search is controlled by specifying comma separated lists for the lmbr parameters (-lmbr-p, -lmbr-r,
|
||||
-lmbr-pruning-factor and -mbr-scale). All other parameters are passed through to moses. If any of the lattice mbr
|
||||
parameters are missing, then they are set to their default values. Output is of the form:
|
||||
@ -58,148 +58,150 @@ using namespace Moses;
|
||||
//keys
|
||||
enum gridkey {lmbr_p,lmbr_r,lmbr_prune,lmbr_scale};
|
||||
|
||||
class Grid {
|
||||
public:
|
||||
/** Add a parameter with key, command line argument, and default value */
|
||||
void addParam(gridkey key, const string& arg, float defaultValue) {
|
||||
m_args[arg] = key;
|
||||
assert(m_grid.find(key) == m_grid.end());
|
||||
m_grid[key].push_back(defaultValue);
|
||||
}
|
||||
|
||||
/** Parse the arguments, removing those that define the grid and returning a copy of the rest */
|
||||
void parseArgs(int& argc, char**& argv) {
|
||||
char** newargv = new char*[argc+1]; //Space to add mbr parameter
|
||||
int newargc = 0;
|
||||
for (int i = 0; i < argc; ++i) {
|
||||
bool consumed = false;
|
||||
for (map<string,gridkey>::const_iterator argi = m_args.begin(); argi != m_args.end(); ++argi) {
|
||||
if (!strcmp(argv[i], argi->first.c_str())) {
|
||||
++i;
|
||||
if (i >= argc) {
|
||||
cerr << "Error: missing parameter for " << argi->first << endl;
|
||||
throw runtime_error("Missing parameter");
|
||||
} else {
|
||||
string value = argv[i];
|
||||
gridkey key = argi->second;
|
||||
if (m_grid[key].size() != 1) {
|
||||
throw runtime_error("Duplicate grid argument");
|
||||
}
|
||||
m_grid[key].clear();
|
||||
char delim = ',';
|
||||
string::size_type lastpos = value.find_first_not_of(delim);
|
||||
string::size_type pos = value.find_first_of(delim,lastpos);
|
||||
while (string::npos != pos || string::npos != lastpos) {
|
||||
float param = atof(value.substr(lastpos, pos-lastpos).c_str());
|
||||
if (!param) {
|
||||
cerr << "Error: Illegal grid parameter for " << argi->first << endl;
|
||||
throw runtime_error("Illegal grid parameter");
|
||||
}
|
||||
m_grid[key].push_back(param);
|
||||
lastpos = value.find_first_not_of(delim,pos);
|
||||
pos = value.find_first_of(delim,lastpos);
|
||||
}
|
||||
consumed = true;
|
||||
}
|
||||
if (consumed) break;
|
||||
}
|
||||
}
|
||||
if (!consumed) {
|
||||
newargv[newargc] = new char[strlen(argv[i]) + 1];
|
||||
strcpy(newargv[newargc],argv[i]);
|
||||
++newargc;
|
||||
}
|
||||
class Grid
|
||||
{
|
||||
public:
|
||||
/** Add a parameter with key, command line argument, and default value */
|
||||
void addParam(gridkey key, const string& arg, float defaultValue) {
|
||||
m_args[arg] = key;
|
||||
assert(m_grid.find(key) == m_grid.end());
|
||||
m_grid[key].push_back(defaultValue);
|
||||
}
|
||||
|
||||
/** Parse the arguments, removing those that define the grid and returning a copy of the rest */
|
||||
void parseArgs(int& argc, char**& argv) {
|
||||
char** newargv = new char*[argc+1]; //Space to add mbr parameter
|
||||
int newargc = 0;
|
||||
for (int i = 0; i < argc; ++i) {
|
||||
bool consumed = false;
|
||||
for (map<string,gridkey>::const_iterator argi = m_args.begin(); argi != m_args.end(); ++argi) {
|
||||
if (!strcmp(argv[i], argi->first.c_str())) {
|
||||
++i;
|
||||
if (i >= argc) {
|
||||
cerr << "Error: missing parameter for " << argi->first << endl;
|
||||
throw runtime_error("Missing parameter");
|
||||
} else {
|
||||
string value = argv[i];
|
||||
gridkey key = argi->second;
|
||||
if (m_grid[key].size() != 1) {
|
||||
throw runtime_error("Duplicate grid argument");
|
||||
}
|
||||
argc = newargc;
|
||||
argv = newargv;
|
||||
m_grid[key].clear();
|
||||
char delim = ',';
|
||||
string::size_type lastpos = value.find_first_not_of(delim);
|
||||
string::size_type pos = value.find_first_of(delim,lastpos);
|
||||
while (string::npos != pos || string::npos != lastpos) {
|
||||
float param = atof(value.substr(lastpos, pos-lastpos).c_str());
|
||||
if (!param) {
|
||||
cerr << "Error: Illegal grid parameter for " << argi->first << endl;
|
||||
throw runtime_error("Illegal grid parameter");
|
||||
}
|
||||
m_grid[key].push_back(param);
|
||||
lastpos = value.find_first_not_of(delim,pos);
|
||||
pos = value.find_first_of(delim,lastpos);
|
||||
}
|
||||
consumed = true;
|
||||
}
|
||||
if (consumed) break;
|
||||
}
|
||||
|
||||
/** Get the grid for a particular key.*/
|
||||
const vector<float>& getGrid(gridkey key) const {
|
||||
map<gridkey,vector<float> >::const_iterator iter = m_grid.find(key);
|
||||
assert (iter != m_grid.end());
|
||||
return iter->second;
|
||||
|
||||
}
|
||||
|
||||
private:
|
||||
map<gridkey,vector<float> > m_grid;
|
||||
map<string,gridkey> m_args;
|
||||
}
|
||||
if (!consumed) {
|
||||
newargv[newargc] = new char[strlen(argv[i]) + 1];
|
||||
strcpy(newargv[newargc],argv[i]);
|
||||
++newargc;
|
||||
}
|
||||
}
|
||||
argc = newargc;
|
||||
argv = newargv;
|
||||
}
|
||||
|
||||
/** Get the grid for a particular key.*/
|
||||
const vector<float>& getGrid(gridkey key) const {
|
||||
map<gridkey,vector<float> >::const_iterator iter = m_grid.find(key);
|
||||
assert (iter != m_grid.end());
|
||||
return iter->second;
|
||||
|
||||
}
|
||||
|
||||
private:
|
||||
map<gridkey,vector<float> > m_grid;
|
||||
map<string,gridkey> m_args;
|
||||
};
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
cerr << "Lattice MBR Grid search" << endl;
|
||||
|
||||
Grid grid;
|
||||
grid.addParam(lmbr_p, "-lmbr-p", 0.5);
|
||||
grid.addParam(lmbr_r, "-lmbr-r", 0.5);
|
||||
grid.addParam(lmbr_prune, "-lmbr-pruning-factor",30.0);
|
||||
grid.addParam(lmbr_scale, "-mbr-scale",1.0);
|
||||
|
||||
grid.parseArgs(argc,argv);
|
||||
|
||||
Parameter* params = new Parameter();
|
||||
if (!params->LoadParam(argc,argv)) {
|
||||
params->Explain();
|
||||
exit(1);
|
||||
}
|
||||
if (!StaticData::LoadDataStatic(params)) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
|
||||
staticData.SetUseLatticeMBR(true);
|
||||
IOWrapper* ioWrapper = GetIODevice(staticData);
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
cerr << "Lattice MBR Grid search" << endl;
|
||||
|
||||
if (!ioWrapper) {
|
||||
throw runtime_error("Failed to initialise IOWrapper");
|
||||
}
|
||||
size_t nBestSize = staticData.GetMBRSize();
|
||||
|
||||
if (nBestSize <= 0){
|
||||
throw new runtime_error("Non-positive size specified for n-best list");
|
||||
}
|
||||
|
||||
size_t lineCount = 0;
|
||||
InputType* source = NULL;
|
||||
|
||||
const vector<float>& pgrid = grid.getGrid(lmbr_p);
|
||||
const vector<float>& rgrid = grid.getGrid(lmbr_r);
|
||||
const vector<float>& prune_grid = grid.getGrid(lmbr_prune);
|
||||
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
|
||||
|
||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||
++lineCount;
|
||||
Sentence sentence(Input);
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
Manager manager(*source,staticData.GetSearchAlgorithm(), &system);
|
||||
manager.ProcessSentence();
|
||||
TrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,true);
|
||||
//grid search
|
||||
for (vector<float>::const_iterator pi = pgrid.begin(); pi != pgrid.end(); ++pi) {
|
||||
float p = *pi;
|
||||
staticData.SetLatticeMBRPrecision(p);
|
||||
for (vector<float>::const_iterator ri = rgrid.begin(); ri != rgrid.end(); ++ri) {
|
||||
float r = *ri;
|
||||
staticData.SetLatticeMBRPRatio(r);
|
||||
for (vector<float>::const_iterator prune_i = prune_grid.begin(); prune_i != prune_grid.end(); ++prune_i) {
|
||||
size_t prune = (size_t)(*prune_i);
|
||||
staticData.SetLatticeMBRPruningFactor(prune);
|
||||
for (vector<float>::const_iterator scale_i = scale_grid.begin(); scale_i != scale_grid.end(); ++scale_i) {
|
||||
float scale = *scale_i;
|
||||
staticData.SetMBRScale(scale);
|
||||
cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| ";
|
||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||
OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),cout);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Grid grid;
|
||||
grid.addParam(lmbr_p, "-lmbr-p", 0.5);
|
||||
grid.addParam(lmbr_r, "-lmbr-r", 0.5);
|
||||
grid.addParam(lmbr_prune, "-lmbr-pruning-factor",30.0);
|
||||
grid.addParam(lmbr_scale, "-mbr-scale",1.0);
|
||||
|
||||
grid.parseArgs(argc,argv);
|
||||
|
||||
Parameter* params = new Parameter();
|
||||
if (!params->LoadParam(argc,argv)) {
|
||||
params->Explain();
|
||||
exit(1);
|
||||
}
|
||||
if (!StaticData::LoadDataStatic(params)) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
StaticData& staticData = const_cast<StaticData&>(StaticData::Instance());
|
||||
staticData.SetUseLatticeMBR(true);
|
||||
IOWrapper* ioWrapper = GetIODevice(staticData);
|
||||
|
||||
if (!ioWrapper) {
|
||||
throw runtime_error("Failed to initialise IOWrapper");
|
||||
}
|
||||
size_t nBestSize = staticData.GetMBRSize();
|
||||
|
||||
if (nBestSize <= 0) {
|
||||
throw new runtime_error("Non-positive size specified for n-best list");
|
||||
}
|
||||
|
||||
size_t lineCount = 0;
|
||||
InputType* source = NULL;
|
||||
|
||||
const vector<float>& pgrid = grid.getGrid(lmbr_p);
|
||||
const vector<float>& rgrid = grid.getGrid(lmbr_r);
|
||||
const vector<float>& prune_grid = grid.getGrid(lmbr_prune);
|
||||
const vector<float>& scale_grid = grid.getGrid(lmbr_scale);
|
||||
|
||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||
++lineCount;
|
||||
Sentence sentence(Input);
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
Manager manager(*source,staticData.GetSearchAlgorithm(), &system);
|
||||
manager.ProcessSentence();
|
||||
TrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,true);
|
||||
//grid search
|
||||
for (vector<float>::const_iterator pi = pgrid.begin(); pi != pgrid.end(); ++pi) {
|
||||
float p = *pi;
|
||||
staticData.SetLatticeMBRPrecision(p);
|
||||
for (vector<float>::const_iterator ri = rgrid.begin(); ri != rgrid.end(); ++ri) {
|
||||
float r = *ri;
|
||||
staticData.SetLatticeMBRPRatio(r);
|
||||
for (vector<float>::const_iterator prune_i = prune_grid.begin(); prune_i != prune_grid.end(); ++prune_i) {
|
||||
size_t prune = (size_t)(*prune_i);
|
||||
staticData.SetLatticeMBRPruningFactor(prune);
|
||||
for (vector<float>::const_iterator scale_i = scale_grid.begin(); scale_i != scale_grid.end(); ++scale_i) {
|
||||
float scale = *scale_i;
|
||||
staticData.SetMBRScale(scale);
|
||||
cout << lineCount << " ||| " << p << " " << r << " " << prune << " " << scale << " ||| ";
|
||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||
OutputBestHypo(mbrBestHypo, lineCount, staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),cout);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -53,206 +53,211 @@ using namespace Moses;
|
||||
static const size_t PRECISION = 3;
|
||||
|
||||
/** Enforce rounding */
|
||||
void fix(std::ostream& stream, size_t size) {
|
||||
stream.setf(std::ios::fixed);
|
||||
stream.precision(size);
|
||||
void fix(std::ostream& stream, size_t size)
|
||||
{
|
||||
stream.setf(std::ios::fixed);
|
||||
stream.precision(size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Translates a sentence.
|
||||
**/
|
||||
class TranslationTask : public Task {
|
||||
class TranslationTask : public Task
|
||||
{
|
||||
|
||||
public:
|
||||
public:
|
||||
|
||||
TranslationTask(size_t lineNumber,
|
||||
InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
|
||||
OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
|
||||
OutputCollector* detailedTranslationCollector,
|
||||
OutputCollector* alignmentInfoCollector ) :
|
||||
m_source(source), m_lineNumber(lineNumber),
|
||||
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
|
||||
m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
|
||||
m_detailedTranslationCollector(detailedTranslationCollector),
|
||||
m_alignmentInfoCollector(alignmentInfoCollector) {}
|
||||
TranslationTask(size_t lineNumber,
|
||||
InputType* source, OutputCollector* outputCollector, OutputCollector* nbestCollector,
|
||||
OutputCollector* wordGraphCollector, OutputCollector* searchGraphCollector,
|
||||
OutputCollector* detailedTranslationCollector,
|
||||
OutputCollector* alignmentInfoCollector ) :
|
||||
m_source(source), m_lineNumber(lineNumber),
|
||||
m_outputCollector(outputCollector), m_nbestCollector(nbestCollector),
|
||||
m_wordGraphCollector(wordGraphCollector), m_searchGraphCollector(searchGraphCollector),
|
||||
m_detailedTranslationCollector(detailedTranslationCollector),
|
||||
m_alignmentInfoCollector(alignmentInfoCollector) {}
|
||||
|
||||
void Run()
|
||||
{
|
||||
void Run() {
|
||||
#ifdef BOOST_HAS_PTHREADS
|
||||
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl);
|
||||
TRACE_ERR("Translating line " << m_lineNumber << " in thread id " << pthread_self() << std::endl);
|
||||
#endif
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
Sentence sentence(Input);
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
Manager manager(*m_source,staticData.GetSearchAlgorithm(), &system);
|
||||
manager.ProcessSentence();
|
||||
|
||||
//Word Graph
|
||||
if (m_wordGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
manager.GetWordGraph(m_lineNumber, out);
|
||||
m_wordGraphCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
|
||||
//Search Graph
|
||||
if (m_searchGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
manager.OutputSearchGraph(m_lineNumber, out);
|
||||
m_searchGraphCollector->Write(m_lineNumber, out.str());
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
Sentence sentence(Input);
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
Manager manager(*m_source,staticData.GetSearchAlgorithm(), &system);
|
||||
manager.ProcessSentence();
|
||||
|
||||
//Word Graph
|
||||
if (m_wordGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
manager.GetWordGraph(m_lineNumber, out);
|
||||
m_wordGraphCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
|
||||
//Search Graph
|
||||
if (m_searchGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
manager.OutputSearchGraph(m_lineNumber, out);
|
||||
m_searchGraphCollector->Write(m_lineNumber, out.str());
|
||||
|
||||
#ifdef HAVE_PROTOBUF
|
||||
if (staticData.GetOutputSearchGraphPB()) {
|
||||
ostringstream sfn;
|
||||
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
|
||||
string fn = sfn.str();
|
||||
VERBOSE(2, "Writing search graph to " << fn << endl);
|
||||
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
|
||||
manager.SerializeSearchGraphPB(m_lineNumber, output);
|
||||
}
|
||||
if (staticData.GetOutputSearchGraphPB()) {
|
||||
ostringstream sfn;
|
||||
sfn << staticData.GetParam("output-search-graph-pb")[0] << '/' << m_lineNumber << ".pb" << ends;
|
||||
string fn = sfn.str();
|
||||
VERBOSE(2, "Writing search graph to " << fn << endl);
|
||||
fstream output(fn.c_str(), ios::trunc | ios::binary | ios::out);
|
||||
manager.SerializeSearchGraphPB(m_lineNumber, output);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (m_outputCollector) {
|
||||
ostringstream out;
|
||||
ostringstream debug;
|
||||
fix(debug,PRECISION);
|
||||
|
||||
//All derivations - send them to debug stream
|
||||
if (staticData.PrintAllDerivations()) {
|
||||
manager.PrintAllDerivations(m_lineNumber, debug);
|
||||
}
|
||||
|
||||
//Best hypothesis
|
||||
const Hypothesis* bestHypo = NULL;
|
||||
if (!staticData.UseMBR()) {
|
||||
bestHypo = manager.GetBestHypothesis();
|
||||
if (bestHypo) {
|
||||
if (staticData.IsPathRecoveryEnabled()) {
|
||||
OutputInput(out, bestHypo);
|
||||
out << "||| ";
|
||||
}
|
||||
OutputSurface(
|
||||
out,
|
||||
bestHypo,
|
||||
staticData.GetOutputFactorOrder(),
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors());
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
|
||||
IFVERBOSE(1) {
|
||||
debug << "BEST TRANSLATION: " << *bestHypo << endl;
|
||||
}
|
||||
}
|
||||
out << endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
size_t nBestSize = staticData.GetMBRSize();
|
||||
if (nBestSize <= 0)
|
||||
{
|
||||
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
|
||||
exit(1);
|
||||
}
|
||||
TrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,true);
|
||||
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
|
||||
IFVERBOSE(2) { PrintUserTime("calculated n-best list for (L)MBR decoding"); }
|
||||
|
||||
if (staticData.UseLatticeMBR())
|
||||
{
|
||||
if (m_nbestCollector)
|
||||
{
|
||||
//lattice mbr nbest
|
||||
vector<LatticeMBRSolution> solutions;
|
||||
size_t n = min(nBestSize, staticData.GetNBestSize());
|
||||
getLatticeMBRNBest(manager,nBestList,solutions,n);
|
||||
ostringstream out;
|
||||
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
|
||||
m_nbestCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
else
|
||||
{
|
||||
//Lattice MBR decoding
|
||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
IFVERBOSE(2) { PrintUserTime("finished Lattice MBR decoding"); }
|
||||
}
|
||||
}
|
||||
else if (staticData.UseConsensusDecoding()) {
|
||||
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
|
||||
OutputBestHypo(conBestHypo, m_lineNumber,
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
|
||||
IFVERBOSE(2) { PrintUserTime("finished Consensus decoding"); }
|
||||
}
|
||||
else
|
||||
{
|
||||
//MBR decoding
|
||||
const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList);
|
||||
OutputBestHypo(mbrBestHypo, m_lineNumber,
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
|
||||
IFVERBOSE(2) { PrintUserTime("finished MBR decoding"); }
|
||||
|
||||
}
|
||||
}
|
||||
m_outputCollector->Write(m_lineNumber,out.str(),debug.str());
|
||||
}
|
||||
if (m_nbestCollector && !staticData.UseLatticeMBR()) {
|
||||
TrellisPathList nBestList;
|
||||
ostringstream out;
|
||||
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
|
||||
OutputNBest(out,nBestList, staticData.GetOutputFactorOrder(), manager.GetTranslationSystem(), m_lineNumber);
|
||||
m_nbestCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
|
||||
//detailed translation reporting
|
||||
if (m_detailedTranslationCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
|
||||
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
|
||||
}
|
||||
}
|
||||
|
||||
IFVERBOSE(2) { PrintUserTime("Sentence Decoding Time:"); }
|
||||
|
||||
manager.CalcDecoderStatistics();
|
||||
|
||||
if (m_outputCollector) {
|
||||
ostringstream out;
|
||||
ostringstream debug;
|
||||
fix(debug,PRECISION);
|
||||
|
||||
//All derivations - send them to debug stream
|
||||
if (staticData.PrintAllDerivations()) {
|
||||
manager.PrintAllDerivations(m_lineNumber, debug);
|
||||
}
|
||||
|
||||
//Best hypothesis
|
||||
const Hypothesis* bestHypo = NULL;
|
||||
if (!staticData.UseMBR()) {
|
||||
bestHypo = manager.GetBestHypothesis();
|
||||
if (bestHypo) {
|
||||
if (staticData.IsPathRecoveryEnabled()) {
|
||||
OutputInput(out, bestHypo);
|
||||
out << "||| ";
|
||||
}
|
||||
OutputSurface(
|
||||
out,
|
||||
bestHypo,
|
||||
staticData.GetOutputFactorOrder(),
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors());
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, bestHypo);
|
||||
IFVERBOSE(1) {
|
||||
debug << "BEST TRANSLATION: " << *bestHypo << endl;
|
||||
}
|
||||
}
|
||||
out << endl;
|
||||
} else {
|
||||
size_t nBestSize = staticData.GetMBRSize();
|
||||
if (nBestSize <= 0) {
|
||||
cerr << "ERROR: negative size for number of MBR candidate translations not allowed (option mbr-size)" << endl;
|
||||
exit(1);
|
||||
}
|
||||
TrellisPathList nBestList;
|
||||
manager.CalcNBest(nBestSize, nBestList,true);
|
||||
VERBOSE(2,"size of n-best: " << nBestList.GetSize() << " (" << nBestSize << ")" << endl);
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("calculated n-best list for (L)MBR decoding");
|
||||
}
|
||||
|
||||
~TranslationTask() {delete m_source;}
|
||||
if (staticData.UseLatticeMBR()) {
|
||||
if (m_nbestCollector) {
|
||||
//lattice mbr nbest
|
||||
vector<LatticeMBRSolution> solutions;
|
||||
size_t n = min(nBestSize, staticData.GetNBestSize());
|
||||
getLatticeMBRNBest(manager,nBestList,solutions,n);
|
||||
ostringstream out;
|
||||
OutputLatticeMBRNBest(out, solutions,m_lineNumber);
|
||||
m_nbestCollector->Write(m_lineNumber, out.str());
|
||||
} else {
|
||||
//Lattice MBR decoding
|
||||
vector<Word> mbrBestHypo = doLatticeMBR(manager,nBestList);
|
||||
OutputBestHypo(mbrBestHypo, m_lineNumber, staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("finished Lattice MBR decoding");
|
||||
}
|
||||
}
|
||||
} else if (staticData.UseConsensusDecoding()) {
|
||||
const TrellisPath &conBestHypo = doConsensusDecoding(manager,nBestList);
|
||||
OutputBestHypo(conBestHypo, m_lineNumber,
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, conBestHypo);
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("finished Consensus decoding");
|
||||
}
|
||||
} else {
|
||||
//MBR decoding
|
||||
const Moses::TrellisPath &mbrBestHypo = doMBR(nBestList);
|
||||
OutputBestHypo(mbrBestHypo, m_lineNumber,
|
||||
staticData.GetReportSegmentation(),
|
||||
staticData.GetReportAllFactors(),out);
|
||||
OutputAlignment(m_alignmentInfoCollector, m_lineNumber, mbrBestHypo);
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("finished MBR decoding");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
m_outputCollector->Write(m_lineNumber,out.str(),debug.str());
|
||||
}
|
||||
if (m_nbestCollector && !staticData.UseLatticeMBR()) {
|
||||
TrellisPathList nBestList;
|
||||
ostringstream out;
|
||||
manager.CalcNBest(staticData.GetNBestSize(), nBestList,staticData.GetDistinctNBest());
|
||||
OutputNBest(out,nBestList, staticData.GetOutputFactorOrder(), manager.GetTranslationSystem(), m_lineNumber);
|
||||
m_nbestCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
|
||||
//detailed translation reporting
|
||||
if (m_detailedTranslationCollector) {
|
||||
ostringstream out;
|
||||
fix(out,PRECISION);
|
||||
TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
|
||||
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
|
||||
}
|
||||
|
||||
IFVERBOSE(2) {
|
||||
PrintUserTime("Sentence Decoding Time:");
|
||||
}
|
||||
|
||||
manager.CalcDecoderStatistics();
|
||||
}
|
||||
|
||||
~TranslationTask() {
|
||||
delete m_source;
|
||||
}
|
||||
|
||||
private:
|
||||
InputType* m_source;
|
||||
size_t m_lineNumber;
|
||||
OutputCollector* m_outputCollector;
|
||||
OutputCollector* m_nbestCollector;
|
||||
OutputCollector* m_wordGraphCollector;
|
||||
OutputCollector* m_searchGraphCollector;
|
||||
OutputCollector* m_detailedTranslationCollector;
|
||||
OutputCollector* m_alignmentInfoCollector;
|
||||
std::ofstream *m_alignmentStream;
|
||||
|
||||
private:
|
||||
InputType* m_source;
|
||||
size_t m_lineNumber;
|
||||
OutputCollector* m_outputCollector;
|
||||
OutputCollector* m_nbestCollector;
|
||||
OutputCollector* m_wordGraphCollector;
|
||||
OutputCollector* m_searchGraphCollector;
|
||||
OutputCollector* m_detailedTranslationCollector;
|
||||
OutputCollector* m_alignmentInfoCollector;
|
||||
std::ofstream *m_alignmentStream;
|
||||
|
||||
|
||||
};
|
||||
|
||||
static void PrintFeatureWeight(const FeatureFunction* ff) {
|
||||
|
||||
static void PrintFeatureWeight(const FeatureFunction* ff)
|
||||
{
|
||||
|
||||
size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID());
|
||||
size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID());
|
||||
for (size_t i = weightStart; i < weightEnd; ++i) {
|
||||
cout << ff->GetScoreProducerDescription() << " " << ff->GetScoreProducerWeightShortName() << " "
|
||||
<< StaticData::Instance().GetAllWeights()[i] << endl;
|
||||
cout << ff->GetScoreProducerDescription() << " " << ff->GetScoreProducerWeightShortName() << " "
|
||||
<< StaticData::Instance().GetAllWeights()[i] << endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void ShowWeights() {
|
||||
static void ShowWeights()
|
||||
{
|
||||
fix(cout,6);
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
@ -274,147 +279,149 @@ static void ShowWeights() {
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
|
||||
#ifdef HAVE_PROTOBUF
|
||||
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
||||
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
||||
#endif
|
||||
IFVERBOSE(1)
|
||||
{
|
||||
TRACE_ERR("command: ");
|
||||
for(int i=0;i<argc;++i) TRACE_ERR(argv[i]<<" ");
|
||||
TRACE_ERR(endl);
|
||||
}
|
||||
IFVERBOSE(1) {
|
||||
TRACE_ERR("command: ");
|
||||
for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" ");
|
||||
TRACE_ERR(endl);
|
||||
}
|
||||
|
||||
fix(cout,PRECISION);
|
||||
fix(cerr,PRECISION);
|
||||
fix(cout,PRECISION);
|
||||
fix(cerr,PRECISION);
|
||||
|
||||
|
||||
Parameter* params = new Parameter();
|
||||
if (!params->LoadParam(argc,argv)) {
|
||||
params->Explain();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
//create threadpool, if necessary
|
||||
int threadcount = (params->GetParam("threads").size() > 0) ?
|
||||
Scan<size_t>(params->GetParam("threads")[0]) : 1;
|
||||
|
||||
Parameter* params = new Parameter();
|
||||
if (!params->LoadParam(argc,argv)) {
|
||||
params->Explain();
|
||||
exit(1);
|
||||
}
|
||||
|
||||
//create threadpool, if necessary
|
||||
int threadcount = (params->GetParam("threads").size() > 0) ?
|
||||
Scan<size_t>(params->GetParam("threads")[0]) : 1;
|
||||
|
||||
#ifdef WITH_THREADS
|
||||
if (threadcount < 1) {
|
||||
cerr << "Error: Need to specify a positive number of threads" << endl;
|
||||
exit(1);
|
||||
}
|
||||
ThreadPool pool(threadcount);
|
||||
if (threadcount < 1) {
|
||||
cerr << "Error: Need to specify a positive number of threads" << endl;
|
||||
exit(1);
|
||||
}
|
||||
ThreadPool pool(threadcount);
|
||||
#else
|
||||
if (threadcount > 1) {
|
||||
cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
|
||||
exit(1);
|
||||
}
|
||||
if (threadcount > 1) {
|
||||
cerr << "Error: Thread count of " << threadcount << " but moses not built with thread support" << endl;
|
||||
exit(1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
if (!StaticData::LoadDataStatic(params)) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (params->isParamSpecified("show-weights")) {
|
||||
ShowWeights();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
// set up read/writing class
|
||||
IOWrapper* ioWrapper = GetIODevice(staticData);
|
||||
if (!StaticData::LoadDataStatic(params)) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (!ioWrapper) {
|
||||
cerr << "Error; Failed to create IO object" << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// check on weights
|
||||
vector<float> weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
|
||||
TRACE_ERR("The global weight vector looks like this:");
|
||||
for (size_t j=0; j<weights.size(); j++) { TRACE_ERR(" " << weights[j]); }
|
||||
TRACE_ERR("\n");
|
||||
}
|
||||
// every score must have a weight! check that here:
|
||||
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
|
||||
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (params->isParamSpecified("show-weights")) {
|
||||
ShowWeights();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
InputType* source = NULL;
|
||||
size_t lineCount = 0;
|
||||
auto_ptr<OutputCollector> outputCollector;//for translations
|
||||
auto_ptr<OutputCollector> nbestCollector;
|
||||
auto_ptr<ofstream> nbestOut;
|
||||
size_t nbestSize = staticData.GetNBestSize();
|
||||
string nbestFile = staticData.GetNBestFilePath();
|
||||
if (nbestSize) {
|
||||
if (nbestFile == "-" || nbestFile == "/dev/stdout") {
|
||||
//nbest to stdout, no 1-best
|
||||
nbestCollector.reset(new OutputCollector());
|
||||
} else {
|
||||
//nbest to file, 1-best to stdout
|
||||
nbestOut.reset(new ofstream(nbestFile.c_str()));
|
||||
assert(nbestOut->good());
|
||||
nbestCollector.reset(new OutputCollector(nbestOut.get()));
|
||||
outputCollector.reset(new OutputCollector());
|
||||
}
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
// set up read/writing class
|
||||
IOWrapper* ioWrapper = GetIODevice(staticData);
|
||||
|
||||
if (!ioWrapper) {
|
||||
cerr << "Error; Failed to create IO object" << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// check on weights
|
||||
vector<float> weights = staticData.GetAllWeights();
|
||||
IFVERBOSE(2) {
|
||||
TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager());
|
||||
TRACE_ERR("The global weight vector looks like this:");
|
||||
for (size_t j=0; j<weights.size(); j++) {
|
||||
TRACE_ERR(" " << weights[j]);
|
||||
}
|
||||
TRACE_ERR("\n");
|
||||
}
|
||||
// every score must have a weight! check that here:
|
||||
if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) {
|
||||
TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
||||
InputType* source = NULL;
|
||||
size_t lineCount = 0;
|
||||
auto_ptr<OutputCollector> outputCollector;//for translations
|
||||
auto_ptr<OutputCollector> nbestCollector;
|
||||
auto_ptr<ofstream> nbestOut;
|
||||
size_t nbestSize = staticData.GetNBestSize();
|
||||
string nbestFile = staticData.GetNBestFilePath();
|
||||
if (nbestSize) {
|
||||
if (nbestFile == "-" || nbestFile == "/dev/stdout") {
|
||||
//nbest to stdout, no 1-best
|
||||
nbestCollector.reset(new OutputCollector());
|
||||
} else {
|
||||
outputCollector.reset(new OutputCollector());
|
||||
//nbest to file, 1-best to stdout
|
||||
nbestOut.reset(new ofstream(nbestFile.c_str()));
|
||||
assert(nbestOut->good());
|
||||
nbestCollector.reset(new OutputCollector(nbestOut.get()));
|
||||
outputCollector.reset(new OutputCollector());
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> wordGraphCollector;
|
||||
if (staticData.GetOutputWordGraph()) {
|
||||
wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream())));
|
||||
} else {
|
||||
outputCollector.reset(new OutputCollector());
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> wordGraphCollector;
|
||||
if (staticData.GetOutputWordGraph()) {
|
||||
wordGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputWordGraphStream())));
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> searchGraphCollector;
|
||||
if (staticData.GetOutputSearchGraph()) {
|
||||
searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream())));
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> detailedTranslationCollector;
|
||||
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
||||
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
|
||||
}
|
||||
auto_ptr<OutputCollector> alignmentInfoCollector;
|
||||
if (!staticData.GetAlignmentOutputFile().empty()) {
|
||||
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
|
||||
}
|
||||
|
||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||
IFVERBOSE(1) {
|
||||
ResetUserTime();
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> searchGraphCollector;
|
||||
if (staticData.GetOutputSearchGraph()) {
|
||||
searchGraphCollector.reset(new OutputCollector(&(ioWrapper->GetOutputSearchGraphStream())));
|
||||
}
|
||||
|
||||
auto_ptr<OutputCollector> detailedTranslationCollector;
|
||||
if (staticData.IsDetailedTranslationReportingEnabled()) {
|
||||
detailedTranslationCollector.reset(new OutputCollector(&(ioWrapper->GetDetailedTranslationReportingStream())));
|
||||
}
|
||||
auto_ptr<OutputCollector> alignmentInfoCollector;
|
||||
if (!staticData.GetAlignmentOutputFile().empty()) {
|
||||
alignmentInfoCollector.reset(new OutputCollector(ioWrapper->GetAlignmentOutputStream()));
|
||||
}
|
||||
|
||||
while(ReadInput(*ioWrapper,staticData.GetInputType(),source)) {
|
||||
IFVERBOSE(1) {
|
||||
ResetUserTime();
|
||||
}
|
||||
TranslationTask* task =
|
||||
new TranslationTask(lineCount,source, outputCollector.get(),
|
||||
nbestCollector.get(), wordGraphCollector.get(),
|
||||
searchGraphCollector.get(),
|
||||
detailedTranslationCollector.get(),
|
||||
alignmentInfoCollector.get() );
|
||||
TranslationTask* task =
|
||||
new TranslationTask(lineCount,source, outputCollector.get(),
|
||||
nbestCollector.get(), wordGraphCollector.get(),
|
||||
searchGraphCollector.get(),
|
||||
detailedTranslationCollector.get(),
|
||||
alignmentInfoCollector.get() );
|
||||
#ifdef WITH_THREADS
|
||||
pool.Submit(task);
|
||||
pool.Submit(task);
|
||||
|
||||
#else
|
||||
task->Run();
|
||||
task->Run();
|
||||
#endif
|
||||
source = NULL; //make sure it doesn't get deleted
|
||||
++lineCount;
|
||||
}
|
||||
source = NULL; //make sure it doesn't get deleted
|
||||
++lineCount;
|
||||
}
|
||||
#ifdef WITH_THREADS
|
||||
pool.Stop(true); //flush remaining jobs
|
||||
pool.Stop(true); //flush remaining jobs
|
||||
#endif
|
||||
|
||||
#ifndef EXIT_RETURN
|
||||
//This avoids that destructors are called (it can take a long time)
|
||||
exit(EXIT_SUCCESS);
|
||||
//This avoids that destructors are called (it can take a long time)
|
||||
exit(EXIT_SUCCESS);
|
||||
#else
|
||||
return EXIT_SUCCESS;
|
||||
return EXIT_SUCCESS;
|
||||
#endif
|
||||
}
|
||||
|
@ -5,28 +5,28 @@ Moses - factored phrase-based language decoder
|
||||
Copyright (c) 2006 University of Edinburgh
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
Redistribution and use in source and binary forms, with or without modification,
|
||||
are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
* Neither the name of the University of Edinburgh nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
* Neither the name of the University of Edinburgh nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
||||
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
||||
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
|
||||
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
|
||||
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
***********************************************************************/
|
||||
|
||||
|
@ -9,11 +9,12 @@
|
||||
|
||||
using namespace Moses;
|
||||
|
||||
namespace TranslationAnalysis {
|
||||
|
||||
void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, const Hypothesis* hypo)
|
||||
namespace TranslationAnalysis
|
||||
{
|
||||
os << std::endl << "TRANSLATION HYPOTHESIS DETAILS:" << std::endl;
|
||||
|
||||
void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, const Hypothesis* hypo)
|
||||
{
|
||||
os << std::endl << "TRANSLATION HYPOTHESIS DETAILS:" << std::endl;
|
||||
std::vector<const Hypothesis*> translationPath;
|
||||
while (hypo) {
|
||||
translationPath.push_back(hypo);
|
||||
@ -24,83 +25,90 @@ namespace TranslationAnalysis {
|
||||
std::vector<std::string> droppedWords;
|
||||
std::vector<const Hypothesis*>::iterator tpi = translationPath.begin();
|
||||
++tpi; // skip initial translation state
|
||||
std::vector<std::string> sourceMap;
|
||||
std::vector<std::string> targetMap;
|
||||
std::vector<unsigned int> lmAcc(0);
|
||||
size_t lmCalls = 0;
|
||||
bool doLMStats = ((*tpi)->GetLMStats() != 0);
|
||||
if (doLMStats)
|
||||
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
|
||||
std::vector<std::string> sourceMap;
|
||||
std::vector<std::string> targetMap;
|
||||
std::vector<unsigned int> lmAcc(0);
|
||||
size_t lmCalls = 0;
|
||||
bool doLMStats = ((*tpi)->GetLMStats() != 0);
|
||||
if (doLMStats)
|
||||
lmAcc.resize((*tpi)->GetLMStats()->size(), 0);
|
||||
for (; tpi != translationPath.end(); ++tpi) {
|
||||
std::ostringstream sms;
|
||||
std::ostringstream tms;
|
||||
std::ostringstream sms;
|
||||
std::ostringstream tms;
|
||||
std::string target = (*tpi)->GetTargetPhraseStringRep();
|
||||
std::string source = (*tpi)->GetSourcePhraseStringRep();
|
||||
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
|
||||
WordsRange swr = (*tpi)->GetCurrSourceWordsRange();
|
||||
WordsRange twr = (*tpi)->GetCurrTargetWordsRange();
|
||||
WordsRange swr = (*tpi)->GetCurrSourceWordsRange();
|
||||
|
||||
const AlignmentInfo &alignmentInfo = (*tpi)->GetCurrTargetPhrase().GetAlignmentInfo();
|
||||
|
||||
// language model backoff stats,
|
||||
if (doLMStats) {
|
||||
std::vector<std::vector<unsigned int> >& lmstats = *(*tpi)->GetLMStats();
|
||||
std::vector<std::vector<unsigned int> >::iterator i = lmstats.begin();
|
||||
std::vector<unsigned int>::iterator acc = lmAcc.begin();
|
||||
const AlignmentInfo &alignmentInfo = (*tpi)->GetCurrTargetPhrase().GetAlignmentInfo();
|
||||
|
||||
for (; i != lmstats.end(); ++i, ++acc) {
|
||||
std::vector<unsigned int>::iterator j = i->begin();
|
||||
lmCalls += i->size();
|
||||
for (; j != i->end(); ++j) {
|
||||
(*acc) += *j;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool epsilon = false;
|
||||
// language model backoff stats,
|
||||
if (doLMStats) {
|
||||
std::vector<std::vector<unsigned int> >& lmstats = *(*tpi)->GetLMStats();
|
||||
std::vector<std::vector<unsigned int> >::iterator i = lmstats.begin();
|
||||
std::vector<unsigned int>::iterator acc = lmAcc.begin();
|
||||
|
||||
for (; i != lmstats.end(); ++i, ++acc) {
|
||||
std::vector<unsigned int>::iterator j = i->begin();
|
||||
lmCalls += i->size();
|
||||
for (; j != i->end(); ++j) {
|
||||
(*acc) += *j;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool epsilon = false;
|
||||
if (target == "") {
|
||||
target="<EPSILON>";
|
||||
epsilon = true;
|
||||
epsilon = true;
|
||||
droppedWords.push_back(source);
|
||||
}
|
||||
os << " SOURCE: " << swr << " " << source << std::endl
|
||||
<< " TRANSLATED AS: " << target << std::endl
|
||||
<< " WORD ALIGNED: " << alignmentInfo << std::endl;
|
||||
size_t twr_i = twr.GetStartPos();
|
||||
size_t swr_i = swr.GetStartPos();
|
||||
if (!epsilon) { sms << twr_i; }
|
||||
if (epsilon) { tms << "del(" << swr_i << ")"; } else { tms << swr_i; }
|
||||
swr_i++; twr_i++;
|
||||
for (; twr_i <= twr.GetEndPos() && twr.GetEndPos() != NOT_FOUND; twr_i++) {
|
||||
sms << '-' << twr_i;
|
||||
}
|
||||
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
|
||||
tms << '-' << swr_i;
|
||||
}
|
||||
if (!epsilon) targetMap.push_back(sms.str());
|
||||
sourceMap.push_back(tms.str());
|
||||
<< " TRANSLATED AS: " << target << std::endl
|
||||
<< " WORD ALIGNED: " << alignmentInfo << std::endl;
|
||||
size_t twr_i = twr.GetStartPos();
|
||||
size_t swr_i = swr.GetStartPos();
|
||||
if (!epsilon) {
|
||||
sms << twr_i;
|
||||
}
|
||||
if (epsilon) {
|
||||
tms << "del(" << swr_i << ")";
|
||||
} else {
|
||||
tms << swr_i;
|
||||
}
|
||||
swr_i++;
|
||||
twr_i++;
|
||||
for (; twr_i <= twr.GetEndPos() && twr.GetEndPos() != NOT_FOUND; twr_i++) {
|
||||
sms << '-' << twr_i;
|
||||
}
|
||||
for (; swr_i <= swr.GetEndPos() && swr.GetEndPos() != NOT_FOUND; swr_i++) {
|
||||
tms << '-' << swr_i;
|
||||
}
|
||||
if (!epsilon) targetMap.push_back(sms.str());
|
||||
sourceMap.push_back(tms.str());
|
||||
}
|
||||
std::vector<std::string>::iterator si = sourceMap.begin();
|
||||
std::vector<std::string>::iterator ti = targetMap.begin();
|
||||
os << std::endl << "SOURCE/TARGET SPANS:";
|
||||
os << std::endl << " SOURCE:";
|
||||
for (; si != sourceMap.end(); ++si) {
|
||||
os << " " << *si;
|
||||
}
|
||||
os << std::endl << " TARGET:";
|
||||
for (; ti != targetMap.end(); ++ti) {
|
||||
os << " " << *ti;
|
||||
}
|
||||
os << std::endl << std::endl;
|
||||
if (doLMStats && lmCalls > 0) {
|
||||
std::vector<unsigned int>::iterator acc = lmAcc.begin();
|
||||
const LMList& lmlist = system->GetLanguageModels();
|
||||
LMList::const_iterator i = lmlist.begin();
|
||||
for (; acc != lmAcc.end(); ++acc, ++i) {
|
||||
char buf[256];
|
||||
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
|
||||
os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
|
||||
}
|
||||
}
|
||||
std::vector<std::string>::iterator si = sourceMap.begin();
|
||||
std::vector<std::string>::iterator ti = targetMap.begin();
|
||||
os << std::endl << "SOURCE/TARGET SPANS:";
|
||||
os << std::endl << " SOURCE:";
|
||||
for (; si != sourceMap.end(); ++si) {
|
||||
os << " " << *si;
|
||||
}
|
||||
os << std::endl << " TARGET:";
|
||||
for (; ti != targetMap.end(); ++ti) {
|
||||
os << " " << *ti;
|
||||
}
|
||||
os << std::endl << std::endl;
|
||||
if (doLMStats && lmCalls > 0) {
|
||||
std::vector<unsigned int>::iterator acc = lmAcc.begin();
|
||||
const LMList& lmlist = system->GetLanguageModels();
|
||||
LMList::const_iterator i = lmlist.begin();
|
||||
for (; acc != lmAcc.end(); ++acc, ++i) {
|
||||
char buf[256];
|
||||
sprintf(buf, "%.4f", (float)(*acc)/(float)lmCalls);
|
||||
os << (*i)->GetScoreProducerDescription() <<", AVG N-GRAM LENGTH: " << buf << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
if (droppedWords.size() > 0) {
|
||||
std::vector<std::string>::iterator dwi = droppedWords.begin();
|
||||
@ -109,9 +117,9 @@ namespace TranslationAnalysis {
|
||||
os << "\tdropped=" << *dwi << std::endl;
|
||||
}
|
||||
}
|
||||
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
|
||||
os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): ";
|
||||
StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights());
|
||||
os << std::endl;
|
||||
os << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -18,7 +18,7 @@ namespace TranslationAnalysis
|
||||
* print details about the translation represented in hypothesis to
|
||||
* os. Included information: phrase alignment, words dropped, scores
|
||||
*/
|
||||
void PrintTranslationAnalysis(const Moses::TranslationSystem* system, std::ostream &os, const Moses::Hypothesis* hypo);
|
||||
void PrintTranslationAnalysis(const Moses::TranslationSystem* system, std::ostream &os, const Moses::Hypothesis* hypo);
|
||||
|
||||
}
|
||||
|
||||
|
@ -4,7 +4,8 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main() {
|
||||
int main()
|
||||
{
|
||||
cerr << "Reading PLF from STDIN...\n";
|
||||
string line;
|
||||
int lc = 0;
|
||||
|
@ -18,11 +18,11 @@ using namespace std ;
|
||||
using namespace Moses;
|
||||
|
||||
|
||||
/* Input :
|
||||
1. a sorted n-best list, with duplicates filtered out in the following format
|
||||
/* Input :
|
||||
1. a sorted n-best list, with duplicates filtered out in the following format
|
||||
0 ||| amr moussa is currently on a visit to libya , tomorrow , sunday , to hold talks with regard to the in sudan . ||| 0 -4.94418 0 0 -2.16036 0 0 -81.4462 -106.593 -114.43 -105.55 -12.7873 -26.9057 -25.3715 -52.9336 7.99917 -24 ||| -4.58432
|
||||
|
||||
2. a weight vector
|
||||
2. a weight vector
|
||||
3. bleu order ( default = 4)
|
||||
4. scaling factor to weigh the weight vector (default = 1.0)
|
||||
|
||||
@ -38,12 +38,9 @@ float min_interval = 1e-4;
|
||||
void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const Factor* >, int > & allngrams)
|
||||
{
|
||||
vector< const Factor* > ngram;
|
||||
for (int k = 0; k < BLEU_ORDER; k++)
|
||||
{
|
||||
for(int i =0; i < max((int)sentence.size()-k,0); i++)
|
||||
{
|
||||
for ( int j = i; j<= i+k; j++)
|
||||
{
|
||||
for (int k = 0; k < BLEU_ORDER; k++) {
|
||||
for(int i =0; i < max((int)sentence.size()-k,0); i++) {
|
||||
for ( int j = i; j<= i+k; j++) {
|
||||
ngram.push_back(sentence[j]);
|
||||
}
|
||||
++allngrams[ngram];
|
||||
@ -52,15 +49,15 @@ void extract_ngrams(const vector<const Factor* >& sentence, map < vector < const
|
||||
}
|
||||
}
|
||||
|
||||
float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats ) {
|
||||
float calculate_score(const vector< vector<const Factor*> > & sents, int ref, int hyp, vector < map < vector < const Factor *>, int > > & ngram_stats )
|
||||
{
|
||||
int comps_n = 2*BLEU_ORDER+1;
|
||||
vector<int> comps(comps_n);
|
||||
float logbleu = 0.0, brevity;
|
||||
|
||||
|
||||
int hyp_length = sents[hyp].size();
|
||||
|
||||
for (int i =0; i<BLEU_ORDER;i++)
|
||||
{
|
||||
for (int i =0; i<BLEU_ORDER; i++) {
|
||||
comps[2*i] = 0;
|
||||
comps[2*i+1] = max(hyp_length-i,0);
|
||||
}
|
||||
@ -69,18 +66,15 @@ float calculate_score(const vector< vector<const Factor*> > & sents, int ref, in
|
||||
map< vector < const Factor * >, int > & ref_ngrams = ngram_stats[ref] ;
|
||||
|
||||
for (map< vector< const Factor * >, int >::iterator it = hyp_ngrams.begin();
|
||||
it != hyp_ngrams.end(); it++)
|
||||
{
|
||||
it != hyp_ngrams.end(); it++) {
|
||||
map< vector< const Factor * >, int >::iterator ref_it = ref_ngrams.find(it->first);
|
||||
if(ref_it != ref_ngrams.end())
|
||||
{
|
||||
if(ref_it != ref_ngrams.end()) {
|
||||
comps[2* (it->first.size()-1)] += min(ref_it->second,it->second);
|
||||
}
|
||||
}
|
||||
comps[comps_n-1] = sents[ref].size();
|
||||
|
||||
for (int i=0; i<BLEU_ORDER; i++)
|
||||
{
|
||||
for (int i=0; i<BLEU_ORDER; i++) {
|
||||
if (comps[0] == 0)
|
||||
return 0.0;
|
||||
if ( i > 0 )
|
||||
@ -95,7 +89,8 @@ float calculate_score(const vector< vector<const Factor*> > & sents, int ref, in
|
||||
return exp(logbleu);
|
||||
}
|
||||
|
||||
const TrellisPath doMBR(const TrellisPathList& nBestList){
|
||||
const TrellisPath doMBR(const TrellisPathList& nBestList)
|
||||
{
|
||||
float marginal = 0;
|
||||
|
||||
vector<float> joint_prob_vec;
|
||||
@ -104,19 +99,17 @@ const TrellisPath doMBR(const TrellisPathList& nBestList){
|
||||
vector< map < vector <const Factor *>, int > > ngram_stats;
|
||||
|
||||
TrellisPathList::const_iterator iter;
|
||||
|
||||
|
||||
// get max score to prevent underflow
|
||||
float maxScore = -1e20;
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
|
||||
{
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
|
||||
const TrellisPath &path = **iter;
|
||||
float score = StaticData::Instance().GetMBRScale()
|
||||
* path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights());
|
||||
* path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights());
|
||||
if (maxScore < score) maxScore = score;
|
||||
}
|
||||
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter)
|
||||
{
|
||||
|
||||
for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) {
|
||||
const TrellisPath &path = **iter;
|
||||
joint_prob = UntransformScore(StaticData::Instance().GetMBRScale() * path.GetScoreBreakdown().InnerProduct(StaticData::Instance().GetAllWeights()) - maxScore);
|
||||
marginal += joint_prob;
|
||||
@ -125,62 +118,61 @@ const TrellisPath doMBR(const TrellisPathList& nBestList){
|
||||
// get words in translation
|
||||
vector<const Factor*> translation;
|
||||
GetOutputFactors(path, translation);
|
||||
|
||||
|
||||
// collect n-gram counts
|
||||
map < vector < const Factor *>, int > counts;
|
||||
extract_ngrams(translation,counts);
|
||||
|
||||
ngram_stats.push_back(counts);
|
||||
translations.push_back(translation);
|
||||
}
|
||||
}
|
||||
|
||||
vector<float> mbr_loss;
|
||||
float bleu, weightedLoss;
|
||||
float weightedLossCumul = 0;
|
||||
float minMBRLoss = 1000000;
|
||||
int minMBRLossIdx = -1;
|
||||
|
||||
/* Main MBR computation done here */
|
||||
iter = nBestList.begin();
|
||||
for (unsigned int i = 0; i < nBestList.GetSize(); i++){
|
||||
weightedLossCumul = 0;
|
||||
for (unsigned int j = 0; j < nBestList.GetSize(); j++){
|
||||
if ( i != j) {
|
||||
bleu = calculate_score(translations, j, i,ngram_stats );
|
||||
weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal);
|
||||
weightedLossCumul += weightedLoss;
|
||||
if (weightedLossCumul > minMBRLoss)
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (weightedLossCumul < minMBRLoss){
|
||||
minMBRLoss = weightedLossCumul;
|
||||
minMBRLossIdx = i;
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
/* Find sentence that minimises Bayes Risk under 1- BLEU loss */
|
||||
return nBestList.at(minMBRLossIdx);
|
||||
//return translations[minMBRLossIdx];
|
||||
vector<float> mbr_loss;
|
||||
float bleu, weightedLoss;
|
||||
float weightedLossCumul = 0;
|
||||
float minMBRLoss = 1000000;
|
||||
int minMBRLossIdx = -1;
|
||||
|
||||
/* Main MBR computation done here */
|
||||
iter = nBestList.begin();
|
||||
for (unsigned int i = 0; i < nBestList.GetSize(); i++) {
|
||||
weightedLossCumul = 0;
|
||||
for (unsigned int j = 0; j < nBestList.GetSize(); j++) {
|
||||
if ( i != j) {
|
||||
bleu = calculate_score(translations, j, i,ngram_stats );
|
||||
weightedLoss = ( 1 - bleu) * ( joint_prob_vec[j]/marginal);
|
||||
weightedLossCumul += weightedLoss;
|
||||
if (weightedLossCumul > minMBRLoss)
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (weightedLossCumul < minMBRLoss) {
|
||||
minMBRLoss = weightedLossCumul;
|
||||
minMBRLossIdx = i;
|
||||
}
|
||||
iter++;
|
||||
}
|
||||
/* Find sentence that minimises Bayes Risk under 1- BLEU loss */
|
||||
return nBestList.at(minMBRLossIdx);
|
||||
//return translations[minMBRLossIdx];
|
||||
}
|
||||
|
||||
void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation){
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
||||
assert (outputFactorOrder.size() == 1);
|
||||
void GetOutputFactors(const TrellisPath &path, vector <const Factor*> &translation)
|
||||
{
|
||||
const std::vector<const Hypothesis *> &edges = path.GetEdges();
|
||||
const std::vector<FactorType>& outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
||||
assert (outputFactorOrder.size() == 1);
|
||||
|
||||
// print the surface factor of the translation
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--)
|
||||
{
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++)
|
||||
{
|
||||
|
||||
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
|
||||
translation.push_back(factor);
|
||||
}
|
||||
}
|
||||
// print the surface factor of the translation
|
||||
for (int currEdge = (int)edges.size() - 1 ; currEdge >= 0 ; currEdge--) {
|
||||
const Hypothesis &edge = *edges[currEdge];
|
||||
const Phrase &phrase = edge.GetCurrTargetPhrase();
|
||||
size_t size = phrase.GetSize();
|
||||
for (size_t pos = 0 ; pos < size ; pos++) {
|
||||
|
||||
const Factor *factor = phrase.GetFactor(pos, outputFactorOrder[0]);
|
||||
translation.push_back(factor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user