2008-06-11 14:52:57 +04:00
|
|
|
// $Id$
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
|
|
|
|
|
|
|
#include <cassert>
|
|
|
|
#include <algorithm>
|
|
|
|
#include "TargetPhrase.h"
|
|
|
|
#include "PhraseDictionaryMemory.h"
|
|
|
|
#include "GenerationDictionary.h"
|
|
|
|
#include "LanguageModel.h"
|
|
|
|
#include "StaticData.h"
|
|
|
|
#include "ScoreIndexManager.h"
|
|
|
|
#include "LMList.h"
|
|
|
|
#include "ScoreComponentCollection.h"
|
|
|
|
#include "Util.h"
|
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
2008-09-12 22:09:06 +04:00
|
|
|
bool TargetPhrase::wordalignflag=StaticData::Instance().UseAlignmentInfo();
|
|
|
|
bool TargetPhrase::printalign=StaticData::Instance().PrintAlignmentInfo();
|
|
|
|
|
|
|
|
//bool TargetPhrase::wordalignflag;
|
|
|
|
//bool TargetPhrase::printalign;
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
TargetPhrase::TargetPhrase(FactorDirection direction)
|
|
|
|
:Phrase(direction),m_transScore(0.0), m_ngramScore(0.0), m_fullScore(0.0), m_sourcePhrase(0)
|
|
|
|
{
|
2008-09-12 22:09:06 +04:00
|
|
|
wordalignflag=StaticData::Instance().UseAlignmentInfo();
|
|
|
|
printalign=StaticData::Instance().PrintAlignmentInfo();
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void TargetPhrase::SetScore()
|
|
|
|
{ // used when creating translations of unknown words:
|
|
|
|
m_transScore = m_ngramScore = 0;
|
|
|
|
m_fullScore = - StaticData::Instance().GetWeightWordPenalty();
|
|
|
|
}
|
|
|
|
|
2008-09-12 22:09:06 +04:00
|
|
|
void TargetPhrase::SetAlignment()
|
|
|
|
{
|
|
|
|
m_alignmentPair.SetIdentityAlignment();
|
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
void TargetPhrase::SetScore(float score)
|
|
|
|
{
|
|
|
|
//we use an existing score producer to figure out information for score setting (number of scores and weights)
|
|
|
|
//TODO: is this a good idea?
|
|
|
|
ScoreProducer* prod = StaticData::Instance().GetPhraseDictionaries()[0];
|
|
|
|
|
|
|
|
//get the weight list
|
|
|
|
unsigned int id = prod->GetScoreBookkeepingID();
|
|
|
|
|
|
|
|
const vector<float> &allWeights = StaticData::Instance().GetAllWeights();
|
|
|
|
|
|
|
|
size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id);
|
|
|
|
size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id);
|
|
|
|
|
|
|
|
vector<float> weights;
|
|
|
|
|
|
|
|
std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights));
|
|
|
|
|
|
|
|
//find out how many items are in the score vector for this producer
|
|
|
|
size_t numScores = prod->GetNumScoreComponents();
|
|
|
|
|
|
|
|
//divide up the score among all of the score vectors
|
|
|
|
vector <float> scoreVector(numScores,score/numScores);
|
|
|
|
|
|
|
|
//Now we have what we need to call the full SetScore method
|
|
|
|
SetScore(prod,scoreVector,weights,StaticData::Instance().GetWeightWordPenalty(),StaticData::Instance().GetAllLM());
|
|
|
|
}
|
|
|
|
|
|
|
|
void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer,
|
2008-09-12 22:09:06 +04:00
|
|
|
const Scores &scoreVector,
|
|
|
|
const vector<float> &weightT,
|
2008-06-11 14:52:57 +04:00
|
|
|
float weightWP, const LMList &languageModels)
|
|
|
|
{
|
|
|
|
assert(weightT.size() == scoreVector.size());
|
|
|
|
// calc average score if non-best
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f);
|
|
|
|
m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector);
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// Replicated from TranslationOptions.cpp
|
|
|
|
float totalFutureScore = 0;
|
|
|
|
float totalNgramScore = 0;
|
|
|
|
float totalFullScore = 0;
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
LMList::const_iterator lmIter;
|
|
|
|
for (lmIter = languageModels.begin(); lmIter != languageModels.end(); ++lmIter)
|
|
|
|
{
|
|
|
|
const LanguageModel &lm = **lmIter;
|
|
|
|
|
|
|
|
if (lm.Useable(*this))
|
|
|
|
{ // contains factors used by this LM
|
|
|
|
const float weightLM = lm.GetWeight();
|
|
|
|
float fullScore, nGramScore;
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
lm.CalcScore(*this, fullScore, nGramScore);
|
|
|
|
m_scoreBreakdown.Assign(&lm, nGramScore);
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
// total LM score so far
|
|
|
|
totalNgramScore += nGramScore * weightLM;
|
|
|
|
totalFullScore += fullScore * weightLM;
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
m_ngramScore = totalNgramScore;
|
2008-09-12 22:09:06 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
m_fullScore = m_transScore + totalFutureScore + totalFullScore
|
2008-09-12 22:09:06 +04:00
|
|
|
- (this->GetSize() * weightWP); // word penalty
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
void TargetPhrase::SetWeights(const ScoreProducer* translationScoreProducer, const vector<float> &weightT)
|
|
|
|
{
|
|
|
|
// calling this function in case of confusion net input is undefined
|
|
|
|
assert(StaticData::Instance().GetInputType()==SentenceInput);
|
|
|
|
|
|
|
|
/* one way to fix this, you have to make sure the weightT contains (in
|
|
|
|
addition to the usual phrase translation scaling factors) the input
|
|
|
|
weight factor as last element
|
|
|
|
*/
|
|
|
|
|
|
|
|
m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT);
|
|
|
|
}
|
|
|
|
|
|
|
|
void TargetPhrase::ResetScore()
|
|
|
|
{
|
|
|
|
m_fullScore = m_ngramScore = 0;
|
|
|
|
m_scoreBreakdown.ZeroAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
TargetPhrase *TargetPhrase::MergeNext(const TargetPhrase &inputPhrase) const
|
|
|
|
{
|
|
|
|
if (! IsCompatible(inputPhrase))
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
// ok, merge
|
|
|
|
TargetPhrase *clone = new TargetPhrase(*this);
|
|
|
|
clone->m_sourcePhrase = m_sourcePhrase;
|
|
|
|
int currWord = 0;
|
|
|
|
const size_t len = GetSize();
|
|
|
|
for (size_t currPos = 0 ; currPos < len ; currPos++)
|
|
|
|
{
|
|
|
|
const Word &inputWord = inputPhrase.GetWord(currPos);
|
|
|
|
Word &cloneWord = clone->GetWord(currPos);
|
|
|
|
cloneWord.Merge(inputWord);
|
|
|
|
|
|
|
|
currWord++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return clone;
|
|
|
|
}
|
|
|
|
|
2008-09-12 22:09:06 +04:00
|
|
|
// helper functions
|
|
|
|
void AddAlignmentElement(AlignmentPhraseInserter &inserter
|
|
|
|
, const string &str
|
|
|
|
, size_t phraseSize
|
|
|
|
, size_t otherPhraseSize
|
|
|
|
, list<size_t> &uniformAlignment)
|
|
|
|
{
|
|
|
|
// input
|
|
|
|
vector<string> alignPhraseVector = Tokenize(str);
|
|
|
|
// from
|
|
|
|
// "(0) (3) (1,2)"
|
|
|
|
// to
|
|
|
|
// "(0)" "(3)" "(1,2)"
|
|
|
|
assert (alignPhraseVector.size() == phraseSize) ;
|
|
|
|
|
|
|
|
const size_t inputSize = alignPhraseVector.size();
|
|
|
|
for (size_t pos = 0 ; pos < inputSize ; ++pos)
|
|
|
|
{
|
|
|
|
string alignElementStr = alignPhraseVector[pos];
|
|
|
|
|
|
|
|
//change "()" into "(-1)" for both source and target word-to-word alignments
|
|
|
|
std::string emtpyAlignStr="()";
|
|
|
|
std::string replaceAlignStr="(-1)";
|
|
|
|
alignElementStr=Replace(alignElementStr,emtpyAlignStr,replaceAlignStr);
|
|
|
|
|
|
|
|
//remove all "(" from both source and target word-to-word alignments
|
|
|
|
emtpyAlignStr="(";
|
|
|
|
replaceAlignStr="";
|
|
|
|
alignElementStr=Replace(alignElementStr,emtpyAlignStr,replaceAlignStr);
|
|
|
|
|
|
|
|
//remove all ")" from both source and target word-to-word alignments
|
|
|
|
emtpyAlignStr=")";
|
|
|
|
replaceAlignStr="";
|
|
|
|
alignElementStr=Replace(alignElementStr,emtpyAlignStr,replaceAlignStr);
|
|
|
|
|
|
|
|
AlignmentElement *alignElement = new AlignmentElement(Tokenize<AlignmentElementType>(alignElementStr, ","));
|
|
|
|
// "(1,2)"
|
|
|
|
// to
|
|
|
|
// [1] [2]
|
|
|
|
if (alignElement->GetSize() == 0)
|
|
|
|
{ // no alignment info. add uniform alignment, ie. can be aligned to any word
|
|
|
|
alignElement->SetUniformAlignment(otherPhraseSize);
|
|
|
|
uniformAlignment.push_back(pos);
|
|
|
|
}
|
|
|
|
|
|
|
|
**inserter = alignElement;
|
|
|
|
(*inserter)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// helper functions
|
|
|
|
void AddAlignmentElement(AlignmentPhraseInserter &inserter
|
|
|
|
, const WordAlignments &wa
|
|
|
|
, size_t phraseSize
|
|
|
|
, size_t otherPhraseSize
|
|
|
|
, list<size_t> &uniformAlignment)
|
|
|
|
{
|
|
|
|
// from
|
|
|
|
// "(0) (3) (1,2)"
|
|
|
|
// to
|
|
|
|
// "(0)" "(3)" "(1,2)"
|
|
|
|
assert (wa.size() == phraseSize) ;
|
|
|
|
|
|
|
|
const size_t inputSize = wa.size();
|
|
|
|
for (size_t pos = 0 ; pos < inputSize ; ++pos)
|
|
|
|
{
|
|
|
|
string alignElementStr = wa[pos];
|
|
|
|
AlignmentElement *alignElement = new AlignmentElement(Tokenize<AlignmentElementType>(alignElementStr, ","));
|
|
|
|
// "(1,2)"
|
|
|
|
// to
|
|
|
|
// [1] [2]
|
|
|
|
if (alignElement->GetSize() == 0)
|
|
|
|
{ // no alignment info. add uniform alignment, ie. can be aligned to any word
|
|
|
|
alignElement->SetUniformAlignment(otherPhraseSize);
|
|
|
|
uniformAlignment.push_back(pos);
|
|
|
|
}
|
|
|
|
|
|
|
|
**inserter = alignElement;
|
|
|
|
(*inserter)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void TargetPhrase::CreateAlignmentInfo(const WordAlignments &swa
|
|
|
|
, const WordAlignments &twa)
|
|
|
|
{
|
|
|
|
AlignmentPhraseInserter sourceInserter = m_alignmentPair.GetInserter(Input);
|
|
|
|
AlignmentPhraseInserter targetInserter = m_alignmentPair.GetInserter(Output);
|
|
|
|
list<size_t> uniformAlignmentSource, uniformAlignmentTarget;
|
|
|
|
|
|
|
|
if (!UseWordAlignment()){ //build uniform word-to-word alignment to fit the internal structure which requires their presence
|
|
|
|
std::string srcAlignStr,trgAlignStr;
|
|
|
|
UniformAlignment(srcAlignStr, m_sourcePhrase->GetSize(), GetSize());
|
|
|
|
UniformAlignment(trgAlignStr, GetSize(), m_sourcePhrase->GetSize());
|
|
|
|
CreateAlignmentInfo(srcAlignStr,trgAlignStr);
|
|
|
|
}
|
|
|
|
else{
|
|
|
|
AddAlignmentElement(sourceInserter
|
|
|
|
, swa
|
|
|
|
, m_sourcePhrase->GetSize()
|
|
|
|
, GetSize()
|
|
|
|
, uniformAlignmentSource);
|
|
|
|
AddAlignmentElement(targetInserter
|
|
|
|
, twa
|
|
|
|
, GetSize()
|
|
|
|
, m_sourcePhrase->GetSize()
|
|
|
|
, uniformAlignmentTarget);
|
|
|
|
}
|
|
|
|
// propergate uniform alignments to other side
|
|
|
|
// m_alignmentPair.GetAlignmentPhrase(Output).AddUniformAlignmentElement(uniformAlignmentSource);
|
|
|
|
// m_alignmentPair.GetAlignmentPhrase(Input).AddUniformAlignmentElement(uniformAlignmentTarget);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void TargetPhrase::CreateAlignmentInfo(const string &sourceStr
|
|
|
|
, const string &targetStr)
|
|
|
|
{
|
|
|
|
AlignmentPhraseInserter sourceInserter = m_alignmentPair.GetInserter(Input);
|
|
|
|
AlignmentPhraseInserter targetInserter = m_alignmentPair.GetInserter(Output);
|
|
|
|
list<size_t> uniformAlignmentSource, uniformAlignmentTarget;
|
|
|
|
|
|
|
|
AddAlignmentElement(sourceInserter
|
|
|
|
, sourceStr
|
|
|
|
, m_sourcePhrase->GetSize()
|
|
|
|
, GetSize()
|
|
|
|
, uniformAlignmentSource);
|
|
|
|
AddAlignmentElement(targetInserter
|
|
|
|
, targetStr
|
|
|
|
, GetSize()
|
|
|
|
, m_sourcePhrase->GetSize()
|
|
|
|
, uniformAlignmentTarget);
|
|
|
|
// propergate uniform alignments to other side
|
|
|
|
// m_alignmentPair.GetAlignmentPhrase(Output).AddUniformAlignmentElement(uniformAlignmentSource);
|
|
|
|
// m_alignmentPair.GetAlignmentPhrase(Input).AddUniformAlignmentElement(uniformAlignmentTarget);
|
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
TO_STRING_BODY(TargetPhrase);
|
|
|
|
|
|
|
|
std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp)
|
|
|
|
{
|
2008-09-12 22:09:06 +04:00
|
|
|
os << static_cast<const Phrase&>(tp);
|
|
|
|
os << ", pC=" << tp.m_transScore << ", c=" << tp.m_fullScore;
|
|
|
|
if (tp.PrintAlignmentInfo())
|
|
|
|
os << ", " << tp.GetAlignmentPair();
|
2008-06-11 14:52:57 +04:00
|
|
|
return os;
|
|
|
|
}
|