2015-10-24 04:27:58 +03:00
// -*- mode: c++; indent-tabs-mode: nil; tab-width:2 -*-
2010-11-17 17:06:21 +03:00
/***********************************************************************
Moses - factored phrase - based language decoder
Copyright ( C ) 2006 University of Edinburgh
This library is free software ; you can redistribute it and / or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation ; either
version 2.1 of the License , or ( at your option ) any later version .
This library is distributed in the hope that it will be useful ,
but WITHOUT ANY WARRANTY ; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the GNU
Lesser General Public License for more details .
You should have received a copy of the GNU Lesser General Public
License along with this library ; if not , write to the Free Software
Foundation , Inc . , 51 Franklin Street , Fifth Floor , Boston , MA 02110 - 1301 USA
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# include <limits>
# include <iostream>
# include <memory>
# include <sstream>
2013-05-24 21:02:49 +04:00
# include "moses/FF/FFState.h"
2012-11-12 23:56:18 +04:00
# include "Implementation.h"
2013-02-21 18:29:51 +04:00
# include "ChartState.h"
2012-11-12 23:56:18 +04:00
# include "moses/TypeDef.h"
# include "moses/Util.h"
# include "moses/Manager.h"
# include "moses/FactorCollection.h"
# include "moses/Phrase.h"
# include "moses/StaticData.h"
# include "moses/ChartManager.h"
# include "moses/ChartHypothesis.h"
2014-01-13 18:37:05 +04:00
# include "util/exception.hh"
2010-11-17 17:06:21 +03:00
using namespace std ;
namespace Moses
{
2013-10-29 22:59:53 +04:00
LanguageModelImplementation : : LanguageModelImplementation ( const std : : string & line )
: LanguageModel ( line )
2016-11-15 03:00:37 +03:00
, m_nGramOrder ( NOT_FOUND )
2013-08-28 14:06:27 +04:00
{
2013-08-28 14:05:47 +04:00
}
void LanguageModelImplementation : : SetParameter ( const std : : string & key , const std : : string & value )
{
2013-08-28 14:06:27 +04:00
if ( key = = " order " ) {
m_nGramOrder = Scan < size_t > ( value ) ;
} else if ( key = = " path " ) {
m_filePath = value ;
} else {
LanguageModel : : SetParameter ( key , value ) ;
}
2013-08-28 14:05:47 +04:00
}
2011-09-21 20:06:48 +04:00
void LanguageModelImplementation : : ShiftOrPush ( std : : vector < const Word * > & contextFactor , const Word & word ) const
{
if ( contextFactor . size ( ) < GetNGramOrder ( ) ) {
contextFactor . push_back ( & word ) ;
2017-01-02 21:57:52 +03:00
} else if ( GetNGramOrder ( ) > 0 ) {
2011-09-21 20:06:48 +04:00
// shift
for ( size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder ( ) - 1 ; currNGramOrder + + ) {
contextFactor [ currNGramOrder ] = contextFactor [ currNGramOrder + 1 ] ;
}
contextFactor [ GetNGramOrder ( ) - 1 ] = & word ;
}
}
2011-03-08 02:21:09 +03:00
LMResult LanguageModelImplementation : : GetValueGivenState (
2011-02-24 16:14:42 +03:00
const std : : vector < const Word * > & contextFactor ,
FFState & state ) const
2010-11-17 17:06:21 +03:00
{
2011-02-24 16:14:42 +03:00
return GetValueForgotState ( contextFactor , state ) ;
2010-11-17 17:06:21 +03:00
}
void LanguageModelImplementation : : GetState (
2011-02-24 16:14:42 +03:00
const std : : vector < const Word * > & contextFactor ,
FFState & state ) const
2010-11-17 17:06:21 +03:00
{
2011-02-24 16:14:42 +03:00
GetValueForgotState ( contextFactor , state ) ;
2010-11-17 17:06:21 +03:00
}
2011-02-24 16:14:42 +03:00
2013-05-29 21:16:15 +04:00
// Calculate score of a phrase.
void LanguageModelImplementation : : CalcScore ( const Phrase & phrase , float & fullScore , float & ngramScore , size_t & oovCount ) const
{
2011-10-11 17:50:44 +04:00
fullScore = 0 ;
ngramScore = 0 ;
oovCount = 0 ;
size_t phraseSize = phrase . GetSize ( ) ;
if ( ! phraseSize ) return ;
vector < const Word * > contextFactor ;
contextFactor . reserve ( GetNGramOrder ( ) ) ;
2013-02-21 15:30:54 +04:00
std : : auto_ptr < FFState > state ( NewState ( ( phrase . GetWord ( 0 ) = = GetSentenceStartWord ( ) ) ?
2013-05-29 21:16:15 +04:00
GetBeginSentenceState ( ) : GetNullContextState ( ) ) ) ;
2011-10-11 17:50:44 +04:00
size_t currPos = 0 ;
while ( currPos < phraseSize ) {
const Word & word = phrase . GetWord ( currPos ) ;
if ( word . IsNonTerminal ( ) ) {
// do nothing. reset ngram. needed to score target phrases during pt loading in chart decoding
if ( ! contextFactor . empty ( ) ) {
// TODO: state operator= ?
state . reset ( NewState ( GetNullContextState ( ) ) ) ;
contextFactor . clear ( ) ;
}
} else {
ShiftOrPush ( contextFactor , word ) ;
2013-11-23 00:27:46 +04:00
UTIL_THROW_IF2 ( contextFactor . size ( ) > GetNGramOrder ( ) ,
2014-01-15 19:42:02 +04:00
" Can only calculate LM score of phrases up to the n-gram order " ) ;
2011-10-11 17:50:44 +04:00
2013-02-21 15:30:54 +04:00
if ( word = = GetSentenceStartWord ( ) ) {
2011-10-11 17:50:44 +04:00
// do nothing, don't include prob for <s> unigram
2011-11-01 14:24:40 +04:00
if ( currPos ! = 0 ) {
2014-01-13 18:37:05 +04:00
UTIL_THROW2 ( " Either your data contains <s> in a position other than the first word or your language model is missing <s>. Did you build your ARPA using IRSTLM and forget to run add-start-end.sh? " ) ;
2011-11-01 14:24:40 +04:00
}
2011-10-11 17:50:44 +04:00
} else {
LMResult result = GetValueGivenState ( contextFactor , * state ) ;
fullScore + = result . score ;
if ( contextFactor . size ( ) = = GetNGramOrder ( ) )
ngramScore + = result . score ;
2013-05-29 21:16:15 +04:00
if ( result . unknown ) + + oovCount ;
2011-10-11 17:50:44 +04:00
}
}
currPos + + ;
}
}
2014-07-10 02:41:08 +04:00
FFState * LanguageModelImplementation : : EvaluateWhenApplied ( const Hypothesis & hypo , const FFState * ps , ScoreComponentCollection * out ) const
2013-05-29 21:16:15 +04:00
{
2011-10-11 17:50:44 +04:00
// In this function, we only compute the LM scores of n-grams that overlap a
// phrase boundary. Phrase-internal scores are taken directly from the
// translation option.
// In the case of unigram language models, there is no overlap, so we don't
// need to do anything.
if ( GetNGramOrder ( ) < = 1 )
return NULL ;
// Empty phrase added? nothing to be done
if ( hypo . GetCurrTargetLength ( ) = = 0 )
return ps ? NewState ( ps ) : NULL ;
2014-01-03 22:45:31 +04:00
IFVERBOSE ( 2 ) {
hypo . GetManager ( ) . GetSentenceStats ( ) . StartTimeCalcLM ( ) ;
}
2011-10-11 17:50:44 +04:00
const size_t currEndPos = hypo . GetCurrTargetWordsRange ( ) . GetEndPos ( ) ;
const size_t startPos = hypo . GetCurrTargetWordsRange ( ) . GetStartPos ( ) ;
// 1st n-gram
vector < const Word * > contextFactor ( GetNGramOrder ( ) ) ;
size_t index = 0 ;
for ( int currPos = ( int ) startPos - ( int ) GetNGramOrder ( ) + 1 ; currPos < = ( int ) startPos ; currPos + + ) {
if ( currPos > = 0 )
contextFactor [ index + + ] = & hypo . GetWord ( currPos ) ;
else {
2013-02-21 15:30:54 +04:00
contextFactor [ index + + ] = & GetSentenceStartWord ( ) ;
2011-10-11 17:50:44 +04:00
}
}
FFState * res = NewState ( ps ) ;
float lmScore = ps ? GetValueGivenState ( contextFactor , * res ) . score : GetValueForgotState ( contextFactor , * res ) . score ;
// main loop
size_t endPos = std : : min ( startPos + GetNGramOrder ( ) - 2
, currEndPos ) ;
for ( size_t currPos = startPos + 1 ; currPos < = endPos ; currPos + + ) {
// shift all args down 1 place
for ( size_t i = 0 ; i < GetNGramOrder ( ) - 1 ; i + + )
contextFactor [ i ] = contextFactor [ i + 1 ] ;
// add last factor
contextFactor . back ( ) = & hypo . GetWord ( currPos ) ;
lmScore + = GetValueGivenState ( contextFactor , * res ) . score ;
}
// end of sentence
if ( hypo . IsSourceCompleted ( ) ) {
const size_t size = hypo . GetSize ( ) ;
2013-02-21 15:30:54 +04:00
contextFactor . back ( ) = & GetSentenceEndWord ( ) ;
2011-10-11 17:50:44 +04:00
for ( size_t i = 0 ; i < GetNGramOrder ( ) - 1 ; i + + ) {
int currPos = ( int ) ( size - GetNGramOrder ( ) + i + 1 ) ;
if ( currPos < 0 )
2013-02-21 15:30:54 +04:00
contextFactor [ i ] = & GetSentenceStartWord ( ) ;
2011-10-11 17:50:44 +04:00
else
contextFactor [ i ] = & hypo . GetWord ( ( size_t ) currPos ) ;
}
lmScore + = GetValueForgotState ( contextFactor , * res ) . score ;
2013-05-29 21:16:15 +04:00
} else {
2011-10-11 17:50:44 +04:00
if ( endPos < currEndPos ) {
//need to get the LM state (otherwise the last LM state is fine)
for ( size_t currPos = endPos + 1 ; currPos < = currEndPos ; currPos + + ) {
for ( size_t i = 0 ; i < GetNGramOrder ( ) - 1 ; i + + )
contextFactor [ i ] = contextFactor [ i + 1 ] ;
contextFactor . back ( ) = & hypo . GetWord ( currPos ) ;
}
GetState ( contextFactor , * res ) ;
}
}
2013-02-21 03:36:04 +04:00
if ( OOVFeatureEnabled ( ) ) {
2011-10-11 17:50:44 +04:00
vector < float > scores ( 2 ) ;
scores [ 0 ] = lmScore ;
scores [ 1 ] = 0 ;
2013-02-21 03:36:04 +04:00
out - > PlusEquals ( this , scores ) ;
2011-10-11 17:50:44 +04:00
} else {
2013-02-21 03:36:04 +04:00
out - > PlusEquals ( this , lmScore ) ;
2011-10-11 17:50:44 +04:00
}
IFVERBOSE ( 2 ) {
2014-01-03 22:45:31 +04:00
hypo . GetManager ( ) . GetSentenceStats ( ) . StopTimeCalcLM ( ) ;
2011-10-11 17:50:44 +04:00
}
return res ;
}
2014-07-10 02:54:16 +04:00
FFState * LanguageModelImplementation : : EvaluateWhenApplied ( const ChartHypothesis & hypo , int featureID , ScoreComponentCollection * out ) const
2013-05-29 21:16:15 +04:00
{
2011-10-10 20:25:56 +04:00
LanguageModelChartState * ret = new LanguageModelChartState ( hypo , featureID , GetNGramOrder ( ) ) ;
2011-09-21 20:06:48 +04:00
// data structure for factored context phrase (history and predicted word)
vector < const Word * > contextFactor ;
contextFactor . reserve ( GetNGramOrder ( ) ) ;
// initialize language model context state
FFState * lmState = NewState ( GetNullContextState ( ) ) ;
// initial language model scores
float prefixScore = 0.0 ; // not yet final for initial words (lack context)
float finalizedScore = 0.0 ; // finalized, has sufficient context
// get index map for underlying hypotheses
const AlignmentInfo : : NonTermIndexMap & nonTermIndexMap =
2013-05-29 21:16:15 +04:00
hypo . GetCurrTargetPhrase ( ) . GetAlignNonTerm ( ) . GetNonTermIndexMap ( ) ;
2011-09-21 20:06:48 +04:00
// loop over rule
for ( size_t phrasePos = 0 , wordPos = 0 ;
2012-10-05 22:12:29 +04:00
phrasePos < hypo . GetCurrTargetPhrase ( ) . GetSize ( ) ;
2013-05-29 21:16:15 +04:00
phrasePos + + ) {
2011-09-21 20:06:48 +04:00
// consult rule for either word or non-terminal
2012-10-05 22:12:29 +04:00
const Word & word = hypo . GetCurrTargetPhrase ( ) . GetWord ( phrasePos ) ;
2011-09-21 20:06:48 +04:00
// regular word
2013-05-29 21:16:15 +04:00
if ( ! word . IsNonTerminal ( ) ) {
2011-09-21 20:06:48 +04:00
ShiftOrPush ( contextFactor , word ) ;
// beginning of sentence symbol <s>? -> just update state
2013-05-29 21:16:15 +04:00
if ( word = = GetSentenceStartWord ( ) ) {
2014-01-15 19:42:02 +04:00
UTIL_THROW_IF2 ( phrasePos ! = 0 ,
" Sentence start symbol must be at the beginning of sentence " ) ;
2011-09-21 20:06:48 +04:00
delete lmState ;
lmState = NewState ( GetBeginSentenceState ( ) ) ;
}
// score a regular word added by the rule
2013-05-29 21:16:15 +04:00
else {
2013-04-26 22:39:29 +04:00
updateChartScore ( & prefixScore , & finalizedScore , GetValueGivenState ( contextFactor , * lmState ) . score , + + wordPos ) ;
2011-09-21 20:06:48 +04:00
}
}
// non-terminal, add phrase from underlying hypothesis
2013-05-29 21:16:15 +04:00
else {
2011-09-21 20:06:48 +04:00
// look up underlying hypothesis
size_t nonTermIndex = nonTermIndexMap [ phrasePos ] ;
const ChartHypothesis * prevHypo = hypo . GetPrevHypo ( nonTermIndex ) ;
2011-10-10 20:25:56 +04:00
const LanguageModelChartState * prevState =
static_cast < const LanguageModelChartState * > ( prevHypo - > GetFFState ( featureID ) ) ;
size_t subPhraseLength = prevState - > GetNumTargetTerminals ( ) ;
2011-09-21 20:06:48 +04:00
// special case: rule starts with non-terminal -> copy everything
if ( phrasePos = = 0 ) {
// get prefixScore and finalizedScore
prefixScore = prevState - > GetPrefixScore ( ) ;
2015-01-07 17:25:43 +03:00
finalizedScore = - prefixScore ;
2011-09-21 20:06:48 +04:00
// get language model state
delete lmState ;
lmState = NewState ( prevState - > GetRightContext ( ) ) ;
// push suffix
2011-10-10 20:25:56 +04:00
int suffixPos = prevState - > GetSuffix ( ) . GetSize ( ) - ( GetNGramOrder ( ) - 1 ) ;
2011-09-21 20:06:48 +04:00
if ( suffixPos < 0 ) suffixPos = 0 ; // push all words if less than order
2013-05-29 21:16:15 +04:00
for ( ; ( size_t ) suffixPos < prevState - > GetSuffix ( ) . GetSize ( ) ; suffixPos + + ) {
2011-10-10 20:25:56 +04:00
const Word & word = prevState - > GetSuffix ( ) . GetWord ( suffixPos ) ;
2011-09-21 20:06:48 +04:00
ShiftOrPush ( contextFactor , word ) ;
wordPos + + ;
}
}
// internal non-terminal
2013-05-29 21:16:15 +04:00
else {
2011-09-21 20:06:48 +04:00
// score its prefix
for ( size_t prefixPos = 0 ;
prefixPos < GetNGramOrder ( ) - 1 // up to LM order window
2013-05-29 21:16:15 +04:00
& & prefixPos < subPhraseLength ; // up to length
prefixPos + + ) {
2011-10-10 20:25:56 +04:00
const Word & word = prevState - > GetPrefix ( ) . GetWord ( prefixPos ) ;
2011-09-21 20:06:48 +04:00
ShiftOrPush ( contextFactor , word ) ;
2013-04-26 22:39:29 +04:00
updateChartScore ( & prefixScore , & finalizedScore , GetValueGivenState ( contextFactor , * lmState ) . score , + + wordPos ) ;
2011-09-21 20:06:48 +04:00
}
2015-01-07 17:25:43 +03:00
finalizedScore - = prevState - > GetPrefixScore ( ) ;
2011-09-21 20:06:48 +04:00
// check if we are dealing with a large sub-phrase
2013-05-29 21:16:15 +04:00
if ( subPhraseLength > GetNGramOrder ( ) - 1 ) {
2011-09-21 20:06:48 +04:00
// copy language model state
delete lmState ;
lmState = NewState ( prevState - > GetRightContext ( ) ) ;
// push its suffix
size_t remainingWords = subPhraseLength - ( GetNGramOrder ( ) - 1 ) ;
if ( remainingWords > GetNGramOrder ( ) - 1 ) {
// only what is needed for the history window
remainingWords = GetNGramOrder ( ) - 1 ;
}
2011-10-10 20:25:56 +04:00
for ( size_t suffixPos = prevState - > GetSuffix ( ) . GetSize ( ) - remainingWords ;
suffixPos < prevState - > GetSuffix ( ) . GetSize ( ) ;
2011-09-21 20:06:48 +04:00
suffixPos + + ) {
2011-10-10 20:25:56 +04:00
const Word & word = prevState - > GetSuffix ( ) . GetWord ( suffixPos ) ;
2011-09-21 20:06:48 +04:00
ShiftOrPush ( contextFactor , word ) ;
}
wordPos + = subPhraseLength ;
}
}
}
}
2015-01-07 17:25:43 +03:00
// add combined score to score breakdown
2014-11-13 13:36:26 +03:00
if ( OOVFeatureEnabled ( ) ) {
vector < float > scores ( 2 ) ;
2015-01-07 17:25:43 +03:00
scores [ 0 ] = prefixScore + finalizedScore - hypo . GetTranslationOption ( ) . GetScores ( ) . GetScoresForProducer ( this ) [ 0 ] ;
// scores[1] = out->GetScoresForProducer(this)[1];
scores [ 1 ] = 0 ;
out - > PlusEquals ( this , scores ) ;
2015-01-14 14:07:42 +03:00
} else {
2015-01-07 17:25:43 +03:00
out - > PlusEquals ( this , prefixScore + finalizedScore - hypo . GetTranslationOption ( ) . GetScores ( ) . GetScoresForProducer ( this ) [ 0 ] ) ;
2014-11-13 13:36:26 +03:00
}
2011-09-21 20:06:48 +04:00
2011-10-10 20:25:56 +04:00
ret - > Set ( prefixScore , lmState ) ;
return ret ;
2011-09-21 20:06:48 +04:00
}
2013-05-29 21:16:15 +04:00
void LanguageModelImplementation : : updateChartScore ( float * prefixScore , float * finalizedScore , float score , size_t wordPos ) const
{
2011-09-21 20:06:48 +04:00
if ( wordPos < GetNGramOrder ( ) ) {
* prefixScore + = score ;
2013-05-29 21:16:15 +04:00
} else {
2011-09-21 20:06:48 +04:00
* finalizedScore + = score ;
}
}
2010-11-17 17:06:21 +03:00
}