mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Added CalcScore to BackwardLanguageModel
This commit is contained in:
parent
d340218fd4
commit
95e3a37469
@ -25,6 +25,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "lm/model.hh"
|
||||
|
||||
#include "moses/FFState.h"
|
||||
#include "moses/Phrase.h"
|
||||
|
||||
#include "moses/LM/Ken.h"
|
||||
#include "moses/LM/Backward.h"
|
||||
|
||||
@ -58,6 +60,51 @@ namespace Moses {
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <class Model> void BackwardLanguageModel<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
|
||||
fullScore = 0;
|
||||
ngramScore = 0;
|
||||
oovCount = 0;
|
||||
|
||||
if (!phrase.GetSize()) return;
|
||||
|
||||
lm::ngram::ChartState discarded_sadly;
|
||||
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
|
||||
|
||||
UTIL_THROW_IF(
|
||||
(m_beginSentenceFactor == phrase.GetWord(0).GetFactor(m_factorType)),
|
||||
util::Exception,
|
||||
"BackwardLanguageModel does not currently support rules that include <s>"
|
||||
);
|
||||
|
||||
float before_boundary = 0.0f;
|
||||
for (size_t position = phrase.GetSize() - 1,
|
||||
ngramBoundary = m_ngram->Order() - 1; position >= 0; position-=1) {
|
||||
|
||||
const Word &word = phrase.GetWord(position);
|
||||
|
||||
UTIL_THROW_IF(
|
||||
(word.IsNonTerminal()),
|
||||
util::Exception,
|
||||
"BackwardLanguageModel does not currently support rules that include non-terminals"
|
||||
);
|
||||
|
||||
lm::WordIndex index = TranslateID(word);
|
||||
scorer.Terminal(index);
|
||||
if (!index) ++oovCount;
|
||||
|
||||
if (position==ngramBoundary) {
|
||||
before_boundary = scorer.Finish();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
fullScore += scorer.Finish();
|
||||
|
||||
ngramScore = TransformLMScore(fullScore - before_boundary);
|
||||
fullScore = TransformLMScore(fullScore);
|
||||
|
||||
}
|
||||
|
||||
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy) {
|
||||
try {
|
||||
lm::ngram::ModelType model_type;
|
||||
|
@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
namespace Moses {
|
||||
|
||||
//! This will also load. Returns a templated KenLM class
|
||||
//! This will also load. Returns a templated backward LM.
|
||||
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy);
|
||||
|
||||
/*
|
||||
@ -40,11 +40,16 @@ template <class Model> class BackwardLanguageModel : public LanguageModelKen<Mod
|
||||
|
||||
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
|
||||
|
||||
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
|
||||
|
||||
private:
|
||||
|
||||
// This line is required to make the parent class's protected member visible to this class
|
||||
// These lines are required to make the parent class's protected members visible to this class
|
||||
using LanguageModelKen<Model>::m_ngram;
|
||||
|
||||
using LanguageModelKen<Model>::m_beginSentenceFactor;
|
||||
using LanguageModelKen<Model>::m_factorType;
|
||||
using LanguageModelKen<Model>::TranslateID;
|
||||
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
@ -129,11 +129,6 @@ template <class Model> const FFState *LanguageModelKen<Model>::EmptyHypothesisSt
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <class Model> lm::WordIndex LanguageModelKen<Model>::TranslateID(const Word &word) const {
|
||||
std::size_t factor = word.GetFactor(m_factorType)->GetId();
|
||||
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
|
||||
}
|
||||
|
||||
template <class Model> lm::WordIndex *LanguageModelKen<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
|
||||
lm::WordIndex *index = indices;
|
||||
lm::WordIndex *end = indices + m_ngram->Order() - 1;
|
||||
@ -159,10 +154,10 @@ template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Increme
|
||||
|
||||
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> ©_from) :
|
||||
m_ngram(copy_from.m_ngram),
|
||||
// TODO: don't copy this.
|
||||
m_lmIdLookup(copy_from.m_lmIdLookup),
|
||||
// TODO: don't copy this.
|
||||
m_beginSentenceFactor(copy_from.m_beginSentenceFactor),
|
||||
m_factorType(copy_from.m_factorType),
|
||||
m_beginSentenceFactor(copy_from.m_beginSentenceFactor) {
|
||||
m_lmIdLookup(copy_from.m_lmIdLookup) {
|
||||
}
|
||||
|
||||
template <class Model> void LanguageModelKen<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
|
||||
|
@ -63,19 +63,23 @@ template <class Model> class LanguageModelKen : public LanguageModel {
|
||||
|
||||
boost::shared_ptr<Model> m_ngram;
|
||||
|
||||
const Factor *m_beginSentenceFactor;
|
||||
|
||||
FactorType m_factorType;
|
||||
|
||||
lm::WordIndex TranslateID(const Word &word) const {
|
||||
std::size_t factor = word.GetFactor(m_factorType)->GetId();
|
||||
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
|
||||
}
|
||||
|
||||
private:
|
||||
LanguageModelKen(const LanguageModelKen<Model> ©_from);
|
||||
|
||||
lm::WordIndex TranslateID(const Word &word) const;
|
||||
|
||||
// Convert last words of hypothesis into vocab ids, returning an end pointer.
|
||||
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const;
|
||||
|
||||
std::vector<lm::WordIndex> m_lmIdLookup;
|
||||
|
||||
FactorType m_factorType;
|
||||
|
||||
const Factor *m_beginSentenceFactor;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
Loading…
Reference in New Issue
Block a user