moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Added CalcScore to BackwardLanguageModel

Browse Source
This commit is contained in:
Lane Schwartz 2013-04-08 16:54:11 -04:00
parent d340218fd4
commit 95e3a37469
4 changed files with 67 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
moses/LM/Backward.cpp
View File

@ -25,6 +25,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "lm/model.hh"
#include "moses/FFState.h"
#include "moses/Phrase.h"
#include "moses/LM/Ken.h"
#include "moses/LM/Backward.h"
@ -58,6 +60,51 @@ namespace Moses {
return ret;
}
template <class Model> void BackwardLanguageModel<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
fullScore = 0;
ngramScore = 0;
oovCount = 0;
if (!phrase.GetSize()) return;
lm::ngram::ChartState discarded_sadly;
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
UTIL_THROW_IF(
(m_beginSentenceFactor == phrase.GetWord(0).GetFactor(m_factorType)),
util::Exception,
"BackwardLanguageModel does not currently support rules that include <s>"
);
float before_boundary = 0.0f;
for (size_t position = phrase.GetSize() - 1,
ngramBoundary = m_ngram->Order() - 1; position >= 0; position-=1) {
const Word &word = phrase.GetWord(position);
UTIL_THROW_IF(
(word.IsNonTerminal()),
util::Exception,
"BackwardLanguageModel does not currently support rules that include non-terminals"
);
lm::WordIndex index = TranslateID(word);
scorer.Terminal(index);
if (!index) ++oovCount;
if (position==ngramBoundary) {
before_boundary = scorer.Finish();
}
}
fullScore += scorer.Finish();
ngramScore = TransformLMScore(fullScore - before_boundary);
fullScore = TransformLMScore(fullScore);
}
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy) {
try {
lm::ngram::ModelType model_type;

11
moses/LM/Backward.h
View File

@ -28,7 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
namespace Moses {
//! This will also load. Returns a templated KenLM class
//! This will also load. Returns a templated backward LM.
LanguageModel *ConstructBackwardLM(const std::string &file, FactorType factorType, bool lazy);
/*
@ -40,11 +40,16 @@ template <class Model> class BackwardLanguageModel : public LanguageModelKen<Mod
virtual const FFState *EmptyHypothesisState(const InputType &/*input*/) const;
virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const;
private:
// This line is required to make the parent class's protected member visible to this class
// These lines are required to make the parent class's protected members visible to this class
using LanguageModelKen<Model>::m_ngram;
using LanguageModelKen<Model>::m_beginSentenceFactor;
using LanguageModelKen<Model>::m_factorType;
using LanguageModelKen<Model>::TranslateID;
};
} // namespace Moses

11
moses/LM/Ken.cpp
View File

@ -129,11 +129,6 @@ template <class Model> const FFState *LanguageModelKen<Model>::EmptyHypothesisSt
return ret;
}
template <class Model> lm::WordIndex LanguageModelKen<Model>::TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
template <class Model> lm::WordIndex *LanguageModelKen<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
@ -159,10 +154,10 @@ template <class Model> void LanguageModelKen<Model>::IncrementalCallback(Increme
template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> &copy_from) :
m_ngram(copy_from.m_ngram),
// TODO: don't copy this.
m_lmIdLookup(copy_from.m_lmIdLookup),
// TODO: don't copy this.
m_beginSentenceFactor(copy_from.m_beginSentenceFactor),
m_factorType(copy_from.m_factorType),
m_beginSentenceFactor(copy_from.m_beginSentenceFactor) {
m_lmIdLookup(copy_from.m_lmIdLookup) {
}
template <class Model> void LanguageModelKen<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {

14
moses/LM/Ken.h
View File

@ -63,19 +63,23 @@ template <class Model> class LanguageModelKen : public LanguageModel {
boost::shared_ptr<Model> m_ngram;
const Factor *m_beginSentenceFactor;
FactorType m_factorType;
lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word.GetFactor(m_factorType)->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
private:
LanguageModelKen(const LanguageModelKen<Model> &copy_from);
lm::WordIndex TranslateID(const Word &word) const;
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const;
std::vector<lm::WordIndex> m_lmIdLookup;
FactorType m_factorType;
const Factor *m_beginSentenceFactor;
};
} // namespace Moses