mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
lattice decoding with sparse features
This commit is contained in:
parent
65e2806b1d
commit
1adf7d0521
@ -47,7 +47,7 @@ void InputFeature::Evaluate(const InputType &input
|
||||
return;
|
||||
}
|
||||
|
||||
const ScoreComponentCollection *scores = inputPath.GetInputScore();
|
||||
const ScorePair *scores = inputPath.GetInputScore();
|
||||
if (scores) {
|
||||
|
||||
}
|
||||
|
@ -11,7 +11,7 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
InputPath::InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, const WordsRange &range, const InputPath *prevNode
|
||||
,const ScoreComponentCollection *inputScore)
|
||||
,const ScorePair *inputScore)
|
||||
:m_prevNode(prevNode)
|
||||
,m_phrase(phrase)
|
||||
,m_sourceNonTerms(sourceNonTerms)
|
||||
|
@ -13,9 +13,10 @@ namespace Moses
|
||||
class PhraseDictionary;
|
||||
class TargetPhraseCollection;
|
||||
class ScoreComponentCollection;
|
||||
class ScorePair;
|
||||
class TargetPhrase;
|
||||
|
||||
class InputPath;
|
||||
|
||||
typedef std::list<InputPath*> InputPathList;
|
||||
|
||||
/** Each node contains
|
||||
@ -32,7 +33,7 @@ protected:
|
||||
const InputPath *m_prevNode;
|
||||
Phrase m_phrase;
|
||||
WordsRange m_range;
|
||||
const ScoreComponentCollection *m_inputScore;
|
||||
const ScorePair *m_inputScore;
|
||||
std::map<const PhraseDictionary*, std::pair<const TargetPhraseCollection*, const void*> > m_targetPhrases;
|
||||
const NonTerminalSet m_sourceNonTerms;
|
||||
|
||||
@ -47,7 +48,7 @@ public:
|
||||
}
|
||||
|
||||
InputPath(const Phrase &phrase, const NonTerminalSet &sourceNonTerms, const WordsRange &range, const InputPath *prevNode
|
||||
,const ScoreComponentCollection *inputScore);
|
||||
,const ScorePair *inputScore);
|
||||
~InputPath();
|
||||
|
||||
const Phrase &GetPhrase() const {
|
||||
@ -72,7 +73,7 @@ public:
|
||||
|
||||
// pointer to internal node in phrase-table. Since this is implementation dependent, this is a void*
|
||||
const void *GetPtNode(const PhraseDictionary &phraseDictionary) const;
|
||||
const ScoreComponentCollection *GetInputScore() const {
|
||||
const ScorePair *GetInputScore() const {
|
||||
return m_inputScore;
|
||||
}
|
||||
|
||||
|
@ -220,6 +220,18 @@ FVector ScoreComponentCollection::GetVectorForProducer(const FeatureFunction* sp
|
||||
return fv;
|
||||
}
|
||||
|
||||
void ScoreComponentCollection::PlusEquals(const FeatureFunction* sp, const ScorePair &scorePair)
|
||||
{
|
||||
PlusEquals(sp, scorePair.denseScores);
|
||||
|
||||
std::map<std::string, float>::const_iterator iter;
|
||||
for (iter = scorePair.sparseScores.begin(); iter != scorePair.sparseScores.end(); ++iter) {
|
||||
const string &key = iter->first;
|
||||
float value = iter->second;
|
||||
PlusEquals(sp, key, value);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -41,6 +41,24 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
/**
|
||||
* Smaller version for just 1 FF.
|
||||
*/
|
||||
struct ScorePair
|
||||
{
|
||||
std::vector<float> denseScores;
|
||||
std::map<std::string, float> sparseScores;
|
||||
|
||||
void PlusEquals(const std::vector<float> &other)
|
||||
{
|
||||
CHECK(denseScores.size() == other.size());
|
||||
std::transform(denseScores.begin(),
|
||||
denseScores.end(),
|
||||
other.begin(),
|
||||
denseScores.begin(),
|
||||
std::plus<float>());
|
||||
}
|
||||
};
|
||||
|
||||
/*** An unweighted collection of scores for a translation or step in a translation.
|
||||
*
|
||||
@ -227,6 +245,8 @@ public:
|
||||
m_scores[fname] += score;
|
||||
}
|
||||
|
||||
void PlusEquals(const FeatureFunction* sp, const ScorePair &scorePair);
|
||||
|
||||
//For features which have an unbounded number of components
|
||||
void SparsePlusEquals(const std::string& full_name, float score) {
|
||||
FName fname(full_name);
|
||||
|
@ -37,6 +37,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "InputPath.h"
|
||||
#include "moses/FF/UnknownWordPenaltyProducer.h"
|
||||
#include "moses/FF/LexicalReordering/LexicalReordering.h"
|
||||
#include "moses/FF/InputFeature.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -514,17 +515,19 @@ void TranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
|
||||
void TranslationOptionCollection::SetInputScore(const InputPath &inputPath, PartialTranslOptColl &oldPtoc)
|
||||
{
|
||||
const ScoreComponentCollection *inputScore = inputPath.GetInputScore();
|
||||
const ScorePair *inputScore = inputPath.GetInputScore();
|
||||
if (inputScore == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
const InputFeature *inputFeature = StaticData::Instance().GetInputFeature();
|
||||
|
||||
const std::vector<TranslationOption*> &transOpts = oldPtoc.GetList();
|
||||
for (size_t i = 0; i < transOpts.size(); ++i) {
|
||||
TranslationOption &transOpt = *transOpts[i];
|
||||
|
||||
ScoreComponentCollection &scores = transOpt.GetScoreBreakdown();
|
||||
scores.PlusEquals(*inputScore);
|
||||
scores.PlusEquals(inputFeature, *inputScore);
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -43,8 +43,8 @@ TranslationOptionCollectionConfusionNet::TranslationOptionCollectionConfusionNet
|
||||
subphrase.AddWord(word);
|
||||
|
||||
const std::vector<float> &scores = col[i].second;
|
||||
ScoreComponentCollection *inputScore = new ScoreComponentCollection();
|
||||
inputScore->Assign(inputFeature, scores);
|
||||
ScorePair *inputScore = new ScorePair();
|
||||
inputScore->denseScores = scores;
|
||||
|
||||
InputPath *node = new InputPath(subphrase, labels, range, NULL, inputScore);
|
||||
list.push_back(node);
|
||||
@ -76,7 +76,7 @@ TranslationOptionCollectionConfusionNet::TranslationOptionCollectionConfusionNet
|
||||
//const InputPath &prevNode = *prevNodes[pathInd];
|
||||
|
||||
const Phrase &prevPhrase = prevNode.GetPhrase();
|
||||
const ScoreComponentCollection *prevInputScore = prevNode.GetInputScore();
|
||||
const ScorePair *prevInputScore = prevNode.GetInputScore();
|
||||
CHECK(prevInputScore);
|
||||
|
||||
// loop thru every word at this position
|
||||
@ -88,8 +88,8 @@ TranslationOptionCollectionConfusionNet::TranslationOptionCollectionConfusionNet
|
||||
subphrase.AddWord(word);
|
||||
|
||||
const std::vector<float> &scores = col[i].second;
|
||||
ScoreComponentCollection *inputScore = new ScoreComponentCollection(*prevInputScore);
|
||||
inputScore->PlusEquals(inputFeature, scores);
|
||||
ScorePair *inputScore = new ScorePair(*prevInputScore);
|
||||
inputScore->PlusEquals(scores);
|
||||
|
||||
InputPath *node = new InputPath(subphrase, labels, range, &prevNode, inputScore);
|
||||
list.push_back(node);
|
||||
|
Loading…
Reference in New Issue
Block a user