2015-02-25 03:11:31 +03:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <string>
|
|
|
|
#include <limits>
|
2016-02-21 23:44:09 +03:00
|
|
|
#include <set>
|
2015-02-25 03:11:31 +03:00
|
|
|
#include <boost/unordered_map.hpp>
|
|
|
|
#include "StatelessFeatureFunction.h"
|
|
|
|
#include "moses/Factor.h"
|
|
|
|
|
2015-02-27 05:39:43 +03:00
|
|
|
#ifdef WITH_THREADS
|
|
|
|
#include <boost/thread/shared_mutex.hpp>
|
|
|
|
#endif
|
|
|
|
|
2015-02-25 03:11:31 +03:00
|
|
|
namespace Moses
|
|
|
|
{
|
|
|
|
|
|
|
|
class Model1Vocabulary
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
|
2015-05-02 13:45:24 +03:00
|
|
|
#define INVALID_ID std::numeric_limits<unsigned>::max() // UINT_MAX
|
2015-02-25 03:11:31 +03:00
|
|
|
static const std::string GIZANULL;
|
|
|
|
|
|
|
|
Model1Vocabulary();
|
|
|
|
bool Store(const Factor* word, const unsigned id);
|
|
|
|
unsigned StoreIfNew(const Factor* word);
|
|
|
|
unsigned GetWordID(const Factor* word) const;
|
|
|
|
const Factor* GetWord(unsigned id) const;
|
|
|
|
void Load(const std::string& fileName);
|
|
|
|
|
|
|
|
protected:
|
|
|
|
boost::unordered_map<const Factor*, unsigned> m_lookup;
|
|
|
|
std::vector< const Factor* > m_vocab;
|
|
|
|
const Factor* m_NULL;
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
class Model1LexicalTable
|
|
|
|
{
|
|
|
|
public:
|
2015-04-30 08:05:11 +03:00
|
|
|
Model1LexicalTable(float floor=1e-7) : m_floor(floor)
|
2015-02-25 03:11:31 +03:00
|
|
|
{}
|
|
|
|
|
|
|
|
void Load(const std::string& fileName, const Model1Vocabulary& vcbS, const Model1Vocabulary& vcbT);
|
|
|
|
|
|
|
|
// p( wordT | wordS )
|
|
|
|
float GetProbability(const Factor* wordS, const Factor* wordT) const;
|
|
|
|
|
|
|
|
protected:
|
|
|
|
boost::unordered_map< const Factor*, boost::unordered_map< const Factor*, float > > m_ltable;
|
|
|
|
const float m_floor;
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Model1Feature : public StatelessFeatureFunction
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
Model1Feature(const std::string &line);
|
|
|
|
|
|
|
|
bool IsUseable(const FactorMask &mask) const {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetParameter(const std::string& key, const std::string& value);
|
|
|
|
|
|
|
|
void EvaluateInIsolation(const Phrase &source
|
|
|
|
, const TargetPhrase &targetPhrase
|
|
|
|
, ScoreComponentCollection &scoreBreakdown
|
2015-11-04 18:10:45 +03:00
|
|
|
, ScoreComponentCollection &estimatedScores) const
|
2015-02-25 03:11:31 +03:00
|
|
|
{};
|
|
|
|
|
|
|
|
void EvaluateWithSourceContext(const InputType &input
|
|
|
|
, const InputPath &inputPath
|
|
|
|
, const TargetPhrase &targetPhrase
|
|
|
|
, const StackVec *stackVec
|
|
|
|
, ScoreComponentCollection &scoreBreakdown
|
2015-11-04 18:10:45 +03:00
|
|
|
, ScoreComponentCollection *estimatedScores = NULL) const;
|
2015-02-25 03:11:31 +03:00
|
|
|
|
|
|
|
void EvaluateTranslationOptionListWithSourceContext(const InputType &input
|
|
|
|
, const TranslationOptionList &translationOptionList) const
|
|
|
|
{}
|
|
|
|
|
|
|
|
void EvaluateWhenApplied(
|
|
|
|
const Hypothesis& cur_hypo,
|
|
|
|
ScoreComponentCollection* accumulator) const
|
|
|
|
{}
|
|
|
|
|
|
|
|
void EvaluateWhenApplied(
|
|
|
|
const ChartHypothesis& cur_hypo,
|
|
|
|
ScoreComponentCollection* accumulator) const
|
|
|
|
{}
|
|
|
|
|
2015-02-26 02:13:15 +03:00
|
|
|
void CleanUpAfterSentenceProcessing(const InputType& source);
|
|
|
|
|
2015-02-25 03:11:31 +03:00
|
|
|
private:
|
|
|
|
std::string m_fileNameVcbS;
|
|
|
|
std::string m_fileNameVcbT;
|
|
|
|
std::string m_fileNameModel1;
|
|
|
|
Model1LexicalTable m_model1;
|
|
|
|
const Factor* m_emptyWord;
|
2016-02-22 21:28:23 +03:00
|
|
|
bool m_skipTargetPunctuation;
|
2016-02-21 23:44:09 +03:00
|
|
|
std::set<const Factor*> m_punctuation;
|
|
|
|
bool m_is_syntax;
|
2015-02-25 03:11:31 +03:00
|
|
|
|
2015-12-10 06:17:36 +03:00
|
|
|
void Load(AllOptions::ptr const& opts);
|
2015-04-30 08:05:11 +03:00
|
|
|
|
2015-02-26 02:13:15 +03:00
|
|
|
// cache
|
|
|
|
mutable boost::unordered_map<const InputType*, boost::unordered_map<const Factor*, float> > m_cache;
|
2015-05-02 13:45:24 +03:00
|
|
|
#ifdef WITH_THREADS
|
2015-02-26 02:13:15 +03:00
|
|
|
// reader-writer lock
|
|
|
|
mutable boost::shared_mutex m_accessLock;
|
2015-05-02 13:45:24 +03:00
|
|
|
#endif
|
2015-02-25 03:11:31 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|