refactor parsing of feature functiona args

This commit is contained in:
Hieu Hoang 2013-06-10 18:11:55 +01:00
parent 96d116dba0
commit abe6bb7c22
86 changed files with 529 additions and 470 deletions

View File

@ -38,8 +38,8 @@ protected:
std::vector<WordPtr> m_words;
public:
Phrase()
{}
Phrase() {
}
virtual ~Phrase() {}

View File

@ -61,8 +61,8 @@ protected:
UINT64 ReadScoresFromFile(std::fstream &fileTPColl);
public:
TargetPhrase()
{}
TargetPhrase() {
}
TargetPhrase(size_t numScores);
TargetPhrase(const TargetPhrase &copy);
virtual ~TargetPhrase();

View File

@ -42,8 +42,8 @@ protected:
public:
Vocab()
:m_nextId(1)
{}
:m_nextId(1) {
}
UINT64 AddVocabId(const std::string &str);
UINT64 GetVocabId(const std::string &str, bool &found) const;
const std::string &GetString(UINT32 vocabId) const {

View File

@ -46,13 +46,13 @@ private:
UINT64 m_vocabId;
public:
explicit Word()
{}
explicit Word() {
}
explicit Word(bool isNonTerminal)
:m_isNonTerminal(isNonTerminal)
,m_vocabId(0)
{}
,m_vocabId(0) {
}
Word(const Word &copy);
~Word();

View File

@ -38,8 +38,8 @@ public:
,m_start_null(start_null)
,m_end_null(end_null)
,m_pre_null(pre_null)
,m_post_null(post_null)
{}
,m_post_null(post_null) {
}
~PhrasePair () {}
void PrintTarget( std::ostream* out ) const;

View File

@ -20,8 +20,8 @@ class _fdstream
{
protected:
_fdstream() :
_file_descriptor(-1), _filebuf(NULL)
{ }
_file_descriptor(-1), _filebuf(NULL) {
}
_fdstream(int file_descriptor, std::ios_base::openmode openmode) :
_file_descriptor(file_descriptor), _openmode(openmode) {
@ -59,8 +59,8 @@ class ifdstream : public _fdstream
{
public:
ifdstream() :
_fdstream(), _stream(NULL)
{ }
_fdstream(), _stream(NULL) {
}
ifdstream(int file_descriptor) :
_fdstream(file_descriptor, std::ios_base::in) {
@ -116,8 +116,8 @@ class ofdstream : public _fdstream
{
public:
ofdstream() :
_fdstream(), _stream(NULL)
{ }
_fdstream(), _stream(NULL) {
}
ofdstream(int file_descriptor) :
_fdstream(file_descriptor, std::ios_base::out) {

View File

@ -81,8 +81,8 @@ class TranslationTask : public Task
public:
TranslationTask(InputType *source, IOWrapper &ioWrapper)
: m_source(source)
, m_ioWrapper(ioWrapper)
{}
, m_ioWrapper(ioWrapper) {
}
~TranslationTask() {
delete m_source;

View File

@ -52,8 +52,8 @@ public:
Stack stack=Stack())
: m_coverage(coverage)
, m_label(label)
, m_stack(stack)
{}
, m_stack(stack) {
}
const WordsRange &GetCoverage() const {
return m_coverage;

View File

@ -95,8 +95,8 @@ public:
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
* currently an empty function
*/
void CalcDecoderStatistics() const
{ }
void CalcDecoderStatistics() const {
}
void ResetSentenceStats(const InputType& source) {
m_sentenceStats = std::auto_ptr<SentenceStats>(new SentenceStats(source));

View File

@ -49,8 +49,8 @@ public:
: m_stackVec(stackVec)
, m_targetPhraseCollection(&targetPhraseColl)
, m_wordsRange(&wordsRange)
, m_estimateOfBestScore(score)
{}
, m_estimateOfBestScore(score) {
}
~ChartTranslationOptions() {}

View File

@ -40,8 +40,7 @@ DecodeFeature::DecodeFeature( const std::string& description
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
}
else {
} else {
++ind;
}
}
@ -73,12 +72,10 @@ bool DecodeFeature::OverrideParameter(const std::string& key, const std::string&
if (key == "input-factor") {
m_input =Tokenize<FactorType>(value, ",");
m_inputFactors = FactorMask(m_input);
}
else if (key == "output-factor") {
} else if (key == "output-factor") {
m_output =Tokenize<FactorType>(value, ",");
m_outputFactors = FactorMask(m_output);
}
else {
} else {
return StatelessFeatureFunction::OverrideParameter(key, value);
}

View File

@ -47,14 +47,14 @@ public:
**/
DecodeGraph(size_t position)
: m_position(position)
, m_maxChartSpan(NOT_FOUND)
{}
, m_maxChartSpan(NOT_FOUND) {
}
// for chart decoding
DecodeGraph(size_t position, size_t maxChartSpan)
: m_position(position)
, m_maxChartSpan(maxChartSpan)
{}
, m_maxChartSpan(maxChartSpan) {
}
//! iterators
typedef std::list<const DecodeStep*>::iterator iterator;

View File

@ -18,8 +18,8 @@ class DistortionScoreProducer : public StatefulFeatureFunction
{
public:
DistortionScoreProducer(const std::string &line)
: StatefulFeatureFunction("Distortion", 1, line)
{}
: StatefulFeatureFunction("Distortion", 1, line) {
}
bool IsUseable(const FactorMask &mask) const {
return true;

View File

@ -54,8 +54,7 @@ void FeatureFunction::Initialize(const std::string& description, const std::stri
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
}
else {
} else {
++ind;
}
}

View File

@ -55,8 +55,8 @@ public:
virtual ~FeatureFunction();
//! override to load model files
virtual void Load()
{}
virtual void Load() {
}
static void ResetDescriptionCounts() {
description_counts.clear();
@ -80,12 +80,12 @@ public:
}
//! Called before search and collecting of translation options
virtual void InitializeForInput(InputType const& source)
{}
virtual void InitializeForInput(InputType const& source) {
}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing(const InputType& source)
{}
virtual void CleanUpAfterSentenceProcessing(const InputType& source) {
}
const std::string &GetArgLine() const {
return m_argLine;
@ -99,12 +99,12 @@ public:
virtual void Evaluate(const Phrase &source
, const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{}
, ScoreComponentCollection &estimatedFutureScore) const {
}
virtual void Evaluate(const InputType &source
, ScoreComponentCollection &scoreBreakdown) const
{}
, ScoreComponentCollection &scoreBreakdown) const {
}
virtual bool OverrideParameter(const std::string& key, const std::string& value);
};

View File

@ -45,6 +45,11 @@ PhrasePairFeature::PhrasePairFeature(const std::string &line)
Load(filePathSource);
}
bool PhrasePairFeature::OverrideParameter(const std::string& key, const std::string& value)
{
}
bool PhrasePairFeature::Load(const std::string &filePathSource/*, const std::string &filePathTarget*/)
{
if (m_domainTrigger) {

View File

@ -45,6 +45,7 @@ public:
}
bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/);
bool OverrideParameter(const std::string& key, const std::string& value);
};

View File

@ -10,6 +10,7 @@
#include "moses/Util.h"
#include "util/string_piece_hash.hh"
#include "util/exception.hh"
namespace Moses
{
@ -22,37 +23,42 @@ SourceWordDeletionFeature::SourceWordDeletionFeature(const std::string &line)
{
std::cerr << "Initializing source word deletion feature.." << std::endl;
string filename;
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "factor") {
m_factorType = Scan<FactorType>(args[1]);
} else if (args[0] == "path") {
filename = args[1];
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
throw "Unknown argument " + args[0];
++ind;
}
}
// load word list for restricted feature set
if (filename != "") {
cerr << "loading source word deletion word list from " << filename << endl;
if (!Load(filename)) {
UserMessage::Add("Unable to load word list for source word deletion feature from file " + filename);
//return false;
}
}
}
bool SourceWordDeletionFeature::Load(const std::string &filePath)
bool SourceWordDeletionFeature::OverrideParameter(const std::string& key, const std::string& value)
{
ifstream inFile(filePath.c_str());
if (!inFile) {
cerr << "could not open file " << filePath << endl;
return false;
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
} else if (key == "path") {
m_filename = value;
} else {
StatelessFeatureFunction::OverrideParameter(key, value);
}
}
void SourceWordDeletionFeature::Load()
{
if (m_filename == "") {
return;
}
cerr << "loading source word deletion word list from " << m_filename << endl;
ifstream inFile(m_filename.c_str());
UTIL_THROW_IF(!inFile, util::Exception, "Can't open file " << m_filename);
std::string line;
while (getline(inFile, line)) {
m_vocab.insert(line);
@ -61,7 +67,6 @@ bool SourceWordDeletionFeature::Load(const std::string &filePath)
inFile.close();
m_unrestricted = false;
return true;
}
void SourceWordDeletionFeature::Evaluate(const Phrase &source

View File

@ -19,11 +19,12 @@ private:
boost::unordered_set<std::string> m_vocab;
FactorType m_factorType;
bool m_unrestricted;
std::string m_filename;
public:
SourceWordDeletionFeature(const std::string &line);
bool Load(const std::string &filePath);
void Load();
bool IsUseable(const FactorMask &mask) const {
return true;
@ -38,6 +39,8 @@ public:
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
bool OverrideParameter(const std::string& key, const std::string& value);
};
}

View File

@ -24,15 +24,15 @@ public:
* This should be implemented for features that apply to phrase-based models.
**/
virtual void Evaluate(const PhraseBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
/**
* Same for chart-based features.
**/
virtual void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const
{}
ScoreComponentCollection* accumulator) const {
}
virtual bool IsStateless() const {
return true;

View File

@ -4,6 +4,7 @@
#include "moses/Hypothesis.h"
#include "moses/ScoreComponentCollection.h"
#include "util/string_piece_hash.hh"
#include "util/exception.hh"
using namespace std;
@ -21,29 +22,42 @@ TargetBigramFeature::TargetBigramFeature(const std::string &line)
{
std::cerr << "Initializing target bigram feature.." << std::endl;
vector<string> tokens = Tokenize(line);
//CHECK(tokens[0] == m_description);
// set factor
m_factorType = Scan<FactorType>(tokens[1]);
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
++ind;
}
}
FactorCollection& factorCollection = FactorCollection::Instance();
const Factor* bosFactor =
factorCollection.AddFactor(Output,m_factorType,BOS_);
m_bos.SetFactor(m_factorType,bosFactor);
const string &filePath = tokens[2];
Load(filePath);
}
bool TargetBigramFeature::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
} else if (key == "path") {
m_filePath = value;
} else {
StatefulFeatureFunction::OverrideParameter(key, value);
}
}
bool TargetBigramFeature::Load(const std::string &filePath)
void TargetBigramFeature::Load()
{
if (filePath == "*") return true; //allow all
ifstream inFile(filePath.c_str());
if (!inFile) {
return false;
}
if (m_filePath == "*")
return ; //allow all
ifstream inFile(m_filePath.c_str());
UTIL_THROW_IF(!inFile, util::Exception, "Can't open file " << m_filePath);
std::string line;
m_vocab.insert(BOS_);
@ -53,7 +67,6 @@ bool TargetBigramFeature::Load(const std::string &filePath)
}
inFile.close();
return true;
}

View File

@ -33,7 +33,7 @@ class TargetBigramFeature : public StatefulFeatureFunction
public:
TargetBigramFeature(const std::string &line);
bool Load(const std::string &filePath);
void Load();
bool IsUseable(const FactorMask &mask) const;
@ -47,10 +47,12 @@ public:
ScoreComponentCollection* ) const {
abort();
}
bool OverrideParameter(const std::string& key, const std::string& value);
private:
FactorType m_factorType;
Word m_bos;
std::string m_filePath;
boost::unordered_set<std::string> m_vocab;
};

View File

@ -42,13 +42,29 @@ TargetNgramFeature::TargetNgramFeature(const std::string &line)
{
std::cerr << "Initializing target ngram feature.." << std::endl;
vector<string> tokens = Tokenize(line);
//CHECK(tokens[0] == m_description);
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
++ind;
}
}
}
CHECK(tokens.size() == 4);
m_factorType = Scan<FactorType>(tokens[1]);
m_n = Scan<size_t>(tokens[2]);
m_lower_ngrams = Scan<bool>(tokens[3]);
bool TargetNgramFeature::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
} else if (key == "n") {
m_n = Scan<size_t>(value);
} else if (key == "lower-ngrams") {
m_lower_ngrams = Scan<bool>(value);
} else {
StatefulFeatureFunction::OverrideParameter(key, value);
}
}

View File

@ -191,6 +191,7 @@ public:
virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId,
ScoreComponentCollection* accumulator) const;
bool OverrideParameter(const std::string& key, const std::string& value);
private:
FactorType m_factorType;

View File

@ -20,18 +20,28 @@ TargetWordInsertionFeature::TargetWordInsertionFeature(const std::string &line)
m_unrestricted(true)
{
std::cerr << "Initializing target word insertion feature.." << std::endl;
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "factor") {
m_factorType = Scan<FactorType>(args[1]);
} else if (args[0] == "path") {
m_filename = args[1];
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
throw "Unknown argument " + args[0];
++ind;
}
}
}
bool TargetWordInsertionFeature::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
} else if (key == "path") {
m_filename = value;
} else {
StatelessFeatureFunction::OverrideParameter(key, value);
}
}
void TargetWordInsertionFeature::Load()

View File

@ -37,6 +37,7 @@ public:
const TargetPhrase& targetPhrase,
ScoreComponentCollection* accumulator,
const AlignmentInfo &alignmentInfo) const;
bool OverrideParameter(const std::string& key, const std::string& value);
};

View File

@ -27,33 +27,14 @@ WordTranslationFeature::WordTranslationFeature(const std::string &line)
{
std::cerr << "Initializing word translation feature.. " << endl;
string texttype;
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "input-factor") {
m_factorTypeSource = Scan<FactorType>(args[1]);
} else if (args[0] == "output-factor") {
m_factorTypeTarget = Scan<FactorType>(args[1]);
} else if (args[0] == "simple") {
m_simple = Scan<bool>(args[1]);
} else if (args[0] == "source-context") {
m_sourceContext = Scan<bool>(args[1]);
} else if (args[0] == "target-context") {
m_targetContext = Scan<bool>(args[1]);
} else if (args[0] == "ignore-punctuation") {
m_ignorePunctuation = Scan<bool>(args[1]);
} else if (args[0] == "domain-trigger") {
m_domainTrigger = Scan<bool>(args[1]);
} else if (args[0] == "texttype") {
texttype = args[1];
} else if (args[0] == "source-path") {
m_filePathSource = args[1];
} else if (args[0] == "target-path") {
m_filePathTarget = args[1];
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
throw "Unknown argument " + args[0];
++ind;
}
}
@ -89,6 +70,33 @@ WordTranslationFeature::WordTranslationFeature(const std::string &line)
}
bool WordTranslationFeature::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "input-factor") {
m_factorTypeSource = Scan<FactorType>(value);
} else if (key == "output-factor") {
m_factorTypeTarget = Scan<FactorType>(value);
} else if (key == "simple") {
m_simple = Scan<bool>(value);
} else if (key == "source-context") {
m_sourceContext = Scan<bool>(value);
} else if (key == "target-context") {
m_targetContext = Scan<bool>(value);
} else if (key == "ignore-punctuation") {
m_ignorePunctuation = Scan<bool>(value);
} else if (key == "domain-trigger") {
m_domainTrigger = Scan<bool>(value);
} else if (key == "texttype") {
//texttype = value; TODO not used
} else if (key == "source-path") {
m_filePathSource = value;
} else if (key == "target-path") {
m_filePathTarget = value;
} else {
StatelessFeatureFunction::OverrideParameter(key, value);
}
}
void WordTranslationFeature::Load()
{
// load word list for restricted feature set

View File

@ -52,6 +52,7 @@ public:
void EvaluateChart(const ChartBasedFeatureContext& context,
ScoreComponentCollection* accumulator) const;
bool OverrideParameter(const std::string& key, const std::string& value);
};
}

View File

@ -87,8 +87,8 @@ class FactorCollection
//! constructor. only the 1 static variable can be created
FactorCollection()
:m_factorId(0)
{}
:m_factorId(0) {
}
public:
static FactorCollection& Instance() {

View File

@ -62,8 +62,8 @@ protected:
//! Usually <s> and </s>
LanguageModelImplementation(const std::string& description, const std::string &line)
:LanguageModel(description, line)
{}
:LanguageModel(description, line) {
}
public:
virtual ~LanguageModelImplementation() {}

View File

@ -42,8 +42,8 @@ protected:
FactorMask m_factorTypes;
LanguageModelMultiFactor(const std::string& description, const std::string &line)
:LanguageModelImplementation(description, line)
{}
:LanguageModelImplementation(description, line) {
}
public:
virtual bool Load(const std::string &filePath

View File

@ -23,8 +23,8 @@ public:
typedef count_t T; // type for ORLM filter
LanguageModelORLM(const std::string &line)
:LanguageModelSingleFactor("ORLM", line)
,m_lm(0)
{}
,m_lm(0) {
}
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
~LanguageModelORLM() {

View File

@ -70,8 +70,8 @@ private:
public:
LanguageModelParallelBackoff(const std::string &line)
:LanguageModelMultiFactor("ParallelBackoffLM", line)
{}
:LanguageModelMultiFactor("ParallelBackoffLM", line) {
}
~LanguageModelParallelBackoff();

View File

@ -46,8 +46,8 @@ class LanguageModelRandLM : public LanguageModelSingleFactor
public:
LanguageModelRandLM(const std::string &line)
:LanguageModelSingleFactor("RandLM", line)
, m_lm(0)
{}
, m_lm(0) {
}
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
virtual LMResult GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
~LanguageModelRandLM() {

View File

@ -40,8 +40,7 @@ protected:
m_obj(p),
useCache(1),
totalE(0),
distinctE(0)
{
distinctE(0) {
m_numInputScores = 0;
const StaticData &staticData = StaticData::Instance();
m_inputFeature = staticData.GetInputFeature();

View File

@ -43,8 +43,8 @@ public:
TranslationDimension(std::size_t pos,
const std::vector<TargetPhrase*> &orderedTargetPhrases)
: m_pos(pos)
, m_orderedTargetPhrases(&orderedTargetPhrases)
{}
, m_orderedTargetPhrases(&orderedTargetPhrases) {
}
std::size_t IncrementPos() {
return m_pos++;
@ -80,8 +80,8 @@ class HypothesisDimension
public:
HypothesisDimension(std::size_t pos, const HypoList &orderedHypos)
: m_pos(pos)
, m_orderedHypos(&orderedHypos)
{}
, m_orderedHypos(&orderedHypos) {
}
std::size_t IncrementPos() {
return m_pos++;

View File

@ -91,8 +91,8 @@ public:
//! Clone a score collection
ScoreComponentCollection(const ScoreComponentCollection& rhs)
: m_scores(rhs.m_scores)
{}
: m_scores(rhs.m_scores) {
}
ScoreComponentCollection& operator=( const ScoreComponentCollection& rhs ) {
m_scores = rhs.m_scores;

View File

@ -38,8 +38,8 @@ public:
virtual void EvaluateChart(const ChartBasedFeatureContext&, ScoreComponentCollection*) const {}
virtual void Evaluate(const TargetPhrase &targetPhrase
, ScoreComponentCollection &scoreBreakdown
, ScoreComponentCollection &estimatedFutureScore) const
{ }
, ScoreComponentCollection &estimatedFutureScore) const {
}
};
class MockSingleFeature : public MockStatelessFeatureFunction

View File

@ -20,8 +20,8 @@ public:
std::vector<wordID_t> words;
SAPhrase(size_t phraseSize)
:words(phraseSize)
{}
:words(phraseSize) {
}
void SetId(size_t pos, wordID_t id) {
CHECK(pos < words.size());
@ -43,8 +43,8 @@ public:
, m_endTarget(endTarget)
, m_startSource(startSource)
, m_endSource(endSource)
, m_sntIndex(sntIndex)
{}
, m_sntIndex(sntIndex) {
}
size_t GetTargetSize() const {
return m_endTarget - m_startTarget + 1;

View File

@ -83,8 +83,8 @@ public:
}
DottedRuleColl(size_t size)
: m_coll(size)
{}
: m_coll(size) {
}
~DottedRuleColl();

View File

@ -57,26 +57,26 @@ public:
MmapAllocator() throw()
: m_file_ptr(std::tmpfile()), m_file_desc(fileno(m_file_ptr)),
m_page_size(sysconf(_SC_PAGE_SIZE)), m_map_size(0), m_data_ptr(0),
m_data_offset(0), m_fixed(false), m_count(new size_t(0))
{ }
m_data_offset(0), m_fixed(false), m_count(new size_t(0)) {
}
MmapAllocator(std::FILE* f_ptr) throw()
: m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)),
m_page_size(sysconf(_SC_PAGE_SIZE)), m_map_size(0), m_data_ptr(0),
m_data_offset(0), m_fixed(false), m_count(new size_t(0))
{ }
m_data_offset(0), m_fixed(false), m_count(new size_t(0)) {
}
MmapAllocator(std::FILE* f_ptr, size_t data_offset) throw()
: m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)),
m_page_size(sysconf(_SC_PAGE_SIZE)), m_map_size(0), m_data_ptr(0),
m_data_offset(data_offset), m_fixed(true), m_count(new size_t(0))
{ }
m_data_offset(data_offset), m_fixed(true), m_count(new size_t(0)) {
}
MmapAllocator(std::string fileName) throw()
: m_file_ptr(std::fopen(fileName.c_str(), "wb+")), m_file_desc(fileno(m_file_ptr)),
m_page_size(sysconf(_SC_PAGE_SIZE)), m_map_size(0), m_data_ptr(0),
m_data_offset(0), m_fixed(false), m_count(new size_t(0))
{ }
m_data_offset(0), m_fixed(false), m_count(new size_t(0)) {
}
MmapAllocator(const MmapAllocator& c) throw()
: m_file_ptr(c.m_file_ptr), m_file_desc(c.m_file_desc),

View File

@ -75,8 +75,8 @@ public:
typedef CacheMap::const_iterator const_iterator;
TargetPhraseCollectionCache(size_t max = 5000, float tolerance = 0.2)
: m_max(max), m_tolerance(tolerance)
{}
: m_max(max), m_tolerance(tolerance) {
}
iterator Begin() {
return m_phraseCache.begin();

View File

@ -338,7 +338,7 @@ template<typename T>
const void* OnlineRLM<T>::getContext(const wordID_t* ngram, int len)
{
int dummy(0);
float* *addresses = new float*[len]; // only interested in addresses of cache
float**addresses = new float*[len]; // only interested in addresses of cache
CHECK(cache_->getCache2(ngram, len, &addresses[0], &dummy) == len);
// return address of cache node

View File

@ -32,24 +32,18 @@ namespace Moses
PhraseDictionary::PhraseDictionary(const std::string &description, const std::string &line)
:DecodeFeature(description, line)
,m_tableLimit(20) // default
{
m_tableLimit= 20; // TODO default?
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "path") {
m_filePath = args[1];
} else if (args[0] == "table-limit") {
m_tableLimit = Scan<size_t>(args[1]);
} else if (args[0] == "target-path") {
m_targetFile = args[1];
} else if (args[0] == "alignment-path") {
m_alignmentsFile = args[1];
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
//throw "Unknown argument " + args[0];
++ind;
}
}
} // for (size_t i = 0; i < toks.size(); ++i) {
// find out which feature function can be applied in this decode step
const std::vector<FeatureFunction*> &allFeatures = FeatureFunction::GetFeatureFunctions();
@ -72,10 +66,15 @@ GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const
bool PhraseDictionary::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "table-limit") {
if (key == "path") {
m_filePath = value;
} else if (key == "table-limit") {
m_tableLimit = Scan<size_t>(value);
}
else {
} else if (key == "target-path") {
m_targetFile = value;
} else if (key == "alignment-path") {
m_alignmentsFile = value;
} else {
return DecodeFeature::OverrideParameter(key, value);
}

View File

@ -56,8 +56,8 @@ class PhraseDictionary : public DecodeFeature
public:
PhraseDictionary(const std::string &description, const std::string &line);
virtual ~PhraseDictionary()
{}
virtual ~PhraseDictionary() {
}
//! table limit number.
size_t GetTableLimit() const {
@ -70,11 +70,11 @@ public:
virtual const TargetPhraseCollection *GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const;
//! Create entry for translation of source to targetPhrase
virtual void InitializeForInput(InputType const& source)
{}
virtual void InitializeForInput(InputType const& source) {
}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing(const InputType& source)
{}
virtual void CleanUpAfterSentenceProcessing(const InputType& source) {
}
//! Create a sentence-specific manager for SCFG rule lookup.
virtual ChartRuleLookupManager *CreateRuleLookupManager(

View File

@ -12,17 +12,14 @@ PhraseDictionaryDynSuffixArray::PhraseDictionaryDynSuffixArray(const std::string
:PhraseDictionary("PhraseDictionaryDynSuffixArray", line)
,m_biSA(new BilingualDynSuffixArray())
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "source") {
m_source = args[1];
} else if (args[0] == "target") {
m_target = args[1];
} else if (args[0] == "alignment") {
m_alignments = args[1];
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
//throw "Unknown argument " + args[0];
++ind;
}
}
@ -80,4 +77,17 @@ ChartRuleLookupManager *PhraseDictionaryDynSuffixArray::CreateRuleLookupManager(
throw "Chart decoding not supported by PhraseDictionaryDynSuffixArray";
}
bool PhraseDictionaryDynSuffixArray::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "source") {
m_source = value;
} else if (key == "target") {
m_target = value;
} else if (key == "alignment") {
m_alignments = value;
} else {
PhraseDictionary::OverrideParameter(key, value);
}
}
}// end namepsace

View File

@ -25,6 +25,9 @@ public:
void insertSnt(string&, string&, string&);
void deleteSnt(unsigned, unsigned);
ChartRuleLookupManager *CreateRuleLookupManager(const InputType&, const ChartCellCollectionBase&);
bool OverrideParameter(const std::string& key, const std::string& value);
private:
BilingualDynSuffixArray *m_biSA;
std::string m_source, m_target, m_alignments;

View File

@ -38,13 +38,13 @@ class PhraseDictionaryMemory : public RuleTableTrie
protected:
PhraseDictionaryMemory(const std::string &description, const std::string &line)
: RuleTableTrie(description, line)
{}
: RuleTableTrie(description, line) {
}
public:
PhraseDictionaryMemory(const std::string &line)
: RuleTableTrie("PhraseDictionaryMemory", line)
{}
: RuleTableTrie("PhraseDictionaryMemory", line) {
}
const PhraseDictionaryNodeMemory &GetRootNode() const {
return m_collection;

View File

@ -28,22 +28,16 @@ namespace Moses
PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
:PhraseDictionary("PhraseDictionaryMultiModel", line)
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "mode") {
m_mode =args[1];
if (m_mode != "interpolate") {
ostringstream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
++ind;
}
} else if (args[0] == "components") {
m_pdStr = Tokenize(args[1], ",");
m_numModels = m_pdStr.size();
} else if (args[0] == "lambda") {
m_multimodelweights = Tokenize<float>(args[1], ",");
}
} // for
size_t numWeights = m_numScoreComponents;
if (m_mode == "interpolate") {
@ -55,15 +49,16 @@ PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &line)
PhraseDictionaryMultiModel::PhraseDictionaryMultiModel(const std::string &description, const std::string &line)
:PhraseDictionary(description, line)
{
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "components") {
m_pdStr = Tokenize(args[1], ",");
m_numModels = m_pdStr.size();
} else if (args[0] == "lambda") {
m_multimodelweights = Tokenize<float>(args[1], ",");
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
++ind;
}
}
} // for
if (description == "PhraseDictionaryMultiModelCounts") {
CHECK(m_pdStr.size() == m_multimodelweights.size() || m_pdStr.size()*4 == m_multimodelweights.size());
@ -329,6 +324,24 @@ void PhraseDictionaryMultiModel::CleanUpComponentModels(const InputType &source
}
}
bool PhraseDictionaryMultiModel::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "mode") {
m_mode = value;
if (m_mode != "interpolate") {
ostringstream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
}
} else if (key == "components") {
m_pdStr = Tokenize(value, ",");
m_numModels = m_pdStr.size();
} else if (key == "lambda") {
m_multimodelweights = Tokenize<float>(value, ",");
} else {
PhraseDictionary::OverrideParameter(key, value);
}
}
#ifdef WITH_DLIB
vector<float> PhraseDictionaryMultiModel::MinimizePerplexity(vector<pair<string, string> > &phrase_pair_vector)

View File

@ -80,6 +80,7 @@ public:
/* Don't do anything source specific here as this object is shared between threads.*/
}
ChartRuleLookupManager *CreateRuleLookupManager(const InputType&, const ChartCellCollectionBase&);
bool OverrideParameter(const std::string& key, const std::string& value);
protected:
std::string m_mode;

View File

@ -68,68 +68,15 @@ PhraseDictionaryMultiModelCounts::PhraseDictionaryMultiModelCounts(const std::st
//m_mode = "interpolate";
//m_combineFunction = LinearInterpolationFromCounts;
for (size_t i = 0; i < m_args.size(); ++i) {
const vector<string> &args = m_args[i];
if (args[0] == "mode") {
m_mode = args[1];
if (m_mode == "instance_weighting")
m_combineFunction = InstanceWeighting;
else if (m_mode == "interpolate") {
m_combineFunction = LinearInterpolationFromCounts;
size_t ind = 0;
while (ind < m_args.size()) {
vector<string> &args = m_args[ind];
bool consumed = OverrideParameter(args[0], args[1]);
if (consumed) {
m_args.erase(m_args.begin() + ind);
} else {
ostringstream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
++ind;
}
} else if (args[0] == "lex-e2f") {
m_lexE2FStr = Tokenize(args[1], ",");
CHECK(m_lexE2FStr.size() == m_pdStr.size());
} else if (args[0] == "lex-f2e") {
m_lexF2EStr = Tokenize(args[1], ",");
CHECK(m_lexF2EStr.size() == m_pdStr.size());
}
else if (args[0] == "target-table") {
m_targetTable = Tokenize(args[1], ",");
CHECK(m_targetTable.size() == m_pdStr.size());
}
} // for
}
bool PhraseDictionaryMultiModelCounts::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "mode") {
m_mode = value;
if (m_mode == "instance_weighting")
m_combineFunction = InstanceWeighting;
else if (m_mode == "interpolate") {
m_combineFunction = LinearInterpolationFromCounts;
} else {
ostringstream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
}
} else if (key == "lex-e2f") {
m_lexE2FStr = Tokenize(value, ",");
CHECK(m_lexE2FStr.size() == m_pdStr.size());
} else if (key == "lex-f2e") {
m_lexF2EStr = Tokenize(value, ",");
CHECK(m_lexF2EStr.size() == m_pdStr.size());
}
else if (key == "target-table") {
m_targetTable = Tokenize(value, ",");
CHECK(m_targetTable.size() == m_pdStr.size());
}
else {
PhraseDictionaryMultiModel::OverrideParameter(key, value);
}
}
@ -742,5 +689,36 @@ double LinearInterpolationFromCounts(vector<float> &joint_counts, vector<float>
return p_weighted;
}
bool PhraseDictionaryMultiModelCounts::OverrideParameter(const std::string& key, const std::string& value)
{
if (key == "mode") {
m_mode = value;
if (m_mode == "instance_weighting")
m_combineFunction = InstanceWeighting;
else if (m_mode == "interpolate") {
m_combineFunction = LinearInterpolationFromCounts;
} else {
ostringstream msg;
msg << "combination mode unknown: " << m_mode;
throw runtime_error(msg.str());
}
} else if (key == "lex-e2f") {
m_lexE2FStr = Tokenize(value, ",");
CHECK(m_lexE2FStr.size() == m_pdStr.size());
} else if (key == "lex-f2e") {
m_lexF2EStr = Tokenize(value, ",");
CHECK(m_lexF2EStr.size() == m_pdStr.size());
}
else if (key == "target-table") {
m_targetTable = Tokenize(value, ",");
CHECK(m_targetTable.size() == m_pdStr.size());
}
else {
PhraseDictionaryMultiModel::OverrideParameter(key, value);
}
}
} //namespace

View File

@ -126,8 +126,8 @@ protected:
TargetPhraseCollection *m_targetPhraseCollection;
PhraseDictionaryNodeMemory()
:m_targetPhraseCollection(NULL)
{}
:m_targetPhraseCollection(NULL) {
}
public:
virtual ~PhraseDictionaryNodeMemory();

View File

@ -41,8 +41,8 @@ public:
TgtCand(const IPhrase& a, const Scores& b , const std::string& alignment)
: e(a)
, sc(b)
, m_alignment(alignment)
{}
, m_alignment(alignment) {
}
TgtCand(const IPhrase& a,const Scores& b) : e(a),sc(b) {}

View File

@ -87,10 +87,5 @@ void PhraseDictionaryOnDisk::InitializeForInput(InputType const& source)
return;
}
void PhraseDictionaryOnDisk::CleanUpAfterSentenceProcessing(InputType const& source)
{
}
}

View File

@ -49,8 +49,8 @@ protected:
public:
PhraseDictionaryOnDisk(const std::string &line)
: MyBase("PhraseDictionaryOnDisk", line)
{}
: MyBase("PhraseDictionaryOnDisk", line) {
}
virtual ~PhraseDictionaryOnDisk();
@ -67,7 +67,6 @@ public:
const ChartCellCollectionBase &);
virtual void InitializeForInput(InputType const& source);
virtual void CleanUpAfterSentenceProcessing(InputType const& source);
};

View File

@ -41,8 +41,8 @@ class RuleTableTrie : public PhraseDictionary
{
public:
RuleTableTrie(const std::string &description, const std::string &line)
: PhraseDictionary(description, line)
{}
: PhraseDictionary(description, line) {
}
virtual ~RuleTableTrie();

View File

@ -45,8 +45,8 @@ class RuleTableUTrie : public RuleTableTrie
{
public:
RuleTableUTrie(const std::string &line)
: RuleTableTrie("RuleTableUTrie", line)
{}
: RuleTableTrie("RuleTableUTrie", line) {
}
const UTrieNode &GetRootNode() const {
return m_root;

View File

@ -25,8 +25,8 @@ public:
int max_cost;
int internal_cost;
Match( int is, int ie, int ts, int te, int min, int max, int i )
:input_start(is), input_end(ie), tm_start(ts), tm_end(te), min_cost(min), max_cost(max), internal_cost(i)
{}
:input_start(is), input_end(ie), tm_start(ts), tm_end(te), min_cost(min), max_cost(max), internal_cost(i) {
}
};
}

View File

@ -21,8 +21,8 @@ struct SentenceAlignment {
std::vector< WORD_ID > target;
std::vector< std::pair<int,int> > alignment;
SentenceAlignment()
{}
SentenceAlignment() {
}
std::string getTargetString(const Vocabulary &vocab) const;

View File

@ -17,8 +17,8 @@ public:
XMLParseOutput(const std::string &label, const WordsRange &range)
: m_label(label)
, m_range(range)
{}
, m_range(range) {
}
};
/** An input to the decoder that represent a parse tree.
@ -45,8 +45,8 @@ protected:
bool ProcessAndStripXMLTags(std::string &line, std::vector<XMLParseOutput> &sourceLabels, std::vector<XmlOption*> &res);
public:
TreeInput()
{}
TreeInput() {
}
InputTypeEnum GetType() const {
return TreeInputType;

View File

@ -45,8 +45,8 @@ public:
inline WordsRange(size_t startPos, size_t endPos) : m_startPos(startPos), m_endPos(endPos) {}
inline WordsRange(const WordsRange &copy)
: m_startPos(copy.GetStartPos())
, m_endPos(copy.GetEndPos())
{}
, m_endPos(copy.GetEndPos()) {
}
inline size_t GetStartPos() const {
return m_startPos;

View File

@ -20,8 +20,8 @@ struct XmlOption {
TargetPhrase targetPhrase;
XmlOption(const WordsRange &r, const TargetPhrase &tp)
: range(r), targetPhrase(tp)
{}
: range(r), targetPhrase(tp) {
}
};

View File

@ -42,8 +42,8 @@ public:
return m_elements.end();
}
AlignmentElement()
{}
AlignmentElement() {
}
size_t GetSize() const {
return m_elements.size();
@ -58,8 +58,8 @@ protected:
std::vector<AlignmentElement> m_elements;
public:
AlignmentPhrase(size_t size)
:m_elements(size)
{}
:m_elements(size) {
}
void Merge(const AlignmentPhrase &newAlignment, const WordsRange &newAlignmentRange);
void Merge(const std::vector< std::vector<size_t> > &source);
size_t GetSize() const {

View File

@ -62,8 +62,8 @@ public:
, startS(sS)
, endS(eS)
, count(0)
, pcfgScore(0.0)
{}
, pcfgScore(0.0) {
}
void SetSpanLength(size_t sourcePos, size_t sourceLength, size_t targetLength) {
m_ntLengths[sourcePos] = std::pair<size_t, size_t>(sourceLength, targetLength);

View File

@ -40,15 +40,15 @@ public:
: m_start(2)
, m_end(2)
, m_pos(2)
, m_label(2)
{}
, m_label(2) {
}
Hole(const Hole &copy)
: m_start(copy.m_start)
, m_end(copy.m_end)
, m_pos(copy.m_pos)
, m_label(copy.m_label)
{}
, m_label(copy.m_label) {
}
Hole(int startS, int endS, int startT, int endT)
: m_start(2)

View File

@ -44,8 +44,8 @@ public:
HoleCollection(int sourcePhraseStart, int sourcePhraseEnd)
: m_scope(1, 0)
, m_sourcePhraseStart(1, sourcePhraseStart)
, m_sourcePhraseEnd(1, sourcePhraseEnd)
{}
, m_sourcePhraseEnd(1, sourcePhraseEnd) {
}
const HoleList &GetHoles() const {
return m_holes;

View File

@ -86,8 +86,8 @@ public:
, gzOutput(false)
, unpairedExtractFormat(false)
, conditionOnTargetLhs(false)
, boundaryRules(false)
{}
, boundaryRules(false) {
}
};
}

View File

@ -70,8 +70,8 @@ struct ScoreFeatureContext {
) :
phrasePair(thePhrasePair),
count(theCount),
maybeLog(theMaybeLog)
{}
maybeLog(theMaybeLog) {
}
const PhraseAlignmentCollection& phrasePair;
float count;

View File

@ -53,8 +53,8 @@ public:
, m_sourceLabelCollection(srcLabelColl)
, m_targetTopLabelCollection(tgtTopLabelColl)
, m_sourceTopLabelCollection(srcTopLabelColl)
, m_options(options)
{}
, m_options(options) {
}
virtual ~SentenceAlignmentWithSyntax() {}

View File

@ -44,8 +44,8 @@ public:
,m_end(endPos)
,m_label(label)
,m_parent(0)
,m_pcfgScore(0.0f)
{}
,m_pcfgScore(0.0f) {
}
int GetStart() const {
return m_start;
}

View File

@ -29,8 +29,8 @@ class XmlException
{
public:
XmlException(const std::string & msg)
: m_msg(msg)
{}
: m_msg(msg) {
}
const std::string &
getMsg() const {

View File

@ -71,14 +71,14 @@ public:
std::map<const std::string*, WordCount> m_coll;
WordCount()
:m_count(0)
{}
:m_count(0) {
}
//WordCount(const WordCount &copy);
WordCount(float count)
:m_count(count)
{}
:m_count(count) {
}
void AddCount(float incr);

View File

@ -5,7 +5,7 @@ use File::Basename;
sub Beautify($);
Beautify("/home/hieu/workspace/github/mosesdecoder");
Beautify("/Users/hieuhoang/unison/workspace/github/mosesdecoder");
sub Beautify($)
{

View File

@ -33,12 +33,12 @@ extern "C" {
#endif
#if defined(__STDC__)
int DeclareParams(char *, ...);
int DeclareParams(char *, ...);
#else
int DeclareParams();
int DeclareParams();
#endif
int GetParams(int *n, char ***a,char *CmdFileName),
int GetParams(int *n, char ***a,char *CmdFileName),
SPrintParams(),
PrintParams();