moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-06 00:08:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

import classes into moses2

Browse Source
This commit is contained in:
Hieu Hoang 2015-11-13 10:40:55 +00:00
parent 8d7e273057
commit e58e2fd777
17 changed files with 54 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
contrib/other-builds/moses2/.cproject
View File

@ -31,6 +31,7 @@
<option id="gnu.cpp.compiler.option.debugging.level.769854045" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
<option id="gnu.cpp.compiler.option.include.paths.1502531988" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../&quot;"/>
</option>
<option id="gnu.cpp.compiler.option.preprocessor.def.1025143565" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
<listOptionValue builtIn="false" value="KENLM_MAX_ORDER=7"/>

3
contrib/other-builds/moses2/FF/FeatureFunction.cpp
View File

@ -9,6 +9,7 @@
#include "FeatureFunction.h"
#include "../System.h"
#include "../legacy/Util2.h"
#include "util/exception.hh"
using namespace std;
@ -54,7 +55,7 @@ void FeatureFunction::ParseLine(const std::string &line)
// name
if (m_name == "") {
m_name = Moses::SPrint(nameStub) + Moses::SPrint(0);
m_name = SPrint(nameStub) + SPrint(0);
}
}

3
contrib/other-builds/moses2/FF/FeatureFunctions.cpp
View File

@ -21,6 +21,7 @@
#include "../TranslationModel/UnknownWordPenalty.h"
#include "../LM/LanguageModel.h"
#include "../LM/KENLM.h"
#include "util/exception.hh"
using namespace std;
@ -33,7 +34,7 @@ FeatureFunctions::FeatureFunctions(System &system)
}
FeatureFunctions::~FeatureFunctions() {
Moses::RemoveAllInColl(m_featureFunctions);
RemoveAllInColl(m_featureFunctions);
}
void FeatureFunctions::Create()

2
contrib/other-builds/moses2/FF/FeatureFunctions.h
View File

@ -10,7 +10,7 @@
#include <vector>
#include <string>
#include "moses/Parameter.h"
#include "../legacy/Parameter.h"
class System;
class FeatureFunction;

8
contrib/other-builds/moses2/LM/KENLM.cpp
View File

@ -14,7 +14,7 @@
#include "../Search/Manager.h"
#include "lm/state.hh"
#include "lm/left.hh"
#include "moses/FactorCollection.h"
#include "../legacy/FactorCollection.h"
using namespace std;
@ -196,7 +196,7 @@ void KENLM::EvaluateWhenApplied(const Manager &mgr,
stateCast.state = *state0;
}
score = Moses::TransformLMScore(score);
score = TransformLMScore(score);
bool OOVFeatureEnabled = false;
if (OOVFeatureEnabled) {
@ -246,8 +246,8 @@ void KENLM::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore,
}
fullScore += scorer.Finish();
ngramScore = Moses::TransformLMScore(fullScore - before_boundary);
fullScore = Moses::TransformLMScore(fullScore);
ngramScore = TransformLMScore(fullScore - before_boundary);
fullScore = TransformLMScore(fullScore);
}
lm::WordIndex KENLM::TranslateID(const Word &word) const {

4
contrib/other-builds/moses2/LM/KENLM.h
View File

@ -12,7 +12,7 @@
#include "../FF/StatefulFeatureFunction.h"
#include "lm/model.hh"
#include "../legacy/Factor.h"
#include "moses/TypeDef.h"
#include "../legacy/Util2.h"
class Word;
@ -45,7 +45,7 @@ public:
protected:
std::string m_path;
Moses::FactorType m_factorType;
FactorType m_factorType;
bool m_lazy;
const Factor *m_bos;
const Factor *m_eos;

4
contrib/other-builds/moses2/LM/LanguageModel.cpp
View File

@ -83,7 +83,7 @@ void LanguageModel::Load(System &system)
assert(substrings.size() == 2 || substrings.size() == 3);
SCORE prob = Moses::TransformLMScore(Scan<SCORE>(substrings[0]));
SCORE prob = TransformLMScore(Scan<SCORE>(substrings[0]));
if (substrings[1] == "<unk>") {
m_oov = prob;
continue;
@ -91,7 +91,7 @@ void LanguageModel::Load(System &system)
SCORE backoff = 0.f;
if (substrings.size() == 3) {
backoff = Moses::TransformLMScore(Scan<SCORE>(substrings[2]));
backoff = TransformLMScore(Scan<SCORE>(substrings[2]));
}
// ngram

4
contrib/other-builds/moses2/LM/LanguageModel.h
View File

@ -12,7 +12,7 @@
#include "../TypeDef.h"
#include "../MorphoTrie/MorphTrie.h"
#include "../legacy/Factor.h"
#include "moses/TypeDef.h"
#include "../legacy/Util2.h"
////////////////////////////////////////////////////////////////////////////////////////
struct LMScores
@ -67,7 +67,7 @@ public:
protected:
std::string m_path;
Moses::FactorType m_factorType;
FactorType m_factorType;
size_t m_order;
MorphTrie<const Factor*, LMScores> m_root;

4
contrib/other-builds/moses2/MorphoTrie/utils.h
View File

@ -4,8 +4,8 @@
#include <string>
#include <vector>
#include "legacy/Util2.h"
#include "moses/Factor.h"
#include "moses/InputFileStream.h"
#include "../legacy/Factor.h"
#include "../legacy/InputFileStream.h"
using namespace std;

2
contrib/other-builds/moses2/Phrase.h
View File

@ -12,7 +12,7 @@
#include <iostream>
#include "Word.h"
#include "MemPool.h"
#include "moses/FactorCollection.h"
#include "legacy/FactorCollection.h"
class SubPhrase;

2
contrib/other-builds/moses2/Scores.cpp
View File

@ -148,7 +148,7 @@ void Scores::CreateFromString(const std::string &str,
{
vector<SCORE> scores = Tokenize<SCORE>(str);
if (transformScores) {
std::transform(scores.begin(), scores.end(), scores.begin(), Moses::TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(), TransformScore);
}
/*

2
contrib/other-builds/moses2/Search/Stack.h
View File

@ -28,7 +28,7 @@ public:
class Stack {
protected:
typedef boost::unordered_set<const Hypothesis*, Moses::UnorderedComparer<Hypothesis>, Moses::UnorderedComparer<Hypothesis> > _HCType;
typedef boost::unordered_set<const Hypothesis*, UnorderedComparer<Hypothesis>, UnorderedComparer<Hypothesis> > _HCType;
_HCType m_hypos;
public:
typedef _HCType::iterator iterator;

8
contrib/other-builds/moses2/System.cpp
View File

@ -23,10 +23,10 @@ System::System(const Parameter &paramsArg)
{
ini_performance_options();
params.SetParameter(stackSize, "stack", Moses::DEFAULT_MAX_HYPOSTACK_SIZE);
params.SetParameter(stackSize, "stack", DEFAULT_MAX_HYPOSTACK_SIZE);
params.SetParameter(maxDistortion, "distortion-limit", -1);
params.SetParameter(maxPhraseLength, "max-phrase-length",
Moses::DEFAULT_MAX_PHRASE_LENGTH);
DEFAULT_MAX_PHRASE_LENGTH);
featureFunctions.Create();
LoadWeights();
@ -50,7 +50,7 @@ void System::LoadWeights()
void System::LoadMappings()
{
const Moses::PARAM_VEC *vec = params.GetParam("mapping");
const PARAM_VEC *vec = params.GetParam("mapping");
UTIL_THROW_IF2(vec == NULL, "Must have [mapping] section");
BOOST_FOREACH(const std::string &line, *vec) {
@ -102,7 +102,7 @@ void
System
::ini_performance_options()
{
const Moses::PARAM_VEC *paramsVec;
const PARAM_VEC *paramsVec;
// m_parameter->SetParameter<size_t>(m_timeout_threshold, "time-out", -1);
// m_timeout = (GetTimeoutThreshold() == (size_t)-1) ? false : true;

4
contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.h
View File

@ -8,7 +8,7 @@
#ifndef PHRASETABLEMEMORY_H_
#define PHRASETABLEMEMORY_H_
#include "PhraseTable.h"
#include "../legacy/Util2.h"
class PhraseTableMemory : public PhraseTable
{
@ -24,7 +24,7 @@ class PhraseTableMemory : public PhraseTable
void SortAndPrune(size_t tableLimit);
protected:
typedef boost::unordered_map<Word, Node, Moses::UnorderedComparer<Word>, Moses::UnorderedComparer<Word> > Children;
typedef boost::unordered_map<Word, Node, UnorderedComparer<Word>, UnorderedComparer<Word> > Children;
Children m_children;
TargetPhrases::shared_ptr m_targetPhrases;

3
contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp
View File

@ -12,6 +12,7 @@
#include "../Search/Manager.h"
#include "../legacy/FactorCollection.h"
#include "../legacy/ProbingPT/quering.hh"
#include "../legacy/Util2.h"
using namespace std;
@ -154,7 +155,7 @@ TargetPhrase *ProbingPT::CreateTargetPhrase(MemPool &pool, const System &system,
// score for this phrase table
vector<SCORE> scores = probingTargetPhrase.prob;
std::transform(scores.begin(), scores.end(), scores.begin(), Moses::TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(), TransformScore);
tp->GetScores().PlusEquals(system, *this, scores);
// // alignment

2
contrib/other-builds/moses2/Word.h
View File

@ -9,7 +9,7 @@
#include <iostream>
#include "TypeDef.h"
#include "moses/Factor.h"
#include "legacy/Factor.h"
#include "legacy/FactorCollection.h"
class Word {

24
contrib/other-builds/moses2/legacy/Util2.h
View File

@ -6,9 +6,12 @@
#include <limits>
#include <sstream>
#include <vector>
#include <cmath>
#define NOT_FOUND std::numeric_limits<size_t>::max()
typedef size_t FactorType;
const size_t DEFAULT_MAX_PHRASE_LENGTH = 20;
const size_t DEFAULT_MAX_HYPOSTACK_SIZE = 200;
template<typename T>
class UnorderedComparer
@ -208,3 +211,24 @@ inline bool FileExists(const std::string& filePath)
const std::string ToLower(const std::string& str);
//! delete and remove every element of a collection object such as set, list etc
template<class COLL>
void RemoveAllInColl(COLL &coll)
{
for (typename COLL::const_iterator iter = coll.begin() ; iter != coll.end() ; ++iter) {
delete (*iter);
}
coll.clear();
}
//! irst number are in log 10, transform to natural log
inline float TransformLMScore(float irstScore)
{
return irstScore * 2.30258509299405f;
}
//! transform prob to natural log score
inline float TransformScore(float prob)
{
return log(prob);
}