moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-10-26 19:37:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
Hieu Hoang 2017-02-01 00:28:37 +00:00
commit 235de89b57
269 changed files with 16306 additions and 17743 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
moses/Util.h
View File

@ -428,7 +428,7 @@ inline float CalcTranslationScore(const std::vector<float> &probVector,
out << *this; \
return out.str(); \
} \
//! delete and remove every element of a collection object such as set, list etc
template<class COLL>
void RemoveAllInColl(COLL &coll)

128
moses2/ArcLists.cpp
View File

@ -19,110 +19,108 @@ namespace Moses2
ArcLists::ArcLists()
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
ArcLists::~ArcLists()
{
BOOST_FOREACH(const Coll::value_type &collPair, m_coll){
const ArcList *arcList = collPair.second;
delete arcList;
}
BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
const ArcList *arcList = collPair.second;
delete arcList;
}
}
void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,
const HypothesisBase *otherHypo)
const HypothesisBase *otherHypo)
{
//cerr << added << " " << currHypo << " " << otherHypo << endl;
ArcList *arcList;
if (added) {
// we're winners!
if (otherHypo) {
// there was a existing losing hypo
arcList = &GetAndDetachArcList(otherHypo);
}
else {
// there was no existing hypo
arcList = new ArcList;
}
m_coll[currHypo] = arcList;
}
else {
// we're losers!
// there should be a winner, we're not doing beam pruning
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
arcList = &GetArcList(otherHypo);
}
//cerr << added << " " << currHypo << " " << otherHypo << endl;
ArcList *arcList;
if (added) {
// we're winners!
if (otherHypo) {
// there was a existing losing hypo
arcList = &GetAndDetachArcList(otherHypo);
} else {
// there was no existing hypo
arcList = new ArcList;
}
m_coll[currHypo] = arcList;
} else {
// we're losers!
// there should be a winner, we're not doing beam pruning
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
arcList = &GetArcList(otherHypo);
}
// in any case, add the curr hypo
arcList->push_back(currHypo);
// in any case, add the curr hypo
arcList->push_back(currHypo);
}
ArcList &ArcLists::GetArcList(const HypothesisBase *hypo)
{
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList &arcList = *iter->second;
return arcList;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList &arcList = *iter->second;
return arcList;
}
const ArcList &ArcLists::GetArcList(const HypothesisBase *hypo) const
{
Coll::const_iterator iter = m_coll.find(hypo);
Coll::const_iterator iter = m_coll.find(hypo);
if (iter == m_coll.end()) {
cerr << "looking for:" << hypo << " have " << m_coll.size() << " :";
BOOST_FOREACH(const Coll::value_type &collPair, m_coll){
const HypothesisBase *hypo = collPair.first;
cerr << hypo << " ";
}
}
if (iter == m_coll.end()) {
cerr << "looking for:" << hypo << " have " << m_coll.size() << " :";
BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
const HypothesisBase *hypo = collPair.first;
cerr << hypo << " ";
}
}
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list for " << hypo);
ArcList &arcList = *iter->second;
return arcList;
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list for " << hypo);
ArcList &arcList = *iter->second;
return arcList;
}
ArcList &ArcLists::GetAndDetachArcList(const HypothesisBase *hypo)
{
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList &arcList = *iter->second;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList &arcList = *iter->second;
m_coll.erase(iter);
m_coll.erase(iter);
return arcList;
return arcList;
}
void ArcLists::Sort()
{
BOOST_FOREACH(Coll::value_type &collPair, m_coll){
ArcList &list = *collPair.second;
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
}
BOOST_FOREACH(Coll::value_type &collPair, m_coll) {
ArcList &list = *collPair.second;
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
}
}
void ArcLists::Delete(const HypothesisBase *hypo)
{
//cerr << "hypo=" << hypo->Debug() << endl;
//cerr << "m_coll=" << m_coll.size() << endl;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
//cerr << "hypo=" << hypo->Debug() << endl;
//cerr << "m_coll=" << m_coll.size() << endl;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
m_coll.erase(iter);
delete arcList;
m_coll.erase(iter);
delete arcList;
}
std::string ArcLists::Debug(const System &system) const
{
stringstream strm;
BOOST_FOREACH(const Coll::value_type &collPair, m_coll){
const ArcList *arcList = collPair.second;
strm << arcList << "(" << arcList->size() << ") ";
}
return strm.str();
stringstream strm;
BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
const ArcList *arcList = collPair.second;
strm << arcList << "(" << arcList->size() << ") ";
}
return strm.str();
}
}

2
moses2/ArcLists.h
View File

@ -23,7 +23,7 @@ public:
virtual ~ArcLists();
void AddArc(bool added, const HypothesisBase *currHypo,
const HypothesisBase *otherHypo);
const HypothesisBase *otherHypo);
void Sort();
void Delete(const HypothesisBase *hypo);

41
moses2/Array.h
View File

@ -13,26 +13,21 @@ public:
typedef T* iterator;
typedef const T* const_iterator;
//! iterators
const_iterator begin() const
{
const_iterator begin() const {
return m_arr;
}
const_iterator end() const
{
const_iterator end() const {
return m_arr + m_size;
}
iterator begin()
{
iterator begin() {
return m_arr;
}
iterator end()
{
iterator end() {
return m_arr + m_size;
}
Array(MemPool &pool, size_t size = 0, const T &val = T())
{
Array(MemPool &pool, size_t size = 0, const T &val = T()) {
m_size = size;
m_maxSize = size;
m_arr = pool.Allocate<T>(size);
@ -41,26 +36,23 @@ public:
}
}
size_t size() const
{
size_t size() const {
return m_size;
}
const T& operator[](size_t ind) const
{
const T& operator[](size_t ind) const {
return m_arr[ind];
}
T& operator[](size_t ind)
{
T& operator[](size_t ind) {
return m_arr[ind];
}
T *GetArray()
{ return m_arr; }
T *GetArray() {
return m_arr;
}
size_t hash() const
{
size_t hash() const {
size_t seed = 0;
for (size_t i = 0; i < m_size; ++i) {
boost::hash_combine(seed, m_arr[i]);
@ -68,21 +60,18 @@ public:
return seed;
}
int Compare(const Array &compare) const
{
int Compare(const Array &compare) const {
int cmp = memcmp(m_arr, compare.m_arr, sizeof(T) * m_size);
return cmp;
}
bool operator==(const Array &compare) const
{
bool operator==(const Array &compare) const {
int cmp = Compare(compare);
return cmp == 0;
}
void resize(size_t newSize)
{
void resize(size_t newSize) {
assert(m_size < m_maxSize);
m_size = newSize;
}

4
moses2/EstimatedScores.cpp
View File

@ -99,8 +99,8 @@ float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap, size_t startPos,
}
// end of a gap?
else if (startGap != notInGap
&& (bitmap.GetValue(currPos) == true
|| (startPos <= currPos && currPos <= endPos))) {
&& (bitmap.GetValue(currPos) == true
|| (startPos <= currPos && currPos <= endPos))) {
estimatedScore += GetValue(startGap, currPos - 1);
startGap = notInGap;
}

6
moses2/EstimatedScores.h
View File

@ -36,8 +36,7 @@ class EstimatedScores: public Matrix<float>
{
public:
EstimatedScores(MemPool &pool, size_t size) :
Matrix<float>(pool, size, size)
{
Matrix<float>(pool, size, size) {
}
~EstimatedScores(); // not implemented
@ -45,8 +44,7 @@ public:
float CalcEstimatedScore(Bitmap const&) const;
float CalcEstimatedScore(Bitmap const&, size_t startPos, size_t endPos) const;
std::ostream &Debug(std::ostream &out, const System &system) const
{
std::ostream &Debug(std::ostream &out, const System &system) const {
for (size_t endPos = 0; endPos < GetSize(); endPos++) {
for (size_t startPos = 0; startPos < GetSize(); startPos++)
out << GetValue(startPos, endPos) << " ";

56
moses2/FF/Distortion.cpp
View File

@ -16,36 +16,30 @@ using namespace std;
namespace Moses2
{
struct DistortionState_traditional: public FFState
{
struct DistortionState_traditional: public FFState {
Range range;
int first_gap;
DistortionState_traditional() :
range()
{
range() {
// uninitialised
}
void Set(const Range& wr, int fg)
{
void Set(const Range& wr, int fg) {
range = wr;
first_gap = fg;
}
size_t hash() const
{
size_t hash() const {
return range.GetEndPos();
}
virtual bool operator==(const FFState& other) const
{
virtual bool operator==(const FFState& other) const {
const DistortionState_traditional& o =
static_cast<const DistortionState_traditional&>(other);
static_cast<const DistortionState_traditional&>(other);
return range.GetEndPos() == o.range.GetEndPos();
}
virtual std::string ToString() const
{
virtual std::string ToString() const {
stringstream sb;
sb << first_gap << " " << range;
return sb.str();
@ -55,7 +49,7 @@ struct DistortionState_traditional: public FFState
///////////////////////////////////////////////////////////////////////
Distortion::Distortion(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
StatefulFeatureFunction(startInd, line)
{
ReadParameters();
}
@ -71,10 +65,10 @@ FFState* Distortion::BlankState(MemPool &pool, const System &sys) const
}
void Distortion::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
const InputType &input, const Hypothesis &hypo) const
{
DistortionState_traditional &stateCast =
static_cast<DistortionState_traditional&>(state);
static_cast<DistortionState_traditional&>(state);
// fake previous translated phrase start and end
size_t start = NOT_FOUND;
@ -92,31 +86,31 @@ void Distortion::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
}
void Distortion::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void Distortion::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void Distortion::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const DistortionState_traditional &prev =
static_cast<const DistortionState_traditional&>(prevState);
static_cast<const DistortionState_traditional&>(prevState);
SCORE distortionScore = CalculateDistortionScore(prev.range,
hypo.GetInputPath().range, prev.first_gap);
hypo.GetInputPath().range, prev.first_gap);
//cerr << "distortionScore=" << distortionScore << endl;
scores.PlusEquals(mgr.system, *this, distortionScore);
DistortionState_traditional &stateCast =
static_cast<DistortionState_traditional&>(state);
static_cast<DistortionState_traditional&>(state);
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
//cerr << "hypo=" << hypo.Debug(mgr.system) << endl;
@ -128,8 +122,7 @@ SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
bool useEarlyDistortionCost = false;
if (!useEarlyDistortionCost) {
return -(SCORE) ComputeDistortionDistance(prev, curr);
}
else {
} else {
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
Definitions:
S : current source range
@ -162,7 +155,7 @@ SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
//IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
return (float) -2
* ((int) curr.GetNumWordsBetween(prev) + (int) curr.GetNumWordsCovered());
* ((int) curr.GetNumWordsBetween(prev) + (int) curr.GetNumWordsCovered());
}
}
@ -173,16 +166,15 @@ int Distortion::ComputeDistortionDistance(const Range& prev,
int dist = 0;
if (prev.GetNumWordsCovered() == 0) {
dist = current.GetStartPos();
}
else {
} else {
dist = (int) prev.GetEndPos() - (int) current.GetStartPos() + 1;
}
return abs(dist);
}
void Distortion::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
UTIL_THROW2("Not implemented");
}

23
moses2/FF/Distortion.h
View File

@ -23,33 +23,32 @@ public:
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const std::deque<Hypothesis*> &hypos) const
{
virtual void EvaluateWhenApplied(const std::deque<Hypothesis*> &hypos) const {
}
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
protected:
SCORE CalculateDistortionScore(const Range &prev, const Range &curr,
const int FirstGap) const;
const int FirstGap) const;
int ComputeDistortionDistance(const Range& prev, const Range& current) const;

1
moses2/FF/FFState.cpp
View File

@ -0,0 +1 @@
#include "FFState.h"

15
moses2/FF/FFState.h
View File

@ -10,14 +10,12 @@ namespace Moses2
class FFState
{
public:
virtual ~FFState()
{
virtual ~FFState() {
}
virtual size_t hash() const = 0;
virtual bool operator==(const FFState& other) const = 0;
virtual bool operator!=(const FFState& other) const
{
virtual bool operator!=(const FFState& other) const {
return !(*this == other);
}
@ -35,17 +33,14 @@ inline std::ostream& operator<<(std::ostream& out, const FFState& obj)
class DummyState: public FFState
{
public:
DummyState()
{
DummyState() {
}
virtual size_t hash() const
{
virtual size_t hash() const {
return 0;
}
virtual bool operator==(const FFState& other) const
{
virtual bool operator==(const FFState& other) const {
return true;
}

21
moses2/FF/FeatureFunction.cpp
View File

@ -17,10 +17,10 @@ namespace Moses2
{
FeatureFunction::FeatureFunction(size_t startInd, const std::string &line)
:m_startInd(startInd)
,m_numScores(1)
,m_PhraseTableInd(NOT_FOUND)
,m_tuneable(true)
:m_startInd(startInd)
,m_numScores(1)
,m_PhraseTableInd(NOT_FOUND)
,m_tuneable(true)
{
ParseLine(line);
//cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
@ -43,18 +43,16 @@ void FeatureFunction::ParseLine(const std::string &line)
for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = TokenizeFirstOnly(toks[i], "=");
UTIL_THROW_IF2(args.size() != 2,
"Incorrect format for feature function arg: " << toks[i]);
"Incorrect format for feature function arg: " << toks[i]);
pair<set<string>::iterator, bool> ret = keys.insert(args[0]);
UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line);
if (args[0] == "num-features") {
m_numScores = Scan<size_t>(args[1]);
}
else if (args[0] == "name") {
} else if (args[0] == "name") {
m_name = args[1];
}
else {
} else {
m_args.push_back(args);
}
}
@ -71,12 +69,11 @@ void FeatureFunction::ReadParameters()
}
void FeatureFunction::SetParameter(const std::string& key,
const std::string& value)
const std::string& value)
{
if (key == "tuneable") {
m_tuneable = Scan<bool>(value);
}
else {
} else {
UTIL_THROW2(GetName() << ": Unknown argument " << key << "=" << value);
}
}

44
moses2/FF/FeatureFunction.h
View File

@ -39,44 +39,35 @@ public:
FeatureFunction(size_t startInd, const std::string &line);
virtual ~FeatureFunction();
virtual void Load(System &system)
{
virtual void Load(System &system) {
}
size_t GetStartInd() const
{
size_t GetStartInd() const {
return m_startInd;
}
size_t GetNumScores() const
{
size_t GetNumScores() const {
return m_numScores;
}
const std::string &GetName() const
{
const std::string &GetName() const {
return m_name;
}
void SetName(const std::string &val)
{
void SetName(const std::string &val) {
m_name = val;
}
virtual size_t HasPhraseTableInd() const
{
virtual size_t HasPhraseTableInd() const {
return false;
}
void SetPhraseTableInd(size_t ind)
{
void SetPhraseTableInd(size_t ind) {
m_PhraseTableInd = ind;
}
size_t GetPhraseTableInd() const
{
size_t GetPhraseTableInd() const {
return m_PhraseTableInd;
}
//! if false, then this feature is not displayed in the n-best list.
// use with care
virtual bool IsTuneable() const
{
virtual bool IsTuneable() const {
return m_tuneable;
}
@ -85,30 +76,27 @@ public:
// may have more factors than actually need, but not guaranteed.
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const = 0;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const = 0;
// For SCFG decoding, the source can contain non-terminals, NOT the raw
// source from the input sentence
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const = 0;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const = 0;
// used by lexicalised reordering model to add scores to tp data structures
virtual void EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const
{
const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const {
}
virtual void EvaluateAfterTablePruning(MemPool &pool,
const SCFG::TargetPhrases &tps, const Phrase<SCFG::Word> &sourcePhrase) const
{
const SCFG::TargetPhrases &tps, const Phrase<SCFG::Word> &sourcePhrase) const {
}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing() const
{
virtual void CleanUpAfterSentenceProcessing() const {
}
protected:

78
moses2/FF/FeatureFunctions.cpp
View File

@ -26,7 +26,7 @@ using namespace std;
namespace Moses2
{
FeatureFunctions::FeatureFunctions(System &system) :
m_system(system), m_ffStartInd(0)
m_system(system), m_ffStartInd(0)
{
}
@ -38,27 +38,26 @@ FeatureFunctions::~FeatureFunctions()
void FeatureFunctions::Load()
{
// load, everything but pts
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
if (pt) {
// do nothing. load pt last
if (pt) {
// do nothing. load pt last
} else {
cerr << "Loading " << nonConstFF->GetName() << endl;
nonConstFF->Load(m_system);
cerr << "Finished loading " << nonConstFF->GetName() << endl;
}
}
else {
cerr << "Loading " << nonConstFF->GetName() << endl;
nonConstFF->Load(m_system);
cerr << "Finished loading " << nonConstFF->GetName() << endl;
}
}
// load pt
BOOST_FOREACH(const PhraseTable *pt, phraseTables) {
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
cerr << "Loading " << nonConstPT->GetName() << endl;
nonConstPT->Load(m_system);
cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
BOOST_FOREACH(const PhraseTable *pt, phraseTables) {
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
cerr << "Loading " << nonConstPT->GetName() << endl;
nonConstPT->Load(m_system);
cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
}
void FeatureFunctions::Create()
@ -68,7 +67,7 @@ void FeatureFunctions::Create()
const PARAM_VEC *ffParams = params.GetParam("feature");
UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");
BOOST_FOREACH(const std::string &line, *ffParams){
BOOST_FOREACH(const std::string &line, *ffParams) {
//cerr << "line=" << line << endl;
FeatureFunction *ff = Create(line);
@ -135,34 +134,33 @@ std::string FeatureFunctions::GetDefaultName(const std::string &stub)
{
size_t ind;
boost::unordered_map<std::string, size_t>::iterator iter =
m_defaultNames.find(stub);
m_defaultNames.find(stub);
if (iter == m_defaultNames.end()) {
m_defaultNames[stub] = 0;
ind = 0;
}
else {
} else {
ind = ++(iter->second);
}
return stub + SPrint(ind);
}
const FeatureFunction *FeatureFunctions::FindFeatureFunction(
const std::string &name) const
const std::string &name) const
{
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
if (ff->GetName() == name) {
return ff;
}
}
return NULL;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
if (ff->GetName() == name) {
return ff;
}
}
return NULL;
}
FeatureFunction *FeatureFunctions::FindFeatureFunction(
const std::string &name)
const std::string &name)
{
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
if (ff->GetName() == name) {
return const_cast<FeatureFunction *>(ff);
return const_cast<FeatureFunction *>(ff);
}
}
return NULL;
@ -190,7 +188,7 @@ void FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
{
SCORE estimatedScore = 0;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, estimatedScore);
}
@ -199,14 +197,14 @@ void FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
}
void FeatureFunctions::EvaluateInIsolation(
MemPool &pool,
const System &system,
const Phrase<SCFG::Word> &source,
SCFG::TargetPhraseImpl &targetPhrase) const
MemPool &pool,
const System &system,
const Phrase<SCFG::Word> &source,
SCFG::TargetPhraseImpl &targetPhrase) const
{
SCORE estimatedScore = 0;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, estimatedScore);
}
@ -240,7 +238,7 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const
void FeatureFunctions::CleanUpAfterSentenceProcessing() const
{
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
ff->CleanUpAfterSentenceProcessing();
ff->CleanUpAfterSentenceProcessing();
}
}
@ -280,7 +278,7 @@ void FeatureFunctions::OverrideFeatures()
UTIL_THROW_IF2(keyVal.size() != 2, "Incorrect format for parameter override: " << keyValStr);
cerr << "Override " << ff->GetName() << " "
<< keyVal[0] << "=" << keyVal[1] << endl;
<< keyVal[0] << "=" << keyVal[1] << endl;
ff->SetParameter(keyVal[0], keyVal[1]);

33
moses2/FF/FeatureFunctions.h
View File

@ -49,17 +49,21 @@ public:
FeatureFunctions(System &system);
virtual ~FeatureFunctions();
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const
{ return m_featureFunctions; }
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const {
return m_featureFunctions;
}
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const
{ return m_statefulFeatureFunctions; }
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const {
return m_statefulFeatureFunctions;
}
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const
{ return m_withPhraseTableInd; }
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const {
return m_withPhraseTableInd;
}
size_t GetNumScores() const
{ return m_ffStartInd; }
size_t GetNumScores() const {
return m_ffStartInd;
}
void Create();
void Load();
@ -67,19 +71,20 @@ public:
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
const PhraseTable *GetPhraseTableExcludeUnknownWordPenalty(size_t ptInd);
const UnknownWordPenalty *GetUnknownWordPenalty() const
{ return m_unkWP; }
const UnknownWordPenalty *GetUnknownWordPenalty() const {
return m_unkWP;
}
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, TargetPhraseImpl &targetPhrase) const;
const Phrase<Moses2::Word> &source, TargetPhraseImpl &targetPhrase) const;
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<SCFG::Word> &source, SCFG::TargetPhraseImpl &targetPhrase) const;
const Phrase<SCFG::Word> &source, SCFG::TargetPhraseImpl &targetPhrase) const;
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
const Phrase<Moses2::Word> &sourcePhrase) const;
const Phrase<Moses2::Word> &sourcePhrase) const;
void EvaluateAfterTablePruning(MemPool &pool, const SCFG::TargetPhrases &tps,
const Phrase<SCFG::Word> &sourcePhrase) const;
const Phrase<SCFG::Word> &sourcePhrase) const;
void EvaluateWhenAppliedBatch(const Batch &batch) const;

6
moses2/FF/FeatureRegistry.cpp
View File

@ -35,8 +35,7 @@ template<class F>
class DefaultFeatureFactory: public FeatureFactory
{
public:
FeatureFunction *Create(size_t startInd, const std::string &line) const
{
FeatureFunction *Create(size_t startInd, const std::string &line) const {
return new F(startInd, line);
}
};
@ -45,8 +44,7 @@ public:
class KenFactory: public FeatureFactory
{
public:
FeatureFunction *Create(size_t startInd, const std::string &line) const
{
FeatureFunction *Create(size_t startInd, const std::string &line) const {
ConstructKenLM(startInd, line);
}
};

8
moses2/FF/FeatureRegistry.h
View File

@ -10,15 +10,13 @@ class FeatureFunction;
class FeatureFactory
{
public:
virtual ~FeatureFactory()
{
virtual ~FeatureFactory() {
}
virtual FeatureFunction *Create(size_t startInd, const std::string &line) const = 0;
protected:
FeatureFactory()
{
FeatureFactory() {
}
};
@ -33,7 +31,7 @@ public:
~FeatureRegistry();
FeatureFunction *Construct(size_t startInd, const std::string &name,
const std::string &line) const;
const std::string &line) const;
void PrintFF() const;
private:

22
moses2/FF/LexicalReordering/BidirectionalReorderingState.cpp
View File

@ -15,9 +15,9 @@ namespace Moses2
{
BidirectionalReorderingState::BidirectionalReorderingState(
const LRModel &config, LRState *bw, LRState *fw, size_t offset) :
LRState(config, LRModel::Bidirectional, offset), m_backward(bw), m_forward(
fw)
const LRModel &config, LRState *bw, LRState *fw, size_t offset) :
LRState(config, LRModel::Bidirectional, offset), m_backward(bw), m_forward(
fw)
{
}
@ -27,8 +27,8 @@ BidirectionalReorderingState::~BidirectionalReorderingState()
}
void BidirectionalReorderingState::Init(const LRState *prev,
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
if (m_backward) {
m_backward->Init(prev, topt, path, first, coverage);
@ -41,7 +41,7 @@ void BidirectionalReorderingState::Init(const LRState *prev,
std::string BidirectionalReorderingState::ToString() const
{
return "BidirectionalReorderingState " + SPrint(this) + " "
+ SPrint(m_backward) + " " + SPrint(m_forward);
+ SPrint(m_backward) + " " + SPrint(m_forward);
}
size_t BidirectionalReorderingState::hash() const
@ -57,10 +57,10 @@ bool BidirectionalReorderingState::operator==(const FFState& o) const
if (&o == this) return true;
BidirectionalReorderingState const &other =
static_cast<BidirectionalReorderingState const&>(o);
static_cast<BidirectionalReorderingState const&>(o);
bool ret = (*m_backward == *other.m_backward)
&& (*m_forward == *other.m_forward);
&& (*m_forward == *other.m_forward);
return ret;
}
@ -69,11 +69,11 @@ void BidirectionalReorderingState::Expand(const ManagerBase &mgr,
Scores &scores, FFState &state) const
{
BidirectionalReorderingState &stateCast =
static_cast<BidirectionalReorderingState&>(state);
static_cast<BidirectionalReorderingState&>(state);
m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores,
*stateCast.m_backward);
*stateCast.m_backward);
m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores,
*stateCast.m_forward);
*stateCast.m_forward);
}
} /* namespace Moses2 */

8
moses2/FF/LexicalReordering/BidirectionalReorderingState.h
View File

@ -14,12 +14,12 @@ class BidirectionalReorderingState: public LRState
{
public:
BidirectionalReorderingState(const LRModel &config, LRState *bw, LRState *fw,
size_t offset);
size_t offset);
virtual ~BidirectionalReorderingState();
void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
@ -27,8 +27,8 @@ public:
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:
LRState *m_backward;

16
moses2/FF/LexicalReordering/HReorderingBackwardState.cpp
View File

@ -14,7 +14,7 @@ namespace Moses2
HReorderingBackwardState::HReorderingBackwardState(MemPool &pool,
const LRModel &config, size_t offset) :
LRState(config, LRModel::Backward, offset), reoStack(pool)
LRState(config, LRModel::Backward, offset), reoStack(pool)
{
// TODO Auto-generated constructor stub
@ -26,8 +26,8 @@ HReorderingBackwardState::~HReorderingBackwardState()
}
void HReorderingBackwardState::Init(const LRState *prev,
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
reoStack.Init();
@ -42,7 +42,7 @@ size_t HReorderingBackwardState::hash() const
bool HReorderingBackwardState::operator==(const FFState& o) const
{
const HReorderingBackwardState& other =
static_cast<const HReorderingBackwardState&>(o);
static_cast<const HReorderingBackwardState&>(o);
bool ret = reoStack == other.reoStack;
return ret;
}
@ -53,13 +53,13 @@ std::string HReorderingBackwardState::ToString() const
}
void HReorderingBackwardState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
HReorderingBackwardState &nextState =
static_cast<HReorderingBackwardState&>(state);
static_cast<HReorderingBackwardState&>(state);
nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
NULL);
NULL);
nextState.reoStack = reoStack;
const Range &swrange = hypo.GetInputPath().range;

6
moses2/FF/LexicalReordering/HReorderingBackwardState.h
View File

@ -20,7 +20,7 @@ public:
HReorderingBackwardState(MemPool &pool, const LRModel &config, size_t offset);
virtual void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
const InputPathBase &path, bool first, const Bitmap *coverage);
virtual ~HReorderingBackwardState();
@ -28,8 +28,8 @@ public:
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
};

22
moses2/FF/LexicalReordering/HReorderingForwardState.cpp
View File

@ -15,7 +15,7 @@ namespace Moses2
HReorderingForwardState::HReorderingForwardState(const LRModel &config,
size_t offset) :
LRState(config, LRModel::Forward, offset), m_first(true)
LRState(config, LRModel::Forward, offset), m_first(true)
{
prevPath = NULL;
m_coverage = NULL;
@ -27,8 +27,8 @@ HReorderingForwardState::~HReorderingForwardState()
}
void HReorderingForwardState::Init(const LRState *prev,
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
prevPath = &path;
@ -48,12 +48,12 @@ bool HReorderingForwardState::operator==(const FFState& o) const
if (&o == this) return true;
HReorderingForwardState const& other =
static_cast<HReorderingForwardState const&>(o);
static_cast<HReorderingForwardState const&>(o);
int compareScores = (
(prevPath->range == other.prevPath->range) ?
ComparePrevScores(other.prevTP) :
(prevPath->range < other.prevPath->range) ? -1 : 1);
(prevPath->range == other.prevPath->range) ?
ComparePrevScores(other.prevTP) :
(prevPath->range < other.prevPath->range) ? -1 : 1);
return compareScores == 0;
}
@ -63,8 +63,8 @@ std::string HReorderingForwardState::ToString() const
}
void HReorderingForwardState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
const Range &cur = hypo.GetInputPath().range;
// keep track of the current coverage ourselves so we don't need the hypothesis
@ -79,9 +79,9 @@ void HReorderingForwardState::Expand(const ManagerBase &mgr,
}
HReorderingForwardState &stateCast =
static_cast<HReorderingForwardState&>(state);
static_cast<HReorderingForwardState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
&cov);
&cov);
}
} /* namespace Moses2 */

6
moses2/FF/LexicalReordering/HReorderingForwardState.h
View File

@ -20,14 +20,14 @@ public:
virtual ~HReorderingForwardState();
void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:
bool m_first;

91
moses2/FF/LexicalReordering/LRModel.cpp
View File

@ -22,8 +22,8 @@ namespace Moses2
{
bool IsMonotonicStep(Range const& prev, // words range of last source phrase
Range const& cur, // words range of current source phrase
Bitmap const& cov) // coverage bitmap
Range const& cur, // words range of current source phrase
Bitmap const& cov) // coverage bitmap
{
size_t e = prev.GetEndPos() + 1;
size_t s = cur.GetStartPos();
@ -38,19 +38,17 @@ bool IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
}
LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(
Backward), m_scoreProducer(&ff)
m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(
Backward), m_scoreProducer(&ff)
{
std::vector<std::string> config = Tokenize(modelType, "-");
for (size_t i = 0; i < config.size(); ++i) {
if (config[i] == "hier") {
m_phraseBased = false;
}
else if (config[i] == "phrase") {
} else if (config[i] == "phrase") {
m_phraseBased = true;
}
else if (config[i] == "wbe") {
} else if (config[i] == "wbe") {
m_phraseBased = true;
}
// no word-based decoding available, fall-back to phrase-based
@ -58,45 +56,36 @@ LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
else if (config[i] == "msd") {
m_modelType = MSD;
}
else if (config[i] == "mslr") {
} else if (config[i] == "mslr") {
m_modelType = MSLR;
}
else if (config[i] == "monotonicity") {
} else if (config[i] == "monotonicity") {
m_modelType = Monotonic;
}
else if (config[i] == "leftright") {
} else if (config[i] == "leftright") {
m_modelType = LeftRight;
}
// unidirectional is deprecated, use backward instead
else if (config[i] == "unidirectional") {
m_direction = Backward;
}
else if (config[i] == "backward") {
} else if (config[i] == "backward") {
m_direction = Backward;
}
else if (config[i] == "forward") {
} else if (config[i] == "forward") {
m_direction = Forward;
}
else if (config[i] == "bidirectional") {
} else if (config[i] == "bidirectional") {
m_direction = Bidirectional;
}
else if (config[i] == "f") {
m_condition = F;
}
else if (config[i] == "fe") {
} else if (config[i] == "fe") {
m_condition = FE;
}
else if (config[i] == "collapseff") {
m_collapseScores = true;
}
else if (config[i] == "allff") {
} else if (config[i] == "allff") {
m_collapseScores = false;
}
else {
} else {
std::cerr
<< "Illegal part in the lexical reordering configuration string: "
<< config[i] << std::endl;
@ -106,7 +95,7 @@ LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
if (m_modelType == None) {
std::cerr << "You need to specify the type of the reordering model "
<< "(msd, monotonicity,...)" << std::endl;
<< "(msd, monotonicity,...)" << std::endl;
exit(1);
}
@ -135,19 +124,19 @@ LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
{
UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified");
return (
(m_modelType == LeftRight) ? prev.GetEndPos() <= cur.GetStartPos() ? R : L
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M :
(m_modelType == Monotonic) ? NM :
(prev.GetStartPos() == cur.GetEndPos() + 1) ? S :
(m_modelType == MSD) ? D :
(cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
(m_modelType == LeftRight) ? prev.GetEndPos() <= cur.GetStartPos() ? R : L
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M :
(m_modelType == Monotonic) ? NM :
(prev.GetStartPos() == cur.GetEndPos() + 1) ? S :
(m_modelType == MSD) ? D :
(cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
}
LRModel::ReorderingType LRModel::GetOrientation(int const reoDistance) const
{
// this one is for HierarchicalReorderingBackwardState
return ((m_modelType == LeftRight) ? (reoDistance >= 1) ? R : L
: (reoDistance == 1) ? M : (m_modelType == Monotonic) ? NM :
: (reoDistance == 1) ? M : (m_modelType == Monotonic) ? NM :
(reoDistance == -1) ? S : (m_modelType == MSD) ? D :
(reoDistance > 1) ? DR : DL);
}
@ -162,28 +151,26 @@ LRState *LRModel::CreateLRState(MemPool &pool) const
case Bidirectional:
if (m_phraseBased) {
bwd =
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Backward, offset);
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Backward, offset);
//cerr << "bwd=" << bwd << bwd->ToString() << endl;
}
else {
} else {
bwd =
new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(
pool, *this, offset);
new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(
pool, *this, offset);
}
offset += m_collapseScores ? 1 : GetNumberOfTypes();
if (m_direction == Backward) return bwd; // else fall through
case Forward:
if (m_phraseBased) {
fwd =
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Forward, offset);
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Forward, offset);
//cerr << "fwd=" << fwd << fwd->ToString() << endl;
}
else {
} else {
fwd =
new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(
*this, offset);
new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(
*this, offset);
}
offset += m_collapseScores ? 1 : GetNumberOfTypes();
if (m_direction == Forward) return fwd;
@ -191,8 +178,8 @@ LRState *LRModel::CreateLRState(MemPool &pool) const
//cerr << "LRStates:" << *bwd << endl << *fwd << endl;
BidirectionalReorderingState *ret =
new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(
*this, bwd, fwd, 0);
new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(
*this, bwd, fwd, 0);
return ret;
}
@ -200,10 +187,10 @@ LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
Range const& cur, Bitmap const& cov) const
{
return (
(m_modelType == LeftRight) ? cur.GetStartPos() > prev.GetEndPos() ? R : L
: IsMonotonicStep(prev, cur, cov) ? M : (m_modelType == Monotonic) ? NM :
IsSwap(prev, cur, cov) ? S : (m_modelType == MSD) ? D :
cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
(m_modelType == LeftRight) ? cur.GetStartPos() > prev.GetEndPos() ? R : L
: IsMonotonicStep(prev, cur, cov) ? M : (m_modelType == Monotonic) ? NM :
IsSwap(prev, cur, cov) ? S : (m_modelType == MSD) ? D :
cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
}
} /* namespace Moses2 */

30
moses2/FF/LexicalReordering/LRModel.h
View File

@ -19,21 +19,17 @@ class LexicalReordering;
class LRModel
{
public:
enum ModelType
{
enum ModelType {
Monotonic, MSD, MSLR, LeftRight, None
};
enum Direction
{
enum Direction {
Forward, Backward, Bidirectional
};
enum Condition
{
enum Condition {
F, E, FE
};
enum ReorderingType
{
enum ReorderingType {
M = 0, // monotonic
NM = 1, // non-monotonic
S = 1, // swap
@ -49,34 +45,28 @@ public:
LRModel(const std::string &modelType, LexicalReordering &ff);
virtual ~LRModel();
ModelType GetModelType() const
{
ModelType GetModelType() const {
return m_modelType;
}
Direction GetDirection() const
{
Direction GetDirection() const {
return m_direction;
}
Condition GetCondition() const
{
Condition GetCondition() const {
return m_condition;
}
bool IsPhraseBased() const
{
bool IsPhraseBased() const {
return m_phraseBased;
}
bool CollapseScores() const
{
bool CollapseScores() const {
return m_collapseScores;
}
size_t GetNumberOfTypes() const;
LexicalReordering*
GetScoreProducer() const
{
GetScoreProducer() const {
return m_scoreProducer;
}

12
moses2/FF/LexicalReordering/LRState.cpp
View File

@ -17,7 +17,7 @@ namespace Moses2
class InputType;
LRState::LRState(const LRModel &config, LRModel::Direction dir, size_t offset) :
m_configuration(config), m_direction(dir), m_offset(offset)
m_configuration(config), m_direction(dir), m_offset(offset)
{
}
@ -43,15 +43,15 @@ int LRState::ComparePrevScores(const TargetPhrase<Moses2::Word> *other) const
}
void LRState::CopyScores(const System &system, Scores &accum,
const TargetPhrase<Moses2::Word> &topt, ReorderingType reoType) const
const TargetPhrase<Moses2::Word> &topt, ReorderingType reoType) const
{
// don't call this on a bidirectional object
UTIL_THROW_IF2(
m_direction != LRModel::Backward && m_direction != LRModel::Forward,
"Unknown direction: " << m_direction);
m_direction != LRModel::Backward && m_direction != LRModel::Forward,
"Unknown direction: " << m_direction);
TargetPhrase<Moses2::Word> const* relevantOpt = (
(m_direction == LRModel::Backward) ? &topt : prevTP);
(m_direction == LRModel::Backward) ? &topt : prevTP);
LexicalReordering* producer = m_configuration.GetScoreProducer();
size_t phraseTableInd = producer->GetPhraseTableInd();
@ -65,7 +65,7 @@ void LRState::CopyScores(const System &system, Scores &accum,
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
UTIL_THROW_IF2(off_local >= producer->GetNumScores(),
"offset out of vector bounds!");
"offset out of vector bounds!");
// look up applicable score from vector of scores
//UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");

8
moses2/FF/LexicalReordering/LRState.h
View File

@ -26,14 +26,14 @@ public:
LRState(const LRModel &config, LRModel::Direction dir, size_t offset);
virtual void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
const InputPathBase &path, bool first, const Bitmap *coverage) = 0;
const InputPathBase &path, bool first, const Bitmap *coverage) = 0;
virtual void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const = 0;
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const = 0;
void CopyScores(const System &system, Scores &accum, const TargetPhrase<Moses2::Word> &topt,
ReorderingType reoType) const;
ReorderingType reoType) const;
protected:
const LRModel& m_configuration;

62
moses2/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -30,8 +30,8 @@ namespace Moses2
///////////////////////////////////////////////////////////////////////
LexicalReordering::LexicalReordering(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line), m_compactModel(NULL), m_blank(
NULL), m_propertyInd(-1), m_coll(NULL), m_configuration(NULL)
StatefulFeatureFunction(startInd, line), m_compactModel(NULL), m_blank(
NULL), m_propertyInd(-1), m_coll(NULL), m_configuration(NULL)
{
ReadParameters();
assert(m_configuration);
@ -51,13 +51,11 @@ void LexicalReordering::Load(System &system)
if (m_propertyInd >= 0) {
// Using integrate Lex RO. No loading needed
}
else if (FileExists(m_path + ".minlexr")) {
} else if (FileExists(m_path + ".minlexr")) {
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr",
m_FactorsF, m_FactorsE, m_FactorsC);
m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
}
else {
} else {
m_coll = new Coll();
InputFileStream file(m_path);
string line;
@ -71,12 +69,12 @@ void LexicalReordering::Load(System &system)
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
assert(toks.size() == 3);
PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(),
system, toks[0]);
system, toks[0]);
PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(),
system, toks[1]);
system, toks[1]);
std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
std::transform(scores.begin(), scores.end(), scores.begin(),
TransformScore);
TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
Key key(source, target);
@ -86,24 +84,19 @@ void LexicalReordering::Load(System &system)
}
void LexicalReordering::SetParameter(const std::string& key,
const std::string& value)
const std::string& value)
{
if (key == "path") {
m_path = value;
}
else if (key == "type") {
} else if (key == "type") {
m_configuration = new LRModel(value, *this);
}
else if (key == "input-factor") {
} else if (key == "input-factor") {
m_FactorsF = Tokenize<FactorType>(value);
}
else if (key == "output-factor") {
} else if (key == "output-factor") {
m_FactorsE = Tokenize<FactorType>(value);
}
else if (key == "property-index") {
} else if (key == "property-index") {
m_propertyInd = Scan<int>(value);
}
else {
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -119,9 +112,9 @@ void LexicalReordering::EmptyHypothesisState(FFState &state,
const Hypothesis &hypo) const
{
BidirectionalReorderingState &stateCast =
static_cast<BidirectionalReorderingState&>(state);
static_cast<BidirectionalReorderingState&>(state);
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true,
&hypo.GetBitmap());
&hypo.GetBitmap());
}
void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system,
@ -141,9 +134,9 @@ void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system,
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps, const Phrase<Moses2::Word> &sourcePhrase) const
{
BOOST_FOREACH(const TargetPhraseImpl *tp, tps){
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
}
BOOST_FOREACH(const TargetPhraseImpl *tp, tps) {
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
}
}
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
@ -152,11 +145,10 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
if (m_propertyInd >= 0) {
SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd);
targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
}
else if (m_compactModel) {
} else if (m_compactModel) {
// using external compact binary model
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase,
*m_blank);
*m_blank);
if (values.size()) {
assert(values.size() == m_numScores);
@ -165,12 +157,10 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
scoreArr[i] = values[i];
}
targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
}
else {
} else {
targetPhrase.ffData[m_PhraseTableInd] = NULL;
}
}
else if (m_coll) {
} else if (m_coll) {
// using external memory model
// cache data in target phrase
@ -183,8 +173,7 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
scoreArr[i] = (*values)[i];
}
targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
}
else {
} else {
targetPhrase.ffData[m_PhraseTableInd] = NULL;
}
}
@ -199,15 +188,14 @@ void LexicalReordering::EvaluateWhenApplied(const ManagerBase &mgr,
}
const LexicalReordering::Values *LexicalReordering::GetValues(
const Phrase<Moses2::Word> &source, const Phrase<Moses2::Word> &target) const
const Phrase<Moses2::Word> &source, const Phrase<Moses2::Word> &target) const
{
Key key(&source, &target);
Coll::const_iterator iter;
iter = m_coll->find(key);
if (iter == m_coll->end()) {
return NULL;
}
else {
} else {
return &iter->second;
}
}

34
moses2/FF/LexicalReordering/LexicalReordering.h
View File

@ -30,36 +30,35 @@ public:
virtual void SetParameter(const std::string& key, const std::string& value);
virtual size_t HasPhraseTableInd() const
{
virtual size_t HasPhraseTableInd() const {
return true;
}
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
const Phrase<Moses2::Word> &sourcePhrase) const;
const Phrase<Moses2::Word> &sourcePhrase) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
@ -71,7 +70,7 @@ protected:
virtual void
EvaluateAfterTablePruning(MemPool &pool, const TargetPhraseImpl &targetPhrase,
const Phrase<Moses2::Word> &sourcePhrase) const;
const Phrase<Moses2::Word> &sourcePhrase) const;
// PROPERTY IN PT
int m_propertyInd;
@ -84,17 +83,14 @@ protected:
typedef std::pair<const Phrase<Moses2::Word>*, const Phrase<Moses2::Word>* > Key;
typedef std::vector<SCORE> Values;
struct KeyComparer
{
size_t operator()(const Key &obj) const
{
struct KeyComparer {
size_t operator()(const Key &obj) const {
size_t seed = obj.first->hash();
boost::hash_combine(seed, obj.second->hash());
return seed;
}
bool operator()(const Key& a, const Key& b) const
{
bool operator()(const Key& a, const Key& b) const {
if ((*a.first) != (*b.first)) {
return false;
}

28
moses2/FF/LexicalReordering/PhraseBasedReorderingState.cpp
View File

@ -18,7 +18,7 @@ namespace Moses2
PhraseBasedReorderingState::PhraseBasedReorderingState(const LRModel &config,
LRModel::Direction dir, size_t offset) :
LRState(config, dir, offset)
LRState(config, dir, offset)
{
// uninitialised
prevPath = NULL;
@ -26,8 +26,8 @@ PhraseBasedReorderingState::PhraseBasedReorderingState(const LRModel &config,
}
void PhraseBasedReorderingState::Init(const LRState *prev,
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
const TargetPhrase<Moses2::Word> &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
prevPath = &path;
@ -48,39 +48,37 @@ bool PhraseBasedReorderingState::operator==(const FFState& o) const
if (&o == this) return true;
const PhraseBasedReorderingState &other =
static_cast<const PhraseBasedReorderingState&>(o);
static_cast<const PhraseBasedReorderingState&>(o);
if (&prevPath->range == &other.prevPath->range) {
if (m_direction == LRModel::Forward) {
int compareScore = ComparePrevScores(other.prevTP);
return compareScore == 0;
}
else {
} else {
return true;
}
}
else {
} else {
return false;
}
}
void PhraseBasedReorderingState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
if ((m_direction != LRModel::Forward) || !m_first) {
LRModel const& lrmodel = m_configuration;
Range const &cur = hypo.GetInputPath().range;
LRModel::ReorderingType reoType = (
m_first ?
lrmodel.GetOrientation(cur) :
lrmodel.GetOrientation(prevPath->range, cur));
m_first ?
lrmodel.GetOrientation(cur) :
lrmodel.GetOrientation(prevPath->range, cur));
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
}
PhraseBasedReorderingState &stateCast =
static_cast<PhraseBasedReorderingState&>(state);
static_cast<PhraseBasedReorderingState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
NULL);
NULL);
}
} /* namespace Moses2 */

11
moses2/FF/LexicalReordering/PhraseBasedReorderingState.h
View File

@ -20,22 +20,21 @@ public:
bool m_first;
PhraseBasedReorderingState(const LRModel &config, LRModel::Direction dir,
size_t offset);
size_t offset);
void Init(const LRState *prev, const TargetPhrase<Moses2::Word> &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const
{
virtual std::string ToString() const {
return "PhraseBasedReorderingState";
}
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:

17
moses2/FF/LexicalReordering/ReorderingStack.cpp
View File

@ -11,7 +11,7 @@
namespace Moses2
{
ReorderingStack::ReorderingStack(MemPool &pool) :
m_stack(pool)
m_stack(pool)
{
}
@ -50,8 +50,7 @@ int ReorderingStack::ShiftReduce(const Range &input_span)
//calculate the distance we are returning
if (input_span.GetStartPos() > prev_span.GetStartPos()) {
distance = input_span.GetStartPos() - prev_span.GetEndPos();
}
else {
} else {
distance = input_span.GetEndPos() - prev_span.GetStartPos();
}
@ -59,13 +58,11 @@ int ReorderingStack::ShiftReduce(const Range &input_span)
m_stack.pop_back();
Range new_span(prev_span.GetStartPos(), input_span.GetEndPos());
Reduce(new_span);
}
else if (distance == -1) { //swap
} else if (distance == -1) { //swap
m_stack.pop_back();
Range new_span(input_span.GetStartPos(), prev_span.GetEndPos());
Reduce(new_span);
}
else { // discontinuous
} else { // discontinuous
m_stack.push_back(input_span);
}
@ -85,13 +82,11 @@ void ReorderingStack::Reduce(Range current)
m_stack.pop_back();
Range t(previous.GetStartPos(), current.GetEndPos());
current = t;
}
else if (previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
} else if (previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
m_stack.pop_back();
Range t(current.GetStartPos(), previous.GetEndPos());
current = t;
}
else { // discontinuous, no more merging
} else { // discontinuous, no more merging
cont_loop = false;
}
} // finished reducing, exit

4
moses2/FF/OSM/OpSequenceModel.cpp
View File

@ -17,7 +17,7 @@ namespace Moses2
////////////////////////////////////////////////////////////////////////////////////////
OpSequenceModel::OpSequenceModel(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
StatefulFeatureFunction(startInd, line)
{
sFactor = 0;
tFactor = 0;
@ -96,7 +96,7 @@ void OpSequenceModel::EvaluateInIsolation(MemPool &pool,
obj.populateScores(scoresVec,numFeatures);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
scoresVec.data());
scoresVec.data());
estimatedScore += weightedScore;
}

20
moses2/FF/OSM/OpSequenceModel.h
View File

@ -16,32 +16,32 @@ public:
int tFactor; // Target Factor ...
util::LoadMethod load_method; // method to load model
OpSequenceModel(size_t startInd, const std::string &line);
OpSequenceModel(size_t startInd, const std::string &line);
virtual ~OpSequenceModel();
virtual void Load(System &system);
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
void SetParameter(const std::string& key, const std::string& value);

5
moses2/FF/OSM/osmHyp.h
View File

@ -22,8 +22,9 @@ public:
virtual size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const
{ return "osmState"; }
virtual std::string ToString() const {
return "osmState";
}
void saveState(int jVal, int eVal, std::map <int , std::string> & gapVal);
int getJ()const {

10
moses2/FF/PhrasePenalty.cpp
View File

@ -12,7 +12,7 @@ namespace Moses2
{
PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line) :
StatelessFeatureFunction(startInd, line)
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
}
@ -23,15 +23,15 @@ PhrasePenalty::~PhrasePenalty()
}
void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
scores.PlusEquals(system, *this, 1);
}
void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
scores.PlusEquals(system, *this, 1);
}

8
moses2/FF/PhrasePenalty.h
View File

@ -20,13 +20,13 @@ public:
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
};

6
moses2/FF/PointerState.cpp
View File

@ -0,0 +1,6 @@
#include "PointerState.h"
namespace Moses2
{
}

18
moses2/FF/PointerState.h
View File

@ -6,31 +6,25 @@
namespace Moses2
{
struct PointerState: public FFState
{
struct PointerState: public FFState {
const void* lmstate;
explicit PointerState()
{
explicit PointerState() {
// uninitialised
}
PointerState(const void* lms)
{
PointerState(const void* lms) {
lmstate = lms;
}
virtual size_t hash() const
{
virtual size_t hash() const {
return (size_t) lmstate;
}
virtual bool operator==(const FFState& other) const
{
virtual bool operator==(const FFState& other) const {
const PointerState& o = static_cast<const PointerState&>(other);
return lmstate == o.lmstate;
}
virtual std::string ToString() const
{
virtual std::string ToString() const {
std::stringstream sb;
sb << lmstate;
return sb.str();

14
moses2/FF/SkeletonStatefulFF.cpp
View File

@ -19,23 +19,19 @@ class SkeletonState: public FFState
public:
int targetLen;
SkeletonState()
{
SkeletonState() {
// uninitialised
}
virtual size_t hash() const
{
virtual size_t hash() const {
return (size_t) targetLen;
}
virtual bool operator==(const FFState& o) const
{
virtual bool operator==(const FFState& o) const {
const SkeletonState& other = static_cast<const SkeletonState&>(o);
return targetLen == other.targetLen;
}
virtual std::string ToString() const
{
virtual std::string ToString() const {
stringstream sb;
sb << targetLen;
return sb.str();
@ -45,7 +41,7 @@ public:
////////////////////////////////////////////////////////////////////////////////////////
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
StatefulFeatureFunction(startInd, line)
{
ReadParameters();
}

18
moses2/FF/SkeletonStatefulFF.h
View File

@ -21,25 +21,25 @@ public:
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
};

2
moses2/FF/SkeletonStatelessFF.cpp
View File

@ -13,7 +13,7 @@ namespace Moses2
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd,
const std::string &line) :
StatelessFeatureFunction(startInd, line)
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
}

8
moses2/FF/SkeletonStatelessFF.h
View File

@ -20,13 +20,13 @@ public:
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
};

8
moses2/FF/StatefulFeatureFunction.cpp
View File

@ -20,7 +20,7 @@ namespace Moses2
StatefulFeatureFunction::StatefulFeatureFunction(size_t startInd,
const std::string &line) :
FeatureFunction(startInd, line)
FeatureFunction(startInd, line)
{
}
@ -30,10 +30,10 @@ StatefulFeatureFunction::~StatefulFeatureFunction()
}
void StatefulFeatureFunction::EvaluateWhenAppliedBatch(
const System &system,
const Batch &batch) const
const System &system,
const Batch &batch) const
{
//cerr << "EvaluateWhenAppliedBatch:" << m_name << endl;
//cerr << "EvaluateWhenAppliedBatch:" << m_name << endl;
#ifdef __linux
/*
pthread_t handle;

20
moses2/FF/StatefulFeatureFunction.h
View File

@ -30,12 +30,10 @@ public:
StatefulFeatureFunction(size_t startInd, const std::string &line);
virtual ~StatefulFeatureFunction();
void SetStatefulInd(size_t ind)
{
void SetStatefulInd(size_t ind) {
m_statefulInd = ind;
}
size_t GetStatefulInd() const
{
size_t GetStatefulInd() const {
return m_statefulInd;
}
@ -44,19 +42,19 @@ public:
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const = 0;
const InputType &input, const Hypothesis &hypo) const = 0;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const = 0;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const = 0;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const = 0;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const = 0;
virtual void EvaluateWhenAppliedBatch(
const System &system,
const Batch &batch) const;
const System &system,
const Batch &batch) const;
protected:
size_t m_statefulInd;

2
moses2/FF/StatelessFeatureFunction.cpp
View File

@ -12,7 +12,7 @@ namespace Moses2
StatelessFeatureFunction::StatelessFeatureFunction(size_t startInd,
const std::string &line) :
FeatureFunction(startInd, line)
FeatureFunction(startInd, line)
{
// TODO Auto-generated constructor stub

10
moses2/FF/WordPenalty.cpp
View File

@ -17,7 +17,7 @@ namespace Moses2
{
WordPenalty::WordPenalty(size_t startInd, const std::string &line) :
StatelessFeatureFunction(startInd, line)
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
}
@ -28,16 +28,16 @@ WordPenalty::~WordPenalty()
}
void WordPenalty::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
SCORE score = -(SCORE) targetPhrase.GetSize();
scores.PlusEquals(system, *this, score);
}
void WordPenalty::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
size_t count = 0;
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {

8
moses2/FF/WordPenalty.h
View File

@ -21,13 +21,13 @@ public:
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
};

16
moses2/HypothesisBase.cpp
View File

@ -29,15 +29,15 @@ HypothesisBase::HypothesisBase(MemPool &pool, const System &system)
// FF states
const std::vector<const StatefulFeatureFunction*> &sfffs =
system.featureFunctions.GetStatefulFeatureFunctions();
system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
m_ffStates = (FFState **) pool.Allocate(sizeof(FFState*) * numStatefulFFs);
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = sfff->BlankState(pool, system);
m_ffStates[statefulInd] = state;
}
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = sfff->BlankState(pool, system);
m_ffStates[statefulInd] = state;
}
}
size_t HypothesisBase::hash() const
@ -48,7 +48,7 @@ size_t HypothesisBase::hash() const
size_t HypothesisBase::hash(size_t seed) const
{
size_t numStatefulFFs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
// states
for (size_t i = 0; i < numStatefulFFs; ++i) {
@ -63,7 +63,7 @@ size_t HypothesisBase::hash(size_t seed) const
bool HypothesisBase::operator==(const HypothesisBase &other) const
{
size_t numStatefulFFs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
// states
for (size_t i = 0; i < numStatefulFFs; ++i) {

34
moses2/HypothesisBase.h
View File

@ -20,28 +20,31 @@ class Scores;
class HypothesisBase
{
public:
virtual ~HypothesisBase()
{
virtual ~HypothesisBase() {
}
inline ManagerBase &GetManager() const
{
inline ManagerBase &GetManager() const {
return *m_mgr;
}
template<typename T>
const T &Cast() const
{ return static_cast<const T&>(*this); }
const T &Cast() const {
return static_cast<const T&>(*this);
}
const Scores &GetScores() const
{ return *m_scores; }
Scores &GetScores()
{ return *m_scores; }
const Scores &GetScores() const {
return *m_scores;
}
Scores &GetScores() {
return *m_scores;
}
const FFState *GetState(size_t ind) const
{ return m_ffStates[ind]; }
FFState *GetState(size_t ind)
{ return m_ffStates[ind]; }
const FFState *GetState(size_t ind) const {
return m_ffStates[ind];
}
FFState *GetState(size_t ind) {
return m_ffStates[ind];
}
virtual size_t hash() const;
virtual size_t hash(size_t seed) const;
@ -64,8 +67,7 @@ protected:
class HypothesisFutureScoreOrderer
{
public:
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const
{
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const {
return a->GetFutureScore() > b->GetFutureScore();
}
};

195
moses2/HypothesisColl.cpp
View File

@ -19,8 +19,8 @@ namespace Moses2
{
HypothesisColl::HypothesisColl(const ManagerBase &mgr)
:m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool()))
,m_sortedHypos(NULL)
:m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool()))
,m_sortedHypos(NULL)
{
m_bestScore = -std::numeric_limits<float>::infinity();
m_worstScore = std::numeric_limits<float>::infinity();
@ -28,29 +28,29 @@ HypothesisColl::HypothesisColl(const ManagerBase &mgr)
const HypothesisBase *HypothesisColl::GetBestHypo() const
{
if (GetSize() == 0) {
return NULL;
}
if (m_sortedHypos) {
return (*m_sortedHypos)[0];
}
if (GetSize() == 0) {
return NULL;
}
if (m_sortedHypos) {
return (*m_sortedHypos)[0];
}
SCORE bestScore = -std::numeric_limits<SCORE>::infinity();
const HypothesisBase *bestHypo;
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
if (hypo->GetFutureScore() > bestScore) {
bestScore = hypo->GetFutureScore();
bestHypo = hypo;
}
}
return bestHypo;
SCORE bestScore = -std::numeric_limits<SCORE>::infinity();
const HypothesisBase *bestHypo;
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
if (hypo->GetFutureScore() > bestScore) {
bestScore = hypo->GetFutureScore();
bestHypo = hypo;
}
}
return bestHypo;
}
void HypothesisColl::Add(
const ManagerBase &mgr,
HypothesisBase *hypo,
Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
const ManagerBase &mgr,
HypothesisBase *hypo,
Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
{
size_t maxStackSize = mgr.system.options.search.stack_size;
@ -76,105 +76,100 @@ void HypothesisColl::Add(
return;
}
StackAdd added = Add(hypo);
StackAdd added = Add(hypo);
size_t nbestSize = mgr.system.options.nbest.nbest_size;
if (nbestSize) {
arcLists.AddArc(added.added, hypo, added.other);
}
else {
if (added.added) {
size_t nbestSize = mgr.system.options.nbest.nbest_size;
if (nbestSize) {
arcLists.AddArc(added.added, hypo, added.other);
} else {
if (added.added) {
if (added.other) {
hypoRecycle.Recycle(added.other);
}
}
else {
} else {
hypoRecycle.Recycle(hypo);
}
}
}
}
// update beam variables
if (added.added) {
if (added.added) {
if (futureScore > m_bestScore) {
m_bestScore = futureScore;
float beamWidth = mgr.system.options.search.beam_width;
if ( m_bestScore + beamWidth > m_worstScore ) {
m_worstScore = m_bestScore + beamWidth;
}
}
else if (GetSize() <= maxStackSize && futureScore < m_worstScore) {
} else if (GetSize() <= maxStackSize && futureScore < m_worstScore) {
m_worstScore = futureScore;
}
}
}
}
StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
{
std::pair<_HCType::iterator, bool> addRet = m_coll.insert(hypo);
//cerr << endl << "new=" << hypo->Debug(hypo->GetManager().system) << endl;
std::pair<_HCType::iterator, bool> addRet = m_coll.insert(hypo);
//cerr << endl << "new=" << hypo->Debug(hypo->GetManager().system) << endl;
// CHECK RECOMBINATION
if (addRet.second) {
// equiv hypo doesn't exists
// CHECK RECOMBINATION
if (addRet.second) {
// equiv hypo doesn't exists
//cerr << "Added " << hypo << endl;
return StackAdd(true, NULL);
}
else {
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
//cerr << "hypoExisting=" << hypoExisting->Debug(hypo->GetManager().system) << endl;
return StackAdd(true, NULL);
} else {
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
//cerr << "hypoExisting=" << hypoExisting->Debug(hypo->GetManager().system) << endl;
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
// incoming hypo is better than the one we have
const HypothesisBase * const &hypoExisting1 = *addRet.first;
const HypothesisBase *&hypoExisting2 =
const_cast<const HypothesisBase *&>(hypoExisting1);
hypoExisting2 = hypo;
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
// incoming hypo is better than the one we have
const HypothesisBase * const &hypoExisting1 = *addRet.first;
const HypothesisBase *&hypoExisting2 =
const_cast<const HypothesisBase *&>(hypoExisting1);
hypoExisting2 = hypo;
//cerr << "Added " << hypo << " dicard existing " << hypoExisting2 << endl;
return StackAdd(true, hypoExisting);
}
else {
// already storing the best hypo. discard incoming hypo
return StackAdd(true, hypoExisting);
} else {
// already storing the best hypo. discard incoming hypo
//cerr << "Keep existing " << hypoExisting << " dicard new " << hypo << endl;
return StackAdd(false, hypoExisting);
}
}
return StackAdd(false, hypoExisting);
}
}
//assert(false);
//assert(false);
}
const Hypotheses &HypothesisColl::GetSortedAndPrunedHypos(
const ManagerBase &mgr,
ArcLists &arcLists) const
const ManagerBase &mgr,
ArcLists &arcLists) const
{
if (m_sortedHypos == NULL) {
// create sortedHypos first
MemPool &pool = mgr.GetPool();
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool,
m_coll.size());
if (m_sortedHypos == NULL) {
// create sortedHypos first
MemPool &pool = mgr.GetPool();
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool,
m_coll.size());
SortHypos(mgr, m_sortedHypos->GetArray());
SortHypos(mgr, m_sortedHypos->GetArray());
// prune
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
// prune
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
size_t maxStackSize = mgr.system.options.search.stack_size;
if (maxStackSize && m_sortedHypos->size() > maxStackSize) {
for (size_t i = maxStackSize; i < m_sortedHypos->size(); ++i) {
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
recycler.Recycle(hypo);
size_t maxStackSize = mgr.system.options.search.stack_size;
if (maxStackSize && m_sortedHypos->size() > maxStackSize) {
for (size_t i = maxStackSize; i < m_sortedHypos->size(); ++i) {
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
recycler.Recycle(hypo);
// delete from arclist
if (mgr.system.options.nbest.nbest_size) {
arcLists.Delete(hypo);
}
}
m_sortedHypos->resize(maxStackSize);
}
// delete from arclist
if (mgr.system.options.nbest.nbest_size) {
arcLists.Delete(hypo);
}
}
m_sortedHypos->resize(maxStackSize);
}
}
}
return *m_sortedHypos;
return *m_sortedHypos;
}
void HypothesisColl::PruneHypos(const ManagerBase &mgr, ArcLists &arcLists)
@ -221,7 +216,7 @@ void HypothesisColl::SortHypos(const ManagerBase &mgr, const HypothesisBase **so
cerr << endl;
*/
size_t ind = 0;
BOOST_FOREACH(const HypothesisBase *hypo, m_coll){
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
sortedHypos[ind] = hypo;
++ind;
}
@ -229,11 +224,9 @@ void HypothesisColl::SortHypos(const ManagerBase &mgr, const HypothesisBase **so
size_t indMiddle;
if (maxStackSize == 0) {
indMiddle = GetSize();
}
else if (GetSize() > maxStackSize) {
} else if (GetSize() > maxStackSize) {
indMiddle = maxStackSize;
}
else {
} else {
// GetSize() <= maxStackSize
indMiddle = GetSize();
}
@ -241,10 +234,10 @@ void HypothesisColl::SortHypos(const ManagerBase &mgr, const HypothesisBase **so
const HypothesisBase **iterMiddle = sortedHypos + indMiddle;
std::partial_sort(
sortedHypos,
iterMiddle,
sortedHypos + GetSize(),
HypothesisFutureScoreOrderer());
sortedHypos,
iterMiddle,
sortedHypos + GetSize(),
HypothesisFutureScoreOrderer());
/*
cerr << "sorted hypos: ";
@ -266,8 +259,8 @@ void HypothesisColl::Delete(const HypothesisBase *hypo)
void HypothesisColl::Clear()
{
m_sortedHypos = NULL;
m_coll.clear();
m_sortedHypos = NULL;
m_coll.clear();
m_bestScore = -std::numeric_limits<float>::infinity();
m_worstScore = std::numeric_limits<float>::infinity();
@ -275,13 +268,13 @@ void HypothesisColl::Clear()
std::string HypothesisColl::Debug(const System &system) const
{
stringstream out;
BOOST_FOREACH (const HypothesisBase *hypo, m_coll) {
out << hypo->Debug(system);
out << std::endl << std::endl;
}
stringstream out;
BOOST_FOREACH (const HypothesisBase *hypo, m_coll) {
out << hypo->Debug(system);
out << std::endl << std::endl;
}
return out.str();
return out.str();
}
} /* namespace Moses2 */

22
moses2/HypothesisColl.h
View File

@ -26,24 +26,24 @@ public:
HypothesisColl(const ManagerBase &mgr);
void Add(const ManagerBase &mgr,
HypothesisBase *hypo,
Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
HypothesisBase *hypo,
Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
size_t GetSize() const
{ return m_coll.size(); }
size_t GetSize() const {
return m_coll.size();
}
void Clear();
const Hypotheses &GetSortedAndPrunedHypos(
const ManagerBase &mgr,
ArcLists &arcLists) const;
const ManagerBase &mgr,
ArcLists &arcLists) const;
const HypothesisBase *GetBestHypo() const;
template<typename T>
const T *GetBestHypo() const
{
const T *GetBestHypo() const {
const HypothesisBase *hypo = GetBestHypo();
return hypo ? &hypo->Cast<T>() : NULL;
}
@ -54,8 +54,8 @@ public:
protected:
typedef boost::unordered_set<const HypothesisBase*,
UnorderedComparer<HypothesisBase>, UnorderedComparer<HypothesisBase>,
MemPoolAllocator<const HypothesisBase*> > _HCType;
UnorderedComparer<HypothesisBase>, UnorderedComparer<HypothesisBase>,
MemPoolAllocator<const HypothesisBase*> > _HCType;
_HCType m_coll;
mutable Hypotheses *m_sortedHypos;

4
moses2/InputPathBase.cpp
View File

@ -11,8 +11,8 @@
namespace Moses2
{
InputPathBase::InputPathBase(MemPool &pool,
const Range &range, size_t numPt, const InputPathBase *prefixPath) :
range(range), prefixPath(prefixPath)
const Range &range, size_t numPt, const InputPathBase *prefixPath) :
range(range), prefixPath(prefixPath)
{
}

2
moses2/InputPathBase.h
View File

@ -24,7 +24,7 @@ public:
Range range;
InputPathBase(MemPool &pool, const Range &range,
size_t numPt, const InputPathBase *prefixPath);
size_t numPt, const InputPathBase *prefixPath);
};

15
moses2/InputPathsBase.h
View File

@ -22,8 +22,7 @@ class InputPathsBase
{
typedef std::vector<InputPathBase*> Coll;
public:
InputPathsBase()
{
InputPathsBase() {
}
virtual ~InputPathsBase();
@ -31,21 +30,17 @@ public:
typedef Coll::iterator iterator;
typedef Coll::const_iterator const_iterator;
const_iterator begin() const
{
const_iterator begin() const {
return m_inputPaths.begin();
}
const_iterator end() const
{
const_iterator end() const {
return m_inputPaths.end();
}
iterator begin()
{
iterator begin() {
return m_inputPaths.begin();
}
iterator end()
{
iterator end() {
return m_inputPaths.end();
}

42
moses2/InputType.cpp
View File

@ -12,35 +12,35 @@ namespace Moses2
{
//////////////////////////////////////////////////////////////////////////////
InputType::XMLOption::XMLOption(MemPool &pool, const std::string &nodeName, size_t vStartPos)
:startPos(vStartPos)
,prob(0)
,m_entity(NULL)
:startPos(vStartPos)
,prob(0)
,m_entity(NULL)
{
m_nodeName = pool.Allocate<char>(nodeName.size() + 1);
strcpy(m_nodeName, nodeName.c_str());
m_nodeName = pool.Allocate<char>(nodeName.size() + 1);
strcpy(m_nodeName, nodeName.c_str());
}
void InputType::XMLOption::SetTranslation(MemPool &pool, const std::string &val)
{
m_translation = pool.Allocate<char>(val.size() + 1);
strcpy(m_translation, val.c_str());
m_translation = pool.Allocate<char>(val.size() + 1);
strcpy(m_translation, val.c_str());
}
void InputType::XMLOption::SetEntity(MemPool &pool, const std::string &val)
{
m_entity = pool.Allocate<char>(val.size() + 1);
strcpy(m_entity, val.c_str());
m_entity = pool.Allocate<char>(val.size() + 1);
strcpy(m_entity, val.c_str());
}
std::string InputType::XMLOption::Debug(const System &system) const
{
std::stringstream out;
out << "[" << startPos << "," << phraseSize << "]="
<< m_nodeName << ","
<< m_translation << ","
<< prob;
<< m_nodeName << ","
<< m_translation << ","
<< prob;
if (m_entity) {
out << "," << m_entity;
out << "," << m_entity;
}
return out.str();
}
@ -48,9 +48,9 @@ std::string InputType::XMLOption::Debug(const System &system) const
//////////////////////////////////////////////////////////////////////////////
InputType::InputType(MemPool &pool)
:m_reorderingConstraint(pool)
,m_xmlOptions(pool)
,m_xmlCoverageMap(pool)
:m_reorderingConstraint(pool)
,m_xmlOptions(pool)
,m_xmlCoverageMap(pool)
{
}
@ -64,18 +64,18 @@ void InputType::Init(const System &system, size_t size, int max_distortion)
m_reorderingConstraint.InitializeWalls(size, max_distortion);
if (system.options.input.xml_policy != XmlPassThrough) {
m_xmlCoverageMap.assign(size, false);
m_xmlCoverageMap.assign(size, false);
}
}
void InputType::AddXMLOption(const System &system, const XMLOption *xmlOption)
{
m_xmlOptions.push_back(xmlOption);
m_xmlOptions.push_back(xmlOption);
if (system.options.input.xml_policy != XmlPassThrough) {
for(size_t j = xmlOption->startPos; j < xmlOption->startPos + xmlOption->phraseSize; ++j) {
m_xmlCoverageMap[j]=true;
}
for(size_t j = xmlOption->startPos; j < xmlOption->startPos + xmlOption->phraseSize; ++j) {
m_xmlCoverageMap[j]=true;
}
}
}

48
moses2/InputType.h
View File

@ -20,29 +20,32 @@ public:
class XMLOption
{
public:
size_t startPos, phraseSize;
size_t startPos, phraseSize;
SCORE prob;
SCORE prob;
XMLOption(MemPool &pool, const std::string &nodeName, size_t vStartPos);
XMLOption(MemPool &pool, const std::string &nodeName, size_t vStartPos);
const char *GetNodeName() const
{ return m_nodeName; }
const char *GetNodeName() const {
return m_nodeName;
}
const char *GetTranslation() const
{ return m_translation; }
const char *GetTranslation() const {
return m_translation;
}
const char *GetEntity() const
{ return m_entity; }
const char *GetEntity() const {
return m_entity;
}
void SetTranslation(MemPool &pool, const std::string &val);
void SetEntity(MemPool &pool, const std::string &val);
void SetTranslation(MemPool &pool, const std::string &val);
void SetEntity(MemPool &pool, const std::string &val);
std::string Debug(const System &system) const;
std::string Debug(const System &system) const;
public:
char *m_nodeName;
char *m_translation;
char *m_entity;
char *m_nodeName;
char *m_translation;
char *m_entity;
};
@ -53,14 +56,17 @@ public:
virtual void Init(const System &system, size_t size, int max_distortion);
ReorderingConstraint &GetReorderingConstraint()
{ return m_reorderingConstraint; }
ReorderingConstraint &GetReorderingConstraint() {
return m_reorderingConstraint;
}
const ReorderingConstraint &GetReorderingConstraint() const
{ return m_reorderingConstraint; }
const ReorderingConstraint &GetReorderingConstraint() const {
return m_reorderingConstraint;
}
const Vector<const XMLOption*> &GetXMLOptions() const
{ return m_xmlOptions; }
const Vector<const XMLOption*> &GetXMLOptions() const {
return m_xmlOptions;
}
void AddXMLOption(const System &system, const XMLOption *xmlOption);

55
moses2/LM/GPULM.cpp
View File

@ -29,28 +29,23 @@ using namespace std;
namespace Moses2
{
struct GPULMState: public FFState
{
virtual std::string ToString() const
{
struct GPULMState: public FFState {
virtual std::string ToString() const {
return "GPULMState";
}
virtual size_t hash() const
{
virtual size_t hash() const {
return boost::hash_value(lastWords);
}
virtual bool operator==(const FFState& other) const
{
virtual bool operator==(const FFState& other) const {
const GPULMState &otherCast = static_cast<const GPULMState&>(other);
bool ret = lastWords == otherCast.lastWords;
return ret;
}
void SetContext(const Context &context)
{
void SetContext(const Context &context) {
lastWords = context;
if (lastWords.size()) {
lastWords.resize(lastWords.size() - 1);
@ -63,7 +58,7 @@ struct GPULMState: public FFState
/////////////////////////////////////////////////////////////////
GPULM::GPULM(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
:StatefulFeatureFunction(startInd, line)
{
cerr << "GPULM::GPULM" << endl;
ReadParameters();
@ -93,15 +88,15 @@ FFState* GPULM::BlankState(MemPool &pool, const System &sys) const
//! return the state associated with the empty hypothesis for a given sentence
void GPULM::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
const InputType &input, const Hypothesis &hypo) const
{
GPULMState &stateCast = static_cast<GPULMState&>(state);
stateCast.lastWords.push_back(m_bos);
}
void GPULM::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
if (targetPhrase.GetSize() == 0) {
return;
@ -120,8 +115,7 @@ void GPULM::EvaluateInIsolation(MemPool &pool, const System &system,
if (context.size() == m_order) {
//std::pair<SCORE, void*> fromScoring = Score(context);
//score += fromScoring.first;
}
else {
} else {
//std::pair<SCORE, void*> fromScoring = Score(context);
//nonFullScore += fromScoring.first;
}
@ -130,33 +124,30 @@ void GPULM::EvaluateInIsolation(MemPool &pool, const System &system,
}
void GPULM::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
UTIL_THROW2("Not implemented");
}
void GPULM::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
UTIL_THROW2("Not implemented");
}
void GPULM::SetParameter(const std::string& key,
const std::string& value)
const std::string& value)
{
//cerr << "key=" << key << " " << value << endl;
if (key == "path") {
m_path = value;
}
else if (key == "order") {
} else if (key == "order") {
m_order = Scan<size_t>(value);
}
else if (key == "factor") {
} else if (key == "factor") {
m_factorType = Scan<FactorType>(value);
}
else {
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
@ -164,8 +155,8 @@ void GPULM::SetParameter(const std::string& key,
}
void GPULM::EvaluateWhenAppliedBatch(
const System &system,
const Batch &batch) const
const System &system,
const Batch &batch) const
{
// create list of ngrams
std::vector<std::pair<Hypothesis*, Context> > contexts;
@ -219,7 +210,7 @@ void GPULM::CreateNGram(std::vector<std::pair<Hypothesis*, Context> > &contexts,
}
void GPULM::ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const
const Factor *factor) const
{
if (context.size() < m_order) {
context.resize(context.size() + 1);
@ -239,8 +230,8 @@ SCORE GPULM::Score(const Context &context) const
}
void GPULM::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
UTIL_THROW2("Not implemented");
}

29
moses2/LM/GPULM.h
View File

@ -33,35 +33,35 @@ public:
virtual void Load(System &system);
void SetParameter(const std::string& key,
const std::string& value);
const std::string& value);
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenAppliedBatch(
const System &system,
const Batch &batch) const;
const System &system,
const Batch &batch) const;
protected:
std::string m_path;
@ -71,8 +71,7 @@ protected:
const Factor *m_eos;
size_t m_order;
inline lm::WordIndex TranslateID(const Word &word) const
{
inline lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word[m_factorType]->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
@ -83,7 +82,7 @@ protected:
void CreateNGram(std::vector<std::pair<Hypothesis*, Context> > &contexts, Hypothesis &hypo) const;
void ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const;
const Factor *factor) const;
SCORE Score(const Context &context) const;
};

155
moses2/LM/KENLM.cpp
View File

@ -28,23 +28,19 @@ using namespace std;
namespace Moses2
{
struct KenLMState: public FFState
{
struct KenLMState: public FFState {
lm::ngram::State state;
virtual size_t hash() const
{
virtual size_t hash() const {
size_t ret = hash_value(state);
return ret;
}
virtual bool operator==(const FFState& o) const
{
virtual bool operator==(const FFState& o) const {
const KenLMState &other = static_cast<const KenLMState &>(o);
bool ret = state == other.state;
return ret;
}
virtual std::string ToString() const
{
virtual std::string ToString() const {
stringstream ss;
for (size_t i = 0; i < state.Length(); ++i) {
ss << state.words[i] << " ";
@ -77,9 +73,8 @@ public:
return ret;
}
virtual std::string ToString() const
{
return "LanguageModelChartStateKenLM";
virtual std::string ToString() const {
return "LanguageModelChartStateKenLM";
}
private:
@ -91,13 +86,11 @@ class MappingBuilder: public lm::EnumerateVocab
{
public:
MappingBuilder(FactorCollection &factorCollection, System &system,
std::vector<lm::WordIndex> &mapping) :
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping)
{
std::vector<lm::WordIndex> &mapping) :
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping) {
}
void Add(lm::WordIndex index, const StringPiece &str)
{
void Add(lm::WordIndex index, const StringPiece &str) {
std::size_t factorId = m_factorCollection.AddFactor(str, m_system, false)->GetId();
if (m_mapping.size() <= factorId) {
// 0 is <unk> :-)
@ -115,10 +108,10 @@ private:
/////////////////////////////////////////////////////////////////
template<class Model>
KENLM<Model>::KENLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method) :
StatefulFeatureFunction(startInd, line), m_path(file), m_factorType(
factorType), m_load_method(load_method)
const std::string &file, FactorType factorType,
util::LoadMethod load_method) :
StatefulFeatureFunction(startInd, line), m_path(file), m_factorType(
factorType), m_load_method(load_method)
{
ReadParameters();
}
@ -154,8 +147,7 @@ FFState* KENLM<Model>::BlankState(MemPool &pool, const System &sys) const
FFState *ret;
if (sys.isPb) {
ret = new (pool.Allocate<KenLMState>()) KenLMState();
}
else {
} else {
ret = new (pool.Allocate<LanguageModelChartStateKenLM>()) LanguageModelChartStateKenLM();
}
return ret;
@ -164,7 +156,7 @@ FFState* KENLM<Model>::BlankState(MemPool &pool, const System &sys) const
//! return the state associated with the empty hypothesis for a given sentence
template<class Model>
void KENLM<Model>::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
const InputType &input, const Hypothesis &hypo) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
stateCast.state = m_ngram->BeginSentenceState();
@ -172,8 +164,8 @@ void KENLM<Model>::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
template<class Model>
void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
// contains factors used by this LM
float fullScore, nGramScore;
@ -193,22 +185,21 @@ void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system,
estimateScoresVec[0] = estimateScore;
estimateScoresVec[1] = 0;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScoresVec);
estimateScoresVec);
estimatedScore += weightedScore;
}
else {
} else {
scores.PlusEquals(system, *this, nGramScore);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScore);
estimateScore);
estimatedScore += weightedScore;
}
}
template<class Model>
void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
// contains factors used by this LM
float fullScore, nGramScore;
@ -232,29 +223,28 @@ void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system, cons
estimateScoresVec[0] = estimateScore;
estimateScoresVec[1] = 0;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScoresVec);
estimateScoresVec);
estimatedScore += weightedScore;
}
else {
} else {
scores.PlusEquals(system, *this, nGramScore);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScore);
estimateScore);
estimatedScore += weightedScore;
}
}
template<class Model>
void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
const System &system = mgr.system;
const lm::ngram::State &in_state =
static_cast<const KenLMState&>(prevState).state;
static_cast<const KenLMState&>(prevState).state;
if (!hypo.GetTargetPhrase().GetSize()) {
stateCast.state = in_state;
@ -271,11 +261,11 @@ void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
typename Model::State *state0 = &stateCast.state, *state1 = &aux_state;
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)),
*state0);
*state0);
++position;
for (; position < adjust_end; ++position) {
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)),
*state1);
*state1);
std::swap(state0, state1);
}
@ -284,15 +274,13 @@ void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
score += m_ngram->FullScoreForgotState(&indices.front(), last,
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
}
else if (adjust_end < end) {
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
} else if (adjust_end < end) {
// Get state after adding a long phrase.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
m_ngram->GetState(&indices.front(), last, stateCast.state);
}
else if (state0 != &stateCast.state) {
} else if (state0 != &stateCast.state) {
// Short enough phrase that we can just reuse the state.
stateCast.state = *state0;
}
@ -305,15 +293,14 @@ void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(system, *this, scoresVec);
}
else {
} else {
scores.PlusEquals(system, *this, score);
}
}
template<class Model>
void KENLM<Model>::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore,
float &ngramScore, std::size_t &oovCount) const
float &ngramScore, std::size_t &oovCount) const
{
fullScore = 0;
ngramScore = 0;
@ -328,8 +315,7 @@ void KENLM<Model>::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScor
if (m_bos == phrase[0][m_factorType]) {
scorer.BeginSentence();
position = 1;
}
else {
} else {
position = 0;
}
@ -357,7 +343,7 @@ void KENLM<Model>::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScor
template<class Model>
void KENLM<Model>::CalcScore(const Phrase<SCFG::Word> &phrase, float &fullScore,
float &ngramScore, std::size_t &oovCount) const
float &ngramScore, std::size_t &oovCount) const
{
fullScore = 0;
ngramScore = 0;
@ -411,7 +397,7 @@ void KENLM<Model>::CalcScore(const Phrase<SCFG::Word> &phrase, float &fullScore,
// Convert last words of hypothesis into vocab ids, returning an end pointer.
template<class Model>
lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo,
lm::WordIndex *indices) const
lm::WordIndex *indices) const
{
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
@ -428,8 +414,8 @@ lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo,
template<class Model>
void KENLM<Model>::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
LanguageModelChartStateKenLM &newState = static_cast<LanguageModelChartStateKenLM&>(state);
lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState.GetChartState());
@ -511,48 +497,38 @@ FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
for (; argument; ++argument) {
const char *equals = std::find(argument->data(),
argument->data() + argument->size(), '=');
argument->data() + argument->size(), '=');
UTIL_THROW_IF2(equals == argument->data() + argument->size(),
"Expected = in KenLM argument " << *argument);
"Expected = in KenLM argument " << *argument);
StringPiece name(argument->data(), equals - argument->data());
StringPiece value(equals + 1,
argument->data() + argument->size() - equals - 1);
argument->data() + argument->size() - equals - 1);
if (name == "factor") {
factorType = boost::lexical_cast<FactorType>(value);
}
else if (name == "order") {
} else if (name == "order") {
// Ignored
}
else if (name == "path") {
} else if (name == "path") {
filePath.assign(value.data(), value.size());
}
else if (name == "lazyken") {
} else if (name == "lazyken") {
// deprecated: use load instead.
load_method =
boost::lexical_cast<bool>(value) ?
util::LAZY : util::POPULATE_OR_READ;
}
else if (name == "load") {
boost::lexical_cast<bool>(value) ?
util::LAZY : util::POPULATE_OR_READ;
} else if (name == "load") {
if (value == "lazy") {
load_method = util::LAZY;
}
else if (value == "populate_or_lazy") {
} else if (value == "populate_or_lazy") {
load_method = util::POPULATE_OR_LAZY;
}
else if (value == "populate_or_read" || value == "populate") {
} else if (value == "populate_or_read" || value == "populate") {
load_method = util::POPULATE_OR_READ;
}
else if (value == "read") {
} else if (value == "read") {
load_method = util::READ;
}
else if (value == "parallel_read") {
} else if (value == "parallel_read") {
load_method = util::PARALLEL_READ;
}
else {
} else {
UTIL_THROW2("Unknown KenLM load method " << value);
}
}
else {
} else {
// pass to base class to interpret
line << " " << name << "=" << value;
}
@ -562,38 +538,37 @@ FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
}
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method)
const std::string &file, FactorType factorType,
util::LoadMethod load_method)
{
lm::ngram::ModelType model_type;
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
switch (model_type) {
case lm::ngram::PROBING:
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file,
factorType, load_method);
factorType, load_method);
case lm::ngram::REST_PROBING:
return new KENLM<lm::ngram::RestProbingModel>(startInd, line, file,
factorType, load_method);
factorType, load_method);
case lm::ngram::TRIE:
return new KENLM<lm::ngram::TrieModel>(startInd, line, file, factorType,
load_method);
load_method);
case lm::ngram::QUANT_TRIE:
return new KENLM<lm::ngram::QuantTrieModel>(startInd, line, file,
factorType, load_method);
factorType, load_method);
case lm::ngram::ARRAY_TRIE:
return new KENLM<lm::ngram::ArrayTrieModel>(startInd, line, file,
factorType, load_method);
factorType, load_method);
case lm::ngram::QUANT_ARRAY_TRIE:
return new KENLM<lm::ngram::QuantArrayTrieModel>(startInd, line, file,
factorType, load_method);
factorType, load_method);
default:
UTIL_THROW2("Unrecognized kenlm model type " << model_type)
;
}
}
else {
} else {
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType,
load_method);
load_method);
}
}

31
moses2/LM/KENLM.h
View File

@ -19,15 +19,15 @@ class Word;
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig);
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method);
const std::string &file, FactorType factorType,
util::LoadMethod load_method);
template<class Model>
class KENLM: public StatefulFeatureFunction
{
public:
KENLM(size_t startInd, const std::string &line, const std::string &file,
FactorType factorType, util::LoadMethod load_method);
FactorType factorType, util::LoadMethod load_method);
virtual ~KENLM();
@ -37,25 +37,25 @@ public:
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
@ -67,13 +67,12 @@ protected:
boost::shared_ptr<Model> m_ngram;
void CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore, float &ngramScore,
std::size_t &oovCount) const;
std::size_t &oovCount) const;
void CalcScore(const Phrase<SCFG::Word> &phrase, float &fullScore, float &ngramScore,
std::size_t &oovCount) const;
std::size_t &oovCount) const;
inline lm::WordIndex TranslateID(const Word &word) const
{
inline lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word[m_factorType]->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}

114
moses2/LM/KENLMBatch.cpp
View File

@ -33,23 +33,19 @@ using namespace std;
namespace Moses2
{
struct KenLMState: public FFState
{
struct KenLMState: public FFState {
lm::ngram::State state;
virtual size_t hash() const
{
virtual size_t hash() const {
size_t ret = hash_value(state);
return ret;
}
virtual bool operator==(const FFState& o) const
{
virtual bool operator==(const FFState& o) const {
const KenLMState &other = static_cast<const KenLMState &>(o);
bool ret = state == other.state;
return ret;
}
virtual std::string ToString() const
{
virtual std::string ToString() const {
stringstream ss;
for (size_t i = 0; i < state.Length(); ++i) {
ss << state.words[i] << " ";
@ -64,13 +60,11 @@ class MappingBuilder: public lm::EnumerateVocab
{
public:
MappingBuilder(FactorCollection &factorCollection, System &system,
std::vector<lm::WordIndex> &mapping) :
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping)
{
std::vector<lm::WordIndex> &mapping) :
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping) {
}
void Add(lm::WordIndex index, const StringPiece &str)
{
void Add(lm::WordIndex index, const StringPiece &str) {
std::size_t factorId = m_factorCollection.AddFactor(str, m_system, false)->GetId();
if (m_mapping.size() <= factorId) {
// 0 is <unk> :-)
@ -87,8 +81,8 @@ private:
/////////////////////////////////////////////////////////////////
KENLMBatch::KENLMBatch(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
,m_numHypos(0)
:StatefulFeatureFunction(startInd, line)
,m_numHypos(0)
{
cerr << "KENLMBatch::KENLMBatch" << endl;
ReadParameters();
@ -126,15 +120,15 @@ FFState* KENLMBatch::BlankState(MemPool &pool, const System &sys) const
//! return the state associated with the empty hypothesis for a given sentence
void KENLMBatch::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
const InputType &input, const Hypothesis &hypo) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
stateCast.state = m_ngram->BeginSentenceState();
}
void KENLMBatch::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
// contains factors used by this LM
float fullScore, nGramScore;
@ -154,34 +148,33 @@ void KENLMBatch::EvaluateInIsolation(MemPool &pool, const System &system,
estimateScoresVec[0] = estimateScore;
estimateScoresVec[1] = 0;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScoresVec);
estimateScoresVec);
estimatedScore += weightedScore;
}
else {
} else {
scores.PlusEquals(system, *this, nGramScore);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScore);
estimateScore);
estimatedScore += weightedScore;
}
}
void KENLMBatch::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void KENLMBatch::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
const System &system = mgr.system;
const lm::ngram::State &in_state =
static_cast<const KenLMState&>(prevState).state;
static_cast<const KenLMState&>(prevState).state;
if (!hypo.GetTargetPhrase().GetSize()) {
stateCast.state = in_state;
@ -198,11 +191,11 @@ void KENLMBatch::EvaluateWhenApplied(const ManagerBase &mgr,
typename Model::State *state0 = &stateCast.state, *state1 = &aux_state;
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)),
*state0);
*state0);
++position;
for (; position < adjust_end; ++position) {
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)),
*state1);
*state1);
std::swap(state0, state1);
}
@ -211,15 +204,13 @@ void KENLMBatch::EvaluateWhenApplied(const ManagerBase &mgr,
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
score += m_ngram->FullScoreForgotState(&indices.front(), last,
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
}
else if (adjust_end < end) {
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
} else if (adjust_end < end) {
// Get state after adding a long phrase.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
m_ngram->GetState(&indices.front(), last, stateCast.state);
}
else if (state0 != &stateCast.state) {
} else if (state0 != &stateCast.state) {
// Short enough phrase that we can just reuse the state.
stateCast.state = *state0;
}
@ -232,14 +223,13 @@ void KENLMBatch::EvaluateWhenApplied(const ManagerBase &mgr,
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(system, *this, scoresVec);
}
else {
} else {
scores.PlusEquals(system, *this, score);
}
}
void KENLMBatch::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore,
float &ngramScore, std::size_t &oovCount) const
float &ngramScore, std::size_t &oovCount) const
{
fullScore = 0;
ngramScore = 0;
@ -254,8 +244,7 @@ void KENLMBatch::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore,
if (m_bos == phrase[0][m_factorType]) {
scorer.BeginSentence();
position = 1;
}
else {
} else {
position = 0;
}
@ -283,7 +272,7 @@ void KENLMBatch::CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore,
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *KENLMBatch::LastIDs(const Hypothesis &hypo,
lm::WordIndex *indices) const
lm::WordIndex *indices) const
{
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
@ -299,44 +288,34 @@ lm::WordIndex *KENLMBatch::LastIDs(const Hypothesis &hypo,
}
void KENLMBatch::SetParameter(const std::string& key,
const std::string& value)
const std::string& value)
{
//cerr << "key=" << key << " " << value << endl;
if (key == "path") {
m_path = value;
}
else if (key == "order") {
} else if (key == "order") {
// ignore
}
else if (key == "factor") {
} else if (key == "factor") {
m_factorType = Scan<FactorType>(value);
}
else if (key == "lazyken") {
} else if (key == "lazyken") {
m_load_method =
boost::lexical_cast<bool>(value) ?
util::LAZY : util::POPULATE_OR_READ;
}
else if (key == "load") {
boost::lexical_cast<bool>(value) ?
util::LAZY : util::POPULATE_OR_READ;
} else if (key == "load") {
if (value == "lazy") {
m_load_method = util::LAZY;
}
else if (value == "populate_or_lazy") {
} else if (value == "populate_or_lazy") {
m_load_method = util::POPULATE_OR_LAZY;
}
else if (value == "populate_or_read" || value == "populate") {
} else if (value == "populate_or_read" || value == "populate") {
m_load_method = util::POPULATE_OR_READ;
}
else if (value == "read") {
} else if (value == "read") {
m_load_method = util::READ;
}
else if (value == "parallel_read") {
} else if (value == "parallel_read") {
m_load_method = util::PARALLEL_READ;
}
else {
} else {
UTIL_THROW2("Unknown KenLM load method " << value);
}
}
else {
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
@ -344,7 +323,7 @@ void KENLMBatch::SetParameter(const std::string& key,
}
void KENLMBatch::EvaluateWhenAppliedBatch(
const Batch &batch) const
const Batch &batch) const
{
{
// write lock
@ -362,8 +341,7 @@ void KENLMBatch::EvaluateWhenAppliedBatch(
m_numHypos = 0;
m_threadNeeded.notify_all();
}
else {
} else {
boost::mutex::scoped_lock lock(m_mutex);
m_threadNeeded.wait(lock);
}
@ -380,8 +358,8 @@ void KENLMBatch::EvaluateWhenAppliedBatch() const
}
void KENLMBatch::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
UTIL_THROW2("Not implemented");
}

27
moses2/LM/KENLMBatch.h
View File

@ -33,34 +33,34 @@ public:
virtual void Load(System &system);
void SetParameter(const std::string& key,
const std::string& value);
const std::string& value);
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenAppliedBatch(
const Batch &batch) const;
const Batch &batch) const;
protected:
std::string m_path;
@ -73,10 +73,9 @@ protected:
boost::shared_ptr<Model> m_ngram;
void CalcScore(const Phrase<Moses2::Word> &phrase, float &fullScore, float &ngramScore,
std::size_t &oovCount) const;
std::size_t &oovCount) const;
inline lm::WordIndex TranslateID(const Word &word) const
{
inline lm::WordIndex TranslateID(const Word &word) const {
std::size_t factor = word[m_factorType]->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}

68
moses2/LM/LanguageModel.cpp
View File

@ -22,28 +22,24 @@ using namespace std;
namespace Moses2
{
struct LMState: public PointerState
{
struct LMState: public PointerState {
LMState() :
PointerState()
{
PointerState() {
// uninitialised
}
void Set(MemPool &pool, void *lms, const std::vector<const Factor*> &context)
{
void Set(MemPool &pool, void *lms, const std::vector<const Factor*> &context) {
lmstate = lms;
numWords = context.size();
lastWords = (const Factor**) pool.Allocate(
sizeof(const Factor*) * numWords);
sizeof(const Factor*) * numWords);
for (size_t i = 0; i < numWords; ++i) {
lastWords[i] = context[i];
}
}
void Init(MemPool &pool, const Factor *factor)
{
void Init(MemPool &pool, const Factor *factor) {
lmstate = NULL;
numWords = 1;
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*));
@ -56,7 +52,7 @@ struct LMState: public PointerState
////////////////////////////////////////////////////////////////////////////////////////
LanguageModel::LanguageModel(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line), m_oov(-100)
StatefulFeatureFunction(startInd, line), m_oov(-100)
{
ReadParameters();
}
@ -112,18 +108,15 @@ void LanguageModel::Load(System &system)
}
void LanguageModel::SetParameter(const std::string& key,
const std::string& value)
const std::string& value)
{
if (key == "path") {
m_path = value;
}
else if (key == "factor") {
} else if (key == "factor") {
m_factorType = Scan<FactorType>(value);
}
else if (key == "order") {
} else if (key == "order") {
m_order = Scan<size_t>(value);
}
else {
} else {
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -143,8 +136,8 @@ void LanguageModel::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
}
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
if (targetPhrase.GetSize() == 0) {
return;
@ -163,8 +156,7 @@ void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
if (context.size() == m_order) {
std::pair<SCORE, void*> fromScoring = Score(context);
score += fromScoring.first;
}
else {
} else {
std::pair<SCORE, void*> fromScoring = Score(context);
nonFullScore += fromScoring.first;
}
@ -176,14 +168,14 @@ void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
}
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const
{
}
void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const LMState &prevLMState = static_cast<const LMState &>(prevState);
size_t numWords = prevLMState.numWords;
@ -214,8 +206,7 @@ void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
score += fromScoring.first;
fromScoring.second = NULL;
context.clear();
}
else {
} else {
assert(context.size());
if (context.size() == m_order) {
context.resize(context.size() - 1);
@ -233,7 +224,7 @@ void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
}
void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const
const Factor *factor) const
{
if (context.size() < m_order) {
context.resize(context.size() + 1);
@ -248,7 +239,7 @@ void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context,
}
std::pair<SCORE, void*> LanguageModel::Score(
const std::vector<const Factor*> &context) const
const std::vector<const Factor*> &context) const
{
//cerr << "context=";
//DebugContext(context);
@ -260,8 +251,7 @@ std::pair<SCORE, void*> LanguageModel::Score(
if (node) {
ret.first = node->getValue().prob;
ret.second = (void*) node;
}
else {
} else {
SCORE backoff = 0;
std::vector<const Factor*> backOffContext(context.begin() + 1,
context.end());
@ -282,7 +272,7 @@ std::pair<SCORE, void*> LanguageModel::Score(
}
SCORE LanguageModel::BackoffScore(
const std::vector<const Factor*> &context) const
const std::vector<const Factor*> &context) const
{
//cerr << "backoff=";
//DebugContext(context);
@ -295,19 +285,17 @@ SCORE LanguageModel::BackoffScore(
if (stoppedAtInd == context.size()) {
// found entire ngram
ret = node.getValue().backoff;
}
else {
} else {
if (stoppedAtInd == 0) {
ret = m_oov;
stoppedAtInd = 1;
}
else {
} else {
ret = node.getValue().backoff;
}
// recursive
std::vector<const Factor*> backoff(context.begin() + stoppedAtInd,
context.end());
context.end());
ret += BackoffScore(backoff);
}
@ -315,7 +303,7 @@ SCORE LanguageModel::BackoffScore(
}
void LanguageModel::DebugContext(
const std::vector<const Factor*> &context) const
const std::vector<const Factor*> &context) const
{
for (size_t i = 0; i < context.size(); ++i) {
cerr << context[i]->GetString() << " ";
@ -324,8 +312,8 @@ void LanguageModel::DebugContext(
}
void LanguageModel::EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const
{
UTIL_THROW2("Not implemented");
}

37
moses2/LM/LanguageModel.h
View File

@ -17,24 +17,19 @@ namespace Moses2
{
////////////////////////////////////////////////////////////////////////////////////////
struct LMScores
{
LMScores()
{
struct LMScores {
LMScores() {
}
LMScores(const LMScores &copy) :
prob(copy.prob), backoff(copy.backoff)
{
prob(copy.prob), backoff(copy.backoff) {
}
LMScores(float inProb, float inBackoff) :
prob(inProb), backoff(inBackoff)
{
prob(inProb), backoff(inBackoff) {
}
void Debug(std::ostream &out, const System &system) const
{
void Debug(std::ostream &out, const System &system) const {
out << "(" << prob << "," << backoff << ")" << std::flush;
}
@ -54,25 +49,25 @@ public:
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
@ -85,9 +80,9 @@ protected:
const Factor *m_eos;
void ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const;
const Factor *factor) const;
std::pair<SCORE, void*> Score(
const std::vector<const Factor*> &context) const;
const std::vector<const Factor*> &context) const;
SCORE BackoffScore(const std::vector<const Factor*> &context) const;
void DebugContext(const std::vector<const Factor*> &context) const;

96
moses2/LM/LanguageModelDALM.cpp
View File

@ -76,8 +76,9 @@ public:
state.refresh();
}
virtual std::string ToString() const
{ return "DALM state"; }
virtual std::string ToString() const {
return "DALM state";
}
};
@ -89,29 +90,30 @@ inline void read_ini(const char *inifile, string &model, string &words, string &
getline(ifs, line);
while(ifs) {
unsigned int pos = line.find("=");
string key = line.substr(0, pos);
string value = line.substr(pos+1, line.size()-pos);
if(key=="MODEL") {
model = value;
} else if(key=="WORDS") {
words = value;
} else if(key=="WORDSTXT") {
wordstxt = value;
}
getline(ifs, line);
unsigned int pos = line.find("=");
string key = line.substr(0, pos);
string value = line.substr(pos+1, line.size()-pos);
if(key=="MODEL") {
model = value;
} else if(key=="WORDS") {
words = value;
} else if(key=="WORDSTXT") {
wordstxt = value;
}
getline(ifs, line);
}
}
/////////////////////////
LanguageModelDALM::LanguageModelDALM(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
:StatefulFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
LanguageModelDALM::~LanguageModelDALM() {
// TODO Auto-generated destructor stub
LanguageModelDALM::~LanguageModelDALM()
{
// TODO Auto-generated destructor stub
}
void LanguageModelDALM::Load(System &system)
@ -165,72 +167,72 @@ void LanguageModelDALM::CreateVocabMapping(const std::string &wordstxt, const Sy
string line;
std::size_t max_fid = 0;
while(getline(vocabStrm, line)) {
const Factor *factor = system.GetVocab().AddFactor(line, system);
std::size_t fid = factor->GetId();
DALM::VocabId wid = m_vocab->lookup(line.c_str());
const Factor *factor = system.GetVocab().AddFactor(line, system);
std::size_t fid = factor->GetId();
DALM::VocabId wid = m_vocab->lookup(line.c_str());
vlist.push_back(std::pair<std::size_t, DALM::VocabId>(fid, wid));
if(max_fid < fid) max_fid = fid;
vlist.push_back(std::pair<std::size_t, DALM::VocabId>(fid, wid));
if(max_fid < fid) max_fid = fid;
}
for(std::size_t i = 0; i < m_vocabMap.size(); i++) {
m_vocabMap[i] = m_vocab->unk();
m_vocabMap[i] = m_vocab->unk();
}
m_vocabMap.resize(max_fid+1, m_vocab->unk());
std::vector< std::pair<std::size_t, DALM::VocabId> >::iterator it = vlist.begin();
while(it != vlist.end()) {
std::pair<std::size_t, DALM::VocabId> &entry = *it;
m_vocabMap[entry.first] = entry.second;
std::pair<std::size_t, DALM::VocabId> &entry = *it;
m_vocabMap[entry.first] = entry.second;
++it;
++it;
}
}
void LanguageModelDALM::SetParameter(const std::string& key, const std::string& value)
{
if (key == "factor") {
m_factorType = Scan<FactorType>(value);
m_factorType = Scan<FactorType>(value);
} else if (key == "order") {
m_nGramOrder = Scan<size_t>(value);
m_nGramOrder = Scan<size_t>(value);
} else if (key == "path") {
m_filePath = value;
m_filePath = value;
} else {
StatefulFeatureFunction::SetParameter(key, value);
StatefulFeatureFunction::SetParameter(key, value);
}
m_ContextSize = m_nGramOrder-1;
}
FFState* LanguageModelDALM::BlankState(MemPool &pool, const System &sys) const
{
DALMState *state = new DALMState();
return state;
DALMState *state = new DALMState();
return state;
}
void LanguageModelDALM::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
{
DALMState &dalmState = static_cast<DALMState&>(state);
m_lm->init_state(dalmState.get_state());
}
void LanguageModelDALM::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhraseImpl &targetPhrase,
Scores &scores,
SCORE &estimatedScore) const
{
void LanguageModelDALM::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhraseImpl &targetPhrase,
Scores &scores,
SCORE &estimatedScore) const
{
}
}
void LanguageModelDALM::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
{
}

14
moses2/LM/LanguageModelDALM.h
View File

@ -36,20 +36,20 @@ public:
virtual FFState* BlankState(MemPool &pool, const System &sys) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
const TargetPhraseImpl &targetPhrase, Scores &scores,
SCORE &estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const SCFG::Manager &mgr,
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
const SCFG::Hypothesis &hypo, int featureID, Scores &scores,
FFState &state) const;
protected:
FactorType m_factorType;

44
moses2/Main.cpp
View File

@ -20,15 +20,15 @@ using namespace std;
int main(int argc, char** argv)
{
cerr << "Starting..." << endl;
cerr << "Starting..." << endl;
Moses2::Timer timer;
timer.start();
//Temp();
//Temp();
Moses2::Parameter params;
if (!params.LoadParam(argc, argv)) {
return EXIT_FAILURE;
return EXIT_FAILURE;
}
Moses2::System system(params);
timer.check("Loaded");
@ -45,8 +45,7 @@ int main(int argc, char** argv)
if (params.GetParam("server")) {
std::cerr << "RUN SERVER" << std::endl;
run_as_server(system);
}
else {
} else {
std::cerr << "RUN BATCH" << std::endl;
batch_run(params, system, pool);
}
@ -71,8 +70,7 @@ istream &GetInputStream(Moses2::Parameter &params)
if (vec && vec->size()) {
Moses2::InputFileStream *stream = new Moses2::InputFileStream(vec->at(0));
return *stream;
}
else {
} else {
return cin;
}
}
@ -86,7 +84,7 @@ void batch_run(Moses2::Parameter &params, Moses2::System &system, Moses2::Thread
string line;
while (getline(inStream, line)) {
//cerr << "line=" << line << endl;
boost::shared_ptr<Moses2::TranslationTask> task(new Moses2::TranslationTask(system, line, translationId));
boost::shared_ptr<Moses2::TranslationTask> task(new Moses2::TranslationTask(system, line, translationId));
//cerr << "START pool.Submit()" << endl;
pool.Submit(task);
@ -106,23 +104,23 @@ void batch_run(Moses2::Parameter &params, Moses2::System &system, Moses2::Thread
////////////////////////////////////////////////////////////////////////////////////////////////
void Temp()
{
Moses2::MemPool pool;
Moses2::MemPoolAllocator<int> a(pool);
Moses2::MemPool pool;
Moses2::MemPoolAllocator<int> a(pool);
boost::unordered_set<int, boost::hash<int>, std::equal_to<int>, Moses2::MemPoolAllocator<int> > s(a);
s.insert(3);
s.insert(4);
s.insert(3);
s.erase(3);
boost::unordered_set<int, boost::hash<int>, std::equal_to<int>, Moses2::MemPoolAllocator<int> > s(a);
s.insert(3);
s.insert(4);
s.insert(3);
s.erase(3);
boost::pool_allocator<int> alloc;
std::vector<int, boost::pool_allocator<int> > v(alloc);
for (int i = 0; i < 1000; ++i)
v.push_back(i);
boost::pool_allocator<int> alloc;
std::vector<int, boost::pool_allocator<int> > v(alloc);
for (int i = 0; i < 1000; ++i)
v.push_back(i);
v.clear();
boost::singleton_pool<boost::pool_allocator_tag, sizeof(int)>::
purge_memory();
v.clear();
boost::singleton_pool<boost::pool_allocator_tag, sizeof(int)>::
purge_memory();
abort();
abort();
}

3
moses2/Main.h
View File

@ -7,7 +7,8 @@
#pragma once
#include <iostream>
namespace Moses2 {
namespace Moses2
{
class Parameter;
class System;
class ThreadPool;

20
moses2/ManagerBase.cpp
View File

@ -21,14 +21,14 @@ using namespace std;
namespace Moses2
{
ManagerBase::ManagerBase(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId)
:system(sys)
,task(task)
,m_inputStr(inputStr)
,m_translationId(translationId)
,m_pool(NULL)
,m_systemPool(NULL)
,m_hypoRecycle(NULL)
const std::string &inputStr, long translationId)
:system(sys)
,task(task)
,m_inputStr(inputStr)
,m_translationId(translationId)
,m_pool(NULL)
,m_systemPool(NULL)
,m_hypoRecycle(NULL)
{
}
@ -37,10 +37,10 @@ ManagerBase::~ManagerBase()
system.featureFunctions.CleanUpAfterSentenceProcessing();
if (m_pool) {
GetPool().Reset();
GetPool().Reset();
}
if (m_hypoRecycle) {
GetHypoRecycle().Clear();
GetHypoRecycle().Clear();
}
}

27
moses2/ManagerBase.h
View File

@ -38,27 +38,32 @@ public:
mutable ArcLists arcLists;
ManagerBase(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId);
const std::string &inputStr, long translationId);
virtual ~ManagerBase();
virtual void Decode() = 0;
virtual std::string OutputBest() const = 0;
virtual std::string OutputNBest() = 0;
virtual std::string OutputTransOpt() = 0;
MemPool &GetPool() const
{ return *m_pool; }
MemPool &GetPool() const {
return *m_pool;
}
MemPool &GetSystemPool() const
{ return *m_systemPool; }
MemPool &GetSystemPool() const {
return *m_systemPool;
}
Recycler<HypothesisBase*> &GetHypoRecycle() const
{ return *m_hypoRecycle; }
Recycler<HypothesisBase*> &GetHypoRecycle() const {
return *m_hypoRecycle;
}
const InputType &GetInput() const
{ return *m_input; }
const InputType &GetInput() const {
return *m_input;
}
long GetTranslationId() const
{ return m_translationId; }
long GetTranslationId() const {
return m_translationId;
}
protected:
std::string m_inputStr;

10
moses2/MemPool.cpp
View File

@ -16,7 +16,7 @@ namespace Moses2
{
MemPool::Page::Page(std::size_t vSize) :
size(vSize)
size(vSize)
{
mem = (uint8_t*) util::MallocOrThrow(size);
end = mem + size;
@ -28,7 +28,7 @@ MemPool::Page::~Page()
}
////////////////////////////////////////////////////
MemPool::MemPool(size_t initSize) :
m_currSize(initSize), m_currPage(0)
m_currSize(initSize), m_currPage(0)
{
Page *page = new Page(m_currSize);
m_pages.push_back(page);
@ -57,16 +57,14 @@ uint8_t *MemPool::More(std::size_t size)
uint8_t *ret = page->mem;
current_ = ret + size;
return ret;
}
else {
} else {
// use existing page
Page &page = *m_pages[m_currPage];
if (size <= page.size) {
uint8_t *ret = page.mem;
current_ = ret + size;
return ret;
}
else {
} else {
// recursive call More()
return More(size);
}

51
moses2/MemPool.h
View File

@ -20,14 +20,12 @@ namespace Moses2
class MemPool
{
struct Page
{
struct Page {
uint8_t *mem;
uint8_t *end;
size_t size;
Page()
{
Page() {
}
Page(std::size_t size);
~Page();
@ -38,8 +36,7 @@ public:
~MemPool();
uint8_t *Allocate(std::size_t size)
{
uint8_t *Allocate(std::size_t size) {
size = (size + 3) & 0xfffffffc;
uint8_t *ret = current_;
@ -48,8 +45,7 @@ public:
Page &page = *m_pages[m_currPage];
if (current_ <= page.end) {
// return what we got
}
else {
} else {
ret = More(size);
}
return ret;
@ -57,15 +53,13 @@ public:
}
template<typename T>
T *Allocate()
{
T *Allocate() {
uint8_t *ret = Allocate(sizeof(T));
return (T*) ret;
}
template<typename T>
T *Allocate(size_t num)
{
T *Allocate(size_t num) {
uint8_t *ret = Allocate(sizeof(T) * num);
return (T*) ret;
}
@ -94,18 +88,15 @@ class ObjectPoolContiguous
public:
ObjectPoolContiguous(std::size_t initSize = 100000) :
m_size(0), m_actualSize(initSize)
{
m_size(0), m_actualSize(initSize) {
m_vec = (T*) malloc(sizeof(T) * initSize);
}
~ObjectPoolContiguous()
{
~ObjectPoolContiguous() {
free(m_vec);
}
void Add(T &obj)
{
void Add(T &obj) {
if (m_size >= m_actualSize) {
//std::cerr << std::endl << "MORE " << m_size << std::endl;
m_actualSize *= 2;
@ -116,46 +107,38 @@ public:
++m_size;
}
bool IsEmpty() const
{
bool IsEmpty() const {
return m_size == 0;
}
void Reset()
{
void Reset() {
m_size = 0;
}
// vector op
size_t GetSize() const
{
size_t GetSize() const {
return m_size;
}
const T& operator[](size_t ind) const
{
const T& operator[](size_t ind) const {
return m_vec[ind];
}
// stack op
const T &Get() const
{
const T &Get() const {
return m_vec[m_size - 1];
}
void Pop()
{
void Pop() {
--m_size;
}
T *GetData()
{
T *GetData() {
return m_vec;
}
template<typename ORDERER>
void Sort(const ORDERER &orderer)
{
void Sort(const ORDERER &orderer) {
std::sort(m_vec, m_vec + m_size, orderer);
}

27
moses2/MemPoolAllocator.h
View File

@ -17,51 +17,42 @@ public:
typedef std::ptrdiff_t difference_type;
template<class U>
struct rebind
{
struct rebind {
typedef MemPoolAllocator<U> other;
};
MemPoolAllocator(Moses2::MemPool &pool) :
m_pool(pool)
{
m_pool(pool) {
}
MemPoolAllocator(const MemPoolAllocator &other) :
m_pool(other.m_pool)
{
m_pool(other.m_pool) {
}
template<class U>
MemPoolAllocator(const MemPoolAllocator<U>& other) :
m_pool(other.m_pool)
{
m_pool(other.m_pool) {
}
size_type max_size() const
{
size_type max_size() const {
return std::numeric_limits<size_type>::max();
}
void deallocate(pointer p, size_type n)
{
void deallocate(pointer p, size_type n) {
//std::cerr << "deallocate " << p << " " << n << std::endl;
}
pointer allocate(size_type n, std::allocator<void>::const_pointer hint = 0)
{
pointer allocate(size_type n, std::allocator<void>::const_pointer hint = 0) {
//std::cerr << "allocate " << n << " " << hint << std::endl;
pointer ret = m_pool.Allocate<T>(n);
return ret;
}
void construct(pointer p, const_reference val)
{
void construct(pointer p, const_reference val) {
//std::cerr << "construct " << p << " " << n << std::endl;
new ((void *) p) T(val);
}
void destroy(pointer p)
{
void destroy(pointer p) {
//std::cerr << "destroy " << p << " " << n << std::endl;
}

20
moses2/MorphoTrie/MorphTrie.h
View File

@ -11,24 +11,23 @@ template<class KeyClass, class ValueClass>
class MorphTrie
{
public:
MorphTrie()
{
MorphTrie() {
}
Node<KeyClass, ValueClass>* insert(const std::vector<KeyClass>& word,
const ValueClass& value);
const ValueClass& value);
const Node<KeyClass, ValueClass>* getNode(
const std::vector<KeyClass>& words) const;
const std::vector<KeyClass>& words) const;
const Node<KeyClass, ValueClass> &getNode(const std::vector<KeyClass>& words,
size_t &stoppedAtInd) const;
std::vector<const Node<KeyClass, ValueClass>*> getNodes(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
private:
Node<KeyClass, ValueClass> root;
};
template<class KeyClass, class ValueClass>
Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(
const std::vector<KeyClass>& word, const ValueClass& value)
const std::vector<KeyClass>& word, const ValueClass& value)
{
Node<KeyClass, ValueClass>* cNode = &root;
for (size_t i = 0; i < word.size(); ++i) {
@ -41,7 +40,7 @@ Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(
const std::vector<KeyClass>& words) const
const std::vector<KeyClass>& words) const
{
size_t stoppedAtInd;
const Node<KeyClass, ValueClass> &ret = getNode(words, stoppedAtInd);
@ -53,7 +52,7 @@ const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
{
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
for (size_t i = 0; i < words.size(); ++i) {
@ -72,7 +71,7 @@ const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(
template<class KeyClass, class ValueClass>
std::vector<const Node<KeyClass, ValueClass>*> MorphTrie<KeyClass, ValueClass>::getNodes(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
{
std::vector<const Node<KeyClass, ValueClass>*> ret;
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
@ -84,8 +83,7 @@ std::vector<const Node<KeyClass, ValueClass>*> MorphTrie<KeyClass, ValueClass>::
if (newNode == NULL) {
stoppedAtInd = i;
return ret;
}
else {
} else {
ret.push_back(newNode);
}
prevNode = newNode;

22
moses2/MorphoTrie/Node.h
View File

@ -12,28 +12,23 @@ template<class KeyClass, class ValueClass>
class Node
{
public:
Node()
{
Node() {
}
Node(const ValueClass& value) :
m_value(value)
{
m_value(value) {
}
~Node();
void setKey(const KeyClass& key);
void setValue(const ValueClass& value)
{
void setValue(const ValueClass& value) {
m_value = value;
}
Node* findSub(const KeyClass& key);
const Node* findSub(const KeyClass& key) const;
Node *addSubnode(const KeyClass& cKey)
{
Node *addSubnode(const KeyClass& cKey) {
Node *node = findSub(cKey);
if (node) {
return node;
}
else {
} else {
node = new Node();
subNodes[cKey] = node;
return node;
@ -41,8 +36,7 @@ public:
}
std::vector<Node*> getSubnodes();
const ValueClass &getValue() const
{
const ValueClass &getValue() const {
return m_value;
}
@ -64,7 +58,7 @@ Node<KeyClass, ValueClass>::~Node()
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
const KeyClass& cKey) const
const KeyClass& cKey) const
{
typename boost::unordered_map<KeyClass, Node*>::const_iterator iter;
iter = subNodes.find(cKey);
@ -77,7 +71,7 @@ const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
template<class KeyClass, class ValueClass>
Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
const KeyClass& cKey)
const KeyClass& cKey)
{
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
iter = subNodes.find(cKey);

29
moses2/Phrase.h
View File

@ -32,17 +32,16 @@ template<typename WORD>
class Phrase
{
public:
virtual ~Phrase()
{
virtual ~Phrase() {
}
virtual const WORD& operator[](size_t pos) const = 0;
virtual size_t GetSize() const = 0;
virtual const WORD& Back() const
{ return (*this)[GetSize() - 1]; }
virtual const WORD& Back() const {
return (*this)[GetSize() - 1];
}
virtual size_t hash() const
{
virtual size_t hash() const {
size_t seed = 0;
for (size_t i = 0; i < GetSize(); ++i) {
@ -54,8 +53,7 @@ public:
return seed;
}
virtual bool operator==(const Phrase &compare) const
{
virtual bool operator==(const Phrase &compare) const {
if (GetSize() != compare.GetSize()) {
return false;
}
@ -71,13 +69,11 @@ public:
return true;
}
virtual bool operator!=(const Phrase &compare) const
{
virtual bool operator!=(const Phrase &compare) const {
return !((*this) == compare);
}
virtual std::string GetString(const FactorList &factorTypes) const
{
virtual std::string GetString(const FactorList &factorTypes) const {
if (GetSize() == 0) {
return "";
}
@ -95,8 +91,7 @@ public:
virtual SubPhrase<WORD> GetSubPhrase(size_t start, size_t size) const = 0;
virtual std::string Debug(const System &system) const
{
virtual std::string Debug(const System &system) const {
std::stringstream out;
size_t size = GetSize();
if (size) {
@ -110,8 +105,7 @@ public:
return out.str();
}
virtual void OutputToStream(const System &system, std::ostream &out) const
{
virtual void OutputToStream(const System &system, std::ostream &out) const {
size_t size = GetSize();
if (size) {
(*this)[0].OutputToStream(system, out);
@ -131,8 +125,7 @@ template<typename WORD>
class PhraseOrdererLexical
{
public:
bool operator()(const Phrase<WORD> &a, const Phrase<WORD> &b) const
{
bool operator()(const Phrase<WORD> &a, const Phrase<WORD> &b) const {
size_t minSize = std::min(a.GetSize(), b.GetSize());
for (size_t i = 0; i < minSize; ++i) {
const Word &aWord = a[i];

40
moses2/PhraseBased/CubePruningMiniStack/Misc.cpp
View File

@ -22,22 +22,20 @@ namespace NSCubePruningMiniStack
////////////////////////////////////////////////////////////////////////
QueueItem *QueueItem::Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
size_t hypoIndex, size_t tpIndex,
QueueItemRecycler &queueItemRecycler)
size_t hypoIndex, size_t tpIndex,
QueueItemRecycler &queueItemRecycler)
{
QueueItem *ret;
if (currItem) {
// reuse incoming queue item to create new item
ret = currItem;
ret->Init(mgr, edge, hypoIndex, tpIndex);
}
else if (!queueItemRecycler.empty()) {
} else if (!queueItemRecycler.empty()) {
// use item from recycle bin
ret = queueItemRecycler.back();
ret->Init(mgr, edge, hypoIndex, tpIndex);
queueItemRecycler.pop_back();
}
else {
} else {
// create new item
ret = new (mgr.GetPool().Allocate<QueueItem>()) QueueItem(mgr, edge,
hypoIndex, tpIndex);
@ -47,14 +45,14 @@ QueueItem *QueueItem::Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
}
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
size_t tpIndex) :
edge(&edge), hypoIndex(hypoIndex), tpIndex(tpIndex)
size_t tpIndex) :
edge(&edge), hypoIndex(hypoIndex), tpIndex(tpIndex)
{
CreateHypothesis(mgr);
}
void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
size_t tpIndex)
size_t tpIndex)
{
this->edge = &edge;
this->hypoIndex = hypoIndex;
@ -66,7 +64,7 @@ void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
void QueueItem::CreateHypothesis(Manager &mgr)
{
const Hypothesis *prevHypo =
static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
const TargetPhraseImpl &tp = edge->tps[tpIndex];
//cerr << "hypoIndex=" << hypoIndex << endl;
@ -76,7 +74,7 @@ void QueueItem::CreateHypothesis(Manager &mgr)
hypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
hypo->Init(mgr, *prevHypo, edge->path, tp, edge->newBitmap,
edge->estimatedScore);
edge->estimatedScore);
if (!mgr.system.options.cube.lazy_scoring) {
hypo->EvaluateWhenApplied();
@ -85,8 +83,8 @@ void QueueItem::CreateHypothesis(Manager &mgr)
////////////////////////////////////////////////////////////////////////
CubeEdge::CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
const TargetPhrases &tps, const Bitmap &newBitmap) :
hypos(hypos), path(path), tps(tps), newBitmap(newBitmap)
const TargetPhrases &tps, const Bitmap &newBitmap) :
hypos(hypos), path(path), tps(tps), newBitmap(newBitmap)
{
estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
}
@ -99,7 +97,7 @@ std::string CubeEdge::Debug(const System &system) const
}
bool CubeEdge::SetSeenPosition(const size_t x, const size_t y,
SeenPositions &seenPositions) const
SeenPositions &seenPositions) const
{
//UTIL_THROW_IF2(x >= (1<<17), "Error");
//UTIL_THROW_IF2(y >= (1<<17), "Error");
@ -110,22 +108,22 @@ bool CubeEdge::SetSeenPosition(const size_t x, const size_t y,
}
void CubeEdge::CreateFirst(Manager &mgr, Queue &queue,
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler)
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler)
{
assert(hypos.size());
assert(tps.GetSize());
QueueItem *item = QueueItem::Create(NULL, mgr, *this, 0, 0,
queueItemRecycler);
queueItemRecycler);
queue.push(item);
bool setSeen = SetSeenPosition(0, 0, seenPositions);
assert(setSeen);
}
void CubeEdge::CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler)
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler)
{
size_t hypoIndex = item->hypoIndex;
size_t tpIndex = item->tpIndex;
@ -134,7 +132,7 @@ void CubeEdge::CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
&& SetSeenPosition(hypoIndex + 1, tpIndex, seenPositions)) {
// reuse incoming queue item to create new item
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex + 1,
tpIndex, queueItemRecycler);
tpIndex, queueItemRecycler);
assert(newItem == item);
queue.push(newItem);
item = NULL;
@ -143,7 +141,7 @@ void CubeEdge::CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
if (tpIndex + 1 < tps.GetSize()
&& SetSeenPosition(hypoIndex, tpIndex + 1, seenPositions)) {
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex,
tpIndex + 1, queueItemRecycler);
tpIndex + 1, queueItemRecycler);
queue.push(newItem);
item = NULL;
}

21
moses2/PhraseBased/CubePruningMiniStack/Misc.h
View File

@ -38,8 +38,8 @@ class QueueItem
~QueueItem(); // NOT IMPLEMENTED. Use MemPool
public:
static QueueItem *Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
size_t hypoIndex, size_t tpIndex,
QueueItemRecycler &queueItemRecycler);
size_t hypoIndex, size_t tpIndex,
QueueItemRecycler &queueItemRecycler);
QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
void Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
@ -56,8 +56,7 @@ protected:
class QueueItemOrderer
{
public:
bool operator()(QueueItem* itemA, QueueItem* itemB) const
{
bool operator()(QueueItem* itemA, QueueItem* itemB) const {
HypothesisFutureScoreOrderer orderer;
return !orderer(itemA->hypo, itemB->hypo);
}
@ -68,11 +67,11 @@ class CubeEdge
{
public:
typedef std::priority_queue<QueueItem*,
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >, QueueItemOrderer> Queue;
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >, QueueItemOrderer> Queue;
typedef std::pair<const CubeEdge*, int> SeenPositionItem;
typedef boost::unordered_set<SeenPositionItem, boost::hash<SeenPositionItem>,
std::equal_to<SeenPositionItem>, MemPoolAllocator<SeenPositionItem> > SeenPositions;
std::equal_to<SeenPositionItem>, MemPoolAllocator<SeenPositionItem> > SeenPositions;
const Hypotheses &hypos;
const InputPath &path;
@ -81,16 +80,16 @@ public:
SCORE estimatedScore;
CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
const TargetPhrases &tps, const Bitmap &newBitmap);
const TargetPhrases &tps, const Bitmap &newBitmap);
bool SetSeenPosition(const size_t x, const size_t y,
SeenPositions &seenPositions) const;
SeenPositions &seenPositions) const;
void CreateFirst(Manager &mgr, Queue &queue, SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler);
QueueItemRecycler &queueItemRecycler);
void CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler);
SeenPositions &seenPositions,
QueueItemRecycler &queueItemRecycler);
std::string Debug(const System &system) const;

240
moses2/PhraseBased/CubePruningMiniStack/Search.cpp
View File

@ -30,16 +30,16 @@ namespace NSCubePruningMiniStack
////////////////////////////////////////////////////////////////////////
Search::Search(Manager &mgr) :
Moses2::Search(mgr), m_stack(mgr), m_cubeEdgeAlloc(mgr.GetPool())
Moses2::Search(mgr), m_stack(mgr), m_cubeEdgeAlloc(mgr.GetPool())
, m_queue(QueueItemOrderer(),
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(
MemPoolAllocator<QueueItem*>(mgr.GetPool())))
, m_queue(QueueItemOrderer(),
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(
MemPoolAllocator<QueueItem*>(mgr.GetPool())))
, m_seenPositions(
MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
, m_seenPositions(
MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
, m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
, m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
{
}
@ -50,6 +50,7 @@ Search::~Search()
void Search::Decode()
{
<<<<<<< HEAD
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
// init cue edges
@ -75,14 +76,42 @@ void Search::Decode()
m_stack.Clear();
Decode(stackInd);
PostDecode(stackInd);
=======
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
//m_stack.DebugCounts();
}
// init cue edges
m_cubeEdges.resize(sentence.GetSize() + 1);
for (size_t i = 0; i < m_cubeEdges.size(); ++i) {
m_cubeEdges[i] = new (mgr.GetPool().Allocate<CubeEdges>()) CubeEdges(
m_cubeEdgeAlloc);
}
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
//cerr << "initHypo=" << *initHypo << endl;
m_stack.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
PostDecode(0);
for (size_t stackInd = 1; stackInd < sentence.GetSize() + 1;
++stackInd) {
//cerr << "stackInd=" << stackInd << endl;
m_stack.Clear();
Decode(stackInd);
PostDecode(stackInd);
>>>>>>> 84b918b389e5a2d5e31cca993e1e53ff1354f1b1
//m_stack.DebugCounts();
}
}
void Search::Decode(size_t stackInd)
{
<<<<<<< HEAD
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
// reuse queue from previous stack. Clear it first
@ -166,89 +195,166 @@ void Search::Decode(size_t stackInd)
sfpt->EvaluateBeforeExtending(stackInd, hypos, mgr);
}
}
=======
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
// reuse queue from previous stack. Clear it first
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> > &container = Container(
m_queue);
//cerr << "container=" << container.size() << endl;
BOOST_FOREACH(QueueItem *item, container) {
// recycle unused hypos from queue
Hypothesis *hypo = item->hypo;
hypoRecycler.Recycle(hypo);
// recycle queue item
m_queueItemRecycler.push_back(item);
}
container.clear();
m_seenPositions.clear();
// add top hypo from every edge into queue
CubeEdges &edges = *m_cubeEdges[stackInd];
BOOST_FOREACH(CubeEdge *edge, edges) {
//cerr << *edge << " ";
edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler);
}
/*
cerr << "edges: ";
boost::unordered_set<const Bitmap*> uniqueBM;
BOOST_FOREACH(CubeEdge *edge, edges) {
uniqueBM.insert(&edge->newBitmap);
//cerr << *edge << " ";
}
cerr << edges.size() << " " << uniqueBM.size();
cerr << endl;
*/
size_t pops = 0;
while (!m_queue.empty() && pops < mgr.system.options.cube.pop_limit) {
// get best hypo from queue, add to stack
//cerr << "queue=" << queue.size() << endl;
QueueItem *item = m_queue.top();
m_queue.pop();
CubeEdge *edge = item->edge;
// add hypo to stack
Hypothesis *hypo = item->hypo;
if (mgr.system.options.cube.lazy_scoring) {
hypo->EvaluateWhenApplied();
}
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler);
++pops;
}
// create hypo from every edge. Increase diversity
if (mgr.system.options.cube.diversity) {
while (!m_queue.empty()) {
QueueItem *item = m_queue.top();
m_queue.pop();
if (item->hypoIndex == 0 && item->tpIndex == 0) {
// add hypo to stack
Hypothesis *hypo = item->hypo;
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
}
}
}
>>>>>>> 84b918b389e5a2d5e31cca993e1e53ff1354f1b1
}
void Search::PostDecode(size_t stackInd)
{
MemPool &pool = mgr.GetPool();
MemPool &pool = mgr.GetPool();
const InputPaths &paths = mgr.GetInputPaths();
const Matrix<InputPath*> &pathMatrix = paths.GetMatrix();
size_t inputSize = pathMatrix.GetRows();
size_t numPaths = pathMatrix.GetCols();
const InputPaths &paths = mgr.GetInputPaths();
const Matrix<InputPath*> &pathMatrix = paths.GetMatrix();
size_t inputSize = pathMatrix.GetRows();
size_t numPaths = pathMatrix.GetCols();
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()){
const Bitmap &hypoBitmap = *val.first.first;
size_t firstGap = hypoBitmap.GetFirstGapPos();
size_t hypoEndPos = val.first.second;
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()) {
const Bitmap &hypoBitmap = *val.first.first;
size_t firstGap = hypoBitmap.GetFirstGapPos();
size_t hypoEndPos = val.first.second;
Moses2::HypothesisColl &hypos = *val.second;
Moses2::HypothesisColl &hypos = *val.second;
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
// create edges to next hypos from existing hypos
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
// create edges to next hypos from existing hypos
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
if (path == NULL) {
break;
}
if (path->GetNumRules() == 0) {
continue;
}
if (path == NULL) {
break;
}
if (path->GetNumRules() == 0) {
continue;
}
const Range &pathRange = path->range;
//cerr << "pathRange=" << pathRange << endl;
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
continue;
}
const Range &pathRange = path->range;
//cerr << "pathRange=" << pathRange << endl;
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
continue;
}
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
if (!reorderingConstraint.Check(hypoBitmap, startPos, pathRange.GetEndPos())) {
continue;
}
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
if (!reorderingConstraint.Check(hypoBitmap, startPos, pathRange.GetEndPos())) {
continue;
}
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
size_t numWords = newBitmap.GetNumWordsCovered();
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
size_t numWords = newBitmap.GetNumWordsCovered();
CubeEdges &edges = *m_cubeEdges[numWords];
CubeEdges &edges = *m_cubeEdges[numWords];
// sort hypo for a particular bitmap and hypoEndPos
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
// sort hypo for a particular bitmap and hypoEndPos
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
size_t numPt = mgr.system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = path->targetPhrases[i];
if (tps && tps->GetSize()) {
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
edges.push_back(edge);
}
}
}
}
}
size_t numPt = mgr.system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = path->targetPhrases[i];
if (tps && tps->GetSize()) {
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
edges.push_back(edge);
}
}
}
}
}
}
const Hypothesis *Search::GetBestHypo() const
{
const Hypothesis *bestHypo = m_stack.GetBestHypo();
return bestHypo;
const Hypothesis *bestHypo = m_stack.GetBestHypo();
return bestHypo;
}
void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
{
const Stack::Coll &coll = m_stack.GetColl();
BOOST_FOREACH(const Stack::Coll::value_type &val, coll){
Moses2::HypothesisColl &hypos = *val.second;
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
const Stack::Coll &coll = m_stack.GetColl();
BOOST_FOREACH(const Stack::Coll::value_type &val, coll) {
Moses2::HypothesisColl &hypos = *val.second;
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
BOOST_FOREACH(const HypothesisBase *hypoBase, sortedHypos) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
BOOST_FOREACH(const HypothesisBase *hypoBase, sortedHypos) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
}
}

124
moses2/PhraseBased/CubePruningMiniStack/Stack.cpp
View File

@ -20,103 +20,101 @@ namespace Moses2
namespace NSCubePruningMiniStack
{
Stack::Stack(const Manager &mgr) :
m_mgr(mgr), m_coll(
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> >(
mgr.GetPool())), m_miniStackRecycler(
MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
m_mgr(mgr), m_coll(
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> >(
mgr.GetPool())), m_miniStackRecycler(
MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
{
}
Stack::~Stack()
{
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl *miniStack = val.second;
delete miniStack;
}
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl *miniStack = val.second;
delete miniStack;
}
while (!m_miniStackRecycler.empty()) {
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
m_miniStackRecycler.pop_back();
delete miniStack;
while (!m_miniStackRecycler.empty()) {
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
m_miniStackRecycler.pop_back();
delete miniStack;
}
}
}
void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
ArcLists &arcLists)
{
HypoCoverage key(&hypo->GetBitmap(), hypo->GetInputPath().range.GetEndPos());
Moses2::HypothesisColl &coll = GetMiniStack(key);
coll.Add(m_mgr, hypo, hypoRecycle, arcLists);
HypoCoverage key(&hypo->GetBitmap(), hypo->GetInputPath().range.GetEndPos());
Moses2::HypothesisColl &coll = GetMiniStack(key);
coll.Add(m_mgr, hypo, hypoRecycle, arcLists);
}
const Hypothesis *Stack::GetBestHypo() const
{
SCORE bestScore = -std::numeric_limits<SCORE>::infinity();
const HypothesisBase *bestHypo = NULL;
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl &hypos = *val.second;
const Moses2::HypothesisBase *hypo = hypos.GetBestHypo();
SCORE bestScore = -std::numeric_limits<SCORE>::infinity();
const HypothesisBase *bestHypo = NULL;
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl &hypos = *val.second;
const Moses2::HypothesisBase *hypo = hypos.GetBestHypo();
if (hypo && hypo->GetFutureScore() > bestScore) {
bestScore = hypo->GetFutureScore();
bestHypo = hypo;
}
}
return &bestHypo->Cast<Hypothesis>();
if (hypo && hypo->GetFutureScore() > bestScore) {
bestScore = hypo->GetFutureScore();
bestHypo = hypo;
}
}
return &bestHypo->Cast<Hypothesis>();
}
size_t Stack::GetHypoSize() const
{
size_t ret = 0;
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl &hypos = *val.second;
ret += hypos.GetSize();
}
return ret;
size_t ret = 0;
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl &hypos = *val.second;
ret += hypos.GetSize();
}
return ret;
}
Moses2::HypothesisColl &Stack::GetMiniStack(const HypoCoverage &key)
{
Moses2::HypothesisColl *ret;
Coll::iterator iter = m_coll.find(key);
if (iter == m_coll.end()) {
if (m_miniStackRecycler.empty()) {
ret = new Moses2::HypothesisColl(m_mgr);
}
else {
ret = m_miniStackRecycler.back();
ret->Clear();
m_miniStackRecycler.pop_back();
}
Moses2::HypothesisColl *ret;
Coll::iterator iter = m_coll.find(key);
if (iter == m_coll.end()) {
if (m_miniStackRecycler.empty()) {
ret = new Moses2::HypothesisColl(m_mgr);
} else {
ret = m_miniStackRecycler.back();
ret->Clear();
m_miniStackRecycler.pop_back();
}
m_coll[key] = ret;
}
else {
ret = iter->second;
}
return *ret;
m_coll[key] = ret;
} else {
ret = iter->second;
}
return *ret;
}
void Stack::Clear()
{
BOOST_FOREACH(const Coll::value_type &val, m_coll){
Moses2::HypothesisColl *miniStack = val.second;
m_miniStackRecycler.push_back(miniStack);
}
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
Moses2::HypothesisColl *miniStack = val.second;
m_miniStackRecycler.push_back(miniStack);
}
m_coll.clear();
m_coll.clear();
}
void Stack::DebugCounts()
{
cerr << "counts=";
BOOST_FOREACH(const Coll::value_type &val, GetColl()){
const Moses2::HypothesisColl &miniStack = *val.second;
size_t count = miniStack.GetSize();
cerr << count << " ";
}
cerr << endl;
cerr << "counts=";
BOOST_FOREACH(const Coll::value_type &val, GetColl()) {
const Moses2::HypothesisColl &miniStack = *val.second;
size_t count = miniStack.GetSize();
cerr << count << " ";
}
cerr << endl;
}
}

12
moses2/PhraseBased/CubePruningMiniStack/Stack.h
View File

@ -36,25 +36,23 @@ public:
// bitmap and current endPos of hypos
typedef boost::unordered_map<HypoCoverage, Moses2::HypothesisColl*,
boost::hash<HypoCoverage>, std::equal_to<HypoCoverage>,
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> > > Coll;
boost::hash<HypoCoverage>, std::equal_to<HypoCoverage>,
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> > > Coll;
Stack(const Manager &mgr);
virtual ~Stack();
size_t GetHypoSize() const;
Coll &GetColl()
{
Coll &GetColl() {
return m_coll;
}
const Coll &GetColl() const
{
const Coll &GetColl() const {
return m_coll;
}
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
ArcLists &arcLists);
Moses2::HypothesisColl &GetMiniStack(const HypoCoverage &key);

38
moses2/PhraseBased/Hypothesis.cpp
View File

@ -30,8 +30,7 @@ Hypothesis *Hypothesis::Create(MemPool &pool, Manager &mgr)
ret = static_cast<Hypothesis*>(recycler.Get());
if (ret) {
// got new hypo from recycler. Do nothing
}
else {
} else {
ret = new (pool.Allocate<Hypothesis>()) Hypothesis(pool, mgr.system);
//cerr << "Hypothesis=" << sizeof(Hypothesis) << " " << ret << endl;
recycler.Keep(ret);
@ -40,7 +39,7 @@ Hypothesis *Hypothesis::Create(MemPool &pool, Manager &mgr)
}
Hypothesis::Hypothesis(MemPool &pool, const System &system) :
HypothesisBase(pool, system), m_currTargetWordsRange()
HypothesisBase(pool, system), m_currTargetWordsRange()
{
}
@ -50,7 +49,7 @@ Hypothesis::~Hypothesis()
}
void Hypothesis::Init(Manager &mgr, const InputPathBase &path,
const TargetPhraseImpl &tp, const Bitmap &bitmap)
const TargetPhraseImpl &tp, const Bitmap &bitmap)
{
m_mgr = &mgr;
m_targetPhrase = &tp;
@ -66,8 +65,8 @@ void Hypothesis::Init(Manager &mgr, const InputPathBase &path,
}
void Hypothesis::Init(Manager &mgr, const Hypothesis &prevHypo,
const InputPathBase &path, const TargetPhraseImpl &tp, const Bitmap &bitmap,
SCORE estimatedScore)
const InputPathBase &path, const TargetPhraseImpl &tp, const Bitmap &bitmap,
SCORE estimatedScore)
{
m_mgr = &mgr;
m_targetPhrase = &tp;
@ -76,9 +75,9 @@ void Hypothesis::Init(Manager &mgr, const Hypothesis &prevHypo,
m_prevHypo = &prevHypo;
m_currTargetWordsRange.SetStartPos(
prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
m_currTargetWordsRange.SetEndPos(
prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
m_estimatedScore = estimatedScore;
@ -116,7 +115,7 @@ std::string Hypothesis::Debug(const System &system) const
// states
const std::vector<const StatefulFeatureFunction*> &sfffs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
for (size_t i = 0; i < numStatefulFFs; ++i) {
const FFState &state = *GetState(i);
@ -152,8 +151,7 @@ void Hypothesis::OutputToStream(std::ostream &out) const
if (m_mgr->system.options.output.ReportSegmentation == 1) {
// just report phrase segmentation
out << "|" << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << "| ";
}
else if (m_mgr->system.options.output.ReportSegmentation == 2) {
} else if (m_mgr->system.options.output.ReportSegmentation == 2) {
// more detailed info about every segment
out << "|";
@ -171,19 +169,19 @@ void Hypothesis::OutputToStream(std::ostream &out) const
void Hypothesis::EmptyHypothesisState(const InputType &input)
{
const std::vector<const StatefulFeatureFunction*> &sfffs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = m_ffStates[statefulInd];
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
}
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = m_ffStates[statefulInd];
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
}
}
void Hypothesis::EvaluateWhenApplied()
{
const std::vector<const StatefulFeatureFunction*> &sfffs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
EvaluateWhenApplied(*sfff);
}
//cerr << *this << endl;
@ -196,7 +194,7 @@ void Hypothesis::EvaluateWhenApplied(const StatefulFeatureFunction &sfff)
FFState *thisState = m_ffStates[statefulInd];
assert(prevState);
sfff.EvaluateWhenApplied(GetManager(), *this, *prevState, *m_scores,
*thisState);
*thisState);
}

28
moses2/PhraseBased/Hypothesis.h
View File

@ -35,36 +35,31 @@ public:
// initial, empty hypo
void Init(Manager &mgr, const InputPathBase &path, const TargetPhraseImpl &tp,
const Bitmap &bitmap);
const Bitmap &bitmap);
void Init(Manager &mgr, const Hypothesis &prevHypo, const InputPathBase &path,
const TargetPhraseImpl &tp, const Bitmap &bitmap, SCORE estimatedScore);
const TargetPhraseImpl &tp, const Bitmap &bitmap, SCORE estimatedScore);
size_t hash() const;
bool operator==(const Hypothesis &other) const;
inline const Bitmap &GetBitmap() const
{
inline const Bitmap &GetBitmap() const {
return *m_sourceCompleted;
}
inline const InputPathBase &GetInputPath() const
{
inline const InputPathBase &GetInputPath() const {
return *m_path;
}
inline const Range &GetCurrTargetWordsRange() const
{
inline const Range &GetCurrTargetWordsRange() const {
return m_currTargetWordsRange;
}
SCORE GetFutureScore() const
{
SCORE GetFutureScore() const {
return GetScores().GetTotalScore() + m_estimatedScore;
}
const TargetPhrase<Moses2::Word> &GetTargetPhrase() const
{
const TargetPhrase<Moses2::Word> &GetTargetPhrase() const {
return *m_targetPhrase;
}
@ -77,16 +72,14 @@ public:
void EvaluateWhenApplied();
void EvaluateWhenApplied(const StatefulFeatureFunction &sfff);
const Hypothesis* GetPrevHypo() const
{
const Hypothesis* GetPrevHypo() const {
return m_prevHypo;
}
/** curr - pos is relative from CURRENT hypothesis's starting index
* (ie, start of sentence would be some negative number, which is
* not allowed- USE WITH CAUTION) */
inline const Word &GetCurrWord(size_t pos) const
{
inline const Word &GetCurrWord(size_t pos) const {
return GetTargetPhrase()[pos];
}
@ -108,8 +101,7 @@ protected:
class HypothesisTargetPhraseOrderer
{
public:
bool operator()(const Hypothesis* a, const Hypothesis* b) const
{
bool operator()(const Hypothesis* a, const Hypothesis* b) const {
PhraseOrdererLexical<Moses2::Word> phraseCmp;
bool ret = phraseCmp(a->GetTargetPhrase(), b->GetTargetPhrase());
/*

10
moses2/PhraseBased/InputPath.cpp
View File

@ -15,10 +15,10 @@ using namespace std;
namespace Moses2
{
InputPath::InputPath(MemPool &pool, const SubPhrase<Moses2::Word> &subPhrase,
const Range &range, size_t numPt, const InputPath *prefixPath)
:InputPathBase(pool, range, numPt, prefixPath)
,m_numRules(0)
,subPhrase(subPhrase)
const Range &range, size_t numPt, const InputPath *prefixPath)
:InputPathBase(pool, range, numPt, prefixPath)
,m_numRules(0)
,subPhrase(subPhrase)
{
targetPhrases = pool.Allocate<const TargetPhrases*>(numPt);
Init<const TargetPhrases*>(targetPhrases, numPt, NULL);
@ -30,7 +30,7 @@ InputPath::~InputPath()
}
void InputPath::AddTargetPhrases(const PhraseTable &pt,
const TargetPhrases *tps)
const TargetPhrases *tps)
{
size_t ptInd = pt.GetPtInd();
targetPhrases[ptInd] = tps;

7
moses2/PhraseBased/InputPath.h
View File

@ -22,14 +22,15 @@ public:
SubPhrase<Moses2::Word> subPhrase;
InputPath(MemPool &pool, const SubPhrase<Moses2::Word> &subPhrase, const Range &range,
size_t numPt, const InputPath *prefixPath);
size_t numPt, const InputPath *prefixPath);
virtual ~InputPath();
void AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps);
const TargetPhrases *GetTargetPhrases(const PhraseTable &pt) const;
size_t GetNumRules() const
{ return m_numRules; }
size_t GetNumRules() const {
return m_numRules;
}
std::string Debug(const System &system) const;

13
moses2/PhraseBased/InputPaths.h
View File

@ -23,16 +23,17 @@ class InputPaths: public InputPathsBase
public:
void Init(const InputType &input, const ManagerBase &mgr);
const InputPath &GetBlank() const
{
const InputPath &GetBlank() const {
return *m_blank;
}
Matrix<InputPath*> &GetMatrix()
{ return *m_matrix; }
Matrix<InputPath*> &GetMatrix() {
return *m_matrix;
}
const Matrix<InputPath*> &GetMatrix() const
{ return *m_matrix; }
const Matrix<InputPath*> &GetMatrix() const {
return *m_matrix;
}
protected:
InputPath *m_blank;

322
moses2/PhraseBased/Manager.cpp
View File

@ -37,71 +37,71 @@ using namespace std;
namespace Moses2
{
Manager::Manager(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId) :
ManagerBase(sys, task, inputStr, translationId)
,m_search(NULL)
,m_bitmaps(NULL)
const std::string &inputStr, long translationId) :
ManagerBase(sys, task, inputStr, translationId)
,m_search(NULL)
,m_bitmaps(NULL)
{
//cerr << translationId << " inputStr=" << inputStr << endl;
//cerr << translationId << " inputStr=" << inputStr << endl;
}
Manager::~Manager()
{
//cerr << "Start ~Manager " << this << endl;
delete m_search;
delete m_bitmaps;
//cerr << "Finish ~Manager " << this << endl;
//cerr << "Start ~Manager " << this << endl;
delete m_search;
delete m_bitmaps;
//cerr << "Finish ~Manager " << this << endl;
}
void Manager::Init()
{
// init pools etc
InitPools();
// init pools etc
InitPools();
FactorCollection &vocab = system.GetVocab();
m_input = Moses2::Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr);
FactorCollection &vocab = system.GetVocab();
m_input = Moses2::Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr);
m_bitmaps = new Bitmaps(GetPool());
m_bitmaps = new Bitmaps(GetPool());
const PhraseTable &firstPt = *system.featureFunctions.phraseTables[0];
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(
GetPool(), firstPt, system, 0);
const PhraseTable &firstPt = *system.featureFunctions.phraseTables[0];
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(
GetPool(), firstPt, system, 0);
const Sentence &sentence = static_cast<const Sentence&>(GetInput());
//cerr << "sentence=" << sentence.GetSize() << " " << sentence.Debug(system) << endl;
const Sentence &sentence = static_cast<const Sentence&>(GetInput());
//cerr << "sentence=" << sentence.GetSize() << " " << sentence.Debug(system) << endl;
m_inputPaths.Init(sentence, *this);
m_inputPaths.Init(sentence, *this);
// xml
const UnknownWordPenalty *unkWP = system.featureFunctions.GetUnknownWordPenalty();
UTIL_THROW_IF2(unkWP == NULL, "There must be a UnknownWordPenalty FF");
unkWP->ProcessXML(*this, GetPool(), sentence, m_inputPaths);
// xml
const UnknownWordPenalty *unkWP = system.featureFunctions.GetUnknownWordPenalty();
UTIL_THROW_IF2(unkWP == NULL, "There must be a UnknownWordPenalty FF");
unkWP->ProcessXML(*this, GetPool(), sentence, m_inputPaths);
// lookup with every pt
const std::vector<const PhraseTable*> &pts = system.mappings;
for (size_t i = 0; i < pts.size(); ++i) {
const PhraseTable &pt = *pts[i];
//cerr << "Looking up from " << pt.GetName() << endl;
pt.Lookup(*this, m_inputPaths);
}
//m_inputPaths.DeleteUnusedPaths();
CalcFutureScore();
// lookup with every pt
const std::vector<const PhraseTable*> &pts = system.mappings;
for (size_t i = 0; i < pts.size(); ++i) {
const PhraseTable &pt = *pts[i];
//cerr << "Looking up from " << pt.GetName() << endl;
pt.Lookup(*this, m_inputPaths);
}
//m_inputPaths.DeleteUnusedPaths();
CalcFutureScore();
m_bitmaps->Init(sentence.GetSize(), vector<bool>(0));
m_bitmaps->Init(sentence.GetSize(), vector<bool>(0));
switch (system.options.search.algo) {
case Normal:
m_search = new NSNormal::Search(*this);
break;
case NormalBatch:
//m_search = new NSBatch::Search(*this);
UTIL_THROW2("Not implemented");
break;
case CubePruning:
case CubePruningMiniStack:
m_search = new NSCubePruningMiniStack::Search(*this);
break;
/*
switch (system.options.search.algo) {
case Normal:
m_search = new NSNormal::Search(*this);
break;
case NormalBatch:
//m_search = new NSBatch::Search(*this);
UTIL_THROW2("Not implemented");
break;
case CubePruning:
case CubePruningMiniStack:
m_search = new NSCubePruningMiniStack::Search(*this);
break;
/*
case CubePruningPerMiniStack:
m_search = new NSCubePruningPerMiniStack::Search(*this);
break;
@ -114,166 +114,164 @@ void Manager::Init()
case CubePruningBitmapStack:
m_search = new NSCubePruningBitmapStack::Search(*this);
break;
*/
default:
*/
default:
UTIL_THROW2("Unknown search algorithm");
}
}
}
void Manager::Decode()
{
//cerr << "Start Decode " << this << endl;
//cerr << "Start Decode " << this << endl;
Init();
m_search->Decode();
Init();
m_search->Decode();
//cerr << "Finished Decode " << this << endl;
//cerr << "Finished Decode " << this << endl;
}
void Manager::CalcFutureScore()
{
const Sentence &sentence = static_cast<const Sentence&>(GetInput());
size_t size = sentence.GetSize();
m_estimatedScores =
new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(),
size);
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
const Sentence &sentence = static_cast<const Sentence&>(GetInput());
size_t size = sentence.GetSize();
m_estimatedScores =
new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(),
size);
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
// walk all the translation options and record the cheapest option for each span
BOOST_FOREACH(const InputPathBase *path, m_inputPaths){
const Range &range = path->range;
SCORE bestScore = -numeric_limits<SCORE>::infinity();
// walk all the translation options and record the cheapest option for each span
BOOST_FOREACH(const InputPathBase *path, m_inputPaths) {
const Range &range = path->range;
SCORE bestScore = -numeric_limits<SCORE>::infinity();
size_t numPt = system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
if (tps) {
BOOST_FOREACH(const TargetPhraseImpl *tp, *tps) {
SCORE score = tp->GetFutureScore();
if (score > bestScore) {
bestScore = score;
}
}
}
}
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
}
size_t numPt = system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
if (tps) {
BOOST_FOREACH(const TargetPhraseImpl *tp, *tps) {
SCORE score = tp->GetFutureScore();
if (score > bestScore) {
bestScore = score;
}
}
}
}
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
}
// now fill all the cells in the strictly upper triangle
// there is no way to modify the diagonal now, in the case
// where no translation option covers a single-word span,
// we leave the +inf in the matrix
// like in chart parsing we want each cell to contain the highest score
// of the full-span trOpt or the sum of scores of joining two smaller spans
// now fill all the cells in the strictly upper triangle
// there is no way to modify the diagonal now, in the case
// where no translation option covers a single-word span,
// we leave the +inf in the matrix
// like in chart parsing we want each cell to contain the highest score
// of the full-span trOpt or the sum of scores of joining two smaller spans
for (size_t colstart = 1; colstart < size; colstart++) {
for (size_t diagshift = 0; diagshift < size - colstart; diagshift++) {
size_t sPos = diagshift;
size_t ePos = colstart + diagshift;
for (size_t joinAt = sPos; joinAt < ePos; joinAt++) {
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
+ m_estimatedScores->GetValue(joinAt + 1, ePos);
// uncomment to see the cell filling scheme
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
// << sPos << "," << joinAt << "]+["
// << joinAt+1 << "," << ePos << "] (colstart: "
// << colstart << ", diagshift: " << diagshift << ")"
// << endl);
for (size_t colstart = 1; colstart < size; colstart++) {
for (size_t diagshift = 0; diagshift < size - colstart; diagshift++) {
size_t sPos = diagshift;
size_t ePos = colstart + diagshift;
for (size_t joinAt = sPos; joinAt < ePos; joinAt++) {
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
+ m_estimatedScores->GetValue(joinAt + 1, ePos);
// uncomment to see the cell filling scheme
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
// << sPos << "," << joinAt << "]+["
// << joinAt+1 << "," << ePos << "] (colstart: "
// << colstart << ", diagshift: " << diagshift << ")"
// << endl);
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos)) m_estimatedScores->SetValue(
sPos, ePos, joinedScore);
}
}
}
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos)) m_estimatedScores->SetValue(
sPos, ePos, joinedScore);
}
}
}
//cerr << "Square matrix:" << endl;
//cerr << *m_estimatedScores << endl;
//cerr << "Square matrix:" << endl;
//cerr << *m_estimatedScores << endl;
}
std::string Manager::OutputBest() const
{
stringstream out;
stringstream out;
Moses2::FixPrecision(out);
const Hypothesis *bestHypo = m_search->GetBestHypo();
if (bestHypo) {
if (system.options.output.ReportHypoScore) {
out << bestHypo->GetScores().GetTotalScore() << " ";
}
const Hypothesis *bestHypo = m_search->GetBestHypo();
if (bestHypo) {
if (system.options.output.ReportHypoScore) {
out << bestHypo->GetScores().GetTotalScore() << " ";
}
bestHypo->OutputToStream(out);
//cerr << "BEST TRANSLATION: " << *bestHypo;
}
else {
if (system.options.output.ReportHypoScore) {
out << "0 ";
}
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
}
bestHypo->OutputToStream(out);
//cerr << "BEST TRANSLATION: " << *bestHypo;
} else {
if (system.options.output.ReportHypoScore) {
out << "0 ";
}
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
}
return out.str();
//cerr << endl;
return out.str();
//cerr << endl;
}
std::string Manager::OutputNBest()
{
arcLists.Sort();
arcLists.Sort();
boost::unordered_set<size_t> distinctHypos;
boost::unordered_set<size_t> distinctHypos;
TrellisPaths<TrellisPath> contenders;
m_search->AddInitialTrellisPaths(contenders);
TrellisPaths<TrellisPath> contenders;
m_search->AddInitialTrellisPaths(contenders);
long transId = GetTranslationId();
long transId = GetTranslationId();
// MAIN LOOP
stringstream out;
//Moses2::FixPrecision(out);
// MAIN LOOP
stringstream out;
//Moses2::FixPrecision(out);
size_t maxIter = system.options.nbest.nbest_size * system.options.nbest.factor;
size_t bestInd = 0;
for (size_t i = 0; i < maxIter; ++i) {
if (bestInd > system.options.nbest.nbest_size || contenders.empty()) {
break;
}
size_t maxIter = system.options.nbest.nbest_size * system.options.nbest.factor;
size_t bestInd = 0;
for (size_t i = 0; i < maxIter; ++i) {
if (bestInd > system.options.nbest.nbest_size || contenders.empty()) {
break;
}
//cerr << "bestInd=" << bestInd << endl;
TrellisPath *path = contenders.Get();
//cerr << "bestInd=" << bestInd << endl;
TrellisPath *path = contenders.Get();
bool ok = false;
if (system.options.nbest.only_distinct) {
string tgtPhrase = path->OutputTargetPhrase(system);
//cerr << "tgtPhrase=" << tgtPhrase << endl;
boost::hash<std::string> string_hash;
size_t hash = string_hash(tgtPhrase);
bool ok = false;
if (system.options.nbest.only_distinct) {
string tgtPhrase = path->OutputTargetPhrase(system);
//cerr << "tgtPhrase=" << tgtPhrase << endl;
boost::hash<std::string> string_hash;
size_t hash = string_hash(tgtPhrase);
if (distinctHypos.insert(hash).second) {
ok = true;
}
}
else {
ok = true;
}
if (distinctHypos.insert(hash).second) {
ok = true;
}
} else {
ok = true;
}
if (ok) {
++bestInd;
out << transId << " ||| ";
path->OutputToStream(out, system);
out << "\n";
}
if (ok) {
++bestInd;
out << transId << " ||| ";
path->OutputToStream(out, system);
out << "\n";
}
// create next paths
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
// create next paths
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
delete path;
}
delete path;
}
return out.str();
return out.str();
}
std::string Manager::OutputTransOpt()
{
return "";
return "";
}
}

22
moses2/PhraseBased/Manager.h
View File

@ -37,21 +37,25 @@ class Manager: public ManagerBase
{
public:
Manager(System &sys, const TranslationTask &task, const std::string &inputStr,
long translationId);
long translationId);
virtual ~Manager();
Bitmaps &GetBitmaps()
{ return *m_bitmaps; }
Bitmaps &GetBitmaps() {
return *m_bitmaps;
}
const EstimatedScores &GetEstimatedScores() const
{ return *m_estimatedScores; }
const EstimatedScores &GetEstimatedScores() const {
return *m_estimatedScores;
}
const InputPaths &GetInputPaths() const
{ return m_inputPaths; }
const InputPaths &GetInputPaths() const {
return m_inputPaths;
}
const TargetPhraseImpl &GetInitPhrase() const
{ return *m_initPhrase; }
const TargetPhraseImpl &GetInitPhrase() const {
return *m_initPhrase;
}
void Decode();
std::string OutputBest() const;

150
moses2/PhraseBased/Normal/Search.cpp
View File

@ -27,46 +27,47 @@ namespace NSNormal
{
Search::Search(Manager &mgr)
:Moses2::Search(mgr)
, m_stacks(mgr)
:Moses2::Search(mgr)
, m_stacks(mgr)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
Search::~Search()
{
// TODO Auto-generated destructor stub
// TODO Auto-generated destructor stub
}
void Search::Decode()
{
// init stacks
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
m_stacks.Init(mgr, sentence.GetSize() + 1);
// init stacks
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
m_stacks.Init(mgr, sentence.GetSize() + 1);
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
Decode(stackInd);
//cerr << m_stacks << endl;
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
Decode(stackInd);
//cerr << m_stacks << endl;
// delete stack to save mem
if (stackInd < m_stacks.GetSize() - 1) {
m_stacks.Delete(stackInd);
}
//cerr << m_stacks.Debug(mgr.system) << endl;
}
// delete stack to save mem
if (stackInd < m_stacks.GetSize() - 1) {
m_stacks.Delete(stackInd);
}
//cerr << m_stacks.Debug(mgr.system) << endl;
}
}
void Search::Decode(size_t stackInd)
{
<<<<<<< HEAD
//cerr << "stackInd=" << stackInd << endl;
Stack &stack = m_stacks[stackInd];
if (&stack == &m_stacks.Back()) {
@ -88,77 +89,96 @@ void Search::Decode(size_t stackInd)
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
}
}
=======
//cerr << "stackInd=" << stackInd << endl;
Stack &stack = m_stacks[stackInd];
if (&stack == &m_stacks.Back()) {
// last stack. don't do anythin
return;
}
const Hypotheses &hypos = stack.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
//cerr << "hypos=" << hypos.size() << endl;
const InputPaths &paths = mgr.GetInputPaths();
BOOST_FOREACH(const InputPathBase *path, paths) {
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
}
}
>>>>>>> 84b918b389e5a2d5e31cca993e1e53ff1354f1b1
}
void Search::Extend(const Hypothesis &hypo, const InputPath &path)
{
const Bitmap &hypoBitmap = hypo.GetBitmap();
const Range &hypoRange = hypo.GetInputPath().range;
const Range &pathRange = path.range;
const Bitmap &hypoBitmap = hypo.GetBitmap();
const Range &hypoRange = hypo.GetInputPath().range;
const Range &pathRange = path.range;
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
return;
}
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
return;
}
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
if (!reorderingConstraint.Check(hypoBitmap, pathRange.GetStartPos(), pathRange.GetEndPos())) {
return;
}
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
if (!reorderingConstraint.Check(hypoBitmap, pathRange.GetStartPos(), pathRange.GetEndPos())) {
return;
}
// extend this hypo
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
// extend this hypo
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
size_t numPt = mgr.system.mappings.size();
const TargetPhrases **tpsAllPt = path.targetPhrases;
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = tpsAllPt[i];
if (tps) {
Extend(hypo, *tps, path, newBitmap, estimatedScore);
}
}
size_t numPt = mgr.system.mappings.size();
const TargetPhrases **tpsAllPt = path.targetPhrases;
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = tpsAllPt[i];
if (tps) {
Extend(hypo, *tps, path, newBitmap, estimatedScore);
}
}
}
void Search::Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
BOOST_FOREACH(const TargetPhraseImpl *tp, tps){
Extend(hypo, *tp, path, newBitmap, estimatedScore);
}
BOOST_FOREACH(const TargetPhraseImpl *tp, tps) {
Extend(hypo, *tp, path, newBitmap, estimatedScore);
}
}
void Search::Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
newHypo->EvaluateWhenApplied();
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
newHypo->EvaluateWhenApplied();
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
}
const Hypothesis *Search::GetBestHypo() const
{
const Stack &lastStack = m_stacks.Back();
const Hypothesis *best = lastStack.GetBestHypo<Hypothesis>();
return best;
const Stack &lastStack = m_stacks.Back();
const Hypothesis *best = lastStack.GetBestHypo<Hypothesis>();
return best;
}
void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
{
const Stack &lastStack = m_stacks.Back();
const Hypotheses &hypos = lastStack.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
const Stack &lastStack = m_stacks.Back();
const Hypotheses &hypos = lastStack.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos){
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
} // namespace

4
moses2/PhraseBased/Normal/Search.h
View File

@ -41,9 +41,9 @@ protected:
void Decode(size_t stackInd);
void Extend(const Hypothesis &hypo, const InputPath &path);
void Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
void Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
};

2
moses2/PhraseBased/Normal/Stack.cpp
View File

@ -20,7 +20,7 @@ namespace NSNormal
{
Stack::Stack(const Manager &mgr) :
HypothesisColl(mgr)
HypothesisColl(mgr)
{
// TODO Auto-generated constructor stub

7
moses2/PhraseBased/Normal/Stacks.cpp
View File

@ -18,7 +18,7 @@ namespace NSNormal
{
Stacks::Stacks(const Manager &mgr) :
m_mgr(mgr)
m_mgr(mgr)
{
// TODO Auto-generated constructor stub
@ -46,8 +46,7 @@ std::string Stacks::Debug(const System &system) const
const Stack *stack = m_stacks[i];
if (stack) {
out << stack->GetSize() << " ";
}
else {
} else {
out << "N ";
}
}
@ -55,7 +54,7 @@ std::string Stacks::Debug(const System &system) const
}
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
ArcLists &arcLists)
{
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
//cerr << "numWordsCovered=" << numWordsCovered << endl;

14
moses2/PhraseBased/Normal/Stacks.h
View File

@ -27,29 +27,25 @@ public:
void Init(const Manager &mgr, size_t numStacks);
size_t GetSize() const
{
size_t GetSize() const {
return m_stacks.size();
}
const Stack &Back() const
{
const Stack &Back() const {
return *m_stacks.back();
}
Stack &operator[](size_t ind)
{
Stack &operator[](size_t ind) {
return *m_stacks[ind];
}
void Delete(size_t ind)
{
void Delete(size_t ind) {
delete m_stacks[ind];
m_stacks[ind] = NULL;
}
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
ArcLists &arcLists);
std::string Debug(const System &system) const;

2
moses2/PhraseBased/PhraseImpl.cpp
View File

@ -11,7 +11,7 @@ using namespace std;
namespace Moses2
{
PhraseImpl *PhraseImpl::CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str)
const System &system, const std::string &str)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();

5
moses2/PhraseBased/PhraseImpl.h
View File

@ -9,11 +9,10 @@ class PhraseImpl: public PhraseImplTemplate<Word>
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str);
const System &system, const std::string &str);
PhraseImpl(MemPool &pool, size_t size) :
PhraseImplTemplate<Word>(pool, size)
{
PhraseImplTemplate<Word>(pool, size) {
}
};

4
moses2/PhraseBased/ReorderingConstraint.cpp
View File

@ -237,12 +237,12 @@ std::ostream &ReorderingConstraint::Debug(std::ostream &out, const System &syste
out << "Walls:";
for (size_t i = 0; i < m_size; ++i) {
out << m_wall[i];
out << m_wall[i];
}
out << " Local walls:";
for (size_t i = 0; i < m_size; ++i) {
out << m_localWall[i] << " ";
out << m_localWall[i] << " ";
}
return out;

11
moses2/PhraseBased/Search.cpp
View File

@ -15,7 +15,7 @@ namespace Moses2
{
Search::Search(Manager &mgr) :
mgr(mgr)
mgr(mgr)
{
// TODO Auto-generated constructor stub
@ -27,7 +27,7 @@ Search::~Search()
}
bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
const Range &pathRange)
const Range &pathRange)
{
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
@ -46,7 +46,7 @@ bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
if (mgr.system.options.reordering.max_distortion >= 0) {
// distortion limit
int distortion = ComputeDistortionDistance(hypoRangeEndPos,
pathRange.GetStartPos());
pathRange.GetStartPos());
if (distortion > mgr.system.options.reordering.max_distortion) {
//cerr << " NO" << endl;
return false;
@ -88,8 +88,7 @@ bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
if (isLeftMostEdge) {
// any length extension is okay if starting at left-most edge
}
else { // starting somewhere other than left-most edge, use caution
} else { // starting somewhere other than left-most edge, use caution
// the basic idea is this: we would like to translate a phrase
// starting from a position further right than the left-most
// open gap. The distortion penalty for the following phrase
@ -101,7 +100,7 @@ bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
if (ComputeDistortionDistance(pathRange.GetEndPos(),
bestNextExtension.GetStartPos()) > mgr.system.options.reordering.max_distortion) {
bestNextExtension.GetStartPos()) > mgr.system.options.reordering.max_distortion) {
//cerr << " NO" << endl;
return false;
}

8
moses2/PhraseBased/Search.h
View File

@ -39,16 +39,14 @@ protected:
//ArcLists m_arcLists;
bool CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
const Range &pathRange);
const Range &pathRange);
inline int ComputeDistortionDistance(size_t prevEndPos,
size_t currStartPos) const
{
size_t currStartPos) const {
int dist = 0;
if (prevEndPos == NOT_FOUND) {
dist = currStartPos;
}
else {
} else {
dist = (int)prevEndPos - (int)currStartPos + 1;
}
return abs(dist);

141
moses2/PhraseBased/Sentence.cpp
View File

@ -17,15 +17,14 @@ namespace Moses2
{
Sentence *Sentence::CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str)
const System &system, const std::string &str)
{
Sentence *ret;
if (system.options.input.xml_policy) {
// xml
ret = CreateFromStringXML(pool, vocab, system, str);
}
else {
ret = CreateFromStringXML(pool, vocab, system, str);
} else {
// no xml
//cerr << "PB Sentence" << endl;
std::vector<std::string> toks = Tokenize(str);
@ -42,82 +41,80 @@ Sentence *Sentence::CreateFromString(MemPool &pool, FactorCollection &vocab,
}
Sentence *Sentence::CreateFromStringXML(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str)
const System &system, const std::string &str)
{
Sentence *ret;
vector<XMLOption*> xmlOptions;
pugi::xml_document doc;
vector<XMLOption*> xmlOptions;
pugi::xml_document doc;
string str2 = "<xml>" + str + "</xml>";
pugi::xml_parse_result result = doc.load(str2.c_str(),
pugi::parse_cdata | pugi::parse_wconv_attribute | pugi::parse_eol | pugi::parse_comments);
pugi::xml_node topNode = doc.child("xml");
string str2 = "<xml>" + str + "</xml>";
pugi::xml_parse_result result = doc.load(str2.c_str(),
pugi::parse_cdata | pugi::parse_wconv_attribute | pugi::parse_eol | pugi::parse_comments);
pugi::xml_node topNode = doc.child("xml");
std::vector<std::string> toks;
XMLParse(pool, system, 0, topNode, toks, xmlOptions);
std::vector<std::string> toks;
XMLParse(pool, system, 0, topNode, toks, xmlOptions);
// debug
/*
cerr << "xmloptions:" << endl;
for (size_t i = 0; i < xmlOptions.size(); ++i) {
cerr << xmlOptions[i]->Debug(system) << endl;
// debug
/*
cerr << "xmloptions:" << endl;
for (size_t i = 0; i < xmlOptions.size(); ++i) {
cerr << xmlOptions[i]->Debug(system) << endl;
}
*/
// create words
size_t size = toks.size();
ret = new (pool.Allocate<Sentence>()) Sentence(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
// xml
ret->Init(system, size, system.options.reordering.max_distortion);
ReorderingConstraint &reorderingConstraint = ret->GetReorderingConstraint();
// set reordering walls, if "-monotone-at-punction" is set
if (system.options.reordering.monotone_at_punct && ret->GetSize()) {
reorderingConstraint.SetMonotoneAtPunctuation(*ret);
}
// set walls obtained from xml
for(size_t i=0; i<xmlOptions.size(); i++) {
const XMLOption *xmlOption = xmlOptions[i];
if(strcmp(xmlOption->GetNodeName(), "wall") == 0) {
UTIL_THROW_IF2(xmlOption->startPos > ret->GetSize(), "wall is beyond the sentence"); // no buggy walls, please
reorderingConstraint.SetWall(xmlOption->startPos - 1, true);
} else if (strcmp(xmlOption->GetNodeName(), "zone") == 0) {
reorderingConstraint.SetZone( xmlOption->startPos, xmlOption->startPos + xmlOption->phraseSize -1 );
} else if (strcmp(xmlOption->GetNodeName(), "ne") == 0) {
FactorType placeholderFactor = system.options.input.placeholder_factor;
UTIL_THROW_IF2(placeholderFactor == NOT_FOUND,
"Placeholder XML in input. Must have argument -placeholder-factor [NUM]");
UTIL_THROW_IF2(xmlOption->phraseSize != 1,
"Placeholder must only cover 1 word");
const Factor *factor = vocab.AddFactor(xmlOption->GetEntity(), system, false);
(*ret)[xmlOption->startPos][placeholderFactor] = factor;
} else {
// default - forced translation. Add to class variable
ret->AddXMLOption(system, xmlOption);
}
*/
}
reorderingConstraint.FinalizeWalls();
// create words
size_t size = toks.size();
ret = new (pool.Allocate<Sentence>()) Sentence(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
// xml
ret->Init(system, size, system.options.reordering.max_distortion);
ReorderingConstraint &reorderingConstraint = ret->GetReorderingConstraint();
// set reordering walls, if "-monotone-at-punction" is set
if (system.options.reordering.monotone_at_punct && ret->GetSize()) {
reorderingConstraint.SetMonotoneAtPunctuation(*ret);
}
// set walls obtained from xml
for(size_t i=0; i<xmlOptions.size(); i++) {
const XMLOption *xmlOption = xmlOptions[i];
if(strcmp(xmlOption->GetNodeName(), "wall") == 0) {
UTIL_THROW_IF2(xmlOption->startPos > ret->GetSize(), "wall is beyond the sentence"); // no buggy walls, please
reorderingConstraint.SetWall(xmlOption->startPos - 1, true);
}
else if (strcmp(xmlOption->GetNodeName(), "zone") == 0) {
reorderingConstraint.SetZone( xmlOption->startPos, xmlOption->startPos + xmlOption->phraseSize -1 );
}
else if (strcmp(xmlOption->GetNodeName(), "ne") == 0) {
FactorType placeholderFactor = system.options.input.placeholder_factor;
UTIL_THROW_IF2(placeholderFactor == NOT_FOUND,
"Placeholder XML in input. Must have argument -placeholder-factor [NUM]");
UTIL_THROW_IF2(xmlOption->phraseSize != 1,
"Placeholder must only cover 1 word");
const Factor *factor = vocab.AddFactor(xmlOption->GetEntity(), system, false);
(*ret)[xmlOption->startPos][placeholderFactor] = factor;
}
else {
// default - forced translation. Add to class variable
ret->AddXMLOption(system, xmlOption);
}
}
reorderingConstraint.FinalizeWalls();
return ret;
return ret;
}
void Sentence::XMLParse(
MemPool &pool,
const System &system,
size_t depth,
const pugi::xml_node &parentNode,
std::vector<std::string> &toks,
vector<XMLOption*> &xmlOptions)
{ // pugixml
MemPool &pool,
const System &system,
size_t depth,
const pugi::xml_node &parentNode,
std::vector<std::string> &toks,
vector<XMLOption*> &xmlOptions)
{
// pugixml
for (pugi::xml_node childNode = parentNode.first_child(); childNode; childNode = childNode.next_sibling()) {
string nodeName = childNode.name();
//cerr << depth << " nodeName=" << nodeName << endl;
@ -139,17 +136,17 @@ void Sentence::XMLParse(
pugi::xml_attribute attr;
attr = childNode.attribute("translation");
if (!attr.empty()) {
xmlOption->SetTranslation(pool, attr.as_string());
xmlOption->SetTranslation(pool, attr.as_string());
}
attr = childNode.attribute("entity");
if (!attr.empty()) {
xmlOption->SetEntity(pool, attr.as_string());
xmlOption->SetEntity(pool, attr.as_string());
}
attr = childNode.attribute("prob");
if (!attr.empty()) {
xmlOption->prob = attr.as_float();
xmlOption->prob = attr.as_float();
}
xmlOptions.push_back(xmlOption);

20
moses2/PhraseBased/Sentence.h
View File

@ -24,11 +24,11 @@ class Sentence: public InputType, public PhraseImpl
public:
static Sentence *CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str);
const System &system, const std::string &str);
Sentence(MemPool &pool, size_t size)
:InputType(pool)
,PhraseImpl(pool, size)
:InputType(pool)
,PhraseImpl(pool, size)
{}
virtual ~Sentence()
@ -36,15 +36,15 @@ public:
protected:
static Sentence *CreateFromStringXML(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str);
const System &system, const std::string &str);
static void XMLParse(
MemPool &pool,
const System &system,
size_t depth,
const pugi::xml_node &parentNode,
std::vector<std::string> &toks,
std::vector<XMLOption*> &xmlOptions);
MemPool &pool,
const System &system,
size_t depth,
const pugi::xml_node &parentNode,
std::vector<std::string> &toks,
std::vector<XMLOption*> &xmlOptions);
};

Some files were not shown because too many files have changed in this diff Show More