moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
Hieu Hoang 2016-03-31 21:00:16 +01:00
parent 52b4d094bf
commit e6096aa8bf
194 changed files with 7568 additions and 7136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
contrib/other-builds/moses2/ArcLists.cpp
View File

@ -16,79 +16,81 @@ using namespace std;
namespace Moses2
{
ArcLists::ArcLists() {
// TODO Auto-generated constructor stub
}
ArcLists::~ArcLists() {
BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
const ArcList *arcList = collPair.second;
delete arcList;
}
}
void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,const HypothesisBase *otherHypo)
ArcLists::ArcLists()
{
//cerr << added << " " << currHypo << " " << otherHypo << endl;
ArcList *arcList;
if (added) {
// we're winners!
if (otherHypo) {
// there was a existing losing hypo
arcList = GetAndDetachArcList(otherHypo);
}
else {
// there was no existing hypo
arcList = new ArcList;
}
m_coll[currHypo] = arcList;
}
else {
// we're losers!
// there should be a winner, we're not doing beam pruning
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
arcList = GetArcList(otherHypo);
}
// TODO Auto-generated constructor stub
// in any case, add the curr hypo
arcList->push_back(currHypo);
}
ArcLists::~ArcLists()
{
BOOST_FOREACH(const Coll::value_type &collPair, m_coll){
const ArcList *arcList = collPair.second;
delete arcList;
}
}
void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,
const HypothesisBase *otherHypo)
{
//cerr << added << " " << currHypo << " " << otherHypo << endl;
ArcList *arcList;
if (added) {
// we're winners!
if (otherHypo) {
// there was a existing losing hypo
arcList = GetAndDetachArcList(otherHypo);
}
else {
// there was no existing hypo
arcList = new ArcList;
}
m_coll[currHypo] = arcList;
}
else {
// we're losers!
// there should be a winner, we're not doing beam pruning
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
arcList = GetArcList(otherHypo);
}
// in any case, add the curr hypo
arcList->push_back(currHypo);
}
ArcList *ArcLists::GetArcList(const HypothesisBase *hypo)
{
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
return arcList;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
return arcList;
}
const ArcList *ArcLists::GetArcList(const HypothesisBase *hypo) const
{
Coll::const_iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
return arcList;
Coll::const_iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
return arcList;
}
ArcList *ArcLists::GetAndDetachArcList(const HypothesisBase *hypo)
{
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
Coll::iterator iter = m_coll.find(hypo);
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
ArcList *arcList = iter->second;
m_coll.erase(iter);
m_coll.erase(iter);
return arcList;
return arcList;
}
void ArcLists::Sort()
{
BOOST_FOREACH(Coll::value_type &collPair, m_coll) {
ArcList &list = *collPair.second;
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
}
BOOST_FOREACH(Coll::value_type &collPair, m_coll){
ArcList &list = *collPair.second;
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
}
}
void ArcLists::Delete(const HypothesisBase *hypo)

24
contrib/other-builds/moses2/ArcLists.h
View File

@ -15,23 +15,25 @@ class HypothesisBase;
typedef std::vector<const HypothesisBase*> ArcList;
class ArcLists {
class ArcLists
{
public:
ArcLists();
virtual ~ArcLists();
ArcLists();
virtual ~ArcLists();
void AddArc(bool added, const HypothesisBase *currHypo, const HypothesisBase *otherHypo);
void Sort();
void Delete(const HypothesisBase *hypo);
void AddArc(bool added, const HypothesisBase *currHypo,
const HypothesisBase *otherHypo);
void Sort();
void Delete(const HypothesisBase *hypo);
const ArcList *GetArcList(const HypothesisBase *hypo) const;
const ArcList *GetArcList(const HypothesisBase *hypo) const;
protected:
typedef boost::unordered_map<const HypothesisBase*, ArcList*> Coll;
Coll m_coll;
typedef boost::unordered_map<const HypothesisBase*, ArcList*> Coll;
Coll m_coll;
ArcList *GetArcList(const HypothesisBase *hypo);
ArcList *GetAndDetachArcList(const HypothesisBase *hypo);
ArcList *GetArcList(const HypothesisBase *hypo);
ArcList *GetAndDetachArcList(const HypothesisBase *hypo);
};

59
contrib/other-builds/moses2/EstimatedScores.cpp
View File

@ -2,23 +2,23 @@
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <string>
#include <iostream>
@ -34,18 +34,18 @@ namespace Moses2
* /param bitmap coverage bitmap
*/
float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap ) const
float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap) const
{
const size_t notInGap= numeric_limits<size_t>::max();
const size_t notInGap = numeric_limits<size_t>::max();
size_t startGap = notInGap;
float estimatedScore = 0.0f;
for(size_t currPos = 0 ; currPos < bitmap.GetSize() ; currPos++) {
for (size_t currPos = 0; currPos < bitmap.GetSize(); currPos++) {
// start of a new gap?
if(bitmap.GetValue(currPos) == false && startGap == notInGap) {
if (bitmap.GetValue(currPos) == false && startGap == notInGap) {
startGap = currPos;
}
// end of a gap?
else if(bitmap.GetValue(currPos) == true && startGap != notInGap) {
else if (bitmap.GetValue(currPos) == true && startGap != notInGap) {
estimatedScore += GetValue(startGap, currPos - 1);
startGap = notInGap;
}
@ -73,37 +73,41 @@ float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap ) const
* /param endPos end of the span that is added to the coverage
*/
float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap, size_t startPos, size_t endPos ) const
float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap, size_t startPos,
size_t endPos) const
{
const size_t notInGap= numeric_limits<size_t>::max();
const size_t notInGap = numeric_limits<size_t>::max();
float estimatedScore = 0.0f;
size_t startGap = bitmap.GetFirstGapPos();
if (startGap == NOT_FOUND) return estimatedScore; // everything filled
// start loop at first gap
size_t startLoop = startGap+1;
size_t startLoop = startGap + 1;
if (startPos == startGap) { // unless covered by phrase
startGap = notInGap;
startLoop = endPos+1; // -> postpone start
startLoop = endPos + 1; // -> postpone start
}
size_t lastCovered = bitmap.GetLastPos();
if (endPos > lastCovered || lastCovered == NOT_FOUND) lastCovered = endPos;
for(size_t currPos = startLoop; currPos <= lastCovered ; currPos++) {
for (size_t currPos = startLoop; currPos <= lastCovered; currPos++) {
// start of a new gap?
if(startGap == notInGap && bitmap.GetValue(currPos) == false && (currPos < startPos || currPos > endPos)) {
if (startGap == notInGap && bitmap.GetValue(currPos) == false
&& (currPos < startPos || currPos > endPos)) {
startGap = currPos;
}
// end of a gap?
else if(startGap != notInGap && (bitmap.GetValue(currPos) == true || (startPos <= currPos && currPos <= endPos))) {
else if (startGap != notInGap
&& (bitmap.GetValue(currPos) == true
|| (startPos <= currPos && currPos <= endPos))) {
estimatedScore += GetValue(startGap, currPos - 1);
startGap = notInGap;
}
}
// coverage ending with gap?
if (lastCovered != bitmap.GetSize() - 1) {
estimatedScore += GetValue(lastCovered+1, bitmap.GetSize() - 1);
estimatedScore += GetValue(lastCovered + 1, bitmap.GetSize() - 1);
}
return estimatedScore;
@ -111,4 +115,3 @@ float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap, size_t startPos
}

51
contrib/other-builds/moses2/EstimatedScores.h
View File

@ -1,23 +1,23 @@
// $Id$
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
@ -31,26 +31,29 @@ namespace Moses2
class MemPool;
//! A square array of floats to store future costs in the phrase-based decoder
class EstimatedScores : public Matrix<float>
class EstimatedScores: public Matrix<float>
{
friend std::ostream& operator<<(std::ostream &out, const EstimatedScores &matrix);
friend std::ostream& operator<<(std::ostream &out,
const EstimatedScores &matrix);
public:
EstimatedScores(MemPool &pool, size_t size)
:Matrix<float>(pool, size, size)
{}
EstimatedScores(MemPool &pool, size_t size) :
Matrix<float>(pool, size, size)
{
}
~EstimatedScores(); // not implemented
float CalcEstimatedScore( Bitmap const& ) const;
float CalcEstimatedScore( Bitmap const&, size_t startPos, size_t endPos ) const;
float CalcEstimatedScore(Bitmap const&) const;
float CalcEstimatedScore(Bitmap const&, size_t startPos, size_t endPos) const;
};
inline std::ostream& operator<<(std::ostream &out, const EstimatedScores &matrix)
inline std::ostream& operator<<(std::ostream &out,
const EstimatedScores &matrix)
{
for (size_t endPos = 0 ; endPos < matrix.GetSize() ; endPos++) {
for (size_t startPos = 0 ; startPos < matrix.GetSize() ; startPos++)
for (size_t endPos = 0; endPos < matrix.GetSize(); endPos++) {
for (size_t startPos = 0; startPos < matrix.GetSize(); startPos++)
out << matrix.GetValue(startPos, endPos) << " ";
out << std::endl;
}

150
contrib/other-builds/moses2/FF/Distortion.cpp
View File

@ -16,50 +16,53 @@ using namespace std;
namespace Moses2
{
struct DistortionState_traditional : public FFState {
struct DistortionState_traditional: public FFState
{
Range range;
int first_gap;
DistortionState_traditional()
:range()
DistortionState_traditional() :
range()
{
// uninitialised
// uninitialised
}
void Set(const Range& wr, int fg)
{
range = wr;
first_gap = fg;
range = wr;
first_gap = fg;
}
size_t hash() const {
size_t hash() const
{
return range.GetEndPos();
}
virtual bool operator==(const FFState& other) const {
virtual bool operator==(const FFState& other) const
{
const DistortionState_traditional& o =
static_cast<const DistortionState_traditional&>(other);
static_cast<const DistortionState_traditional&>(other);
return range.GetEndPos() == o.range.GetEndPos();
}
virtual std::string ToString() const
{
stringstream sb;
sb << first_gap << " " << range;
return sb.str();
stringstream sb;
sb << first_gap << " " << range;
return sb.str();
}
};
///////////////////////////////////////////////////////////////////////
Distortion::Distortion(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
Distortion::Distortion(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
Distortion::~Distortion() {
// TODO Auto-generated destructor stub
Distortion::~Distortion()
{
// TODO Auto-generated destructor stub
}
FFState* Distortion::BlankState(MemPool &pool) const
@ -67,76 +70,70 @@ FFState* Distortion::BlankState(MemPool &pool) const
return new (pool.Allocate<DistortionState_traditional>()) DistortionState_traditional();
}
void Distortion::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
void Distortion::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
{
DistortionState_traditional &stateCast = static_cast<DistortionState_traditional&>(state);
DistortionState_traditional &stateCast =
static_cast<DistortionState_traditional&>(state);
// fake previous translated phrase start and end
size_t start = NOT_FOUND;
size_t end = NOT_FOUND;
/*
if (input.m_frontSpanCoveredLength > 0) {
// can happen with --continue-partial-translation
start = 0;
end = input.m_frontSpanCoveredLength -1;
}
*/
// fake previous translated phrase start and end
size_t start = NOT_FOUND;
size_t end = NOT_FOUND;
/*
if (input.m_frontSpanCoveredLength > 0) {
// can happen with --continue-partial-translation
start = 0;
end = input.m_frontSpanCoveredLength -1;
}
*/
stateCast.range = Range(start, end);
stateCast.first_gap = NOT_FOUND;
stateCast.range = Range(start, end);
stateCast.first_gap = NOT_FOUND;
}
void
Distortion::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void Distortion::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
}
void Distortion::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const DistortionState_traditional &prev = static_cast<const DistortionState_traditional&>(prevState);
SCORE distortionScore = CalculateDistortionScore(
prev.range,
hypo.GetInputPath().range,
prev.first_gap);
//cerr << "distortionScore=" << distortionScore << endl;
const DistortionState_traditional &prev =
static_cast<const DistortionState_traditional&>(prevState);
SCORE distortionScore = CalculateDistortionScore(prev.range,
hypo.GetInputPath().range, prev.first_gap);
//cerr << "distortionScore=" << distortionScore << endl;
scores.PlusEquals(mgr.system, *this, distortionScore);
scores.PlusEquals(mgr.system, *this, distortionScore);
DistortionState_traditional &stateCast = static_cast<DistortionState_traditional&>(state);
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
DistortionState_traditional &stateCast =
static_cast<DistortionState_traditional&>(state);
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
}
SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr, const int FirstGap) const
SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
const int FirstGap) const
{
bool useEarlyDistortionCost = false;
if(!useEarlyDistortionCost) {
return - (SCORE) ComputeDistortionDistance(prev, curr);
} else {
if (!useEarlyDistortionCost) {
return -(SCORE) ComputeDistortionDistance(prev, curr);
}
else {
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
Definitions:
S : current source range
S' : last translated source phrase range
S'' : longest fully-translated initial segment
*/
Definitions:
S : current source range
S' : last translated source phrase range
S'' : longest fully-translated initial segment
*/
int prefixEndPos = (int)FirstGap-1;
if((int)FirstGap==-1)
prefixEndPos = -1;
int prefixEndPos = (int) FirstGap - 1;
if ((int) FirstGap == -1) prefixEndPos = -1;
// case1: S is adjacent to S'' => return 0
if ((int) curr.GetStartPos() == prefixEndPos+1) {
if ((int) curr.GetStartPos() == prefixEndPos + 1) {
//IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
return 0;
}
@ -144,30 +141,33 @@ SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
// case2: S is to the left of S' => return 2(length(S))
if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
//IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
return (float) -2*(int)curr.GetNumWordsCovered();
return (float) -2 * (int) curr.GetNumWordsCovered();
}
// case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
if ((int) prev.GetEndPos() <= prefixEndPos) {
//IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
int z = (int)curr.GetStartPos()-prefixEndPos - 1;
return (float) -2*(z + (int)curr.GetNumWordsCovered());
int z = (int) curr.GetStartPos() - prefixEndPos - 1;
return (float) -2 * (z + (int) curr.GetNumWordsCovered());
}
// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
//IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());
return (float) -2
* ((int) curr.GetNumWordsBetween(prev) + (int) curr.GetNumWordsCovered());
}
}
int Distortion::ComputeDistortionDistance(const Range& prev, const Range& current) const
int Distortion::ComputeDistortionDistance(const Range& prev,
const Range& current) const
{
int dist = 0;
if (prev.GetNumWordsCovered() == 0) {
dist = current.GetStartPos();
} else {
dist = (int)prev.GetEndPos() - (int)current.GetStartPos() + 1 ;
}
else {
dist = (int) prev.GetEndPos() - (int) current.GetStartPos() + 1;
}
return abs(dist);
}

33
contrib/other-builds/moses2/FF/Distortion.h
View File

@ -15,37 +15,32 @@
namespace Moses2
{
class Distortion : public StatefulFeatureFunction
class Distortion: public StatefulFeatureFunction
{
public:
Distortion(size_t startInd, const std::string &line);
virtual ~Distortion();
Distortion(size_t startInd, const std::string &line);
virtual ~Distortion();
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void EvaluateWhenApplied(const std::deque<Hypothesis*> &hypos) const
{}
{
}
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
protected:
SCORE CalculateDistortionScore(const Range &prev, const Range &curr, const int FirstGap) const;
SCORE CalculateDistortionScore(const Range &prev, const Range &curr,
const int FirstGap) const;
int ComputeDistortionDistance(const Range& prev, const Range& current) const;

28
contrib/other-builds/moses2/FF/FeatureFunction.cpp
View File

@ -16,17 +16,16 @@ using namespace std;
namespace Moses2
{
FeatureFunction::FeatureFunction(size_t startInd, const std::string &line)
:m_startInd(startInd)
,m_numScores(1)
,m_PhraseTableInd(NOT_FOUND)
{
ParseLine(line);
cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
}
FeatureFunction::FeatureFunction(size_t startInd, const std::string &line) :
m_startInd(startInd), m_numScores(1), m_PhraseTableInd(NOT_FOUND)
{
ParseLine(line);
cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
}
FeatureFunction::~FeatureFunction() {
// TODO Auto-generated destructor stub
FeatureFunction::~FeatureFunction()
{
// TODO Auto-generated destructor stub
}
void FeatureFunction::ParseLine(const std::string &line)
@ -41,13 +40,13 @@ void FeatureFunction::ParseLine(const std::string &line)
for (size_t i = 1; i < toks.size(); ++i) {
vector<string> args = TokenizeFirstOnly(toks[i], "=");
UTIL_THROW_IF2(args.size() != 2,
"Incorrect format for feature function arg: " << toks[i]);
"Incorrect format for feature function arg: " << toks[i]);
pair<set<string>::iterator,bool> ret = keys.insert(args[0]);
pair<set<string>::iterator, bool> ret = keys.insert(args[0]);
UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line);
if (args[0] == "num-features") {
m_numScores = Scan<size_t>(args[1]);
m_numScores = Scan<size_t>(args[1]);
}
else if (args[0] == "name") {
m_name = args[1];
@ -68,7 +67,8 @@ void FeatureFunction::ReadParameters()
}
}
void FeatureFunction::SetParameter(const std::string& key, const std::string& value)
void FeatureFunction::SetParameter(const std::string& key,
const std::string& value)
{
if (key == "tuneable") {
m_tuneable = Scan<bool>(value);

122
contrib/other-builds/moses2/FF/FeatureFunction.h
View File

@ -24,70 +24,88 @@ class Scores;
class ManagerBase;
class MemPool;
namespace SCFG
{
class TargetPhrase;
}
namespace SCFG
{
class TargetPhrase;
}
class FeatureFunction {
class FeatureFunction
{
public:
FeatureFunction(size_t startInd, const std::string &line);
virtual ~FeatureFunction();
virtual void Load(System &system)
{}
FeatureFunction(size_t startInd, const std::string &line);
virtual ~FeatureFunction();
virtual void Load(System &system)
{
}
size_t GetStartInd() const
{ return m_startInd; }
size_t GetNumScores() const
{ return m_numScores; }
const std::string &GetName() const
{ return m_name; }
void SetName(const std::string &val)
{ m_name = val; }
size_t GetStartInd() const
{
return m_startInd;
}
size_t GetNumScores() const
{
return m_numScores;
}
const std::string &GetName() const
{
return m_name;
}
void SetName(const std::string &val)
{
m_name = val;
}
virtual size_t HasPhraseTableInd() const
{ return false; }
void SetPhraseTableInd(size_t ind)
{ m_PhraseTableInd = ind; }
size_t GetPhraseTableInd() const
{ return m_PhraseTableInd; }
virtual size_t HasPhraseTableInd() const
{
return false;
}
void SetPhraseTableInd(size_t ind)
{
m_PhraseTableInd = ind;
}
size_t GetPhraseTableInd() const
{
return m_PhraseTableInd;
}
// may have more factors than actually need, but not guaranteed.
// For SCFG decoding, the source contains non-terminals, NOT the raw
// source from the input sentence
virtual void
EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const = 0;
// may have more factors than actually need, but not guaranteed.
// For SCFG decoding, the source contains non-terminals, NOT the raw
// source from the input sentence
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const = 0;
/*
virtual void
EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const SCFG::TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const {}
*/
/*
virtual void
EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const SCFG::TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const {}
*/
virtual void
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const
{}
virtual void EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps, const Phrase &sourcePhrase) const
{
}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing() {}
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing()
{
}
protected:
size_t m_startInd;
size_t m_numScores;
size_t m_PhraseTableInd;
std::string m_name;
std::vector<std::vector<std::string> > m_args;
bool m_tuneable;
size_t m_startInd;
size_t m_numScores;
size_t m_PhraseTableInd;
std::string m_name;
std::vector<std::vector<std::string> > m_args;
bool m_tuneable;
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void ReadParameters();
void ParseLine(const std::string &line);
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void ReadParameters();
void ParseLine(const std::string &line);
};
}

224
contrib/other-builds/moses2/FF/FeatureFunctions.cpp
View File

@ -31,41 +31,41 @@ using namespace std;
namespace Moses2
{
FeatureFunctions::FeatureFunctions(System &system)
:m_system(system)
,m_ffStartInd(0)
FeatureFunctions::FeatureFunctions(System &system) :
m_system(system), m_ffStartInd(0)
{
//m_registry.PrintFF();
//m_registry.PrintFF();
}
FeatureFunctions::~FeatureFunctions() {
RemoveAllInColl(m_featureFunctions);
FeatureFunctions::~FeatureFunctions()
{
RemoveAllInColl(m_featureFunctions);
}
void FeatureFunctions::Load()
{
// load, everything but pts
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
if (pt) {
// do nothing. load pt last
}
else {
cerr << "Loading " << nonConstFF->GetName() << endl;
nonConstFF->Load(m_system);
cerr << "Finished loading " << nonConstFF->GetName() << endl;
}
if (pt) {
// do nothing. load pt last
}
else {
cerr << "Loading " << nonConstFF->GetName() << endl;
nonConstFF->Load(m_system);
cerr << "Finished loading " << nonConstFF->GetName() << endl;
}
}
// load pt
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
cerr << "Loading " << nonConstPT->GetName() << endl;
nonConstPT->Load(m_system);
cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
// load pt
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
cerr << "Loading " << nonConstPT->GetName() << endl;
nonConstPT->Load(m_system);
cerr << "Finished loading " << nonConstPT->GetName() << endl;
}
}
void FeatureFunctions::Create()
@ -75,125 +75,129 @@ void FeatureFunctions::Create()
const PARAM_VEC *ffParams = params.GetParam("feature");
UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");
BOOST_FOREACH(const std::string &line, *ffParams) {
cerr << "line=" << line << endl;
FeatureFunction *ff = Create(line);
BOOST_FOREACH(const std::string &line, *ffParams){
cerr << "line=" << line << endl;
FeatureFunction *ff = Create(line);
m_featureFunctions.push_back(ff);
m_featureFunctions.push_back(ff);
StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
if (sfff) {
sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
m_statefulFeatureFunctions.push_back(sfff);
}
if (ff->HasPhraseTableInd()) {
ff->SetPhraseTableInd(m_withPhraseTableInd.size());
m_withPhraseTableInd.push_back(ff);
}
PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
if (pt) {
pt->SetPtInd(m_phraseTables.size());
m_phraseTables.push_back(pt);
}
StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
if (sfff) {
sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
m_statefulFeatureFunctions.push_back(sfff);
}
if (ff->HasPhraseTableInd()) {
ff->SetPhraseTableInd(m_withPhraseTableInd.size());
m_withPhraseTableInd.push_back(ff);
}
PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
if (pt) {
pt->SetPtInd(m_phraseTables.size());
m_phraseTables.push_back(pt);
}
}
}
FeatureFunction *FeatureFunctions::Create(const std::string &line)
{
vector<string> toks = Tokenize(line);
vector<string> toks = Tokenize(line);
FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
UTIL_THROW_IF2(ff == NULL, "Feature function not created");
FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
UTIL_THROW_IF2(ff == NULL, "Feature function not created");
// name
if (ff->GetName() == "") {
ff->SetName(GetDefaultName(toks[0]));
}
// name
if (ff->GetName() == "") {
ff->SetName(GetDefaultName(toks[0]));
}
m_ffStartInd += ff->GetNumScores();
m_ffStartInd += ff->GetNumScores();
return ff;
return ff;
}
std::string FeatureFunctions::GetDefaultName(const std::string &stub)
{
size_t ind;
boost::unordered_map<std::string, size_t>::iterator iter = m_defaultNames.find(stub);
if (iter == m_defaultNames.end()) {
m_defaultNames[stub] = 0;
ind = 0;
}
else {
ind = ++(iter->second);
}
return stub + SPrint(ind);
}
const FeatureFunction *FeatureFunctions::FindFeatureFunction(const std::string &name) const
{
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
if (ff->GetName() == name) {
return ff;
}
size_t ind;
boost::unordered_map<std::string, size_t>::iterator iter =
m_defaultNames.find(stub);
if (iter == m_defaultNames.end()) {
m_defaultNames[stub] = 0;
ind = 0;
}
return NULL;
else {
ind = ++(iter->second);
}
return stub + SPrint(ind);
}
const PhraseTable *FeatureFunctions::GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd)
const FeatureFunction *FeatureFunctions::FindFeatureFunction(
const std::string &name) const
{
// assume only 1 unk wp
std::vector<const PhraseTable*> tmpVec(m_phraseTables);
std::vector<const PhraseTable*>::iterator iter;
for (iter = tmpVec.begin(); iter != tmpVec.end(); ++iter) {
const PhraseTable *pt = *iter;
const UnknownWordPenalty *unkWP = dynamic_cast<const UnknownWordPenalty *>(pt);
if (unkWP) {
tmpVec.erase(iter);
break;
}
}
const PhraseTable *pt = tmpVec[ptInd];
return pt;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
if (ff->GetName() == name) {
return ff;
}
}
return NULL;
}
void
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, TargetPhrase &targetPhrase) const
const PhraseTable *FeatureFunctions::GetPhraseTablesExcludeUnknownWordPenalty(
size_t ptInd)
{
// assume only 1 unk wp
std::vector<const PhraseTable*> tmpVec(m_phraseTables);
std::vector<const PhraseTable*>::iterator iter;
for (iter = tmpVec.begin(); iter != tmpVec.end(); ++iter) {
const PhraseTable *pt = *iter;
const UnknownWordPenalty *unkWP =
dynamic_cast<const UnknownWordPenalty *>(pt);
if (unkWP) {
tmpVec.erase(iter);
break;
}
}
const PhraseTable *pt = tmpVec[ptInd];
return pt;
}
void FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, TargetPhrase &targetPhrase) const
{
SCORE estimatedScore = 0;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
}
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
}
targetPhrase.SetEstimatedScore(estimatedScore);
}
void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const
void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps, const Phrase &sourcePhrase) const
{
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
}
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
}
}
/*
void
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const
{
SCORE estimatedScore = 0;
void
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const
{
SCORE estimatedScore = 0;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
}
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
}
//targetPhrase.SetEstimatedScore(estimatedScore);
}
*/
//targetPhrase.SetEstimatedScore(estimatedScore);
}
*/
}

78
contrib/other-builds/moses2/FF/FeatureFunctions.h
View File

@ -30,55 +30,65 @@ class Scores;
namespace SCFG
{
class TargetPhrase;
class TargetPhrase;
}
class FeatureFunctions {
class FeatureFunctions
{
public:
std::vector<const PhraseTable*> m_phraseTables;
std::vector<const PhraseTable*> m_phraseTables;
FeatureFunctions(System &system);
virtual ~FeatureFunctions();
FeatureFunctions(System &system);
virtual ~FeatureFunctions();
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const
{ return m_featureFunctions; }
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const
{
return m_featureFunctions;
}
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const
{ return m_statefulFeatureFunctions; }
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const
{
return m_statefulFeatureFunctions;
}
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const
{ return m_withPhraseTableInd; }
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const
{
return m_withPhraseTableInd;
}
size_t GetNumScores() const
{ return m_ffStartInd; }
size_t GetNumScores() const
{
return m_ffStartInd;
}
void Create();
void Load();
void Create();
void Load();
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
const PhraseTable *GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd);
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
const PhraseTable *GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd);
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, TargetPhrase &targetPhrase) const;
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const;
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, TargetPhrase &targetPhrase) const;
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
const Phrase &sourcePhrase) const;
/*
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const;
*/
/*
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const;
*/
protected:
std::vector<const FeatureFunction*> m_featureFunctions;
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
std::vector<const FeatureFunction*> m_withPhraseTableInd;
boost::unordered_map<std::string, size_t> m_defaultNames;
System &m_system;
size_t m_ffStartInd;
std::vector<const FeatureFunction*> m_featureFunctions;
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
std::vector<const FeatureFunction*> m_withPhraseTableInd;
boost::unordered_map<std::string, size_t> m_defaultNames;
System &m_system;
size_t m_ffStartInd;
FeatureFunction *Create(const std::string &line);
std::string GetDefaultName(const std::string &stub);
FeatureFunction *Create(const std::string &line);
std::string GetDefaultName(const std::string &stub);
FeatureRegistry m_registry;
FeatureRegistry m_registry;
};
}

34
contrib/other-builds/moses2/FF/FeatureRegistry.cpp
View File

@ -19,20 +19,22 @@ using namespace std;
namespace Moses2
{
template <class F>
class DefaultFeatureFactory : public FeatureFactory
template<class F>
class DefaultFeatureFactory: public FeatureFactory
{
public:
FeatureFunction *Create(size_t startInd, const std::string &line) {
return new F(startInd, line);
}
FeatureFunction *Create(size_t startInd, const std::string &line)
{
return new F(startInd, line);
}
};
////////////////////////////////////////////////////////////////////
class KenFactory : public FeatureFactory
class KenFactory: public FeatureFactory
{
public:
FeatureFunction *Create(size_t startInd, const std::string &line) {
FeatureFunction *Create(size_t startInd, const std::string &line)
{
ConstructKenLM(startInd, line);
}
};
@ -41,9 +43,9 @@ public:
FeatureRegistry::FeatureRegistry()
{
// Feature with same name as class
#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >());
#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >());
// Feature with different name than class.
#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >());
#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >());
MOSES_FNAME2("PhraseDictionaryMemory", PhraseTableMemory);
MOSES_FNAME(ProbingPT);
@ -69,19 +71,21 @@ FeatureRegistry::~FeatureRegistry()
void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory)
{
std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name, boost::shared_ptr<FeatureFactory>(factory));
std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name,
boost::shared_ptr<FeatureFactory>(factory));
if (!registry_.insert(to_ins).second) {
cerr << "Duplicate feature name " << name << endl;
abort();
cerr << "Duplicate feature name " << name << endl;
abort();
}
}
FeatureFunction *FeatureRegistry::Construct(size_t startInd, const std::string &name, const std::string &line)
FeatureFunction *FeatureRegistry::Construct(size_t startInd,
const std::string &name, const std::string &line)
{
Map::iterator i = registry_.find(name);
if (i == registry_.end()) {
cerr << "Feature name " << name << " is not registered.";
abort();
cerr << "Feature name " << name << " is not registered.";
abort();
}
FeatureFactory *fact = i->second.get();
FeatureFunction *ff = fact->Create(startInd, line);

12
contrib/other-builds/moses2/FF/FeatureRegistry.h
View File

@ -10,12 +10,16 @@ class FeatureFunction;
class FeatureFactory
{
public:
virtual ~FeatureFactory() {}
virtual ~FeatureFactory()
{
}
virtual FeatureFunction *Create(size_t startInd, const std::string &line) = 0;
protected:
FeatureFactory() {}
FeatureFactory()
{
}
};
////////////////////////////////////////////////////////////////////
@ -26,7 +30,8 @@ public:
~FeatureRegistry();
FeatureFunction *Construct(size_t startInd, const std::string &name, const std::string &line);
FeatureFunction *Construct(size_t startInd, const std::string &name,
const std::string &line);
void PrintFF() const;
private:
@ -39,6 +44,5 @@ private:
////////////////////////////////////////////////////////////////////
}

71
contrib/other-builds/moses2/FF/LexicalReordering/BidirectionalReorderingState.cpp
View File

@ -11,43 +11,37 @@
using namespace std;
namespace Moses2 {
namespace Moses2
{
BidirectionalReorderingState::BidirectionalReorderingState(
const LRModel &config,
LRState *bw,
LRState *fw,
size_t offset)
: LRState(config,
LRModel::Bidirectional,
offset)
, m_backward(bw)
, m_forward(fw)
{}
const LRModel &config, LRState *bw, LRState *fw, size_t offset) :
LRState(config, LRModel::Bidirectional, offset), m_backward(bw), m_forward(
fw)
{
}
BidirectionalReorderingState::~BidirectionalReorderingState() {
// TODO Auto-generated destructor stub
BidirectionalReorderingState::~BidirectionalReorderingState()
{
// TODO Auto-generated destructor stub
}
void BidirectionalReorderingState::Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage)
const TargetPhrase &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
if (m_backward) {
m_backward->Init(prev, topt, path, first, coverage);
}
if (m_forward) {
m_forward->Init(prev, topt, path, first, coverage);
}
if (m_backward) {
m_backward->Init(prev, topt, path, first, coverage);
}
if (m_forward) {
m_forward->Init(prev, topt, path, first, coverage);
}
}
std::string BidirectionalReorderingState::ToString() const
{ return "BidirectionalReorderingState "
+ SPrint(this) + " "
+ SPrint(m_backward) + " "
+ SPrint(m_forward);
{
return "BidirectionalReorderingState " + SPrint(this) + " "
+ SPrint(m_backward) + " " + SPrint(m_forward);
}
size_t BidirectionalReorderingState::hash() const
@ -62,23 +56,24 @@ bool BidirectionalReorderingState::operator==(const FFState& o) const
{
if (&o == this) return 0;
BidirectionalReorderingState const &other
= static_cast<BidirectionalReorderingState const&>(o);
BidirectionalReorderingState const &other =
static_cast<BidirectionalReorderingState const&>(o);
bool ret = (*m_backward == *other.m_backward) && (*m_forward == *other.m_forward);
bool ret = (*m_backward == *other.m_backward)
&& (*m_forward == *other.m_forward);
return ret;
}
void BidirectionalReorderingState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
BidirectionalReorderingState &stateCast = static_cast<BidirectionalReorderingState&>(state);
m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores, *stateCast.m_backward);
m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores, *stateCast.m_forward);
BidirectionalReorderingState &stateCast =
static_cast<BidirectionalReorderingState&>(state);
m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores,
*stateCast.m_backward);
m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores,
*stateCast.m_forward);
}
} /* namespace Moses2 */

26
contrib/other-builds/moses2/FF/LexicalReordering/BidirectionalReorderingState.h
View File

@ -7,36 +7,28 @@
#pragma once
#include "LRState.h"
namespace Moses2 {
namespace Moses2
{
class BidirectionalReorderingState: public LRState
{
public:
BidirectionalReorderingState(
const LRModel &config,
LRState *bw,
LRState *fw,
size_t offset);
BidirectionalReorderingState(const LRModel &config, LRState *bw, LRState *fw,
size_t offset);
virtual ~BidirectionalReorderingState();
void Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage);
void Init(const LRState *prev, const TargetPhrase &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:
LRState *m_backward;

44
contrib/other-builds/moses2/FF/LexicalReordering/HReorderingBackwardState.cpp
View File

@ -9,27 +9,24 @@
#include "../../PhraseBased/Hypothesis.h"
#include "../../PhraseBased/Manager.h"
namespace Moses2 {
HReorderingBackwardState::HReorderingBackwardState(
MemPool &pool,
const LRModel &config,
size_t offset)
:LRState(config, LRModel::Backward, offset)
,reoStack(pool)
namespace Moses2
{
// TODO Auto-generated constructor stub
HReorderingBackwardState::HReorderingBackwardState(MemPool &pool,
const LRModel &config, size_t offset) :
LRState(config, LRModel::Backward, offset), reoStack(pool)
{
// TODO Auto-generated constructor stub
}
HReorderingBackwardState::~HReorderingBackwardState() {
// TODO Auto-generated destructor stub
HReorderingBackwardState::~HReorderingBackwardState()
{
// TODO Auto-generated destructor stub
}
void HReorderingBackwardState::Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const TargetPhrase &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
@ -44,26 +41,25 @@ size_t HReorderingBackwardState::hash() const
bool HReorderingBackwardState::operator==(const FFState& o) const
{
const HReorderingBackwardState& other
= static_cast<const HReorderingBackwardState&>(o);
const HReorderingBackwardState& other =
static_cast<const HReorderingBackwardState&>(o);
bool ret = reoStack == other.reoStack;
return ret;
}
std::string HReorderingBackwardState::ToString() const
{
return "HReorderingBackwardState " + SPrint(m_offset);
return "HReorderingBackwardState " + SPrint(m_offset);
}
void HReorderingBackwardState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
HReorderingBackwardState &nextState = static_cast<HReorderingBackwardState&>(state);
nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, NULL);
HReorderingBackwardState &nextState =
static_cast<HReorderingBackwardState&>(state);
nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
NULL);
nextState.reoStack = reoStack;
const Range &swrange = hypo.GetInputPath().range;

26
contrib/other-builds/moses2/FF/LexicalReordering/HReorderingBackwardState.h
View File

@ -8,36 +8,28 @@
#include "LRState.h"
#include "ReorderingStack.h"
namespace Moses2 {
namespace Moses2
{
class HReorderingBackwardState : public LRState
class HReorderingBackwardState: public LRState
{
private:
ReorderingStack reoStack;
public:
HReorderingBackwardState(
MemPool &pool,
const LRModel &config,
size_t offset);
HReorderingBackwardState(MemPool &pool, const LRModel &config, size_t offset);
virtual void Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage);
virtual void Init(const LRState *prev, const TargetPhrase &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
virtual ~HReorderingBackwardState();
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
};

49
contrib/other-builds/moses2/FF/LexicalReordering/HReorderingForwardState.cpp
View File

@ -10,27 +10,24 @@
#include "../../PhraseBased/Manager.h"
#include "../../PhraseBased/Hypothesis.h"
namespace Moses2 {
namespace Moses2
{
HReorderingForwardState::HReorderingForwardState(
const LRModel &config,
size_t offset)
: LRState(config, LRModel::Forward, offset)
, m_first(true)
HReorderingForwardState::HReorderingForwardState(const LRModel &config,
size_t offset) :
LRState(config, LRModel::Forward, offset), m_first(true)
{
prevPath = NULL;
m_coverage = NULL;
}
HReorderingForwardState::~HReorderingForwardState() {
// TODO Auto-generated destructor stub
HReorderingForwardState::~HReorderingForwardState()
{
// TODO Auto-generated destructor stub
}
void HReorderingForwardState::Init(
const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
void HReorderingForwardState::Init(const LRState *prev,
const TargetPhrase &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
@ -50,26 +47,24 @@ bool HReorderingForwardState::operator==(const FFState& o) const
{
if (&o == this) return true;
HReorderingForwardState const& other
= static_cast<HReorderingForwardState const&>(o);
HReorderingForwardState const& other =
static_cast<HReorderingForwardState const&>(o);
int compareScores = ((prevPath->range == other.prevPath->range)
? ComparePrevScores(other.prevTP)
: (prevPath->range < other.prevPath->range) ? -1 : 1);
int compareScores = (
(prevPath->range == other.prevPath->range) ?
ComparePrevScores(other.prevTP) :
(prevPath->range < other.prevPath->range) ? -1 : 1);
return compareScores == 0;
}
std::string HReorderingForwardState::ToString() const
{
return "HReorderingForwardState " + SPrint(m_offset);
return "HReorderingForwardState " + SPrint(m_offset);
}
void HReorderingForwardState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
const Range &cur = hypo.GetInputPath().range;
// keep track of the current coverage ourselves so we don't need the hypothesis
@ -83,8 +78,10 @@ void HReorderingForwardState::Expand(const ManagerBase &mgr,
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
}
HReorderingForwardState &stateCast = static_cast<HReorderingForwardState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, &cov);
HReorderingForwardState &stateCast =
static_cast<HReorderingForwardState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
&cov);
}
} /* namespace Moses2 */

26
contrib/other-builds/moses2/FF/LexicalReordering/HReorderingForwardState.h
View File

@ -7,35 +7,27 @@
#pragma once
#include "LRState.h"
namespace Moses2 {
namespace Moses2
{
class Range;
class Bitmap;
class InputPathBase;
class HReorderingForwardState : public LRState
class HReorderingForwardState: public LRState
{
public:
HReorderingForwardState(
const LRModel &config,
size_t offset);
HReorderingForwardState(const LRModel &config, size_t offset);
virtual ~HReorderingForwardState();
void Init(
const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage);
void Init(const LRState *prev, const TargetPhrase &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const;
void Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:
bool m_first;

226
contrib/other-builds/moses2/FF/LexicalReordering/LRModel.cpp
View File

@ -18,144 +18,138 @@
using namespace std;
namespace Moses2 {
namespace Moses2
{
bool
IsMonotonicStep(Range const& prev, // words range of last source phrase
Range const& cur, // words range of current source phrase
Bitmap const& cov) // coverage bitmap
bool IsMonotonicStep(Range const& prev, // words range of last source phrase
Range const& cur, // words range of current source phrase
Bitmap const& cov) // coverage bitmap
{
size_t e = prev.GetEndPos() + 1;
size_t s = cur.GetStartPos();
return (s == e || (s >= e && !cov.GetValue(e)));
}
bool
IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
bool IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
{
size_t s = prev.GetStartPos();
size_t e = cur.GetEndPos();
return (e+1 == s || (e < s && !cov.GetValue(s-1)));
return (e + 1 == s || (e < s && !cov.GetValue(s - 1)));
}
LRModel::LRModel(const std::string &modelType, LexicalReordering &ff)
: m_modelType(None)
, m_phraseBased(true)
, m_collapseScores(false)
, m_direction(Backward)
, m_scoreProducer(&ff)
LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(
Backward), m_scoreProducer(&ff)
{
std::vector<std::string> config = Tokenize(modelType, "-");
for (size_t i=0; i<config.size(); ++i) {
if (config[i] == "hier") {
m_phraseBased = false;
} else if (config[i] == "phrase") {
m_phraseBased = true;
} else if (config[i] == "wbe") {
m_phraseBased = true;
}
// no word-based decoding available, fall-back to phrase-based
// This is the old lexical reordering model combination of moses
for (size_t i = 0; i < config.size(); ++i) {
if (config[i] == "hier") {
m_phraseBased = false;
}
else if (config[i] == "phrase") {
m_phraseBased = true;
}
else if (config[i] == "wbe") {
m_phraseBased = true;
}
// no word-based decoding available, fall-back to phrase-based
// This is the old lexical reordering model combination of moses
else if (config[i] == "msd") {
m_modelType = MSD;
} else if (config[i] == "mslr") {
m_modelType = MSLR;
} else if (config[i] == "monotonicity") {
m_modelType = Monotonic;
} else if (config[i] == "leftright") {
m_modelType = LeftRight;
}
else if (config[i] == "msd") {
m_modelType = MSD;
}
else if (config[i] == "mslr") {
m_modelType = MSLR;
}
else if (config[i] == "monotonicity") {
m_modelType = Monotonic;
}
else if (config[i] == "leftright") {
m_modelType = LeftRight;
}
// unidirectional is deprecated, use backward instead
else if (config[i] == "unidirectional") {
m_direction = Backward;
} else if (config[i] == "backward") {
m_direction = Backward;
} else if (config[i] == "forward") {
m_direction = Forward;
} else if (config[i] == "bidirectional") {
m_direction = Bidirectional;
}
// unidirectional is deprecated, use backward instead
else if (config[i] == "unidirectional") {
m_direction = Backward;
}
else if (config[i] == "backward") {
m_direction = Backward;
}
else if (config[i] == "forward") {
m_direction = Forward;
}
else if (config[i] == "bidirectional") {
m_direction = Bidirectional;
}
else if (config[i] == "f") {
m_condition = F;
} else if (config[i] == "fe") {
m_condition = FE;
}
else if (config[i] == "f") {
m_condition = F;
}
else if (config[i] == "fe") {
m_condition = FE;
}
else if (config[i] == "collapseff") {
m_collapseScores = true;
} else if (config[i] == "allff") {
m_collapseScores = false;
} else {
std::cerr
<< "Illegal part in the lexical reordering configuration string: "
<< config[i] << std::endl;
exit(1);
}
else if (config[i] == "collapseff") {
m_collapseScores = true;
}
else if (config[i] == "allff") {
m_collapseScores = false;
}
else {
std::cerr
<< "Illegal part in the lexical reordering configuration string: "
<< config[i] << std::endl;
exit(1);
}
}
if (m_modelType == None) {
std::cerr
<< "You need to specify the type of the reordering model "
<< "(msd, monotonicity,...)" << std::endl;
exit(1);
std::cerr << "You need to specify the type of the reordering model "
<< "(msd, monotonicity,...)" << std::endl;
exit(1);
}
}
LRModel::~LRModel() {
// TODO Auto-generated destructor stub
LRModel::~LRModel()
{
// TODO Auto-generated destructor stub
}
size_t
LRModel::
GetNumberOfTypes() const
size_t LRModel::GetNumberOfTypes() const
{
return ((m_modelType == MSD) ? 3 :
(m_modelType == MSLR) ? 4 : 2);
return ((m_modelType == MSD) ? 3 : (m_modelType == MSLR) ? 4 : 2);
}
/// return orientation for the first phrase
LRModel::ReorderingType
LRModel::
GetOrientation(Range const& cur) const
LRModel::ReorderingType LRModel::GetOrientation(Range const& cur) const
{
UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None");
return ((m_modelType == LeftRight) ? R :
(cur.GetStartPos() == 0) ? M :
(m_modelType == MSD) ? D :
(m_modelType == MSLR) ? DR : NM);
return ((m_modelType == LeftRight) ? R : (cur.GetStartPos() == 0) ? M :
(m_modelType == MSD) ? D : (m_modelType == MSLR) ? DR : NM);
}
LRModel::ReorderingType
LRModel::
GetOrientation(Range const& prev, Range const& cur) const
LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
Range const& cur) const
{
UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified");
return ((m_modelType == LeftRight)
? prev.GetEndPos() <= cur.GetStartPos() ? R : L
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M
: (m_modelType == Monotonic) ? NM
: (prev.GetStartPos() == cur.GetEndPos() + 1) ? S
: (m_modelType == MSD) ? D
: (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
return (
(m_modelType == LeftRight) ? prev.GetEndPos() <= cur.GetStartPos() ? R : L
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M :
(m_modelType == Monotonic) ? NM :
(prev.GetStartPos() == cur.GetEndPos() + 1) ? S :
(m_modelType == MSD) ? D :
(cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
}
LRModel::ReorderingType
LRModel::
GetOrientation(int const reoDistance) const
LRModel::ReorderingType LRModel::GetOrientation(int const reoDistance) const
{
// this one is for HierarchicalReorderingBackwardState
return ((m_modelType == LeftRight)
? (reoDistance >= 1) ? R : L
: (reoDistance == 1) ? M
: (m_modelType == Monotonic) ? NM
: (reoDistance == -1) ? S
: (m_modelType == MSD) ? D
: (reoDistance > 1) ? DR : DL);
return ((m_modelType == LeftRight) ? (reoDistance >= 1) ? R : L
: (reoDistance == 1) ? M : (m_modelType == Monotonic) ? NM :
(reoDistance == -1) ? S : (m_modelType == MSD) ? D :
(reoDistance > 1) ? DR : DL);
}
LRState *LRModel::CreateLRState(MemPool &pool) const
@ -163,47 +157,53 @@ LRState *LRModel::CreateLRState(MemPool &pool) const
LRState *bwd = NULL, *fwd = NULL;
size_t offset = 0;
switch(m_direction) {
switch (m_direction) {
case Backward:
case Bidirectional:
if (m_phraseBased) {
bwd = new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(*this, Backward, offset);
bwd =
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Backward, offset);
//cerr << "bwd=" << bwd << bwd->ToString() << endl;
}
else {
bwd = new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(pool, *this, offset);
bwd =
new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(
pool, *this, offset);
}
offset += m_collapseScores ? 1 : GetNumberOfTypes();
if (m_direction == Backward) return bwd; // else fall through
case Forward:
if (m_phraseBased) {
fwd = new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(*this, Forward, offset);
fwd =
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
*this, Forward, offset);
//cerr << "fwd=" << fwd << fwd->ToString() << endl;
}
else {
fwd = new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(*this, offset);
fwd =
new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(
*this, offset);
}
offset += m_collapseScores ? 1 : GetNumberOfTypes();
if (m_direction == Forward) return fwd;
}
//cerr << "LRStates:" << *bwd << endl << *fwd << endl;
BidirectionalReorderingState *ret = new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(*this, bwd, fwd, 0);
BidirectionalReorderingState *ret =
new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(
*this, bwd, fwd, 0);
return ret;
}
LRModel::ReorderingType
LRModel::
GetOrientation(Range const& prev, Range const& cur,
Bitmap const& cov) const
LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
Range const& cur, Bitmap const& cov) const
{
return ((m_modelType == LeftRight)
? cur.GetStartPos() > prev.GetEndPos() ? R : L
: IsMonotonicStep(prev,cur,cov) ? M
: (m_modelType == Monotonic) ? NM
: IsSwap(prev,cur,cov) ? S
: (m_modelType == MSD) ? D
: cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
return (
(m_modelType == LeftRight) ? cur.GetStartPos() > prev.GetEndPos() ? R : L
: IsMonotonicStep(prev, cur, cov) ? M : (m_modelType == Monotonic) ? NM :
IsSwap(prev, cur, cov) ? S : (m_modelType == MSD) ? D :
cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
}
} /* namespace Moses2 */

117
contrib/other-builds/moses2/FF/LexicalReordering/LRModel.h
View File

@ -7,7 +7,8 @@
#pragma once
#include <string>
namespace Moses2 {
namespace Moses2
{
class MemPool;
class Range;
@ -15,69 +16,83 @@ class Bitmap;
class LRState;
class LexicalReordering;
class LRModel {
class LRModel
{
public:
enum ModelType { Monotonic, MSD, MSLR, LeftRight, None };
enum Direction { Forward, Backward, Bidirectional };
enum Condition { F, E, FE };
enum ReorderingType {
M = 0, // monotonic
NM = 1, // non-monotonic
S = 1, // swap
D = 2, // discontinuous
DL = 2, // discontinuous, left
DR = 3, // discontinuous, right
R = 0, // right
L = 1, // left
MAX = 3, // largest possible
NONE = 4 // largest possible
enum ModelType
{
Monotonic, MSD, MSLR, LeftRight, None
};
enum Direction
{
Forward, Backward, Bidirectional
};
enum Condition
{
F, E, FE
};
LRModel(const std::string &modelType, LexicalReordering &ff);
virtual ~LRModel();
enum ReorderingType
{
M = 0, // monotonic
NM = 1, // non-monotonic
S = 1, // swap
D = 2, // discontinuous
DL = 2, // discontinuous, left
DR = 3, // discontinuous, right
R = 0, // right
L = 1, // left
MAX = 3, // largest possible
NONE = 4 // largest possible
};
ModelType GetModelType() const {
return m_modelType;
}
Direction GetDirection() const {
return m_direction;
}
Condition GetCondition() const {
return m_condition;
}
LRModel(const std::string &modelType, LexicalReordering &ff);
virtual ~LRModel();
bool
IsPhraseBased() const {
return m_phraseBased;
}
ModelType GetModelType() const
{
return m_modelType;
}
Direction GetDirection() const
{
return m_direction;
}
Condition GetCondition() const
{
return m_condition;
}
bool
CollapseScores() const {
return m_collapseScores;
}
bool IsPhraseBased() const
{
return m_phraseBased;
}
size_t GetNumberOfTypes() const;
bool CollapseScores() const
{
return m_collapseScores;
}
LexicalReordering*
GetScoreProducer() const {
return m_scoreProducer;
}
size_t GetNumberOfTypes() const;
LRState *CreateLRState(MemPool &pool) const;
LexicalReordering*
GetScoreProducer() const
{
return m_scoreProducer;
}
ReorderingType // for first phrase in phrase-based
GetOrientation(Range const& cur) const;
LRState *CreateLRState(MemPool &pool) const;
ReorderingType // for non-first phrases in phrase-based
GetOrientation(Range const& prev, Range const& cur) const;
ReorderingType // for first phrase in phrase-based
GetOrientation(Range const& cur) const;
ReorderingType // for HReorderingForwardState
GetOrientation(Range const& prev, Range const& cur,
Bitmap const& cov) const;
ReorderingType // for non-first phrases in phrase-based
GetOrientation(Range const& prev, Range const& cur) const;
ReorderingType // for HReorderingBackwarddState
GetOrientation(int const reoDistance) const;
ReorderingType // for HReorderingForwardState
GetOrientation(Range const& prev, Range const& cur, Bitmap const& cov) const;
ReorderingType // for HReorderingBackwarddState
GetOrientation(int const reoDistance) const;
protected:

72
contrib/other-builds/moses2/FF/LexicalReordering/LRState.cpp
View File

@ -10,54 +10,47 @@
using namespace std;
namespace Moses2 {
namespace Moses2
{
class InputType;
LRState::LRState(const LRModel &config,
LRModel::Direction dir,
size_t offset)
:m_configuration(config)
,m_direction(dir)
,m_offset(offset)
LRState::LRState(const LRModel &config, LRModel::Direction dir, size_t offset) :
m_configuration(config), m_direction(dir), m_offset(offset)
{
}
int
LRState::
ComparePrevScores(const TargetPhrase *other) const
int LRState::ComparePrevScores(const TargetPhrase *other) const
{
LexicalReordering* producer = m_configuration.GetScoreProducer();
size_t phraseTableInd = producer->GetPhraseTableInd();
const SCORE *myScores = (const SCORE*) prevTP->ffData[phraseTableInd]; //producer->
const SCORE *yrScores = (const SCORE*) other->ffData[phraseTableInd]; //producer->
if(myScores == yrScores) return 0;
if (myScores == yrScores) return 0;
// The pointers are NULL if a phrase pair isn't found in the reordering table.
if(yrScores == NULL) return -1;
if(myScores == NULL) return 1;
if (yrScores == NULL) return -1;
if (myScores == NULL) return 1;
size_t stop = m_offset + m_configuration.GetNumberOfTypes();
for(size_t i = m_offset; i < stop; i++) {
if((myScores)[i] < (yrScores)[i]) return -1;
if((myScores)[i] > (yrScores)[i]) return 1;
for (size_t i = m_offset; i < stop; i++) {
if ((myScores)[i] < (yrScores)[i]) return -1;
if ((myScores)[i] > (yrScores)[i]) return 1;
}
return 0;
}
void LRState::CopyScores(const System &system,
Scores &accum,
const TargetPhrase &topt,
ReorderingType reoType) const
void LRState::CopyScores(const System &system, Scores &accum,
const TargetPhrase &topt, ReorderingType reoType) const
{
// don't call this on a bidirectional object
UTIL_THROW_IF2(m_direction != LRModel::Backward &&
m_direction != LRModel::Forward,
"Unknown direction: " << m_direction);
UTIL_THROW_IF2(
m_direction != LRModel::Backward && m_direction != LRModel::Forward,
"Unknown direction: " << m_direction);
TargetPhrase const* relevantOpt = ((m_direction == LRModel::Backward)
? &topt : prevTP);
TargetPhrase const* relevantOpt = (
(m_direction == LRModel::Backward) ? &topt : prevTP);
LexicalReordering* producer = m_configuration.GetScoreProducer();
size_t phraseTableInd = producer->GetPhraseTableInd();
@ -68,32 +61,31 @@ void LRState::CopyScores(const System &system,
}
size_t off_remote = m_offset + reoType;
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
UTIL_THROW_IF2(off_local >= producer->GetNumScores(),
"offset out of vector bounds!");
"offset out of vector bounds!");
// look up applicable score from vector of scores
//UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
//Scores scores(producer->GetNumScoreComponents(),0);
//UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
//Scores scores(producer->GetNumScoreComponents(),0);
SCORE score = cached[off_remote];
accum.PlusEquals(system, *producer, score, off_local);
// else: use default scores (if specified)
/*
else if (producer->GetHaveDefaultScores()) {
Scores scores(producer->GetNumScoreComponents(),0);
scores[off_local] = producer->GetDefaultScore(off_remote);
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
}
*/
else if (producer->GetHaveDefaultScores()) {
Scores scores(producer->GetNumScoreComponents(),0);
scores[off_local] = producer->GetDefaultScore(off_remote);
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
}
*/
// note: if no default score, no cost
/*
const SparseReordering* sparse = m_configuration.GetSparseReordering();
if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType,
m_direction, accum);
*/
const SparseReordering* sparse = m_configuration.GetSparseReordering();
if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType,
m_direction, accum);
*/
}
}

31
contrib/other-builds/moses2/FF/LexicalReordering/LRState.h
View File

@ -2,7 +2,8 @@
#include "../../legacy/FFState.h"
#include "LRModel.h"
namespace Moses2 {
namespace Moses2
{
class LexicalReordering;
class Hypothesis;
@ -14,33 +15,23 @@ class TargetPhrase;
class InputType;
class InputPathBase;
class LRState : public FFState
class LRState: public FFState
{
public:
typedef LRModel::ReorderingType ReorderingType;
const TargetPhrase *prevTP;
LRState(const LRModel &config,
LRModel::Direction dir,
size_t offset);
LRState(const LRModel &config, LRModel::Direction dir, size_t offset);
virtual void Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage) = 0;
virtual void Init(const LRState *prev, const TargetPhrase &topt,
const InputPathBase &path, bool first, const Bitmap *coverage) = 0;
virtual void Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const = 0;
virtual void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const = 0;
void CopyScores(const System &system,
Scores &accum,
const TargetPhrase &topt,
ReorderingType reoType) const;
void CopyScores(const System &system, Scores &accum, const TargetPhrase &topt,
ReorderingType reoType) const;
protected:
const LRModel& m_configuration;

151
contrib/other-builds/moses2/FF/LexicalReordering/LexicalReordering.cpp
View File

@ -22,28 +22,25 @@
using namespace std;
namespace Moses2 {
namespace Moses2
{
///////////////////////////////////////////////////////////////////////
LexicalReordering::LexicalReordering(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
,m_compactModel(NULL)
,m_blank(NULL)
,m_propertyInd(-1)
,m_coll(NULL)
,m_configuration(NULL)
LexicalReordering::LexicalReordering(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line), m_compactModel(NULL), m_blank(
NULL), m_propertyInd(-1), m_coll(NULL), m_configuration(NULL)
{
ReadParameters();
assert(m_configuration);
//assert(m_numScores == 6);
ReadParameters();
assert(m_configuration);
//assert(m_numScores == 6);
}
LexicalReordering::~LexicalReordering()
{
delete m_compactModel;
delete m_coll;
delete m_configuration;
delete m_compactModel;
delete m_coll;
delete m_configuration;
}
void LexicalReordering::Load(System &system)
@ -51,57 +48,61 @@ void LexicalReordering::Load(System &system)
MemPool &pool = system.GetSystemPool();
if (m_propertyInd >= 0) {
// Using integrate Lex RO. No loading needed
// Using integrate Lex RO. No loading needed
}
else if (FileExists(m_path + ".minlexr") ) {
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr", m_FactorsF,
m_FactorsE, m_FactorsC);
m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
else if (FileExists(m_path + ".minlexr")) {
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr",
m_FactorsF, m_FactorsE, m_FactorsC);
m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
}
else {
m_coll = new Coll();
InputFileStream file(m_path);
string line;
size_t lineNum = 0;
m_coll = new Coll();
InputFileStream file(m_path);
string line;
size_t lineNum = 0;
while(getline(file, line)) {
if (++lineNum % 1000000 == 0) {
cerr << lineNum << " ";
}
while (getline(file, line)) {
if (++lineNum % 1000000 == 0) {
cerr << lineNum << " ";
}
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
assert(toks.size() == 3);
PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(), system, toks[0]);
PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(), system, toks[1]);
std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
std::transform(scores.begin(), scores.end(), scores.begin(), TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
assert(toks.size() == 3);
PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(),
system, toks[0]);
PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(),
system, toks[1]);
std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
std::transform(scores.begin(), scores.end(), scores.begin(),
TransformScore);
std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
Key key(source, target);
(*m_coll)[key] = scores;
}
Key key(source, target);
(*m_coll)[key] = scores;
}
}
}
void LexicalReordering::SetParameter(const std::string& key, const std::string& value)
void LexicalReordering::SetParameter(const std::string& key,
const std::string& value)
{
if (key == "path") {
m_path = value;
m_path = value;
}
else if (key == "type") {
m_configuration = new LRModel(value, *this);
m_configuration = new LRModel(value, *this);
}
else if (key == "input-factor") {
m_FactorsF = Tokenize<FactorType>(value);
m_FactorsF = Tokenize<FactorType>(value);
}
else if (key == "output-factor") {
m_FactorsE = Tokenize<FactorType>(value);
m_FactorsE = Tokenize<FactorType>(value);
}
else if (key == "property-index") {
m_propertyInd = Scan<int>(value);
m_propertyInd = Scan<int>(value);
}
else {
StatefulFeatureFunction::SetParameter(key, value);
StatefulFeatureFunction::SetParameter(key, value);
}
}
@ -112,35 +113,31 @@ FFState* LexicalReordering::BlankState(MemPool &pool) const
}
void LexicalReordering::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
const ManagerBase &mgr, const InputType &input,
const Hypothesis &hypo) const
{
BidirectionalReorderingState &stateCast = static_cast<BidirectionalReorderingState&>(state);
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true, &hypo.GetBitmap());
BidirectionalReorderingState &stateCast =
static_cast<BidirectionalReorderingState&>(state);
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true,
&hypo.GetBitmap());
}
void LexicalReordering::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
}
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps,
const Phrase &sourcePhrase) const
const TargetPhrases &tps, const Phrase &sourcePhrase) const
{
BOOST_FOREACH(const TargetPhrase *tp, tps) {
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
}
BOOST_FOREACH(const TargetPhrase *tp, tps){
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
}
}
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrase &targetPhrase,
const Phrase &sourcePhrase) const
const TargetPhrase &targetPhrase, const Phrase &sourcePhrase) const
{
if (m_propertyInd >= 0) {
SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd);
@ -148,7 +145,8 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
}
else if (m_compactModel) {
// using external compact binary model
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase, *m_blank);
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase,
*m_blank);
if (values.size()) {
assert(values.size() == m_numScores);
@ -183,26 +181,25 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
}
void LexicalReordering::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const LRState &prevStateCast = static_cast<const LRState&>(prevState);
prevStateCast.Expand(mgr, *this, hypo, m_PhraseTableInd, scores, state);
}
const LexicalReordering::Values *LexicalReordering::GetValues(const Phrase &source, const Phrase &target) const
const LexicalReordering::Values *LexicalReordering::GetValues(
const Phrase &source, const Phrase &target) const
{
Key key(&source, &target);
Coll::const_iterator iter;
iter = m_coll->find(key);
if (iter == m_coll->end()) {
return NULL;
}
else {
return &iter->second;
}
Key key(&source, &target);
Coll::const_iterator iter;
iter = m_coll->find(key);
if (iter == m_coll->end()) {
return NULL;
}
else {
return &iter->second;
}
}
} /* namespace Moses2 */

70
contrib/other-builds/moses2/FF/LexicalReordering/LexicalReordering.h
View File

@ -13,12 +13,13 @@
#include "../../Phrase.h"
#include "../../legacy/Range.h"
namespace Moses2 {
namespace Moses2
{
class LexicalReorderingTableCompact;
class LRModel;
class LexicalReordering : public StatefulFeatureFunction
class LexicalReordering: public StatefulFeatureFunction
{
public:
LexicalReordering(size_t startInd, const std::string &line);
@ -29,32 +30,26 @@ public:
virtual void SetParameter(const std::string& key, const std::string& value);
virtual size_t HasPhraseTableInd() const
{ return true; }
{
return true;
}
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrases &tps,
const Phrase &sourcePhrase) const;
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
const Phrase &sourcePhrase) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
@ -65,9 +60,8 @@ protected:
LRModel *m_configuration;
virtual void
EvaluateAfterTablePruning(MemPool &pool,
const TargetPhrase &targetPhrase,
const Phrase &sourcePhrase) const;
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrase &targetPhrase,
const Phrase &sourcePhrase) const;
// PROPERTY IN PT
int m_propertyInd;
@ -82,21 +76,23 @@ protected:
struct KeyComparer
{
size_t operator()(const Key &obj) const {
size_t seed = obj.first->hash();
boost::hash_combine(seed, obj.second->hash());
return seed;
}
size_t operator()(const Key &obj) const
{
size_t seed = obj.first->hash();
boost::hash_combine(seed, obj.second->hash());
return seed;
}
bool operator()(const Key& a, const Key& b) const {
if ((*a.first) != (*b.first)) {
return false;
}
if ((*a.second) != (*b.second)) {
return false;
}
return true;
}
bool operator()(const Key& a, const Key& b) const
{
if ((*a.first) != (*b.first)) {
return false;
}
if ((*a.second) != (*b.second)) {
return false;
}
return true;
}
};

80
contrib/other-builds/moses2/FF/LexicalReordering/PhraseBasedReorderingState.cpp
View File

@ -13,32 +13,29 @@
using namespace std;
namespace Moses2 {
PhraseBasedReorderingState::PhraseBasedReorderingState(
const LRModel &config,
LRModel::Direction dir,
size_t offset)
:LRState(config, dir, offset)
namespace Moses2
{
// uninitialised
prevPath = NULL;
prevTP = NULL;
PhraseBasedReorderingState::PhraseBasedReorderingState(const LRModel &config,
LRModel::Direction dir, size_t offset) :
LRState(config, dir, offset)
{
// uninitialised
prevPath = NULL;
prevTP = NULL;
}
void PhraseBasedReorderingState::Init(
const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage)
void PhraseBasedReorderingState::Init(const LRState *prev,
const TargetPhrase &topt, const InputPathBase &path, bool first,
const Bitmap *coverage)
{
prevTP = &topt;
prevPath = &path;
m_first = first;
}
size_t PhraseBasedReorderingState::hash() const {
size_t PhraseBasedReorderingState::hash() const
{
size_t ret;
ret = (size_t) &prevPath->range;
boost::hash_combine(ret, m_direction);
@ -46,39 +43,44 @@ size_t PhraseBasedReorderingState::hash() const {
return ret;
}
bool PhraseBasedReorderingState::operator==(const FFState& o) const {
if (&o == this) return true;
bool PhraseBasedReorderingState::operator==(const FFState& o) const
{
if (&o == this) return true;
const PhraseBasedReorderingState &other = static_cast<const PhraseBasedReorderingState&>(o);
if (&prevPath->range == &other.prevPath->range) {
if (m_direction == LRModel::Forward) {
int compareScore = ComparePrevScores(other.prevTP);
return compareScore == 0;
} else {
return true;
}
} else {
return false;
}
const PhraseBasedReorderingState &other =
static_cast<const PhraseBasedReorderingState&>(o);
if (&prevPath->range == &other.prevPath->range) {
if (m_direction == LRModel::Forward) {
int compareScore = ComparePrevScores(other.prevTP);
return compareScore == 0;
}
else {
return true;
}
}
else {
return false;
}
}
void PhraseBasedReorderingState::Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
Scores &scores, FFState &state) const
{
if ((m_direction != LRModel::Forward) || !m_first) {
LRModel const& lrmodel = m_configuration;
Range const &cur = hypo.GetInputPath().range;
LRModel::ReorderingType reoType = (m_first ? lrmodel.GetOrientation(cur)
: lrmodel.GetOrientation(prevPath->range, cur));
LRModel::ReorderingType reoType = (
m_first ?
lrmodel.GetOrientation(cur) :
lrmodel.GetOrientation(prevPath->range, cur));
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
}
PhraseBasedReorderingState &stateCast = static_cast<PhraseBasedReorderingState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, NULL);
PhraseBasedReorderingState &stateCast =
static_cast<PhraseBasedReorderingState&>(state);
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
NULL);
}
} /* namespace Moses2 */

28
contrib/other-builds/moses2/FF/LexicalReordering/PhraseBasedReorderingState.h
View File

@ -8,40 +8,34 @@
#pragma once
#include "LRState.h"
namespace Moses2 {
namespace Moses2
{
class InputPathBase;
class PhraseBasedReorderingState : public LRState
class PhraseBasedReorderingState: public LRState
{
public:
const InputPathBase *prevPath;
bool m_first;
PhraseBasedReorderingState(const LRModel &config,
LRModel::Direction dir,
size_t offset);
PhraseBasedReorderingState(const LRModel &config, LRModel::Direction dir,
size_t offset);
void Init(const LRState *prev,
const TargetPhrase &topt,
const InputPathBase &path,
bool first,
const Bitmap *coverage);
void Init(const LRState *prev, const TargetPhrase &topt,
const InputPathBase &path, bool first, const Bitmap *coverage);
size_t hash() const;
virtual bool operator==(const FFState& other) const;
virtual std::string ToString() const
{
return "PhraseBasedReorderingState";
return "PhraseBasedReorderingState";
}
void Expand(const ManagerBase &mgr,
const LexicalReordering &ff,
const Hypothesis &hypo,
size_t phraseTableInd,
Scores &scores,
FFState &state) const;
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
FFState &state) const;
protected:

33
contrib/other-builds/moses2/FF/LexicalReordering/ReorderingStack.cpp
View File

@ -2,7 +2,7 @@
* ReorderingStack.cpp
** Author: Ankit K. Srivastava
** Date: Jan 26, 2010
*/
*/
#include <vector>
#include "ReorderingStack.h"
@ -10,8 +10,8 @@
namespace Moses2
{
ReorderingStack::ReorderingStack(MemPool &pool)
:m_stack(pool)
ReorderingStack::ReorderingStack(MemPool &pool) :
m_stack(pool)
{
}
@ -36,10 +36,10 @@ bool ReorderingStack::operator==(const ReorderingStack& o) const
// Method to push (shift element into the stack and reduce if reqd)
int ReorderingStack::ShiftReduce(const Range &input_span)
{
int distance; // value to return: the initial distance between this and previous span
int distance; // value to return: the initial distance between this and previous span
// stack is empty
if(m_stack.empty()) {
if (m_stack.empty()) {
m_stack.push_back(input_span);
return input_span.GetStartPos() + 1; // - (-1)
}
@ -48,21 +48,24 @@ int ReorderingStack::ShiftReduce(const Range &input_span)
Range prev_span = m_stack.back(); //access last element added
//calculate the distance we are returning
if(input_span.GetStartPos() > prev_span.GetStartPos()) {
if (input_span.GetStartPos() > prev_span.GetStartPos()) {
distance = input_span.GetStartPos() - prev_span.GetEndPos();
} else {
}
else {
distance = input_span.GetEndPos() - prev_span.GetStartPos();
}
if(distance == 1) { //monotone
if (distance == 1) { //monotone
m_stack.pop_back();
Range new_span(prev_span.GetStartPos(), input_span.GetEndPos());
Reduce(new_span);
} else if(distance == -1) { //swap
}
else if (distance == -1) { //swap
m_stack.pop_back();
Range new_span(input_span.GetStartPos(), prev_span.GetEndPos());
Reduce(new_span);
} else { // discontinuous
}
else { // discontinuous
m_stack.push_back(input_span);
}
@ -78,16 +81,18 @@ void ReorderingStack::Reduce(Range current)
Range previous = m_stack.back();
if(current.GetStartPos() - previous.GetEndPos() == 1) { //mono&merge
if (current.GetStartPos() - previous.GetEndPos() == 1) { //mono&merge
m_stack.pop_back();
Range t(previous.GetStartPos(), current.GetEndPos());
current = t;
} else if(previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
}
else if (previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
m_stack.pop_back();
Range t(current.GetStartPos(), previous.GetEndPos());
current = t;
} else { // discontinuous, no more merging
cont_loop=false;
}
else { // discontinuous, no more merging
cont_loop = false;
}
} // finished reducing, exit

1
contrib/other-builds/moses2/FF/LexicalReordering/ReorderingStack.h
View File

@ -38,5 +38,4 @@ private:
void Reduce(Range input_span);
};
}

20
contrib/other-builds/moses2/FF/PhrasePenalty.cpp
View File

@ -11,22 +11,20 @@
namespace Moses2
{
PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line)
:StatelessFeatureFunction(startInd, line)
PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line) :
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
PhrasePenalty::~PhrasePenalty() {
// TODO Auto-generated destructor stub
PhrasePenalty::~PhrasePenalty()
{
// TODO Auto-generated destructor stub
}
void
PhrasePenalty::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source, const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
scores.PlusEquals(system, *this, 1);
}

17
contrib/other-builds/moses2/FF/PhrasePenalty.h
View File

@ -12,19 +12,16 @@
namespace Moses2
{
class PhrasePenalty : public StatelessFeatureFunction
class PhrasePenalty: public StatelessFeatureFunction
{
public:
PhrasePenalty(size_t startInd, const std::string &line);
virtual ~PhrasePenalty();
PhrasePenalty(size_t startInd, const std::string &line);
virtual ~PhrasePenalty();
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
};

59
contrib/other-builds/moses2/FF/SkeletonStatefulFF.cpp
View File

@ -14,76 +14,73 @@ using namespace std;
namespace Moses2
{
class SkeletonState : public FFState
class SkeletonState: public FFState
{
public:
int targetLen;
SkeletonState()
{
// uninitialised
// uninitialised
}
virtual size_t hash() const {
virtual size_t hash() const
{
return (size_t) targetLen;
}
virtual bool operator==(const FFState& o) const {
virtual bool operator==(const FFState& o) const
{
const SkeletonState& other = static_cast<const SkeletonState&>(o);
return targetLen == other.targetLen;
}
virtual std::string ToString() const
{
stringstream sb;
sb << targetLen;
return sb.str();
stringstream sb;
sb << targetLen;
return sb.str();
}
};
////////////////////////////////////////////////////////////////////////////////////////
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
SkeletonStatefulFF::~SkeletonStatefulFF() {
// TODO Auto-generated destructor stub
SkeletonStatefulFF::~SkeletonStatefulFF()
{
// TODO Auto-generated destructor stub
}
FFState* SkeletonStatefulFF::BlankState(MemPool &pool) const
{
return new (pool.Allocate<SkeletonState>()) SkeletonState();
return new (pool.Allocate<SkeletonState>()) SkeletonState();
}
void SkeletonStatefulFF::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
const ManagerBase &mgr, const InputType &input,
const Hypothesis &hypo) const
{
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
stateCast.targetLen = 0;
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
stateCast.targetLen = 0;
}
void
SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool,
const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
}
void SkeletonStatefulFF::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
}
}

27
contrib/other-builds/moses2/FF/SkeletonStatefulFF.h
View File

@ -13,31 +13,24 @@
namespace Moses2
{
class SkeletonStatefulFF : public StatefulFeatureFunction
class SkeletonStatefulFF: public StatefulFeatureFunction
{
public:
SkeletonStatefulFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatefulFF();
SkeletonStatefulFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatefulFF();
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
};

23
contrib/other-builds/moses2/FF/SkeletonStatelessFF.cpp
View File

@ -11,23 +11,22 @@
namespace Moses2
{
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd, const std::string &line)
:StatelessFeatureFunction(startInd, line)
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd,
const std::string &line) :
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
SkeletonStatelessFF::~SkeletonStatelessFF() {
// TODO Auto-generated destructor stub
SkeletonStatelessFF::~SkeletonStatelessFF()
{
// TODO Auto-generated destructor stub
}
void
SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool,
const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
}

17
contrib/other-builds/moses2/FF/SkeletonStatelessFF.h
View File

@ -12,19 +12,16 @@
namespace Moses2
{
class SkeletonStatelessFF : public StatelessFeatureFunction
class SkeletonStatelessFF: public StatelessFeatureFunction
{
public:
SkeletonStatelessFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatelessFF();
SkeletonStatelessFF(size_t startInd, const std::string &line);
virtual ~SkeletonStatelessFF();
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
};

45
contrib/other-builds/moses2/FF/StatefulFeatureFunction.cpp
View File

@ -16,43 +16,46 @@
namespace Moses2
{
StatefulFeatureFunction::StatefulFeatureFunction(size_t startInd, const std::string &line)
:FeatureFunction(startInd, line)
StatefulFeatureFunction::StatefulFeatureFunction(size_t startInd,
const std::string &line) :
FeatureFunction(startInd, line)
{
}
StatefulFeatureFunction::~StatefulFeatureFunction() {
// TODO Auto-generated destructor stub
StatefulFeatureFunction::~StatefulFeatureFunction()
{
// TODO Auto-generated destructor stub
}
void StatefulFeatureFunction::EvaluateWhenApplied(const ObjectPoolContiguous<Hypothesis*> &hypos) const
void StatefulFeatureFunction::EvaluateWhenApplied(
const ObjectPoolContiguous<Hypothesis*> &hypos) const
{
#ifdef __linux
/*
pthread_t handle;
handle = pthread_self();
pthread_t handle;
handle = pthread_self();
int s;
cpu_set_t cpusetOrig, cpuset;
s = pthread_getaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
int s;
cpu_set_t cpusetOrig, cpuset;
s = pthread_getaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
CPU_ZERO(&cpuset);
CPU_ZERO(&cpuset);
int core = handle % 8;
core += 24;
CPU_SET(core, &cpuset);
int core = handle % 8;
core += 24;
CPU_SET(core, &cpuset);
s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpuset);
*/
s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpuset);
*/
#endif
for (size_t i = 0; i < hypos.GetSize(); ++i) {
Hypothesis *hypo = hypos[i];
hypo->EvaluateWhenApplied(*this);
}
for (size_t i = 0; i < hypos.GetSize(); ++i) {
Hypothesis *hypo = hypos[i];
hypo->EvaluateWhenApplied(*this);
}
#ifdef __linux
// s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
// s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
#endif
}

56
contrib/other-builds/moses2/FF/StatefulFeatureFunction.h
View File

@ -18,48 +18,46 @@ namespace Moses2
class Hypothesis;
class InputType;
class StatefulFeatureFunction : public FeatureFunction
class StatefulFeatureFunction: public FeatureFunction
{
public:
StatefulFeatureFunction(size_t startInd, const std::string &line);
virtual ~StatefulFeatureFunction();
StatefulFeatureFunction(size_t startInd, const std::string &line);
virtual ~StatefulFeatureFunction();
void SetStatefulInd(size_t ind)
{ m_statefulInd = ind; }
size_t GetStatefulInd() const
{ return m_statefulInd; }
void SetStatefulInd(size_t ind)
{
m_statefulInd = ind;
}
size_t GetStatefulInd() const
{
return m_statefulInd;
}
//! return uninitialise state
virtual FFState* BlankState(MemPool &pool) const = 0;
//! return uninitialise state
virtual FFState* BlankState(MemPool &pool) const = 0;
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const = 0;
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const = 0;
virtual void EvaluateWhenApplied(const ObjectPoolContiguous<Hypothesis*> &hypos) const;
virtual void EvaluateWhenApplied(
const ObjectPoolContiguous<Hypothesis*> &hypos) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const = 0;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const = 0;
virtual void EvaluateWhenAppliedNonBatch(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
{ }
virtual void EvaluateWhenAppliedNonBatch(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
}
protected:
size_t m_statefulInd;
size_t m_statefulInd;
};
}
#endif /* STATEFULFEATUREFUNCTION_H_ */

12
contrib/other-builds/moses2/FF/StatelessFeatureFunction.cpp
View File

@ -10,15 +10,17 @@
namespace Moses2
{
StatelessFeatureFunction::StatelessFeatureFunction(size_t startInd, const std::string &line)
:FeatureFunction(startInd, line)
StatelessFeatureFunction::StatelessFeatureFunction(size_t startInd,
const std::string &line) :
FeatureFunction(startInd, line)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
StatelessFeatureFunction::~StatelessFeatureFunction() {
// TODO Auto-generated destructor stub
StatelessFeatureFunction::~StatelessFeatureFunction()
{
// TODO Auto-generated destructor stub
}
}

6
contrib/other-builds/moses2/FF/StatelessFeatureFunction.h
View File

@ -13,11 +13,11 @@
namespace Moses2
{
class StatelessFeatureFunction : public FeatureFunction
class StatelessFeatureFunction: public FeatureFunction
{
public:
StatelessFeatureFunction(size_t startInd, const std::string &line);
virtual ~StatelessFeatureFunction();
StatelessFeatureFunction(size_t startInd, const std::string &line);
virtual ~StatelessFeatureFunction();
};
}

24
contrib/other-builds/moses2/FF/WordPenalty.cpp
View File

@ -13,28 +13,24 @@
namespace Moses2
{
WordPenalty::WordPenalty(size_t startInd, const std::string &line)
:StatelessFeatureFunction(startInd, line)
WordPenalty::WordPenalty(size_t startInd, const std::string &line) :
StatelessFeatureFunction(startInd, line)
{
ReadParameters();
ReadParameters();
}
WordPenalty::~WordPenalty() {
// TODO Auto-generated destructor stub
WordPenalty::~WordPenalty()
{
// TODO Auto-generated destructor stub
}
void
WordPenalty::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void WordPenalty::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
SCORE score = - (SCORE) targetPhrase.GetSize();
SCORE score = -(SCORE) targetPhrase.GetSize();
scores.PlusEquals(system, *this, score);
}
}

17
contrib/other-builds/moses2/FF/WordPenalty.h
View File

@ -13,19 +13,16 @@
namespace Moses2
{
class WordPenalty : public StatelessFeatureFunction
class WordPenalty: public StatelessFeatureFunction
{
public:
WordPenalty(size_t startInd, const std::string &line);
virtual ~WordPenalty();
WordPenalty(size_t startInd, const std::string &line);
virtual ~WordPenalty();
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
};

47
contrib/other-builds/moses2/HypothesisBase.cpp
View File

@ -25,34 +25,37 @@ namespace Moses2
HypothesisBase::HypothesisBase(MemPool &pool, const System &system)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
system.featureFunctions.GetNumScores());
// FF states
const std::vector<const StatefulFeatureFunction*> &sfffs = system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
m_ffStates = (FFState **) pool.Allocate(sizeof(FFState*) * numStatefulFFs);
// FF states
const std::vector<const StatefulFeatureFunction*> &sfffs =
system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
m_ffStates = (FFState **) pool.Allocate(sizeof(FFState*) * numStatefulFFs);
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = sfff->BlankState(pool);
m_ffStates[statefulInd] = state;
}
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = sfff->BlankState(pool);
m_ffStates[statefulInd] = state;
}
}
size_t HypothesisBase::hash() const
{
return hash(0);
return hash(0);
}
size_t HypothesisBase::hash(size_t seed) const
{
size_t numStatefulFFs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
size_t numStatefulFFs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
// states
for (size_t i = 0; i < numStatefulFFs; ++i) {
const FFState *state = m_ffStates[i];
size_t hash = state->hash();
boost::hash_combine(seed, hash);
const FFState *state = m_ffStates[i];
size_t hash = state->hash();
boost::hash_combine(seed, hash);
}
return seed;
@ -60,20 +63,20 @@ size_t HypothesisBase::hash(size_t seed) const
bool HypothesisBase::operator==(const HypothesisBase &other) const
{
size_t numStatefulFFs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
size_t numStatefulFFs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
// states
for (size_t i = 0; i < numStatefulFFs; ++i) {
const FFState &thisState = *m_ffStates[i];
const FFState &otherState = *other.m_ffStates[i];
if (thisState != otherState) {
return false;
}
const FFState &thisState = *m_ffStates[i];
const FFState &otherState = *other.m_ffStates[i];
if (thisState != otherState) {
return false;
}
}
return true;
}
}

45
contrib/other-builds/moses2/HypothesisBase.h
View File

@ -20,42 +20,49 @@ class Scores;
class HypothesisBase
{
public:
virtual ~HypothesisBase() {}
virtual ~HypothesisBase()
{
}
inline ManagerBase &GetManager() const
{ return *m_mgr; }
inline ManagerBase &GetManager() const
{
return *m_mgr;
}
const Scores &GetScores() const
{ return *m_scores; }
const Scores &GetScores() const
{
return *m_scores;
}
const FFState *GetState(size_t ind) const
{ return m_ffStates[ind]; }
const FFState *GetState(size_t ind) const
{
return m_ffStates[ind];
}
virtual size_t hash() const;
virtual size_t hash(size_t seed) const;
virtual bool operator==(const HypothesisBase &other) const;
virtual SCORE GetFutureScore() const = 0;
virtual void EvaluateWhenApplied() = 0;
virtual size_t hash() const;
virtual size_t hash(size_t seed) const;
virtual bool operator==(const HypothesisBase &other) const;
virtual SCORE GetFutureScore() const = 0;
virtual void EvaluateWhenApplied() = 0;
protected:
ManagerBase *m_mgr;
Scores *m_scores;
FFState **m_ffStates;
ManagerBase *m_mgr;
Scores *m_scores;
FFState **m_ffStates;
HypothesisBase(MemPool &pool, const System &system);
HypothesisBase(MemPool &pool, const System &system);
};
////////////////////////////////////////////////////////////////////////////////////
class HypothesisFutureScoreOrderer
{
public:
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const {
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const
{
return a->GetFutureScore() > b->GetFutureScore();
}
};
}

118
contrib/other-builds/moses2/HypothesisColl.cpp
View File

@ -10,12 +10,14 @@
#include "ManagerBase.h"
#include "System.h"
namespace Moses2 {
namespace Moses2
{
HypothesisColl::HypothesisColl(const ManagerBase &mgr)
:m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool()))
,m_sortedHypos(NULL)
{}
HypothesisColl::HypothesisColl(const ManagerBase &mgr) :
m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool())), m_sortedHypos(
NULL)
{
}
StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
{
@ -23,40 +25,43 @@ StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
// CHECK RECOMBINATION
if (addRet.second) {
// equiv hypo doesn't exists
return StackAdd(true, NULL);
// equiv hypo doesn't exists
return StackAdd(true, NULL);
}
else {
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
// incoming hypo is better than the one we have
const HypothesisBase *const &hypoExisting1 = *addRet.first;
const HypothesisBase *&hypoExisting2 = const_cast<const HypothesisBase *&>(hypoExisting1);
hypoExisting2 = hypo;
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
// incoming hypo is better than the one we have
const HypothesisBase * const &hypoExisting1 = *addRet.first;
const HypothesisBase *&hypoExisting2 =
const_cast<const HypothesisBase *&>(hypoExisting1);
hypoExisting2 = hypo;
return StackAdd(true, hypoExisting);
}
else {
// already storing the best hypo. discard incoming hypo
return StackAdd(false, hypoExisting);
}
return StackAdd(true, hypoExisting);
}
else {
// already storing the best hypo. discard incoming hypo
return StackAdd(false, hypoExisting);
}
}
assert(false);
}
Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const
Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr,
ArcLists &arcLists) const
{
if (m_sortedHypos == NULL) {
// create sortedHypos first
MemPool &pool = mgr.GetPool();
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool, m_coll.size());
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool,
m_coll.size());
size_t ind = 0;
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
(*m_sortedHypos)[ind] = hypo;
++ind;
}
size_t ind = 0;
BOOST_FOREACH(const HypothesisBase *hypo, m_coll){
(*m_sortedHypos)[ind] = hypo;
++ind;
}
SortAndPruneHypos(mgr, arcLists);
}
@ -64,56 +69,57 @@ Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLi
return *m_sortedHypos;
}
void HypothesisColl::SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const
void HypothesisColl::SortAndPruneHypos(const ManagerBase &mgr,
ArcLists &arcLists) const
{
size_t stackSize = mgr.system.stackSize;
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
/*
cerr << "UNSORTED hypos:" << endl;
for (size_t i = 0; i < hypos.size(); ++i) {
const Hypothesis *hypo = hypos[i];
cerr << *hypo << endl;
}
cerr << endl;
*/
cerr << "UNSORTED hypos:" << endl;
for (size_t i = 0; i < hypos.size(); ++i) {
const Hypothesis *hypo = hypos[i];
cerr << *hypo << endl;
}
cerr << endl;
*/
Hypotheses::iterator iterMiddle;
iterMiddle = (stackSize == 0 || m_sortedHypos->size() < stackSize)
? m_sortedHypos->end()
: m_sortedHypos->begin() + stackSize;
iterMiddle =
(stackSize == 0 || m_sortedHypos->size() < stackSize) ?
m_sortedHypos->end() : m_sortedHypos->begin() + stackSize;
std::partial_sort(m_sortedHypos->begin(), iterMiddle, m_sortedHypos->end(),
HypothesisFutureScoreOrderer());
HypothesisFutureScoreOrderer());
// prune
if (stackSize && m_sortedHypos->size() > stackSize) {
for (size_t i = stackSize; i < m_sortedHypos->size(); ++i) {
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
recycler.Recycle(hypo);
for (size_t i = stackSize; i < m_sortedHypos->size(); ++i) {
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
recycler.Recycle(hypo);
// delete from arclist
if (mgr.system.nbestSize) {
arcLists.Delete(hypo);
}
}
m_sortedHypos->resize(stackSize);
// delete from arclist
if (mgr.system.nbestSize) {
arcLists.Delete(hypo);
}
}
m_sortedHypos->resize(stackSize);
}
/*
cerr << "sorted hypos:" << endl;
for (size_t i = 0; i < hypos.size(); ++i) {
const Hypothesis *hypo = hypos[i];
cerr << hypo << " " << *hypo << endl;
}
cerr << endl;
*/
cerr << "sorted hypos:" << endl;
for (size_t i = 0; i < hypos.size(); ++i) {
const Hypothesis *hypo = hypos[i];
cerr << hypo << " " << *hypo << endl;
}
cerr << endl;
*/
}
void HypothesisColl::Clear()
{
m_sortedHypos = NULL;
m_coll.clear();
m_sortedHypos = NULL;
m_coll.clear();
}
} /* namespace Moses2 */

54
contrib/other-builds/moses2/HypothesisColl.h
View File

@ -11,54 +11,62 @@
#include "Recycler.h"
#include "legacy/Util2.h"
namespace Moses2 {
namespace Moses2
{
class ManagerBase;
class ArcLists;
typedef Array<const HypothesisBase*> Hypotheses;
typedef Array<const HypothesisBase*> Hypotheses;
class HypothesisColl
{
public:
typedef boost::unordered_set<const HypothesisBase*,
UnorderedComparer<HypothesisBase>,
UnorderedComparer<HypothesisBase>,
MemPoolAllocator<const HypothesisBase*>
> _HCType;
UnorderedComparer<HypothesisBase>, UnorderedComparer<HypothesisBase>,
MemPoolAllocator<const HypothesisBase*> > _HCType;
typedef _HCType::iterator iterator;
typedef _HCType::const_iterator const_iterator;
//! iterators
const_iterator begin() const {
return m_coll.begin();
const_iterator begin() const
{
return m_coll.begin();
}
const_iterator end() const {
return m_coll.end();
const_iterator end() const
{
return m_coll.end();
}
HypothesisColl(const ManagerBase &mgr);
HypothesisColl(const ManagerBase &mgr);
StackAdd Add(const HypothesisBase *hypo);
StackAdd Add(const HypothesisBase *hypo);
_HCType &GetColl()
{ return m_coll; }
_HCType &GetColl()
{
return m_coll;
}
const _HCType &GetColl() const
{ return m_coll; }
const _HCType &GetColl() const
{
return m_coll;
}
size_t GetSize() const
{ return m_coll.size(); }
size_t GetSize() const
{
return m_coll.size();
}
void Clear();
void Clear();
Hypotheses &GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
Hypotheses &GetSortedAndPruneHypos(const ManagerBase &mgr,
ArcLists &arcLists) const;
protected:
_HCType m_coll;
mutable Hypotheses *m_sortedHypos;
_HCType m_coll;
mutable Hypotheses *m_sortedHypos;
void SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
void SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
};

8
contrib/other-builds/moses2/InputPathBase.cpp
View File

@ -10,14 +10,12 @@
namespace Moses2
{
InputPathBase::InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPathBase *prefixPath)
:subPhrase(subPhrase)
,range(range)
,prefixPath(prefixPath)
InputPathBase::InputPathBase(MemPool &pool, const SubPhrase &subPhrase,
const Range &range, size_t numPt, const InputPathBase *prefixPath) :
subPhrase(subPhrase), range(range), prefixPath(prefixPath)
{
}
}

13
contrib/other-builds/moses2/InputPathBase.h
View File

@ -18,16 +18,17 @@ namespace Moses2
class PhraseTable;
class InputPathBase {
class InputPathBase
{
public:
const InputPathBase *prefixPath;
SubPhrase subPhrase;
Range range;
const InputPathBase *prefixPath;
SubPhrase subPhrase;
Range range;
InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPathBase *prefixPath);
InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
size_t numPt, const InputPathBase *prefixPath);
};
}

1
contrib/other-builds/moses2/InputPathsBase.cpp
View File

@ -20,6 +20,5 @@ InputPathsBase::~InputPathsBase()
{
}
}

31
contrib/other-builds/moses2/InputPathsBase.h
View File

@ -20,35 +20,40 @@ class InputPathBase;
class InputPathsBase
{
typedef std::vector<InputPathBase*> Coll;
typedef std::vector<InputPathBase*> Coll;
public:
InputPathsBase() {}
virtual ~InputPathsBase();
InputPathsBase()
{
}
virtual ~InputPathsBase();
//! iterators
typedef Coll::iterator iterator;
typedef Coll::const_iterator const_iterator;
const_iterator begin() const {
return m_inputPaths.begin();
const_iterator begin() const
{
return m_inputPaths.begin();
}
const_iterator end() const {
return m_inputPaths.end();
const_iterator end() const
{
return m_inputPaths.end();
}
iterator begin() {
return m_inputPaths.begin();
iterator begin()
{
return m_inputPaths.begin();
}
iterator end() {
return m_inputPaths.end();
iterator end()
{
return m_inputPaths.end();
}
virtual void Init(const Sentence &input, const ManagerBase &mgr) = 0;
protected:
Coll m_inputPaths;
Coll m_inputPaths;
};
}

8
contrib/other-builds/moses2/InputType.cpp
View File

@ -7,10 +7,12 @@
#include "InputType.h"
namespace Moses2 {
namespace Moses2
{
InputType::~InputType() {
// TODO Auto-generated destructor stub
InputType::~InputType()
{
// TODO Auto-generated destructor stub
}
} /* namespace Moses2 */

23
contrib/other-builds/moses2/InputType.h
View File

@ -8,24 +8,27 @@
#ifndef INPUTTYPE_H_
#define INPUTTYPE_H_
namespace Moses2 {
namespace Moses2
{
class InputType {
class InputType
{
public:
InputType(long translationId)
:m_translationId(translationId)
{}
InputType(long translationId) :
m_translationId(translationId)
{
}
virtual ~InputType();
virtual ~InputType();
long GetTranslationId() const {
return m_translationId;
}
long GetTranslationId() const
{
return m_translationId;
}
protected:
long m_translationId; //< contiguous Id
};
} /* namespace Moses2 */

370
contrib/other-builds/moses2/LM/KENLM.cpp
View File

@ -24,13 +24,16 @@ using namespace std;
namespace Moses2
{
struct KenLMState : public FFState {
struct KenLMState: public FFState
{
lm::ngram::State state;
virtual size_t hash() const {
virtual size_t hash() const
{
size_t ret = hash_value(state);
return ret;
}
virtual bool operator==(const FFState& o) const {
virtual bool operator==(const FFState& o) const
{
const KenLMState &other = static_cast<const KenLMState &>(o);
bool ret = state == other.state;
return ret;
@ -38,32 +41,33 @@ struct KenLMState : public FFState {
virtual std::string ToString() const
{
stringstream ss;
for (size_t i = 0; i < state.Length(); ++i) {
ss << state.words[i] << " ";
}
return ss.str();
stringstream ss;
for (size_t i = 0; i < state.Length(); ++i) {
ss << state.words[i] << " ";
}
return ss.str();
}
};
/////////////////////////////////////////////////////////////////
class MappingBuilder : public lm::EnumerateVocab
class MappingBuilder: public lm::EnumerateVocab
{
public:
MappingBuilder(FactorCollection &factorCollection, System &system, std::vector<lm::WordIndex> &mapping)
: m_factorCollection(factorCollection)
, m_system(system)
, m_mapping(mapping)
{}
MappingBuilder(FactorCollection &factorCollection, System &system,
std::vector<lm::WordIndex> &mapping) :
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping)
{
}
void Add(lm::WordIndex index, const StringPiece &str) {
std::size_t factorId = m_factorCollection.AddFactor(str, m_system)->GetId();
if (m_mapping.size() <= factorId) {
// 0 is <unk> :-)
m_mapping.resize(factorId + 1);
}
m_mapping[factorId] = index;
void Add(lm::WordIndex index, const StringPiece &str)
{
std::size_t factorId = m_factorCollection.AddFactor(str, m_system)->GetId();
if (m_mapping.size() <= factorId) {
// 0 is <unk> :-)
m_mapping.resize(factorId + 1);
}
m_mapping[factorId] = index;
}
private:
@ -73,23 +77,23 @@ private:
};
/////////////////////////////////////////////////////////////////
template <class Model>
KENLM<Model>::KENLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
:StatefulFeatureFunction(startInd, line)
,m_path(file)
,m_factorType(factorType)
,m_load_method(load_method)
template<class Model>
KENLM<Model>::KENLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method) :
StatefulFeatureFunction(startInd, line), m_path(file), m_factorType(
factorType), m_load_method(load_method)
{
ReadParameters();
}
template <class Model>
template<class Model>
KENLM<Model>::~KENLM()
{
// TODO Auto-generated destructor stub
// TODO Auto-generated destructor stub
}
template <class Model>
template<class Model>
void KENLM<Model>::Load(System &system)
{
FactorCollection &fc = system.GetVocab();
@ -108,7 +112,7 @@ void KENLM<Model>::Load(System &system)
m_ngram.reset(new Model(m_path.c_str(), config));
}
template <class Model>
template<class Model>
FFState* KENLM<Model>::BlankState(MemPool &pool) const
{
KenLMState *ret = new (pool.Allocate<KenLMState>()) KenLMState();
@ -116,23 +120,18 @@ FFState* KENLM<Model>::BlankState(MemPool &pool) const
}
//! return the state associated with the empty hypothesis for a given sentence
template <class Model>
void KENLM<Model>::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
template<class Model>
void KENLM<Model>::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
stateCast.state = m_ngram->BeginSentenceState();
}
template <class Model>
void KENLM<Model>::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
template<class Model>
void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
// contains factors used by this LM
float fullScore, nGramScore;
@ -144,40 +143,41 @@ void KENLM<Model>::EvaluateInIsolation(MemPool &pool,
bool GetLMEnableOOVFeature = false;
if (GetLMEnableOOVFeature) {
float scoresVec[2], estimateScoresVec[2];
scoresVec[0] = nGramScore;
scoresVec[1] = oovCount;
scores.PlusEquals(system, *this, scoresVec);
float scoresVec[2], estimateScoresVec[2];
scoresVec[0] = nGramScore;
scoresVec[1] = oovCount;
scores.PlusEquals(system, *this, scoresVec);
estimateScoresVec[0] = estimateScore;
estimateScoresVec[1] = 0;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, estimateScoresVec);
(*estimatedScore) += weightedScore;
estimateScoresVec[0] = estimateScore;
estimateScoresVec[1] = 0;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScoresVec);
(*estimatedScore) += weightedScore;
}
else {
scores.PlusEquals(system, *this, nGramScore);
scores.PlusEquals(system, *this, nGramScore);
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, estimateScore);
(*estimatedScore) += weightedScore;
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
estimateScore);
(*estimatedScore) += weightedScore;
}
}
template <class Model>
template<class Model>
void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
KenLMState &stateCast = static_cast<KenLMState&>(state);
const System &system = mgr.system;
const lm::ngram::State &in_state = static_cast<const KenLMState&>(prevState).state;
const lm::ngram::State &in_state =
static_cast<const KenLMState&>(prevState).state;
if (!hypo.GetTargetPhrase().GetSize()) {
stateCast.state = in_state;
return;
return;
}
const std::size_t begin = hypo.GetCurrTargetWordsRange().GetStartPos();
@ -189,90 +189,100 @@ void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
typename Model::State aux_state;
typename Model::State *state0 = &stateCast.state, *state1 = &aux_state;
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)), *state0);
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)),
*state0);
++position;
for (; position < adjust_end; ++position) {
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)), *state1);
std::swap(state0, state1);
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)),
*state1);
std::swap(state0, state1);
}
if (hypo.GetBitmap().IsComplete()) {
// Score end of sentence.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
score += m_ngram->FullScoreForgotState(&indices.front(), last, m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
} else if (adjust_end < end) {
// Get state after adding a long phrase.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
m_ngram->GetState(&indices.front(), last, stateCast.state);
} else if (state0 != &stateCast.state) {
// Short enough phrase that we can just reuse the state.
stateCast.state = *state0;
// Score end of sentence.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
score += m_ngram->FullScoreForgotState(&indices.front(), last,
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
}
else if (adjust_end < end) {
// Get state after adding a long phrase.
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
m_ngram->GetState(&indices.front(), last, stateCast.state);
}
else if (state0 != &stateCast.state) {
// Short enough phrase that we can just reuse the state.
stateCast.state = *state0;
}
score = TransformLMScore(score);
bool OOVFeatureEnabled = false;
if (OOVFeatureEnabled) {
std::vector<float> scoresVec(2);
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(system, *this, scoresVec);
} else {
scores.PlusEquals(system, *this, score);
std::vector<float> scoresVec(2);
scoresVec[0] = score;
scoresVec[1] = 0.0;
scores.PlusEquals(system, *this, scoresVec);
}
else {
scores.PlusEquals(system, *this, score);
}
}
template <class Model>
void KENLM<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const
template<class Model>
void KENLM<Model>::CalcScore(const Phrase &phrase, float &fullScore,
float &ngramScore, std::size_t &oovCount) const
{
fullScore = 0;
ngramScore = 0;
oovCount = 0;
fullScore = 0;
ngramScore = 0;
oovCount = 0;
if (!phrase.GetSize()) return;
if (!phrase.GetSize()) return;
lm::ngram::ChartState discarded_sadly;
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
lm::ngram::ChartState discarded_sadly;
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
size_t position;
if (m_bos == phrase[0][m_factorType]) {
scorer.BeginSentence();
position = 1;
} else {
position = 0;
}
size_t position;
if (m_bos == phrase[0][m_factorType]) {
scorer.BeginSentence();
position = 1;
}
else {
position = 0;
}
size_t ngramBoundary = m_ngram->Order() - 1;
size_t ngramBoundary = m_ngram->Order() - 1;
size_t end_loop = std::min(ngramBoundary, phrase.GetSize());
for (; position < end_loop; ++position) {
const Word &word = phrase[position];
lm::WordIndex index = TranslateID(word);
scorer.Terminal(index);
if (!index) ++oovCount;
}
float before_boundary = fullScore + scorer.Finish();
for (; position < phrase.GetSize(); ++position) {
const Word &word = phrase[position];
lm::WordIndex index = TranslateID(word);
scorer.Terminal(index);
if (!index) ++oovCount;
}
fullScore += scorer.Finish();
size_t end_loop = std::min(ngramBoundary, phrase.GetSize());
for (; position < end_loop; ++position) {
const Word &word = phrase[position];
lm::WordIndex index = TranslateID(word);
scorer.Terminal(index);
if (!index) ++oovCount;
}
float before_boundary = fullScore + scorer.Finish();
for (; position < phrase.GetSize(); ++position) {
const Word &word = phrase[position];
lm::WordIndex index = TranslateID(word);
scorer.Terminal(index);
if (!index) ++oovCount;
}
fullScore += scorer.Finish();
ngramScore = TransformLMScore(fullScore - before_boundary);
fullScore = TransformLMScore(fullScore);
ngramScore = TransformLMScore(fullScore - before_boundary);
fullScore = TransformLMScore(fullScore);
}
// Convert last words of hypothesis into vocab ids, returning an end pointer.
template <class Model>
lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
template<class Model>
lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo,
lm::WordIndex *indices) const
{
lm::WordIndex *index = indices;
lm::WordIndex *end = indices + m_ngram->Order() - 1;
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
for (; ; ++index, --position) {
for (;; ++index, --position) {
if (index == end) return index;
if (position == -1) {
*index = m_ngram->GetVocabulary().BeginSentence();
@ -289,13 +299,12 @@ lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indi
* Otherwise, depending on the compiler, those functions may not be present
* at link time.
*/
template class KENLM<lm::ngram::ProbingModel>;
template class KENLM<lm::ngram::RestProbingModel>;
template class KENLM<lm::ngram::TrieModel>;
template class KENLM<lm::ngram::ArrayTrieModel>;
template class KENLM<lm::ngram::QuantTrieModel>;
template class KENLM<lm::ngram::QuantArrayTrieModel>;
template class KENLM<lm::ngram::ProbingModel> ;
template class KENLM<lm::ngram::RestProbingModel> ;
template class KENLM<lm::ngram::TrieModel> ;
template class KENLM<lm::ngram::ArrayTrieModel> ;
template class KENLM<lm::ngram::QuantTrieModel> ;
template class KENLM<lm::ngram::QuantArrayTrieModel> ;
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
{
@ -310,65 +319,90 @@ FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
line << "KENLM";
for (; argument; ++argument) {
const char *equals = std::find(argument->data(), argument->data() + argument->size(), '=');
UTIL_THROW_IF2(equals == argument->data() + argument->size(),
"Expected = in KenLM argument " << *argument);
StringPiece name(argument->data(), equals - argument->data());
StringPiece value(equals + 1, argument->data() + argument->size() - equals - 1);
if (name == "factor") {
factorType = boost::lexical_cast<FactorType>(value);
} else if (name == "order") {
// Ignored
} else if (name == "path") {
filePath.assign(value.data(), value.size());
} else if (name == "lazyken") {
// deprecated: use load instead.
load_method = boost::lexical_cast<bool>(value) ? util::LAZY : util::POPULATE_OR_READ;
} else if (name == "load") {
if (value == "lazy") {
load_method = util::LAZY;
} else if (value == "populate_or_lazy") {
load_method = util::POPULATE_OR_LAZY;
} else if (value == "populate_or_read" || value == "populate") {
load_method = util::POPULATE_OR_READ;
} else if (value == "read") {
load_method = util::READ;
} else if (value == "parallel_read") {
load_method = util::PARALLEL_READ;
} else {
UTIL_THROW2("Unknown KenLM load method " << value);
}
} else {
// pass to base class to interpret
line << " " << name << "=" << value;
}
const char *equals = std::find(argument->data(),
argument->data() + argument->size(), '=');
UTIL_THROW_IF2(equals == argument->data() + argument->size(),
"Expected = in KenLM argument " << *argument);
StringPiece name(argument->data(), equals - argument->data());
StringPiece value(equals + 1,
argument->data() + argument->size() - equals - 1);
if (name == "factor") {
factorType = boost::lexical_cast<FactorType>(value);
}
else if (name == "order") {
// Ignored
}
else if (name == "path") {
filePath.assign(value.data(), value.size());
}
else if (name == "lazyken") {
// deprecated: use load instead.
load_method =
boost::lexical_cast<bool>(value) ?
util::LAZY : util::POPULATE_OR_READ;
}
else if (name == "load") {
if (value == "lazy") {
load_method = util::LAZY;
}
else if (value == "populate_or_lazy") {
load_method = util::POPULATE_OR_LAZY;
}
else if (value == "populate_or_read" || value == "populate") {
load_method = util::POPULATE_OR_READ;
}
else if (value == "read") {
load_method = util::READ;
}
else if (value == "parallel_read") {
load_method = util::PARALLEL_READ;
}
else {
UTIL_THROW2("Unknown KenLM load method " << value);
}
}
else {
// pass to base class to interpret
line << " " << name << "=" << value;
}
}
return ConstructKenLM(startInd, line.str(), filePath, factorType, load_method);
}
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method)
{
lm::ngram::ModelType model_type;
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
switch(model_type) {
switch (model_type) {
case lm::ngram::PROBING:
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file,
factorType, load_method);
case lm::ngram::REST_PROBING:
return new KENLM<lm::ngram::RestProbingModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::RestProbingModel>(startInd, line, file,
factorType, load_method);
case lm::ngram::TRIE:
return new KENLM<lm::ngram::TrieModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::TrieModel>(startInd, line, file, factorType,
load_method);
case lm::ngram::QUANT_TRIE:
return new KENLM<lm::ngram::QuantTrieModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::QuantTrieModel>(startInd, line, file,
factorType, load_method);
case lm::ngram::ARRAY_TRIE:
return new KENLM<lm::ngram::ArrayTrieModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::ArrayTrieModel>(startInd, line, file,
factorType, load_method);
case lm::ngram::QUANT_ARRAY_TRIE:
return new KENLM<lm::ngram::QuantArrayTrieModel>(startInd, line, file, factorType, load_method);
return new KENLM<lm::ngram::QuantArrayTrieModel>(startInd, line, file,
factorType, load_method);
default:
UTIL_THROW2("Unrecognized kenlm model type " << model_type);
UTIL_THROW2("Unrecognized kenlm model type " << model_type)
;
}
} else {
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType, load_method);
}
else {
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType,
load_method);
}
}

39
contrib/other-builds/moses2/LM/KENLM.h
View File

@ -21,13 +21,16 @@ namespace Moses2
class Word;
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig);
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
const std::string &file, FactorType factorType,
util::LoadMethod load_method);
template <class Model>
class KENLM : public StatefulFeatureFunction
template<class Model>
class KENLM: public StatefulFeatureFunction
{
public:
KENLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
KENLM(size_t startInd, const std::string &line, const std::string &file,
FactorType factorType, util::LoadMethod load_method);
virtual ~KENLM();
@ -36,24 +39,17 @@ public:
virtual FFState* BlankState(MemPool &pool) const;
//! return the state associated with the empty hypothesis for a given sentence
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
@ -64,12 +60,13 @@ protected:
boost::shared_ptr<Model> m_ngram;
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const;
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore,
std::size_t &oovCount) const;
inline lm::WordIndex TranslateID(const Word &word) const
{
std::size_t factor = word[m_factorType]->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
std::size_t factor = word[m_factorType]->GetId();
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
}
// Convert last words of hypothesis into vocab ids, returning an end pointer.
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const;

383
contrib/other-builds/moses2/LM/LanguageModel.cpp
View File

@ -21,31 +21,32 @@ using namespace std;
namespace Moses2
{
struct LMState : public PointerState
struct LMState: public PointerState
{
LMState()
:PointerState()
LMState() :
PointerState()
{
// uninitialised
// uninitialised
}
void Set(MemPool &pool, void *lms, const std::vector<const Factor*> &context)
{
lmstate = lms;
lmstate = lms;
numWords = context.size();
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*) * numWords);
for (size_t i = 0; i < numWords; ++i) {
lastWords[i] = context[i];
}
numWords = context.size();
lastWords = (const Factor**) pool.Allocate(
sizeof(const Factor*) * numWords);
for (size_t i = 0; i < numWords; ++i) {
lastWords[i] = context[i];
}
}
void Init(MemPool &pool, const Factor *factor)
{
lmstate = NULL;
numWords = 1;
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*));
lastWords[0] = factor;
lmstate = NULL;
numWords = 1;
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*));
lastWords[0] = factor;
}
size_t numWords;
@ -53,15 +54,15 @@ struct LMState : public PointerState
};
////////////////////////////////////////////////////////////////////////////////////////
LanguageModel::LanguageModel(size_t startInd, const std::string &line)
:StatefulFeatureFunction(startInd, line)
,m_oov(-100)
LanguageModel::LanguageModel(size_t startInd, const std::string &line) :
StatefulFeatureFunction(startInd, line), m_oov(-100)
{
ReadParameters();
ReadParameters();
}
LanguageModel::~LanguageModel() {
// TODO Auto-generated destructor stub
LanguageModel::~LanguageModel()
{
// TODO Auto-generated destructor stub
}
void LanguageModel::Load(System &system)
@ -75,247 +76,247 @@ void LanguageModel::Load(System &system)
size_t lineNum = 0;
string line;
while (getline(infile, line)) {
if (++lineNum % 100000 == 0) {
cerr << lineNum << " ";
}
if (++lineNum % 100000 == 0) {
cerr << lineNum << " ";
}
vector<string> substrings = Tokenize(line, "\t");
vector<string> substrings = Tokenize(line, "\t");
if (substrings.size() < 2)
continue;
if (substrings.size() < 2) continue;
assert(substrings.size() == 2 || substrings.size() == 3);
assert(substrings.size() == 2 || substrings.size() == 3);
SCORE prob = TransformLMScore(Scan<SCORE>(substrings[0]));
if (substrings[1] == "<unk>") {
m_oov = prob;
continue;
}
SCORE prob = TransformLMScore(Scan<SCORE>(substrings[0]));
if (substrings[1] == "<unk>") {
m_oov = prob;
continue;
}
SCORE backoff = 0.f;
if (substrings.size() == 3) {
backoff = TransformLMScore(Scan<SCORE>(substrings[2]));
}
SCORE backoff = 0.f;
if (substrings.size() == 3) {
backoff = TransformLMScore(Scan<SCORE>(substrings[2]));
}
// ngram
vector<string> key = Tokenize(substrings[1], " ");
// ngram
vector<string> key = Tokenize(substrings[1], " ");
vector<const Factor*> factorKey(key.size());
for (size_t i = 0; i < key.size(); ++i) {
factorKey[factorKey.size() - i - 1] = fc.AddFactor(key[i], system, false);
}
vector<const Factor*> factorKey(key.size());
for (size_t i = 0; i < key.size(); ++i) {
factorKey[factorKey.size() - i - 1] = fc.AddFactor(key[i], system, false);
}
m_root.insert(factorKey, LMScores(prob, backoff));
m_root.insert(factorKey, LMScores(prob, backoff));
}
}
void LanguageModel::SetParameter(const std::string& key, const std::string& value)
void LanguageModel::SetParameter(const std::string& key,
const std::string& value)
{
if (key == "path") {
m_path = value;
m_path = value;
}
else if (key == "factor") {
m_factorType = Scan<FactorType>(value);
m_factorType = Scan<FactorType>(value);
}
else if (key == "order") {
m_order = Scan<size_t>(value);
m_order = Scan<size_t>(value);
}
else {
StatefulFeatureFunction::SetParameter(key, value);
StatefulFeatureFunction::SetParameter(key, value);
}
}
FFState* LanguageModel::BlankState(MemPool &pool) const
{
return new (pool.Allocate<LMState>()) LMState();
return new (pool.Allocate<LMState>()) LMState();
}
void LanguageModel::EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const
void LanguageModel::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const
{
LMState &stateCast = static_cast<LMState&>(state);
LMState &stateCast = static_cast<LMState&>(state);
MemPool &pool = mgr.GetPool();
stateCast.Init(pool, m_bos);
MemPool &pool = mgr.GetPool();
stateCast.Init(pool, m_bos);
}
void
LanguageModel::EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const
{
if (targetPhrase.GetSize() == 0) {
return;
}
if (targetPhrase.GetSize() == 0) {
return;
}
SCORE score = 0;
SCORE nonFullScore = 0;
vector<const Factor*> context;
SCORE score = 0;
SCORE nonFullScore = 0;
vector<const Factor*> context;
// context.push_back(m_bos);
context.reserve(m_order);
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Factor *factor = targetPhrase[i][m_factorType];
ShiftOrPush(context, factor);
context.reserve(m_order);
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
const Factor *factor = targetPhrase[i][m_factorType];
ShiftOrPush(context, factor);
if (context.size() == m_order) {
std::pair<SCORE, void*> fromScoring = Score(context);
score += fromScoring.first;
}
else if (estimatedScore) {
std::pair<SCORE, void*> fromScoring = Score(context);
nonFullScore += fromScoring.first;
}
}
if (context.size() == m_order) {
std::pair<SCORE, void*> fromScoring = Score(context);
score += fromScoring.first;
}
else if (estimatedScore) {
std::pair<SCORE, void*> fromScoring = Score(context);
nonFullScore += fromScoring.first;
}
}
scores.PlusEquals(system, *this, score);
if (estimatedScore) {
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, nonFullScore);
(*estimatedScore) += weightedScore;
}
scores.PlusEquals(system, *this, score);
if (estimatedScore) {
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
nonFullScore);
(*estimatedScore) += weightedScore;
}
}
void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const
{
const LMState &prevLMState = static_cast<const LMState &>(prevState);
size_t numWords = prevLMState.numWords;
const LMState &prevLMState = static_cast<const LMState &>(prevState);
size_t numWords = prevLMState.numWords;
// context is held backwards
vector<const Factor*> context(numWords);
for (size_t i = 0; i < numWords; ++i) {
context[i] = prevLMState.lastWords[i];
}
//DebugContext(context);
// context is held backwards
vector<const Factor*> context(numWords);
for (size_t i = 0; i < numWords; ++i) {
context[i] = prevLMState.lastWords[i];
}
//DebugContext(context);
SCORE score = 0;
std::pair<SCORE, void*> fromScoring;
const TargetPhrase &tp = hypo.GetTargetPhrase();
for (size_t i = 0; i < tp.GetSize(); ++i) {
const Word &word = tp[i];
const Factor *factor = word[m_factorType];
ShiftOrPush(context, factor);
fromScoring = Score(context);
score += fromScoring.first;
}
SCORE score = 0;
std::pair<SCORE, void*> fromScoring;
const TargetPhrase &tp = hypo.GetTargetPhrase();
for (size_t i = 0; i < tp.GetSize(); ++i) {
const Word &word = tp[i];
const Factor *factor = word[m_factorType];
ShiftOrPush(context, factor);
fromScoring = Score(context);
score += fromScoring.first;
}
const Bitmap &bm = hypo.GetBitmap();
if (bm.IsComplete()) {
// everything translated
ShiftOrPush(context, m_eos);
fromScoring = Score(context);
score += fromScoring.first;
fromScoring.second = NULL;
context.clear();
}
else {
assert(context.size());
if (context.size() == m_order) {
context.resize(context.size() - 1);
}
}
const Bitmap &bm = hypo.GetBitmap();
if (bm.IsComplete()) {
// everything translated
ShiftOrPush(context, m_eos);
fromScoring = Score(context);
score += fromScoring.first;
fromScoring.second = NULL;
context.clear();
}
else {
assert(context.size());
if (context.size() == m_order) {
context.resize(context.size() - 1);
}
}
scores.PlusEquals(mgr.system, *this, score);
scores.PlusEquals(mgr.system, *this, score);
// return state
//DebugContext(context);
// return state
//DebugContext(context);
LMState &stateCast = static_cast<LMState&>(state);
MemPool &pool = mgr.GetPool();
stateCast.Set(pool, fromScoring.second, context);
LMState &stateCast = static_cast<LMState&>(state);
MemPool &pool = mgr.GetPool();
stateCast.Set(pool, fromScoring.second, context);
}
void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context, const Factor *factor) const
void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const
{
if (context.size() < m_order) {
context.resize(context.size() + 1);
}
assert(context.size());
if (context.size() < m_order) {
context.resize(context.size() + 1);
}
assert(context.size());
for (size_t i = context.size() - 1; i > 0; --i) {
context[i] = context[i - 1];
}
for (size_t i = context.size() - 1; i > 0; --i) {
context[i] = context[i - 1];
}
context[0] = factor;
context[0] = factor;
}
std::pair<SCORE, void*> LanguageModel::Score(const std::vector<const Factor*> &context) const
std::pair<SCORE, void*> LanguageModel::Score(
const std::vector<const Factor*> &context) const
{
//cerr << "context=";
//DebugContext(context);
//cerr << "context=";
//DebugContext(context);
std::pair<SCORE, void*> ret;
std::pair<SCORE, void*> ret;
typedef Node<const Factor*, LMScores> LMNode;
const LMNode *node = m_root.getNode(context);
if (node) {
ret.first = node->getValue().prob;
ret.second = (void*) node;
}
else {
SCORE backoff = 0;
std::vector<const Factor*> backOffContext(context.begin() + 1, context.end());
node = m_root.getNode(backOffContext);
if (node) {
backoff = node->getValue().backoff;
}
typedef Node<const Factor*, LMScores> LMNode;
const LMNode *node = m_root.getNode(context);
if (node) {
ret.first = node->getValue().prob;
ret.second = (void*) node;
}
else {
SCORE backoff = 0;
std::vector<const Factor*> backOffContext(context.begin() + 1,
context.end());
node = m_root.getNode(backOffContext);
if (node) {
backoff = node->getValue().backoff;
}
std::vector<const Factor*> newContext(context.begin(), context.end() - 1);
std::pair<SCORE, void*> newRet = Score(newContext);
std::vector<const Factor*> newContext(context.begin(), context.end() - 1);
std::pair<SCORE, void*> newRet = Score(newContext);
ret.first = backoff + newRet.first;
ret.second = newRet.second;
}
ret.first = backoff + newRet.first;
ret.second = newRet.second;
}
//cerr << "score=" << ret.first << endl;
return ret;
//cerr << "score=" << ret.first << endl;
return ret;
}
SCORE LanguageModel::BackoffScore(const std::vector<const Factor*> &context) const
SCORE LanguageModel::BackoffScore(
const std::vector<const Factor*> &context) const
{
//cerr << "backoff=";
//DebugContext(context);
//cerr << "backoff=";
//DebugContext(context);
SCORE ret;
size_t stoppedAtInd;
const Node<const Factor*, LMScores> &node = m_root.getNode(context, stoppedAtInd);
SCORE ret;
size_t stoppedAtInd;
const Node<const Factor*, LMScores> &node = m_root.getNode(context,
stoppedAtInd);
if (stoppedAtInd == context.size()) {
// found entire ngram
ret = node.getValue().backoff;
}
else {
if (stoppedAtInd == 0) {
ret = m_oov;
stoppedAtInd = 1;
}
else {
ret = node.getValue().backoff;
}
if (stoppedAtInd == context.size()) {
// found entire ngram
ret = node.getValue().backoff;
}
else {
if (stoppedAtInd == 0) {
ret = m_oov;
stoppedAtInd = 1;
}
else {
ret = node.getValue().backoff;
}
// recursive
std::vector<const Factor*> backoff(context.begin() + stoppedAtInd, context.end());
ret += BackoffScore(backoff);
}
// recursive
std::vector<const Factor*> backoff(context.begin() + stoppedAtInd,
context.end());
ret += BackoffScore(backoff);
}
return ret;
return ret;
}
void LanguageModel::DebugContext(const std::vector<const Factor*> &context) const
void LanguageModel::DebugContext(
const std::vector<const Factor*> &context) const
{
for (size_t i = 0; i < context.size(); ++i) {
cerr << context[i]->GetString() << " ";
}
cerr << endl;
for (size_t i = 0; i < context.size(); ++i) {
cerr << context[i]->GetString() << " ";
}
cerr << endl;
}
}

89
contrib/other-builds/moses2/LM/LanguageModel.h
View File

@ -20,77 +20,72 @@ namespace Moses2
////////////////////////////////////////////////////////////////////////////////////////
struct LMScores
{
LMScores()
{}
LMScores()
{
}
LMScores(const LMScores &copy)
:prob(copy.prob)
,backoff(copy.backoff)
{}
LMScores(const LMScores &copy) :
prob(copy.prob), backoff(copy.backoff)
{
}
LMScores(float inProb, float inBackoff)
:prob(inProb)
,backoff(inBackoff)
{}
LMScores(float inProb, float inBackoff) :
prob(inProb), backoff(inBackoff)
{
}
float prob, backoff;
float prob, backoff;
};
inline std::ostream& operator<<(std::ostream &out, const LMScores &obj)
{
out << "(" << obj.prob << "," << obj.backoff << ")" << std::flush;
return out;
out << "(" << obj.prob << "," << obj.backoff << ")" << std::flush;
return out;
}
////////////////////////////////////////////////////////////////////////////////////////
class LanguageModel : public StatefulFeatureFunction
class LanguageModel: public StatefulFeatureFunction
{
public:
LanguageModel(size_t startInd, const std::string &line);
virtual ~LanguageModel();
LanguageModel(size_t startInd, const std::string &line);
virtual ~LanguageModel();
virtual void Load(System &system);
virtual void Load(System &system);
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void SetParameter(const std::string& key, const std::string& value);
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source,
const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
protected:
std::string m_path;
FactorType m_factorType;
size_t m_order;
std::string m_path;
FactorType m_factorType;
size_t m_order;
MorphTrie<const Factor*, LMScores> m_root;
SCORE m_oov;
const Factor *m_bos;
const Factor *m_eos;
MorphTrie<const Factor*, LMScores> m_root;
SCORE m_oov;
const Factor *m_bos;
const Factor *m_eos;
void ShiftOrPush(std::vector<const Factor*> &context, const Factor *factor) const;
std::pair<SCORE, void*> Score(const std::vector<const Factor*> &context) const;
SCORE BackoffScore(const std::vector<const Factor*> &context) const;
void ShiftOrPush(std::vector<const Factor*> &context,
const Factor *factor) const;
std::pair<SCORE, void*> Score(
const std::vector<const Factor*> &context) const;
SCORE BackoffScore(const std::vector<const Factor*> &context) const;
void DebugContext(const std::vector<const Factor*> &context) const;
void DebugContext(const std::vector<const Factor*> &context) const;
};
}
#endif /* LANGUAGEMODEL_H_ */

43
contrib/other-builds/moses2/LM/LanguageModelDALM.h
View File

@ -10,7 +10,6 @@
#include "../legacy/Util2.h"
#include "../legacy/Factor.h"
namespace DALM
{
class Logger;
@ -26,40 +25,34 @@ typedef unsigned int VocabId;
namespace Moses2
{
class LanguageModelDALM : public StatefulFeatureFunction
class LanguageModelDALM: public StatefulFeatureFunction
{
public:
LanguageModelDALM(size_t startInd, const std::string &line);
virtual ~LanguageModelDALM();
LanguageModelDALM(size_t startInd, const std::string &line);
virtual ~LanguageModelDALM();
virtual void Load(System &system);
virtual void SetParameter(const std::string& key, const std::string& value);
virtual void Load(System &system);
virtual void SetParameter(const std::string& key, const std::string& value);
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state,
const ManagerBase &mgr,
const InputType &input,
const Hypothesis &hypo) const;
virtual FFState* BlankState(MemPool &pool) const;
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
const InputType &input, const Hypothesis &hypo) const;
virtual void
EvaluateInIsolation(MemPool &pool,
const System &system,
const Phrase &source, const TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
const TargetPhrase &targetPhrase, Scores &scores,
SCORE *estimatedScore) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo,
const FFState &prevState,
Scores &scores,
FFState &state) const;
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
FFState &state) const;
protected:
FactorType m_factorType;
std::string m_filePath;
size_t m_nGramOrder; //! max n-gram length contained in this LM
size_t m_ContextSize;
std::string m_filePath;
size_t m_nGramOrder; //! max n-gram length contained in this LM
size_t m_ContextSize;
DALM::Logger *m_logger;
DALM::Vocabulary *m_vocab;

28
contrib/other-builds/moses2/ManagerBase.cpp
View File

@ -21,33 +21,33 @@ using namespace std;
namespace Moses2
{
ManagerBase::ManagerBase(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId)
:system(sys)
,task(task)
,m_inputStr(inputStr)
,m_translationId(translationId)
{}
ManagerBase::ManagerBase(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId) :
system(sys), task(task), m_inputStr(inputStr), m_translationId(
translationId)
{
}
ManagerBase::~ManagerBase()
{
GetPool().Reset();
GetHypoRecycle().Clear();
GetPool().Reset();
GetHypoRecycle().Clear();
}
void ManagerBase::InitPools()
{
m_pool = &system.GetManagerPool();
m_systemPool = &system.GetSystemPool();
m_hypoRecycle = &system.GetHypoRecycler();
m_pool = &system.GetManagerPool();
m_systemPool = &system.GetSystemPool();
m_hypoRecycle = &system.GetHypoRecycler();
}
void ManagerBase::ParseInput(bool addBOSEOS)
{
FactorCollection &vocab = system.GetVocab();
FactorCollection &vocab = system.GetVocab();
m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr, m_translationId, addBOSEOS);
m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr,
m_translationId, addBOSEOS);
}
}

52
contrib/other-builds/moses2/ManagerBase.h
View File

@ -33,41 +33,47 @@ class HypothesisBase;
class ManagerBase
{
public:
const System &system;
const TranslationTask &task;
const System &system;
const TranslationTask &task;
ArcLists arcLists;
ManagerBase(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId);
virtual ~ManagerBase();
virtual void Decode() = 0;
ManagerBase(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId);
virtual ~ManagerBase();
virtual void Decode() = 0;
MemPool &GetPool() const
{ return *m_pool; }
MemPool &GetPool() const
{
return *m_pool;
}
MemPool &GetSystemPool() const
{ return *m_systemPool; }
MemPool &GetSystemPool() const
{
return *m_systemPool;
}
Recycler<HypothesisBase*> &GetHypoRecycle() const
{
return *m_hypoRecycle;
}
Recycler<HypothesisBase*> &GetHypoRecycle() const
{
return *m_hypoRecycle;
}
const Sentence &GetInput() const
{ return *m_input; }
const Sentence &GetInput() const
{
return *m_input;
}
protected:
std::string m_inputStr;
long m_translationId;
Sentence *m_input;
std::string m_inputStr;
long m_translationId;
Sentence *m_input;
mutable MemPool *m_pool, *m_systemPool;
mutable Recycler<HypothesisBase*> *m_hypoRecycle;
mutable MemPool *m_pool, *m_systemPool;
mutable Recycler<HypothesisBase*> *m_hypoRecycle;
void InitPools();
void ParseInput(bool addBOSEOS);
void InitPools();
void ParseInput(bool addBOSEOS);
};
}

72
contrib/other-builds/moses2/MemPool.cpp
View File

@ -15,8 +15,8 @@ using namespace std;
namespace Moses2
{
MemPool::Page::Page(std::size_t vSize)
:size(vSize)
MemPool::Page::Page(std::size_t vSize) :
size(vSize)
{
mem = (uint8_t*) util::MallocOrThrow(size);
end = mem + size;
@ -24,18 +24,17 @@ MemPool::Page::Page(std::size_t vSize)
MemPool::Page::~Page()
{
free(mem);
free(mem);
}
////////////////////////////////////////////////////
MemPool::MemPool(size_t initSize)
:m_currSize(initSize)
,m_currPage(0)
MemPool::MemPool(size_t initSize) :
m_currSize(initSize), m_currPage(0)
{
Page *page = new Page(m_currSize);
m_pages.push_back(page);
Page *page = new Page(m_currSize);
m_pages.push_back(page);
current_ = page->mem;
//cerr << "new memory pool";
current_ = page->mem;
//cerr << "new memory pool";
}
MemPool::~MemPool()
@ -44,41 +43,40 @@ MemPool::~MemPool()
RemoveAllInColl(m_pages);
}
uint8_t *MemPool::More(std::size_t size)
{
++m_currPage;
if (m_currPage >= m_pages.size()) {
// add new page
m_currSize <<= 1;
std::size_t amount = std::max(m_currSize, size);
++m_currPage;
if (m_currPage >= m_pages.size()) {
// add new page
m_currSize <<= 1;
std::size_t amount = std::max(m_currSize, size);
Page *page = new Page(amount);
m_pages.push_back(page);
Page *page = new Page(amount);
m_pages.push_back(page);
uint8_t *ret = page->mem;
current_ = ret + size;
return ret;
}
else {
// use existing page
Page &page = *m_pages[m_currPage];
if (size <= page.size) {
uint8_t *ret = page.mem;
current_ = ret + size;
return ret;
}
else {
// recursive call More()
return More(size);
}
}
uint8_t *ret = page->mem;
current_ = ret + size;
return ret;
}
else {
// use existing page
Page &page = *m_pages[m_currPage];
if (size <= page.size) {
uint8_t *ret = page.mem;
current_ = ret + size;
return ret;
}
else {
// recursive call More()
return More(size);
}
}
}
void MemPool::Reset()
{
m_currPage = 0;
current_ = m_pages[0]->mem;
m_currPage = 0;
current_ = m_pages[0]->mem;
}
}

317
contrib/other-builds/moses2/MemPool.h
View File

@ -18,197 +18,218 @@
namespace Moses2
{
class MemPool {
struct Page {
uint8_t *mem;
uint8_t *end;
size_t size;
Page() {}
Page(std::size_t size);
~Page();
};
public:
MemPool(std::size_t initSize = 10000);
~MemPool();
uint8_t *Allocate(std::size_t size) {
uint8_t *ret = current_;
current_ += size;
Page &page = *m_pages[m_currPage];
if (current_ <= page.end) {
// return what we got
} else {
ret = More(size);
}
return ret;
class MemPool
{
struct Page
{
uint8_t *mem;
uint8_t *end;
size_t size;
Page()
{
}
Page(std::size_t size);
~Page();
};
template<typename T>
T *Allocate() {
uint8_t *ret = Allocate(sizeof(T));
return (T*) ret;
public:
MemPool(std::size_t initSize = 10000);
~MemPool();
uint8_t *Allocate(std::size_t size)
{
uint8_t *ret = current_;
current_ += size;
Page &page = *m_pages[m_currPage];
if (current_ <= page.end) {
// return what we got
}
template<typename T>
T *Allocate(size_t num) {
uint8_t *ret = Allocate(sizeof(T) * num);
return (T*) ret;
else {
ret = More(size);
}
return ret;
void Reset();
}
private:
uint8_t *More(std::size_t size);
template<typename T>
T *Allocate()
{
uint8_t *ret = Allocate(sizeof(T));
return (T*) ret;
}
std::vector<Page*> m_pages;
template<typename T>
T *Allocate(size_t num)
{
uint8_t *ret = Allocate(sizeof(T) * num);
return (T*) ret;
}
size_t m_currSize;
size_t m_currPage;
uint8_t *current_;
void Reset();
// no copying
MemPool(const MemPool &);
MemPool &operator=(const MemPool &);
private:
uint8_t *More(std::size_t size);
std::vector<Page*> m_pages;
size_t m_currSize;
size_t m_currPage;
uint8_t *current_;
// no copying
MemPool(const MemPool &);
MemPool &operator=(const MemPool &);
};
////////////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
class ObjectPoolContiguous {
template<typename T>
class ObjectPoolContiguous
{
public:
ObjectPoolContiguous(std::size_t initSize = 100000)
:m_size(0)
,m_actualSize(initSize)
{
m_vec = (T*) malloc(sizeof(T) * initSize);
}
public:
ObjectPoolContiguous(std::size_t initSize = 100000) :
m_size(0), m_actualSize(initSize)
{
m_vec = (T*) malloc(sizeof(T) * initSize);
}
~ObjectPoolContiguous()
{
free(m_vec);
}
void Add(T &obj)
{
if (m_size >= m_actualSize) {
//std::cerr << std::endl << "MORE " << m_size << std::endl;
m_actualSize *= 2;
m_vec = (T*) realloc(m_vec, sizeof(T) * m_actualSize);
~ObjectPoolContiguous()
{
free (m_vec);
}
m_vec[m_size] = obj;
++m_size;
}
void Add(T &obj) {
if (m_size >= m_actualSize) {
//std::cerr << std::endl << "MORE " << m_size << std::endl;
m_actualSize *= 2;
m_vec = (T*) realloc(m_vec, sizeof(T) * m_actualSize);
}
m_vec[m_size] = obj;
++m_size;
}
bool IsEmpty() const
{
return m_size == 0;
}
bool IsEmpty() const
{ return m_size == 0; }
void Reset()
{
m_size = 0;
}
void Reset()
{
m_size = 0;
}
// vector op
size_t GetSize() const
{
return m_size;
}
// vector op
size_t GetSize() const
{ return m_size; }
const T& operator[](size_t ind) const
{
return m_vec[ind];
}
// stack op
const T &Get() const
{
return m_vec[m_size - 1];
}
const T& operator[](size_t ind) const {
return m_vec[ind];
}
void Pop()
{
--m_size;
}
// stack op
const T &Get() const {
return m_vec[m_size - 1];
}
T *GetData()
{
return m_vec;
}
void Pop()
{
--m_size;
}
template<typename ORDERER>
void Sort(const ORDERER &orderer)
{
std::sort(m_vec, m_vec + m_size, orderer);
}
T *GetData()
{
return m_vec;
}
private:
T *m_vec;
size_t m_size, m_actualSize;
template<typename ORDERER>
void Sort(const ORDERER &orderer)
{
std::sort(m_vec, m_vec + m_size, orderer);
}
private:
T *m_vec;
size_t m_size, m_actualSize;
// no copying
ObjectPoolContiguous(const ObjectPoolContiguous &);
ObjectPoolContiguous &operator=(const ObjectPoolContiguous &);
// no copying
ObjectPoolContiguous(const ObjectPoolContiguous &);
ObjectPoolContiguous &operator=(const ObjectPoolContiguous &);
};
//////////////////////////////////////////////////////////////////////////////////////////
template <typename T>
template<typename T>
class MemPoolAllocator
{
public:
typedef T value_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
typedef T value_type;
typedef T* pointer;
typedef const T* const_pointer;
typedef T& reference;
typedef const T& const_reference;
typedef std::size_t size_type;
typedef std::ptrdiff_t difference_type;
template< class U >
struct rebind { typedef MemPoolAllocator<U> other; };
template<class U>
struct rebind
{
typedef MemPoolAllocator<U> other;
};
MemPoolAllocator(Moses2::MemPool &pool)
:m_pool(pool)
{}
MemPoolAllocator(const MemPoolAllocator &other)
:m_pool(other.m_pool)
{}
MemPoolAllocator(Moses2::MemPool &pool) :
m_pool(pool)
{
}
MemPoolAllocator(const MemPoolAllocator &other) :
m_pool(other.m_pool)
{
}
template< class U >
MemPoolAllocator( const MemPoolAllocator<U>& other )
:m_pool(other.m_pool)
{}
template<class U>
MemPoolAllocator(const MemPoolAllocator<U>& other) :
m_pool(other.m_pool)
{
}
size_type max_size() const
{ return std::numeric_limits<size_type>::max(); }
size_type max_size() const
{
return std::numeric_limits<size_type>::max();
}
void deallocate( pointer p, size_type n )
{
//std::cerr << "deallocate " << p << " " << n << std::endl;
}
void deallocate(pointer p, size_type n)
{
//std::cerr << "deallocate " << p << " " << n << std::endl;
}
pointer allocate( size_type n, std::allocator<void>::const_pointer hint = 0 )
{
//std::cerr << "allocate " << n << " " << hint << std::endl;
pointer ret = m_pool.Allocate<T>(n);
return ret;
}
pointer allocate(size_type n, std::allocator<void>::const_pointer hint = 0)
{
//std::cerr << "allocate " << n << " " << hint << std::endl;
pointer ret = m_pool.Allocate<T>(n);
return ret;
}
void construct( pointer p, const_reference val )
{
//std::cerr << "construct " << p << " " << n << std::endl;
new((void *)p) T(val);
}
void construct(pointer p, const_reference val)
{
//std::cerr << "construct " << p << " " << n << std::endl;
new ((void *) p) T(val);
}
void destroy( pointer p )
{
//std::cerr << "destroy " << p << " " << n << std::endl;
}
void destroy(pointer p)
{
//std::cerr << "destroy " << p << " " << n << std::endl;
}
MemPool &m_pool;
MemPool &m_pool;
protected:
};
}

121
contrib/other-builds/moses2/MorphoTrie/MorphTrie.h
View File

@ -10,88 +10,91 @@ namespace Moses2
template<class KeyClass, class ValueClass>
class MorphTrie
{
public:
MorphTrie() {}
Node<KeyClass, ValueClass>* insert(const std::vector<KeyClass>& word, const ValueClass& value);
const Node<KeyClass, ValueClass>* getNode(const std::vector<KeyClass>& words) const;
const Node<KeyClass, ValueClass> &getNode(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
std::vector<const Node<KeyClass, ValueClass>* > getNodes(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
private:
Node<KeyClass, ValueClass> root;
public:
MorphTrie()
{
}
Node<KeyClass, ValueClass>* insert(const std::vector<KeyClass>& word,
const ValueClass& value);
const Node<KeyClass, ValueClass>* getNode(
const std::vector<KeyClass>& words) const;
const Node<KeyClass, ValueClass> &getNode(const std::vector<KeyClass>& words,
size_t &stoppedAtInd) const;
std::vector<const Node<KeyClass, ValueClass>*> getNodes(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
private:
Node<KeyClass, ValueClass> root;
};
template<class KeyClass, class ValueClass>
Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(const std::vector<KeyClass>& word,
const ValueClass& value)
Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(
const std::vector<KeyClass>& word, const ValueClass& value)
{
Node<KeyClass, ValueClass>* cNode = &root;
for (size_t i = 0; i < word.size(); ++i)
{
KeyClass cKey = word[i];
cNode = cNode->addSubnode(cKey);
}
cNode->setValue(value);
return cNode;
Node<KeyClass, ValueClass>* cNode = &root;
for (size_t i = 0; i < word.size(); ++i) {
KeyClass cKey = word[i];
cNode = cNode->addSubnode(cKey);
}
cNode->setValue(value);
return cNode;
}
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(const std::vector<KeyClass>& words) const
const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(
const std::vector<KeyClass>& words) const
{
size_t stoppedAtInd;
const Node<KeyClass, ValueClass> &ret = getNode(words, stoppedAtInd);
if (stoppedAtInd < words.size()) {
return NULL;
}
return &ret;
size_t stoppedAtInd;
const Node<KeyClass, ValueClass> &ret = getNode(words, stoppedAtInd);
if (stoppedAtInd < words.size()) {
return NULL;
}
return &ret;
}
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
{
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
for (size_t i = 0; i < words.size(); ++i)
{
const KeyClass &cKey = words[i];
newNode = prevNode->findSub(cKey);
if (newNode == NULL)
{
stoppedAtInd = i;
return *prevNode;
}
prevNode = newNode;
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
for (size_t i = 0; i < words.size(); ++i) {
const KeyClass &cKey = words[i];
newNode = prevNode->findSub(cKey);
if (newNode == NULL) {
stoppedAtInd = i;
return *prevNode;
}
prevNode = newNode;
}
stoppedAtInd = words.size();
return *newNode;
stoppedAtInd = words.size();
return *newNode;
}
template<class KeyClass, class ValueClass>
std::vector<const Node<KeyClass, ValueClass>* > MorphTrie<KeyClass, ValueClass>::getNodes(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
std::vector<const Node<KeyClass, ValueClass>*> MorphTrie<KeyClass, ValueClass>::getNodes(
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
{
std::vector<const Node<KeyClass, ValueClass>* > ret;
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
ret.push_back(prevNode);
std::vector<const Node<KeyClass, ValueClass>*> ret;
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
ret.push_back(prevNode);
for (size_t i = 0; i < words.size(); ++i)
{
const KeyClass &cKey = words[i];
newNode = prevNode->findSub(cKey);
if (newNode == NULL)
{
stoppedAtInd = i;
return ret;
}
else {
ret.push_back(newNode);
}
prevNode = newNode;
for (size_t i = 0; i < words.size(); ++i) {
const KeyClass &cKey = words[i];
newNode = prevNode->findSub(cKey);
if (newNode == NULL) {
stoppedAtInd = i;
return ret;
}
else {
ret.push_back(newNode);
}
prevNode = newNode;
}
stoppedAtInd = words.size();
return ret;
stoppedAtInd = words.size();
return ret;
}
}
#endif /* end of include guard: MORPHTRIE_H_ */

107
contrib/other-builds/moses2/MorphoTrie/Node.h
View File

@ -8,42 +8,47 @@
namespace Moses2
{
template<class KeyClass, class ValueClass>
template<class KeyClass, class ValueClass>
class Node
{
public:
Node() { }
Node(const ValueClass& value)
: m_value(value)
{}
~Node();
void setKey(const KeyClass& key);
void setValue(const ValueClass& value)
{
m_value = value;
}
Node* findSub(const KeyClass& key);
const Node* findSub(const KeyClass& key) const;
Node *addSubnode(const KeyClass& cKey)
{
Node *node = findSub(cKey);
if (node) {
return node;
}
else {
node = new Node();
subNodes[cKey] = node;
return node;
}
}
public:
Node()
{
}
Node(const ValueClass& value) :
m_value(value)
{
}
~Node();
void setKey(const KeyClass& key);
void setValue(const ValueClass& value)
{
m_value = value;
}
Node* findSub(const KeyClass& key);
const Node* findSub(const KeyClass& key) const;
Node *addSubnode(const KeyClass& cKey)
{
Node *node = findSub(cKey);
if (node) {
return node;
}
else {
node = new Node();
subNodes[cKey] = node;
return node;
}
}
std::vector<Node*> getSubnodes();
const ValueClass &getValue() const
{ return m_value; }
std::vector<Node*> getSubnodes();
const ValueClass &getValue() const
{
return m_value;
}
private:
boost::unordered_map<KeyClass, Node*> subNodes;
ValueClass m_value;
private:
boost::unordered_map<KeyClass, Node*> subNodes;
ValueClass m_value;
};
@ -52,33 +57,35 @@ Node<KeyClass, ValueClass>::~Node()
{
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
for (iter = subNodes.begin(); iter != subNodes.end(); ++iter) {
Node *node = iter->second;
delete node;
Node *node = iter->second;
delete node;
}
}
template<class KeyClass, class ValueClass>
const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(const KeyClass& cKey) const
const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
const KeyClass& cKey) const
{
typename boost::unordered_map<KeyClass, Node*>::const_iterator iter;
iter = subNodes.find(cKey);
if (iter != subNodes.end()) {
Node *node = iter->second;
return node;
}
return NULL;
typename boost::unordered_map<KeyClass, Node*>::const_iterator iter;
iter = subNodes.find(cKey);
if (iter != subNodes.end()) {
Node *node = iter->second;
return node;
}
return NULL;
}
template<class KeyClass, class ValueClass>
Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(const KeyClass& cKey)
Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
const KeyClass& cKey)
{
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
iter = subNodes.find(cKey);
if (iter != subNodes.end()) {
Node *node = iter->second;
return node;
}
return NULL;
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
iter = subNodes.find(cKey);
if (iter != subNodes.end()) {
Node *node = iter->second;
return node;
}
return NULL;
}
}

20
contrib/other-builds/moses2/MorphoTrie/utils.h
View File

@ -12,18 +12,18 @@ using namespace std;
namespace Moses2
{
inline void ParseLineByChar(string& line, char c, vector<string>& substrings) {
size_t i = 0;
size_t j = line.find(c);
inline void ParseLineByChar(string& line, char c, vector<string>& substrings)
{
size_t i = 0;
size_t j = line.find(c);
while (j != string::npos) {
substrings.push_back(line.substr(i, j-i));
i = ++j;
j = line.find(c, j);
while (j != string::npos) {
substrings.push_back(line.substr(i, j - i));
i = ++j;
j = line.find(c, j);
if (j == string::npos)
substrings.push_back(line.substr(i, line.length()));
}
if (j == string::npos) substrings.push_back(line.substr(i, line.length()));
}
}
}

94
contrib/other-builds/moses2/Phrase.cpp
View File

@ -18,12 +18,12 @@ namespace Moses2
size_t Phrase::hash() const
{
size_t seed = 0;
size_t seed = 0;
for (size_t i = 0; i < GetSize(); ++i) {
const Word &word = (*this)[i];
size_t wordHash = word.hash();
boost::hash_combine(seed, wordHash);
const Word &word = (*this)[i];
size_t wordHash = word.hash();
boost::hash_combine(seed, wordHash);
}
return seed;
@ -32,15 +32,15 @@ size_t Phrase::hash() const
bool Phrase::operator==(const Phrase &compare) const
{
if (GetSize() != compare.GetSize()) {
return false;
return false;
}
for (size_t i = 0; i < GetSize(); ++i) {
const Word &word = (*this)[i];
const Word &otherWord = compare[i];
if (word != otherWord) {
return false;
}
const Word &word = (*this)[i];
const Word &otherWord = compare[i];
if (word != otherWord) {
return false;
}
}
return true;
@ -48,68 +48,70 @@ bool Phrase::operator==(const Phrase &compare) const
std::string Phrase::GetString(const FactorList &factorTypes) const
{
if (GetSize() == 0) {
return "";
}
if (GetSize() == 0) {
return "";
}
std::stringstream ret;
std::stringstream ret;
const Word &word = (*this)[0];
ret << word.GetString(factorTypes);
for (size_t i = 1; i < GetSize(); ++i) {
const Word &word = (*this)[i];
ret << " " << word.GetString(factorTypes);
}
return ret.str();
const Word &word = (*this)[0];
ret << word.GetString(factorTypes);
for (size_t i = 1; i < GetSize(); ++i) {
const Word &word = (*this)[i];
ret << " " << word.GetString(factorTypes);
}
return ret.str();
}
std::ostream& operator<<(std::ostream &out, const Phrase &obj)
{
if (obj.GetSize()) {
out << obj[0];
for (size_t i = 1; i < obj.GetSize(); ++i) {
const Word &word = obj[i];
out << " " << word;
}
}
return out;
if (obj.GetSize()) {
out << obj[0];
for (size_t i = 1; i < obj.GetSize(); ++i) {
const Word &word = obj[i];
out << " " << word;
}
}
return out;
}
void Phrase::OutputToStream(std::ostream &out) const
{
size_t size = GetSize();
if (size) {
out << (*this)[0];
for (size_t i = 1; i < size; ++i) {
const Word &word = (*this)[i];
out << " " << word;
}
}
size_t size = GetSize();
if (size) {
out << (*this)[0];
for (size_t i = 1; i < size; ++i) {
const Word &word = (*this)[i];
out << " " << word;
}
}
}
////////////////////////////////////////////////////////////////////////
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system)
:pt(pt)
,scoreProperties(NULL)
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt,
const System &system) :
pt(pt), scoreProperties(NULL)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
system.featureFunctions.GetNumScores());
}
SCORE TargetPhrase::GetFutureScore() const
{ return m_scores->GetTotalScore() + m_estimatedScore; }
{
return m_scores->GetTotalScore() + m_estimatedScore;
}
SCORE *TargetPhrase::GetScoresProperty(int propertyInd) const
{
return scoreProperties ? scoreProperties + propertyInd : NULL;
return scoreProperties ? scoreProperties + propertyInd : NULL;
}
std::ostream& operator<<(std::ostream &out, const TargetPhrase &obj)
{
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
return out;
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
return out;
}
} // namespace

62
contrib/other-builds/moses2/Phrase.h
View File

@ -26,16 +26,18 @@ class System;
class Phrase
{
friend std::ostream& operator<<(std::ostream &, const Phrase &);
friend std::ostream& operator<<(std::ostream &, const Phrase &);
public:
virtual ~Phrase() {}
virtual ~Phrase()
{
}
virtual const Word& operator[](size_t pos) const = 0;
virtual size_t GetSize() const = 0;
virtual size_t hash() const;
virtual bool operator==(const Phrase &compare) const;
virtual bool operator!=(const Phrase &compare) const
{
return !( (*this) == compare );
return !((*this) == compare);
}
virtual std::string GetString(const FactorList &factorTypes) const;
virtual SubPhrase GetSubPhrase(size_t start, size_t size) const = 0;
@ -44,7 +46,7 @@ public:
};
////////////////////////////////////////////////////////////////////////
class TargetPhrase : public Phrase
class TargetPhrase: public Phrase
{
friend std::ostream& operator<<(std::ostream &, const TargetPhrase &);
@ -56,34 +58,41 @@ public:
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system);
Scores &GetScores()
{ return *m_scores; }
{
return *m_scores;
}
const Scores &GetScores() const
{ return *m_scores; }
{
return *m_scores;
}
SCORE GetFutureScore() const;
void SetEstimatedScore(const SCORE &value)
{ m_estimatedScore = value; }
{
m_estimatedScore = value;
}
SCORE *GetScoresProperty(int propertyInd) const;
protected:
Scores *m_scores;
SCORE m_estimatedScore;
Scores *m_scores;
SCORE m_estimatedScore;
};
//////////////////////////////////////////
struct CompareFutureScore {
bool operator() (const TargetPhrase *a, const TargetPhrase *b) const
struct CompareFutureScore
{
bool operator()(const TargetPhrase *a, const TargetPhrase *b) const
{
return a->GetFutureScore() > b->GetFutureScore();
return a->GetFutureScore() > b->GetFutureScore();
}
bool operator() (const TargetPhrase &a, const TargetPhrase &b) const
bool operator()(const TargetPhrase &a, const TargetPhrase &b) const
{
return a.GetFutureScore() > b.GetFutureScore();
return a.GetFutureScore() > b.GetFutureScore();
}
};
@ -91,18 +100,19 @@ struct CompareFutureScore {
class PhraseOrdererLexical
{
public:
bool operator()(const Phrase &a, const Phrase &b) const {
size_t minSize = std::min(a.GetSize(), b.GetSize());
for (size_t i = 0; i < minSize; ++i) {
const Word &aWord = a[i];
const Word &bWord = b[i];
int cmp = aWord.Compare(bWord);
//std::cerr << "WORD: " << aWord << " ||| " << bWord << " ||| " << lessThan << std::endl;
if (cmp) {
return (cmp < 0);
}
}
return a.GetSize() < b.GetSize();
bool operator()(const Phrase &a, const Phrase &b) const
{
size_t minSize = std::min(a.GetSize(), b.GetSize());
for (size_t i = 0; i < minSize; ++i) {
const Word &aWord = a[i];
const Word &bWord = b[i];
int cmp = aWord.Compare(bWord);
//std::cerr << "WORD: " << aWord << " ||| " << bWord << " ||| " << lessThan << std::endl;
if (cmp) {
return (cmp < 0);
}
}
return a.GetSize() < b.GetSize();
}
};

182
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Misc.cpp
View File

@ -20,145 +20,139 @@ namespace NSCubePruningMiniStack
{
////////////////////////////////////////////////////////////////////////
QueueItem *QueueItem::Create(QueueItem *currItem,
Manager &mgr,
CubeEdge &edge,
size_t hypoIndex,
size_t tpIndex,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
QueueItem *QueueItem::Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
size_t hypoIndex, size_t tpIndex,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
{
QueueItem *ret;
if (currItem) {
// reuse incoming queue item to create new item
ret = currItem;
ret->Init(mgr, edge, hypoIndex, tpIndex);
}
else if (!queueItemRecycler.empty()) {
// use item from recycle bin
ret = queueItemRecycler.back();
ret->Init(mgr, edge, hypoIndex, tpIndex);
queueItemRecycler.pop_back();
}
else {
// create new item
ret = new (mgr.GetPool().Allocate<QueueItem>()) QueueItem(mgr, edge, hypoIndex, tpIndex);
}
QueueItem *ret;
if (currItem) {
// reuse incoming queue item to create new item
ret = currItem;
ret->Init(mgr, edge, hypoIndex, tpIndex);
}
else if (!queueItemRecycler.empty()) {
// use item from recycle bin
ret = queueItemRecycler.back();
ret->Init(mgr, edge, hypoIndex, tpIndex);
queueItemRecycler.pop_back();
}
else {
// create new item
ret = new (mgr.GetPool().Allocate<QueueItem>()) QueueItem(mgr, edge,
hypoIndex, tpIndex);
}
return ret;
return ret;
}
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
:edge(&edge)
,hypoIndex(hypoIndex)
,tpIndex(tpIndex)
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
size_t tpIndex) :
edge(&edge), hypoIndex(hypoIndex), tpIndex(tpIndex)
{
CreateHypothesis(mgr);
CreateHypothesis(mgr);
}
void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
size_t tpIndex)
{
this->edge = &edge;
this->hypoIndex = hypoIndex;
this->tpIndex = tpIndex;
this->edge = &edge;
this->hypoIndex = hypoIndex;
this->tpIndex = tpIndex;
CreateHypothesis(mgr);
CreateHypothesis(mgr);
}
void QueueItem::CreateHypothesis(Manager &mgr)
{
const Hypothesis *prevHypo = static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
const TargetPhrase &tp = edge->tps[tpIndex];
const Hypothesis *prevHypo =
static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
const TargetPhrase &tp = edge->tps[tpIndex];
//cerr << "hypoIndex=" << hypoIndex << endl;
//cerr << "edge.hypos=" << edge.hypos.size() << endl;
//cerr << prevHypo << endl;
//cerr << *prevHypo << endl;
//cerr << "hypoIndex=" << hypoIndex << endl;
//cerr << "edge.hypos=" << edge.hypos.size() << endl;
//cerr << prevHypo << endl;
//cerr << *prevHypo << endl;
hypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
hypo->Init(mgr, *prevHypo, edge->path, tp, edge->newBitmap, edge->estimatedScore);
hypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
hypo->Init(mgr, *prevHypo, edge->path, tp, edge->newBitmap,
edge->estimatedScore);
if (!mgr.system.cubePruningLazyScoring) {
hypo->EvaluateWhenApplied();
}
if (!mgr.system.cubePruningLazyScoring) {
hypo->EvaluateWhenApplied();
}
}
////////////////////////////////////////////////////////////////////////
CubeEdge::CubeEdge(
Manager &mgr,
const Hypotheses &hypos,
const InputPath &path,
const TargetPhrases &tps,
const Bitmap &newBitmap)
:hypos(hypos)
,path(path)
,tps(tps)
,newBitmap(newBitmap)
CubeEdge::CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
const TargetPhrases &tps, const Bitmap &newBitmap) :
hypos(hypos), path(path), tps(tps), newBitmap(newBitmap)
{
estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
}
std::ostream& operator<<(std::ostream &out, const CubeEdge &obj)
{
out << obj.newBitmap;
return out;
out << obj.newBitmap;
return out;
}
bool
CubeEdge::SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
bool CubeEdge::SetSeenPosition(const size_t x, const size_t y,
SeenPositions &seenPositions) const
{
//UTIL_THROW_IF2(x >= (1<<17), "Error");
//UTIL_THROW_IF2(y >= (1<<17), "Error");
SeenPositionItem val(this, (x<<16) + y);
SeenPositionItem val(this, (x << 16) + y);
std::pair<SeenPositions::iterator, bool> pairRet = seenPositions.insert(val);
return pairRet.second;
}
void CubeEdge::CreateFirst(Manager &mgr,
Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
void CubeEdge::CreateFirst(Manager &mgr, Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
{
assert(hypos.size());
assert(tps.GetSize());
assert(hypos.size());
assert(tps.GetSize());
QueueItem *item = QueueItem::Create(NULL, mgr, *this, 0, 0, queueItemRecycler);
queue.push(item);
bool setSeen = SetSeenPosition(0, 0, seenPositions);
assert(setSeen);
QueueItem *item = QueueItem::Create(NULL, mgr, *this, 0, 0,
queueItemRecycler);
queue.push(item);
bool setSeen = SetSeenPosition(0, 0, seenPositions);
assert(setSeen);
}
void CubeEdge::CreateNext(Manager &mgr,
QueueItem *item,
Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
void CubeEdge::CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
{
size_t hypoIndex = item->hypoIndex;
size_t tpIndex = item->tpIndex;
size_t hypoIndex = item->hypoIndex;
size_t tpIndex = item->tpIndex;
if (hypoIndex + 1 < hypos.size() && SetSeenPosition(hypoIndex + 1, tpIndex, seenPositions)) {
// reuse incoming queue item to create new item
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex + 1, tpIndex, queueItemRecycler);
assert(newItem == item);
queue.push(newItem);
item = NULL;
}
if (hypoIndex + 1 < hypos.size()
&& SetSeenPosition(hypoIndex + 1, tpIndex, seenPositions)) {
// reuse incoming queue item to create new item
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex + 1,
tpIndex, queueItemRecycler);
assert(newItem == item);
queue.push(newItem);
item = NULL;
}
if (tpIndex + 1 < tps.GetSize() && SetSeenPosition(hypoIndex, tpIndex + 1, seenPositions)) {
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex, tpIndex + 1, queueItemRecycler);
queue.push(newItem);
item = NULL;
}
if (tpIndex + 1 < tps.GetSize()
&& SetSeenPosition(hypoIndex, tpIndex + 1, seenPositions)) {
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex,
tpIndex + 1, queueItemRecycler);
queue.push(newItem);
item = NULL;
}
if (item) {
// recycle unused queue item
queueItemRecycler.push_back(item);
}
if (item) {
// recycle unused queue item
queueItemRecycler.push_back(item);
}
}
}
}

78
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Misc.h
View File

@ -31,33 +31,31 @@ class CubeEdge;
///////////////////////////////////////////
class QueueItem
{
~QueueItem(); // NOT IMPLEMENTED. Use MemPool
~QueueItem(); // NOT IMPLEMENTED. Use MemPool
public:
static QueueItem *Create(QueueItem *currItem,
Manager &mgr,
CubeEdge &edge,
size_t hypoIndex,
size_t tpIndex,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
static QueueItem *Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
size_t hypoIndex, size_t tpIndex,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
void Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
void Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
CubeEdge *edge;
size_t hypoIndex, tpIndex;
Hypothesis *hypo;
CubeEdge *edge;
size_t hypoIndex, tpIndex;
Hypothesis *hypo;
protected:
void CreateHypothesis(Manager &mgr);
void CreateHypothesis(Manager &mgr);
};
///////////////////////////////////////////
class QueueItemOrderer
{
public:
bool operator()(QueueItem* itemA, QueueItem* itemB) const {
HypothesisFutureScoreOrderer orderer;
return !orderer(itemA->hypo, itemB->hypo);
bool operator()(QueueItem* itemA, QueueItem* itemB) const
{
HypothesisFutureScoreOrderer orderer;
return !orderer(itemA->hypo, itemB->hypo);
}
};
@ -67,39 +65,30 @@ class CubeEdge
friend std::ostream& operator<<(std::ostream &, const CubeEdge &);
public:
typedef std::priority_queue<QueueItem*,
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >,
QueueItemOrderer> Queue;
typedef std::priority_queue<QueueItem*,
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >, QueueItemOrderer> Queue;
typedef std::pair<const CubeEdge*, int> SeenPositionItem;
typedef boost::unordered_set<SeenPositionItem,
boost::hash<SeenPositionItem>,
std::equal_to<SeenPositionItem>,
MemPoolAllocator<SeenPositionItem> > SeenPositions;
typedef std::pair<const CubeEdge*, int> SeenPositionItem;
typedef boost::unordered_set<SeenPositionItem, boost::hash<SeenPositionItem>,
std::equal_to<SeenPositionItem>, MemPoolAllocator<SeenPositionItem> > SeenPositions;
const Hypotheses &hypos;
const InputPath &path;
const TargetPhrases &tps;
const Bitmap &newBitmap;
SCORE estimatedScore;
const Hypotheses &hypos;
const InputPath &path;
const TargetPhrases &tps;
const Bitmap &newBitmap;
SCORE estimatedScore;
CubeEdge(Manager &mgr,
const Hypotheses &hypos,
const InputPath &path,
const TargetPhrases &tps,
const Bitmap &newBitmap);
CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
const TargetPhrases &tps, const Bitmap &newBitmap);
bool SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const;
bool SetSeenPosition(const size_t x, const size_t y,
SeenPositions &seenPositions) const;
void CreateFirst(Manager &mgr,
Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
void CreateNext(Manager &mgr,
QueueItem *item,
Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
void CreateFirst(Manager &mgr, Queue &queue, SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
void CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
SeenPositions &seenPositions,
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
protected:
@ -109,4 +98,3 @@ protected:
}

276
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Search.cpp
View File

@ -27,17 +27,17 @@ namespace NSCubePruningMiniStack
{
////////////////////////////////////////////////////////////////////////
Search::Search(Manager &mgr)
:Moses2::Search(mgr)
,m_stack(mgr)
,m_cubeEdgeAlloc(mgr.GetPool())
Search::Search(Manager &mgr) :
Moses2::Search(mgr), m_stack(mgr), m_cubeEdgeAlloc(mgr.GetPool())
,m_queue(QueueItemOrderer(),
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(MemPoolAllocator<QueueItem*>(mgr.GetPool())) )
, m_queue(QueueItemOrderer(),
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(
MemPoolAllocator<QueueItem*>(mgr.GetPool())))
,m_seenPositions(MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
, m_seenPositions(
MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
,m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
, m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
{
}
@ -48,109 +48,113 @@ Search::~Search()
void Search::Decode()
{
// init cue edges
m_cubeEdges.resize(mgr.GetInput().GetSize() + 1);
for (size_t i = 0; i < m_cubeEdges.size(); ++i) {
m_cubeEdges[i] = new (mgr.GetPool().Allocate<CubeEdges>()) CubeEdges(m_cubeEdgeAlloc);
}
// init cue edges
m_cubeEdges.resize(mgr.GetInput().GetSize() + 1);
for (size_t i = 0; i < m_cubeEdges.size(); ++i) {
m_cubeEdges[i] = new (mgr.GetPool().Allocate<CubeEdges>()) CubeEdges(
m_cubeEdgeAlloc);
}
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
//cerr << "initHypo=" << *initHypo << endl;
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
//cerr << "initHypo=" << *initHypo << endl;
m_stack.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
PostDecode(0);
m_stack.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
PostDecode(0);
for (size_t stackInd = 1; stackInd < mgr.GetInput().GetSize() + 1; ++stackInd) {
//cerr << "stackInd=" << stackInd << endl;
m_stack.Clear();
Decode(stackInd);
PostDecode(stackInd);
for (size_t stackInd = 1; stackInd < mgr.GetInput().GetSize() + 1;
++stackInd) {
//cerr << "stackInd=" << stackInd << endl;
m_stack.Clear();
Decode(stackInd);
PostDecode(stackInd);
//m_stack.DebugCounts();
//cerr << m_stacks << endl;
}
//m_stack.DebugCounts();
//cerr << m_stacks << endl;
}
}
void Search::Decode(size_t stackInd)
{
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
// reuse queue from previous stack. Clear it first
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> > &container = Container(m_queue);
//cerr << "container=" << container.size() << endl;
BOOST_FOREACH(QueueItem *item, container) {
// recycle unused hypos from queue
Hypothesis *hypo = item->hypo;
hypoRecycler.Recycle(hypo);
// reuse queue from previous stack. Clear it first
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> > &container = Container(
m_queue);
//cerr << "container=" << container.size() << endl;
BOOST_FOREACH(QueueItem *item, container){
// recycle unused hypos from queue
Hypothesis *hypo = item->hypo;
hypoRecycler.Recycle(hypo);
// recycle queue item
m_queueItemRecycler.push_back(item);
}
container.clear();
// recycle queue item
m_queueItemRecycler.push_back(item);
}
container.clear();
m_seenPositions.clear();
m_seenPositions.clear();
// add top hypo from every edge into queue
CubeEdges &edges = *m_cubeEdges[stackInd];
// add top hypo from every edge into queue
CubeEdges &edges = *m_cubeEdges[stackInd];
BOOST_FOREACH(CubeEdge *edge, edges) {
//cerr << *edge << " ";
edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler);
}
BOOST_FOREACH(CubeEdge *edge, edges){
//cerr << *edge << " ";
edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler);
}
/*
cerr << "edges: ";
boost::unordered_set<const Bitmap*> uniqueBM;
BOOST_FOREACH(CubeEdge *edge, edges) {
uniqueBM.insert(&edge->newBitmap);
//cerr << *edge << " ";
}
cerr << edges.size() << " " << uniqueBM.size();
cerr << endl;
*/
/*
cerr << "edges: ";
boost::unordered_set<const Bitmap*> uniqueBM;
BOOST_FOREACH(CubeEdge *edge, edges) {
uniqueBM.insert(&edge->newBitmap);
//cerr << *edge << " ";
}
cerr << edges.size() << " " << uniqueBM.size();
cerr << endl;
*/
size_t pops = 0;
while (!m_queue.empty() && pops < mgr.system.popLimit) {
// get best hypo from queue, add to stack
//cerr << "queue=" << queue.size() << endl;
QueueItem *item = m_queue.top();
m_queue.pop();
size_t pops = 0;
while (!m_queue.empty() && pops < mgr.system.popLimit) {
// get best hypo from queue, add to stack
//cerr << "queue=" << queue.size() << endl;
QueueItem *item = m_queue.top();
m_queue.pop();
CubeEdge *edge = item->edge;
CubeEdge *edge = item->edge;
// add hypo to stack
Hypothesis *hypo = item->hypo;
// add hypo to stack
Hypothesis *hypo = item->hypo;
if (mgr.system.cubePruningLazyScoring) {
hypo->EvaluateWhenApplied();
}
if (mgr.system.cubePruningLazyScoring) {
hypo->EvaluateWhenApplied();
}
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler);
edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler);
++pops;
}
++pops;
}
// create hypo from every edge. Increase diversity
if (mgr.system.cubePruningDiversity) {
while (!m_queue.empty()) {
QueueItem *item = m_queue.top();
m_queue.pop();
// create hypo from every edge. Increase diversity
if (mgr.system.cubePruningDiversity) {
while (!m_queue.empty()) {
QueueItem *item = m_queue.top();
m_queue.pop();
if (item->hypoIndex == 0 && item->tpIndex == 0) {
// add hypo to stack
Hypothesis *hypo = item->hypo;
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
}
}
}
if (item->hypoIndex == 0 && item->tpIndex == 0) {
// add hypo to stack
Hypothesis *hypo = item->hypo;
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
}
}
}
}
void Search::PostDecode(size_t stackInd)
@ -162,78 +166,76 @@ void Search::PostDecode(size_t stackInd)
size_t inputSize = pathMatrix.GetRows();
size_t numPaths = pathMatrix.GetCols();
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()) {
const Bitmap &hypoBitmap = *val.first.first;
size_t firstGap = hypoBitmap.GetFirstGapPos();
size_t hypoEndPos = val.first.second;
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()){
const Bitmap &hypoBitmap = *val.first.first;
size_t firstGap = hypoBitmap.GetFirstGapPos();
size_t hypoEndPos = val.first.second;
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
// create edges to next hypos from existing hypos
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
// create edges to next hypos from existing hypos
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
if (path == NULL) {
break;
}
if (!path->IsUsed()) {
continue;
}
if (path == NULL) {
break;
}
if (!path->IsUsed()) {
continue;
}
const Range &pathRange = path->range;
//cerr << "pathRange=" << pathRange << endl;
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
continue;
}
const Range &pathRange = path->range;
//cerr << "pathRange=" << pathRange << endl;
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
continue;
}
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
size_t numWords = newBitmap.GetNumWordsCovered();
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
size_t numWords = newBitmap.GetNumWordsCovered();
CubeEdges &edges = *m_cubeEdges[numWords];
CubeEdges &edges = *m_cubeEdges[numWords];
// sort hypo for a particular bitmap and hypoEndPos
Hypotheses &sortedHypos = val.second->GetSortedAndPruneHypos(mgr, mgr.arcLists);
// sort hypo for a particular bitmap and hypoEndPos
Hypotheses &sortedHypos = val.second->GetSortedAndPruneHypos(mgr, mgr.arcLists);
size_t numPt = mgr.system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = path->targetPhrases[i];
if (tps && tps->GetSize()) {
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
edges.push_back(edge);
}
}
}
}
size_t numPt = mgr.system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = path->targetPhrases[i];
if (tps && tps->GetSize()) {
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
edges.push_back(edge);
}
}
}
}
}
}
const Hypothesis *Search::GetBestHypothesis() const
{
std::vector<const Hypothesis*> sortedHypos = m_stack.GetBestHypos(1);
std::vector<const Hypothesis*> sortedHypos = m_stack.GetBestHypos(1);
const Hypothesis *best = NULL;
if (sortedHypos.size()) {
best = sortedHypos[0];
}
return best;
const Hypothesis *best = NULL;
if (sortedHypos.size()) {
best = sortedHypos[0];
}
return best;
}
void Search::AddInitialTrellisPaths(TrellisPaths &paths) const
{
const Stack::Coll &coll = m_stack.GetColl();
BOOST_FOREACH(const Stack::Coll::value_type &val, coll) {
const Moses2::HypothesisColl &hypos = *val.second;
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
BOOST_FOREACH(const Stack::Coll::value_type &val, coll){
const Moses2::HypothesisColl &hypos = *val.second;
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
}
}
}

38
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Search.h
View File

@ -23,35 +23,35 @@ class TargetPhrases;
namespace NSCubePruningMiniStack
{
class Search : public Moses2::Search
class Search: public Moses2::Search
{
public:
Search(Manager &mgr);
virtual ~Search();
Search(Manager &mgr);
virtual ~Search();
virtual void Decode();
const Hypothesis *GetBestHypothesis() const;
virtual void Decode();
const Hypothesis *GetBestHypothesis() const;
void AddInitialTrellisPaths(TrellisPaths &paths) const;
void AddInitialTrellisPaths(TrellisPaths &paths) const;
protected:
Stack m_stack;
Stack m_stack;
CubeEdge::Queue m_queue;
CubeEdge::SeenPositions m_seenPositions;
CubeEdge::Queue m_queue;
CubeEdge::SeenPositions m_seenPositions;
// CUBE PRUNING VARIABLES
// setup
MemPoolAllocator<CubeEdge*> m_cubeEdgeAlloc;
typedef std::vector<CubeEdge*, MemPoolAllocator<CubeEdge*> > CubeEdges;
std::vector<CubeEdges*> m_cubeEdges;
// CUBE PRUNING VARIABLES
// setup
MemPoolAllocator<CubeEdge*> m_cubeEdgeAlloc;
typedef std::vector<CubeEdge*, MemPoolAllocator<CubeEdge*> > CubeEdges;
std::vector<CubeEdges*> m_cubeEdges;
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > m_queueItemRecycler;
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > m_queueItemRecycler;
// CUBE PRUNING
// decoding
void Decode(size_t stackInd);
void PostDecode(size_t stackInd);
// CUBE PRUNING
// decoding
void Decode(size_t stackInd);
void PostDecode(size_t stackInd);
};
}

139
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Stack.cpp
View File

@ -19,120 +19,121 @@ namespace Moses2
namespace NSCubePruningMiniStack
{
Stack::Stack(const Manager &mgr)
:m_mgr(mgr)
,m_coll(MemPoolAllocator< std::pair<HypoCoverage, Moses2::HypothesisColl*> >(mgr.GetPool()))
,m_miniStackRecycler(MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
Stack::Stack(const Manager &mgr) :
m_mgr(mgr), m_coll(
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> >(
mgr.GetPool())), m_miniStackRecycler(
MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
{
}
Stack::~Stack() {
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl *miniStack = val.second;
delete miniStack;
}
while (!m_miniStackRecycler.empty()) {
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
m_miniStackRecycler.pop_back();
delete miniStack;
}
Stack::~Stack()
{
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl *miniStack = val.second;
delete miniStack;
}
void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists)
while (!m_miniStackRecycler.empty()) {
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
m_miniStackRecycler.pop_back();
delete miniStack;
}
}
void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
{
HypoCoverage key(&hypo->GetBitmap(), hypo->GetInputPath().range.GetEndPos());
StackAdd added = GetMiniStack(key).Add(hypo);
size_t nbestSize = m_mgr.system.nbestSize;
if (nbestSize) {
arcLists.AddArc(added.added, hypo, added.other);
arcLists.AddArc(added.added, hypo, added.other);
}
else {
if (!added.added) {
hypoRecycle.Recycle(hypo);
}
else if (added.other) {
hypoRecycle.Recycle(added.other);
}
if (!added.added) {
hypoRecycle.Recycle(hypo);
}
else if (added.other) {
hypoRecycle.Recycle(added.other);
}
}
}
std::vector<const Hypothesis*> Stack::GetBestHypos(size_t num) const
{
std::vector<const Hypothesis*> ret;
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
ret.reserve(ret.size() + hypos.size());
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
ret.push_back(static_cast<const Hypothesis*>(hypo));
}
ret.reserve(ret.size() + hypos.size());
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
ret.push_back(static_cast<const Hypothesis*>(hypo));
}
}
std::vector<const Hypothesis*>::iterator iterMiddle;
iterMiddle = (num == 0 || ret.size() < num)
? ret.end()
: ret.begin()+num;
iterMiddle = (num == 0 || ret.size() < num) ? ret.end() : ret.begin() + num;
std::partial_sort(ret.begin(), iterMiddle, ret.end(),
HypothesisFutureScoreOrderer());
HypothesisFutureScoreOrderer());
return ret;
}
size_t Stack::GetHypoSize() const
{
size_t ret = 0;
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
ret += hypos.size();
}
return ret;
size_t ret = 0;
BOOST_FOREACH(const Coll::value_type &val, m_coll){
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
ret += hypos.size();
}
return ret;
}
Moses2::HypothesisColl &Stack::GetMiniStack(const HypoCoverage &key)
{
Moses2::HypothesisColl *ret;
Coll::iterator iter = m_coll.find(key);
if (iter == m_coll.end()) {
if (m_miniStackRecycler.empty()) {
ret = new Moses2::HypothesisColl(m_mgr);
}
else {
ret = m_miniStackRecycler.back();
ret->Clear();
m_miniStackRecycler.pop_back();
}
Moses2::HypothesisColl *ret;
Coll::iterator iter = m_coll.find(key);
if (iter == m_coll.end()) {
if (m_miniStackRecycler.empty()) {
ret = new Moses2::HypothesisColl(m_mgr);
}
else {
ret = m_miniStackRecycler.back();
ret->Clear();
m_miniStackRecycler.pop_back();
}
m_coll[key] = ret;
}
else {
ret = iter->second;
}
return *ret;
m_coll[key] = ret;
}
else {
ret = iter->second;
}
return *ret;
}
void Stack::Clear()
{
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
Moses2::HypothesisColl *miniStack = val.second;
m_miniStackRecycler.push_back(miniStack);
}
BOOST_FOREACH(const Coll::value_type &val, m_coll){
Moses2::HypothesisColl *miniStack = val.second;
m_miniStackRecycler.push_back(miniStack);
}
m_coll.clear();
m_coll.clear();
}
void Stack::DebugCounts()
{
cerr << "counts=";
BOOST_FOREACH(const Coll::value_type &val, GetColl()) {
const Moses2::HypothesisColl &miniStack = *val.second;
size_t count = miniStack.GetColl().size();
cerr << count << " ";
}
cerr << endl;
cerr << "counts=";
BOOST_FOREACH(const Coll::value_type &val, GetColl()){
const Moses2::HypothesisColl &miniStack = *val.second;
size_t count = miniStack.GetColl().size();
cerr << count << " ";
}
cerr << endl;
}
}

50
contrib/other-builds/moses2/PhraseBased/CubePruningMiniStack/Stack.h
View File

@ -26,46 +26,47 @@ class ArcLists;
namespace NSCubePruningMiniStack
{
class Stack {
class Stack
{
protected:
public:
typedef std::pair<const Bitmap*, size_t> HypoCoverage;
// bitmap and current endPos of hypos
// bitmap and current endPos of hypos
typedef boost::unordered_map<HypoCoverage, Moses2::HypothesisColl*,
boost::hash<HypoCoverage>,
std::equal_to<HypoCoverage>,
MemPoolAllocator< std::pair<HypoCoverage, Moses2::HypothesisColl*> >
> Coll;
boost::hash<HypoCoverage>, std::equal_to<HypoCoverage>,
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> > > Coll;
Stack(const Manager &mgr);
virtual ~Stack();
Stack(const Manager &mgr);
virtual ~Stack();
size_t GetHypoSize() const;
size_t GetHypoSize() const;
Coll &GetColl()
{
return m_coll;
}
const Coll &GetColl() const
{
return m_coll;
}
Coll &GetColl()
{ return m_coll; }
const Coll &GetColl() const
{ return m_coll; }
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists);
Moses2::HypothesisColl &GetMiniStack(const HypoCoverage &key);
Moses2::HypothesisColl &GetMiniStack(const HypoCoverage &key);
std::vector<const Hypothesis*> GetBestHypos(size_t num) const;
void Clear();
std::vector<const Hypothesis*> GetBestHypos(size_t num) const;
void Clear();
void DebugCounts();
void DebugCounts();
protected:
const Manager &m_mgr;
Coll m_coll;
std::deque<Moses2::HypothesisColl*, MemPoolAllocator<Moses2::HypothesisColl*> > m_miniStackRecycler;
const Manager &m_mgr;
Coll m_coll;
std::deque<Moses2::HypothesisColl*, MemPoolAllocator<Moses2::HypothesisColl*> > m_miniStackRecycler;
};
@ -73,4 +74,3 @@ protected:
}

206
contrib/other-builds/moses2/PhraseBased/Hypothesis.cpp
View File

@ -23,66 +23,67 @@ namespace Moses2
Hypothesis *Hypothesis::Create(MemPool &pool, Manager &mgr)
{
// ++g_numHypos;
Hypothesis *ret;
Hypothesis *ret;
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
ret = static_cast<Hypothesis*>(recycler.Get());
if (ret) {
// got new hypo from recycler. Do nothing
}
else {
ret = new (pool.Allocate<Hypothesis>()) Hypothesis(pool, mgr.system);
//cerr << "Hypothesis=" << sizeof(Hypothesis) << " " << ret << endl;
recycler.Keep(ret);
}
return ret;
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
ret = static_cast<Hypothesis*>(recycler.Get());
if (ret) {
// got new hypo from recycler. Do nothing
}
else {
ret = new (pool.Allocate<Hypothesis>()) Hypothesis(pool, mgr.system);
//cerr << "Hypothesis=" << sizeof(Hypothesis) << " " << ret << endl;
recycler.Keep(ret);
}
return ret;
}
Hypothesis::Hypothesis(MemPool &pool, const System &system)
:HypothesisBase(pool, system)
,m_currTargetWordsRange()
Hypothesis::Hypothesis(MemPool &pool, const System &system) :
HypothesisBase(pool, system), m_currTargetWordsRange()
{
}
Hypothesis::~Hypothesis() {
// TODO Auto-generated destructor stub
Hypothesis::~Hypothesis()
{
// TODO Auto-generated destructor stub
}
void Hypothesis::Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap)
void Hypothesis::Init(Manager &mgr, const InputPathBase &path,
const TargetPhrase &tp, const Bitmap &bitmap)
{
m_mgr = &mgr;
m_targetPhrase = &tp;
m_sourceCompleted = &bitmap;
m_path = &path;
m_prevHypo = NULL;
m_mgr = &mgr;
m_targetPhrase = &tp;
m_sourceCompleted = &bitmap;
m_path = &path;
m_prevHypo = NULL;
m_currTargetWordsRange.SetStartPos(NOT_FOUND);
m_currTargetWordsRange.SetEndPos(NOT_FOUND);
m_currTargetWordsRange.SetStartPos(NOT_FOUND);
m_currTargetWordsRange.SetEndPos(NOT_FOUND);
m_estimatedScore = 0;
m_scores->Reset(mgr.system);
m_estimatedScore = 0;
m_scores->Reset(mgr.system);
}
void Hypothesis::Init(Manager &mgr, const Hypothesis &prevHypo,
const InputPathBase &path,
const TargetPhrase &tp,
const Bitmap &bitmap,
SCORE estimatedScore)
const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap,
SCORE estimatedScore)
{
m_mgr = &mgr;
m_targetPhrase = &tp;
m_sourceCompleted = &bitmap;
m_path = &path;
m_prevHypo = &prevHypo;
m_mgr = &mgr;
m_targetPhrase = &tp;
m_sourceCompleted = &bitmap;
m_path = &path;
m_prevHypo = &prevHypo;
m_currTargetWordsRange.SetStartPos(prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
m_currTargetWordsRange.SetEndPos(prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
m_currTargetWordsRange.SetStartPos(
prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
m_currTargetWordsRange.SetEndPos(
prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
m_estimatedScore = estimatedScore;
m_estimatedScore = estimatedScore;
m_scores->Reset(mgr.system);
m_scores->PlusEquals(mgr.system, prevHypo.GetScores());
m_scores->PlusEquals(mgr.system, GetTargetPhrase().GetScores());
m_scores->Reset(mgr.system);
m_scores->PlusEquals(mgr.system, prevHypo.GetScores());
m_scores->PlusEquals(mgr.system, GetTargetPhrase().GetScores());
}
size_t Hypothesis::hash() const
@ -98,7 +99,7 @@ bool Hypothesis::operator==(const Hypothesis &other) const
{
// coverage
if (m_sourceCompleted != other.m_sourceCompleted) {
return false;
return false;
}
bool ret = HypothesisBase::operator ==(other);
@ -108,87 +109,92 @@ bool Hypothesis::operator==(const Hypothesis &other) const
void Hypothesis::OutputToStream(std::ostream &out) const
{
if (m_prevHypo) {
m_prevHypo->OutputToStream(out);
m_prevHypo->OutputToStream(out);
}
//cerr << *this << endl;
if (GetTargetPhrase().GetSize()) {
const Phrase &phrase = GetTargetPhrase();
out << phrase << " ";
const Phrase &phrase = GetTargetPhrase();
out << phrase << " ";
}
if (m_path->range.GetStartPos() != NOT_FOUND) {
if (m_mgr->system.reportSegmentation == 1) {
// just report phrase segmentation
out << "|" << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << "| ";
}
else if (m_mgr->system.reportSegmentation == 2) {
// more detailed info about every segment
out << "|";
if (m_mgr->system.reportSegmentation == 1) {
// just report phrase segmentation
out << "|" << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << "| ";
}
else if (m_mgr->system.reportSegmentation == 2) {
// more detailed info about every segment
out << "|";
// phrase segmentation
out << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << ",";
// phrase segmentation
out << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << ",";
// score breakdown
m_scores->OutputBreakdownToStream(out, m_mgr->system);
// score breakdown
m_scores->OutputBreakdownToStream(out, m_mgr->system);
out << "| ";
}
out << "| ";
}
}
}
std::ostream& operator<<(std::ostream &out, const Hypothesis &obj)
{
// coverage
out << obj.GetBitmap() << " " << obj.GetInputPath().range << " ";
// coverage
out << obj.GetBitmap() << " " << obj.GetInputPath().range << " ";
// states
const std::vector<const StatefulFeatureFunction*> &sfffs = obj.GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
for (size_t i = 0; i < numStatefulFFs; ++i) {
const FFState &state = *obj.GetState(i);
out << "(" << state << ") ";
}
// states
const std::vector<const StatefulFeatureFunction*> &sfffs =
obj.GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
size_t numStatefulFFs = sfffs.size();
for (size_t i = 0; i < numStatefulFFs; ++i) {
const FFState &state = *obj.GetState(i);
out << "(" << state << ") ";
}
// string
obj.OutputToStream(out);
out << " ";
out << "fc=" << obj.GetFutureScore() << " ";
obj.GetScores().Debug(out, obj.GetManager().system);
return out;
// string
obj.OutputToStream(out);
out << " ";
out << "fc=" << obj.GetFutureScore() << " ";
obj.GetScores().Debug(out, obj.GetManager().system);
return out;
}
void Hypothesis::EmptyHypothesisState(const InputType &input)
{
const std::vector<const StatefulFeatureFunction*> &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = m_ffStates[statefulInd];
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
}
const std::vector<const StatefulFeatureFunction*> &sfffs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
size_t statefulInd = sfff->GetStatefulInd();
FFState *state = m_ffStates[statefulInd];
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
}
}
void Hypothesis::EvaluateWhenApplied()
{
const std::vector<const StatefulFeatureFunction*> &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
EvaluateWhenApplied(*sfff);
}
//cerr << *this << endl;
const std::vector<const StatefulFeatureFunction*> &sfffs =
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
EvaluateWhenApplied(*sfff);
}
//cerr << *this << endl;
}
void Hypothesis::EvaluateWhenApplied(const StatefulFeatureFunction &sfff)
{
size_t statefulInd = sfff.GetStatefulInd();
const FFState *prevState = m_prevHypo->GetState(statefulInd);
FFState *thisState = m_ffStates[statefulInd];
assert(prevState);
sfff.EvaluateWhenApplied(GetManager(), *this, *prevState, *m_scores, *thisState);
size_t statefulInd = sfff.GetStatefulInd();
const FFState *prevState = m_prevHypo->GetState(statefulInd);
FFState *thisState = m_ffStates[statefulInd];
assert(prevState);
sfff.EvaluateWhenApplied(GetManager(), *this, *prevState, *m_scores,
*thisState);
}
/** recursive - pos is relative from start of sentence */
const Word &Hypothesis::GetWord(size_t pos) const {
const Word &Hypothesis::GetWord(size_t pos) const
{
const Hypothesis *hypo = this;
while (pos < hypo->GetCurrTargetWordsRange().GetStartPos()) {
hypo = hypo->GetPrevHypo();
@ -199,15 +205,15 @@ const Word &Hypothesis::GetWord(size_t pos) const {
void Hypothesis::Swap(Hypothesis &other)
{
/*
Swap(m_targetPhrase, other.m_targetPhrase);
Swap(m_sourceCompleted, other.m_sourceCompleted);
Swap(m_range, other.m_range);
Swap(m_prevHypo, other.m_prevHypo);
Swap(m_ffStates, other.m_ffStates);
Swap(m_estimatedScore, other.m_estimatedScore);
Swap(m_currTargetWordsRange, other.m_currTargetWordsRange);
*/
/*
Swap(m_targetPhrase, other.m_targetPhrase);
Swap(m_sourceCompleted, other.m_sourceCompleted);
Swap(m_range, other.m_range);
Swap(m_prevHypo, other.m_prevHypo);
Swap(m_ffStates, other.m_ffStates);
Swap(m_estimatedScore, other.m_estimatedScore);
Swap(m_currTargetWordsRange, other.m_currTargetWordsRange);
*/
}
}

59
contrib/other-builds/moses2/PhraseBased/Hypothesis.h
View File

@ -22,11 +22,11 @@ class Manager;
class InputType;
class StatefulFeatureFunction;
class Hypothesis : public HypothesisBase
class Hypothesis: public HypothesisBase
{
friend std::ostream& operator<<(std::ostream &, const Hypothesis &);
friend std::ostream& operator<<(std::ostream &, const Hypothesis &);
Hypothesis(MemPool &pool, const System &system);
Hypothesis(MemPool &pool, const System &system);
public:
@ -34,32 +34,39 @@ public:
virtual ~Hypothesis();
// initial, empty hypo
void Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap);
void Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp,
const Bitmap &bitmap);
void Init(Manager &mgr, const Hypothesis &prevHypo,
const InputPathBase &path,
const TargetPhrase &tp,
const Bitmap &bitmap,
SCORE estimatedScore);
void Init(Manager &mgr, const Hypothesis &prevHypo, const InputPathBase &path,
const TargetPhrase &tp, const Bitmap &bitmap, SCORE estimatedScore);
size_t hash() const;
bool operator==(const Hypothesis &other) const;
inline const Bitmap &GetBitmap() const
{ return *m_sourceCompleted; }
{
return *m_sourceCompleted;
}
inline const InputPathBase &GetInputPath() const
{ return *m_path; }
{
return *m_path;
}
inline const Range &GetCurrTargetWordsRange() const {
inline const Range &GetCurrTargetWordsRange() const
{
return m_currTargetWordsRange;
}
SCORE GetFutureScore() const
{ return GetScores().GetTotalScore() + m_estimatedScore; }
{
return GetScores().GetTotalScore() + m_estimatedScore;
}
const TargetPhrase &GetTargetPhrase() const
{ return *m_targetPhrase; }
{
return *m_targetPhrase;
}
void OutputToStream(std::ostream &out) const;
@ -69,12 +76,15 @@ public:
void EvaluateWhenApplied(const StatefulFeatureFunction &sfff);
const Hypothesis* GetPrevHypo() const
{ return m_prevHypo; }
{
return m_prevHypo;
}
/** curr - pos is relative from CURRENT hypothesis's starting index
* (ie, start of sentence would be some negative number, which is
* not allowed- USE WITH CAUTION) */
inline const Word &GetCurrWord(size_t pos) const {
inline const Word &GetCurrWord(size_t pos) const
{
return GetTargetPhrase()[pos];
}
@ -96,14 +106,15 @@ protected:
class HypothesisTargetPhraseOrderer
{
public:
bool operator()(const Hypothesis* a, const Hypothesis* b) const {
PhraseOrdererLexical phraseCmp;
bool ret = phraseCmp(a->GetTargetPhrase(), b->GetTargetPhrase());
/*
std::cerr << (const Phrase&) a->GetTargetPhrase() << " ||| "
<< (const Phrase&) b->GetTargetPhrase() << " ||| "
<< ret << std::endl;
*/
bool operator()(const Hypothesis* a, const Hypothesis* b) const
{
PhraseOrdererLexical phraseCmp;
bool ret = phraseCmp(a->GetTargetPhrase(), b->GetTargetPhrase());
/*
std::cerr << (const Phrase&) a->GetTargetPhrase() << " ||| "
<< (const Phrase&) b->GetTargetPhrase() << " ||| "
<< ret << std::endl;
*/
return ret;
}
};

36
contrib/other-builds/moses2/PhraseBased/InputPath.cpp
View File

@ -10,42 +10,40 @@
namespace Moses2
{
InputPath::InputPath(MemPool &pool,
const SubPhrase &subPhrase,
const Range &range,
size_t numPt,
const InputPath *prefixPath)
:InputPathBase(pool, subPhrase, range, numPt, prefixPath)
,m_isUsed(false)
InputPath::InputPath(MemPool &pool, const SubPhrase &subPhrase,
const Range &range, size_t numPt, const InputPath *prefixPath) :
InputPathBase(pool, subPhrase, range, numPt, prefixPath), m_isUsed(false)
{
targetPhrases = pool.Allocate<const TargetPhrases*>(numPt);
Init<const TargetPhrases*>(targetPhrases, numPt, NULL);
}
InputPath::~InputPath() {
// TODO Auto-generated destructor stub
InputPath::~InputPath()
{
// TODO Auto-generated destructor stub
}
void InputPath::AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps)
void InputPath::AddTargetPhrases(const PhraseTable &pt,
const TargetPhrases *tps)
{
size_t ptInd = pt.GetPtInd();
targetPhrases[ptInd] = tps;
size_t ptInd = pt.GetPtInd();
targetPhrases[ptInd] = tps;
if (tps && tps->GetSize()) {
m_isUsed = true;
}
if (tps && tps->GetSize()) {
m_isUsed = true;
}
}
const TargetPhrases *InputPath::GetTargetPhrases(const PhraseTable &pt) const
{
size_t ptInd = pt.GetPtInd();
return targetPhrases[ptInd];
size_t ptInd = pt.GetPtInd();
return targetPhrases[ptInd];
}
std::ostream& operator<<(std::ostream &out, const InputPath &obj)
{
out << obj.range << " " << obj.subPhrase;
return out;
out << obj.range << " " << obj.subPhrase;
return out;
}
}

23
contrib/other-builds/moses2/PhraseBased/InputPath.h
View File

@ -14,23 +14,26 @@
namespace Moses2
{
class InputPath : public InputPathBase
class InputPath: public InputPathBase
{
friend std::ostream& operator<<(std::ostream &, const InputPath &);
friend std::ostream& operator<<(std::ostream &, const InputPath &);
public:
const TargetPhrases** targetPhrases;
const TargetPhrases** targetPhrases;
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPath *prefixPath);
virtual ~InputPath();
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
size_t numPt, const InputPath *prefixPath);
virtual ~InputPath();
void AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps);
const TargetPhrases *GetTargetPhrases(const PhraseTable &pt) const;
void AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps);
const TargetPhrases *GetTargetPhrases(const PhraseTable &pt) const;
inline bool IsUsed() const
{ return m_isUsed; }
inline bool IsUsed() const
{
return m_isUsed;
}
protected:
bool m_isUsed;
bool m_isUsed;
};
}

31
contrib/other-builds/moses2/PhraseBased/InputPaths.cpp
View File

@ -23,35 +23,38 @@ void InputPaths::Init(const Sentence &input, const ManagerBase &mgr)
size_t size = input.GetSize();
size_t maxLength = min(size, mgr.system.maxPhraseLength);
m_matrix = new (pool.Allocate< Matrix<InputPath*> >()) Matrix<InputPath*>(pool, size, maxLength);
m_matrix = new (pool.Allocate<Matrix<InputPath*> >()) Matrix<InputPath*>(pool,
size, maxLength);
m_matrix->Init(NULL);
// create blank path for initial hypo
Range range(NOT_FOUND, NOT_FOUND);
SubPhrase subPhrase = input.GetSubPhrase(NOT_FOUND, NOT_FOUND);
m_blank = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, NULL);
m_blank = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range,
numPt, NULL);
// create normal paths of subphrases through the sentence
for (size_t startPos = 0; startPos < size; ++startPos) {
const InputPath *prefixPath = NULL;
const InputPath *prefixPath = NULL;
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
size_t endPos = startPos + phaseSize - 1;
size_t endPos = startPos + phaseSize - 1;
if (endPos >= size) {
break;
}
if (endPos >= size) {
break;
}
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
Range range(startPos, endPos);
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
Range range(startPos, endPos);
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, prefixPath);
m_inputPaths.push_back(path);
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool,
subPhrase, range, numPt, prefixPath);
m_inputPaths.push_back(path);
prefixPath = path;
prefixPath = path;
m_matrix->SetValue(startPos, phaseSize - 1, path);
}
m_matrix->SetValue(startPos, phaseSize - 1, path);
}
}
}

11
contrib/other-builds/moses2/PhraseBased/InputPaths.h
View File

@ -19,16 +19,20 @@ namespace Moses2
class Sentence;
class System;
class InputPaths : public InputPathsBase
class InputPaths: public InputPathsBase
{
public:
void Init(const Sentence &input, const ManagerBase &mgr);
const InputPath &GetBlank() const
{ return *m_blank; }
{
return *m_blank;
}
const Matrix<InputPath*> &GetMatrix() const
{ return *m_matrix; }
{
return *m_matrix;
}
protected:
InputPath *m_blank;
@ -38,4 +42,3 @@ protected:
}

322
contrib/other-builds/moses2/PhraseBased/Manager.cpp
View File

@ -15,11 +15,11 @@
#include "Normal/Search.h"
#include "CubePruningMiniStack/Search.h"
/*
#include "CubePruningPerMiniStack/Search.h"
#include "CubePruningPerBitmap/Search.h"
#include "CubePruningCardinalStack/Search.h"
#include "CubePruningBitmapStack/Search.h"
*/
#include "CubePruningPerMiniStack/Search.h"
#include "CubePruningPerBitmap/Search.h"
#include "CubePruningCardinalStack/Search.h"
#include "CubePruningBitmapStack/Search.h"
*/
#include "../System.h"
#include "../TargetPhrases.h"
#include "../Phrase.h"
@ -32,205 +32,211 @@ using namespace std;
namespace Moses2
{
Manager::Manager(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId)
:ManagerBase(sys, task, inputStr, translationId)
{}
Manager::Manager(System &sys, const TranslationTask &task,
const std::string &inputStr, long translationId) :
ManagerBase(sys, task, inputStr, translationId)
{
}
Manager::~Manager() {
Manager::~Manager()
{
delete m_search;
delete m_bitmaps;
delete m_search;
delete m_bitmaps;
}
void Manager::Init()
{
// init pools etc
InitPools();
ParseInput(false);
// init pools etc
InitPools();
ParseInput(false);
m_bitmaps = new Bitmaps(GetPool());
m_bitmaps = new Bitmaps(GetPool());
const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0];
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(GetPool(), firstPt, system, 0);
const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0];
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(
GetPool(), firstPt, system, 0);
m_inputPaths.Init(*m_input, *this);
m_inputPaths.Init(*m_input, *this);
const std::vector<const PhraseTable*> &pts = system.mappings;
for (size_t i = 0; i < pts.size(); ++i) {
const PhraseTable &pt = *pts[i];
//cerr << "Looking up from " << pt.GetName() << endl;
pt.Lookup(*this, m_inputPaths);
}
//m_inputPaths.DeleteUnusedPaths();
CalcFutureScore();
const std::vector<const PhraseTable*> &pts = system.mappings;
for (size_t i = 0; i < pts.size(); ++i) {
const PhraseTable &pt = *pts[i];
//cerr << "Looking up from " << pt.GetName() << endl;
pt.Lookup(*this, m_inputPaths);
}
//m_inputPaths.DeleteUnusedPaths();
CalcFutureScore();
m_bitmaps->Init(m_input->GetSize(), vector<bool>(0));
m_bitmaps->Init(m_input->GetSize(), vector<bool>(0));
switch (system.searchAlgorithm) {
case Normal:
m_search = new SearchNormal(*this);
break;
case CubePruning:
case CubePruningMiniStack:
m_search = new NSCubePruningMiniStack::Search(*this);
break;
/*
case CubePruningPerMiniStack:
m_search = new NSCubePruningPerMiniStack::Search(*this);
break;
case CubePruningPerBitmap:
m_search = new NSCubePruningPerBitmap::Search(*this);
break;
case CubePruningCardinalStack:
m_search = new NSCubePruningCardinalStack::Search(*this);
break;
case CubePruningBitmapStack:
m_search = new NSCubePruningBitmapStack::Search(*this);
break;
*/
default:
cerr << "Unknown search algorithm" << endl;
abort();
}
switch (system.searchAlgorithm) {
case Normal:
m_search = new SearchNormal(*this);
break;
case CubePruning:
case CubePruningMiniStack:
m_search = new NSCubePruningMiniStack::Search(*this);
break;
/*
case CubePruningPerMiniStack:
m_search = new NSCubePruningPerMiniStack::Search(*this);
break;
case CubePruningPerBitmap:
m_search = new NSCubePruningPerBitmap::Search(*this);
break;
case CubePruningCardinalStack:
m_search = new NSCubePruningCardinalStack::Search(*this);
break;
case CubePruningBitmapStack:
m_search = new NSCubePruningBitmapStack::Search(*this);
break;
*/
default:
cerr << "Unknown search algorithm" << endl;
abort();
}
}
void Manager::Decode()
{
Init();
m_search->Decode();
OutputBest();
Init();
m_search->Decode();
OutputBest();
if (system.nbestSize) {
OutputNBest();
}
if (system.nbestSize) {
OutputNBest();
}
}
void Manager::CalcFutureScore()
{
size_t size = m_input->GetSize();
m_estimatedScores = new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(), size);
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
size_t size = m_input->GetSize();
m_estimatedScores =
new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(),
size);
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
// walk all the translation options and record the cheapest option for each span
BOOST_FOREACH(const InputPathBase *path, m_inputPaths) {
const Range &range = path->range;
SCORE bestScore = -numeric_limits<SCORE>::infinity();
// walk all the translation options and record the cheapest option for each span
BOOST_FOREACH(const InputPathBase *path, m_inputPaths){
const Range &range = path->range;
SCORE bestScore = -numeric_limits<SCORE>::infinity();
size_t numPt = system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
if (tps) {
BOOST_FOREACH(const TargetPhrase *tp, *tps) {
SCORE score = tp->GetFutureScore();
if (score > bestScore) {
bestScore = score;
}
}
}
}
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
}
size_t numPt = system.mappings.size();
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
if (tps) {
BOOST_FOREACH(const TargetPhrase *tp, *tps) {
SCORE score = tp->GetFutureScore();
if (score > bestScore) {
bestScore = score;
}
}
}
}
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
}
// now fill all the cells in the strictly upper triangle
// there is no way to modify the diagonal now, in the case
// where no translation option covers a single-word span,
// we leave the +inf in the matrix
// like in chart parsing we want each cell to contain the highest score
// of the full-span trOpt or the sum of scores of joining two smaller spans
// now fill all the cells in the strictly upper triangle
// there is no way to modify the diagonal now, in the case
// where no translation option covers a single-word span,
// we leave the +inf in the matrix
// like in chart parsing we want each cell to contain the highest score
// of the full-span trOpt or the sum of scores of joining two smaller spans
for(size_t colstart = 1; colstart < size ; colstart++) {
for(size_t diagshift = 0; diagshift < size-colstart ; diagshift++) {
size_t sPos = diagshift;
size_t ePos = colstart+diagshift;
for(size_t joinAt = sPos; joinAt < ePos ; joinAt++) {
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
+ m_estimatedScores->GetValue(joinAt+1, ePos);
// uncomment to see the cell filling scheme
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
// << sPos << "," << joinAt << "]+["
// << joinAt+1 << "," << ePos << "] (colstart: "
// << colstart << ", diagshift: " << diagshift << ")"
// << endl);
for (size_t colstart = 1; colstart < size; colstart++) {
for (size_t diagshift = 0; diagshift < size - colstart; diagshift++) {
size_t sPos = diagshift;
size_t ePos = colstart + diagshift;
for (size_t joinAt = sPos; joinAt < ePos; joinAt++) {
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
+ m_estimatedScores->GetValue(joinAt + 1, ePos);
// uncomment to see the cell filling scheme
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
// << sPos << "," << joinAt << "]+["
// << joinAt+1 << "," << ePos << "] (colstart: "
// << colstart << ", diagshift: " << diagshift << ")"
// << endl);
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos))
m_estimatedScores->SetValue(sPos, ePos, joinedScore);
}
}
}
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos)) m_estimatedScores->SetValue(
sPos, ePos, joinedScore);
}
}
}
//cerr << "Square matrix:" << endl;
//cerr << *m_estimatedScores << endl;
//cerr << "Square matrix:" << endl;
//cerr << *m_estimatedScores << endl;
}
void Manager::OutputBest() const
{
stringstream out;
const Hypothesis *bestHypo = m_search->GetBestHypothesis();
if (bestHypo) {
if (system.outputHypoScore) {
out << bestHypo->GetScores().GetTotalScore() << " ";
}
stringstream out;
const Hypothesis *bestHypo = m_search->GetBestHypothesis();
if (bestHypo) {
if (system.outputHypoScore) {
out << bestHypo->GetScores().GetTotalScore() << " ";
}
bestHypo->OutputToStream(out);
//cerr << "BEST TRANSLATION: " << *bestHypo;
}
else {
if (system.outputHypoScore) {
out << "0 ";
}
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
}
out << "\n";
bestHypo->OutputToStream(out);
//cerr << "BEST TRANSLATION: " << *bestHypo;
}
else {
if (system.outputHypoScore) {
out << "0 ";
}
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
}
out << "\n";
system.bestCollector->Write(m_input->GetTranslationId(), out.str());
//cerr << endl;
system.bestCollector->Write(m_input->GetTranslationId(), out.str());
//cerr << endl;
}
void Manager::OutputNBest()
{
arcLists.Sort();
arcLists.Sort();
set<string> distinctHypos;
set<string> distinctHypos;
TrellisPaths contenders;
//cerr << "START AddInitialTrellisPaths" << endl;
m_search->AddInitialTrellisPaths(contenders);
//cerr << "END AddInitialTrellisPaths" << endl;
TrellisPaths contenders;
//cerr << "START AddInitialTrellisPaths" << endl;
m_search->AddInitialTrellisPaths(contenders);
//cerr << "END AddInitialTrellisPaths" << endl;
long transId = m_input->GetTranslationId();
long transId = m_input->GetTranslationId();
// MAIN LOOP
stringstream out;
size_t bestInd = 0;
while (bestInd < system.nbestSize && !contenders.empty()) {
//cerr << "bestInd=" << bestInd << endl;
TrellisPath *path = contenders.Get();
// MAIN LOOP
stringstream out;
size_t bestInd = 0;
while (bestInd < system.nbestSize && !contenders.empty()) {
//cerr << "bestInd=" << bestInd << endl;
TrellisPath *path = contenders.Get();
bool ok = false;
if (system.distinctNBest) {
string tgtPhrase = path->ToString();
if (distinctHypos.insert(tgtPhrase).second) {
ok = true;
}
}
else {
ok = true;
}
bool ok = false;
if (system.distinctNBest) {
string tgtPhrase = path->ToString();
if (distinctHypos.insert(tgtPhrase).second) {
ok = true;
}
}
else {
ok = true;
}
if (ok) {
out << transId << " |||";
path->OutputToStream(out, system);
out << "\n";
}
if (ok) {
out << transId << " |||";
path->OutputToStream(out, system);
out << "\n";
}
// create next paths
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
// create next paths
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
delete path;
delete path;
++bestInd;
}
system.nbestCollector->Write(transId, out.str());
++bestInd;
}
system.nbestCollector->Write(transId, out.str());
}

53
contrib/other-builds/moses2/PhraseBased/Manager.h
View File

@ -31,41 +31,50 @@ class Hypothesis;
class Sentence;
class OutputCollector;
class Manager : public ManagerBase
class Manager: public ManagerBase
{
public:
Manager(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId);
Manager(System &sys, const TranslationTask &task, const std::string &inputStr,
long translationId);
virtual ~Manager();
virtual ~Manager();
Bitmaps &GetBitmaps()
{ return *m_bitmaps; }
Bitmaps &GetBitmaps()
{
return *m_bitmaps;
}
const EstimatedScores &GetEstimatedScores() const
{ return *m_estimatedScores; }
const EstimatedScores &GetEstimatedScores() const
{
return *m_estimatedScores;
}
const InputPaths &GetInputPaths() const
{ return m_inputPaths; }
const InputPaths &GetInputPaths() const
{
return m_inputPaths;
}
const TargetPhrase &GetInitPhrase() const
{ return *m_initPhrase; }
const TargetPhrase &GetInitPhrase() const
{
return *m_initPhrase;
}
void Decode();
void Decode();
protected:
InputPaths m_inputPaths;
Bitmaps *m_bitmaps;
EstimatedScores *m_estimatedScores;
TargetPhrase *m_initPhrase;
InputPaths m_inputPaths;
Bitmaps *m_bitmaps;
EstimatedScores *m_estimatedScores;
TargetPhrase *m_initPhrase;
Search *m_search;
Search *m_search;
// must be run in same thread as Decode()
void Init();
void CalcFutureScore();
void OutputBest() const;
void OutputNBest();
// must be run in same thread as Decode()
void Init();
void CalcFutureScore();
void OutputBest() const;
void OutputNBest();
};

160
contrib/other-builds/moses2/PhraseBased/Normal/Search.cpp
View File

@ -23,135 +23,131 @@ using namespace std;
namespace Moses2
{
SearchNormal::SearchNormal(Manager &mgr)
:Search(mgr)
,m_stacks(mgr)
SearchNormal::SearchNormal(Manager &mgr) :
Search(mgr), m_stacks(mgr)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
SearchNormal::~SearchNormal() {
// TODO Auto-generated destructor stub
SearchNormal::~SearchNormal()
{
// TODO Auto-generated destructor stub
}
void SearchNormal::Decode()
{
// init stacks
m_stacks.Init(mgr, mgr.GetInput().GetSize() + 1);
// init stacks
m_stacks.Init(mgr, mgr.GetInput().GetSize() + 1);
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
Decode(stackInd);
//cerr << m_stacks << endl;
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
Decode(stackInd);
//cerr << m_stacks << endl;
// delete stack to save mem
if (stackInd < m_stacks.GetSize() - 1) {
m_stacks.Delete(stackInd);
}
//cerr << m_stacks << endl;
}
// delete stack to save mem
if (stackInd < m_stacks.GetSize() - 1) {
m_stacks.Delete(stackInd);
}
//cerr << m_stacks << endl;
}
}
void SearchNormal::Decode(size_t stackInd)
{
Stack &stack = m_stacks[stackInd];
if (&stack == &m_stacks.Back()) {
// last stack. don't do anythin
return;
// last stack. don't do anythin
return;
}
Hypotheses &hypos = stack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
const InputPaths &paths = mgr.GetInputPaths();
const InputPaths &paths = mgr.GetInputPaths();
BOOST_FOREACH(const InputPathBase *path, paths) {
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
}
}
BOOST_FOREACH(const InputPathBase *path, paths){
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
}
}
}
void SearchNormal::Extend(const Hypothesis &hypo, const InputPath &path)
{
const Bitmap &hypoBitmap = hypo.GetBitmap();
const Range &hypoRange = hypo.GetInputPath().range;
const Range &pathRange = path.range;
const Bitmap &hypoBitmap = hypo.GetBitmap();
const Range &hypoRange = hypo.GetInputPath().range;
const Range &pathRange = path.range;
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
return;
}
//cerr << " YES" << endl;
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
return;
}
//cerr << " YES" << endl;
// extend this hypo
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
// extend this hypo
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
size_t numPt = mgr.system.mappings.size();
const TargetPhrases **tpsAllPt = path.targetPhrases;
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = tpsAllPt[i];
if (tps) {
Extend(hypo, *tps, path, newBitmap, estimatedScore);
}
}
}
void SearchNormal::Extend(const Hypothesis &hypo,
const TargetPhrases &tps,
const InputPath &path,
const Bitmap &newBitmap,
SCORE estimatedScore)
{
BOOST_FOREACH(const TargetPhrase *tp, tps) {
Extend(hypo, *tp, path, newBitmap, estimatedScore);
size_t numPt = mgr.system.mappings.size();
const TargetPhrases **tpsAllPt = path.targetPhrases;
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = tpsAllPt[i];
if (tps) {
Extend(hypo, *tps, path, newBitmap, estimatedScore);
}
}
}
void SearchNormal::Extend(const Hypothesis &hypo,
const TargetPhrase &tp,
const InputPath &path,
const Bitmap &newBitmap,
SCORE estimatedScore)
void SearchNormal::Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
newHypo->EvaluateWhenApplied();
BOOST_FOREACH(const TargetPhrase *tp, tps){
Extend(hypo, *tp, path, newBitmap, estimatedScore);
}
}
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
void SearchNormal::Extend(const Hypothesis &hypo, const TargetPhrase &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
newHypo->EvaluateWhenApplied();
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
}
const Hypothesis *SearchNormal::GetBestHypothesis() const
{
const Stack &lastStack = m_stacks.Back();
const Hypotheses &sortedHypos = lastStack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
const Stack &lastStack = m_stacks.Back();
const Hypotheses &sortedHypos = lastStack.GetSortedAndPruneHypos(mgr,
mgr.arcLists);
const Hypothesis *best = NULL;
if (sortedHypos.size()) {
best = static_cast<const Hypothesis*>(sortedHypos[0]);
}
return best;
const Hypothesis *best = NULL;
if (sortedHypos.size()) {
best = static_cast<const Hypothesis*>(sortedHypos[0]);
}
return best;
}
void SearchNormal::AddInitialTrellisPaths(TrellisPaths &paths) const
{
const Stack &lastStack = m_stacks.Back();
BOOST_FOREACH(const HypothesisBase *hypoBase, lastStack) {
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
BOOST_FOREACH(const HypothesisBase *hypoBase, lastStack){
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
}

33
contrib/other-builds/moses2/PhraseBased/Normal/Search.h
View File

@ -23,36 +23,29 @@ class TargetPhrases;
class TargetPhrase;
class Stacks;
class SearchNormal : public Search
class SearchNormal: public Search
{
public:
SearchNormal(Manager &mgr);
virtual ~SearchNormal();
SearchNormal(Manager &mgr);
virtual ~SearchNormal();
virtual void Decode();
const Hypothesis *GetBestHypothesis() const;
virtual void Decode();
const Hypothesis *GetBestHypothesis() const;
void AddInitialTrellisPaths(TrellisPaths &paths) const;
void AddInitialTrellisPaths(TrellisPaths &paths) const;
protected:
Stacks m_stacks;
Stacks m_stacks;
void Decode(size_t stackInd);
void Extend(const Hypothesis &hypo, const InputPath &path);
void Extend(const Hypothesis &hypo,
const TargetPhrases &tps,
const InputPath &path,
const Bitmap &newBitmap,
SCORE estimatedScore);
void Extend(const Hypothesis &hypo,
const TargetPhrase &tp,
const InputPath &path,
const Bitmap &newBitmap,
SCORE estimatedScore);
void Decode(size_t stackInd);
void Extend(const Hypothesis &hypo, const InputPath &path);
void Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
void Extend(const Hypothesis &hypo, const TargetPhrase &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
};
}
#endif /* SEARCHNORMAL_H_ */

12
contrib/other-builds/moses2/PhraseBased/Normal/Stack.cpp
View File

@ -16,17 +16,17 @@ using namespace std;
namespace Moses2
{
Stack::Stack(const Manager &mgr)
:HypothesisColl(mgr)
Stack::Stack(const Manager &mgr) :
HypothesisColl(mgr)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
Stack::~Stack() {
// TODO Auto-generated destructor stub
Stack::~Stack()
{
// TODO Auto-generated destructor stub
}
}

7
contrib/other-builds/moses2/PhraseBased/Normal/Stack.h
View File

@ -16,15 +16,14 @@
namespace Moses2
{
class Stack : public HypothesisColl
class Stack: public HypothesisColl
{
public:
Stack(const Manager &mgr);
virtual ~Stack();
Stack(const Manager &mgr);
virtual ~Stack();
protected:
};
}

54
contrib/other-builds/moses2/PhraseBased/Normal/Stacks.cpp
View File

@ -14,43 +14,45 @@ using namespace std;
namespace Moses2
{
Stacks::Stacks(const Manager &mgr)
:m_mgr(mgr)
Stacks::Stacks(const Manager &mgr) :
m_mgr(mgr)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
Stacks::~Stacks() {
for (size_t i = 0; i < m_stacks.size(); ++i) {
delete m_stacks[i];
}
Stacks::~Stacks()
{
for (size_t i = 0; i < m_stacks.size(); ++i) {
delete m_stacks[i];
}
}
void Stacks::Init(const Manager &mgr, size_t numStacks)
{
m_stacks.resize(numStacks);
for (size_t i = 0; i < m_stacks.size(); ++i) {
m_stacks[i] = new Stack(mgr);
}
m_stacks.resize(numStacks);
for (size_t i = 0; i < m_stacks.size(); ++i) {
m_stacks[i] = new Stack(mgr);
}
}
std::ostream& operator<<(std::ostream &out, const Stacks &obj)
{
for (size_t i = 0; i < obj.GetSize(); ++i) {
const Stack *stack = obj.m_stacks[i];
if (stack) {
out << stack->GetSize() << " ";
}
else {
out << "N ";
}
const Stack *stack = obj.m_stacks[i];
if (stack) {
out << stack->GetSize() << " ";
}
else {
out << "N ";
}
}
return out;
}
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists)
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
{
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
//cerr << "numWordsCovered=" << numWordsCovered << endl;
@ -59,15 +61,15 @@ void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLi
size_t nbestSize = m_mgr.system.nbestSize;
if (nbestSize) {
arcLists.AddArc(added.added, hypo, added.other);
arcLists.AddArc(added.added, hypo, added.other);
}
else {
if (!added.added) {
hypoRecycle.Recycle(hypo);
}
else if (added.other) {
hypoRecycle.Recycle(added.other);
}
if (!added.added) {
hypoRecycle.Recycle(hypo);
}
else if (added.other) {
hypoRecycle.Recycle(added.other);
}
}
}

45
contrib/other-builds/moses2/PhraseBased/Normal/Stacks.h
View File

@ -16,33 +16,42 @@ namespace Moses2
class Manager;
class ArcLists;
class Stacks {
friend std::ostream& operator<<(std::ostream &, const Stacks &);
class Stacks
{
friend std::ostream& operator<<(std::ostream &, const Stacks &);
public:
Stacks(const Manager &mgr);
virtual ~Stacks();
Stacks(const Manager &mgr);
virtual ~Stacks();
void Init(const Manager &mgr, size_t numStacks);
void Init(const Manager &mgr, size_t numStacks);
size_t GetSize() const
{ return m_stacks.size(); }
size_t GetSize() const
{
return m_stacks.size();
}
const Stack &Back() const
{ return *m_stacks.back(); }
const Stack &Back() const
{
return *m_stacks.back();
}
Stack &operator[](size_t ind)
{ return *m_stacks[ind]; }
Stack &operator[](size_t ind)
{
return *m_stacks[ind];
}
void Delete(size_t ind) {
delete m_stacks[ind];
m_stacks[ind] = NULL;
}
void Delete(size_t ind)
{
delete m_stacks[ind];
m_stacks[ind] = NULL;
}
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists);
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
protected:
const Manager &m_mgr;
std::vector<Stack*> m_stacks;
const Manager &m_mgr;
std::vector<Stack*> m_stacks;
};
}

148
contrib/other-builds/moses2/PhraseBased/Search.cpp
View File

@ -14,97 +14,99 @@
namespace Moses2
{
Search::Search(Manager &mgr)
:mgr(mgr)
Search::Search(Manager &mgr) :
mgr(mgr)
{
// TODO Auto-generated constructor stub
// TODO Auto-generated constructor stub
}
Search::~Search() {
// TODO Auto-generated destructor stub
Search::~Search()
{
// TODO Auto-generated destructor stub
}
bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos, const Range &pathRange)
bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
const Range &pathRange)
{
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
//cerr << "DOING " << hypoBitmap << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
// " [" << pathRange.GetStartPos() << " " << pathRange.GetEndPos() << "]";
//cerr << "DOING " << hypoBitmap << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
// " [" << pathRange.GetStartPos() << " " << pathRange.GetEndPos() << "]";
if (hypoBitmap.Overlap(pathRange)) {
//cerr << " NO" << endl;
return false;
}
if (hypoBitmap.Overlap(pathRange)) {
//cerr << " NO" << endl;
return false;
}
if (mgr.system.maxDistortion >= 0) {
// distortion limit
int distortion = ComputeDistortionDistance(hypoRangeEndPos, pathRange.GetStartPos());
if (distortion > mgr.system.maxDistortion) {
//cerr << " NO" << endl;
return false;
}
}
// first question: is there a path from the closest translated word to the left
// of the hypothesized extension to the start of the hypothesized extension?
// long version:
// - is there anything to our left?
// - is it farther left than where we're starting anyway?
// - can we get to it?
// closestLeft is exclusive: a value of 3 means 2 is covered, our
// arc is currently ENDING at 3 and can start at 3 implicitly
// TODO is this relevant? only for lattice input?
// ask second question here: we already know we can get to our
// starting point from the closest thing to the left. We now ask the
// follow up: can we get from our end to the closest thing on the
// right?
//
// long version: is anything to our right? is it farther
// right than our (inclusive) end? can our end reach it?
bool isLeftMostEdge = (hypoFirstGapPos == pathRange.GetStartPos());
size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(pathRange.GetEndPos());
/*
if (isWordLattice) {
if (closestRight != endPos
&& ((closestRight + 1) < sourceSize)
&& !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
continue;
}
if (mgr.system.maxDistortion >= 0) {
// distortion limit
int distortion = ComputeDistortionDistance(hypoRangeEndPos,
pathRange.GetStartPos());
if (distortion > mgr.system.maxDistortion) {
//cerr << " NO" << endl;
return false;
}
*/
}
if (isLeftMostEdge) {
// any length extension is okay if starting at left-most edge
// first question: is there a path from the closest translated word to the left
// of the hypothesized extension to the start of the hypothesized extension?
// long version:
// - is there anything to our left?
// - is it farther left than where we're starting anyway?
// - can we get to it?
} else { // starting somewhere other than left-most edge, use caution
// the basic idea is this: we would like to translate a phrase
// starting from a position further right than the left-most
// open gap. The distortion penalty for the following phrase
// will be computed relative to the ending position of the
// current extension, so we ask now what its maximum value will
// be (which will always be the value of the hypothesis starting
// at the left-most edge). If this value is less than the
// distortion limit, we don't allow this extension to be made.
Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
// closestLeft is exclusive: a value of 3 means 2 is covered, our
// arc is currently ENDING at 3 and can start at 3 implicitly
if (ComputeDistortionDistance(pathRange.GetEndPos(), bestNextExtension.GetStartPos())
> mgr.system.maxDistortion) {
//cerr << " NO" << endl;
return false;
}
// TODO is this relevant? only for lattice input?
// everything is fine, we're good to go
// ask second question here: we already know we can get to our
// starting point from the closest thing to the left. We now ask the
// follow up: can we get from our end to the closest thing on the
// right?
//
// long version: is anything to our right? is it farther
// right than our (inclusive) end? can our end reach it?
bool isLeftMostEdge = (hypoFirstGapPos == pathRange.GetStartPos());
size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(pathRange.GetEndPos());
/*
if (isWordLattice) {
if (closestRight != endPos
&& ((closestRight + 1) < sourceSize)
&& !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
continue;
}
}
*/
if (isLeftMostEdge) {
// any length extension is okay if starting at left-most edge
}
else { // starting somewhere other than left-most edge, use caution
// the basic idea is this: we would like to translate a phrase
// starting from a position further right than the left-most
// open gap. The distortion penalty for the following phrase
// will be computed relative to the ending position of the
// current extension, so we ask now what its maximum value will
// be (which will always be the value of the hypothesis starting
// at the left-most edge). If this value is less than the
// distortion limit, we don't allow this extension to be made.
Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
if (ComputeDistortionDistance(pathRange.GetEndPos(),
bestNextExtension.GetStartPos()) > mgr.system.maxDistortion) {
//cerr << " NO" << endl;
return false;
}
return true;
// everything is fine, we're good to go
}
return true;
}
}

43
contrib/other-builds/moses2/PhraseBased/Search.h
View File

@ -20,35 +20,38 @@ class Bitmap;
class Range;
class TrellisPaths;
class Search {
class Search
{
public:
Search(Manager &mgr);
virtual ~Search();
Search(Manager &mgr);
virtual ~Search();
virtual void Decode() = 0;
virtual const Hypothesis *GetBestHypothesis() const = 0;
virtual void Decode() = 0;
virtual const Hypothesis *GetBestHypothesis() const = 0;
virtual void AddInitialTrellisPaths(TrellisPaths &paths) const = 0;
virtual void AddInitialTrellisPaths(TrellisPaths &paths) const = 0;
protected:
Manager &mgr;
//ArcLists m_arcLists;
Manager &mgr;
//ArcLists m_arcLists;
bool CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos, const Range &pathRange);
bool CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
const Range &pathRange);
inline int ComputeDistortionDistance(size_t prevEndPos, size_t currStartPos) const
{
int dist = 0;
if (prevEndPos == NOT_FOUND) {
dist = currStartPos;
} else {
dist = (int)prevEndPos - (int)currStartPos + 1 ;
}
return abs(dist);
}
inline int ComputeDistortionDistance(size_t prevEndPos,
size_t currStartPos) const
{
int dist = 0;
if (prevEndPos == NOT_FOUND) {
dist = currStartPos;
}
else {
dist = (int)prevEndPos - (int)currStartPos + 1;
}
return abs(dist);
}
};
}

40
contrib/other-builds/moses2/PhraseBased/TargetPhraseImpl.cpp
View File

@ -17,37 +17,41 @@ using namespace std;
namespace Moses2
{
TargetPhraseImpl *TargetPhraseImpl::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
TargetPhraseImpl *TargetPhraseImpl::CreateFromString(MemPool &pool,
const PhraseTable &pt, const System &system, const std::string &str)
{
FactorCollection &vocab = system.GetVocab();
FactorCollection &vocab = system.GetVocab();
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhraseImpl *ret = new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, pt, system, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhraseImpl *ret =
new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, pt, system,
size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
return ret;
}
TargetPhraseImpl::TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
:TargetPhrase(pool, pt, system)
,PhraseImplTemplate<Word>(pool, size)
TargetPhraseImpl::TargetPhraseImpl(MemPool &pool, const PhraseTable &pt,
const System &system, size_t size) :
TargetPhrase(pool, pt, system), PhraseImplTemplate<Word>(pool, size)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
system.featureFunctions.GetNumScores());
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
TargetPhraseImpl::~TargetPhraseImpl() {
// TODO Auto-generated destructor stub
TargetPhraseImpl::~TargetPhraseImpl()
{
// TODO Auto-generated destructor stub
}
std::ostream& operator<<(std::ostream &out, const TargetPhraseImpl &obj)
{
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
return out;
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
return out;
}
}

26
contrib/other-builds/moses2/PhraseBased/TargetPhraseImpl.h
View File

@ -22,30 +22,38 @@ class Manager;
class System;
class PhraseTable;
class TargetPhraseImpl : public TargetPhrase, public PhraseImplTemplate<Word>
class TargetPhraseImpl: public TargetPhrase, public PhraseImplTemplate<Word>
{
friend std::ostream& operator<<(std::ostream &, const TargetPhraseImpl &);
friend std::ostream& operator<<(std::ostream &, const TargetPhraseImpl &);
public:
static TargetPhraseImpl *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
static TargetPhraseImpl *CreateFromString(MemPool &pool,
const PhraseTable &pt, const System &system, const std::string &str);
TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system,
size_t size);
//TargetPhraseImpl(MemPool &pool, const System &system, const TargetPhraseImpl &copy);
virtual ~TargetPhraseImpl();
const Word& operator[](size_t pos) const
{ return m_words[pos]; }
{
return m_words[pos];
}
Word& operator[](size_t pos)
{ return m_words[pos]; }
{
return m_words[pos];
}
size_t GetSize() const
{ return m_size; }
{
return m_size;
}
SubPhrase GetSubPhrase(size_t start, size_t end) const
{
SubPhrase ret(*this, start, end);
return ret;
SubPhrase ret(*this, start, end);
return ret;
}
//mutable void *chartState;

150
contrib/other-builds/moses2/PhraseBased/TrellisPath.cpp
View File

@ -13,34 +13,32 @@
using namespace std;
namespace Moses2 {
namespace Moses2
{
std::ostream& operator<<(std::ostream &out, const TrellisNode &node)
{
out << "arcList=" << node.arcList->size() << " " << node.ind;
return out;
out << "arcList=" << node.arcList->size() << " " << node.ind;
return out;
}
/////////////////////////////////////////////////////////////////////////////////
TrellisPath::TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists)
:prevEdgeChanged(-1)
TrellisPath::TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists) :
prevEdgeChanged(-1)
{
AddNodes(hypo, arcLists);
m_scores = &hypo->GetScores();
AddNodes(hypo, arcLists);
m_scores = &hypo->GetScores();
}
TrellisPath::TrellisPath(const TrellisPath &origPath,
size_t edgeIndex,
const TrellisNode &newNode,
const ArcLists &arcLists,
MemPool &pool,
const System &system)
:prevEdgeChanged(edgeIndex)
TrellisPath::TrellisPath(const TrellisPath &origPath, size_t edgeIndex,
const TrellisNode &newNode, const ArcLists &arcLists, MemPool &pool,
const System &system) :
prevEdgeChanged(edgeIndex)
{
nodes.reserve(origPath.nodes.size());
for (size_t currEdge = 0 ; currEdge < edgeIndex ; currEdge++) {
// copy path from parent
nodes.push_back(origPath.nodes[currEdge]);
for (size_t currEdge = 0; currEdge < edgeIndex; currEdge++) {
// copy path from parent
nodes.push_back(origPath.nodes[currEdge]);
}
// 1 deviation
@ -51,9 +49,9 @@ TrellisPath::TrellisPath(const TrellisPath &origPath,
const Hypothesis *prevHypo = arc->GetPrevHypo();
while (prevHypo != NULL) {
const ArcList *arcList = arcLists.GetArcList(prevHypo);
assert(arcList);
TrellisNode node(*arcList, 0);
const ArcList *arcList = arcLists.GetArcList(prevHypo);
assert(arcList);
TrellisNode node(*arcList, 0);
nodes.push_back(node);
prevHypo = prevHypo->GetPrevHypo();
@ -63,104 +61,96 @@ TrellisPath::TrellisPath(const TrellisPath &origPath,
const HypothesisBase *origHypo = origNode.GetHypo();
const HypothesisBase *newHypo = newNode.GetHypo();
CalcScores(origPath.GetScores(),
origHypo->GetScores(),
newHypo->GetScores(),
pool,
system);
CalcScores(origPath.GetScores(), origHypo->GetScores(), newHypo->GetScores(),
pool, system);
}
TrellisPath::~TrellisPath() {
// TODO Auto-generated destructor stub
TrellisPath::~TrellisPath()
{
// TODO Auto-generated destructor stub
}
SCORE TrellisPath::GetFutureScore() const
{
return m_scores->GetTotalScore();
return m_scores->GetTotalScore();
}
void TrellisPath::OutputToStream(std::ostream &out, const System &system) const
{
out << ToString();
out << "||| ";
out << ToString();
out << "||| ";
GetScores().OutputBreakdownToStream(out, system);
out << "||| ";
GetScores().OutputBreakdownToStream(out, system);
out << "||| ";
out << GetScores().GetTotalScore();
out << GetScores().GetTotalScore();
}
std::string TrellisPath::ToString() const
{
//cerr << "path=" << this << " " << nodes.size() << endl;
std::stringstream out;
for (int i = nodes.size() - 1; i >= 0; --i) {
const TrellisNode &node = nodes[i];
const Hypothesis *hypo = static_cast<const Hypothesis*>(node.GetHypo());
//cerr << "hypo=" << hypo << " " << *hypo << endl;
hypo->GetTargetPhrase().OutputToStream(out);
out << " ";
}
return out.str();
//cerr << "path=" << this << " " << nodes.size() << endl;
std::stringstream out;
for (int i = nodes.size() - 1; i >= 0; --i) {
const TrellisNode &node = nodes[i];
const Hypothesis *hypo = static_cast<const Hypothesis*>(node.GetHypo());
//cerr << "hypo=" << hypo << " " << *hypo << endl;
hypo->GetTargetPhrase().OutputToStream(out);
out << " ";
}
return out.str();
}
void TrellisPath::CreateDeviantPaths(TrellisPaths &paths,
const ArcLists &arcLists,
MemPool &pool,
const System &system) const
const ArcLists &arcLists, MemPool &pool, const System &system) const
{
const size_t sizePath = nodes.size();
//cerr << "prevEdgeChanged=" << prevEdgeChanged << endl;
for (size_t currEdge = prevEdgeChanged + 1 ; currEdge < sizePath ; currEdge++) {
TrellisNode newNode = nodes[currEdge];
for (size_t currEdge = prevEdgeChanged + 1; currEdge < sizePath; currEdge++) {
TrellisNode newNode = nodes[currEdge];
assert(newNode.ind == 0);
const ArcList &arcList = *newNode.arcList;
const ArcList &arcList = *newNode.arcList;
//cerr << "arcList=" << arcList.size() << endl;
for (size_t i = 1; i < arcList.size(); ++i) {
//cerr << "i=" << i << endl;
newNode.ind = i;
for (size_t i = 1; i < arcList.size(); ++i) {
//cerr << "i=" << i << endl;
newNode.ind = i;
TrellisPath *deviantPath = new TrellisPath(*this, currEdge, newNode, arcLists, pool, system);
//cerr << "deviantPath=" << deviantPath << endl;
paths.Add(deviantPath);
}
TrellisPath *deviantPath = new TrellisPath(*this, currEdge, newNode,
arcLists, pool, system);
//cerr << "deviantPath=" << deviantPath << endl;
paths.Add(deviantPath);
}
}
}
void TrellisPath::CalcScores(const Scores &origScores,
const Scores &origHypoScores,
const Scores &newHypoScores,
MemPool &pool,
const System &system)
const Scores &origHypoScores, const Scores &newHypoScores, MemPool &pool,
const System &system)
{
Scores *scores = new (pool.Allocate<Scores>())
Scores(system,
pool,
system.featureFunctions.GetNumScores(),
origScores);
scores->PlusEquals(system, newHypoScores);
scores->MinusEquals(system, origHypoScores);
Scores *scores = new (pool.Allocate<Scores>()) Scores(system, pool,
system.featureFunctions.GetNumScores(), origScores);
scores->PlusEquals(system, newHypoScores);
scores->MinusEquals(system, origHypoScores);
m_scores = scores;
m_scores = scores;
}
void TrellisPath::AddNodes(const Hypothesis *hypo, const ArcLists &arcLists)
{
if (hypo) {
// add this hypo
//cerr << "hypo=" << hypo << " " << flush;
//cerr << *hypo << endl;
const ArcList *list = arcLists.GetArcList(hypo);
assert(list);
TrellisNode node(*list, 0);
nodes.push_back(node);
if (hypo) {
// add this hypo
//cerr << "hypo=" << hypo << " " << flush;
//cerr << *hypo << endl;
const ArcList *list = arcLists.GetArcList(hypo);
assert(list);
TrellisNode node(*list, 0);
nodes.push_back(node);
// add prev hypos
const Hypothesis *prev = hypo->GetPrevHypo();
AddNodes(prev, arcLists);
}
// add prev hypos
const Hypothesis *prev = hypo->GetPrevHypo();
AddNodes(prev, arcLists);
}
}
} /* namespace Moses2 */

78
contrib/other-builds/moses2/PhraseBased/TrellisPath.h
View File

@ -9,7 +9,8 @@
#include "../TypeDef.h"
#include "../ArcLists.h"
namespace Moses2 {
namespace Moses2
{
class Scores;
class MemPool;
@ -22,62 +23,59 @@ class TrellisNode
friend std::ostream& operator<<(std::ostream &, const TrellisNode &);
public:
const ArcList *arcList;
size_t ind;
const ArcList *arcList;
size_t ind;
TrellisNode(const ArcList &varcList, size_t vind)
:arcList(&varcList)
,ind(vind)
{}
TrellisNode(const ArcList &varcList, size_t vind) :
arcList(&varcList), ind(vind)
{
}
const HypothesisBase *GetHypo() const
{ return (*arcList)[ind]; }
const HypothesisBase *GetHypo() const
{
return (*arcList)[ind];
}
};
class TrellisPath {
class TrellisPath
{
public:
std::vector<TrellisNode> nodes;
int prevEdgeChanged;
std::vector<TrellisNode> nodes;
int prevEdgeChanged;
/**< the last node that was wiggled to create this path
, or NOT_FOUND if this path is the best trans so consist of only hypos
*/
TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists);
/**< the last node that was wiggled to create this path
, or NOT_FOUND if this path is the best trans so consist of only hypos
*/
TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists);
/** create path from another path, deviate at edgeIndex by using arc instead,
* which may change other hypo back from there
*/
TrellisPath(const TrellisPath &origPath,
size_t edgeIndex,
const TrellisNode &newNode,
const ArcLists &arcLists,
MemPool &pool,
const System &system);
/** create path from another path, deviate at edgeIndex by using arc instead,
* which may change other hypo back from there
*/
TrellisPath(const TrellisPath &origPath, size_t edgeIndex,
const TrellisNode &newNode, const ArcLists &arcLists, MemPool &pool,
const System &system);
virtual ~TrellisPath();
virtual ~TrellisPath();
const Scores &GetScores() const
{ return *m_scores; }
SCORE GetFutureScore() const;
const Scores &GetScores() const
{
return *m_scores;
}
SCORE GetFutureScore() const;
void OutputToStream(std::ostream &out, const System &system) const;
std::string ToString() const;
void OutputToStream(std::ostream &out, const System &system) const;
std::string ToString() const;
//! create a set of next best paths by wiggling 1 of the node at a time.
void CreateDeviantPaths(TrellisPaths &paths,
const ArcLists &arcLists,
MemPool &pool,
const System &system) const;
void CreateDeviantPaths(TrellisPaths &paths, const ArcLists &arcLists,
MemPool &pool, const System &system) const;
protected:
const Scores *m_scores;
void AddNodes(const Hypothesis *hypo, const ArcLists &arcLists);
void CalcScores(const Scores &origScores,
const Scores &origHypoScores,
const Scores &newHypoScores,
MemPool &pool,
const System &system);
void CalcScores(const Scores &origScores, const Scores &origHypoScores,
const Scores &newHypoScores, MemPool &pool, const System &system);
};
} /* namespace Moses2 */

19
contrib/other-builds/moses2/PhraseBased/TrellisPaths.cpp
View File

@ -8,18 +8,21 @@
#include "TrellisPaths.h"
#include "../legacy/Util2.h"
namespace Moses2 {
namespace Moses2
{
TrellisPaths::TrellisPaths() {
// TODO Auto-generated constructor stub
TrellisPaths::TrellisPaths()
{
// TODO Auto-generated constructor stub
}
TrellisPaths::~TrellisPaths() {
while (!empty()) {
TrellisPath *path = Get();
delete path;
}
TrellisPaths::~TrellisPaths()
{
while (!empty()) {
TrellisPath *path = Get();
delete path;
}
}
void TrellisPaths::Add(TrellisPath *trellisPath)

21
contrib/other-builds/moses2/PhraseBased/TrellisPaths.h
View File

@ -10,27 +10,33 @@
#include <queue>
#include "TrellisPath.h"
namespace Moses2 {
namespace Moses2
{
struct CompareTrellisPathCollection {
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const {
struct CompareTrellisPathCollection
{
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const
{
return (pathA->GetFutureScore() < pathB->GetFutureScore());
}
};
class TrellisPaths {
class TrellisPaths
{
public:
TrellisPaths();
virtual ~TrellisPaths();
bool empty() const {
bool empty() const
{
return m_collection.empty();
}
//! add a new entry into collection
void Add(TrellisPath *trellisPath);
TrellisPath *Get() {
TrellisPath *Get()
{
TrellisPath *top = m_collection.top();
// Detach
@ -39,7 +45,8 @@ public:
}
protected:
typedef std::priority_queue<TrellisPath*, std::vector<TrellisPath*>, CompareTrellisPathCollection> CollectionType;
typedef std::priority_queue<TrellisPath*, std::vector<TrellisPath*>,
CompareTrellisPathCollection> CollectionType;
CollectionType m_collection;
};

1
contrib/other-builds/moses2/PhraseImpl.cpp
View File

@ -11,6 +11,5 @@ using namespace std;
namespace Moses2
{
}

42
contrib/other-builds/moses2/PhraseImpl.h
View File

@ -5,42 +5,48 @@
namespace Moses2
{
class PhraseImpl : public Phrase, public PhraseImplTemplate<Word>
class PhraseImpl: public Phrase, public PhraseImplTemplate<Word>
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
}
PhraseImpl(MemPool &pool, size_t size)
:PhraseImplTemplate<Word>(pool, size)
{}
PhraseImpl(MemPool &pool, size_t size) :
PhraseImplTemplate<Word>(pool, size)
{
}
const Word& operator[](size_t pos) const
{ return m_words[pos]; }
{
return m_words[pos];
}
Word& operator[](size_t pos) {
return m_words[pos];
Word& operator[](size_t pos)
{
return m_words[pos];
}
size_t GetSize() const
{ return m_size; }
{
return m_size;
}
SubPhrase GetSubPhrase(size_t start, size_t size) const
{
SubPhrase ret(*this, start, size);
return ret;
SubPhrase ret(*this, start, size);
return ret;
}
};
}

46
contrib/other-builds/moses2/PhraseImplTemplate.h
View File

@ -12,8 +12,6 @@
#include "Phrase.h"
#include "legacy/Util2.h"
namespace Moses2
{
class SubPhrase;
@ -22,50 +20,50 @@ template<typename WORD>
class PhraseImplTemplate
{
public:
PhraseImplTemplate(MemPool &pool, size_t size)
:m_size(size)
PhraseImplTemplate(MemPool &pool, size_t size) :
m_size(size)
{
m_words = new (pool.Allocate<WORD>(size)) WORD[size];
}
PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate &copy)
:m_size(copy.GetSize())
PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate &copy) :
m_size(copy.GetSize())
{
m_words = new (pool.Allocate<WORD>(m_size)) WORD[m_size];
for (size_t i = 0; i < m_size; ++i) {
const WORD &word = copy[i];
(*this)[i] = word;
}
for (size_t i = 0; i < m_size; ++i) {
const WORD &word = copy[i];
(*this)[i] = word;
}
}
virtual ~PhraseImplTemplate()
{}
{
}
size_t GetSize() const
{ return m_size; }
{
return m_size;
}
protected:
size_t m_size;
WORD *m_words;
WORD& operator[](size_t pos) {
return m_words[pos];
WORD& operator[](size_t pos)
{
return m_words[pos];
}
void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks, bool addBOSEOS = false)
void CreateFromString(FactorCollection &vocab, const System &system,
const std::vector<std::string> &toks, bool addBOSEOS = false)
{
for (size_t i = 0; i < m_size; ++i) {
WORD &word = (*this)[i];
word.CreateFromString(vocab, system, toks[i]);
}
for (size_t i = 0; i < m_size; ++i) {
WORD &word = (*this)[i];
word.CreateFromString(vocab, system, toks[i]);
}
}
};
}

4
contrib/other-builds/moses2/Recycler.cpp
View File

@ -7,7 +7,7 @@
#include "Recycler.h"
namespace Moses2 {
namespace Moses2
{
} /* namespace Moses2 */

89
contrib/other-builds/moses2/Recycler.h
View File

@ -10,59 +10,64 @@
#include <deque>
#include <vector>
namespace Moses2 {
namespace Moses2
{
template<typename T>
class Recycler {
class Recycler
{
public:
Recycler()
:m_currInd(0)
{}
virtual ~Recycler()
{}
Recycler() :
m_currInd(0)
{
}
virtual ~Recycler()
{
}
T Get()
{
if (!m_coll.empty()) {
T &obj = m_coll.back();
m_coll.pop_back();
return obj;
}
else if (m_currInd) {
--m_currInd;
T &obj = m_all[m_currInd];
return obj;
}
else {
return NULL;
}
}
T Get()
{
if (!m_coll.empty()) {
T &obj = m_coll.back();
m_coll.pop_back();
return obj;
}
else if (m_currInd) {
--m_currInd;
T &obj = m_all[m_currInd];
return obj;
}
else {
return NULL;
}
}
void Clear()
{
m_coll.clear();
m_currInd = m_all.size();
}
void Clear()
{
m_coll.clear();
m_currInd = m_all.size();
}
void Keep(const T& val) {
m_all.push_back(val);
}
void Keep(const T& val)
{
m_all.push_back(val);
}
void Recycle(const T& val)
{
m_coll.push_back(val);
}
void Recycle(const T& val)
{
m_coll.push_back(val);
}
protected:
// all objects we're looking after
std::vector<T> m_all;
// all objects we're looking after
std::vector<T> m_all;
// pointer to the object that's just been given out.
// to give out another obj, must decrement THEN give out
size_t m_currInd;
// pointer to the object that's just been given out.
// to give out another obj, must decrement THEN give out
size_t m_currInd;
// objects that have been give back to us
std::deque<T> m_coll;
// objects that have been give back to us
std::deque<T> m_coll;
};
} /* namespace Moses2 */

23
contrib/other-builds/moses2/SCFG/InputPath.cpp
View File

@ -14,31 +14,28 @@ namespace Moses2
namespace SCFG
{
InputPath::InputPath(MemPool &pool,
const SubPhrase &subPhrase,
const Range &range,
size_t numPt,
const InputPath *prefixPath)
:InputPathBase(pool, subPhrase, range, numPt, prefixPath)
InputPath::InputPath(MemPool &pool, const SubPhrase &subPhrase,
const Range &range, size_t numPt, const InputPath *prefixPath) :
InputPathBase(pool, subPhrase, range, numPt, prefixPath)
{
m_activeChart = pool.Allocate<ActiveChart>(numPt);
for (size_t i = 0; i < numPt; ++i) {
ActiveChart &memAddr = m_activeChart[i];
ActiveChart *obj =new (&memAddr) ActiveChart();
ActiveChart &memAddr = m_activeChart[i];
ActiveChart *obj = new (&memAddr) ActiveChart();
}
}
InputPath::~InputPath() {
// TODO Auto-generated destructor stub
InputPath::~InputPath()
{
// TODO Auto-generated destructor stub
}
std::ostream& operator<<(std::ostream &out, const InputPath &obj)
{
out << obj.range << " " << obj.subPhrase;
return out;
out << obj.range << " " << obj.subPhrase;
return out;
}
}
}

28
contrib/other-builds/moses2/SCFG/InputPath.h
View File

@ -18,33 +18,37 @@ namespace SCFG
class ActiveChartEntry
{
public:
const void *data;
const void *data;
ActiveChartEntry(const void *vdata)
:data(vdata)
{}
ActiveChartEntry(const void *vdata) :
data(vdata)
{
}
};
////////////////////////////////////////////////////////////////////////////
class ActiveChart
{
public:
std::vector<ActiveChartEntry*> entries;
std::vector<ActiveChartEntry*> entries;
};
////////////////////////////////////////////////////////////////////////////
class InputPath : public InputPathBase
class InputPath: public InputPathBase
{
friend std::ostream& operator<<(std::ostream &, const InputPath &);
friend std::ostream& operator<<(std::ostream &, const InputPath &);
public:
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPath *prefixPath);
virtual ~InputPath();
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
size_t numPt, const InputPath *prefixPath);
virtual ~InputPath();
ActiveChart &GetActiveChart(size_t ptInd)
{ return m_activeChart[ptInd]; }
ActiveChart &GetActiveChart(size_t ptInd)
{
return m_activeChart[ptInd];
}
protected:
ActiveChart *m_activeChart;
ActiveChart *m_activeChart;
};
}

31
contrib/other-builds/moses2/SCFG/InputPaths.cpp
View File

@ -25,35 +25,36 @@ void InputPaths::Init(const Sentence &input, const ManagerBase &mgr)
size_t size = input.GetSize();
size_t maxLength = min(size, mgr.system.maxPhraseLength);
m_matrix = new (pool.Allocate< Matrix<InputPath*> >()) Matrix<InputPath*>(pool, size, maxLength);
m_matrix = new (pool.Allocate<Matrix<InputPath*> >()) Matrix<InputPath*>(pool,
size, maxLength);
m_matrix->Init(NULL);
// create normal paths of subphrases through the sentence
for (size_t startPos = 0; startPos < size; ++startPos) {
const InputPath *prefixPath = NULL;
const InputPath *prefixPath = NULL;
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
size_t endPos = startPos + phaseSize - 1;
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
size_t endPos = startPos + phaseSize - 1;
if (endPos >= size) {
break;
}
if (endPos >= size) {
break;
}
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
Range range(startPos, endPos);
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
Range range(startPos, endPos);
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, prefixPath);
m_inputPaths.push_back(path);
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool,
subPhrase, range, numPt, prefixPath);
m_inputPaths.push_back(path);
prefixPath = path;
prefixPath = path;
m_matrix->SetValue(startPos, phaseSize - 1, path);
}
m_matrix->SetValue(startPos, phaseSize - 1, path);
}
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More