mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
format
This commit is contained in:
parent
52b4d094bf
commit
e6096aa8bf
@ -16,79 +16,81 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
ArcLists::ArcLists() {
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
ArcLists::~ArcLists() {
|
||||
BOOST_FOREACH(const Coll::value_type &collPair, m_coll) {
|
||||
const ArcList *arcList = collPair.second;
|
||||
delete arcList;
|
||||
}
|
||||
}
|
||||
|
||||
void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,const HypothesisBase *otherHypo)
|
||||
ArcLists::ArcLists()
|
||||
{
|
||||
//cerr << added << " " << currHypo << " " << otherHypo << endl;
|
||||
ArcList *arcList;
|
||||
if (added) {
|
||||
// we're winners!
|
||||
if (otherHypo) {
|
||||
// there was a existing losing hypo
|
||||
arcList = GetAndDetachArcList(otherHypo);
|
||||
}
|
||||
else {
|
||||
// there was no existing hypo
|
||||
arcList = new ArcList;
|
||||
}
|
||||
m_coll[currHypo] = arcList;
|
||||
}
|
||||
else {
|
||||
// we're losers!
|
||||
// there should be a winner, we're not doing beam pruning
|
||||
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
|
||||
arcList = GetArcList(otherHypo);
|
||||
}
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
// in any case, add the curr hypo
|
||||
arcList->push_back(currHypo);
|
||||
}
|
||||
|
||||
ArcLists::~ArcLists()
|
||||
{
|
||||
BOOST_FOREACH(const Coll::value_type &collPair, m_coll){
|
||||
const ArcList *arcList = collPair.second;
|
||||
delete arcList;
|
||||
}
|
||||
}
|
||||
|
||||
void ArcLists::AddArc(bool added, const HypothesisBase *currHypo,
|
||||
const HypothesisBase *otherHypo)
|
||||
{
|
||||
//cerr << added << " " << currHypo << " " << otherHypo << endl;
|
||||
ArcList *arcList;
|
||||
if (added) {
|
||||
// we're winners!
|
||||
if (otherHypo) {
|
||||
// there was a existing losing hypo
|
||||
arcList = GetAndDetachArcList(otherHypo);
|
||||
}
|
||||
else {
|
||||
// there was no existing hypo
|
||||
arcList = new ArcList;
|
||||
}
|
||||
m_coll[currHypo] = arcList;
|
||||
}
|
||||
else {
|
||||
// we're losers!
|
||||
// there should be a winner, we're not doing beam pruning
|
||||
UTIL_THROW_IF2(otherHypo == NULL, "There must have been a winning hypo");
|
||||
arcList = GetArcList(otherHypo);
|
||||
}
|
||||
|
||||
// in any case, add the curr hypo
|
||||
arcList->push_back(currHypo);
|
||||
}
|
||||
|
||||
ArcList *ArcLists::GetArcList(const HypothesisBase *hypo)
|
||||
{
|
||||
Coll::iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
return arcList;
|
||||
Coll::iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
return arcList;
|
||||
}
|
||||
|
||||
const ArcList *ArcLists::GetArcList(const HypothesisBase *hypo) const
|
||||
{
|
||||
Coll::const_iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
return arcList;
|
||||
Coll::const_iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
return arcList;
|
||||
}
|
||||
|
||||
|
||||
ArcList *ArcLists::GetAndDetachArcList(const HypothesisBase *hypo)
|
||||
{
|
||||
Coll::iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
Coll::iterator iter = m_coll.find(hypo);
|
||||
UTIL_THROW_IF2(iter == m_coll.end(), "Can't find arc list");
|
||||
ArcList *arcList = iter->second;
|
||||
|
||||
m_coll.erase(iter);
|
||||
m_coll.erase(iter);
|
||||
|
||||
return arcList;
|
||||
return arcList;
|
||||
}
|
||||
|
||||
void ArcLists::Sort()
|
||||
{
|
||||
BOOST_FOREACH(Coll::value_type &collPair, m_coll) {
|
||||
ArcList &list = *collPair.second;
|
||||
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
|
||||
}
|
||||
BOOST_FOREACH(Coll::value_type &collPair, m_coll){
|
||||
ArcList &list = *collPair.second;
|
||||
std::sort(list.begin(), list.end(), HypothesisFutureScoreOrderer() );
|
||||
}
|
||||
}
|
||||
|
||||
void ArcLists::Delete(const HypothesisBase *hypo)
|
||||
|
@ -15,23 +15,25 @@ class HypothesisBase;
|
||||
|
||||
typedef std::vector<const HypothesisBase*> ArcList;
|
||||
|
||||
class ArcLists {
|
||||
class ArcLists
|
||||
{
|
||||
public:
|
||||
ArcLists();
|
||||
virtual ~ArcLists();
|
||||
ArcLists();
|
||||
virtual ~ArcLists();
|
||||
|
||||
void AddArc(bool added, const HypothesisBase *currHypo, const HypothesisBase *otherHypo);
|
||||
void Sort();
|
||||
void Delete(const HypothesisBase *hypo);
|
||||
void AddArc(bool added, const HypothesisBase *currHypo,
|
||||
const HypothesisBase *otherHypo);
|
||||
void Sort();
|
||||
void Delete(const HypothesisBase *hypo);
|
||||
|
||||
const ArcList *GetArcList(const HypothesisBase *hypo) const;
|
||||
const ArcList *GetArcList(const HypothesisBase *hypo) const;
|
||||
|
||||
protected:
|
||||
typedef boost::unordered_map<const HypothesisBase*, ArcList*> Coll;
|
||||
Coll m_coll;
|
||||
typedef boost::unordered_map<const HypothesisBase*, ArcList*> Coll;
|
||||
Coll m_coll;
|
||||
|
||||
ArcList *GetArcList(const HypothesisBase *hypo);
|
||||
ArcList *GetAndDetachArcList(const HypothesisBase *hypo);
|
||||
ArcList *GetArcList(const HypothesisBase *hypo);
|
||||
ArcList *GetAndDetachArcList(const HypothesisBase *hypo);
|
||||
|
||||
};
|
||||
|
||||
|
@ -2,23 +2,23 @@
|
||||
// vim:tabstop=2
|
||||
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2006 University of Edinburgh
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2006 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
@ -34,18 +34,18 @@ namespace Moses2
|
||||
* /param bitmap coverage bitmap
|
||||
*/
|
||||
|
||||
float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap ) const
|
||||
float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap) const
|
||||
{
|
||||
const size_t notInGap= numeric_limits<size_t>::max();
|
||||
const size_t notInGap = numeric_limits<size_t>::max();
|
||||
size_t startGap = notInGap;
|
||||
float estimatedScore = 0.0f;
|
||||
for(size_t currPos = 0 ; currPos < bitmap.GetSize() ; currPos++) {
|
||||
for (size_t currPos = 0; currPos < bitmap.GetSize(); currPos++) {
|
||||
// start of a new gap?
|
||||
if(bitmap.GetValue(currPos) == false && startGap == notInGap) {
|
||||
if (bitmap.GetValue(currPos) == false && startGap == notInGap) {
|
||||
startGap = currPos;
|
||||
}
|
||||
// end of a gap?
|
||||
else if(bitmap.GetValue(currPos) == true && startGap != notInGap) {
|
||||
else if (bitmap.GetValue(currPos) == true && startGap != notInGap) {
|
||||
estimatedScore += GetValue(startGap, currPos - 1);
|
||||
startGap = notInGap;
|
||||
}
|
||||
@ -73,37 +73,41 @@ float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap ) const
|
||||
* /param endPos end of the span that is added to the coverage
|
||||
*/
|
||||
|
||||
float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap, size_t startPos, size_t endPos ) const
|
||||
float EstimatedScores::CalcEstimatedScore(Bitmap const &bitmap, size_t startPos,
|
||||
size_t endPos) const
|
||||
{
|
||||
const size_t notInGap= numeric_limits<size_t>::max();
|
||||
const size_t notInGap = numeric_limits<size_t>::max();
|
||||
float estimatedScore = 0.0f;
|
||||
size_t startGap = bitmap.GetFirstGapPos();
|
||||
if (startGap == NOT_FOUND) return estimatedScore; // everything filled
|
||||
|
||||
// start loop at first gap
|
||||
size_t startLoop = startGap+1;
|
||||
size_t startLoop = startGap + 1;
|
||||
if (startPos == startGap) { // unless covered by phrase
|
||||
startGap = notInGap;
|
||||
startLoop = endPos+1; // -> postpone start
|
||||
startLoop = endPos + 1; // -> postpone start
|
||||
}
|
||||
|
||||
size_t lastCovered = bitmap.GetLastPos();
|
||||
if (endPos > lastCovered || lastCovered == NOT_FOUND) lastCovered = endPos;
|
||||
|
||||
for(size_t currPos = startLoop; currPos <= lastCovered ; currPos++) {
|
||||
for (size_t currPos = startLoop; currPos <= lastCovered; currPos++) {
|
||||
// start of a new gap?
|
||||
if(startGap == notInGap && bitmap.GetValue(currPos) == false && (currPos < startPos || currPos > endPos)) {
|
||||
if (startGap == notInGap && bitmap.GetValue(currPos) == false
|
||||
&& (currPos < startPos || currPos > endPos)) {
|
||||
startGap = currPos;
|
||||
}
|
||||
// end of a gap?
|
||||
else if(startGap != notInGap && (bitmap.GetValue(currPos) == true || (startPos <= currPos && currPos <= endPos))) {
|
||||
else if (startGap != notInGap
|
||||
&& (bitmap.GetValue(currPos) == true
|
||||
|| (startPos <= currPos && currPos <= endPos))) {
|
||||
estimatedScore += GetValue(startGap, currPos - 1);
|
||||
startGap = notInGap;
|
||||
}
|
||||
}
|
||||
// coverage ending with gap?
|
||||
if (lastCovered != bitmap.GetSize() - 1) {
|
||||
estimatedScore += GetValue(lastCovered+1, bitmap.GetSize() - 1);
|
||||
estimatedScore += GetValue(lastCovered + 1, bitmap.GetSize() - 1);
|
||||
}
|
||||
|
||||
return estimatedScore;
|
||||
@ -111,4 +115,3 @@ float EstimatedScores::CalcEstimatedScore( Bitmap const &bitmap, size_t startPos
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -1,23 +1,23 @@
|
||||
// $Id$
|
||||
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2006 University of Edinburgh
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2006 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
@ -31,26 +31,29 @@ namespace Moses2
|
||||
class MemPool;
|
||||
|
||||
//! A square array of floats to store future costs in the phrase-based decoder
|
||||
class EstimatedScores : public Matrix<float>
|
||||
class EstimatedScores: public Matrix<float>
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &out, const EstimatedScores &matrix);
|
||||
friend std::ostream& operator<<(std::ostream &out,
|
||||
const EstimatedScores &matrix);
|
||||
|
||||
public:
|
||||
EstimatedScores(MemPool &pool, size_t size)
|
||||
:Matrix<float>(pool, size, size)
|
||||
{}
|
||||
EstimatedScores(MemPool &pool, size_t size) :
|
||||
Matrix<float>(pool, size, size)
|
||||
{
|
||||
}
|
||||
|
||||
~EstimatedScores(); // not implemented
|
||||
|
||||
float CalcEstimatedScore( Bitmap const& ) const;
|
||||
float CalcEstimatedScore( Bitmap const&, size_t startPos, size_t endPos ) const;
|
||||
float CalcEstimatedScore(Bitmap const&) const;
|
||||
float CalcEstimatedScore(Bitmap const&, size_t startPos, size_t endPos) const;
|
||||
|
||||
};
|
||||
|
||||
inline std::ostream& operator<<(std::ostream &out, const EstimatedScores &matrix)
|
||||
inline std::ostream& operator<<(std::ostream &out,
|
||||
const EstimatedScores &matrix)
|
||||
{
|
||||
for (size_t endPos = 0 ; endPos < matrix.GetSize() ; endPos++) {
|
||||
for (size_t startPos = 0 ; startPos < matrix.GetSize() ; startPos++)
|
||||
for (size_t endPos = 0; endPos < matrix.GetSize(); endPos++) {
|
||||
for (size_t startPos = 0; startPos < matrix.GetSize(); startPos++)
|
||||
out << matrix.GetValue(startPos, endPos) << " ";
|
||||
out << std::endl;
|
||||
}
|
||||
|
@ -16,50 +16,53 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
struct DistortionState_traditional : public FFState {
|
||||
struct DistortionState_traditional: public FFState
|
||||
{
|
||||
Range range;
|
||||
int first_gap;
|
||||
|
||||
DistortionState_traditional()
|
||||
:range()
|
||||
DistortionState_traditional() :
|
||||
range()
|
||||
{
|
||||
// uninitialised
|
||||
// uninitialised
|
||||
}
|
||||
|
||||
void Set(const Range& wr, int fg)
|
||||
{
|
||||
range = wr;
|
||||
first_gap = fg;
|
||||
range = wr;
|
||||
first_gap = fg;
|
||||
}
|
||||
|
||||
size_t hash() const {
|
||||
size_t hash() const
|
||||
{
|
||||
return range.GetEndPos();
|
||||
}
|
||||
virtual bool operator==(const FFState& other) const {
|
||||
virtual bool operator==(const FFState& other) const
|
||||
{
|
||||
const DistortionState_traditional& o =
|
||||
static_cast<const DistortionState_traditional&>(other);
|
||||
static_cast<const DistortionState_traditional&>(other);
|
||||
return range.GetEndPos() == o.range.GetEndPos();
|
||||
}
|
||||
|
||||
virtual std::string ToString() const
|
||||
{
|
||||
stringstream sb;
|
||||
sb << first_gap << " " << range;
|
||||
return sb.str();
|
||||
stringstream sb;
|
||||
sb << first_gap << " " << range;
|
||||
return sb.str();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
Distortion::Distortion(size_t startInd, const std::string &line)
|
||||
:StatefulFeatureFunction(startInd, line)
|
||||
Distortion::Distortion(size_t startInd, const std::string &line) :
|
||||
StatefulFeatureFunction(startInd, line)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
Distortion::~Distortion() {
|
||||
// TODO Auto-generated destructor stub
|
||||
Distortion::~Distortion()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
FFState* Distortion::BlankState(MemPool &pool) const
|
||||
@ -67,76 +70,70 @@ FFState* Distortion::BlankState(MemPool &pool) const
|
||||
return new (pool.Allocate<DistortionState_traditional>()) DistortionState_traditional();
|
||||
}
|
||||
|
||||
void Distortion::EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
void Distortion::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const
|
||||
{
|
||||
DistortionState_traditional &stateCast = static_cast<DistortionState_traditional&>(state);
|
||||
DistortionState_traditional &stateCast =
|
||||
static_cast<DistortionState_traditional&>(state);
|
||||
|
||||
// fake previous translated phrase start and end
|
||||
size_t start = NOT_FOUND;
|
||||
size_t end = NOT_FOUND;
|
||||
/*
|
||||
if (input.m_frontSpanCoveredLength > 0) {
|
||||
// can happen with --continue-partial-translation
|
||||
start = 0;
|
||||
end = input.m_frontSpanCoveredLength -1;
|
||||
}
|
||||
*/
|
||||
// fake previous translated phrase start and end
|
||||
size_t start = NOT_FOUND;
|
||||
size_t end = NOT_FOUND;
|
||||
/*
|
||||
if (input.m_frontSpanCoveredLength > 0) {
|
||||
// can happen with --continue-partial-translation
|
||||
start = 0;
|
||||
end = input.m_frontSpanCoveredLength -1;
|
||||
}
|
||||
*/
|
||||
|
||||
stateCast.range = Range(start, end);
|
||||
stateCast.first_gap = NOT_FOUND;
|
||||
stateCast.range = Range(start, end);
|
||||
stateCast.first_gap = NOT_FOUND;
|
||||
}
|
||||
|
||||
void
|
||||
Distortion::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void Distortion::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
}
|
||||
|
||||
void Distortion::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
const DistortionState_traditional &prev = static_cast<const DistortionState_traditional&>(prevState);
|
||||
SCORE distortionScore = CalculateDistortionScore(
|
||||
prev.range,
|
||||
hypo.GetInputPath().range,
|
||||
prev.first_gap);
|
||||
//cerr << "distortionScore=" << distortionScore << endl;
|
||||
const DistortionState_traditional &prev =
|
||||
static_cast<const DistortionState_traditional&>(prevState);
|
||||
SCORE distortionScore = CalculateDistortionScore(prev.range,
|
||||
hypo.GetInputPath().range, prev.first_gap);
|
||||
//cerr << "distortionScore=" << distortionScore << endl;
|
||||
|
||||
scores.PlusEquals(mgr.system, *this, distortionScore);
|
||||
scores.PlusEquals(mgr.system, *this, distortionScore);
|
||||
|
||||
DistortionState_traditional &stateCast = static_cast<DistortionState_traditional&>(state);
|
||||
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
|
||||
DistortionState_traditional &stateCast =
|
||||
static_cast<DistortionState_traditional&>(state);
|
||||
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
|
||||
}
|
||||
|
||||
SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr, const int FirstGap) const
|
||||
SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
|
||||
const int FirstGap) const
|
||||
{
|
||||
bool useEarlyDistortionCost = false;
|
||||
if(!useEarlyDistortionCost) {
|
||||
return - (SCORE) ComputeDistortionDistance(prev, curr);
|
||||
} else {
|
||||
if (!useEarlyDistortionCost) {
|
||||
return -(SCORE) ComputeDistortionDistance(prev, curr);
|
||||
}
|
||||
else {
|
||||
/* Pay distortion score as soon as possible, from Moore and Quirk MT Summit 2007
|
||||
Definitions:
|
||||
S : current source range
|
||||
S' : last translated source phrase range
|
||||
S'' : longest fully-translated initial segment
|
||||
*/
|
||||
Definitions:
|
||||
S : current source range
|
||||
S' : last translated source phrase range
|
||||
S'' : longest fully-translated initial segment
|
||||
*/
|
||||
|
||||
int prefixEndPos = (int)FirstGap-1;
|
||||
if((int)FirstGap==-1)
|
||||
prefixEndPos = -1;
|
||||
int prefixEndPos = (int) FirstGap - 1;
|
||||
if ((int) FirstGap == -1) prefixEndPos = -1;
|
||||
|
||||
// case1: S is adjacent to S'' => return 0
|
||||
if ((int) curr.GetStartPos() == prefixEndPos+1) {
|
||||
if ((int) curr.GetStartPos() == prefixEndPos + 1) {
|
||||
//IFVERBOSE(4) std::cerr<< "MQ07disto:case1" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
@ -144,30 +141,33 @@ SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
|
||||
// case2: S is to the left of S' => return 2(length(S))
|
||||
if ((int) curr.GetEndPos() < (int) prev.GetEndPos()) {
|
||||
//IFVERBOSE(4) std::cerr<< "MQ07disto:case2" << std::endl;
|
||||
return (float) -2*(int)curr.GetNumWordsCovered();
|
||||
return (float) -2 * (int) curr.GetNumWordsCovered();
|
||||
}
|
||||
|
||||
// case3: S' is a subsequence of S'' => return 2(nbWordBetween(S,S'')+length(S))
|
||||
if ((int) prev.GetEndPos() <= prefixEndPos) {
|
||||
//IFVERBOSE(4) std::cerr<< "MQ07disto:case3" << std::endl;
|
||||
int z = (int)curr.GetStartPos()-prefixEndPos - 1;
|
||||
return (float) -2*(z + (int)curr.GetNumWordsCovered());
|
||||
int z = (int) curr.GetStartPos() - prefixEndPos - 1;
|
||||
return (float) -2 * (z + (int) curr.GetNumWordsCovered());
|
||||
}
|
||||
|
||||
// case4: otherwise => return 2(nbWordBetween(S,S')+length(S))
|
||||
//IFVERBOSE(4) std::cerr<< "MQ07disto:case4" << std::endl;
|
||||
return (float) -2*((int)curr.GetNumWordsBetween(prev) + (int)curr.GetNumWordsCovered());
|
||||
return (float) -2
|
||||
* ((int) curr.GetNumWordsBetween(prev) + (int) curr.GetNumWordsCovered());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
int Distortion::ComputeDistortionDistance(const Range& prev, const Range& current) const
|
||||
int Distortion::ComputeDistortionDistance(const Range& prev,
|
||||
const Range& current) const
|
||||
{
|
||||
int dist = 0;
|
||||
if (prev.GetNumWordsCovered() == 0) {
|
||||
dist = current.GetStartPos();
|
||||
} else {
|
||||
dist = (int)prev.GetEndPos() - (int)current.GetStartPos() + 1 ;
|
||||
}
|
||||
else {
|
||||
dist = (int) prev.GetEndPos() - (int) current.GetStartPos() + 1;
|
||||
}
|
||||
return abs(dist);
|
||||
}
|
||||
|
@ -15,37 +15,32 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class Distortion : public StatefulFeatureFunction
|
||||
class Distortion: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
Distortion(size_t startInd, const std::string &line);
|
||||
virtual ~Distortion();
|
||||
Distortion(size_t startInd, const std::string &line);
|
||||
virtual ~Distortion();
|
||||
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const std::deque<Hypothesis*> &hypos) const
|
||||
{}
|
||||
{
|
||||
}
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
SCORE CalculateDistortionScore(const Range &prev, const Range &curr, const int FirstGap) const;
|
||||
SCORE CalculateDistortionScore(const Range &prev, const Range &curr,
|
||||
const int FirstGap) const;
|
||||
|
||||
int ComputeDistortionDistance(const Range& prev, const Range& current) const;
|
||||
|
||||
|
@ -16,17 +16,16 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
FeatureFunction::FeatureFunction(size_t startInd, const std::string &line)
|
||||
:m_startInd(startInd)
|
||||
,m_numScores(1)
|
||||
,m_PhraseTableInd(NOT_FOUND)
|
||||
{
|
||||
ParseLine(line);
|
||||
cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
|
||||
}
|
||||
FeatureFunction::FeatureFunction(size_t startInd, const std::string &line) :
|
||||
m_startInd(startInd), m_numScores(1), m_PhraseTableInd(NOT_FOUND)
|
||||
{
|
||||
ParseLine(line);
|
||||
cerr << GetName() << " " << m_startInd << "-" << (m_startInd + m_numScores - 1) << endl;
|
||||
}
|
||||
|
||||
FeatureFunction::~FeatureFunction() {
|
||||
// TODO Auto-generated destructor stub
|
||||
FeatureFunction::~FeatureFunction()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void FeatureFunction::ParseLine(const std::string &line)
|
||||
@ -41,13 +40,13 @@ void FeatureFunction::ParseLine(const std::string &line)
|
||||
for (size_t i = 1; i < toks.size(); ++i) {
|
||||
vector<string> args = TokenizeFirstOnly(toks[i], "=");
|
||||
UTIL_THROW_IF2(args.size() != 2,
|
||||
"Incorrect format for feature function arg: " << toks[i]);
|
||||
"Incorrect format for feature function arg: " << toks[i]);
|
||||
|
||||
pair<set<string>::iterator,bool> ret = keys.insert(args[0]);
|
||||
pair<set<string>::iterator, bool> ret = keys.insert(args[0]);
|
||||
UTIL_THROW_IF2(!ret.second, "Duplicate key in line " << line);
|
||||
|
||||
if (args[0] == "num-features") {
|
||||
m_numScores = Scan<size_t>(args[1]);
|
||||
m_numScores = Scan<size_t>(args[1]);
|
||||
}
|
||||
else if (args[0] == "name") {
|
||||
m_name = args[1];
|
||||
@ -68,7 +67,8 @@ void FeatureFunction::ReadParameters()
|
||||
}
|
||||
}
|
||||
|
||||
void FeatureFunction::SetParameter(const std::string& key, const std::string& value)
|
||||
void FeatureFunction::SetParameter(const std::string& key,
|
||||
const std::string& value)
|
||||
{
|
||||
if (key == "tuneable") {
|
||||
m_tuneable = Scan<bool>(value);
|
||||
|
@ -24,70 +24,88 @@ class Scores;
|
||||
class ManagerBase;
|
||||
class MemPool;
|
||||
|
||||
namespace SCFG
|
||||
{
|
||||
class TargetPhrase;
|
||||
}
|
||||
namespace SCFG
|
||||
{
|
||||
class TargetPhrase;
|
||||
}
|
||||
|
||||
class FeatureFunction {
|
||||
class FeatureFunction
|
||||
{
|
||||
public:
|
||||
|
||||
FeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~FeatureFunction();
|
||||
virtual void Load(System &system)
|
||||
{}
|
||||
FeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~FeatureFunction();
|
||||
virtual void Load(System &system)
|
||||
{
|
||||
}
|
||||
|
||||
size_t GetStartInd() const
|
||||
{ return m_startInd; }
|
||||
size_t GetNumScores() const
|
||||
{ return m_numScores; }
|
||||
const std::string &GetName() const
|
||||
{ return m_name; }
|
||||
void SetName(const std::string &val)
|
||||
{ m_name = val; }
|
||||
size_t GetStartInd() const
|
||||
{
|
||||
return m_startInd;
|
||||
}
|
||||
size_t GetNumScores() const
|
||||
{
|
||||
return m_numScores;
|
||||
}
|
||||
const std::string &GetName() const
|
||||
{
|
||||
return m_name;
|
||||
}
|
||||
void SetName(const std::string &val)
|
||||
{
|
||||
m_name = val;
|
||||
}
|
||||
|
||||
virtual size_t HasPhraseTableInd() const
|
||||
{ return false; }
|
||||
void SetPhraseTableInd(size_t ind)
|
||||
{ m_PhraseTableInd = ind; }
|
||||
size_t GetPhraseTableInd() const
|
||||
{ return m_PhraseTableInd; }
|
||||
virtual size_t HasPhraseTableInd() const
|
||||
{
|
||||
return false;
|
||||
}
|
||||
void SetPhraseTableInd(size_t ind)
|
||||
{
|
||||
m_PhraseTableInd = ind;
|
||||
}
|
||||
size_t GetPhraseTableInd() const
|
||||
{
|
||||
return m_PhraseTableInd;
|
||||
}
|
||||
|
||||
// may have more factors than actually need, but not guaranteed.
|
||||
// For SCFG decoding, the source contains non-terminals, NOT the raw
|
||||
// source from the input sentence
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const = 0;
|
||||
// may have more factors than actually need, but not guaranteed.
|
||||
// For SCFG decoding, the source contains non-terminals, NOT the raw
|
||||
// source from the input sentence
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const = 0;
|
||||
|
||||
/*
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const SCFG::TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const {}
|
||||
*/
|
||||
/*
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const SCFG::TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const {}
|
||||
*/
|
||||
|
||||
virtual void
|
||||
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const
|
||||
{}
|
||||
virtual void EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrases &tps, const Phrase &sourcePhrase) const
|
||||
{
|
||||
}
|
||||
|
||||
// clean up temporary memory, called after processing each sentence
|
||||
virtual void CleanUpAfterSentenceProcessing() {}
|
||||
// clean up temporary memory, called after processing each sentence
|
||||
virtual void CleanUpAfterSentenceProcessing()
|
||||
{
|
||||
}
|
||||
|
||||
protected:
|
||||
size_t m_startInd;
|
||||
size_t m_numScores;
|
||||
size_t m_PhraseTableInd;
|
||||
std::string m_name;
|
||||
std::vector<std::vector<std::string> > m_args;
|
||||
bool m_tuneable;
|
||||
size_t m_startInd;
|
||||
size_t m_numScores;
|
||||
size_t m_PhraseTableInd;
|
||||
std::string m_name;
|
||||
std::vector<std::vector<std::string> > m_args;
|
||||
bool m_tuneable;
|
||||
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
virtual void ReadParameters();
|
||||
void ParseLine(const std::string &line);
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
virtual void ReadParameters();
|
||||
void ParseLine(const std::string &line);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -31,41 +31,41 @@ using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
FeatureFunctions::FeatureFunctions(System &system)
|
||||
:m_system(system)
|
||||
,m_ffStartInd(0)
|
||||
FeatureFunctions::FeatureFunctions(System &system) :
|
||||
m_system(system), m_ffStartInd(0)
|
||||
{
|
||||
//m_registry.PrintFF();
|
||||
//m_registry.PrintFF();
|
||||
}
|
||||
|
||||
FeatureFunctions::~FeatureFunctions() {
|
||||
RemoveAllInColl(m_featureFunctions);
|
||||
FeatureFunctions::~FeatureFunctions()
|
||||
{
|
||||
RemoveAllInColl(m_featureFunctions);
|
||||
}
|
||||
|
||||
void FeatureFunctions::Load()
|
||||
{
|
||||
// load, everything but pts
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
|
||||
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
|
||||
FeatureFunction *nonConstFF = const_cast<FeatureFunction*>(ff);
|
||||
PhraseTable *pt = dynamic_cast<PhraseTable*>(nonConstFF);
|
||||
|
||||
if (pt) {
|
||||
// do nothing. load pt last
|
||||
}
|
||||
else {
|
||||
cerr << "Loading " << nonConstFF->GetName() << endl;
|
||||
nonConstFF->Load(m_system);
|
||||
cerr << "Finished loading " << nonConstFF->GetName() << endl;
|
||||
}
|
||||
if (pt) {
|
||||
// do nothing. load pt last
|
||||
}
|
||||
else {
|
||||
cerr << "Loading " << nonConstFF->GetName() << endl;
|
||||
nonConstFF->Load(m_system);
|
||||
cerr << "Finished loading " << nonConstFF->GetName() << endl;
|
||||
}
|
||||
}
|
||||
|
||||
// load pt
|
||||
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
|
||||
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
|
||||
cerr << "Loading " << nonConstPT->GetName() << endl;
|
||||
nonConstPT->Load(m_system);
|
||||
cerr << "Finished loading " << nonConstPT->GetName() << endl;
|
||||
}
|
||||
// load pt
|
||||
BOOST_FOREACH(const PhraseTable *pt, m_phraseTables) {
|
||||
PhraseTable *nonConstPT = const_cast<PhraseTable*>(pt);
|
||||
cerr << "Loading " << nonConstPT->GetName() << endl;
|
||||
nonConstPT->Load(m_system);
|
||||
cerr << "Finished loading " << nonConstPT->GetName() << endl;
|
||||
}
|
||||
}
|
||||
|
||||
void FeatureFunctions::Create()
|
||||
@ -75,125 +75,129 @@ void FeatureFunctions::Create()
|
||||
const PARAM_VEC *ffParams = params.GetParam("feature");
|
||||
UTIL_THROW_IF2(ffParams == NULL, "Must have [feature] section");
|
||||
|
||||
BOOST_FOREACH(const std::string &line, *ffParams) {
|
||||
cerr << "line=" << line << endl;
|
||||
FeatureFunction *ff = Create(line);
|
||||
BOOST_FOREACH(const std::string &line, *ffParams){
|
||||
cerr << "line=" << line << endl;
|
||||
FeatureFunction *ff = Create(line);
|
||||
|
||||
m_featureFunctions.push_back(ff);
|
||||
m_featureFunctions.push_back(ff);
|
||||
|
||||
StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
|
||||
if (sfff) {
|
||||
sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
|
||||
m_statefulFeatureFunctions.push_back(sfff);
|
||||
}
|
||||
|
||||
if (ff->HasPhraseTableInd()) {
|
||||
ff->SetPhraseTableInd(m_withPhraseTableInd.size());
|
||||
m_withPhraseTableInd.push_back(ff);
|
||||
}
|
||||
|
||||
PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
|
||||
if (pt) {
|
||||
pt->SetPtInd(m_phraseTables.size());
|
||||
m_phraseTables.push_back(pt);
|
||||
}
|
||||
StatefulFeatureFunction *sfff = dynamic_cast<StatefulFeatureFunction*>(ff);
|
||||
if (sfff) {
|
||||
sfff->SetStatefulInd(m_statefulFeatureFunctions.size());
|
||||
m_statefulFeatureFunctions.push_back(sfff);
|
||||
}
|
||||
|
||||
if (ff->HasPhraseTableInd()) {
|
||||
ff->SetPhraseTableInd(m_withPhraseTableInd.size());
|
||||
m_withPhraseTableInd.push_back(ff);
|
||||
}
|
||||
|
||||
PhraseTable *pt = dynamic_cast<PhraseTable*>(ff);
|
||||
if (pt) {
|
||||
pt->SetPtInd(m_phraseTables.size());
|
||||
m_phraseTables.push_back(pt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FeatureFunction *FeatureFunctions::Create(const std::string &line)
|
||||
{
|
||||
vector<string> toks = Tokenize(line);
|
||||
vector<string> toks = Tokenize(line);
|
||||
|
||||
FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
|
||||
UTIL_THROW_IF2(ff == NULL, "Feature function not created");
|
||||
FeatureFunction *ff = m_registry.Construct(m_ffStartInd, toks[0], line);
|
||||
UTIL_THROW_IF2(ff == NULL, "Feature function not created");
|
||||
|
||||
// name
|
||||
if (ff->GetName() == "") {
|
||||
ff->SetName(GetDefaultName(toks[0]));
|
||||
}
|
||||
// name
|
||||
if (ff->GetName() == "") {
|
||||
ff->SetName(GetDefaultName(toks[0]));
|
||||
}
|
||||
|
||||
m_ffStartInd += ff->GetNumScores();
|
||||
m_ffStartInd += ff->GetNumScores();
|
||||
|
||||
return ff;
|
||||
return ff;
|
||||
}
|
||||
|
||||
std::string FeatureFunctions::GetDefaultName(const std::string &stub)
|
||||
{
|
||||
size_t ind;
|
||||
boost::unordered_map<std::string, size_t>::iterator iter = m_defaultNames.find(stub);
|
||||
if (iter == m_defaultNames.end()) {
|
||||
m_defaultNames[stub] = 0;
|
||||
ind = 0;
|
||||
}
|
||||
else {
|
||||
ind = ++(iter->second);
|
||||
}
|
||||
return stub + SPrint(ind);
|
||||
}
|
||||
|
||||
const FeatureFunction *FeatureFunctions::FindFeatureFunction(const std::string &name) const
|
||||
{
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
if (ff->GetName() == name) {
|
||||
return ff;
|
||||
}
|
||||
size_t ind;
|
||||
boost::unordered_map<std::string, size_t>::iterator iter =
|
||||
m_defaultNames.find(stub);
|
||||
if (iter == m_defaultNames.end()) {
|
||||
m_defaultNames[stub] = 0;
|
||||
ind = 0;
|
||||
}
|
||||
return NULL;
|
||||
else {
|
||||
ind = ++(iter->second);
|
||||
}
|
||||
return stub + SPrint(ind);
|
||||
}
|
||||
|
||||
const PhraseTable *FeatureFunctions::GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd)
|
||||
const FeatureFunction *FeatureFunctions::FindFeatureFunction(
|
||||
const std::string &name) const
|
||||
{
|
||||
// assume only 1 unk wp
|
||||
std::vector<const PhraseTable*> tmpVec(m_phraseTables);
|
||||
std::vector<const PhraseTable*>::iterator iter;
|
||||
for (iter = tmpVec.begin(); iter != tmpVec.end(); ++iter) {
|
||||
const PhraseTable *pt = *iter;
|
||||
const UnknownWordPenalty *unkWP = dynamic_cast<const UnknownWordPenalty *>(pt);
|
||||
if (unkWP) {
|
||||
tmpVec.erase(iter);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const PhraseTable *pt = tmpVec[ptInd];
|
||||
return pt;
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
|
||||
if (ff->GetName() == name) {
|
||||
return ff;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void
|
||||
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, TargetPhrase &targetPhrase) const
|
||||
const PhraseTable *FeatureFunctions::GetPhraseTablesExcludeUnknownWordPenalty(
|
||||
size_t ptInd)
|
||||
{
|
||||
// assume only 1 unk wp
|
||||
std::vector<const PhraseTable*> tmpVec(m_phraseTables);
|
||||
std::vector<const PhraseTable*>::iterator iter;
|
||||
for (iter = tmpVec.begin(); iter != tmpVec.end(); ++iter) {
|
||||
const PhraseTable *pt = *iter;
|
||||
const UnknownWordPenalty *unkWP =
|
||||
dynamic_cast<const UnknownWordPenalty *>(pt);
|
||||
if (unkWP) {
|
||||
tmpVec.erase(iter);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const PhraseTable *pt = tmpVec[ptInd];
|
||||
return pt;
|
||||
}
|
||||
|
||||
void FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, TargetPhrase &targetPhrase) const
|
||||
{
|
||||
SCORE estimatedScore = 0;
|
||||
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
Scores& scores = targetPhrase.GetScores();
|
||||
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
|
||||
}
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
|
||||
Scores& scores = targetPhrase.GetScores();
|
||||
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
|
||||
}
|
||||
|
||||
targetPhrase.SetEstimatedScore(estimatedScore);
|
||||
}
|
||||
|
||||
void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const
|
||||
void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrases &tps, const Phrase &sourcePhrase) const
|
||||
{
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
|
||||
}
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions){
|
||||
ff->EvaluateAfterTablePruning(pool, tps, sourcePhrase);
|
||||
}
|
||||
|
||||
}
|
||||
/*
|
||||
void
|
||||
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const
|
||||
{
|
||||
SCORE estimatedScore = 0;
|
||||
void
|
||||
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const
|
||||
{
|
||||
SCORE estimatedScore = 0;
|
||||
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
Scores& scores = targetPhrase.GetScores();
|
||||
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
|
||||
}
|
||||
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
|
||||
Scores& scores = targetPhrase.GetScores();
|
||||
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
|
||||
}
|
||||
|
||||
//targetPhrase.SetEstimatedScore(estimatedScore);
|
||||
}
|
||||
*/
|
||||
//targetPhrase.SetEstimatedScore(estimatedScore);
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
|
@ -30,55 +30,65 @@ class Scores;
|
||||
|
||||
namespace SCFG
|
||||
{
|
||||
class TargetPhrase;
|
||||
class TargetPhrase;
|
||||
}
|
||||
|
||||
class FeatureFunctions {
|
||||
class FeatureFunctions
|
||||
{
|
||||
public:
|
||||
std::vector<const PhraseTable*> m_phraseTables;
|
||||
std::vector<const PhraseTable*> m_phraseTables;
|
||||
|
||||
FeatureFunctions(System &system);
|
||||
virtual ~FeatureFunctions();
|
||||
FeatureFunctions(System &system);
|
||||
virtual ~FeatureFunctions();
|
||||
|
||||
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const
|
||||
{ return m_featureFunctions; }
|
||||
const std::vector<const FeatureFunction*> &GetFeatureFunctions() const
|
||||
{
|
||||
return m_featureFunctions;
|
||||
}
|
||||
|
||||
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const
|
||||
{ return m_statefulFeatureFunctions; }
|
||||
const std::vector<const StatefulFeatureFunction*> &GetStatefulFeatureFunctions() const
|
||||
{
|
||||
return m_statefulFeatureFunctions;
|
||||
}
|
||||
|
||||
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const
|
||||
{ return m_withPhraseTableInd; }
|
||||
const std::vector<const FeatureFunction*> &GetWithPhraseTableInd() const
|
||||
{
|
||||
return m_withPhraseTableInd;
|
||||
}
|
||||
|
||||
size_t GetNumScores() const
|
||||
{ return m_ffStartInd; }
|
||||
size_t GetNumScores() const
|
||||
{
|
||||
return m_ffStartInd;
|
||||
}
|
||||
|
||||
void Create();
|
||||
void Load();
|
||||
void Create();
|
||||
void Load();
|
||||
|
||||
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
|
||||
const PhraseTable *GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd);
|
||||
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
|
||||
const PhraseTable *GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd);
|
||||
|
||||
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
|
||||
void EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, TargetPhrase &targetPhrase) const;
|
||||
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const;
|
||||
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
|
||||
void EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, TargetPhrase &targetPhrase) const;
|
||||
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
|
||||
const Phrase &sourcePhrase) const;
|
||||
|
||||
/*
|
||||
void EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const;
|
||||
*/
|
||||
/*
|
||||
void EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const;
|
||||
*/
|
||||
protected:
|
||||
std::vector<const FeatureFunction*> m_featureFunctions;
|
||||
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
|
||||
std::vector<const FeatureFunction*> m_withPhraseTableInd;
|
||||
boost::unordered_map<std::string, size_t> m_defaultNames;
|
||||
System &m_system;
|
||||
size_t m_ffStartInd;
|
||||
std::vector<const FeatureFunction*> m_featureFunctions;
|
||||
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
|
||||
std::vector<const FeatureFunction*> m_withPhraseTableInd;
|
||||
boost::unordered_map<std::string, size_t> m_defaultNames;
|
||||
System &m_system;
|
||||
size_t m_ffStartInd;
|
||||
|
||||
FeatureFunction *Create(const std::string &line);
|
||||
std::string GetDefaultName(const std::string &stub);
|
||||
FeatureFunction *Create(const std::string &line);
|
||||
std::string GetDefaultName(const std::string &stub);
|
||||
|
||||
FeatureRegistry m_registry;
|
||||
FeatureRegistry m_registry;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -19,20 +19,22 @@ using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
template <class F>
|
||||
class DefaultFeatureFactory : public FeatureFactory
|
||||
template<class F>
|
||||
class DefaultFeatureFactory: public FeatureFactory
|
||||
{
|
||||
public:
|
||||
FeatureFunction *Create(size_t startInd, const std::string &line) {
|
||||
return new F(startInd, line);
|
||||
}
|
||||
FeatureFunction *Create(size_t startInd, const std::string &line)
|
||||
{
|
||||
return new F(startInd, line);
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
class KenFactory : public FeatureFactory
|
||||
class KenFactory: public FeatureFactory
|
||||
{
|
||||
public:
|
||||
FeatureFunction *Create(size_t startInd, const std::string &line) {
|
||||
FeatureFunction *Create(size_t startInd, const std::string &line)
|
||||
{
|
||||
ConstructKenLM(startInd, line);
|
||||
}
|
||||
};
|
||||
@ -41,9 +43,9 @@ public:
|
||||
FeatureRegistry::FeatureRegistry()
|
||||
{
|
||||
// Feature with same name as class
|
||||
#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >());
|
||||
#define MOSES_FNAME(name) Add(#name, new DefaultFeatureFactory< name >());
|
||||
// Feature with different name than class.
|
||||
#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >());
|
||||
#define MOSES_FNAME2(name, type) Add(name, new DefaultFeatureFactory< type >());
|
||||
|
||||
MOSES_FNAME2("PhraseDictionaryMemory", PhraseTableMemory);
|
||||
MOSES_FNAME(ProbingPT);
|
||||
@ -69,19 +71,21 @@ FeatureRegistry::~FeatureRegistry()
|
||||
|
||||
void FeatureRegistry::Add(const std::string &name, FeatureFactory *factory)
|
||||
{
|
||||
std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name, boost::shared_ptr<FeatureFactory>(factory));
|
||||
std::pair<std::string, boost::shared_ptr<FeatureFactory> > to_ins(name,
|
||||
boost::shared_ptr<FeatureFactory>(factory));
|
||||
if (!registry_.insert(to_ins).second) {
|
||||
cerr << "Duplicate feature name " << name << endl;
|
||||
abort();
|
||||
cerr << "Duplicate feature name " << name << endl;
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
FeatureFunction *FeatureRegistry::Construct(size_t startInd, const std::string &name, const std::string &line)
|
||||
FeatureFunction *FeatureRegistry::Construct(size_t startInd,
|
||||
const std::string &name, const std::string &line)
|
||||
{
|
||||
Map::iterator i = registry_.find(name);
|
||||
if (i == registry_.end()) {
|
||||
cerr << "Feature name " << name << " is not registered.";
|
||||
abort();
|
||||
cerr << "Feature name " << name << " is not registered.";
|
||||
abort();
|
||||
}
|
||||
FeatureFactory *fact = i->second.get();
|
||||
FeatureFunction *ff = fact->Create(startInd, line);
|
||||
|
@ -10,12 +10,16 @@ class FeatureFunction;
|
||||
class FeatureFactory
|
||||
{
|
||||
public:
|
||||
virtual ~FeatureFactory() {}
|
||||
virtual ~FeatureFactory()
|
||||
{
|
||||
}
|
||||
|
||||
virtual FeatureFunction *Create(size_t startInd, const std::string &line) = 0;
|
||||
|
||||
protected:
|
||||
FeatureFactory() {}
|
||||
FeatureFactory()
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
@ -26,7 +30,8 @@ public:
|
||||
|
||||
~FeatureRegistry();
|
||||
|
||||
FeatureFunction *Construct(size_t startInd, const std::string &name, const std::string &line);
|
||||
FeatureFunction *Construct(size_t startInd, const std::string &name,
|
||||
const std::string &line);
|
||||
void PrintFF() const;
|
||||
|
||||
private:
|
||||
@ -39,6 +44,5 @@ private:
|
||||
|
||||
////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -11,43 +11,37 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
BidirectionalReorderingState::BidirectionalReorderingState(
|
||||
const LRModel &config,
|
||||
LRState *bw,
|
||||
LRState *fw,
|
||||
size_t offset)
|
||||
: LRState(config,
|
||||
LRModel::Bidirectional,
|
||||
offset)
|
||||
, m_backward(bw)
|
||||
, m_forward(fw)
|
||||
{}
|
||||
const LRModel &config, LRState *bw, LRState *fw, size_t offset) :
|
||||
LRState(config, LRModel::Bidirectional, offset), m_backward(bw), m_forward(
|
||||
fw)
|
||||
{
|
||||
}
|
||||
|
||||
BidirectionalReorderingState::~BidirectionalReorderingState() {
|
||||
// TODO Auto-generated destructor stub
|
||||
BidirectionalReorderingState::~BidirectionalReorderingState()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void BidirectionalReorderingState::Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage)
|
||||
const TargetPhrase &topt, const InputPathBase &path, bool first,
|
||||
const Bitmap *coverage)
|
||||
{
|
||||
if (m_backward) {
|
||||
m_backward->Init(prev, topt, path, first, coverage);
|
||||
}
|
||||
if (m_forward) {
|
||||
m_forward->Init(prev, topt, path, first, coverage);
|
||||
}
|
||||
if (m_backward) {
|
||||
m_backward->Init(prev, topt, path, first, coverage);
|
||||
}
|
||||
if (m_forward) {
|
||||
m_forward->Init(prev, topt, path, first, coverage);
|
||||
}
|
||||
}
|
||||
|
||||
std::string BidirectionalReorderingState::ToString() const
|
||||
{ return "BidirectionalReorderingState "
|
||||
+ SPrint(this) + " "
|
||||
+ SPrint(m_backward) + " "
|
||||
+ SPrint(m_forward);
|
||||
{
|
||||
return "BidirectionalReorderingState " + SPrint(this) + " "
|
||||
+ SPrint(m_backward) + " " + SPrint(m_forward);
|
||||
}
|
||||
|
||||
size_t BidirectionalReorderingState::hash() const
|
||||
@ -62,23 +56,24 @@ bool BidirectionalReorderingState::operator==(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return 0;
|
||||
|
||||
BidirectionalReorderingState const &other
|
||||
= static_cast<BidirectionalReorderingState const&>(o);
|
||||
BidirectionalReorderingState const &other =
|
||||
static_cast<BidirectionalReorderingState const&>(o);
|
||||
|
||||
bool ret = (*m_backward == *other.m_backward) && (*m_forward == *other.m_forward);
|
||||
bool ret = (*m_backward == *other.m_backward)
|
||||
&& (*m_forward == *other.m_forward);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void BidirectionalReorderingState::Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
|
||||
Scores &scores, FFState &state) const
|
||||
{
|
||||
BidirectionalReorderingState &stateCast = static_cast<BidirectionalReorderingState&>(state);
|
||||
m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores, *stateCast.m_backward);
|
||||
m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores, *stateCast.m_forward);
|
||||
BidirectionalReorderingState &stateCast =
|
||||
static_cast<BidirectionalReorderingState&>(state);
|
||||
m_backward->Expand(mgr, ff, hypo, phraseTableInd, scores,
|
||||
*stateCast.m_backward);
|
||||
m_forward->Expand(mgr, ff, hypo, phraseTableInd, scores,
|
||||
*stateCast.m_forward);
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -7,36 +7,28 @@
|
||||
#pragma once
|
||||
#include "LRState.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class BidirectionalReorderingState: public LRState
|
||||
{
|
||||
public:
|
||||
BidirectionalReorderingState(
|
||||
const LRModel &config,
|
||||
LRState *bw,
|
||||
LRState *fw,
|
||||
size_t offset);
|
||||
BidirectionalReorderingState(const LRModel &config, LRState *bw, LRState *fw,
|
||||
size_t offset);
|
||||
|
||||
virtual ~BidirectionalReorderingState();
|
||||
|
||||
void Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage);
|
||||
void Init(const LRState *prev, const TargetPhrase &topt,
|
||||
const InputPathBase &path, bool first, const Bitmap *coverage);
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
virtual std::string ToString() const;
|
||||
|
||||
void Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
|
||||
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
LRState *m_backward;
|
||||
|
@ -9,27 +9,24 @@
|
||||
#include "../../PhraseBased/Hypothesis.h"
|
||||
#include "../../PhraseBased/Manager.h"
|
||||
|
||||
namespace Moses2 {
|
||||
|
||||
HReorderingBackwardState::HReorderingBackwardState(
|
||||
MemPool &pool,
|
||||
const LRModel &config,
|
||||
size_t offset)
|
||||
:LRState(config, LRModel::Backward, offset)
|
||||
,reoStack(pool)
|
||||
namespace Moses2
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
HReorderingBackwardState::HReorderingBackwardState(MemPool &pool,
|
||||
const LRModel &config, size_t offset) :
|
||||
LRState(config, LRModel::Backward, offset), reoStack(pool)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
HReorderingBackwardState::~HReorderingBackwardState() {
|
||||
// TODO Auto-generated destructor stub
|
||||
HReorderingBackwardState::~HReorderingBackwardState()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void HReorderingBackwardState::Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const TargetPhrase &topt, const InputPathBase &path, bool first,
|
||||
const Bitmap *coverage)
|
||||
{
|
||||
prevTP = &topt;
|
||||
@ -44,26 +41,25 @@ size_t HReorderingBackwardState::hash() const
|
||||
|
||||
bool HReorderingBackwardState::operator==(const FFState& o) const
|
||||
{
|
||||
const HReorderingBackwardState& other
|
||||
= static_cast<const HReorderingBackwardState&>(o);
|
||||
const HReorderingBackwardState& other =
|
||||
static_cast<const HReorderingBackwardState&>(o);
|
||||
bool ret = reoStack == other.reoStack;
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::string HReorderingBackwardState::ToString() const
|
||||
{
|
||||
return "HReorderingBackwardState " + SPrint(m_offset);
|
||||
return "HReorderingBackwardState " + SPrint(m_offset);
|
||||
}
|
||||
|
||||
void HReorderingBackwardState::Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
|
||||
Scores &scores, FFState &state) const
|
||||
{
|
||||
HReorderingBackwardState &nextState = static_cast<HReorderingBackwardState&>(state);
|
||||
nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, NULL);
|
||||
HReorderingBackwardState &nextState =
|
||||
static_cast<HReorderingBackwardState&>(state);
|
||||
nextState.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
|
||||
NULL);
|
||||
nextState.reoStack = reoStack;
|
||||
|
||||
const Range &swrange = hypo.GetInputPath().range;
|
||||
|
@ -8,36 +8,28 @@
|
||||
#include "LRState.h"
|
||||
#include "ReorderingStack.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class HReorderingBackwardState : public LRState
|
||||
class HReorderingBackwardState: public LRState
|
||||
{
|
||||
private:
|
||||
ReorderingStack reoStack;
|
||||
|
||||
public:
|
||||
HReorderingBackwardState(
|
||||
MemPool &pool,
|
||||
const LRModel &config,
|
||||
size_t offset);
|
||||
HReorderingBackwardState(MemPool &pool, const LRModel &config, size_t offset);
|
||||
|
||||
virtual void Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage);
|
||||
virtual void Init(const LRState *prev, const TargetPhrase &topt,
|
||||
const InputPathBase &path, bool first, const Bitmap *coverage);
|
||||
|
||||
virtual ~HReorderingBackwardState();
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
virtual std::string ToString() const;
|
||||
void Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
|
||||
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -10,27 +10,24 @@
|
||||
#include "../../PhraseBased/Manager.h"
|
||||
#include "../../PhraseBased/Hypothesis.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
HReorderingForwardState::HReorderingForwardState(
|
||||
const LRModel &config,
|
||||
size_t offset)
|
||||
: LRState(config, LRModel::Forward, offset)
|
||||
, m_first(true)
|
||||
HReorderingForwardState::HReorderingForwardState(const LRModel &config,
|
||||
size_t offset) :
|
||||
LRState(config, LRModel::Forward, offset), m_first(true)
|
||||
{
|
||||
prevPath = NULL;
|
||||
m_coverage = NULL;
|
||||
}
|
||||
|
||||
HReorderingForwardState::~HReorderingForwardState() {
|
||||
// TODO Auto-generated destructor stub
|
||||
HReorderingForwardState::~HReorderingForwardState()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void HReorderingForwardState::Init(
|
||||
const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
void HReorderingForwardState::Init(const LRState *prev,
|
||||
const TargetPhrase &topt, const InputPathBase &path, bool first,
|
||||
const Bitmap *coverage)
|
||||
{
|
||||
prevTP = &topt;
|
||||
@ -50,26 +47,24 @@ bool HReorderingForwardState::operator==(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return true;
|
||||
|
||||
HReorderingForwardState const& other
|
||||
= static_cast<HReorderingForwardState const&>(o);
|
||||
HReorderingForwardState const& other =
|
||||
static_cast<HReorderingForwardState const&>(o);
|
||||
|
||||
int compareScores = ((prevPath->range == other.prevPath->range)
|
||||
? ComparePrevScores(other.prevTP)
|
||||
: (prevPath->range < other.prevPath->range) ? -1 : 1);
|
||||
int compareScores = (
|
||||
(prevPath->range == other.prevPath->range) ?
|
||||
ComparePrevScores(other.prevTP) :
|
||||
(prevPath->range < other.prevPath->range) ? -1 : 1);
|
||||
return compareScores == 0;
|
||||
}
|
||||
|
||||
std::string HReorderingForwardState::ToString() const
|
||||
{
|
||||
return "HReorderingForwardState " + SPrint(m_offset);
|
||||
return "HReorderingForwardState " + SPrint(m_offset);
|
||||
}
|
||||
|
||||
void HReorderingForwardState::Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
|
||||
Scores &scores, FFState &state) const
|
||||
{
|
||||
const Range &cur = hypo.GetInputPath().range;
|
||||
// keep track of the current coverage ourselves so we don't need the hypothesis
|
||||
@ -83,8 +78,10 @@ void HReorderingForwardState::Expand(const ManagerBase &mgr,
|
||||
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
|
||||
}
|
||||
|
||||
HReorderingForwardState &stateCast = static_cast<HReorderingForwardState&>(state);
|
||||
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, &cov);
|
||||
HReorderingForwardState &stateCast =
|
||||
static_cast<HReorderingForwardState&>(state);
|
||||
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
|
||||
&cov);
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -7,35 +7,27 @@
|
||||
#pragma once
|
||||
#include "LRState.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
class Range;
|
||||
class Bitmap;
|
||||
class InputPathBase;
|
||||
|
||||
class HReorderingForwardState : public LRState
|
||||
class HReorderingForwardState: public LRState
|
||||
{
|
||||
public:
|
||||
HReorderingForwardState(
|
||||
const LRModel &config,
|
||||
size_t offset);
|
||||
HReorderingForwardState(const LRModel &config, size_t offset);
|
||||
virtual ~HReorderingForwardState();
|
||||
|
||||
void Init(
|
||||
const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage);
|
||||
void Init(const LRState *prev, const TargetPhrase &topt,
|
||||
const InputPathBase &path, bool first, const Bitmap *coverage);
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
virtual std::string ToString() const;
|
||||
void Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
|
||||
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
bool m_first;
|
||||
|
@ -18,144 +18,138 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
bool
|
||||
IsMonotonicStep(Range const& prev, // words range of last source phrase
|
||||
Range const& cur, // words range of current source phrase
|
||||
Bitmap const& cov) // coverage bitmap
|
||||
bool IsMonotonicStep(Range const& prev, // words range of last source phrase
|
||||
Range const& cur, // words range of current source phrase
|
||||
Bitmap const& cov) // coverage bitmap
|
||||
{
|
||||
size_t e = prev.GetEndPos() + 1;
|
||||
size_t s = cur.GetStartPos();
|
||||
return (s == e || (s >= e && !cov.GetValue(e)));
|
||||
}
|
||||
|
||||
bool
|
||||
IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
|
||||
bool IsSwap(Range const& prev, Range const& cur, Bitmap const& cov)
|
||||
{
|
||||
size_t s = prev.GetStartPos();
|
||||
size_t e = cur.GetEndPos();
|
||||
return (e+1 == s || (e < s && !cov.GetValue(s-1)));
|
||||
return (e + 1 == s || (e < s && !cov.GetValue(s - 1)));
|
||||
}
|
||||
|
||||
LRModel::LRModel(const std::string &modelType, LexicalReordering &ff)
|
||||
: m_modelType(None)
|
||||
, m_phraseBased(true)
|
||||
, m_collapseScores(false)
|
||||
, m_direction(Backward)
|
||||
, m_scoreProducer(&ff)
|
||||
LRModel::LRModel(const std::string &modelType, LexicalReordering &ff) :
|
||||
m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(
|
||||
Backward), m_scoreProducer(&ff)
|
||||
{
|
||||
std::vector<std::string> config = Tokenize(modelType, "-");
|
||||
|
||||
for (size_t i=0; i<config.size(); ++i) {
|
||||
if (config[i] == "hier") {
|
||||
m_phraseBased = false;
|
||||
} else if (config[i] == "phrase") {
|
||||
m_phraseBased = true;
|
||||
} else if (config[i] == "wbe") {
|
||||
m_phraseBased = true;
|
||||
}
|
||||
// no word-based decoding available, fall-back to phrase-based
|
||||
// This is the old lexical reordering model combination of moses
|
||||
for (size_t i = 0; i < config.size(); ++i) {
|
||||
if (config[i] == "hier") {
|
||||
m_phraseBased = false;
|
||||
}
|
||||
else if (config[i] == "phrase") {
|
||||
m_phraseBased = true;
|
||||
}
|
||||
else if (config[i] == "wbe") {
|
||||
m_phraseBased = true;
|
||||
}
|
||||
// no word-based decoding available, fall-back to phrase-based
|
||||
// This is the old lexical reordering model combination of moses
|
||||
|
||||
else if (config[i] == "msd") {
|
||||
m_modelType = MSD;
|
||||
} else if (config[i] == "mslr") {
|
||||
m_modelType = MSLR;
|
||||
} else if (config[i] == "monotonicity") {
|
||||
m_modelType = Monotonic;
|
||||
} else if (config[i] == "leftright") {
|
||||
m_modelType = LeftRight;
|
||||
}
|
||||
else if (config[i] == "msd") {
|
||||
m_modelType = MSD;
|
||||
}
|
||||
else if (config[i] == "mslr") {
|
||||
m_modelType = MSLR;
|
||||
}
|
||||
else if (config[i] == "monotonicity") {
|
||||
m_modelType = Monotonic;
|
||||
}
|
||||
else if (config[i] == "leftright") {
|
||||
m_modelType = LeftRight;
|
||||
}
|
||||
|
||||
// unidirectional is deprecated, use backward instead
|
||||
else if (config[i] == "unidirectional") {
|
||||
m_direction = Backward;
|
||||
} else if (config[i] == "backward") {
|
||||
m_direction = Backward;
|
||||
} else if (config[i] == "forward") {
|
||||
m_direction = Forward;
|
||||
} else if (config[i] == "bidirectional") {
|
||||
m_direction = Bidirectional;
|
||||
}
|
||||
// unidirectional is deprecated, use backward instead
|
||||
else if (config[i] == "unidirectional") {
|
||||
m_direction = Backward;
|
||||
}
|
||||
else if (config[i] == "backward") {
|
||||
m_direction = Backward;
|
||||
}
|
||||
else if (config[i] == "forward") {
|
||||
m_direction = Forward;
|
||||
}
|
||||
else if (config[i] == "bidirectional") {
|
||||
m_direction = Bidirectional;
|
||||
}
|
||||
|
||||
else if (config[i] == "f") {
|
||||
m_condition = F;
|
||||
} else if (config[i] == "fe") {
|
||||
m_condition = FE;
|
||||
}
|
||||
else if (config[i] == "f") {
|
||||
m_condition = F;
|
||||
}
|
||||
else if (config[i] == "fe") {
|
||||
m_condition = FE;
|
||||
}
|
||||
|
||||
else if (config[i] == "collapseff") {
|
||||
m_collapseScores = true;
|
||||
} else if (config[i] == "allff") {
|
||||
m_collapseScores = false;
|
||||
} else {
|
||||
std::cerr
|
||||
<< "Illegal part in the lexical reordering configuration string: "
|
||||
<< config[i] << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
else if (config[i] == "collapseff") {
|
||||
m_collapseScores = true;
|
||||
}
|
||||
else if (config[i] == "allff") {
|
||||
m_collapseScores = false;
|
||||
}
|
||||
else {
|
||||
std::cerr
|
||||
<< "Illegal part in the lexical reordering configuration string: "
|
||||
<< config[i] << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (m_modelType == None) {
|
||||
std::cerr
|
||||
<< "You need to specify the type of the reordering model "
|
||||
<< "(msd, monotonicity,...)" << std::endl;
|
||||
exit(1);
|
||||
std::cerr << "You need to specify the type of the reordering model "
|
||||
<< "(msd, monotonicity,...)" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
LRModel::~LRModel() {
|
||||
// TODO Auto-generated destructor stub
|
||||
LRModel::~LRModel()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
size_t
|
||||
LRModel::
|
||||
GetNumberOfTypes() const
|
||||
size_t LRModel::GetNumberOfTypes() const
|
||||
{
|
||||
return ((m_modelType == MSD) ? 3 :
|
||||
(m_modelType == MSLR) ? 4 : 2);
|
||||
return ((m_modelType == MSD) ? 3 : (m_modelType == MSLR) ? 4 : 2);
|
||||
}
|
||||
|
||||
/// return orientation for the first phrase
|
||||
LRModel::ReorderingType
|
||||
LRModel::
|
||||
GetOrientation(Range const& cur) const
|
||||
LRModel::ReorderingType LRModel::GetOrientation(Range const& cur) const
|
||||
{
|
||||
UTIL_THROW_IF2(m_modelType == None, "Reordering Model Type is None");
|
||||
return ((m_modelType == LeftRight) ? R :
|
||||
(cur.GetStartPos() == 0) ? M :
|
||||
(m_modelType == MSD) ? D :
|
||||
(m_modelType == MSLR) ? DR : NM);
|
||||
return ((m_modelType == LeftRight) ? R : (cur.GetStartPos() == 0) ? M :
|
||||
(m_modelType == MSD) ? D : (m_modelType == MSLR) ? DR : NM);
|
||||
}
|
||||
|
||||
LRModel::ReorderingType
|
||||
LRModel::
|
||||
GetOrientation(Range const& prev, Range const& cur) const
|
||||
LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
|
||||
Range const& cur) const
|
||||
{
|
||||
UTIL_THROW_IF2(m_modelType == None, "No reordering model type specified");
|
||||
return ((m_modelType == LeftRight)
|
||||
? prev.GetEndPos() <= cur.GetStartPos() ? R : L
|
||||
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M
|
||||
: (m_modelType == Monotonic) ? NM
|
||||
: (prev.GetStartPos() == cur.GetEndPos() + 1) ? S
|
||||
: (m_modelType == MSD) ? D
|
||||
: (cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
|
||||
return (
|
||||
(m_modelType == LeftRight) ? prev.GetEndPos() <= cur.GetStartPos() ? R : L
|
||||
: (cur.GetStartPos() == prev.GetEndPos() + 1) ? M :
|
||||
(m_modelType == Monotonic) ? NM :
|
||||
(prev.GetStartPos() == cur.GetEndPos() + 1) ? S :
|
||||
(m_modelType == MSD) ? D :
|
||||
(cur.GetStartPos() > prev.GetEndPos()) ? DR : DL);
|
||||
}
|
||||
|
||||
LRModel::ReorderingType
|
||||
LRModel::
|
||||
GetOrientation(int const reoDistance) const
|
||||
LRModel::ReorderingType LRModel::GetOrientation(int const reoDistance) const
|
||||
{
|
||||
// this one is for HierarchicalReorderingBackwardState
|
||||
return ((m_modelType == LeftRight)
|
||||
? (reoDistance >= 1) ? R : L
|
||||
: (reoDistance == 1) ? M
|
||||
: (m_modelType == Monotonic) ? NM
|
||||
: (reoDistance == -1) ? S
|
||||
: (m_modelType == MSD) ? D
|
||||
: (reoDistance > 1) ? DR : DL);
|
||||
return ((m_modelType == LeftRight) ? (reoDistance >= 1) ? R : L
|
||||
: (reoDistance == 1) ? M : (m_modelType == Monotonic) ? NM :
|
||||
(reoDistance == -1) ? S : (m_modelType == MSD) ? D :
|
||||
(reoDistance > 1) ? DR : DL);
|
||||
}
|
||||
|
||||
LRState *LRModel::CreateLRState(MemPool &pool) const
|
||||
@ -163,47 +157,53 @@ LRState *LRModel::CreateLRState(MemPool &pool) const
|
||||
LRState *bwd = NULL, *fwd = NULL;
|
||||
size_t offset = 0;
|
||||
|
||||
switch(m_direction) {
|
||||
switch (m_direction) {
|
||||
case Backward:
|
||||
case Bidirectional:
|
||||
if (m_phraseBased) {
|
||||
bwd = new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(*this, Backward, offset);
|
||||
bwd =
|
||||
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
|
||||
*this, Backward, offset);
|
||||
//cerr << "bwd=" << bwd << bwd->ToString() << endl;
|
||||
}
|
||||
else {
|
||||
bwd = new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(pool, *this, offset);
|
||||
bwd =
|
||||
new (pool.Allocate<HReorderingBackwardState>()) HReorderingBackwardState(
|
||||
pool, *this, offset);
|
||||
}
|
||||
offset += m_collapseScores ? 1 : GetNumberOfTypes();
|
||||
if (m_direction == Backward) return bwd; // else fall through
|
||||
case Forward:
|
||||
if (m_phraseBased) {
|
||||
fwd = new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(*this, Forward, offset);
|
||||
fwd =
|
||||
new (pool.Allocate<PhraseBasedReorderingState>()) PhraseBasedReorderingState(
|
||||
*this, Forward, offset);
|
||||
//cerr << "fwd=" << fwd << fwd->ToString() << endl;
|
||||
}
|
||||
else {
|
||||
fwd = new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(*this, offset);
|
||||
fwd =
|
||||
new (pool.Allocate<HReorderingForwardState>()) HReorderingForwardState(
|
||||
*this, offset);
|
||||
}
|
||||
offset += m_collapseScores ? 1 : GetNumberOfTypes();
|
||||
if (m_direction == Forward) return fwd;
|
||||
}
|
||||
|
||||
//cerr << "LRStates:" << *bwd << endl << *fwd << endl;
|
||||
BidirectionalReorderingState *ret = new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(*this, bwd, fwd, 0);
|
||||
BidirectionalReorderingState *ret =
|
||||
new (pool.Allocate<BidirectionalReorderingState>()) BidirectionalReorderingState(
|
||||
*this, bwd, fwd, 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
LRModel::ReorderingType
|
||||
LRModel::
|
||||
GetOrientation(Range const& prev, Range const& cur,
|
||||
Bitmap const& cov) const
|
||||
LRModel::ReorderingType LRModel::GetOrientation(Range const& prev,
|
||||
Range const& cur, Bitmap const& cov) const
|
||||
{
|
||||
return ((m_modelType == LeftRight)
|
||||
? cur.GetStartPos() > prev.GetEndPos() ? R : L
|
||||
: IsMonotonicStep(prev,cur,cov) ? M
|
||||
: (m_modelType == Monotonic) ? NM
|
||||
: IsSwap(prev,cur,cov) ? S
|
||||
: (m_modelType == MSD) ? D
|
||||
: cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
|
||||
return (
|
||||
(m_modelType == LeftRight) ? cur.GetStartPos() > prev.GetEndPos() ? R : L
|
||||
: IsMonotonicStep(prev, cur, cov) ? M : (m_modelType == Monotonic) ? NM :
|
||||
IsSwap(prev, cur, cov) ? S : (m_modelType == MSD) ? D :
|
||||
cur.GetStartPos() > prev.GetEndPos() ? DR : DL);
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -7,7 +7,8 @@
|
||||
#pragma once
|
||||
#include <string>
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class MemPool;
|
||||
class Range;
|
||||
@ -15,69 +16,83 @@ class Bitmap;
|
||||
class LRState;
|
||||
class LexicalReordering;
|
||||
|
||||
class LRModel {
|
||||
class LRModel
|
||||
{
|
||||
public:
|
||||
enum ModelType { Monotonic, MSD, MSLR, LeftRight, None };
|
||||
enum Direction { Forward, Backward, Bidirectional };
|
||||
enum Condition { F, E, FE };
|
||||
|
||||
enum ReorderingType {
|
||||
M = 0, // monotonic
|
||||
NM = 1, // non-monotonic
|
||||
S = 1, // swap
|
||||
D = 2, // discontinuous
|
||||
DL = 2, // discontinuous, left
|
||||
DR = 3, // discontinuous, right
|
||||
R = 0, // right
|
||||
L = 1, // left
|
||||
MAX = 3, // largest possible
|
||||
NONE = 4 // largest possible
|
||||
enum ModelType
|
||||
{
|
||||
Monotonic, MSD, MSLR, LeftRight, None
|
||||
};
|
||||
enum Direction
|
||||
{
|
||||
Forward, Backward, Bidirectional
|
||||
};
|
||||
enum Condition
|
||||
{
|
||||
F, E, FE
|
||||
};
|
||||
|
||||
LRModel(const std::string &modelType, LexicalReordering &ff);
|
||||
virtual ~LRModel();
|
||||
enum ReorderingType
|
||||
{
|
||||
M = 0, // monotonic
|
||||
NM = 1, // non-monotonic
|
||||
S = 1, // swap
|
||||
D = 2, // discontinuous
|
||||
DL = 2, // discontinuous, left
|
||||
DR = 3, // discontinuous, right
|
||||
R = 0, // right
|
||||
L = 1, // left
|
||||
MAX = 3, // largest possible
|
||||
NONE = 4 // largest possible
|
||||
};
|
||||
|
||||
ModelType GetModelType() const {
|
||||
return m_modelType;
|
||||
}
|
||||
Direction GetDirection() const {
|
||||
return m_direction;
|
||||
}
|
||||
Condition GetCondition() const {
|
||||
return m_condition;
|
||||
}
|
||||
LRModel(const std::string &modelType, LexicalReordering &ff);
|
||||
virtual ~LRModel();
|
||||
|
||||
bool
|
||||
IsPhraseBased() const {
|
||||
return m_phraseBased;
|
||||
}
|
||||
ModelType GetModelType() const
|
||||
{
|
||||
return m_modelType;
|
||||
}
|
||||
Direction GetDirection() const
|
||||
{
|
||||
return m_direction;
|
||||
}
|
||||
Condition GetCondition() const
|
||||
{
|
||||
return m_condition;
|
||||
}
|
||||
|
||||
bool
|
||||
CollapseScores() const {
|
||||
return m_collapseScores;
|
||||
}
|
||||
bool IsPhraseBased() const
|
||||
{
|
||||
return m_phraseBased;
|
||||
}
|
||||
|
||||
size_t GetNumberOfTypes() const;
|
||||
bool CollapseScores() const
|
||||
{
|
||||
return m_collapseScores;
|
||||
}
|
||||
|
||||
LexicalReordering*
|
||||
GetScoreProducer() const {
|
||||
return m_scoreProducer;
|
||||
}
|
||||
size_t GetNumberOfTypes() const;
|
||||
|
||||
LRState *CreateLRState(MemPool &pool) const;
|
||||
LexicalReordering*
|
||||
GetScoreProducer() const
|
||||
{
|
||||
return m_scoreProducer;
|
||||
}
|
||||
|
||||
ReorderingType // for first phrase in phrase-based
|
||||
GetOrientation(Range const& cur) const;
|
||||
LRState *CreateLRState(MemPool &pool) const;
|
||||
|
||||
ReorderingType // for non-first phrases in phrase-based
|
||||
GetOrientation(Range const& prev, Range const& cur) const;
|
||||
ReorderingType // for first phrase in phrase-based
|
||||
GetOrientation(Range const& cur) const;
|
||||
|
||||
ReorderingType // for HReorderingForwardState
|
||||
GetOrientation(Range const& prev, Range const& cur,
|
||||
Bitmap const& cov) const;
|
||||
ReorderingType // for non-first phrases in phrase-based
|
||||
GetOrientation(Range const& prev, Range const& cur) const;
|
||||
|
||||
ReorderingType // for HReorderingBackwarddState
|
||||
GetOrientation(int const reoDistance) const;
|
||||
ReorderingType // for HReorderingForwardState
|
||||
GetOrientation(Range const& prev, Range const& cur, Bitmap const& cov) const;
|
||||
|
||||
ReorderingType // for HReorderingBackwarddState
|
||||
GetOrientation(int const reoDistance) const;
|
||||
|
||||
protected:
|
||||
|
||||
|
@ -10,54 +10,47 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class InputType;
|
||||
|
||||
LRState::LRState(const LRModel &config,
|
||||
LRModel::Direction dir,
|
||||
size_t offset)
|
||||
:m_configuration(config)
|
||||
,m_direction(dir)
|
||||
,m_offset(offset)
|
||||
LRState::LRState(const LRModel &config, LRModel::Direction dir, size_t offset) :
|
||||
m_configuration(config), m_direction(dir), m_offset(offset)
|
||||
{
|
||||
}
|
||||
|
||||
int
|
||||
LRState::
|
||||
ComparePrevScores(const TargetPhrase *other) const
|
||||
int LRState::ComparePrevScores(const TargetPhrase *other) const
|
||||
{
|
||||
LexicalReordering* producer = m_configuration.GetScoreProducer();
|
||||
size_t phraseTableInd = producer->GetPhraseTableInd();
|
||||
const SCORE *myScores = (const SCORE*) prevTP->ffData[phraseTableInd]; //producer->
|
||||
const SCORE *yrScores = (const SCORE*) other->ffData[phraseTableInd]; //producer->
|
||||
|
||||
if(myScores == yrScores) return 0;
|
||||
if (myScores == yrScores) return 0;
|
||||
|
||||
// The pointers are NULL if a phrase pair isn't found in the reordering table.
|
||||
if(yrScores == NULL) return -1;
|
||||
if(myScores == NULL) return 1;
|
||||
if (yrScores == NULL) return -1;
|
||||
if (myScores == NULL) return 1;
|
||||
|
||||
size_t stop = m_offset + m_configuration.GetNumberOfTypes();
|
||||
for(size_t i = m_offset; i < stop; i++) {
|
||||
if((myScores)[i] < (yrScores)[i]) return -1;
|
||||
if((myScores)[i] > (yrScores)[i]) return 1;
|
||||
for (size_t i = m_offset; i < stop; i++) {
|
||||
if ((myScores)[i] < (yrScores)[i]) return -1;
|
||||
if ((myScores)[i] > (yrScores)[i]) return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void LRState::CopyScores(const System &system,
|
||||
Scores &accum,
|
||||
const TargetPhrase &topt,
|
||||
ReorderingType reoType) const
|
||||
void LRState::CopyScores(const System &system, Scores &accum,
|
||||
const TargetPhrase &topt, ReorderingType reoType) const
|
||||
{
|
||||
// don't call this on a bidirectional object
|
||||
UTIL_THROW_IF2(m_direction != LRModel::Backward &&
|
||||
m_direction != LRModel::Forward,
|
||||
"Unknown direction: " << m_direction);
|
||||
UTIL_THROW_IF2(
|
||||
m_direction != LRModel::Backward && m_direction != LRModel::Forward,
|
||||
"Unknown direction: " << m_direction);
|
||||
|
||||
TargetPhrase const* relevantOpt = ((m_direction == LRModel::Backward)
|
||||
? &topt : prevTP);
|
||||
TargetPhrase const* relevantOpt = (
|
||||
(m_direction == LRModel::Backward) ? &topt : prevTP);
|
||||
|
||||
LexicalReordering* producer = m_configuration.GetScoreProducer();
|
||||
size_t phraseTableInd = producer->GetPhraseTableInd();
|
||||
@ -68,32 +61,31 @@ void LRState::CopyScores(const System &system,
|
||||
}
|
||||
|
||||
size_t off_remote = m_offset + reoType;
|
||||
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
|
||||
size_t off_local = m_configuration.CollapseScores() ? m_offset : off_remote;
|
||||
|
||||
UTIL_THROW_IF2(off_local >= producer->GetNumScores(),
|
||||
"offset out of vector bounds!");
|
||||
"offset out of vector bounds!");
|
||||
|
||||
// look up applicable score from vector of scores
|
||||
//UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
|
||||
//Scores scores(producer->GetNumScoreComponents(),0);
|
||||
//UTIL_THROW_IF2(off_remote >= cached->size(), "offset out of vector bounds!");
|
||||
//Scores scores(producer->GetNumScoreComponents(),0);
|
||||
SCORE score = cached[off_remote];
|
||||
accum.PlusEquals(system, *producer, score, off_local);
|
||||
|
||||
// else: use default scores (if specified)
|
||||
/*
|
||||
else if (producer->GetHaveDefaultScores()) {
|
||||
Scores scores(producer->GetNumScoreComponents(),0);
|
||||
scores[off_local] = producer->GetDefaultScore(off_remote);
|
||||
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
|
||||
}
|
||||
*/
|
||||
else if (producer->GetHaveDefaultScores()) {
|
||||
Scores scores(producer->GetNumScoreComponents(),0);
|
||||
scores[off_local] = producer->GetDefaultScore(off_remote);
|
||||
accum->PlusEquals(m_configuration.GetScoreProducer(), scores);
|
||||
}
|
||||
*/
|
||||
// note: if no default score, no cost
|
||||
|
||||
/*
|
||||
const SparseReordering* sparse = m_configuration.GetSparseReordering();
|
||||
if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType,
|
||||
m_direction, accum);
|
||||
*/
|
||||
const SparseReordering* sparse = m_configuration.GetSparseReordering();
|
||||
if (sparse) sparse->CopyScores(*relevantOpt, m_prevOption, input, reoType,
|
||||
m_direction, accum);
|
||||
*/
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -2,7 +2,8 @@
|
||||
#include "../../legacy/FFState.h"
|
||||
#include "LRModel.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class LexicalReordering;
|
||||
class Hypothesis;
|
||||
@ -14,33 +15,23 @@ class TargetPhrase;
|
||||
class InputType;
|
||||
class InputPathBase;
|
||||
|
||||
class LRState : public FFState
|
||||
class LRState: public FFState
|
||||
{
|
||||
public:
|
||||
typedef LRModel::ReorderingType ReorderingType;
|
||||
const TargetPhrase *prevTP;
|
||||
|
||||
LRState(const LRModel &config,
|
||||
LRModel::Direction dir,
|
||||
size_t offset);
|
||||
LRState(const LRModel &config, LRModel::Direction dir, size_t offset);
|
||||
|
||||
virtual void Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage) = 0;
|
||||
virtual void Init(const LRState *prev, const TargetPhrase &topt,
|
||||
const InputPathBase &path, bool first, const Bitmap *coverage) = 0;
|
||||
|
||||
virtual void Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const = 0;
|
||||
virtual void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
|
||||
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
|
||||
FFState &state) const = 0;
|
||||
|
||||
void CopyScores(const System &system,
|
||||
Scores &accum,
|
||||
const TargetPhrase &topt,
|
||||
ReorderingType reoType) const;
|
||||
void CopyScores(const System &system, Scores &accum, const TargetPhrase &topt,
|
||||
ReorderingType reoType) const;
|
||||
|
||||
protected:
|
||||
const LRModel& m_configuration;
|
||||
|
@ -22,28 +22,25 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
|
||||
LexicalReordering::LexicalReordering(size_t startInd, const std::string &line)
|
||||
:StatefulFeatureFunction(startInd, line)
|
||||
,m_compactModel(NULL)
|
||||
,m_blank(NULL)
|
||||
,m_propertyInd(-1)
|
||||
,m_coll(NULL)
|
||||
,m_configuration(NULL)
|
||||
LexicalReordering::LexicalReordering(size_t startInd, const std::string &line) :
|
||||
StatefulFeatureFunction(startInd, line), m_compactModel(NULL), m_blank(
|
||||
NULL), m_propertyInd(-1), m_coll(NULL), m_configuration(NULL)
|
||||
{
|
||||
ReadParameters();
|
||||
assert(m_configuration);
|
||||
//assert(m_numScores == 6);
|
||||
ReadParameters();
|
||||
assert(m_configuration);
|
||||
//assert(m_numScores == 6);
|
||||
}
|
||||
|
||||
LexicalReordering::~LexicalReordering()
|
||||
{
|
||||
delete m_compactModel;
|
||||
delete m_coll;
|
||||
delete m_configuration;
|
||||
delete m_compactModel;
|
||||
delete m_coll;
|
||||
delete m_configuration;
|
||||
}
|
||||
|
||||
void LexicalReordering::Load(System &system)
|
||||
@ -51,57 +48,61 @@ void LexicalReordering::Load(System &system)
|
||||
MemPool &pool = system.GetSystemPool();
|
||||
|
||||
if (m_propertyInd >= 0) {
|
||||
// Using integrate Lex RO. No loading needed
|
||||
// Using integrate Lex RO. No loading needed
|
||||
}
|
||||
else if (FileExists(m_path + ".minlexr") ) {
|
||||
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr", m_FactorsF,
|
||||
m_FactorsE, m_FactorsC);
|
||||
m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
|
||||
else if (FileExists(m_path + ".minlexr")) {
|
||||
m_compactModel = new LexicalReorderingTableCompact(m_path + ".minlexr",
|
||||
m_FactorsF, m_FactorsE, m_FactorsC);
|
||||
m_blank = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, 0);
|
||||
}
|
||||
else {
|
||||
m_coll = new Coll();
|
||||
InputFileStream file(m_path);
|
||||
string line;
|
||||
size_t lineNum = 0;
|
||||
m_coll = new Coll();
|
||||
InputFileStream file(m_path);
|
||||
string line;
|
||||
size_t lineNum = 0;
|
||||
|
||||
while(getline(file, line)) {
|
||||
if (++lineNum % 1000000 == 0) {
|
||||
cerr << lineNum << " ";
|
||||
}
|
||||
while (getline(file, line)) {
|
||||
if (++lineNum % 1000000 == 0) {
|
||||
cerr << lineNum << " ";
|
||||
}
|
||||
|
||||
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
|
||||
assert(toks.size() == 3);
|
||||
PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(), system, toks[0]);
|
||||
PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(), system, toks[1]);
|
||||
std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(), TransformScore);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
|
||||
std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
|
||||
assert(toks.size() == 3);
|
||||
PhraseImpl *source = PhraseImpl::CreateFromString(pool, system.GetVocab(),
|
||||
system, toks[0]);
|
||||
PhraseImpl *target = PhraseImpl::CreateFromString(pool, system.GetVocab(),
|
||||
system, toks[1]);
|
||||
std::vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(),
|
||||
TransformScore);
|
||||
std::transform(scores.begin(), scores.end(), scores.begin(), FloorScore);
|
||||
|
||||
Key key(source, target);
|
||||
(*m_coll)[key] = scores;
|
||||
}
|
||||
Key key(source, target);
|
||||
(*m_coll)[key] = scores;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LexicalReordering::SetParameter(const std::string& key, const std::string& value)
|
||||
void LexicalReordering::SetParameter(const std::string& key,
|
||||
const std::string& value)
|
||||
{
|
||||
if (key == "path") {
|
||||
m_path = value;
|
||||
m_path = value;
|
||||
}
|
||||
else if (key == "type") {
|
||||
m_configuration = new LRModel(value, *this);
|
||||
m_configuration = new LRModel(value, *this);
|
||||
}
|
||||
else if (key == "input-factor") {
|
||||
m_FactorsF = Tokenize<FactorType>(value);
|
||||
m_FactorsF = Tokenize<FactorType>(value);
|
||||
}
|
||||
else if (key == "output-factor") {
|
||||
m_FactorsE = Tokenize<FactorType>(value);
|
||||
m_FactorsE = Tokenize<FactorType>(value);
|
||||
}
|
||||
else if (key == "property-index") {
|
||||
m_propertyInd = Scan<int>(value);
|
||||
m_propertyInd = Scan<int>(value);
|
||||
}
|
||||
else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
@ -112,35 +113,31 @@ FFState* LexicalReordering::BlankState(MemPool &pool) const
|
||||
}
|
||||
|
||||
void LexicalReordering::EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
const ManagerBase &mgr, const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
{
|
||||
BidirectionalReorderingState &stateCast = static_cast<BidirectionalReorderingState&>(state);
|
||||
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true, &hypo.GetBitmap());
|
||||
BidirectionalReorderingState &stateCast =
|
||||
static_cast<BidirectionalReorderingState&>(state);
|
||||
stateCast.Init(NULL, hypo.GetTargetPhrase(), hypo.GetInputPath(), true,
|
||||
&hypo.GetBitmap());
|
||||
}
|
||||
|
||||
void LexicalReordering::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void LexicalReordering::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
}
|
||||
|
||||
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrases &tps,
|
||||
const Phrase &sourcePhrase) const
|
||||
const TargetPhrases &tps, const Phrase &sourcePhrase) const
|
||||
{
|
||||
BOOST_FOREACH(const TargetPhrase *tp, tps) {
|
||||
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
|
||||
}
|
||||
BOOST_FOREACH(const TargetPhrase *tp, tps){
|
||||
EvaluateAfterTablePruning(pool, *tp, sourcePhrase);
|
||||
}
|
||||
}
|
||||
|
||||
void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrase &targetPhrase,
|
||||
const Phrase &sourcePhrase) const
|
||||
const TargetPhrase &targetPhrase, const Phrase &sourcePhrase) const
|
||||
{
|
||||
if (m_propertyInd >= 0) {
|
||||
SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd);
|
||||
@ -148,7 +145,8 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
}
|
||||
else if (m_compactModel) {
|
||||
// using external compact binary model
|
||||
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase, *m_blank);
|
||||
const Values values = m_compactModel->GetScore(sourcePhrase, targetPhrase,
|
||||
*m_blank);
|
||||
if (values.size()) {
|
||||
assert(values.size() == m_numScores);
|
||||
|
||||
@ -183,26 +181,25 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
|
||||
}
|
||||
|
||||
void LexicalReordering::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
const LRState &prevStateCast = static_cast<const LRState&>(prevState);
|
||||
prevStateCast.Expand(mgr, *this, hypo, m_PhraseTableInd, scores, state);
|
||||
}
|
||||
|
||||
const LexicalReordering::Values *LexicalReordering::GetValues(const Phrase &source, const Phrase &target) const
|
||||
const LexicalReordering::Values *LexicalReordering::GetValues(
|
||||
const Phrase &source, const Phrase &target) const
|
||||
{
|
||||
Key key(&source, &target);
|
||||
Coll::const_iterator iter;
|
||||
iter = m_coll->find(key);
|
||||
if (iter == m_coll->end()) {
|
||||
return NULL;
|
||||
}
|
||||
else {
|
||||
return &iter->second;
|
||||
}
|
||||
Key key(&source, &target);
|
||||
Coll::const_iterator iter;
|
||||
iter = m_coll->find(key);
|
||||
if (iter == m_coll->end()) {
|
||||
return NULL;
|
||||
}
|
||||
else {
|
||||
return &iter->second;
|
||||
}
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -13,12 +13,13 @@
|
||||
#include "../../Phrase.h"
|
||||
#include "../../legacy/Range.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class LexicalReorderingTableCompact;
|
||||
class LRModel;
|
||||
|
||||
class LexicalReordering : public StatefulFeatureFunction
|
||||
class LexicalReordering: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
LexicalReordering(size_t startInd, const std::string &line);
|
||||
@ -29,32 +30,26 @@ public:
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
virtual size_t HasPhraseTableInd() const
|
||||
{ return true; }
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void
|
||||
EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrases &tps,
|
||||
const Phrase &sourcePhrase) const;
|
||||
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps,
|
||||
const Phrase &sourcePhrase) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
std::string m_path;
|
||||
@ -65,9 +60,8 @@ protected:
|
||||
LRModel *m_configuration;
|
||||
|
||||
virtual void
|
||||
EvaluateAfterTablePruning(MemPool &pool,
|
||||
const TargetPhrase &targetPhrase,
|
||||
const Phrase &sourcePhrase) const;
|
||||
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrase &targetPhrase,
|
||||
const Phrase &sourcePhrase) const;
|
||||
|
||||
// PROPERTY IN PT
|
||||
int m_propertyInd;
|
||||
@ -82,21 +76,23 @@ protected:
|
||||
|
||||
struct KeyComparer
|
||||
{
|
||||
size_t operator()(const Key &obj) const {
|
||||
size_t seed = obj.first->hash();
|
||||
boost::hash_combine(seed, obj.second->hash());
|
||||
return seed;
|
||||
}
|
||||
size_t operator()(const Key &obj) const
|
||||
{
|
||||
size_t seed = obj.first->hash();
|
||||
boost::hash_combine(seed, obj.second->hash());
|
||||
return seed;
|
||||
}
|
||||
|
||||
bool operator()(const Key& a, const Key& b) const {
|
||||
if ((*a.first) != (*b.first)) {
|
||||
return false;
|
||||
}
|
||||
if ((*a.second) != (*b.second)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool operator()(const Key& a, const Key& b) const
|
||||
{
|
||||
if ((*a.first) != (*b.first)) {
|
||||
return false;
|
||||
}
|
||||
if ((*a.second) != (*b.second)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
@ -13,32 +13,29 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
|
||||
PhraseBasedReorderingState::PhraseBasedReorderingState(
|
||||
const LRModel &config,
|
||||
LRModel::Direction dir,
|
||||
size_t offset)
|
||||
:LRState(config, dir, offset)
|
||||
namespace Moses2
|
||||
{
|
||||
// uninitialised
|
||||
prevPath = NULL;
|
||||
prevTP = NULL;
|
||||
|
||||
PhraseBasedReorderingState::PhraseBasedReorderingState(const LRModel &config,
|
||||
LRModel::Direction dir, size_t offset) :
|
||||
LRState(config, dir, offset)
|
||||
{
|
||||
// uninitialised
|
||||
prevPath = NULL;
|
||||
prevTP = NULL;
|
||||
}
|
||||
|
||||
void PhraseBasedReorderingState::Init(
|
||||
const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage)
|
||||
void PhraseBasedReorderingState::Init(const LRState *prev,
|
||||
const TargetPhrase &topt, const InputPathBase &path, bool first,
|
||||
const Bitmap *coverage)
|
||||
{
|
||||
prevTP = &topt;
|
||||
prevPath = &path;
|
||||
m_first = first;
|
||||
}
|
||||
|
||||
size_t PhraseBasedReorderingState::hash() const {
|
||||
size_t PhraseBasedReorderingState::hash() const
|
||||
{
|
||||
size_t ret;
|
||||
ret = (size_t) &prevPath->range;
|
||||
boost::hash_combine(ret, m_direction);
|
||||
@ -46,39 +43,44 @@ size_t PhraseBasedReorderingState::hash() const {
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool PhraseBasedReorderingState::operator==(const FFState& o) const {
|
||||
if (&o == this) return true;
|
||||
bool PhraseBasedReorderingState::operator==(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return true;
|
||||
|
||||
const PhraseBasedReorderingState &other = static_cast<const PhraseBasedReorderingState&>(o);
|
||||
if (&prevPath->range == &other.prevPath->range) {
|
||||
if (m_direction == LRModel::Forward) {
|
||||
int compareScore = ComparePrevScores(other.prevTP);
|
||||
return compareScore == 0;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
const PhraseBasedReorderingState &other =
|
||||
static_cast<const PhraseBasedReorderingState&>(o);
|
||||
if (&prevPath->range == &other.prevPath->range) {
|
||||
if (m_direction == LRModel::Forward) {
|
||||
int compareScore = ComparePrevScores(other.prevTP);
|
||||
return compareScore == 0;
|
||||
}
|
||||
else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void PhraseBasedReorderingState::Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const LexicalReordering &ff, const Hypothesis &hypo, size_t phraseTableInd,
|
||||
Scores &scores, FFState &state) const
|
||||
{
|
||||
if ((m_direction != LRModel::Forward) || !m_first) {
|
||||
LRModel const& lrmodel = m_configuration;
|
||||
Range const &cur = hypo.GetInputPath().range;
|
||||
LRModel::ReorderingType reoType = (m_first ? lrmodel.GetOrientation(cur)
|
||||
: lrmodel.GetOrientation(prevPath->range, cur));
|
||||
LRModel::ReorderingType reoType = (
|
||||
m_first ?
|
||||
lrmodel.GetOrientation(cur) :
|
||||
lrmodel.GetOrientation(prevPath->range, cur));
|
||||
CopyScores(mgr.system, scores, hypo.GetTargetPhrase(), reoType);
|
||||
}
|
||||
|
||||
PhraseBasedReorderingState &stateCast = static_cast<PhraseBasedReorderingState&>(state);
|
||||
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false, NULL);
|
||||
PhraseBasedReorderingState &stateCast =
|
||||
static_cast<PhraseBasedReorderingState&>(state);
|
||||
stateCast.Init(this, hypo.GetTargetPhrase(), hypo.GetInputPath(), false,
|
||||
NULL);
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -8,40 +8,34 @@
|
||||
#pragma once
|
||||
#include "LRState.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class InputPathBase;
|
||||
|
||||
class PhraseBasedReorderingState : public LRState
|
||||
class PhraseBasedReorderingState: public LRState
|
||||
{
|
||||
public:
|
||||
const InputPathBase *prevPath;
|
||||
bool m_first;
|
||||
|
||||
PhraseBasedReorderingState(const LRModel &config,
|
||||
LRModel::Direction dir,
|
||||
size_t offset);
|
||||
PhraseBasedReorderingState(const LRModel &config, LRModel::Direction dir,
|
||||
size_t offset);
|
||||
|
||||
void Init(const LRState *prev,
|
||||
const TargetPhrase &topt,
|
||||
const InputPathBase &path,
|
||||
bool first,
|
||||
const Bitmap *coverage);
|
||||
void Init(const LRState *prev, const TargetPhrase &topt,
|
||||
const InputPathBase &path, bool first, const Bitmap *coverage);
|
||||
|
||||
size_t hash() const;
|
||||
virtual bool operator==(const FFState& other) const;
|
||||
|
||||
virtual std::string ToString() const
|
||||
{
|
||||
return "PhraseBasedReorderingState";
|
||||
return "PhraseBasedReorderingState";
|
||||
}
|
||||
|
||||
void Expand(const ManagerBase &mgr,
|
||||
const LexicalReordering &ff,
|
||||
const Hypothesis &hypo,
|
||||
size_t phraseTableInd,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
void Expand(const ManagerBase &mgr, const LexicalReordering &ff,
|
||||
const Hypothesis &hypo, size_t phraseTableInd, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
* ReorderingStack.cpp
|
||||
** Author: Ankit K. Srivastava
|
||||
** Date: Jan 26, 2010
|
||||
*/
|
||||
*/
|
||||
|
||||
#include <vector>
|
||||
#include "ReorderingStack.h"
|
||||
@ -10,8 +10,8 @@
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
ReorderingStack::ReorderingStack(MemPool &pool)
|
||||
:m_stack(pool)
|
||||
ReorderingStack::ReorderingStack(MemPool &pool) :
|
||||
m_stack(pool)
|
||||
{
|
||||
|
||||
}
|
||||
@ -36,10 +36,10 @@ bool ReorderingStack::operator==(const ReorderingStack& o) const
|
||||
// Method to push (shift element into the stack and reduce if reqd)
|
||||
int ReorderingStack::ShiftReduce(const Range &input_span)
|
||||
{
|
||||
int distance; // value to return: the initial distance between this and previous span
|
||||
int distance; // value to return: the initial distance between this and previous span
|
||||
|
||||
// stack is empty
|
||||
if(m_stack.empty()) {
|
||||
if (m_stack.empty()) {
|
||||
m_stack.push_back(input_span);
|
||||
return input_span.GetStartPos() + 1; // - (-1)
|
||||
}
|
||||
@ -48,21 +48,24 @@ int ReorderingStack::ShiftReduce(const Range &input_span)
|
||||
Range prev_span = m_stack.back(); //access last element added
|
||||
|
||||
//calculate the distance we are returning
|
||||
if(input_span.GetStartPos() > prev_span.GetStartPos()) {
|
||||
if (input_span.GetStartPos() > prev_span.GetStartPos()) {
|
||||
distance = input_span.GetStartPos() - prev_span.GetEndPos();
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
distance = input_span.GetEndPos() - prev_span.GetStartPos();
|
||||
}
|
||||
|
||||
if(distance == 1) { //monotone
|
||||
if (distance == 1) { //monotone
|
||||
m_stack.pop_back();
|
||||
Range new_span(prev_span.GetStartPos(), input_span.GetEndPos());
|
||||
Reduce(new_span);
|
||||
} else if(distance == -1) { //swap
|
||||
}
|
||||
else if (distance == -1) { //swap
|
||||
m_stack.pop_back();
|
||||
Range new_span(input_span.GetStartPos(), prev_span.GetEndPos());
|
||||
Reduce(new_span);
|
||||
} else { // discontinuous
|
||||
}
|
||||
else { // discontinuous
|
||||
m_stack.push_back(input_span);
|
||||
}
|
||||
|
||||
@ -78,16 +81,18 @@ void ReorderingStack::Reduce(Range current)
|
||||
|
||||
Range previous = m_stack.back();
|
||||
|
||||
if(current.GetStartPos() - previous.GetEndPos() == 1) { //mono&merge
|
||||
if (current.GetStartPos() - previous.GetEndPos() == 1) { //mono&merge
|
||||
m_stack.pop_back();
|
||||
Range t(previous.GetStartPos(), current.GetEndPos());
|
||||
current = t;
|
||||
} else if(previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
|
||||
}
|
||||
else if (previous.GetStartPos() - current.GetEndPos() == 1) { //swap&merge
|
||||
m_stack.pop_back();
|
||||
Range t(current.GetStartPos(), previous.GetEndPos());
|
||||
current = t;
|
||||
} else { // discontinuous, no more merging
|
||||
cont_loop=false;
|
||||
}
|
||||
else { // discontinuous, no more merging
|
||||
cont_loop = false;
|
||||
}
|
||||
} // finished reducing, exit
|
||||
|
||||
|
@ -38,5 +38,4 @@ private:
|
||||
void Reduce(Range input_span);
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
@ -11,22 +11,20 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line)
|
||||
:StatelessFeatureFunction(startInd, line)
|
||||
PhrasePenalty::PhrasePenalty(size_t startInd, const std::string &line) :
|
||||
StatelessFeatureFunction(startInd, line)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
PhrasePenalty::~PhrasePenalty() {
|
||||
// TODO Auto-generated destructor stub
|
||||
PhrasePenalty::~PhrasePenalty()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void
|
||||
PhrasePenalty::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void PhrasePenalty::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
scores.PlusEquals(system, *this, 1);
|
||||
}
|
||||
|
@ -12,19 +12,16 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class PhrasePenalty : public StatelessFeatureFunction
|
||||
class PhrasePenalty: public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
PhrasePenalty(size_t startInd, const std::string &line);
|
||||
virtual ~PhrasePenalty();
|
||||
PhrasePenalty(size_t startInd, const std::string &line);
|
||||
virtual ~PhrasePenalty();
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -14,76 +14,73 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class SkeletonState : public FFState
|
||||
class SkeletonState: public FFState
|
||||
{
|
||||
public:
|
||||
int targetLen;
|
||||
|
||||
SkeletonState()
|
||||
{
|
||||
// uninitialised
|
||||
// uninitialised
|
||||
}
|
||||
|
||||
virtual size_t hash() const {
|
||||
virtual size_t hash() const
|
||||
{
|
||||
return (size_t) targetLen;
|
||||
}
|
||||
virtual bool operator==(const FFState& o) const {
|
||||
virtual bool operator==(const FFState& o) const
|
||||
{
|
||||
const SkeletonState& other = static_cast<const SkeletonState&>(o);
|
||||
return targetLen == other.targetLen;
|
||||
}
|
||||
|
||||
virtual std::string ToString() const
|
||||
{
|
||||
stringstream sb;
|
||||
sb << targetLen;
|
||||
return sb.str();
|
||||
stringstream sb;
|
||||
sb << targetLen;
|
||||
return sb.str();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line)
|
||||
:StatefulFeatureFunction(startInd, line)
|
||||
SkeletonStatefulFF::SkeletonStatefulFF(size_t startInd, const std::string &line) :
|
||||
StatefulFeatureFunction(startInd, line)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
SkeletonStatefulFF::~SkeletonStatefulFF() {
|
||||
// TODO Auto-generated destructor stub
|
||||
SkeletonStatefulFF::~SkeletonStatefulFF()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
FFState* SkeletonStatefulFF::BlankState(MemPool &pool) const
|
||||
{
|
||||
return new (pool.Allocate<SkeletonState>()) SkeletonState();
|
||||
return new (pool.Allocate<SkeletonState>()) SkeletonState();
|
||||
}
|
||||
|
||||
void SkeletonStatefulFF::EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
const ManagerBase &mgr, const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
{
|
||||
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
|
||||
stateCast.targetLen = 0;
|
||||
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
|
||||
stateCast.targetLen = 0;
|
||||
}
|
||||
|
||||
void
|
||||
SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void SkeletonStatefulFF::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
}
|
||||
|
||||
void SkeletonStatefulFF::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
|
||||
stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
|
||||
SkeletonState &stateCast = static_cast<SkeletonState&>(state);
|
||||
stateCast.targetLen = hypo.GetTargetPhrase().GetSize();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -13,31 +13,24 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class SkeletonStatefulFF : public StatefulFeatureFunction
|
||||
class SkeletonStatefulFF: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
SkeletonStatefulFF(size_t startInd, const std::string &line);
|
||||
virtual ~SkeletonStatefulFF();
|
||||
SkeletonStatefulFF(size_t startInd, const std::string &line);
|
||||
virtual ~SkeletonStatefulFF();
|
||||
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -11,23 +11,22 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd, const std::string &line)
|
||||
:StatelessFeatureFunction(startInd, line)
|
||||
SkeletonStatelessFF::SkeletonStatelessFF(size_t startInd,
|
||||
const std::string &line) :
|
||||
StatelessFeatureFunction(startInd, line)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
SkeletonStatelessFF::~SkeletonStatelessFF() {
|
||||
// TODO Auto-generated destructor stub
|
||||
SkeletonStatelessFF::~SkeletonStatelessFF()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void
|
||||
SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void SkeletonStatelessFF::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
|
||||
}
|
||||
|
@ -12,19 +12,16 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class SkeletonStatelessFF : public StatelessFeatureFunction
|
||||
class SkeletonStatelessFF: public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
SkeletonStatelessFF(size_t startInd, const std::string &line);
|
||||
virtual ~SkeletonStatelessFF();
|
||||
SkeletonStatelessFF(size_t startInd, const std::string &line);
|
||||
virtual ~SkeletonStatelessFF();
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -16,43 +16,46 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
StatefulFeatureFunction::StatefulFeatureFunction(size_t startInd, const std::string &line)
|
||||
:FeatureFunction(startInd, line)
|
||||
StatefulFeatureFunction::StatefulFeatureFunction(size_t startInd,
|
||||
const std::string &line) :
|
||||
FeatureFunction(startInd, line)
|
||||
{
|
||||
}
|
||||
|
||||
StatefulFeatureFunction::~StatefulFeatureFunction() {
|
||||
// TODO Auto-generated destructor stub
|
||||
StatefulFeatureFunction::~StatefulFeatureFunction()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void StatefulFeatureFunction::EvaluateWhenApplied(const ObjectPoolContiguous<Hypothesis*> &hypos) const
|
||||
void StatefulFeatureFunction::EvaluateWhenApplied(
|
||||
const ObjectPoolContiguous<Hypothesis*> &hypos) const
|
||||
{
|
||||
#ifdef __linux
|
||||
/*
|
||||
pthread_t handle;
|
||||
handle = pthread_self();
|
||||
pthread_t handle;
|
||||
handle = pthread_self();
|
||||
|
||||
int s;
|
||||
cpu_set_t cpusetOrig, cpuset;
|
||||
s = pthread_getaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
|
||||
int s;
|
||||
cpu_set_t cpusetOrig, cpuset;
|
||||
s = pthread_getaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
|
||||
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_ZERO(&cpuset);
|
||||
|
||||
int core = handle % 8;
|
||||
core += 24;
|
||||
CPU_SET(core, &cpuset);
|
||||
int core = handle % 8;
|
||||
core += 24;
|
||||
CPU_SET(core, &cpuset);
|
||||
|
||||
s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpuset);
|
||||
*/
|
||||
s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpuset);
|
||||
*/
|
||||
#endif
|
||||
|
||||
for (size_t i = 0; i < hypos.GetSize(); ++i) {
|
||||
Hypothesis *hypo = hypos[i];
|
||||
hypo->EvaluateWhenApplied(*this);
|
||||
}
|
||||
for (size_t i = 0; i < hypos.GetSize(); ++i) {
|
||||
Hypothesis *hypo = hypos[i];
|
||||
hypo->EvaluateWhenApplied(*this);
|
||||
}
|
||||
|
||||
#ifdef __linux
|
||||
// s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
|
||||
// s = pthread_setaffinity_np(handle, sizeof(cpu_set_t), &cpusetOrig);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
@ -18,48 +18,46 @@ namespace Moses2
|
||||
class Hypothesis;
|
||||
class InputType;
|
||||
|
||||
class StatefulFeatureFunction : public FeatureFunction
|
||||
class StatefulFeatureFunction: public FeatureFunction
|
||||
{
|
||||
public:
|
||||
StatefulFeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~StatefulFeatureFunction();
|
||||
StatefulFeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~StatefulFeatureFunction();
|
||||
|
||||
void SetStatefulInd(size_t ind)
|
||||
{ m_statefulInd = ind; }
|
||||
size_t GetStatefulInd() const
|
||||
{ return m_statefulInd; }
|
||||
void SetStatefulInd(size_t ind)
|
||||
{
|
||||
m_statefulInd = ind;
|
||||
}
|
||||
size_t GetStatefulInd() const
|
||||
{
|
||||
return m_statefulInd;
|
||||
}
|
||||
|
||||
//! return uninitialise state
|
||||
virtual FFState* BlankState(MemPool &pool) const = 0;
|
||||
//! return uninitialise state
|
||||
virtual FFState* BlankState(MemPool &pool) const = 0;
|
||||
|
||||
//! return the state associated with the empty hypothesis for a given sentence
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const = 0;
|
||||
//! return the state associated with the empty hypothesis for a given sentence
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const = 0;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ObjectPoolContiguous<Hypothesis*> &hypos) const;
|
||||
virtual void EvaluateWhenApplied(
|
||||
const ObjectPoolContiguous<Hypothesis*> &hypos) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const = 0;
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const = 0;
|
||||
|
||||
virtual void EvaluateWhenAppliedNonBatch(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
{ }
|
||||
virtual void EvaluateWhenAppliedNonBatch(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
}
|
||||
|
||||
protected:
|
||||
size_t m_statefulInd;
|
||||
|
||||
size_t m_statefulInd;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif /* STATEFULFEATUREFUNCTION_H_ */
|
||||
|
@ -10,15 +10,17 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
StatelessFeatureFunction::StatelessFeatureFunction(size_t startInd, const std::string &line)
|
||||
:FeatureFunction(startInd, line)
|
||||
StatelessFeatureFunction::StatelessFeatureFunction(size_t startInd,
|
||||
const std::string &line) :
|
||||
FeatureFunction(startInd, line)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
StatelessFeatureFunction::~StatelessFeatureFunction() {
|
||||
// TODO Auto-generated destructor stub
|
||||
StatelessFeatureFunction::~StatelessFeatureFunction()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -13,11 +13,11 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class StatelessFeatureFunction : public FeatureFunction
|
||||
class StatelessFeatureFunction: public FeatureFunction
|
||||
{
|
||||
public:
|
||||
StatelessFeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~StatelessFeatureFunction();
|
||||
StatelessFeatureFunction(size_t startInd, const std::string &line);
|
||||
virtual ~StatelessFeatureFunction();
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -13,28 +13,24 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
WordPenalty::WordPenalty(size_t startInd, const std::string &line)
|
||||
:StatelessFeatureFunction(startInd, line)
|
||||
WordPenalty::WordPenalty(size_t startInd, const std::string &line) :
|
||||
StatelessFeatureFunction(startInd, line)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
WordPenalty::~WordPenalty() {
|
||||
// TODO Auto-generated destructor stub
|
||||
WordPenalty::~WordPenalty()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void
|
||||
WordPenalty::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void WordPenalty::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
SCORE score = - (SCORE) targetPhrase.GetSize();
|
||||
SCORE score = -(SCORE) targetPhrase.GetSize();
|
||||
scores.PlusEquals(system, *this, score);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -13,19 +13,16 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class WordPenalty : public StatelessFeatureFunction
|
||||
class WordPenalty: public StatelessFeatureFunction
|
||||
{
|
||||
public:
|
||||
WordPenalty(size_t startInd, const std::string &line);
|
||||
virtual ~WordPenalty();
|
||||
WordPenalty(size_t startInd, const std::string &line);
|
||||
virtual ~WordPenalty();
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -25,34 +25,37 @@ namespace Moses2
|
||||
|
||||
HypothesisBase::HypothesisBase(MemPool &pool, const System &system)
|
||||
{
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
|
||||
system.featureFunctions.GetNumScores());
|
||||
|
||||
// FF states
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs = system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
size_t numStatefulFFs = sfffs.size();
|
||||
m_ffStates = (FFState **) pool.Allocate(sizeof(FFState*) * numStatefulFFs);
|
||||
// FF states
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs =
|
||||
system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
size_t numStatefulFFs = sfffs.size();
|
||||
m_ffStates = (FFState **) pool.Allocate(sizeof(FFState*) * numStatefulFFs);
|
||||
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
|
||||
size_t statefulInd = sfff->GetStatefulInd();
|
||||
FFState *state = sfff->BlankState(pool);
|
||||
m_ffStates[statefulInd] = state;
|
||||
}
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
|
||||
size_t statefulInd = sfff->GetStatefulInd();
|
||||
FFState *state = sfff->BlankState(pool);
|
||||
m_ffStates[statefulInd] = state;
|
||||
}
|
||||
}
|
||||
|
||||
size_t HypothesisBase::hash() const
|
||||
{
|
||||
return hash(0);
|
||||
return hash(0);
|
||||
}
|
||||
|
||||
size_t HypothesisBase::hash(size_t seed) const
|
||||
{
|
||||
size_t numStatefulFFs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
|
||||
size_t numStatefulFFs =
|
||||
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
|
||||
|
||||
// states
|
||||
for (size_t i = 0; i < numStatefulFFs; ++i) {
|
||||
const FFState *state = m_ffStates[i];
|
||||
size_t hash = state->hash();
|
||||
boost::hash_combine(seed, hash);
|
||||
const FFState *state = m_ffStates[i];
|
||||
size_t hash = state->hash();
|
||||
boost::hash_combine(seed, hash);
|
||||
}
|
||||
return seed;
|
||||
|
||||
@ -60,20 +63,20 @@ size_t HypothesisBase::hash(size_t seed) const
|
||||
|
||||
bool HypothesisBase::operator==(const HypothesisBase &other) const
|
||||
{
|
||||
size_t numStatefulFFs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
|
||||
size_t numStatefulFFs =
|
||||
GetManager().system.featureFunctions.GetStatefulFeatureFunctions().size();
|
||||
|
||||
// states
|
||||
for (size_t i = 0; i < numStatefulFFs; ++i) {
|
||||
const FFState &thisState = *m_ffStates[i];
|
||||
const FFState &otherState = *other.m_ffStates[i];
|
||||
if (thisState != otherState) {
|
||||
return false;
|
||||
}
|
||||
const FFState &thisState = *m_ffStates[i];
|
||||
const FFState &otherState = *other.m_ffStates[i];
|
||||
if (thisState != otherState) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -20,42 +20,49 @@ class Scores;
|
||||
class HypothesisBase
|
||||
{
|
||||
public:
|
||||
virtual ~HypothesisBase() {}
|
||||
virtual ~HypothesisBase()
|
||||
{
|
||||
}
|
||||
|
||||
inline ManagerBase &GetManager() const
|
||||
{ return *m_mgr; }
|
||||
inline ManagerBase &GetManager() const
|
||||
{
|
||||
return *m_mgr;
|
||||
}
|
||||
|
||||
const Scores &GetScores() const
|
||||
{ return *m_scores; }
|
||||
const Scores &GetScores() const
|
||||
{
|
||||
return *m_scores;
|
||||
}
|
||||
|
||||
const FFState *GetState(size_t ind) const
|
||||
{ return m_ffStates[ind]; }
|
||||
const FFState *GetState(size_t ind) const
|
||||
{
|
||||
return m_ffStates[ind];
|
||||
}
|
||||
|
||||
virtual size_t hash() const;
|
||||
virtual size_t hash(size_t seed) const;
|
||||
virtual bool operator==(const HypothesisBase &other) const;
|
||||
|
||||
virtual SCORE GetFutureScore() const = 0;
|
||||
virtual void EvaluateWhenApplied() = 0;
|
||||
virtual size_t hash() const;
|
||||
virtual size_t hash(size_t seed) const;
|
||||
virtual bool operator==(const HypothesisBase &other) const;
|
||||
|
||||
virtual SCORE GetFutureScore() const = 0;
|
||||
virtual void EvaluateWhenApplied() = 0;
|
||||
|
||||
protected:
|
||||
ManagerBase *m_mgr;
|
||||
Scores *m_scores;
|
||||
FFState **m_ffStates;
|
||||
ManagerBase *m_mgr;
|
||||
Scores *m_scores;
|
||||
FFState **m_ffStates;
|
||||
|
||||
HypothesisBase(MemPool &pool, const System &system);
|
||||
HypothesisBase(MemPool &pool, const System &system);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////
|
||||
class HypothesisFutureScoreOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const {
|
||||
bool operator()(const HypothesisBase* a, const HypothesisBase* b) const
|
||||
{
|
||||
return a->GetFutureScore() > b->GetFutureScore();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -10,12 +10,14 @@
|
||||
#include "ManagerBase.h"
|
||||
#include "System.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
HypothesisColl::HypothesisColl(const ManagerBase &mgr)
|
||||
:m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool()))
|
||||
,m_sortedHypos(NULL)
|
||||
{}
|
||||
HypothesisColl::HypothesisColl(const ManagerBase &mgr) :
|
||||
m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool())), m_sortedHypos(
|
||||
NULL)
|
||||
{
|
||||
}
|
||||
|
||||
StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
|
||||
{
|
||||
@ -23,40 +25,43 @@ StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
|
||||
|
||||
// CHECK RECOMBINATION
|
||||
if (addRet.second) {
|
||||
// equiv hypo doesn't exists
|
||||
return StackAdd(true, NULL);
|
||||
// equiv hypo doesn't exists
|
||||
return StackAdd(true, NULL);
|
||||
}
|
||||
else {
|
||||
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
|
||||
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
|
||||
// incoming hypo is better than the one we have
|
||||
const HypothesisBase *const &hypoExisting1 = *addRet.first;
|
||||
const HypothesisBase *&hypoExisting2 = const_cast<const HypothesisBase *&>(hypoExisting1);
|
||||
hypoExisting2 = hypo;
|
||||
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
|
||||
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
|
||||
// incoming hypo is better than the one we have
|
||||
const HypothesisBase * const &hypoExisting1 = *addRet.first;
|
||||
const HypothesisBase *&hypoExisting2 =
|
||||
const_cast<const HypothesisBase *&>(hypoExisting1);
|
||||
hypoExisting2 = hypo;
|
||||
|
||||
return StackAdd(true, hypoExisting);
|
||||
}
|
||||
else {
|
||||
// already storing the best hypo. discard incoming hypo
|
||||
return StackAdd(false, hypoExisting);
|
||||
}
|
||||
return StackAdd(true, hypoExisting);
|
||||
}
|
||||
else {
|
||||
// already storing the best hypo. discard incoming hypo
|
||||
return StackAdd(false, hypoExisting);
|
||||
}
|
||||
}
|
||||
|
||||
assert(false);
|
||||
}
|
||||
|
||||
Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const
|
||||
Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const
|
||||
{
|
||||
if (m_sortedHypos == NULL) {
|
||||
// create sortedHypos first
|
||||
MemPool &pool = mgr.GetPool();
|
||||
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool, m_coll.size());
|
||||
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool,
|
||||
m_coll.size());
|
||||
|
||||
size_t ind = 0;
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
|
||||
(*m_sortedHypos)[ind] = hypo;
|
||||
++ind;
|
||||
}
|
||||
size_t ind = 0;
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, m_coll){
|
||||
(*m_sortedHypos)[ind] = hypo;
|
||||
++ind;
|
||||
}
|
||||
|
||||
SortAndPruneHypos(mgr, arcLists);
|
||||
}
|
||||
@ -64,56 +69,57 @@ Hypotheses &HypothesisColl::GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLi
|
||||
return *m_sortedHypos;
|
||||
}
|
||||
|
||||
void HypothesisColl::SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const
|
||||
void HypothesisColl::SortAndPruneHypos(const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const
|
||||
{
|
||||
size_t stackSize = mgr.system.stackSize;
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
|
||||
/*
|
||||
cerr << "UNSORTED hypos:" << endl;
|
||||
for (size_t i = 0; i < hypos.size(); ++i) {
|
||||
const Hypothesis *hypo = hypos[i];
|
||||
cerr << *hypo << endl;
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
cerr << "UNSORTED hypos:" << endl;
|
||||
for (size_t i = 0; i < hypos.size(); ++i) {
|
||||
const Hypothesis *hypo = hypos[i];
|
||||
cerr << *hypo << endl;
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
Hypotheses::iterator iterMiddle;
|
||||
iterMiddle = (stackSize == 0 || m_sortedHypos->size() < stackSize)
|
||||
? m_sortedHypos->end()
|
||||
: m_sortedHypos->begin() + stackSize;
|
||||
iterMiddle =
|
||||
(stackSize == 0 || m_sortedHypos->size() < stackSize) ?
|
||||
m_sortedHypos->end() : m_sortedHypos->begin() + stackSize;
|
||||
|
||||
std::partial_sort(m_sortedHypos->begin(), iterMiddle, m_sortedHypos->end(),
|
||||
HypothesisFutureScoreOrderer());
|
||||
HypothesisFutureScoreOrderer());
|
||||
|
||||
// prune
|
||||
if (stackSize && m_sortedHypos->size() > stackSize) {
|
||||
for (size_t i = stackSize; i < m_sortedHypos->size(); ++i) {
|
||||
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
|
||||
recycler.Recycle(hypo);
|
||||
for (size_t i = stackSize; i < m_sortedHypos->size(); ++i) {
|
||||
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
|
||||
recycler.Recycle(hypo);
|
||||
|
||||
// delete from arclist
|
||||
if (mgr.system.nbestSize) {
|
||||
arcLists.Delete(hypo);
|
||||
}
|
||||
}
|
||||
m_sortedHypos->resize(stackSize);
|
||||
// delete from arclist
|
||||
if (mgr.system.nbestSize) {
|
||||
arcLists.Delete(hypo);
|
||||
}
|
||||
}
|
||||
m_sortedHypos->resize(stackSize);
|
||||
}
|
||||
|
||||
/*
|
||||
cerr << "sorted hypos:" << endl;
|
||||
for (size_t i = 0; i < hypos.size(); ++i) {
|
||||
const Hypothesis *hypo = hypos[i];
|
||||
cerr << hypo << " " << *hypo << endl;
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
cerr << "sorted hypos:" << endl;
|
||||
for (size_t i = 0; i < hypos.size(); ++i) {
|
||||
const Hypothesis *hypo = hypos[i];
|
||||
cerr << hypo << " " << *hypo << endl;
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
void HypothesisColl::Clear()
|
||||
{
|
||||
m_sortedHypos = NULL;
|
||||
m_coll.clear();
|
||||
m_sortedHypos = NULL;
|
||||
m_coll.clear();
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -11,54 +11,62 @@
|
||||
#include "Recycler.h"
|
||||
#include "legacy/Util2.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class ManagerBase;
|
||||
class ArcLists;
|
||||
|
||||
typedef Array<const HypothesisBase*> Hypotheses;
|
||||
typedef Array<const HypothesisBase*> Hypotheses;
|
||||
|
||||
class HypothesisColl
|
||||
{
|
||||
public:
|
||||
typedef boost::unordered_set<const HypothesisBase*,
|
||||
UnorderedComparer<HypothesisBase>,
|
||||
UnorderedComparer<HypothesisBase>,
|
||||
MemPoolAllocator<const HypothesisBase*>
|
||||
> _HCType;
|
||||
UnorderedComparer<HypothesisBase>, UnorderedComparer<HypothesisBase>,
|
||||
MemPoolAllocator<const HypothesisBase*> > _HCType;
|
||||
|
||||
typedef _HCType::iterator iterator;
|
||||
typedef _HCType::const_iterator const_iterator;
|
||||
//! iterators
|
||||
const_iterator begin() const {
|
||||
return m_coll.begin();
|
||||
const_iterator begin() const
|
||||
{
|
||||
return m_coll.begin();
|
||||
}
|
||||
const_iterator end() const {
|
||||
return m_coll.end();
|
||||
const_iterator end() const
|
||||
{
|
||||
return m_coll.end();
|
||||
}
|
||||
|
||||
HypothesisColl(const ManagerBase &mgr);
|
||||
HypothesisColl(const ManagerBase &mgr);
|
||||
|
||||
StackAdd Add(const HypothesisBase *hypo);
|
||||
StackAdd Add(const HypothesisBase *hypo);
|
||||
|
||||
_HCType &GetColl()
|
||||
{ return m_coll; }
|
||||
_HCType &GetColl()
|
||||
{
|
||||
return m_coll;
|
||||
}
|
||||
|
||||
const _HCType &GetColl() const
|
||||
{ return m_coll; }
|
||||
const _HCType &GetColl() const
|
||||
{
|
||||
return m_coll;
|
||||
}
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_coll.size(); }
|
||||
size_t GetSize() const
|
||||
{
|
||||
return m_coll.size();
|
||||
}
|
||||
|
||||
void Clear();
|
||||
void Clear();
|
||||
|
||||
Hypotheses &GetSortedAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
|
||||
Hypotheses &GetSortedAndPruneHypos(const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const;
|
||||
|
||||
protected:
|
||||
_HCType m_coll;
|
||||
mutable Hypotheses *m_sortedHypos;
|
||||
_HCType m_coll;
|
||||
mutable Hypotheses *m_sortedHypos;
|
||||
|
||||
void SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
|
||||
void SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -10,14 +10,12 @@
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
InputPathBase::InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPathBase *prefixPath)
|
||||
:subPhrase(subPhrase)
|
||||
,range(range)
|
||||
,prefixPath(prefixPath)
|
||||
InputPathBase::InputPathBase(MemPool &pool, const SubPhrase &subPhrase,
|
||||
const Range &range, size_t numPt, const InputPathBase *prefixPath) :
|
||||
subPhrase(subPhrase), range(range), prefixPath(prefixPath)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -18,16 +18,17 @@ namespace Moses2
|
||||
|
||||
class PhraseTable;
|
||||
|
||||
class InputPathBase {
|
||||
class InputPathBase
|
||||
{
|
||||
public:
|
||||
const InputPathBase *prefixPath;
|
||||
SubPhrase subPhrase;
|
||||
Range range;
|
||||
const InputPathBase *prefixPath;
|
||||
SubPhrase subPhrase;
|
||||
Range range;
|
||||
|
||||
InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPathBase *prefixPath);
|
||||
InputPathBase(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
|
||||
size_t numPt, const InputPathBase *prefixPath);
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -20,6 +20,5 @@ InputPathsBase::~InputPathsBase()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -20,35 +20,40 @@ class InputPathBase;
|
||||
|
||||
class InputPathsBase
|
||||
{
|
||||
typedef std::vector<InputPathBase*> Coll;
|
||||
typedef std::vector<InputPathBase*> Coll;
|
||||
public:
|
||||
InputPathsBase() {}
|
||||
virtual ~InputPathsBase();
|
||||
InputPathsBase()
|
||||
{
|
||||
}
|
||||
virtual ~InputPathsBase();
|
||||
|
||||
//! iterators
|
||||
typedef Coll::iterator iterator;
|
||||
typedef Coll::const_iterator const_iterator;
|
||||
|
||||
const_iterator begin() const {
|
||||
return m_inputPaths.begin();
|
||||
const_iterator begin() const
|
||||
{
|
||||
return m_inputPaths.begin();
|
||||
}
|
||||
const_iterator end() const {
|
||||
return m_inputPaths.end();
|
||||
const_iterator end() const
|
||||
{
|
||||
return m_inputPaths.end();
|
||||
}
|
||||
|
||||
iterator begin() {
|
||||
return m_inputPaths.begin();
|
||||
iterator begin()
|
||||
{
|
||||
return m_inputPaths.begin();
|
||||
}
|
||||
iterator end() {
|
||||
return m_inputPaths.end();
|
||||
iterator end()
|
||||
{
|
||||
return m_inputPaths.end();
|
||||
}
|
||||
|
||||
virtual void Init(const Sentence &input, const ManagerBase &mgr) = 0;
|
||||
|
||||
protected:
|
||||
Coll m_inputPaths;
|
||||
Coll m_inputPaths;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -7,10 +7,12 @@
|
||||
|
||||
#include "InputType.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
InputType::~InputType() {
|
||||
// TODO Auto-generated destructor stub
|
||||
InputType::~InputType()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -8,24 +8,27 @@
|
||||
#ifndef INPUTTYPE_H_
|
||||
#define INPUTTYPE_H_
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class InputType {
|
||||
class InputType
|
||||
{
|
||||
public:
|
||||
InputType(long translationId)
|
||||
:m_translationId(translationId)
|
||||
{}
|
||||
InputType(long translationId) :
|
||||
m_translationId(translationId)
|
||||
{
|
||||
}
|
||||
|
||||
virtual ~InputType();
|
||||
virtual ~InputType();
|
||||
|
||||
long GetTranslationId() const {
|
||||
return m_translationId;
|
||||
}
|
||||
long GetTranslationId() const
|
||||
{
|
||||
return m_translationId;
|
||||
}
|
||||
|
||||
protected:
|
||||
long m_translationId; //< contiguous Id
|
||||
|
||||
|
||||
};
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -24,13 +24,16 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
struct KenLMState : public FFState {
|
||||
struct KenLMState: public FFState
|
||||
{
|
||||
lm::ngram::State state;
|
||||
virtual size_t hash() const {
|
||||
virtual size_t hash() const
|
||||
{
|
||||
size_t ret = hash_value(state);
|
||||
return ret;
|
||||
}
|
||||
virtual bool operator==(const FFState& o) const {
|
||||
virtual bool operator==(const FFState& o) const
|
||||
{
|
||||
const KenLMState &other = static_cast<const KenLMState &>(o);
|
||||
bool ret = state == other.state;
|
||||
return ret;
|
||||
@ -38,32 +41,33 @@ struct KenLMState : public FFState {
|
||||
|
||||
virtual std::string ToString() const
|
||||
{
|
||||
stringstream ss;
|
||||
for (size_t i = 0; i < state.Length(); ++i) {
|
||||
ss << state.words[i] << " ";
|
||||
}
|
||||
return ss.str();
|
||||
stringstream ss;
|
||||
for (size_t i = 0; i < state.Length(); ++i) {
|
||||
ss << state.words[i] << " ";
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
class MappingBuilder : public lm::EnumerateVocab
|
||||
class MappingBuilder: public lm::EnumerateVocab
|
||||
{
|
||||
public:
|
||||
MappingBuilder(FactorCollection &factorCollection, System &system, std::vector<lm::WordIndex> &mapping)
|
||||
: m_factorCollection(factorCollection)
|
||||
, m_system(system)
|
||||
, m_mapping(mapping)
|
||||
{}
|
||||
MappingBuilder(FactorCollection &factorCollection, System &system,
|
||||
std::vector<lm::WordIndex> &mapping) :
|
||||
m_factorCollection(factorCollection), m_system(system), m_mapping(mapping)
|
||||
{
|
||||
}
|
||||
|
||||
void Add(lm::WordIndex index, const StringPiece &str) {
|
||||
std::size_t factorId = m_factorCollection.AddFactor(str, m_system)->GetId();
|
||||
if (m_mapping.size() <= factorId) {
|
||||
// 0 is <unk> :-)
|
||||
m_mapping.resize(factorId + 1);
|
||||
}
|
||||
m_mapping[factorId] = index;
|
||||
void Add(lm::WordIndex index, const StringPiece &str)
|
||||
{
|
||||
std::size_t factorId = m_factorCollection.AddFactor(str, m_system)->GetId();
|
||||
if (m_mapping.size() <= factorId) {
|
||||
// 0 is <unk> :-)
|
||||
m_mapping.resize(factorId + 1);
|
||||
}
|
||||
m_mapping[factorId] = index;
|
||||
}
|
||||
|
||||
private:
|
||||
@ -73,23 +77,23 @@ private:
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
template <class Model>
|
||||
KENLM<Model>::KENLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
|
||||
:StatefulFeatureFunction(startInd, line)
|
||||
,m_path(file)
|
||||
,m_factorType(factorType)
|
||||
,m_load_method(load_method)
|
||||
template<class Model>
|
||||
KENLM<Model>::KENLM(size_t startInd, const std::string &line,
|
||||
const std::string &file, FactorType factorType,
|
||||
util::LoadMethod load_method) :
|
||||
StatefulFeatureFunction(startInd, line), m_path(file), m_factorType(
|
||||
factorType), m_load_method(load_method)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
template<class Model>
|
||||
KENLM<Model>::~KENLM()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
template<class Model>
|
||||
void KENLM<Model>::Load(System &system)
|
||||
{
|
||||
FactorCollection &fc = system.GetVocab();
|
||||
@ -108,7 +112,7 @@ void KENLM<Model>::Load(System &system)
|
||||
m_ngram.reset(new Model(m_path.c_str(), config));
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
template<class Model>
|
||||
FFState* KENLM<Model>::BlankState(MemPool &pool) const
|
||||
{
|
||||
KenLMState *ret = new (pool.Allocate<KenLMState>()) KenLMState();
|
||||
@ -116,23 +120,18 @@ FFState* KENLM<Model>::BlankState(MemPool &pool) const
|
||||
}
|
||||
|
||||
//! return the state associated with the empty hypothesis for a given sentence
|
||||
template <class Model>
|
||||
void KENLM<Model>::EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
template<class Model>
|
||||
void KENLM<Model>::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const
|
||||
{
|
||||
KenLMState &stateCast = static_cast<KenLMState&>(state);
|
||||
stateCast.state = m_ngram->BeginSentenceState();
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
void KENLM<Model>::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
template<class Model>
|
||||
void KENLM<Model>::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
// contains factors used by this LM
|
||||
float fullScore, nGramScore;
|
||||
@ -144,40 +143,41 @@ void KENLM<Model>::EvaluateInIsolation(MemPool &pool,
|
||||
|
||||
bool GetLMEnableOOVFeature = false;
|
||||
if (GetLMEnableOOVFeature) {
|
||||
float scoresVec[2], estimateScoresVec[2];
|
||||
scoresVec[0] = nGramScore;
|
||||
scoresVec[1] = oovCount;
|
||||
scores.PlusEquals(system, *this, scoresVec);
|
||||
float scoresVec[2], estimateScoresVec[2];
|
||||
scoresVec[0] = nGramScore;
|
||||
scoresVec[1] = oovCount;
|
||||
scores.PlusEquals(system, *this, scoresVec);
|
||||
|
||||
estimateScoresVec[0] = estimateScore;
|
||||
estimateScoresVec[1] = 0;
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, estimateScoresVec);
|
||||
(*estimatedScore) += weightedScore;
|
||||
estimateScoresVec[0] = estimateScore;
|
||||
estimateScoresVec[1] = 0;
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
|
||||
estimateScoresVec);
|
||||
(*estimatedScore) += weightedScore;
|
||||
}
|
||||
else {
|
||||
scores.PlusEquals(system, *this, nGramScore);
|
||||
scores.PlusEquals(system, *this, nGramScore);
|
||||
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, estimateScore);
|
||||
(*estimatedScore) += weightedScore;
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
|
||||
estimateScore);
|
||||
(*estimatedScore) += weightedScore;
|
||||
}
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
template<class Model>
|
||||
void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
KenLMState &stateCast = static_cast<KenLMState&>(state);
|
||||
|
||||
const System &system = mgr.system;
|
||||
|
||||
const lm::ngram::State &in_state = static_cast<const KenLMState&>(prevState).state;
|
||||
const lm::ngram::State &in_state =
|
||||
static_cast<const KenLMState&>(prevState).state;
|
||||
|
||||
if (!hypo.GetTargetPhrase().GetSize()) {
|
||||
stateCast.state = in_state;
|
||||
return;
|
||||
return;
|
||||
}
|
||||
|
||||
const std::size_t begin = hypo.GetCurrTargetWordsRange().GetStartPos();
|
||||
@ -189,90 +189,100 @@ void KENLM<Model>::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
typename Model::State aux_state;
|
||||
typename Model::State *state0 = &stateCast.state, *state1 = &aux_state;
|
||||
|
||||
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)), *state0);
|
||||
float score = m_ngram->Score(in_state, TranslateID(hypo.GetWord(position)),
|
||||
*state0);
|
||||
++position;
|
||||
for (; position < adjust_end; ++position) {
|
||||
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)), *state1);
|
||||
std::swap(state0, state1);
|
||||
score += m_ngram->Score(*state0, TranslateID(hypo.GetWord(position)),
|
||||
*state1);
|
||||
std::swap(state0, state1);
|
||||
}
|
||||
|
||||
if (hypo.GetBitmap().IsComplete()) {
|
||||
// Score end of sentence.
|
||||
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
|
||||
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
|
||||
score += m_ngram->FullScoreForgotState(&indices.front(), last, m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
|
||||
} else if (adjust_end < end) {
|
||||
// Get state after adding a long phrase.
|
||||
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
|
||||
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
|
||||
m_ngram->GetState(&indices.front(), last, stateCast.state);
|
||||
} else if (state0 != &stateCast.state) {
|
||||
// Short enough phrase that we can just reuse the state.
|
||||
stateCast.state = *state0;
|
||||
// Score end of sentence.
|
||||
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
|
||||
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
|
||||
score += m_ngram->FullScoreForgotState(&indices.front(), last,
|
||||
m_ngram->GetVocabulary().EndSentence(), stateCast.state).prob;
|
||||
}
|
||||
else if (adjust_end < end) {
|
||||
// Get state after adding a long phrase.
|
||||
std::vector<lm::WordIndex> indices(m_ngram->Order() - 1);
|
||||
const lm::WordIndex *last = LastIDs(hypo, &indices.front());
|
||||
m_ngram->GetState(&indices.front(), last, stateCast.state);
|
||||
}
|
||||
else if (state0 != &stateCast.state) {
|
||||
// Short enough phrase that we can just reuse the state.
|
||||
stateCast.state = *state0;
|
||||
}
|
||||
|
||||
score = TransformLMScore(score);
|
||||
|
||||
bool OOVFeatureEnabled = false;
|
||||
if (OOVFeatureEnabled) {
|
||||
std::vector<float> scoresVec(2);
|
||||
scoresVec[0] = score;
|
||||
scoresVec[1] = 0.0;
|
||||
scores.PlusEquals(system, *this, scoresVec);
|
||||
} else {
|
||||
scores.PlusEquals(system, *this, score);
|
||||
std::vector<float> scoresVec(2);
|
||||
scoresVec[0] = score;
|
||||
scoresVec[1] = 0.0;
|
||||
scores.PlusEquals(system, *this, scoresVec);
|
||||
}
|
||||
else {
|
||||
scores.PlusEquals(system, *this, score);
|
||||
}
|
||||
}
|
||||
|
||||
template <class Model>
|
||||
void KENLM<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const
|
||||
template<class Model>
|
||||
void KENLM<Model>::CalcScore(const Phrase &phrase, float &fullScore,
|
||||
float &ngramScore, std::size_t &oovCount) const
|
||||
{
|
||||
fullScore = 0;
|
||||
ngramScore = 0;
|
||||
oovCount = 0;
|
||||
fullScore = 0;
|
||||
ngramScore = 0;
|
||||
oovCount = 0;
|
||||
|
||||
if (!phrase.GetSize()) return;
|
||||
if (!phrase.GetSize()) return;
|
||||
|
||||
lm::ngram::ChartState discarded_sadly;
|
||||
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
|
||||
lm::ngram::ChartState discarded_sadly;
|
||||
lm::ngram::RuleScore<Model> scorer(*m_ngram, discarded_sadly);
|
||||
|
||||
size_t position;
|
||||
if (m_bos == phrase[0][m_factorType]) {
|
||||
scorer.BeginSentence();
|
||||
position = 1;
|
||||
} else {
|
||||
position = 0;
|
||||
}
|
||||
size_t position;
|
||||
if (m_bos == phrase[0][m_factorType]) {
|
||||
scorer.BeginSentence();
|
||||
position = 1;
|
||||
}
|
||||
else {
|
||||
position = 0;
|
||||
}
|
||||
|
||||
size_t ngramBoundary = m_ngram->Order() - 1;
|
||||
size_t ngramBoundary = m_ngram->Order() - 1;
|
||||
|
||||
size_t end_loop = std::min(ngramBoundary, phrase.GetSize());
|
||||
for (; position < end_loop; ++position) {
|
||||
const Word &word = phrase[position];
|
||||
lm::WordIndex index = TranslateID(word);
|
||||
scorer.Terminal(index);
|
||||
if (!index) ++oovCount;
|
||||
}
|
||||
float before_boundary = fullScore + scorer.Finish();
|
||||
for (; position < phrase.GetSize(); ++position) {
|
||||
const Word &word = phrase[position];
|
||||
lm::WordIndex index = TranslateID(word);
|
||||
scorer.Terminal(index);
|
||||
if (!index) ++oovCount;
|
||||
}
|
||||
fullScore += scorer.Finish();
|
||||
size_t end_loop = std::min(ngramBoundary, phrase.GetSize());
|
||||
for (; position < end_loop; ++position) {
|
||||
const Word &word = phrase[position];
|
||||
lm::WordIndex index = TranslateID(word);
|
||||
scorer.Terminal(index);
|
||||
if (!index) ++oovCount;
|
||||
}
|
||||
float before_boundary = fullScore + scorer.Finish();
|
||||
for (; position < phrase.GetSize(); ++position) {
|
||||
const Word &word = phrase[position];
|
||||
lm::WordIndex index = TranslateID(word);
|
||||
scorer.Terminal(index);
|
||||
if (!index) ++oovCount;
|
||||
}
|
||||
fullScore += scorer.Finish();
|
||||
|
||||
ngramScore = TransformLMScore(fullScore - before_boundary);
|
||||
fullScore = TransformLMScore(fullScore);
|
||||
ngramScore = TransformLMScore(fullScore - before_boundary);
|
||||
fullScore = TransformLMScore(fullScore);
|
||||
}
|
||||
|
||||
// Convert last words of hypothesis into vocab ids, returning an end pointer.
|
||||
template <class Model>
|
||||
lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const {
|
||||
template<class Model>
|
||||
lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo,
|
||||
lm::WordIndex *indices) const
|
||||
{
|
||||
lm::WordIndex *index = indices;
|
||||
lm::WordIndex *end = indices + m_ngram->Order() - 1;
|
||||
int position = hypo.GetCurrTargetWordsRange().GetEndPos();
|
||||
for (; ; ++index, --position) {
|
||||
for (;; ++index, --position) {
|
||||
if (index == end) return index;
|
||||
if (position == -1) {
|
||||
*index = m_ngram->GetVocabulary().BeginSentence();
|
||||
@ -289,13 +299,12 @@ lm::WordIndex *KENLM<Model>::LastIDs(const Hypothesis &hypo, lm::WordIndex *indi
|
||||
* Otherwise, depending on the compiler, those functions may not be present
|
||||
* at link time.
|
||||
*/
|
||||
template class KENLM<lm::ngram::ProbingModel>;
|
||||
template class KENLM<lm::ngram::RestProbingModel>;
|
||||
template class KENLM<lm::ngram::TrieModel>;
|
||||
template class KENLM<lm::ngram::ArrayTrieModel>;
|
||||
template class KENLM<lm::ngram::QuantTrieModel>;
|
||||
template class KENLM<lm::ngram::QuantArrayTrieModel>;
|
||||
|
||||
template class KENLM<lm::ngram::ProbingModel> ;
|
||||
template class KENLM<lm::ngram::RestProbingModel> ;
|
||||
template class KENLM<lm::ngram::TrieModel> ;
|
||||
template class KENLM<lm::ngram::ArrayTrieModel> ;
|
||||
template class KENLM<lm::ngram::QuantTrieModel> ;
|
||||
template class KENLM<lm::ngram::QuantArrayTrieModel> ;
|
||||
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
|
||||
{
|
||||
@ -310,65 +319,90 @@ FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig)
|
||||
line << "KENLM";
|
||||
|
||||
for (; argument; ++argument) {
|
||||
const char *equals = std::find(argument->data(), argument->data() + argument->size(), '=');
|
||||
UTIL_THROW_IF2(equals == argument->data() + argument->size(),
|
||||
"Expected = in KenLM argument " << *argument);
|
||||
StringPiece name(argument->data(), equals - argument->data());
|
||||
StringPiece value(equals + 1, argument->data() + argument->size() - equals - 1);
|
||||
if (name == "factor") {
|
||||
factorType = boost::lexical_cast<FactorType>(value);
|
||||
} else if (name == "order") {
|
||||
// Ignored
|
||||
} else if (name == "path") {
|
||||
filePath.assign(value.data(), value.size());
|
||||
} else if (name == "lazyken") {
|
||||
// deprecated: use load instead.
|
||||
load_method = boost::lexical_cast<bool>(value) ? util::LAZY : util::POPULATE_OR_READ;
|
||||
} else if (name == "load") {
|
||||
if (value == "lazy") {
|
||||
load_method = util::LAZY;
|
||||
} else if (value == "populate_or_lazy") {
|
||||
load_method = util::POPULATE_OR_LAZY;
|
||||
} else if (value == "populate_or_read" || value == "populate") {
|
||||
load_method = util::POPULATE_OR_READ;
|
||||
} else if (value == "read") {
|
||||
load_method = util::READ;
|
||||
} else if (value == "parallel_read") {
|
||||
load_method = util::PARALLEL_READ;
|
||||
} else {
|
||||
UTIL_THROW2("Unknown KenLM load method " << value);
|
||||
}
|
||||
} else {
|
||||
// pass to base class to interpret
|
||||
line << " " << name << "=" << value;
|
||||
}
|
||||
const char *equals = std::find(argument->data(),
|
||||
argument->data() + argument->size(), '=');
|
||||
UTIL_THROW_IF2(equals == argument->data() + argument->size(),
|
||||
"Expected = in KenLM argument " << *argument);
|
||||
StringPiece name(argument->data(), equals - argument->data());
|
||||
StringPiece value(equals + 1,
|
||||
argument->data() + argument->size() - equals - 1);
|
||||
if (name == "factor") {
|
||||
factorType = boost::lexical_cast<FactorType>(value);
|
||||
}
|
||||
else if (name == "order") {
|
||||
// Ignored
|
||||
}
|
||||
else if (name == "path") {
|
||||
filePath.assign(value.data(), value.size());
|
||||
}
|
||||
else if (name == "lazyken") {
|
||||
// deprecated: use load instead.
|
||||
load_method =
|
||||
boost::lexical_cast<bool>(value) ?
|
||||
util::LAZY : util::POPULATE_OR_READ;
|
||||
}
|
||||
else if (name == "load") {
|
||||
if (value == "lazy") {
|
||||
load_method = util::LAZY;
|
||||
}
|
||||
else if (value == "populate_or_lazy") {
|
||||
load_method = util::POPULATE_OR_LAZY;
|
||||
}
|
||||
else if (value == "populate_or_read" || value == "populate") {
|
||||
load_method = util::POPULATE_OR_READ;
|
||||
}
|
||||
else if (value == "read") {
|
||||
load_method = util::READ;
|
||||
}
|
||||
else if (value == "parallel_read") {
|
||||
load_method = util::PARALLEL_READ;
|
||||
}
|
||||
else {
|
||||
UTIL_THROW2("Unknown KenLM load method " << value);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// pass to base class to interpret
|
||||
line << " " << name << "=" << value;
|
||||
}
|
||||
}
|
||||
|
||||
return ConstructKenLM(startInd, line.str(), filePath, factorType, load_method);
|
||||
}
|
||||
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
|
||||
const std::string &file, FactorType factorType,
|
||||
util::LoadMethod load_method)
|
||||
{
|
||||
lm::ngram::ModelType model_type;
|
||||
if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) {
|
||||
switch(model_type) {
|
||||
switch (model_type) {
|
||||
case lm::ngram::PROBING:
|
||||
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file,
|
||||
factorType, load_method);
|
||||
case lm::ngram::REST_PROBING:
|
||||
return new KENLM<lm::ngram::RestProbingModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::RestProbingModel>(startInd, line, file,
|
||||
factorType, load_method);
|
||||
case lm::ngram::TRIE:
|
||||
return new KENLM<lm::ngram::TrieModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::TrieModel>(startInd, line, file, factorType,
|
||||
load_method);
|
||||
case lm::ngram::QUANT_TRIE:
|
||||
return new KENLM<lm::ngram::QuantTrieModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::QuantTrieModel>(startInd, line, file,
|
||||
factorType, load_method);
|
||||
case lm::ngram::ARRAY_TRIE:
|
||||
return new KENLM<lm::ngram::ArrayTrieModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::ArrayTrieModel>(startInd, line, file,
|
||||
factorType, load_method);
|
||||
case lm::ngram::QUANT_ARRAY_TRIE:
|
||||
return new KENLM<lm::ngram::QuantArrayTrieModel>(startInd, line, file, factorType, load_method);
|
||||
return new KENLM<lm::ngram::QuantArrayTrieModel>(startInd, line, file,
|
||||
factorType, load_method);
|
||||
default:
|
||||
UTIL_THROW2("Unrecognized kenlm model type " << model_type);
|
||||
UTIL_THROW2("Unrecognized kenlm model type " << model_type)
|
||||
;
|
||||
}
|
||||
} else {
|
||||
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType, load_method);
|
||||
}
|
||||
else {
|
||||
return new KENLM<lm::ngram::ProbingModel>(startInd, line, file, factorType,
|
||||
load_method);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -21,13 +21,16 @@ namespace Moses2
|
||||
class Word;
|
||||
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &lineOrig);
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
|
||||
FeatureFunction *ConstructKenLM(size_t startInd, const std::string &line,
|
||||
const std::string &file, FactorType factorType,
|
||||
util::LoadMethod load_method);
|
||||
|
||||
template <class Model>
|
||||
class KENLM : public StatefulFeatureFunction
|
||||
template<class Model>
|
||||
class KENLM: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
KENLM(size_t startInd, const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method);
|
||||
KENLM(size_t startInd, const std::string &line, const std::string &file,
|
||||
FactorType factorType, util::LoadMethod load_method);
|
||||
|
||||
virtual ~KENLM();
|
||||
|
||||
@ -36,24 +39,17 @@ public:
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
|
||||
//! return the state associated with the empty hypothesis for a given sentence
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
std::string m_path;
|
||||
@ -64,12 +60,13 @@ protected:
|
||||
|
||||
boost::shared_ptr<Model> m_ngram;
|
||||
|
||||
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const;
|
||||
void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore,
|
||||
std::size_t &oovCount) const;
|
||||
|
||||
inline lm::WordIndex TranslateID(const Word &word) const
|
||||
{
|
||||
std::size_t factor = word[m_factorType]->GetId();
|
||||
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
|
||||
std::size_t factor = word[m_factorType]->GetId();
|
||||
return (factor >= m_lmIdLookup.size() ? 0 : m_lmIdLookup[factor]);
|
||||
}
|
||||
// Convert last words of hypothesis into vocab ids, returning an end pointer.
|
||||
lm::WordIndex *LastIDs(const Hypothesis &hypo, lm::WordIndex *indices) const;
|
||||
|
@ -21,31 +21,32 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
struct LMState : public PointerState
|
||||
struct LMState: public PointerState
|
||||
{
|
||||
LMState()
|
||||
:PointerState()
|
||||
LMState() :
|
||||
PointerState()
|
||||
{
|
||||
// uninitialised
|
||||
// uninitialised
|
||||
}
|
||||
|
||||
void Set(MemPool &pool, void *lms, const std::vector<const Factor*> &context)
|
||||
{
|
||||
lmstate = lms;
|
||||
lmstate = lms;
|
||||
|
||||
numWords = context.size();
|
||||
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*) * numWords);
|
||||
for (size_t i = 0; i < numWords; ++i) {
|
||||
lastWords[i] = context[i];
|
||||
}
|
||||
numWords = context.size();
|
||||
lastWords = (const Factor**) pool.Allocate(
|
||||
sizeof(const Factor*) * numWords);
|
||||
for (size_t i = 0; i < numWords; ++i) {
|
||||
lastWords[i] = context[i];
|
||||
}
|
||||
}
|
||||
|
||||
void Init(MemPool &pool, const Factor *factor)
|
||||
{
|
||||
lmstate = NULL;
|
||||
numWords = 1;
|
||||
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*));
|
||||
lastWords[0] = factor;
|
||||
lmstate = NULL;
|
||||
numWords = 1;
|
||||
lastWords = (const Factor**) pool.Allocate(sizeof(const Factor*));
|
||||
lastWords[0] = factor;
|
||||
}
|
||||
|
||||
size_t numWords;
|
||||
@ -53,15 +54,15 @@ struct LMState : public PointerState
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
LanguageModel::LanguageModel(size_t startInd, const std::string &line)
|
||||
:StatefulFeatureFunction(startInd, line)
|
||||
,m_oov(-100)
|
||||
LanguageModel::LanguageModel(size_t startInd, const std::string &line) :
|
||||
StatefulFeatureFunction(startInd, line), m_oov(-100)
|
||||
{
|
||||
ReadParameters();
|
||||
ReadParameters();
|
||||
}
|
||||
|
||||
LanguageModel::~LanguageModel() {
|
||||
// TODO Auto-generated destructor stub
|
||||
LanguageModel::~LanguageModel()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void LanguageModel::Load(System &system)
|
||||
@ -75,247 +76,247 @@ void LanguageModel::Load(System &system)
|
||||
size_t lineNum = 0;
|
||||
string line;
|
||||
while (getline(infile, line)) {
|
||||
if (++lineNum % 100000 == 0) {
|
||||
cerr << lineNum << " ";
|
||||
}
|
||||
if (++lineNum % 100000 == 0) {
|
||||
cerr << lineNum << " ";
|
||||
}
|
||||
|
||||
vector<string> substrings = Tokenize(line, "\t");
|
||||
vector<string> substrings = Tokenize(line, "\t");
|
||||
|
||||
if (substrings.size() < 2)
|
||||
continue;
|
||||
if (substrings.size() < 2) continue;
|
||||
|
||||
assert(substrings.size() == 2 || substrings.size() == 3);
|
||||
assert(substrings.size() == 2 || substrings.size() == 3);
|
||||
|
||||
SCORE prob = TransformLMScore(Scan<SCORE>(substrings[0]));
|
||||
if (substrings[1] == "<unk>") {
|
||||
m_oov = prob;
|
||||
continue;
|
||||
}
|
||||
SCORE prob = TransformLMScore(Scan<SCORE>(substrings[0]));
|
||||
if (substrings[1] == "<unk>") {
|
||||
m_oov = prob;
|
||||
continue;
|
||||
}
|
||||
|
||||
SCORE backoff = 0.f;
|
||||
if (substrings.size() == 3) {
|
||||
backoff = TransformLMScore(Scan<SCORE>(substrings[2]));
|
||||
}
|
||||
SCORE backoff = 0.f;
|
||||
if (substrings.size() == 3) {
|
||||
backoff = TransformLMScore(Scan<SCORE>(substrings[2]));
|
||||
}
|
||||
|
||||
// ngram
|
||||
vector<string> key = Tokenize(substrings[1], " ");
|
||||
// ngram
|
||||
vector<string> key = Tokenize(substrings[1], " ");
|
||||
|
||||
vector<const Factor*> factorKey(key.size());
|
||||
for (size_t i = 0; i < key.size(); ++i) {
|
||||
factorKey[factorKey.size() - i - 1] = fc.AddFactor(key[i], system, false);
|
||||
}
|
||||
vector<const Factor*> factorKey(key.size());
|
||||
for (size_t i = 0; i < key.size(); ++i) {
|
||||
factorKey[factorKey.size() - i - 1] = fc.AddFactor(key[i], system, false);
|
||||
}
|
||||
|
||||
m_root.insert(factorKey, LMScores(prob, backoff));
|
||||
m_root.insert(factorKey, LMScores(prob, backoff));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void LanguageModel::SetParameter(const std::string& key, const std::string& value)
|
||||
void LanguageModel::SetParameter(const std::string& key,
|
||||
const std::string& value)
|
||||
{
|
||||
if (key == "path") {
|
||||
m_path = value;
|
||||
m_path = value;
|
||||
}
|
||||
else if (key == "factor") {
|
||||
m_factorType = Scan<FactorType>(value);
|
||||
m_factorType = Scan<FactorType>(value);
|
||||
}
|
||||
else if (key == "order") {
|
||||
m_order = Scan<size_t>(value);
|
||||
m_order = Scan<size_t>(value);
|
||||
}
|
||||
else {
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
StatefulFeatureFunction::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
FFState* LanguageModel::BlankState(MemPool &pool) const
|
||||
{
|
||||
return new (pool.Allocate<LMState>()) LMState();
|
||||
return new (pool.Allocate<LMState>()) LMState();
|
||||
}
|
||||
|
||||
void LanguageModel::EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const
|
||||
void LanguageModel::EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const
|
||||
{
|
||||
LMState &stateCast = static_cast<LMState&>(state);
|
||||
LMState &stateCast = static_cast<LMState&>(state);
|
||||
|
||||
MemPool &pool = mgr.GetPool();
|
||||
stateCast.Init(pool, m_bos);
|
||||
MemPool &pool = mgr.GetPool();
|
||||
stateCast.Init(pool, m_bos);
|
||||
}
|
||||
|
||||
void
|
||||
LanguageModel::EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const
|
||||
{
|
||||
if (targetPhrase.GetSize() == 0) {
|
||||
return;
|
||||
}
|
||||
if (targetPhrase.GetSize() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
SCORE score = 0;
|
||||
SCORE nonFullScore = 0;
|
||||
vector<const Factor*> context;
|
||||
SCORE score = 0;
|
||||
SCORE nonFullScore = 0;
|
||||
vector<const Factor*> context;
|
||||
// context.push_back(m_bos);
|
||||
|
||||
context.reserve(m_order);
|
||||
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
|
||||
const Factor *factor = targetPhrase[i][m_factorType];
|
||||
ShiftOrPush(context, factor);
|
||||
context.reserve(m_order);
|
||||
for (size_t i = 0; i < targetPhrase.GetSize(); ++i) {
|
||||
const Factor *factor = targetPhrase[i][m_factorType];
|
||||
ShiftOrPush(context, factor);
|
||||
|
||||
if (context.size() == m_order) {
|
||||
std::pair<SCORE, void*> fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
}
|
||||
else if (estimatedScore) {
|
||||
std::pair<SCORE, void*> fromScoring = Score(context);
|
||||
nonFullScore += fromScoring.first;
|
||||
}
|
||||
}
|
||||
if (context.size() == m_order) {
|
||||
std::pair<SCORE, void*> fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
}
|
||||
else if (estimatedScore) {
|
||||
std::pair<SCORE, void*> fromScoring = Score(context);
|
||||
nonFullScore += fromScoring.first;
|
||||
}
|
||||
}
|
||||
|
||||
scores.PlusEquals(system, *this, score);
|
||||
if (estimatedScore) {
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, nonFullScore);
|
||||
(*estimatedScore) += weightedScore;
|
||||
}
|
||||
scores.PlusEquals(system, *this, score);
|
||||
if (estimatedScore) {
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this,
|
||||
nonFullScore);
|
||||
(*estimatedScore) += weightedScore;
|
||||
}
|
||||
}
|
||||
|
||||
void LanguageModel::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const
|
||||
{
|
||||
const LMState &prevLMState = static_cast<const LMState &>(prevState);
|
||||
size_t numWords = prevLMState.numWords;
|
||||
const LMState &prevLMState = static_cast<const LMState &>(prevState);
|
||||
size_t numWords = prevLMState.numWords;
|
||||
|
||||
// context is held backwards
|
||||
vector<const Factor*> context(numWords);
|
||||
for (size_t i = 0; i < numWords; ++i) {
|
||||
context[i] = prevLMState.lastWords[i];
|
||||
}
|
||||
//DebugContext(context);
|
||||
// context is held backwards
|
||||
vector<const Factor*> context(numWords);
|
||||
for (size_t i = 0; i < numWords; ++i) {
|
||||
context[i] = prevLMState.lastWords[i];
|
||||
}
|
||||
//DebugContext(context);
|
||||
|
||||
SCORE score = 0;
|
||||
std::pair<SCORE, void*> fromScoring;
|
||||
const TargetPhrase &tp = hypo.GetTargetPhrase();
|
||||
for (size_t i = 0; i < tp.GetSize(); ++i) {
|
||||
const Word &word = tp[i];
|
||||
const Factor *factor = word[m_factorType];
|
||||
ShiftOrPush(context, factor);
|
||||
fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
}
|
||||
SCORE score = 0;
|
||||
std::pair<SCORE, void*> fromScoring;
|
||||
const TargetPhrase &tp = hypo.GetTargetPhrase();
|
||||
for (size_t i = 0; i < tp.GetSize(); ++i) {
|
||||
const Word &word = tp[i];
|
||||
const Factor *factor = word[m_factorType];
|
||||
ShiftOrPush(context, factor);
|
||||
fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
}
|
||||
|
||||
const Bitmap &bm = hypo.GetBitmap();
|
||||
if (bm.IsComplete()) {
|
||||
// everything translated
|
||||
ShiftOrPush(context, m_eos);
|
||||
fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
fromScoring.second = NULL;
|
||||
context.clear();
|
||||
}
|
||||
else {
|
||||
assert(context.size());
|
||||
if (context.size() == m_order) {
|
||||
context.resize(context.size() - 1);
|
||||
}
|
||||
}
|
||||
const Bitmap &bm = hypo.GetBitmap();
|
||||
if (bm.IsComplete()) {
|
||||
// everything translated
|
||||
ShiftOrPush(context, m_eos);
|
||||
fromScoring = Score(context);
|
||||
score += fromScoring.first;
|
||||
fromScoring.second = NULL;
|
||||
context.clear();
|
||||
}
|
||||
else {
|
||||
assert(context.size());
|
||||
if (context.size() == m_order) {
|
||||
context.resize(context.size() - 1);
|
||||
}
|
||||
}
|
||||
|
||||
scores.PlusEquals(mgr.system, *this, score);
|
||||
scores.PlusEquals(mgr.system, *this, score);
|
||||
|
||||
// return state
|
||||
//DebugContext(context);
|
||||
// return state
|
||||
//DebugContext(context);
|
||||
|
||||
LMState &stateCast = static_cast<LMState&>(state);
|
||||
MemPool &pool = mgr.GetPool();
|
||||
stateCast.Set(pool, fromScoring.second, context);
|
||||
LMState &stateCast = static_cast<LMState&>(state);
|
||||
MemPool &pool = mgr.GetPool();
|
||||
stateCast.Set(pool, fromScoring.second, context);
|
||||
}
|
||||
|
||||
void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context, const Factor *factor) const
|
||||
void LanguageModel::ShiftOrPush(std::vector<const Factor*> &context,
|
||||
const Factor *factor) const
|
||||
{
|
||||
if (context.size() < m_order) {
|
||||
context.resize(context.size() + 1);
|
||||
}
|
||||
assert(context.size());
|
||||
if (context.size() < m_order) {
|
||||
context.resize(context.size() + 1);
|
||||
}
|
||||
assert(context.size());
|
||||
|
||||
for (size_t i = context.size() - 1; i > 0; --i) {
|
||||
context[i] = context[i - 1];
|
||||
}
|
||||
for (size_t i = context.size() - 1; i > 0; --i) {
|
||||
context[i] = context[i - 1];
|
||||
}
|
||||
|
||||
context[0] = factor;
|
||||
context[0] = factor;
|
||||
}
|
||||
|
||||
std::pair<SCORE, void*> LanguageModel::Score(const std::vector<const Factor*> &context) const
|
||||
std::pair<SCORE, void*> LanguageModel::Score(
|
||||
const std::vector<const Factor*> &context) const
|
||||
{
|
||||
//cerr << "context=";
|
||||
//DebugContext(context);
|
||||
//cerr << "context=";
|
||||
//DebugContext(context);
|
||||
|
||||
std::pair<SCORE, void*> ret;
|
||||
std::pair<SCORE, void*> ret;
|
||||
|
||||
typedef Node<const Factor*, LMScores> LMNode;
|
||||
const LMNode *node = m_root.getNode(context);
|
||||
if (node) {
|
||||
ret.first = node->getValue().prob;
|
||||
ret.second = (void*) node;
|
||||
}
|
||||
else {
|
||||
SCORE backoff = 0;
|
||||
std::vector<const Factor*> backOffContext(context.begin() + 1, context.end());
|
||||
node = m_root.getNode(backOffContext);
|
||||
if (node) {
|
||||
backoff = node->getValue().backoff;
|
||||
}
|
||||
typedef Node<const Factor*, LMScores> LMNode;
|
||||
const LMNode *node = m_root.getNode(context);
|
||||
if (node) {
|
||||
ret.first = node->getValue().prob;
|
||||
ret.second = (void*) node;
|
||||
}
|
||||
else {
|
||||
SCORE backoff = 0;
|
||||
std::vector<const Factor*> backOffContext(context.begin() + 1,
|
||||
context.end());
|
||||
node = m_root.getNode(backOffContext);
|
||||
if (node) {
|
||||
backoff = node->getValue().backoff;
|
||||
}
|
||||
|
||||
std::vector<const Factor*> newContext(context.begin(), context.end() - 1);
|
||||
std::pair<SCORE, void*> newRet = Score(newContext);
|
||||
std::vector<const Factor*> newContext(context.begin(), context.end() - 1);
|
||||
std::pair<SCORE, void*> newRet = Score(newContext);
|
||||
|
||||
ret.first = backoff + newRet.first;
|
||||
ret.second = newRet.second;
|
||||
}
|
||||
ret.first = backoff + newRet.first;
|
||||
ret.second = newRet.second;
|
||||
}
|
||||
|
||||
//cerr << "score=" << ret.first << endl;
|
||||
return ret;
|
||||
//cerr << "score=" << ret.first << endl;
|
||||
return ret;
|
||||
}
|
||||
|
||||
SCORE LanguageModel::BackoffScore(const std::vector<const Factor*> &context) const
|
||||
SCORE LanguageModel::BackoffScore(
|
||||
const std::vector<const Factor*> &context) const
|
||||
{
|
||||
//cerr << "backoff=";
|
||||
//DebugContext(context);
|
||||
//cerr << "backoff=";
|
||||
//DebugContext(context);
|
||||
|
||||
SCORE ret;
|
||||
size_t stoppedAtInd;
|
||||
const Node<const Factor*, LMScores> &node = m_root.getNode(context, stoppedAtInd);
|
||||
SCORE ret;
|
||||
size_t stoppedAtInd;
|
||||
const Node<const Factor*, LMScores> &node = m_root.getNode(context,
|
||||
stoppedAtInd);
|
||||
|
||||
if (stoppedAtInd == context.size()) {
|
||||
// found entire ngram
|
||||
ret = node.getValue().backoff;
|
||||
}
|
||||
else {
|
||||
if (stoppedAtInd == 0) {
|
||||
ret = m_oov;
|
||||
stoppedAtInd = 1;
|
||||
}
|
||||
else {
|
||||
ret = node.getValue().backoff;
|
||||
}
|
||||
if (stoppedAtInd == context.size()) {
|
||||
// found entire ngram
|
||||
ret = node.getValue().backoff;
|
||||
}
|
||||
else {
|
||||
if (stoppedAtInd == 0) {
|
||||
ret = m_oov;
|
||||
stoppedAtInd = 1;
|
||||
}
|
||||
else {
|
||||
ret = node.getValue().backoff;
|
||||
}
|
||||
|
||||
// recursive
|
||||
std::vector<const Factor*> backoff(context.begin() + stoppedAtInd, context.end());
|
||||
ret += BackoffScore(backoff);
|
||||
}
|
||||
// recursive
|
||||
std::vector<const Factor*> backoff(context.begin() + stoppedAtInd,
|
||||
context.end());
|
||||
ret += BackoffScore(backoff);
|
||||
}
|
||||
|
||||
return ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
void LanguageModel::DebugContext(const std::vector<const Factor*> &context) const
|
||||
void LanguageModel::DebugContext(
|
||||
const std::vector<const Factor*> &context) const
|
||||
{
|
||||
for (size_t i = 0; i < context.size(); ++i) {
|
||||
cerr << context[i]->GetString() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
for (size_t i = 0; i < context.size(); ++i) {
|
||||
cerr << context[i]->GetString() << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -20,77 +20,72 @@ namespace Moses2
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
struct LMScores
|
||||
{
|
||||
LMScores()
|
||||
{}
|
||||
LMScores()
|
||||
{
|
||||
}
|
||||
|
||||
LMScores(const LMScores ©)
|
||||
:prob(copy.prob)
|
||||
,backoff(copy.backoff)
|
||||
{}
|
||||
LMScores(const LMScores ©) :
|
||||
prob(copy.prob), backoff(copy.backoff)
|
||||
{
|
||||
}
|
||||
|
||||
LMScores(float inProb, float inBackoff)
|
||||
:prob(inProb)
|
||||
,backoff(inBackoff)
|
||||
{}
|
||||
LMScores(float inProb, float inBackoff) :
|
||||
prob(inProb), backoff(inBackoff)
|
||||
{
|
||||
}
|
||||
|
||||
float prob, backoff;
|
||||
float prob, backoff;
|
||||
};
|
||||
|
||||
inline std::ostream& operator<<(std::ostream &out, const LMScores &obj)
|
||||
{
|
||||
out << "(" << obj.prob << "," << obj.backoff << ")" << std::flush;
|
||||
return out;
|
||||
out << "(" << obj.prob << "," << obj.backoff << ")" << std::flush;
|
||||
return out;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////
|
||||
class LanguageModel : public StatefulFeatureFunction
|
||||
class LanguageModel: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
LanguageModel(size_t startInd, const std::string &line);
|
||||
virtual ~LanguageModel();
|
||||
LanguageModel(size_t startInd, const std::string &line);
|
||||
virtual ~LanguageModel();
|
||||
|
||||
virtual void Load(System &system);
|
||||
virtual void Load(System &system);
|
||||
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source,
|
||||
const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
std::string m_path;
|
||||
FactorType m_factorType;
|
||||
size_t m_order;
|
||||
std::string m_path;
|
||||
FactorType m_factorType;
|
||||
size_t m_order;
|
||||
|
||||
MorphTrie<const Factor*, LMScores> m_root;
|
||||
SCORE m_oov;
|
||||
const Factor *m_bos;
|
||||
const Factor *m_eos;
|
||||
MorphTrie<const Factor*, LMScores> m_root;
|
||||
SCORE m_oov;
|
||||
const Factor *m_bos;
|
||||
const Factor *m_eos;
|
||||
|
||||
void ShiftOrPush(std::vector<const Factor*> &context, const Factor *factor) const;
|
||||
std::pair<SCORE, void*> Score(const std::vector<const Factor*> &context) const;
|
||||
SCORE BackoffScore(const std::vector<const Factor*> &context) const;
|
||||
void ShiftOrPush(std::vector<const Factor*> &context,
|
||||
const Factor *factor) const;
|
||||
std::pair<SCORE, void*> Score(
|
||||
const std::vector<const Factor*> &context) const;
|
||||
SCORE BackoffScore(const std::vector<const Factor*> &context) const;
|
||||
|
||||
void DebugContext(const std::vector<const Factor*> &context) const;
|
||||
void DebugContext(const std::vector<const Factor*> &context) const;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif /* LANGUAGEMODEL_H_ */
|
||||
|
@ -10,7 +10,6 @@
|
||||
#include "../legacy/Util2.h"
|
||||
#include "../legacy/Factor.h"
|
||||
|
||||
|
||||
namespace DALM
|
||||
{
|
||||
class Logger;
|
||||
@ -26,40 +25,34 @@ typedef unsigned int VocabId;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class LanguageModelDALM : public StatefulFeatureFunction
|
||||
class LanguageModelDALM: public StatefulFeatureFunction
|
||||
{
|
||||
public:
|
||||
LanguageModelDALM(size_t startInd, const std::string &line);
|
||||
virtual ~LanguageModelDALM();
|
||||
LanguageModelDALM(size_t startInd, const std::string &line);
|
||||
virtual ~LanguageModelDALM();
|
||||
|
||||
virtual void Load(System &system);
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
virtual void Load(System &system);
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state,
|
||||
const ManagerBase &mgr,
|
||||
const InputType &input,
|
||||
const Hypothesis &hypo) const;
|
||||
virtual FFState* BlankState(MemPool &pool) const;
|
||||
virtual void EmptyHypothesisState(FFState &state, const ManagerBase &mgr,
|
||||
const InputType &input, const Hypothesis &hypo) const;
|
||||
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool,
|
||||
const System &system,
|
||||
const Phrase &source, const TargetPhrase &targetPhrase,
|
||||
Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
virtual void
|
||||
EvaluateInIsolation(MemPool &pool, const System &system, const Phrase &source,
|
||||
const TargetPhrase &targetPhrase, Scores &scores,
|
||||
SCORE *estimatedScore) const;
|
||||
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo,
|
||||
const FFState &prevState,
|
||||
Scores &scores,
|
||||
FFState &state) const;
|
||||
virtual void EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
const Hypothesis &hypo, const FFState &prevState, Scores &scores,
|
||||
FFState &state) const;
|
||||
|
||||
protected:
|
||||
FactorType m_factorType;
|
||||
|
||||
std::string m_filePath;
|
||||
size_t m_nGramOrder; //! max n-gram length contained in this LM
|
||||
size_t m_ContextSize;
|
||||
std::string m_filePath;
|
||||
size_t m_nGramOrder; //! max n-gram length contained in this LM
|
||||
size_t m_ContextSize;
|
||||
|
||||
DALM::Logger *m_logger;
|
||||
DALM::Vocabulary *m_vocab;
|
||||
|
@ -21,33 +21,33 @@ using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
ManagerBase::ManagerBase(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId)
|
||||
:system(sys)
|
||||
,task(task)
|
||||
,m_inputStr(inputStr)
|
||||
,m_translationId(translationId)
|
||||
{}
|
||||
ManagerBase::ManagerBase(System &sys, const TranslationTask &task,
|
||||
const std::string &inputStr, long translationId) :
|
||||
system(sys), task(task), m_inputStr(inputStr), m_translationId(
|
||||
translationId)
|
||||
{
|
||||
}
|
||||
|
||||
ManagerBase::~ManagerBase()
|
||||
{
|
||||
GetPool().Reset();
|
||||
GetHypoRecycle().Clear();
|
||||
GetPool().Reset();
|
||||
GetHypoRecycle().Clear();
|
||||
}
|
||||
|
||||
void ManagerBase::InitPools()
|
||||
{
|
||||
m_pool = &system.GetManagerPool();
|
||||
m_systemPool = &system.GetSystemPool();
|
||||
m_hypoRecycle = &system.GetHypoRecycler();
|
||||
m_pool = &system.GetManagerPool();
|
||||
m_systemPool = &system.GetSystemPool();
|
||||
m_hypoRecycle = &system.GetHypoRecycler();
|
||||
}
|
||||
|
||||
void ManagerBase::ParseInput(bool addBOSEOS)
|
||||
{
|
||||
FactorCollection &vocab = system.GetVocab();
|
||||
FactorCollection &vocab = system.GetVocab();
|
||||
|
||||
m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr, m_translationId, addBOSEOS);
|
||||
m_input = Sentence::CreateFromString(GetPool(), vocab, system, m_inputStr,
|
||||
m_translationId, addBOSEOS);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -33,41 +33,47 @@ class HypothesisBase;
|
||||
class ManagerBase
|
||||
{
|
||||
public:
|
||||
const System &system;
|
||||
const TranslationTask &task;
|
||||
const System &system;
|
||||
const TranslationTask &task;
|
||||
ArcLists arcLists;
|
||||
|
||||
ManagerBase(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId);
|
||||
virtual ~ManagerBase();
|
||||
virtual void Decode() = 0;
|
||||
ManagerBase(System &sys, const TranslationTask &task,
|
||||
const std::string &inputStr, long translationId);
|
||||
virtual ~ManagerBase();
|
||||
virtual void Decode() = 0;
|
||||
|
||||
MemPool &GetPool() const
|
||||
{ return *m_pool; }
|
||||
MemPool &GetPool() const
|
||||
{
|
||||
return *m_pool;
|
||||
}
|
||||
|
||||
MemPool &GetSystemPool() const
|
||||
{ return *m_systemPool; }
|
||||
MemPool &GetSystemPool() const
|
||||
{
|
||||
return *m_systemPool;
|
||||
}
|
||||
|
||||
Recycler<HypothesisBase*> &GetHypoRecycle() const
|
||||
{
|
||||
return *m_hypoRecycle;
|
||||
}
|
||||
Recycler<HypothesisBase*> &GetHypoRecycle() const
|
||||
{
|
||||
return *m_hypoRecycle;
|
||||
}
|
||||
|
||||
const Sentence &GetInput() const
|
||||
{ return *m_input; }
|
||||
const Sentence &GetInput() const
|
||||
{
|
||||
return *m_input;
|
||||
}
|
||||
|
||||
protected:
|
||||
std::string m_inputStr;
|
||||
long m_translationId;
|
||||
Sentence *m_input;
|
||||
std::string m_inputStr;
|
||||
long m_translationId;
|
||||
Sentence *m_input;
|
||||
|
||||
mutable MemPool *m_pool, *m_systemPool;
|
||||
mutable Recycler<HypothesisBase*> *m_hypoRecycle;
|
||||
mutable MemPool *m_pool, *m_systemPool;
|
||||
mutable Recycler<HypothesisBase*> *m_hypoRecycle;
|
||||
|
||||
void InitPools();
|
||||
void ParseInput(bool addBOSEOS);
|
||||
void InitPools();
|
||||
void ParseInput(bool addBOSEOS);
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -15,8 +15,8 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
MemPool::Page::Page(std::size_t vSize)
|
||||
:size(vSize)
|
||||
MemPool::Page::Page(std::size_t vSize) :
|
||||
size(vSize)
|
||||
{
|
||||
mem = (uint8_t*) util::MallocOrThrow(size);
|
||||
end = mem + size;
|
||||
@ -24,18 +24,17 @@ MemPool::Page::Page(std::size_t vSize)
|
||||
|
||||
MemPool::Page::~Page()
|
||||
{
|
||||
free(mem);
|
||||
free(mem);
|
||||
}
|
||||
////////////////////////////////////////////////////
|
||||
MemPool::MemPool(size_t initSize)
|
||||
:m_currSize(initSize)
|
||||
,m_currPage(0)
|
||||
MemPool::MemPool(size_t initSize) :
|
||||
m_currSize(initSize), m_currPage(0)
|
||||
{
|
||||
Page *page = new Page(m_currSize);
|
||||
m_pages.push_back(page);
|
||||
Page *page = new Page(m_currSize);
|
||||
m_pages.push_back(page);
|
||||
|
||||
current_ = page->mem;
|
||||
//cerr << "new memory pool";
|
||||
current_ = page->mem;
|
||||
//cerr << "new memory pool";
|
||||
}
|
||||
|
||||
MemPool::~MemPool()
|
||||
@ -44,41 +43,40 @@ MemPool::~MemPool()
|
||||
RemoveAllInColl(m_pages);
|
||||
}
|
||||
|
||||
|
||||
uint8_t *MemPool::More(std::size_t size)
|
||||
{
|
||||
++m_currPage;
|
||||
if (m_currPage >= m_pages.size()) {
|
||||
// add new page
|
||||
m_currSize <<= 1;
|
||||
std::size_t amount = std::max(m_currSize, size);
|
||||
++m_currPage;
|
||||
if (m_currPage >= m_pages.size()) {
|
||||
// add new page
|
||||
m_currSize <<= 1;
|
||||
std::size_t amount = std::max(m_currSize, size);
|
||||
|
||||
Page *page = new Page(amount);
|
||||
m_pages.push_back(page);
|
||||
Page *page = new Page(amount);
|
||||
m_pages.push_back(page);
|
||||
|
||||
uint8_t *ret = page->mem;
|
||||
current_ = ret + size;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
// use existing page
|
||||
Page &page = *m_pages[m_currPage];
|
||||
if (size <= page.size) {
|
||||
uint8_t *ret = page.mem;
|
||||
current_ = ret + size;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
// recursive call More()
|
||||
return More(size);
|
||||
}
|
||||
}
|
||||
uint8_t *ret = page->mem;
|
||||
current_ = ret + size;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
// use existing page
|
||||
Page &page = *m_pages[m_currPage];
|
||||
if (size <= page.size) {
|
||||
uint8_t *ret = page.mem;
|
||||
current_ = ret + size;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
// recursive call More()
|
||||
return More(size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void MemPool::Reset()
|
||||
{
|
||||
m_currPage = 0;
|
||||
current_ = m_pages[0]->mem;
|
||||
m_currPage = 0;
|
||||
current_ = m_pages[0]->mem;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -18,197 +18,218 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class MemPool {
|
||||
struct Page {
|
||||
uint8_t *mem;
|
||||
uint8_t *end;
|
||||
size_t size;
|
||||
|
||||
Page() {}
|
||||
Page(std::size_t size);
|
||||
~Page();
|
||||
};
|
||||
|
||||
public:
|
||||
MemPool(std::size_t initSize = 10000);
|
||||
|
||||
~MemPool();
|
||||
|
||||
uint8_t *Allocate(std::size_t size) {
|
||||
uint8_t *ret = current_;
|
||||
current_ += size;
|
||||
|
||||
Page &page = *m_pages[m_currPage];
|
||||
if (current_ <= page.end) {
|
||||
// return what we got
|
||||
} else {
|
||||
ret = More(size);
|
||||
}
|
||||
return ret;
|
||||
class MemPool
|
||||
{
|
||||
struct Page
|
||||
{
|
||||
uint8_t *mem;
|
||||
uint8_t *end;
|
||||
size_t size;
|
||||
|
||||
Page()
|
||||
{
|
||||
}
|
||||
Page(std::size_t size);
|
||||
~Page();
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
T *Allocate() {
|
||||
uint8_t *ret = Allocate(sizeof(T));
|
||||
return (T*) ret;
|
||||
public:
|
||||
MemPool(std::size_t initSize = 10000);
|
||||
|
||||
~MemPool();
|
||||
|
||||
uint8_t *Allocate(std::size_t size)
|
||||
{
|
||||
uint8_t *ret = current_;
|
||||
current_ += size;
|
||||
|
||||
Page &page = *m_pages[m_currPage];
|
||||
if (current_ <= page.end) {
|
||||
// return what we got
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
T *Allocate(size_t num) {
|
||||
uint8_t *ret = Allocate(sizeof(T) * num);
|
||||
return (T*) ret;
|
||||
else {
|
||||
ret = More(size);
|
||||
}
|
||||
return ret;
|
||||
|
||||
void Reset();
|
||||
}
|
||||
|
||||
private:
|
||||
uint8_t *More(std::size_t size);
|
||||
template<typename T>
|
||||
T *Allocate()
|
||||
{
|
||||
uint8_t *ret = Allocate(sizeof(T));
|
||||
return (T*) ret;
|
||||
}
|
||||
|
||||
std::vector<Page*> m_pages;
|
||||
template<typename T>
|
||||
T *Allocate(size_t num)
|
||||
{
|
||||
uint8_t *ret = Allocate(sizeof(T) * num);
|
||||
return (T*) ret;
|
||||
}
|
||||
|
||||
size_t m_currSize;
|
||||
size_t m_currPage;
|
||||
uint8_t *current_;
|
||||
void Reset();
|
||||
|
||||
// no copying
|
||||
MemPool(const MemPool &);
|
||||
MemPool &operator=(const MemPool &);
|
||||
private:
|
||||
uint8_t *More(std::size_t size);
|
||||
|
||||
std::vector<Page*> m_pages;
|
||||
|
||||
size_t m_currSize;
|
||||
size_t m_currPage;
|
||||
uint8_t *current_;
|
||||
|
||||
// no copying
|
||||
MemPool(const MemPool &);
|
||||
MemPool &operator=(const MemPool &);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
class ObjectPoolContiguous {
|
||||
template<typename T>
|
||||
class ObjectPoolContiguous
|
||||
{
|
||||
|
||||
public:
|
||||
ObjectPoolContiguous(std::size_t initSize = 100000)
|
||||
:m_size(0)
|
||||
,m_actualSize(initSize)
|
||||
{
|
||||
m_vec = (T*) malloc(sizeof(T) * initSize);
|
||||
}
|
||||
public:
|
||||
ObjectPoolContiguous(std::size_t initSize = 100000) :
|
||||
m_size(0), m_actualSize(initSize)
|
||||
{
|
||||
m_vec = (T*) malloc(sizeof(T) * initSize);
|
||||
}
|
||||
|
||||
~ObjectPoolContiguous()
|
||||
{
|
||||
free(m_vec);
|
||||
}
|
||||
|
||||
void Add(T &obj)
|
||||
{
|
||||
if (m_size >= m_actualSize) {
|
||||
//std::cerr << std::endl << "MORE " << m_size << std::endl;
|
||||
m_actualSize *= 2;
|
||||
m_vec = (T*) realloc(m_vec, sizeof(T) * m_actualSize);
|
||||
|
||||
~ObjectPoolContiguous()
|
||||
{
|
||||
free (m_vec);
|
||||
}
|
||||
m_vec[m_size] = obj;
|
||||
++m_size;
|
||||
}
|
||||
|
||||
void Add(T &obj) {
|
||||
if (m_size >= m_actualSize) {
|
||||
//std::cerr << std::endl << "MORE " << m_size << std::endl;
|
||||
m_actualSize *= 2;
|
||||
m_vec = (T*) realloc(m_vec, sizeof(T) * m_actualSize);
|
||||
|
||||
}
|
||||
m_vec[m_size] = obj;
|
||||
++m_size;
|
||||
}
|
||||
bool IsEmpty() const
|
||||
{
|
||||
return m_size == 0;
|
||||
}
|
||||
|
||||
bool IsEmpty() const
|
||||
{ return m_size == 0; }
|
||||
void Reset()
|
||||
{
|
||||
m_size = 0;
|
||||
}
|
||||
|
||||
void Reset()
|
||||
{
|
||||
m_size = 0;
|
||||
}
|
||||
// vector op
|
||||
size_t GetSize() const
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
// vector op
|
||||
size_t GetSize() const
|
||||
{ return m_size; }
|
||||
const T& operator[](size_t ind) const
|
||||
{
|
||||
return m_vec[ind];
|
||||
}
|
||||
|
||||
// stack op
|
||||
const T &Get() const
|
||||
{
|
||||
return m_vec[m_size - 1];
|
||||
}
|
||||
|
||||
const T& operator[](size_t ind) const {
|
||||
return m_vec[ind];
|
||||
}
|
||||
void Pop()
|
||||
{
|
||||
--m_size;
|
||||
}
|
||||
|
||||
// stack op
|
||||
const T &Get() const {
|
||||
return m_vec[m_size - 1];
|
||||
}
|
||||
T *GetData()
|
||||
{
|
||||
return m_vec;
|
||||
}
|
||||
|
||||
void Pop()
|
||||
{
|
||||
--m_size;
|
||||
}
|
||||
template<typename ORDERER>
|
||||
void Sort(const ORDERER &orderer)
|
||||
{
|
||||
std::sort(m_vec, m_vec + m_size, orderer);
|
||||
}
|
||||
|
||||
T *GetData()
|
||||
{
|
||||
return m_vec;
|
||||
}
|
||||
private:
|
||||
T *m_vec;
|
||||
size_t m_size, m_actualSize;
|
||||
|
||||
template<typename ORDERER>
|
||||
void Sort(const ORDERER &orderer)
|
||||
{
|
||||
std::sort(m_vec, m_vec + m_size, orderer);
|
||||
}
|
||||
|
||||
private:
|
||||
T *m_vec;
|
||||
size_t m_size, m_actualSize;
|
||||
|
||||
// no copying
|
||||
ObjectPoolContiguous(const ObjectPoolContiguous &);
|
||||
ObjectPoolContiguous &operator=(const ObjectPoolContiguous &);
|
||||
// no copying
|
||||
ObjectPoolContiguous(const ObjectPoolContiguous &);
|
||||
ObjectPoolContiguous &operator=(const ObjectPoolContiguous &);
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename T>
|
||||
template<typename T>
|
||||
class MemPoolAllocator
|
||||
{
|
||||
public:
|
||||
typedef T value_type;
|
||||
typedef T* pointer;
|
||||
typedef const T* const_pointer;
|
||||
typedef T& reference;
|
||||
typedef const T& const_reference;
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef T value_type;
|
||||
typedef T* pointer;
|
||||
typedef const T* const_pointer;
|
||||
typedef T& reference;
|
||||
typedef const T& const_reference;
|
||||
typedef std::size_t size_type;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
|
||||
template< class U >
|
||||
struct rebind { typedef MemPoolAllocator<U> other; };
|
||||
template<class U>
|
||||
struct rebind
|
||||
{
|
||||
typedef MemPoolAllocator<U> other;
|
||||
};
|
||||
|
||||
MemPoolAllocator(Moses2::MemPool &pool)
|
||||
:m_pool(pool)
|
||||
{}
|
||||
MemPoolAllocator(const MemPoolAllocator &other)
|
||||
:m_pool(other.m_pool)
|
||||
{}
|
||||
MemPoolAllocator(Moses2::MemPool &pool) :
|
||||
m_pool(pool)
|
||||
{
|
||||
}
|
||||
MemPoolAllocator(const MemPoolAllocator &other) :
|
||||
m_pool(other.m_pool)
|
||||
{
|
||||
}
|
||||
|
||||
template< class U >
|
||||
MemPoolAllocator( const MemPoolAllocator<U>& other )
|
||||
:m_pool(other.m_pool)
|
||||
{}
|
||||
template<class U>
|
||||
MemPoolAllocator(const MemPoolAllocator<U>& other) :
|
||||
m_pool(other.m_pool)
|
||||
{
|
||||
}
|
||||
|
||||
size_type max_size() const
|
||||
{ return std::numeric_limits<size_type>::max(); }
|
||||
size_type max_size() const
|
||||
{
|
||||
return std::numeric_limits<size_type>::max();
|
||||
}
|
||||
|
||||
void deallocate( pointer p, size_type n )
|
||||
{
|
||||
//std::cerr << "deallocate " << p << " " << n << std::endl;
|
||||
}
|
||||
void deallocate(pointer p, size_type n)
|
||||
{
|
||||
//std::cerr << "deallocate " << p << " " << n << std::endl;
|
||||
}
|
||||
|
||||
pointer allocate( size_type n, std::allocator<void>::const_pointer hint = 0 )
|
||||
{
|
||||
//std::cerr << "allocate " << n << " " << hint << std::endl;
|
||||
pointer ret = m_pool.Allocate<T>(n);
|
||||
return ret;
|
||||
}
|
||||
pointer allocate(size_type n, std::allocator<void>::const_pointer hint = 0)
|
||||
{
|
||||
//std::cerr << "allocate " << n << " " << hint << std::endl;
|
||||
pointer ret = m_pool.Allocate<T>(n);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void construct( pointer p, const_reference val )
|
||||
{
|
||||
//std::cerr << "construct " << p << " " << n << std::endl;
|
||||
new((void *)p) T(val);
|
||||
}
|
||||
void construct(pointer p, const_reference val)
|
||||
{
|
||||
//std::cerr << "construct " << p << " " << n << std::endl;
|
||||
new ((void *) p) T(val);
|
||||
}
|
||||
|
||||
void destroy( pointer p )
|
||||
{
|
||||
//std::cerr << "destroy " << p << " " << n << std::endl;
|
||||
}
|
||||
void destroy(pointer p)
|
||||
{
|
||||
//std::cerr << "destroy " << p << " " << n << std::endl;
|
||||
}
|
||||
|
||||
MemPool &m_pool;
|
||||
MemPool &m_pool;
|
||||
protected:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -10,88 +10,91 @@ namespace Moses2
|
||||
template<class KeyClass, class ValueClass>
|
||||
class MorphTrie
|
||||
{
|
||||
public:
|
||||
MorphTrie() {}
|
||||
Node<KeyClass, ValueClass>* insert(const std::vector<KeyClass>& word, const ValueClass& value);
|
||||
const Node<KeyClass, ValueClass>* getNode(const std::vector<KeyClass>& words) const;
|
||||
const Node<KeyClass, ValueClass> &getNode(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
|
||||
std::vector<const Node<KeyClass, ValueClass>* > getNodes(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
|
||||
private:
|
||||
Node<KeyClass, ValueClass> root;
|
||||
public:
|
||||
MorphTrie()
|
||||
{
|
||||
}
|
||||
Node<KeyClass, ValueClass>* insert(const std::vector<KeyClass>& word,
|
||||
const ValueClass& value);
|
||||
const Node<KeyClass, ValueClass>* getNode(
|
||||
const std::vector<KeyClass>& words) const;
|
||||
const Node<KeyClass, ValueClass> &getNode(const std::vector<KeyClass>& words,
|
||||
size_t &stoppedAtInd) const;
|
||||
std::vector<const Node<KeyClass, ValueClass>*> getNodes(
|
||||
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const;
|
||||
private:
|
||||
Node<KeyClass, ValueClass> root;
|
||||
};
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(const std::vector<KeyClass>& word,
|
||||
const ValueClass& value)
|
||||
Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::insert(
|
||||
const std::vector<KeyClass>& word, const ValueClass& value)
|
||||
{
|
||||
Node<KeyClass, ValueClass>* cNode = &root;
|
||||
for (size_t i = 0; i < word.size(); ++i)
|
||||
{
|
||||
KeyClass cKey = word[i];
|
||||
cNode = cNode->addSubnode(cKey);
|
||||
}
|
||||
cNode->setValue(value);
|
||||
return cNode;
|
||||
Node<KeyClass, ValueClass>* cNode = &root;
|
||||
for (size_t i = 0; i < word.size(); ++i) {
|
||||
KeyClass cKey = word[i];
|
||||
cNode = cNode->addSubnode(cKey);
|
||||
}
|
||||
cNode->setValue(value);
|
||||
return cNode;
|
||||
}
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(const std::vector<KeyClass>& words) const
|
||||
const Node<KeyClass, ValueClass>* MorphTrie<KeyClass, ValueClass>::getNode(
|
||||
const std::vector<KeyClass>& words) const
|
||||
{
|
||||
size_t stoppedAtInd;
|
||||
const Node<KeyClass, ValueClass> &ret = getNode(words, stoppedAtInd);
|
||||
if (stoppedAtInd < words.size()) {
|
||||
return NULL;
|
||||
}
|
||||
return &ret;
|
||||
size_t stoppedAtInd;
|
||||
const Node<KeyClass, ValueClass> &ret = getNode(words, stoppedAtInd);
|
||||
if (stoppedAtInd < words.size()) {
|
||||
return NULL;
|
||||
}
|
||||
return &ret;
|
||||
}
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
|
||||
const Node<KeyClass, ValueClass> &MorphTrie<KeyClass, ValueClass>::getNode(
|
||||
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
|
||||
{
|
||||
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
|
||||
for (size_t i = 0; i < words.size(); ++i)
|
||||
{
|
||||
const KeyClass &cKey = words[i];
|
||||
newNode = prevNode->findSub(cKey);
|
||||
if (newNode == NULL)
|
||||
{
|
||||
stoppedAtInd = i;
|
||||
return *prevNode;
|
||||
}
|
||||
prevNode = newNode;
|
||||
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
|
||||
for (size_t i = 0; i < words.size(); ++i) {
|
||||
const KeyClass &cKey = words[i];
|
||||
newNode = prevNode->findSub(cKey);
|
||||
if (newNode == NULL) {
|
||||
stoppedAtInd = i;
|
||||
return *prevNode;
|
||||
}
|
||||
prevNode = newNode;
|
||||
}
|
||||
|
||||
stoppedAtInd = words.size();
|
||||
return *newNode;
|
||||
stoppedAtInd = words.size();
|
||||
return *newNode;
|
||||
}
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
std::vector<const Node<KeyClass, ValueClass>* > MorphTrie<KeyClass, ValueClass>::getNodes(const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
|
||||
std::vector<const Node<KeyClass, ValueClass>*> MorphTrie<KeyClass, ValueClass>::getNodes(
|
||||
const std::vector<KeyClass>& words, size_t &stoppedAtInd) const
|
||||
{
|
||||
std::vector<const Node<KeyClass, ValueClass>* > ret;
|
||||
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
|
||||
ret.push_back(prevNode);
|
||||
std::vector<const Node<KeyClass, ValueClass>*> ret;
|
||||
const Node<KeyClass, ValueClass> *prevNode = &root, *newNode;
|
||||
ret.push_back(prevNode);
|
||||
|
||||
for (size_t i = 0; i < words.size(); ++i)
|
||||
{
|
||||
const KeyClass &cKey = words[i];
|
||||
newNode = prevNode->findSub(cKey);
|
||||
if (newNode == NULL)
|
||||
{
|
||||
stoppedAtInd = i;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
ret.push_back(newNode);
|
||||
}
|
||||
prevNode = newNode;
|
||||
for (size_t i = 0; i < words.size(); ++i) {
|
||||
const KeyClass &cKey = words[i];
|
||||
newNode = prevNode->findSub(cKey);
|
||||
if (newNode == NULL) {
|
||||
stoppedAtInd = i;
|
||||
return ret;
|
||||
}
|
||||
else {
|
||||
ret.push_back(newNode);
|
||||
}
|
||||
prevNode = newNode;
|
||||
}
|
||||
|
||||
stoppedAtInd = words.size();
|
||||
return ret;
|
||||
stoppedAtInd = words.size();
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif /* end of include guard: MORPHTRIE_H_ */
|
||||
|
@ -8,42 +8,47 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
template<class KeyClass, class ValueClass>
|
||||
class Node
|
||||
{
|
||||
public:
|
||||
Node() { }
|
||||
Node(const ValueClass& value)
|
||||
: m_value(value)
|
||||
{}
|
||||
~Node();
|
||||
void setKey(const KeyClass& key);
|
||||
void setValue(const ValueClass& value)
|
||||
{
|
||||
m_value = value;
|
||||
}
|
||||
Node* findSub(const KeyClass& key);
|
||||
const Node* findSub(const KeyClass& key) const;
|
||||
Node *addSubnode(const KeyClass& cKey)
|
||||
{
|
||||
Node *node = findSub(cKey);
|
||||
if (node) {
|
||||
return node;
|
||||
}
|
||||
else {
|
||||
node = new Node();
|
||||
subNodes[cKey] = node;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
public:
|
||||
Node()
|
||||
{
|
||||
}
|
||||
Node(const ValueClass& value) :
|
||||
m_value(value)
|
||||
{
|
||||
}
|
||||
~Node();
|
||||
void setKey(const KeyClass& key);
|
||||
void setValue(const ValueClass& value)
|
||||
{
|
||||
m_value = value;
|
||||
}
|
||||
Node* findSub(const KeyClass& key);
|
||||
const Node* findSub(const KeyClass& key) const;
|
||||
Node *addSubnode(const KeyClass& cKey)
|
||||
{
|
||||
Node *node = findSub(cKey);
|
||||
if (node) {
|
||||
return node;
|
||||
}
|
||||
else {
|
||||
node = new Node();
|
||||
subNodes[cKey] = node;
|
||||
return node;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<Node*> getSubnodes();
|
||||
const ValueClass &getValue() const
|
||||
{ return m_value; }
|
||||
std::vector<Node*> getSubnodes();
|
||||
const ValueClass &getValue() const
|
||||
{
|
||||
return m_value;
|
||||
}
|
||||
|
||||
private:
|
||||
boost::unordered_map<KeyClass, Node*> subNodes;
|
||||
ValueClass m_value;
|
||||
private:
|
||||
boost::unordered_map<KeyClass, Node*> subNodes;
|
||||
ValueClass m_value;
|
||||
|
||||
};
|
||||
|
||||
@ -52,33 +57,35 @@ Node<KeyClass, ValueClass>::~Node()
|
||||
{
|
||||
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
|
||||
for (iter = subNodes.begin(); iter != subNodes.end(); ++iter) {
|
||||
Node *node = iter->second;
|
||||
delete node;
|
||||
Node *node = iter->second;
|
||||
delete node;
|
||||
}
|
||||
}
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(const KeyClass& cKey) const
|
||||
const Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
|
||||
const KeyClass& cKey) const
|
||||
{
|
||||
typename boost::unordered_map<KeyClass, Node*>::const_iterator iter;
|
||||
iter = subNodes.find(cKey);
|
||||
if (iter != subNodes.end()) {
|
||||
Node *node = iter->second;
|
||||
return node;
|
||||
}
|
||||
return NULL;
|
||||
typename boost::unordered_map<KeyClass, Node*>::const_iterator iter;
|
||||
iter = subNodes.find(cKey);
|
||||
if (iter != subNodes.end()) {
|
||||
Node *node = iter->second;
|
||||
return node;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
template<class KeyClass, class ValueClass>
|
||||
Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(const KeyClass& cKey)
|
||||
Node<KeyClass, ValueClass>* Node<KeyClass, ValueClass>::findSub(
|
||||
const KeyClass& cKey)
|
||||
{
|
||||
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
|
||||
iter = subNodes.find(cKey);
|
||||
if (iter != subNodes.end()) {
|
||||
Node *node = iter->second;
|
||||
return node;
|
||||
}
|
||||
return NULL;
|
||||
typename boost::unordered_map<KeyClass, Node*>::iterator iter;
|
||||
iter = subNodes.find(cKey);
|
||||
if (iter != subNodes.end()) {
|
||||
Node *node = iter->second;
|
||||
return node;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -12,18 +12,18 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
inline void ParseLineByChar(string& line, char c, vector<string>& substrings) {
|
||||
size_t i = 0;
|
||||
size_t j = line.find(c);
|
||||
inline void ParseLineByChar(string& line, char c, vector<string>& substrings)
|
||||
{
|
||||
size_t i = 0;
|
||||
size_t j = line.find(c);
|
||||
|
||||
while (j != string::npos) {
|
||||
substrings.push_back(line.substr(i, j-i));
|
||||
i = ++j;
|
||||
j = line.find(c, j);
|
||||
while (j != string::npos) {
|
||||
substrings.push_back(line.substr(i, j - i));
|
||||
i = ++j;
|
||||
j = line.find(c, j);
|
||||
|
||||
if (j == string::npos)
|
||||
substrings.push_back(line.substr(i, line.length()));
|
||||
}
|
||||
if (j == string::npos) substrings.push_back(line.substr(i, line.length()));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -18,12 +18,12 @@ namespace Moses2
|
||||
|
||||
size_t Phrase::hash() const
|
||||
{
|
||||
size_t seed = 0;
|
||||
size_t seed = 0;
|
||||
|
||||
for (size_t i = 0; i < GetSize(); ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
size_t wordHash = word.hash();
|
||||
boost::hash_combine(seed, wordHash);
|
||||
const Word &word = (*this)[i];
|
||||
size_t wordHash = word.hash();
|
||||
boost::hash_combine(seed, wordHash);
|
||||
}
|
||||
|
||||
return seed;
|
||||
@ -32,15 +32,15 @@ size_t Phrase::hash() const
|
||||
bool Phrase::operator==(const Phrase &compare) const
|
||||
{
|
||||
if (GetSize() != compare.GetSize()) {
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < GetSize(); ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
const Word &otherWord = compare[i];
|
||||
if (word != otherWord) {
|
||||
return false;
|
||||
}
|
||||
const Word &word = (*this)[i];
|
||||
const Word &otherWord = compare[i];
|
||||
if (word != otherWord) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
@ -48,68 +48,70 @@ bool Phrase::operator==(const Phrase &compare) const
|
||||
|
||||
std::string Phrase::GetString(const FactorList &factorTypes) const
|
||||
{
|
||||
if (GetSize() == 0) {
|
||||
return "";
|
||||
}
|
||||
if (GetSize() == 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
std::stringstream ret;
|
||||
std::stringstream ret;
|
||||
|
||||
const Word &word = (*this)[0];
|
||||
ret << word.GetString(factorTypes);
|
||||
for (size_t i = 1; i < GetSize(); ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
ret << " " << word.GetString(factorTypes);
|
||||
}
|
||||
return ret.str();
|
||||
const Word &word = (*this)[0];
|
||||
ret << word.GetString(factorTypes);
|
||||
for (size_t i = 1; i < GetSize(); ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
ret << " " << word.GetString(factorTypes);
|
||||
}
|
||||
return ret.str();
|
||||
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const Phrase &obj)
|
||||
{
|
||||
if (obj.GetSize()) {
|
||||
out << obj[0];
|
||||
for (size_t i = 1; i < obj.GetSize(); ++i) {
|
||||
const Word &word = obj[i];
|
||||
out << " " << word;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
if (obj.GetSize()) {
|
||||
out << obj[0];
|
||||
for (size_t i = 1; i < obj.GetSize(); ++i) {
|
||||
const Word &word = obj[i];
|
||||
out << " " << word;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void Phrase::OutputToStream(std::ostream &out) const
|
||||
{
|
||||
size_t size = GetSize();
|
||||
if (size) {
|
||||
out << (*this)[0];
|
||||
for (size_t i = 1; i < size; ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
out << " " << word;
|
||||
}
|
||||
}
|
||||
size_t size = GetSize();
|
||||
if (size) {
|
||||
out << (*this)[0];
|
||||
for (size_t i = 1; i < size; ++i) {
|
||||
const Word &word = (*this)[i];
|
||||
out << " " << word;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system)
|
||||
:pt(pt)
|
||||
,scoreProperties(NULL)
|
||||
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt,
|
||||
const System &system) :
|
||||
pt(pt), scoreProperties(NULL)
|
||||
{
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
|
||||
system.featureFunctions.GetNumScores());
|
||||
}
|
||||
|
||||
SCORE TargetPhrase::GetFutureScore() const
|
||||
{ return m_scores->GetTotalScore() + m_estimatedScore; }
|
||||
{
|
||||
return m_scores->GetTotalScore() + m_estimatedScore;
|
||||
}
|
||||
|
||||
SCORE *TargetPhrase::GetScoresProperty(int propertyInd) const
|
||||
{
|
||||
return scoreProperties ? scoreProperties + propertyInd : NULL;
|
||||
return scoreProperties ? scoreProperties + propertyInd : NULL;
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TargetPhrase &obj)
|
||||
{
|
||||
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
|
||||
return out;
|
||||
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
} // namespace
|
||||
|
||||
|
@ -26,16 +26,18 @@ class System;
|
||||
|
||||
class Phrase
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const Phrase &);
|
||||
friend std::ostream& operator<<(std::ostream &, const Phrase &);
|
||||
public:
|
||||
virtual ~Phrase() {}
|
||||
virtual ~Phrase()
|
||||
{
|
||||
}
|
||||
virtual const Word& operator[](size_t pos) const = 0;
|
||||
virtual size_t GetSize() const = 0;
|
||||
virtual size_t hash() const;
|
||||
virtual bool operator==(const Phrase &compare) const;
|
||||
virtual bool operator!=(const Phrase &compare) const
|
||||
{
|
||||
return !( (*this) == compare );
|
||||
return !((*this) == compare);
|
||||
}
|
||||
virtual std::string GetString(const FactorList &factorTypes) const;
|
||||
virtual SubPhrase GetSubPhrase(size_t start, size_t size) const = 0;
|
||||
@ -44,7 +46,7 @@ public:
|
||||
|
||||
};
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
class TargetPhrase : public Phrase
|
||||
class TargetPhrase: public Phrase
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const TargetPhrase &);
|
||||
|
||||
@ -56,34 +58,41 @@ public:
|
||||
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system);
|
||||
|
||||
Scores &GetScores()
|
||||
{ return *m_scores; }
|
||||
{
|
||||
return *m_scores;
|
||||
}
|
||||
|
||||
const Scores &GetScores() const
|
||||
{ return *m_scores; }
|
||||
{
|
||||
return *m_scores;
|
||||
}
|
||||
|
||||
SCORE GetFutureScore() const;
|
||||
|
||||
void SetEstimatedScore(const SCORE &value)
|
||||
{ m_estimatedScore = value; }
|
||||
{
|
||||
m_estimatedScore = value;
|
||||
}
|
||||
|
||||
SCORE *GetScoresProperty(int propertyInd) const;
|
||||
|
||||
protected:
|
||||
Scores *m_scores;
|
||||
SCORE m_estimatedScore;
|
||||
Scores *m_scores;
|
||||
SCORE m_estimatedScore;
|
||||
|
||||
};
|
||||
|
||||
//////////////////////////////////////////
|
||||
struct CompareFutureScore {
|
||||
bool operator() (const TargetPhrase *a, const TargetPhrase *b) const
|
||||
struct CompareFutureScore
|
||||
{
|
||||
bool operator()(const TargetPhrase *a, const TargetPhrase *b) const
|
||||
{
|
||||
return a->GetFutureScore() > b->GetFutureScore();
|
||||
return a->GetFutureScore() > b->GetFutureScore();
|
||||
}
|
||||
|
||||
bool operator() (const TargetPhrase &a, const TargetPhrase &b) const
|
||||
bool operator()(const TargetPhrase &a, const TargetPhrase &b) const
|
||||
{
|
||||
return a.GetFutureScore() > b.GetFutureScore();
|
||||
return a.GetFutureScore() > b.GetFutureScore();
|
||||
}
|
||||
};
|
||||
|
||||
@ -91,18 +100,19 @@ struct CompareFutureScore {
|
||||
class PhraseOrdererLexical
|
||||
{
|
||||
public:
|
||||
bool operator()(const Phrase &a, const Phrase &b) const {
|
||||
size_t minSize = std::min(a.GetSize(), b.GetSize());
|
||||
for (size_t i = 0; i < minSize; ++i) {
|
||||
const Word &aWord = a[i];
|
||||
const Word &bWord = b[i];
|
||||
int cmp = aWord.Compare(bWord);
|
||||
//std::cerr << "WORD: " << aWord << " ||| " << bWord << " ||| " << lessThan << std::endl;
|
||||
if (cmp) {
|
||||
return (cmp < 0);
|
||||
}
|
||||
}
|
||||
return a.GetSize() < b.GetSize();
|
||||
bool operator()(const Phrase &a, const Phrase &b) const
|
||||
{
|
||||
size_t minSize = std::min(a.GetSize(), b.GetSize());
|
||||
for (size_t i = 0; i < minSize; ++i) {
|
||||
const Word &aWord = a[i];
|
||||
const Word &bWord = b[i];
|
||||
int cmp = aWord.Compare(bWord);
|
||||
//std::cerr << "WORD: " << aWord << " ||| " << bWord << " ||| " << lessThan << std::endl;
|
||||
if (cmp) {
|
||||
return (cmp < 0);
|
||||
}
|
||||
}
|
||||
return a.GetSize() < b.GetSize();
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -20,145 +20,139 @@ namespace NSCubePruningMiniStack
|
||||
{
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
QueueItem *QueueItem::Create(QueueItem *currItem,
|
||||
Manager &mgr,
|
||||
CubeEdge &edge,
|
||||
size_t hypoIndex,
|
||||
size_t tpIndex,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
QueueItem *QueueItem::Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
|
||||
size_t hypoIndex, size_t tpIndex,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
{
|
||||
QueueItem *ret;
|
||||
if (currItem) {
|
||||
// reuse incoming queue item to create new item
|
||||
ret = currItem;
|
||||
ret->Init(mgr, edge, hypoIndex, tpIndex);
|
||||
}
|
||||
else if (!queueItemRecycler.empty()) {
|
||||
// use item from recycle bin
|
||||
ret = queueItemRecycler.back();
|
||||
ret->Init(mgr, edge, hypoIndex, tpIndex);
|
||||
queueItemRecycler.pop_back();
|
||||
}
|
||||
else {
|
||||
// create new item
|
||||
ret = new (mgr.GetPool().Allocate<QueueItem>()) QueueItem(mgr, edge, hypoIndex, tpIndex);
|
||||
}
|
||||
QueueItem *ret;
|
||||
if (currItem) {
|
||||
// reuse incoming queue item to create new item
|
||||
ret = currItem;
|
||||
ret->Init(mgr, edge, hypoIndex, tpIndex);
|
||||
}
|
||||
else if (!queueItemRecycler.empty()) {
|
||||
// use item from recycle bin
|
||||
ret = queueItemRecycler.back();
|
||||
ret->Init(mgr, edge, hypoIndex, tpIndex);
|
||||
queueItemRecycler.pop_back();
|
||||
}
|
||||
else {
|
||||
// create new item
|
||||
ret = new (mgr.GetPool().Allocate<QueueItem>()) QueueItem(mgr, edge,
|
||||
hypoIndex, tpIndex);
|
||||
}
|
||||
|
||||
return ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
|
||||
:edge(&edge)
|
||||
,hypoIndex(hypoIndex)
|
||||
,tpIndex(tpIndex)
|
||||
QueueItem::QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
|
||||
size_t tpIndex) :
|
||||
edge(&edge), hypoIndex(hypoIndex), tpIndex(tpIndex)
|
||||
{
|
||||
CreateHypothesis(mgr);
|
||||
CreateHypothesis(mgr);
|
||||
}
|
||||
|
||||
void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex)
|
||||
void QueueItem::Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex,
|
||||
size_t tpIndex)
|
||||
{
|
||||
this->edge = &edge;
|
||||
this->hypoIndex = hypoIndex;
|
||||
this->tpIndex = tpIndex;
|
||||
this->edge = &edge;
|
||||
this->hypoIndex = hypoIndex;
|
||||
this->tpIndex = tpIndex;
|
||||
|
||||
CreateHypothesis(mgr);
|
||||
CreateHypothesis(mgr);
|
||||
}
|
||||
|
||||
void QueueItem::CreateHypothesis(Manager &mgr)
|
||||
{
|
||||
const Hypothesis *prevHypo = static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
|
||||
const TargetPhrase &tp = edge->tps[tpIndex];
|
||||
const Hypothesis *prevHypo =
|
||||
static_cast<const Hypothesis*>(edge->hypos[hypoIndex]);
|
||||
const TargetPhrase &tp = edge->tps[tpIndex];
|
||||
|
||||
//cerr << "hypoIndex=" << hypoIndex << endl;
|
||||
//cerr << "edge.hypos=" << edge.hypos.size() << endl;
|
||||
//cerr << prevHypo << endl;
|
||||
//cerr << *prevHypo << endl;
|
||||
//cerr << "hypoIndex=" << hypoIndex << endl;
|
||||
//cerr << "edge.hypos=" << edge.hypos.size() << endl;
|
||||
//cerr << prevHypo << endl;
|
||||
//cerr << *prevHypo << endl;
|
||||
|
||||
hypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
hypo->Init(mgr, *prevHypo, edge->path, tp, edge->newBitmap, edge->estimatedScore);
|
||||
hypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
hypo->Init(mgr, *prevHypo, edge->path, tp, edge->newBitmap,
|
||||
edge->estimatedScore);
|
||||
|
||||
if (!mgr.system.cubePruningLazyScoring) {
|
||||
hypo->EvaluateWhenApplied();
|
||||
}
|
||||
if (!mgr.system.cubePruningLazyScoring) {
|
||||
hypo->EvaluateWhenApplied();
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
CubeEdge::CubeEdge(
|
||||
Manager &mgr,
|
||||
const Hypotheses &hypos,
|
||||
const InputPath &path,
|
||||
const TargetPhrases &tps,
|
||||
const Bitmap &newBitmap)
|
||||
:hypos(hypos)
|
||||
,path(path)
|
||||
,tps(tps)
|
||||
,newBitmap(newBitmap)
|
||||
CubeEdge::CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
|
||||
const TargetPhrases &tps, const Bitmap &newBitmap) :
|
||||
hypos(hypos), path(path), tps(tps), newBitmap(newBitmap)
|
||||
{
|
||||
estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
|
||||
estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const CubeEdge &obj)
|
||||
{
|
||||
out << obj.newBitmap;
|
||||
return out;
|
||||
out << obj.newBitmap;
|
||||
return out;
|
||||
}
|
||||
|
||||
bool
|
||||
CubeEdge::SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const
|
||||
bool CubeEdge::SetSeenPosition(const size_t x, const size_t y,
|
||||
SeenPositions &seenPositions) const
|
||||
{
|
||||
//UTIL_THROW_IF2(x >= (1<<17), "Error");
|
||||
//UTIL_THROW_IF2(y >= (1<<17), "Error");
|
||||
|
||||
SeenPositionItem val(this, (x<<16) + y);
|
||||
SeenPositionItem val(this, (x << 16) + y);
|
||||
std::pair<SeenPositions::iterator, bool> pairRet = seenPositions.insert(val);
|
||||
return pairRet.second;
|
||||
}
|
||||
|
||||
void CubeEdge::CreateFirst(Manager &mgr,
|
||||
Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
void CubeEdge::CreateFirst(Manager &mgr, Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
{
|
||||
assert(hypos.size());
|
||||
assert(tps.GetSize());
|
||||
assert(hypos.size());
|
||||
assert(tps.GetSize());
|
||||
|
||||
QueueItem *item = QueueItem::Create(NULL, mgr, *this, 0, 0, queueItemRecycler);
|
||||
queue.push(item);
|
||||
bool setSeen = SetSeenPosition(0, 0, seenPositions);
|
||||
assert(setSeen);
|
||||
QueueItem *item = QueueItem::Create(NULL, mgr, *this, 0, 0,
|
||||
queueItemRecycler);
|
||||
queue.push(item);
|
||||
bool setSeen = SetSeenPosition(0, 0, seenPositions);
|
||||
assert(setSeen);
|
||||
}
|
||||
|
||||
void CubeEdge::CreateNext(Manager &mgr,
|
||||
QueueItem *item,
|
||||
Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
void CubeEdge::CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler)
|
||||
{
|
||||
size_t hypoIndex = item->hypoIndex;
|
||||
size_t tpIndex = item->tpIndex;
|
||||
size_t hypoIndex = item->hypoIndex;
|
||||
size_t tpIndex = item->tpIndex;
|
||||
|
||||
if (hypoIndex + 1 < hypos.size() && SetSeenPosition(hypoIndex + 1, tpIndex, seenPositions)) {
|
||||
// reuse incoming queue item to create new item
|
||||
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex + 1, tpIndex, queueItemRecycler);
|
||||
assert(newItem == item);
|
||||
queue.push(newItem);
|
||||
item = NULL;
|
||||
}
|
||||
if (hypoIndex + 1 < hypos.size()
|
||||
&& SetSeenPosition(hypoIndex + 1, tpIndex, seenPositions)) {
|
||||
// reuse incoming queue item to create new item
|
||||
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex + 1,
|
||||
tpIndex, queueItemRecycler);
|
||||
assert(newItem == item);
|
||||
queue.push(newItem);
|
||||
item = NULL;
|
||||
}
|
||||
|
||||
if (tpIndex + 1 < tps.GetSize() && SetSeenPosition(hypoIndex, tpIndex + 1, seenPositions)) {
|
||||
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex, tpIndex + 1, queueItemRecycler);
|
||||
queue.push(newItem);
|
||||
item = NULL;
|
||||
}
|
||||
if (tpIndex + 1 < tps.GetSize()
|
||||
&& SetSeenPosition(hypoIndex, tpIndex + 1, seenPositions)) {
|
||||
QueueItem *newItem = QueueItem::Create(item, mgr, *this, hypoIndex,
|
||||
tpIndex + 1, queueItemRecycler);
|
||||
queue.push(newItem);
|
||||
item = NULL;
|
||||
}
|
||||
|
||||
if (item) {
|
||||
// recycle unused queue item
|
||||
queueItemRecycler.push_back(item);
|
||||
}
|
||||
if (item) {
|
||||
// recycle unused queue item
|
||||
queueItemRecycler.push_back(item);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -31,33 +31,31 @@ class CubeEdge;
|
||||
///////////////////////////////////////////
|
||||
class QueueItem
|
||||
{
|
||||
~QueueItem(); // NOT IMPLEMENTED. Use MemPool
|
||||
~QueueItem(); // NOT IMPLEMENTED. Use MemPool
|
||||
public:
|
||||
static QueueItem *Create(QueueItem *currItem,
|
||||
Manager &mgr,
|
||||
CubeEdge &edge,
|
||||
size_t hypoIndex,
|
||||
size_t tpIndex,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
|
||||
static QueueItem *Create(QueueItem *currItem, Manager &mgr, CubeEdge &edge,
|
||||
size_t hypoIndex, size_t tpIndex,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
QueueItem(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
|
||||
|
||||
void Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
|
||||
void Init(Manager &mgr, CubeEdge &edge, size_t hypoIndex, size_t tpIndex);
|
||||
|
||||
CubeEdge *edge;
|
||||
size_t hypoIndex, tpIndex;
|
||||
Hypothesis *hypo;
|
||||
CubeEdge *edge;
|
||||
size_t hypoIndex, tpIndex;
|
||||
Hypothesis *hypo;
|
||||
|
||||
protected:
|
||||
void CreateHypothesis(Manager &mgr);
|
||||
void CreateHypothesis(Manager &mgr);
|
||||
};
|
||||
|
||||
///////////////////////////////////////////
|
||||
class QueueItemOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(QueueItem* itemA, QueueItem* itemB) const {
|
||||
HypothesisFutureScoreOrderer orderer;
|
||||
return !orderer(itemA->hypo, itemB->hypo);
|
||||
bool operator()(QueueItem* itemA, QueueItem* itemB) const
|
||||
{
|
||||
HypothesisFutureScoreOrderer orderer;
|
||||
return !orderer(itemA->hypo, itemB->hypo);
|
||||
}
|
||||
};
|
||||
|
||||
@ -67,39 +65,30 @@ class CubeEdge
|
||||
friend std::ostream& operator<<(std::ostream &, const CubeEdge &);
|
||||
|
||||
public:
|
||||
typedef std::priority_queue<QueueItem*,
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >,
|
||||
QueueItemOrderer> Queue;
|
||||
typedef std::priority_queue<QueueItem*,
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >, QueueItemOrderer> Queue;
|
||||
|
||||
typedef std::pair<const CubeEdge*, int> SeenPositionItem;
|
||||
typedef boost::unordered_set<SeenPositionItem,
|
||||
boost::hash<SeenPositionItem>,
|
||||
std::equal_to<SeenPositionItem>,
|
||||
MemPoolAllocator<SeenPositionItem> > SeenPositions;
|
||||
typedef std::pair<const CubeEdge*, int> SeenPositionItem;
|
||||
typedef boost::unordered_set<SeenPositionItem, boost::hash<SeenPositionItem>,
|
||||
std::equal_to<SeenPositionItem>, MemPoolAllocator<SeenPositionItem> > SeenPositions;
|
||||
|
||||
const Hypotheses &hypos;
|
||||
const InputPath &path;
|
||||
const TargetPhrases &tps;
|
||||
const Bitmap &newBitmap;
|
||||
SCORE estimatedScore;
|
||||
const Hypotheses &hypos;
|
||||
const InputPath &path;
|
||||
const TargetPhrases &tps;
|
||||
const Bitmap &newBitmap;
|
||||
SCORE estimatedScore;
|
||||
|
||||
CubeEdge(Manager &mgr,
|
||||
const Hypotheses &hypos,
|
||||
const InputPath &path,
|
||||
const TargetPhrases &tps,
|
||||
const Bitmap &newBitmap);
|
||||
CubeEdge(Manager &mgr, const Hypotheses &hypos, const InputPath &path,
|
||||
const TargetPhrases &tps, const Bitmap &newBitmap);
|
||||
|
||||
bool SetSeenPosition(const size_t x, const size_t y, SeenPositions &seenPositions) const;
|
||||
bool SetSeenPosition(const size_t x, const size_t y,
|
||||
SeenPositions &seenPositions) const;
|
||||
|
||||
void CreateFirst(Manager &mgr,
|
||||
Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
void CreateNext(Manager &mgr,
|
||||
QueueItem *item,
|
||||
Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
void CreateFirst(Manager &mgr, Queue &queue, SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
void CreateNext(Manager &mgr, QueueItem *item, Queue &queue,
|
||||
SeenPositions &seenPositions,
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > &queueItemRecycler);
|
||||
|
||||
protected:
|
||||
|
||||
@ -109,4 +98,3 @@ protected:
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -27,17 +27,17 @@ namespace NSCubePruningMiniStack
|
||||
{
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
Search::Search(Manager &mgr)
|
||||
:Moses2::Search(mgr)
|
||||
,m_stack(mgr)
|
||||
,m_cubeEdgeAlloc(mgr.GetPool())
|
||||
Search::Search(Manager &mgr) :
|
||||
Moses2::Search(mgr), m_stack(mgr), m_cubeEdgeAlloc(mgr.GetPool())
|
||||
|
||||
,m_queue(QueueItemOrderer(),
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(MemPoolAllocator<QueueItem*>(mgr.GetPool())) )
|
||||
, m_queue(QueueItemOrderer(),
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> >(
|
||||
MemPoolAllocator<QueueItem*>(mgr.GetPool())))
|
||||
|
||||
,m_seenPositions(MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
|
||||
, m_seenPositions(
|
||||
MemPoolAllocator<CubeEdge::SeenPositionItem>(mgr.GetPool()))
|
||||
|
||||
,m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
|
||||
, m_queueItemRecycler(MemPoolAllocator<QueueItem*>(mgr.GetPool()))
|
||||
|
||||
{
|
||||
}
|
||||
@ -48,109 +48,113 @@ Search::~Search()
|
||||
|
||||
void Search::Decode()
|
||||
{
|
||||
// init cue edges
|
||||
m_cubeEdges.resize(mgr.GetInput().GetSize() + 1);
|
||||
for (size_t i = 0; i < m_cubeEdges.size(); ++i) {
|
||||
m_cubeEdges[i] = new (mgr.GetPool().Allocate<CubeEdges>()) CubeEdges(m_cubeEdgeAlloc);
|
||||
}
|
||||
// init cue edges
|
||||
m_cubeEdges.resize(mgr.GetInput().GetSize() + 1);
|
||||
for (size_t i = 0; i < m_cubeEdges.size(); ++i) {
|
||||
m_cubeEdges[i] = new (mgr.GetPool().Allocate<CubeEdges>()) CubeEdges(
|
||||
m_cubeEdgeAlloc);
|
||||
}
|
||||
|
||||
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
|
||||
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap);
|
||||
initHypo->EmptyHypothesisState(mgr.GetInput());
|
||||
//cerr << "initHypo=" << *initHypo << endl;
|
||||
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
|
||||
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
|
||||
initBitmap);
|
||||
initHypo->EmptyHypothesisState(mgr.GetInput());
|
||||
//cerr << "initHypo=" << *initHypo << endl;
|
||||
|
||||
m_stack.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
PostDecode(0);
|
||||
m_stack.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
PostDecode(0);
|
||||
|
||||
for (size_t stackInd = 1; stackInd < mgr.GetInput().GetSize() + 1; ++stackInd) {
|
||||
//cerr << "stackInd=" << stackInd << endl;
|
||||
m_stack.Clear();
|
||||
Decode(stackInd);
|
||||
PostDecode(stackInd);
|
||||
for (size_t stackInd = 1; stackInd < mgr.GetInput().GetSize() + 1;
|
||||
++stackInd) {
|
||||
//cerr << "stackInd=" << stackInd << endl;
|
||||
m_stack.Clear();
|
||||
Decode(stackInd);
|
||||
PostDecode(stackInd);
|
||||
|
||||
//m_stack.DebugCounts();
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
//m_stack.DebugCounts();
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void Search::Decode(size_t stackInd)
|
||||
{
|
||||
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
|
||||
Recycler<HypothesisBase*> &hypoRecycler = mgr.GetHypoRecycle();
|
||||
|
||||
// reuse queue from previous stack. Clear it first
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> > &container = Container(m_queue);
|
||||
//cerr << "container=" << container.size() << endl;
|
||||
BOOST_FOREACH(QueueItem *item, container) {
|
||||
// recycle unused hypos from queue
|
||||
Hypothesis *hypo = item->hypo;
|
||||
hypoRecycler.Recycle(hypo);
|
||||
// reuse queue from previous stack. Clear it first
|
||||
std::vector<QueueItem*, MemPoolAllocator<QueueItem*> > &container = Container(
|
||||
m_queue);
|
||||
//cerr << "container=" << container.size() << endl;
|
||||
BOOST_FOREACH(QueueItem *item, container){
|
||||
// recycle unused hypos from queue
|
||||
Hypothesis *hypo = item->hypo;
|
||||
hypoRecycler.Recycle(hypo);
|
||||
|
||||
// recycle queue item
|
||||
m_queueItemRecycler.push_back(item);
|
||||
}
|
||||
container.clear();
|
||||
// recycle queue item
|
||||
m_queueItemRecycler.push_back(item);
|
||||
}
|
||||
container.clear();
|
||||
|
||||
m_seenPositions.clear();
|
||||
m_seenPositions.clear();
|
||||
|
||||
// add top hypo from every edge into queue
|
||||
CubeEdges &edges = *m_cubeEdges[stackInd];
|
||||
// add top hypo from every edge into queue
|
||||
CubeEdges &edges = *m_cubeEdges[stackInd];
|
||||
|
||||
BOOST_FOREACH(CubeEdge *edge, edges) {
|
||||
//cerr << *edge << " ";
|
||||
edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler);
|
||||
}
|
||||
BOOST_FOREACH(CubeEdge *edge, edges){
|
||||
//cerr << *edge << " ";
|
||||
edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler);
|
||||
}
|
||||
|
||||
/*
|
||||
cerr << "edges: ";
|
||||
boost::unordered_set<const Bitmap*> uniqueBM;
|
||||
BOOST_FOREACH(CubeEdge *edge, edges) {
|
||||
uniqueBM.insert(&edge->newBitmap);
|
||||
//cerr << *edge << " ";
|
||||
}
|
||||
cerr << edges.size() << " " << uniqueBM.size();
|
||||
cerr << endl;
|
||||
*/
|
||||
/*
|
||||
cerr << "edges: ";
|
||||
boost::unordered_set<const Bitmap*> uniqueBM;
|
||||
BOOST_FOREACH(CubeEdge *edge, edges) {
|
||||
uniqueBM.insert(&edge->newBitmap);
|
||||
//cerr << *edge << " ";
|
||||
}
|
||||
cerr << edges.size() << " " << uniqueBM.size();
|
||||
cerr << endl;
|
||||
*/
|
||||
|
||||
size_t pops = 0;
|
||||
while (!m_queue.empty() && pops < mgr.system.popLimit) {
|
||||
// get best hypo from queue, add to stack
|
||||
//cerr << "queue=" << queue.size() << endl;
|
||||
QueueItem *item = m_queue.top();
|
||||
m_queue.pop();
|
||||
size_t pops = 0;
|
||||
while (!m_queue.empty() && pops < mgr.system.popLimit) {
|
||||
// get best hypo from queue, add to stack
|
||||
//cerr << "queue=" << queue.size() << endl;
|
||||
QueueItem *item = m_queue.top();
|
||||
m_queue.pop();
|
||||
|
||||
CubeEdge *edge = item->edge;
|
||||
CubeEdge *edge = item->edge;
|
||||
|
||||
// add hypo to stack
|
||||
Hypothesis *hypo = item->hypo;
|
||||
// add hypo to stack
|
||||
Hypothesis *hypo = item->hypo;
|
||||
|
||||
if (mgr.system.cubePruningLazyScoring) {
|
||||
hypo->EvaluateWhenApplied();
|
||||
}
|
||||
if (mgr.system.cubePruningLazyScoring) {
|
||||
hypo->EvaluateWhenApplied();
|
||||
}
|
||||
|
||||
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
|
||||
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
|
||||
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
|
||||
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
|
||||
|
||||
edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler);
|
||||
edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler);
|
||||
|
||||
++pops;
|
||||
}
|
||||
++pops;
|
||||
}
|
||||
|
||||
// create hypo from every edge. Increase diversity
|
||||
if (mgr.system.cubePruningDiversity) {
|
||||
while (!m_queue.empty()) {
|
||||
QueueItem *item = m_queue.top();
|
||||
m_queue.pop();
|
||||
// create hypo from every edge. Increase diversity
|
||||
if (mgr.system.cubePruningDiversity) {
|
||||
while (!m_queue.empty()) {
|
||||
QueueItem *item = m_queue.top();
|
||||
m_queue.pop();
|
||||
|
||||
if (item->hypoIndex == 0 && item->tpIndex == 0) {
|
||||
// add hypo to stack
|
||||
Hypothesis *hypo = item->hypo;
|
||||
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
|
||||
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (item->hypoIndex == 0 && item->tpIndex == 0) {
|
||||
// add hypo to stack
|
||||
Hypothesis *hypo = item->hypo;
|
||||
//cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl;
|
||||
m_stack.Add(hypo, hypoRecycler, mgr.arcLists);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Search::PostDecode(size_t stackInd)
|
||||
@ -162,78 +166,76 @@ void Search::PostDecode(size_t stackInd)
|
||||
size_t inputSize = pathMatrix.GetRows();
|
||||
size_t numPaths = pathMatrix.GetCols();
|
||||
|
||||
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()) {
|
||||
const Bitmap &hypoBitmap = *val.first.first;
|
||||
size_t firstGap = hypoBitmap.GetFirstGapPos();
|
||||
size_t hypoEndPos = val.first.second;
|
||||
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
|
||||
BOOST_FOREACH(const Stack::Coll::value_type &val, m_stack.GetColl()){
|
||||
const Bitmap &hypoBitmap = *val.first.first;
|
||||
size_t firstGap = hypoBitmap.GetFirstGapPos();
|
||||
size_t hypoEndPos = val.first.second;
|
||||
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
|
||||
|
||||
// create edges to next hypos from existing hypos
|
||||
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
|
||||
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
|
||||
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
|
||||
// create edges to next hypos from existing hypos
|
||||
for (size_t startPos = firstGap; startPos < inputSize; ++startPos) {
|
||||
for (size_t pathInd = 0; pathInd < numPaths; ++pathInd) {
|
||||
const InputPath *path = pathMatrix.GetValue(startPos, pathInd);
|
||||
|
||||
if (path == NULL) {
|
||||
break;
|
||||
}
|
||||
if (!path->IsUsed()) {
|
||||
continue;
|
||||
}
|
||||
if (path == NULL) {
|
||||
break;
|
||||
}
|
||||
if (!path->IsUsed()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const Range &pathRange = path->range;
|
||||
//cerr << "pathRange=" << pathRange << endl;
|
||||
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
|
||||
continue;
|
||||
}
|
||||
const Range &pathRange = path->range;
|
||||
//cerr << "pathRange=" << pathRange << endl;
|
||||
if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
|
||||
size_t numWords = newBitmap.GetNumWordsCovered();
|
||||
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
|
||||
size_t numWords = newBitmap.GetNumWordsCovered();
|
||||
|
||||
CubeEdges &edges = *m_cubeEdges[numWords];
|
||||
CubeEdges &edges = *m_cubeEdges[numWords];
|
||||
|
||||
// sort hypo for a particular bitmap and hypoEndPos
|
||||
Hypotheses &sortedHypos = val.second->GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
// sort hypo for a particular bitmap and hypoEndPos
|
||||
Hypotheses &sortedHypos = val.second->GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = path->targetPhrases[i];
|
||||
if (tps && tps->GetSize()) {
|
||||
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
|
||||
edges.push_back(edge);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = path->targetPhrases[i];
|
||||
if (tps && tps->GetSize()) {
|
||||
CubeEdge *edge = new (pool.Allocate<CubeEdge>()) CubeEdge(mgr, sortedHypos, *path, *tps, newBitmap);
|
||||
edges.push_back(edge);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Hypothesis *Search::GetBestHypothesis() const
|
||||
{
|
||||
std::vector<const Hypothesis*> sortedHypos = m_stack.GetBestHypos(1);
|
||||
std::vector<const Hypothesis*> sortedHypos = m_stack.GetBestHypos(1);
|
||||
|
||||
const Hypothesis *best = NULL;
|
||||
if (sortedHypos.size()) {
|
||||
best = sortedHypos[0];
|
||||
}
|
||||
return best;
|
||||
const Hypothesis *best = NULL;
|
||||
if (sortedHypos.size()) {
|
||||
best = sortedHypos[0];
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
void Search::AddInitialTrellisPaths(TrellisPaths &paths) const
|
||||
{
|
||||
const Stack::Coll &coll = m_stack.GetColl();
|
||||
BOOST_FOREACH(const Stack::Coll::value_type &val, coll) {
|
||||
const Moses2::HypothesisColl &hypos = *val.second;
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos) {
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
|
||||
paths.Add(path);
|
||||
}
|
||||
BOOST_FOREACH(const Stack::Coll::value_type &val, coll){
|
||||
const Moses2::HypothesisColl &hypos = *val.second;
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos) {
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
|
||||
paths.Add(path);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -23,35 +23,35 @@ class TargetPhrases;
|
||||
namespace NSCubePruningMiniStack
|
||||
{
|
||||
|
||||
class Search : public Moses2::Search
|
||||
class Search: public Moses2::Search
|
||||
{
|
||||
public:
|
||||
Search(Manager &mgr);
|
||||
virtual ~Search();
|
||||
Search(Manager &mgr);
|
||||
virtual ~Search();
|
||||
|
||||
virtual void Decode();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
virtual void Decode();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
|
||||
void AddInitialTrellisPaths(TrellisPaths &paths) const;
|
||||
void AddInitialTrellisPaths(TrellisPaths &paths) const;
|
||||
|
||||
protected:
|
||||
Stack m_stack;
|
||||
Stack m_stack;
|
||||
|
||||
CubeEdge::Queue m_queue;
|
||||
CubeEdge::SeenPositions m_seenPositions;
|
||||
CubeEdge::Queue m_queue;
|
||||
CubeEdge::SeenPositions m_seenPositions;
|
||||
|
||||
// CUBE PRUNING VARIABLES
|
||||
// setup
|
||||
MemPoolAllocator<CubeEdge*> m_cubeEdgeAlloc;
|
||||
typedef std::vector<CubeEdge*, MemPoolAllocator<CubeEdge*> > CubeEdges;
|
||||
std::vector<CubeEdges*> m_cubeEdges;
|
||||
// CUBE PRUNING VARIABLES
|
||||
// setup
|
||||
MemPoolAllocator<CubeEdge*> m_cubeEdgeAlloc;
|
||||
typedef std::vector<CubeEdge*, MemPoolAllocator<CubeEdge*> > CubeEdges;
|
||||
std::vector<CubeEdges*> m_cubeEdges;
|
||||
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > m_queueItemRecycler;
|
||||
std::deque<QueueItem*, MemPoolAllocator<QueueItem*> > m_queueItemRecycler;
|
||||
|
||||
// CUBE PRUNING
|
||||
// decoding
|
||||
void Decode(size_t stackInd);
|
||||
void PostDecode(size_t stackInd);
|
||||
// CUBE PRUNING
|
||||
// decoding
|
||||
void Decode(size_t stackInd);
|
||||
void PostDecode(size_t stackInd);
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -19,120 +19,121 @@ namespace Moses2
|
||||
|
||||
namespace NSCubePruningMiniStack
|
||||
{
|
||||
Stack::Stack(const Manager &mgr)
|
||||
:m_mgr(mgr)
|
||||
,m_coll(MemPoolAllocator< std::pair<HypoCoverage, Moses2::HypothesisColl*> >(mgr.GetPool()))
|
||||
,m_miniStackRecycler(MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
|
||||
Stack::Stack(const Manager &mgr) :
|
||||
m_mgr(mgr), m_coll(
|
||||
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> >(
|
||||
mgr.GetPool())), m_miniStackRecycler(
|
||||
MemPoolAllocator<Moses2::HypothesisColl*>(mgr.GetPool()))
|
||||
{
|
||||
}
|
||||
|
||||
Stack::~Stack() {
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
|
||||
const Moses2::HypothesisColl *miniStack = val.second;
|
||||
delete miniStack;
|
||||
}
|
||||
|
||||
while (!m_miniStackRecycler.empty()) {
|
||||
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
|
||||
m_miniStackRecycler.pop_back();
|
||||
delete miniStack;
|
||||
|
||||
}
|
||||
Stack::~Stack()
|
||||
{
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll){
|
||||
const Moses2::HypothesisColl *miniStack = val.second;
|
||||
delete miniStack;
|
||||
}
|
||||
|
||||
void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists)
|
||||
while (!m_miniStackRecycler.empty()) {
|
||||
Moses2::HypothesisColl *miniStack = m_miniStackRecycler.back();
|
||||
m_miniStackRecycler.pop_back();
|
||||
delete miniStack;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists)
|
||||
{
|
||||
HypoCoverage key(&hypo->GetBitmap(), hypo->GetInputPath().range.GetEndPos());
|
||||
StackAdd added = GetMiniStack(key).Add(hypo);
|
||||
|
||||
size_t nbestSize = m_mgr.system.nbestSize;
|
||||
if (nbestSize) {
|
||||
arcLists.AddArc(added.added, hypo, added.other);
|
||||
arcLists.AddArc(added.added, hypo, added.other);
|
||||
}
|
||||
else {
|
||||
if (!added.added) {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
}
|
||||
else if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
}
|
||||
if (!added.added) {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
}
|
||||
else if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const Hypothesis*> Stack::GetBestHypos(size_t num) const
|
||||
{
|
||||
std::vector<const Hypothesis*> ret;
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
|
||||
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll){
|
||||
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
|
||||
|
||||
ret.reserve(ret.size() + hypos.size());
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
|
||||
ret.push_back(static_cast<const Hypothesis*>(hypo));
|
||||
}
|
||||
ret.reserve(ret.size() + hypos.size());
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
|
||||
ret.push_back(static_cast<const Hypothesis*>(hypo));
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<const Hypothesis*>::iterator iterMiddle;
|
||||
iterMiddle = (num == 0 || ret.size() < num)
|
||||
? ret.end()
|
||||
: ret.begin()+num;
|
||||
iterMiddle = (num == 0 || ret.size() < num) ? ret.end() : ret.begin() + num;
|
||||
|
||||
std::partial_sort(ret.begin(), iterMiddle, ret.end(),
|
||||
HypothesisFutureScoreOrderer());
|
||||
HypothesisFutureScoreOrderer());
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
size_t Stack::GetHypoSize() const
|
||||
{
|
||||
size_t ret = 0;
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
|
||||
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
|
||||
ret += hypos.size();
|
||||
}
|
||||
return ret;
|
||||
size_t ret = 0;
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll){
|
||||
const Moses2::HypothesisColl::_HCType &hypos = val.second->GetColl();
|
||||
ret += hypos.size();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
Moses2::HypothesisColl &Stack::GetMiniStack(const HypoCoverage &key)
|
||||
{
|
||||
Moses2::HypothesisColl *ret;
|
||||
Coll::iterator iter = m_coll.find(key);
|
||||
if (iter == m_coll.end()) {
|
||||
if (m_miniStackRecycler.empty()) {
|
||||
ret = new Moses2::HypothesisColl(m_mgr);
|
||||
}
|
||||
else {
|
||||
ret = m_miniStackRecycler.back();
|
||||
ret->Clear();
|
||||
m_miniStackRecycler.pop_back();
|
||||
}
|
||||
Moses2::HypothesisColl *ret;
|
||||
Coll::iterator iter = m_coll.find(key);
|
||||
if (iter == m_coll.end()) {
|
||||
if (m_miniStackRecycler.empty()) {
|
||||
ret = new Moses2::HypothesisColl(m_mgr);
|
||||
}
|
||||
else {
|
||||
ret = m_miniStackRecycler.back();
|
||||
ret->Clear();
|
||||
m_miniStackRecycler.pop_back();
|
||||
}
|
||||
|
||||
m_coll[key] = ret;
|
||||
}
|
||||
else {
|
||||
ret = iter->second;
|
||||
}
|
||||
return *ret;
|
||||
m_coll[key] = ret;
|
||||
}
|
||||
else {
|
||||
ret = iter->second;
|
||||
}
|
||||
return *ret;
|
||||
}
|
||||
|
||||
void Stack::Clear()
|
||||
{
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll) {
|
||||
Moses2::HypothesisColl *miniStack = val.second;
|
||||
m_miniStackRecycler.push_back(miniStack);
|
||||
}
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll){
|
||||
Moses2::HypothesisColl *miniStack = val.second;
|
||||
m_miniStackRecycler.push_back(miniStack);
|
||||
}
|
||||
|
||||
m_coll.clear();
|
||||
m_coll.clear();
|
||||
}
|
||||
|
||||
void Stack::DebugCounts()
|
||||
{
|
||||
cerr << "counts=";
|
||||
BOOST_FOREACH(const Coll::value_type &val, GetColl()) {
|
||||
const Moses2::HypothesisColl &miniStack = *val.second;
|
||||
size_t count = miniStack.GetColl().size();
|
||||
cerr << count << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
cerr << "counts=";
|
||||
BOOST_FOREACH(const Coll::value_type &val, GetColl()){
|
||||
const Moses2::HypothesisColl &miniStack = *val.second;
|
||||
size_t count = miniStack.GetColl().size();
|
||||
cerr << count << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -26,46 +26,47 @@ class ArcLists;
|
||||
namespace NSCubePruningMiniStack
|
||||
{
|
||||
|
||||
class Stack {
|
||||
class Stack
|
||||
{
|
||||
protected:
|
||||
|
||||
|
||||
public:
|
||||
typedef std::pair<const Bitmap*, size_t> HypoCoverage;
|
||||
// bitmap and current endPos of hypos
|
||||
// bitmap and current endPos of hypos
|
||||
|
||||
typedef boost::unordered_map<HypoCoverage, Moses2::HypothesisColl*,
|
||||
boost::hash<HypoCoverage>,
|
||||
std::equal_to<HypoCoverage>,
|
||||
MemPoolAllocator< std::pair<HypoCoverage, Moses2::HypothesisColl*> >
|
||||
> Coll;
|
||||
boost::hash<HypoCoverage>, std::equal_to<HypoCoverage>,
|
||||
MemPoolAllocator<std::pair<HypoCoverage, Moses2::HypothesisColl*> > > Coll;
|
||||
|
||||
Stack(const Manager &mgr);
|
||||
virtual ~Stack();
|
||||
|
||||
Stack(const Manager &mgr);
|
||||
virtual ~Stack();
|
||||
size_t GetHypoSize() const;
|
||||
|
||||
size_t GetHypoSize() const;
|
||||
Coll &GetColl()
|
||||
{
|
||||
return m_coll;
|
||||
}
|
||||
const Coll &GetColl() const
|
||||
{
|
||||
return m_coll;
|
||||
}
|
||||
|
||||
Coll &GetColl()
|
||||
{ return m_coll; }
|
||||
const Coll &GetColl() const
|
||||
{ return m_coll; }
|
||||
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists);
|
||||
|
||||
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists);
|
||||
Moses2::HypothesisColl &GetMiniStack(const HypoCoverage &key);
|
||||
|
||||
Moses2::HypothesisColl &GetMiniStack(const HypoCoverage &key);
|
||||
std::vector<const Hypothesis*> GetBestHypos(size_t num) const;
|
||||
void Clear();
|
||||
|
||||
std::vector<const Hypothesis*> GetBestHypos(size_t num) const;
|
||||
void Clear();
|
||||
|
||||
void DebugCounts();
|
||||
void DebugCounts();
|
||||
|
||||
protected:
|
||||
const Manager &m_mgr;
|
||||
Coll m_coll;
|
||||
|
||||
std::deque<Moses2::HypothesisColl*, MemPoolAllocator<Moses2::HypothesisColl*> > m_miniStackRecycler;
|
||||
const Manager &m_mgr;
|
||||
Coll m_coll;
|
||||
|
||||
std::deque<Moses2::HypothesisColl*, MemPoolAllocator<Moses2::HypothesisColl*> > m_miniStackRecycler;
|
||||
|
||||
};
|
||||
|
||||
@ -73,4 +74,3 @@ protected:
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -23,66 +23,67 @@ namespace Moses2
|
||||
Hypothesis *Hypothesis::Create(MemPool &pool, Manager &mgr)
|
||||
{
|
||||
// ++g_numHypos;
|
||||
Hypothesis *ret;
|
||||
Hypothesis *ret;
|
||||
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
ret = static_cast<Hypothesis*>(recycler.Get());
|
||||
if (ret) {
|
||||
// got new hypo from recycler. Do nothing
|
||||
}
|
||||
else {
|
||||
ret = new (pool.Allocate<Hypothesis>()) Hypothesis(pool, mgr.system);
|
||||
//cerr << "Hypothesis=" << sizeof(Hypothesis) << " " << ret << endl;
|
||||
recycler.Keep(ret);
|
||||
}
|
||||
return ret;
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
ret = static_cast<Hypothesis*>(recycler.Get());
|
||||
if (ret) {
|
||||
// got new hypo from recycler. Do nothing
|
||||
}
|
||||
else {
|
||||
ret = new (pool.Allocate<Hypothesis>()) Hypothesis(pool, mgr.system);
|
||||
//cerr << "Hypothesis=" << sizeof(Hypothesis) << " " << ret << endl;
|
||||
recycler.Keep(ret);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
Hypothesis::Hypothesis(MemPool &pool, const System &system)
|
||||
:HypothesisBase(pool, system)
|
||||
,m_currTargetWordsRange()
|
||||
Hypothesis::Hypothesis(MemPool &pool, const System &system) :
|
||||
HypothesisBase(pool, system), m_currTargetWordsRange()
|
||||
{
|
||||
}
|
||||
|
||||
Hypothesis::~Hypothesis() {
|
||||
// TODO Auto-generated destructor stub
|
||||
Hypothesis::~Hypothesis()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void Hypothesis::Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap)
|
||||
void Hypothesis::Init(Manager &mgr, const InputPathBase &path,
|
||||
const TargetPhrase &tp, const Bitmap &bitmap)
|
||||
{
|
||||
m_mgr = &mgr;
|
||||
m_targetPhrase = &tp;
|
||||
m_sourceCompleted = &bitmap;
|
||||
m_path = &path;
|
||||
m_prevHypo = NULL;
|
||||
m_mgr = &mgr;
|
||||
m_targetPhrase = &tp;
|
||||
m_sourceCompleted = &bitmap;
|
||||
m_path = &path;
|
||||
m_prevHypo = NULL;
|
||||
|
||||
m_currTargetWordsRange.SetStartPos(NOT_FOUND);
|
||||
m_currTargetWordsRange.SetEndPos(NOT_FOUND);
|
||||
m_currTargetWordsRange.SetStartPos(NOT_FOUND);
|
||||
m_currTargetWordsRange.SetEndPos(NOT_FOUND);
|
||||
|
||||
m_estimatedScore = 0;
|
||||
m_scores->Reset(mgr.system);
|
||||
m_estimatedScore = 0;
|
||||
m_scores->Reset(mgr.system);
|
||||
}
|
||||
|
||||
void Hypothesis::Init(Manager &mgr, const Hypothesis &prevHypo,
|
||||
const InputPathBase &path,
|
||||
const TargetPhrase &tp,
|
||||
const Bitmap &bitmap,
|
||||
SCORE estimatedScore)
|
||||
const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap,
|
||||
SCORE estimatedScore)
|
||||
{
|
||||
m_mgr = &mgr;
|
||||
m_targetPhrase = &tp;
|
||||
m_sourceCompleted = &bitmap;
|
||||
m_path = &path;
|
||||
m_prevHypo = &prevHypo;
|
||||
m_mgr = &mgr;
|
||||
m_targetPhrase = &tp;
|
||||
m_sourceCompleted = &bitmap;
|
||||
m_path = &path;
|
||||
m_prevHypo = &prevHypo;
|
||||
|
||||
m_currTargetWordsRange.SetStartPos(prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
|
||||
m_currTargetWordsRange.SetEndPos(prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
|
||||
m_currTargetWordsRange.SetStartPos(
|
||||
prevHypo.m_currTargetWordsRange.GetEndPos() + 1);
|
||||
m_currTargetWordsRange.SetEndPos(
|
||||
prevHypo.m_currTargetWordsRange.GetEndPos() + tp.GetSize());
|
||||
|
||||
m_estimatedScore = estimatedScore;
|
||||
m_estimatedScore = estimatedScore;
|
||||
|
||||
m_scores->Reset(mgr.system);
|
||||
m_scores->PlusEquals(mgr.system, prevHypo.GetScores());
|
||||
m_scores->PlusEquals(mgr.system, GetTargetPhrase().GetScores());
|
||||
m_scores->Reset(mgr.system);
|
||||
m_scores->PlusEquals(mgr.system, prevHypo.GetScores());
|
||||
m_scores->PlusEquals(mgr.system, GetTargetPhrase().GetScores());
|
||||
}
|
||||
|
||||
size_t Hypothesis::hash() const
|
||||
@ -98,7 +99,7 @@ bool Hypothesis::operator==(const Hypothesis &other) const
|
||||
{
|
||||
// coverage
|
||||
if (m_sourceCompleted != other.m_sourceCompleted) {
|
||||
return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ret = HypothesisBase::operator ==(other);
|
||||
@ -108,87 +109,92 @@ bool Hypothesis::operator==(const Hypothesis &other) const
|
||||
void Hypothesis::OutputToStream(std::ostream &out) const
|
||||
{
|
||||
if (m_prevHypo) {
|
||||
m_prevHypo->OutputToStream(out);
|
||||
m_prevHypo->OutputToStream(out);
|
||||
}
|
||||
//cerr << *this << endl;
|
||||
|
||||
if (GetTargetPhrase().GetSize()) {
|
||||
const Phrase &phrase = GetTargetPhrase();
|
||||
out << phrase << " ";
|
||||
const Phrase &phrase = GetTargetPhrase();
|
||||
out << phrase << " ";
|
||||
}
|
||||
|
||||
if (m_path->range.GetStartPos() != NOT_FOUND) {
|
||||
if (m_mgr->system.reportSegmentation == 1) {
|
||||
// just report phrase segmentation
|
||||
out << "|" << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << "| ";
|
||||
}
|
||||
else if (m_mgr->system.reportSegmentation == 2) {
|
||||
// more detailed info about every segment
|
||||
out << "|";
|
||||
if (m_mgr->system.reportSegmentation == 1) {
|
||||
// just report phrase segmentation
|
||||
out << "|" << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << "| ";
|
||||
}
|
||||
else if (m_mgr->system.reportSegmentation == 2) {
|
||||
// more detailed info about every segment
|
||||
out << "|";
|
||||
|
||||
// phrase segmentation
|
||||
out << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << ",";
|
||||
// phrase segmentation
|
||||
out << m_path->range.GetStartPos() << "-" << m_path->range.GetEndPos() << ",";
|
||||
|
||||
// score breakdown
|
||||
m_scores->OutputBreakdownToStream(out, m_mgr->system);
|
||||
// score breakdown
|
||||
m_scores->OutputBreakdownToStream(out, m_mgr->system);
|
||||
|
||||
out << "| ";
|
||||
}
|
||||
out << "| ";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const Hypothesis &obj)
|
||||
{
|
||||
// coverage
|
||||
out << obj.GetBitmap() << " " << obj.GetInputPath().range << " ";
|
||||
// coverage
|
||||
out << obj.GetBitmap() << " " << obj.GetInputPath().range << " ";
|
||||
|
||||
// states
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs = obj.GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
size_t numStatefulFFs = sfffs.size();
|
||||
for (size_t i = 0; i < numStatefulFFs; ++i) {
|
||||
const FFState &state = *obj.GetState(i);
|
||||
out << "(" << state << ") ";
|
||||
}
|
||||
// states
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs =
|
||||
obj.GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
size_t numStatefulFFs = sfffs.size();
|
||||
for (size_t i = 0; i < numStatefulFFs; ++i) {
|
||||
const FFState &state = *obj.GetState(i);
|
||||
out << "(" << state << ") ";
|
||||
}
|
||||
|
||||
// string
|
||||
obj.OutputToStream(out);
|
||||
out << " ";
|
||||
out << "fc=" << obj.GetFutureScore() << " ";
|
||||
obj.GetScores().Debug(out, obj.GetManager().system);
|
||||
return out;
|
||||
// string
|
||||
obj.OutputToStream(out);
|
||||
out << " ";
|
||||
out << "fc=" << obj.GetFutureScore() << " ";
|
||||
obj.GetScores().Debug(out, obj.GetManager().system);
|
||||
return out;
|
||||
}
|
||||
|
||||
void Hypothesis::EmptyHypothesisState(const InputType &input)
|
||||
{
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
|
||||
size_t statefulInd = sfff->GetStatefulInd();
|
||||
FFState *state = m_ffStates[statefulInd];
|
||||
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
|
||||
}
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs =
|
||||
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
|
||||
size_t statefulInd = sfff->GetStatefulInd();
|
||||
FFState *state = m_ffStates[statefulInd];
|
||||
sfff->EmptyHypothesisState(*state, GetManager(), input, *this);
|
||||
}
|
||||
}
|
||||
|
||||
void Hypothesis::EvaluateWhenApplied()
|
||||
{
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
|
||||
EvaluateWhenApplied(*sfff);
|
||||
}
|
||||
//cerr << *this << endl;
|
||||
const std::vector<const StatefulFeatureFunction*> &sfffs =
|
||||
GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
|
||||
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs){
|
||||
EvaluateWhenApplied(*sfff);
|
||||
}
|
||||
//cerr << *this << endl;
|
||||
}
|
||||
|
||||
void Hypothesis::EvaluateWhenApplied(const StatefulFeatureFunction &sfff)
|
||||
{
|
||||
size_t statefulInd = sfff.GetStatefulInd();
|
||||
const FFState *prevState = m_prevHypo->GetState(statefulInd);
|
||||
FFState *thisState = m_ffStates[statefulInd];
|
||||
assert(prevState);
|
||||
sfff.EvaluateWhenApplied(GetManager(), *this, *prevState, *m_scores, *thisState);
|
||||
size_t statefulInd = sfff.GetStatefulInd();
|
||||
const FFState *prevState = m_prevHypo->GetState(statefulInd);
|
||||
FFState *thisState = m_ffStates[statefulInd];
|
||||
assert(prevState);
|
||||
sfff.EvaluateWhenApplied(GetManager(), *this, *prevState, *m_scores,
|
||||
*thisState);
|
||||
|
||||
}
|
||||
|
||||
/** recursive - pos is relative from start of sentence */
|
||||
const Word &Hypothesis::GetWord(size_t pos) const {
|
||||
const Word &Hypothesis::GetWord(size_t pos) const
|
||||
{
|
||||
const Hypothesis *hypo = this;
|
||||
while (pos < hypo->GetCurrTargetWordsRange().GetStartPos()) {
|
||||
hypo = hypo->GetPrevHypo();
|
||||
@ -199,15 +205,15 @@ const Word &Hypothesis::GetWord(size_t pos) const {
|
||||
|
||||
void Hypothesis::Swap(Hypothesis &other)
|
||||
{
|
||||
/*
|
||||
Swap(m_targetPhrase, other.m_targetPhrase);
|
||||
Swap(m_sourceCompleted, other.m_sourceCompleted);
|
||||
Swap(m_range, other.m_range);
|
||||
Swap(m_prevHypo, other.m_prevHypo);
|
||||
Swap(m_ffStates, other.m_ffStates);
|
||||
Swap(m_estimatedScore, other.m_estimatedScore);
|
||||
Swap(m_currTargetWordsRange, other.m_currTargetWordsRange);
|
||||
*/
|
||||
/*
|
||||
Swap(m_targetPhrase, other.m_targetPhrase);
|
||||
Swap(m_sourceCompleted, other.m_sourceCompleted);
|
||||
Swap(m_range, other.m_range);
|
||||
Swap(m_prevHypo, other.m_prevHypo);
|
||||
Swap(m_ffStates, other.m_ffStates);
|
||||
Swap(m_estimatedScore, other.m_estimatedScore);
|
||||
Swap(m_currTargetWordsRange, other.m_currTargetWordsRange);
|
||||
*/
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -22,11 +22,11 @@ class Manager;
|
||||
class InputType;
|
||||
class StatefulFeatureFunction;
|
||||
|
||||
class Hypothesis : public HypothesisBase
|
||||
class Hypothesis: public HypothesisBase
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const Hypothesis &);
|
||||
friend std::ostream& operator<<(std::ostream &, const Hypothesis &);
|
||||
|
||||
Hypothesis(MemPool &pool, const System &system);
|
||||
Hypothesis(MemPool &pool, const System &system);
|
||||
|
||||
public:
|
||||
|
||||
@ -34,32 +34,39 @@ public:
|
||||
virtual ~Hypothesis();
|
||||
|
||||
// initial, empty hypo
|
||||
void Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp, const Bitmap &bitmap);
|
||||
void Init(Manager &mgr, const InputPathBase &path, const TargetPhrase &tp,
|
||||
const Bitmap &bitmap);
|
||||
|
||||
void Init(Manager &mgr, const Hypothesis &prevHypo,
|
||||
const InputPathBase &path,
|
||||
const TargetPhrase &tp,
|
||||
const Bitmap &bitmap,
|
||||
SCORE estimatedScore);
|
||||
void Init(Manager &mgr, const Hypothesis &prevHypo, const InputPathBase &path,
|
||||
const TargetPhrase &tp, const Bitmap &bitmap, SCORE estimatedScore);
|
||||
|
||||
size_t hash() const;
|
||||
bool operator==(const Hypothesis &other) const;
|
||||
|
||||
inline const Bitmap &GetBitmap() const
|
||||
{ return *m_sourceCompleted; }
|
||||
{
|
||||
return *m_sourceCompleted;
|
||||
}
|
||||
|
||||
inline const InputPathBase &GetInputPath() const
|
||||
{ return *m_path; }
|
||||
{
|
||||
return *m_path;
|
||||
}
|
||||
|
||||
inline const Range &GetCurrTargetWordsRange() const {
|
||||
inline const Range &GetCurrTargetWordsRange() const
|
||||
{
|
||||
return m_currTargetWordsRange;
|
||||
}
|
||||
|
||||
SCORE GetFutureScore() const
|
||||
{ return GetScores().GetTotalScore() + m_estimatedScore; }
|
||||
{
|
||||
return GetScores().GetTotalScore() + m_estimatedScore;
|
||||
}
|
||||
|
||||
const TargetPhrase &GetTargetPhrase() const
|
||||
{ return *m_targetPhrase; }
|
||||
{
|
||||
return *m_targetPhrase;
|
||||
}
|
||||
|
||||
void OutputToStream(std::ostream &out) const;
|
||||
|
||||
@ -69,12 +76,15 @@ public:
|
||||
void EvaluateWhenApplied(const StatefulFeatureFunction &sfff);
|
||||
|
||||
const Hypothesis* GetPrevHypo() const
|
||||
{ return m_prevHypo; }
|
||||
{
|
||||
return m_prevHypo;
|
||||
}
|
||||
|
||||
/** curr - pos is relative from CURRENT hypothesis's starting index
|
||||
* (ie, start of sentence would be some negative number, which is
|
||||
* not allowed- USE WITH CAUTION) */
|
||||
inline const Word &GetCurrWord(size_t pos) const {
|
||||
inline const Word &GetCurrWord(size_t pos) const
|
||||
{
|
||||
return GetTargetPhrase()[pos];
|
||||
}
|
||||
|
||||
@ -96,14 +106,15 @@ protected:
|
||||
class HypothesisTargetPhraseOrderer
|
||||
{
|
||||
public:
|
||||
bool operator()(const Hypothesis* a, const Hypothesis* b) const {
|
||||
PhraseOrdererLexical phraseCmp;
|
||||
bool ret = phraseCmp(a->GetTargetPhrase(), b->GetTargetPhrase());
|
||||
/*
|
||||
std::cerr << (const Phrase&) a->GetTargetPhrase() << " ||| "
|
||||
<< (const Phrase&) b->GetTargetPhrase() << " ||| "
|
||||
<< ret << std::endl;
|
||||
*/
|
||||
bool operator()(const Hypothesis* a, const Hypothesis* b) const
|
||||
{
|
||||
PhraseOrdererLexical phraseCmp;
|
||||
bool ret = phraseCmp(a->GetTargetPhrase(), b->GetTargetPhrase());
|
||||
/*
|
||||
std::cerr << (const Phrase&) a->GetTargetPhrase() << " ||| "
|
||||
<< (const Phrase&) b->GetTargetPhrase() << " ||| "
|
||||
<< ret << std::endl;
|
||||
*/
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
@ -10,42 +10,40 @@
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
InputPath::InputPath(MemPool &pool,
|
||||
const SubPhrase &subPhrase,
|
||||
const Range &range,
|
||||
size_t numPt,
|
||||
const InputPath *prefixPath)
|
||||
:InputPathBase(pool, subPhrase, range, numPt, prefixPath)
|
||||
,m_isUsed(false)
|
||||
InputPath::InputPath(MemPool &pool, const SubPhrase &subPhrase,
|
||||
const Range &range, size_t numPt, const InputPath *prefixPath) :
|
||||
InputPathBase(pool, subPhrase, range, numPt, prefixPath), m_isUsed(false)
|
||||
{
|
||||
targetPhrases = pool.Allocate<const TargetPhrases*>(numPt);
|
||||
Init<const TargetPhrases*>(targetPhrases, numPt, NULL);
|
||||
}
|
||||
|
||||
InputPath::~InputPath() {
|
||||
// TODO Auto-generated destructor stub
|
||||
InputPath::~InputPath()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void InputPath::AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps)
|
||||
void InputPath::AddTargetPhrases(const PhraseTable &pt,
|
||||
const TargetPhrases *tps)
|
||||
{
|
||||
size_t ptInd = pt.GetPtInd();
|
||||
targetPhrases[ptInd] = tps;
|
||||
size_t ptInd = pt.GetPtInd();
|
||||
targetPhrases[ptInd] = tps;
|
||||
|
||||
if (tps && tps->GetSize()) {
|
||||
m_isUsed = true;
|
||||
}
|
||||
if (tps && tps->GetSize()) {
|
||||
m_isUsed = true;
|
||||
}
|
||||
}
|
||||
|
||||
const TargetPhrases *InputPath::GetTargetPhrases(const PhraseTable &pt) const
|
||||
{
|
||||
size_t ptInd = pt.GetPtInd();
|
||||
return targetPhrases[ptInd];
|
||||
size_t ptInd = pt.GetPtInd();
|
||||
return targetPhrases[ptInd];
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const InputPath &obj)
|
||||
{
|
||||
out << obj.range << " " << obj.subPhrase;
|
||||
return out;
|
||||
out << obj.range << " " << obj.subPhrase;
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -14,23 +14,26 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class InputPath : public InputPathBase
|
||||
class InputPath: public InputPathBase
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const InputPath &);
|
||||
friend std::ostream& operator<<(std::ostream &, const InputPath &);
|
||||
public:
|
||||
const TargetPhrases** targetPhrases;
|
||||
const TargetPhrases** targetPhrases;
|
||||
|
||||
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPath *prefixPath);
|
||||
virtual ~InputPath();
|
||||
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
|
||||
size_t numPt, const InputPath *prefixPath);
|
||||
virtual ~InputPath();
|
||||
|
||||
void AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps);
|
||||
const TargetPhrases *GetTargetPhrases(const PhraseTable &pt) const;
|
||||
void AddTargetPhrases(const PhraseTable &pt, const TargetPhrases *tps);
|
||||
const TargetPhrases *GetTargetPhrases(const PhraseTable &pt) const;
|
||||
|
||||
inline bool IsUsed() const
|
||||
{ return m_isUsed; }
|
||||
inline bool IsUsed() const
|
||||
{
|
||||
return m_isUsed;
|
||||
}
|
||||
|
||||
protected:
|
||||
bool m_isUsed;
|
||||
bool m_isUsed;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -23,35 +23,38 @@ void InputPaths::Init(const Sentence &input, const ManagerBase &mgr)
|
||||
size_t size = input.GetSize();
|
||||
size_t maxLength = min(size, mgr.system.maxPhraseLength);
|
||||
|
||||
m_matrix = new (pool.Allocate< Matrix<InputPath*> >()) Matrix<InputPath*>(pool, size, maxLength);
|
||||
m_matrix = new (pool.Allocate<Matrix<InputPath*> >()) Matrix<InputPath*>(pool,
|
||||
size, maxLength);
|
||||
m_matrix->Init(NULL);
|
||||
|
||||
// create blank path for initial hypo
|
||||
Range range(NOT_FOUND, NOT_FOUND);
|
||||
SubPhrase subPhrase = input.GetSubPhrase(NOT_FOUND, NOT_FOUND);
|
||||
m_blank = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, NULL);
|
||||
m_blank = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range,
|
||||
numPt, NULL);
|
||||
|
||||
// create normal paths of subphrases through the sentence
|
||||
for (size_t startPos = 0; startPos < size; ++startPos) {
|
||||
const InputPath *prefixPath = NULL;
|
||||
const InputPath *prefixPath = NULL;
|
||||
|
||||
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
|
||||
size_t endPos = startPos + phaseSize - 1;
|
||||
size_t endPos = startPos + phaseSize - 1;
|
||||
|
||||
if (endPos >= size) {
|
||||
break;
|
||||
}
|
||||
if (endPos >= size) {
|
||||
break;
|
||||
}
|
||||
|
||||
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
|
||||
Range range(startPos, endPos);
|
||||
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
|
||||
Range range(startPos, endPos);
|
||||
|
||||
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, prefixPath);
|
||||
m_inputPaths.push_back(path);
|
||||
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool,
|
||||
subPhrase, range, numPt, prefixPath);
|
||||
m_inputPaths.push_back(path);
|
||||
|
||||
prefixPath = path;
|
||||
prefixPath = path;
|
||||
|
||||
m_matrix->SetValue(startPos, phaseSize - 1, path);
|
||||
}
|
||||
m_matrix->SetValue(startPos, phaseSize - 1, path);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -19,16 +19,20 @@ namespace Moses2
|
||||
class Sentence;
|
||||
class System;
|
||||
|
||||
class InputPaths : public InputPathsBase
|
||||
class InputPaths: public InputPathsBase
|
||||
{
|
||||
public:
|
||||
void Init(const Sentence &input, const ManagerBase &mgr);
|
||||
|
||||
const InputPath &GetBlank() const
|
||||
{ return *m_blank; }
|
||||
{
|
||||
return *m_blank;
|
||||
}
|
||||
|
||||
const Matrix<InputPath*> &GetMatrix() const
|
||||
{ return *m_matrix; }
|
||||
{
|
||||
return *m_matrix;
|
||||
}
|
||||
|
||||
protected:
|
||||
InputPath *m_blank;
|
||||
@ -38,4 +42,3 @@ protected:
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -15,11 +15,11 @@
|
||||
#include "Normal/Search.h"
|
||||
#include "CubePruningMiniStack/Search.h"
|
||||
/*
|
||||
#include "CubePruningPerMiniStack/Search.h"
|
||||
#include "CubePruningPerBitmap/Search.h"
|
||||
#include "CubePruningCardinalStack/Search.h"
|
||||
#include "CubePruningBitmapStack/Search.h"
|
||||
*/
|
||||
#include "CubePruningPerMiniStack/Search.h"
|
||||
#include "CubePruningPerBitmap/Search.h"
|
||||
#include "CubePruningCardinalStack/Search.h"
|
||||
#include "CubePruningBitmapStack/Search.h"
|
||||
*/
|
||||
#include "../System.h"
|
||||
#include "../TargetPhrases.h"
|
||||
#include "../Phrase.h"
|
||||
@ -32,205 +32,211 @@ using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
Manager::Manager(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId)
|
||||
:ManagerBase(sys, task, inputStr, translationId)
|
||||
{}
|
||||
Manager::Manager(System &sys, const TranslationTask &task,
|
||||
const std::string &inputStr, long translationId) :
|
||||
ManagerBase(sys, task, inputStr, translationId)
|
||||
{
|
||||
}
|
||||
|
||||
Manager::~Manager() {
|
||||
Manager::~Manager()
|
||||
{
|
||||
|
||||
delete m_search;
|
||||
delete m_bitmaps;
|
||||
delete m_search;
|
||||
delete m_bitmaps;
|
||||
}
|
||||
|
||||
void Manager::Init()
|
||||
{
|
||||
// init pools etc
|
||||
InitPools();
|
||||
ParseInput(false);
|
||||
// init pools etc
|
||||
InitPools();
|
||||
ParseInput(false);
|
||||
|
||||
m_bitmaps = new Bitmaps(GetPool());
|
||||
m_bitmaps = new Bitmaps(GetPool());
|
||||
|
||||
const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0];
|
||||
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(GetPool(), firstPt, system, 0);
|
||||
const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0];
|
||||
m_initPhrase = new (GetPool().Allocate<TargetPhraseImpl>()) TargetPhraseImpl(
|
||||
GetPool(), firstPt, system, 0);
|
||||
|
||||
m_inputPaths.Init(*m_input, *this);
|
||||
m_inputPaths.Init(*m_input, *this);
|
||||
|
||||
const std::vector<const PhraseTable*> &pts = system.mappings;
|
||||
for (size_t i = 0; i < pts.size(); ++i) {
|
||||
const PhraseTable &pt = *pts[i];
|
||||
//cerr << "Looking up from " << pt.GetName() << endl;
|
||||
pt.Lookup(*this, m_inputPaths);
|
||||
}
|
||||
//m_inputPaths.DeleteUnusedPaths();
|
||||
CalcFutureScore();
|
||||
const std::vector<const PhraseTable*> &pts = system.mappings;
|
||||
for (size_t i = 0; i < pts.size(); ++i) {
|
||||
const PhraseTable &pt = *pts[i];
|
||||
//cerr << "Looking up from " << pt.GetName() << endl;
|
||||
pt.Lookup(*this, m_inputPaths);
|
||||
}
|
||||
//m_inputPaths.DeleteUnusedPaths();
|
||||
CalcFutureScore();
|
||||
|
||||
m_bitmaps->Init(m_input->GetSize(), vector<bool>(0));
|
||||
m_bitmaps->Init(m_input->GetSize(), vector<bool>(0));
|
||||
|
||||
switch (system.searchAlgorithm) {
|
||||
case Normal:
|
||||
m_search = new SearchNormal(*this);
|
||||
break;
|
||||
case CubePruning:
|
||||
case CubePruningMiniStack:
|
||||
m_search = new NSCubePruningMiniStack::Search(*this);
|
||||
break;
|
||||
/*
|
||||
case CubePruningPerMiniStack:
|
||||
m_search = new NSCubePruningPerMiniStack::Search(*this);
|
||||
break;
|
||||
case CubePruningPerBitmap:
|
||||
m_search = new NSCubePruningPerBitmap::Search(*this);
|
||||
break;
|
||||
case CubePruningCardinalStack:
|
||||
m_search = new NSCubePruningCardinalStack::Search(*this);
|
||||
break;
|
||||
case CubePruningBitmapStack:
|
||||
m_search = new NSCubePruningBitmapStack::Search(*this);
|
||||
break;
|
||||
*/
|
||||
default:
|
||||
cerr << "Unknown search algorithm" << endl;
|
||||
abort();
|
||||
}
|
||||
switch (system.searchAlgorithm) {
|
||||
case Normal:
|
||||
m_search = new SearchNormal(*this);
|
||||
break;
|
||||
case CubePruning:
|
||||
case CubePruningMiniStack:
|
||||
m_search = new NSCubePruningMiniStack::Search(*this);
|
||||
break;
|
||||
/*
|
||||
case CubePruningPerMiniStack:
|
||||
m_search = new NSCubePruningPerMiniStack::Search(*this);
|
||||
break;
|
||||
case CubePruningPerBitmap:
|
||||
m_search = new NSCubePruningPerBitmap::Search(*this);
|
||||
break;
|
||||
case CubePruningCardinalStack:
|
||||
m_search = new NSCubePruningCardinalStack::Search(*this);
|
||||
break;
|
||||
case CubePruningBitmapStack:
|
||||
m_search = new NSCubePruningBitmapStack::Search(*this);
|
||||
break;
|
||||
*/
|
||||
default:
|
||||
cerr << "Unknown search algorithm" << endl;
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::Decode()
|
||||
{
|
||||
Init();
|
||||
m_search->Decode();
|
||||
OutputBest();
|
||||
Init();
|
||||
m_search->Decode();
|
||||
OutputBest();
|
||||
|
||||
if (system.nbestSize) {
|
||||
OutputNBest();
|
||||
}
|
||||
if (system.nbestSize) {
|
||||
OutputNBest();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::CalcFutureScore()
|
||||
{
|
||||
size_t size = m_input->GetSize();
|
||||
m_estimatedScores = new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(), size);
|
||||
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
|
||||
size_t size = m_input->GetSize();
|
||||
m_estimatedScores =
|
||||
new (GetPool().Allocate<EstimatedScores>()) EstimatedScores(GetPool(),
|
||||
size);
|
||||
m_estimatedScores->InitTriangle(-numeric_limits<SCORE>::infinity());
|
||||
|
||||
// walk all the translation options and record the cheapest option for each span
|
||||
BOOST_FOREACH(const InputPathBase *path, m_inputPaths) {
|
||||
const Range &range = path->range;
|
||||
SCORE bestScore = -numeric_limits<SCORE>::infinity();
|
||||
// walk all the translation options and record the cheapest option for each span
|
||||
BOOST_FOREACH(const InputPathBase *path, m_inputPaths){
|
||||
const Range &range = path->range;
|
||||
SCORE bestScore = -numeric_limits<SCORE>::infinity();
|
||||
|
||||
size_t numPt = system.mappings.size();
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
|
||||
if (tps) {
|
||||
BOOST_FOREACH(const TargetPhrase *tp, *tps) {
|
||||
SCORE score = tp->GetFutureScore();
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
|
||||
}
|
||||
size_t numPt = system.mappings.size();
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = static_cast<const InputPath*>(path)->targetPhrases[i];
|
||||
if (tps) {
|
||||
BOOST_FOREACH(const TargetPhrase *tp, *tps) {
|
||||
SCORE score = tp->GetFutureScore();
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
m_estimatedScores->SetValue(range.GetStartPos(), range.GetEndPos(), bestScore);
|
||||
}
|
||||
|
||||
// now fill all the cells in the strictly upper triangle
|
||||
// there is no way to modify the diagonal now, in the case
|
||||
// where no translation option covers a single-word span,
|
||||
// we leave the +inf in the matrix
|
||||
// like in chart parsing we want each cell to contain the highest score
|
||||
// of the full-span trOpt or the sum of scores of joining two smaller spans
|
||||
// now fill all the cells in the strictly upper triangle
|
||||
// there is no way to modify the diagonal now, in the case
|
||||
// where no translation option covers a single-word span,
|
||||
// we leave the +inf in the matrix
|
||||
// like in chart parsing we want each cell to contain the highest score
|
||||
// of the full-span trOpt or the sum of scores of joining two smaller spans
|
||||
|
||||
for(size_t colstart = 1; colstart < size ; colstart++) {
|
||||
for(size_t diagshift = 0; diagshift < size-colstart ; diagshift++) {
|
||||
size_t sPos = diagshift;
|
||||
size_t ePos = colstart+diagshift;
|
||||
for(size_t joinAt = sPos; joinAt < ePos ; joinAt++) {
|
||||
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
|
||||
+ m_estimatedScores->GetValue(joinAt+1, ePos);
|
||||
// uncomment to see the cell filling scheme
|
||||
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
|
||||
// << sPos << "," << joinAt << "]+["
|
||||
// << joinAt+1 << "," << ePos << "] (colstart: "
|
||||
// << colstart << ", diagshift: " << diagshift << ")"
|
||||
// << endl);
|
||||
for (size_t colstart = 1; colstart < size; colstart++) {
|
||||
for (size_t diagshift = 0; diagshift < size - colstart; diagshift++) {
|
||||
size_t sPos = diagshift;
|
||||
size_t ePos = colstart + diagshift;
|
||||
for (size_t joinAt = sPos; joinAt < ePos; joinAt++) {
|
||||
float joinedScore = m_estimatedScores->GetValue(sPos, joinAt)
|
||||
+ m_estimatedScores->GetValue(joinAt + 1, ePos);
|
||||
// uncomment to see the cell filling scheme
|
||||
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
|
||||
// << sPos << "," << joinAt << "]+["
|
||||
// << joinAt+1 << "," << ePos << "] (colstart: "
|
||||
// << colstart << ", diagshift: " << diagshift << ")"
|
||||
// << endl);
|
||||
|
||||
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos))
|
||||
m_estimatedScores->SetValue(sPos, ePos, joinedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (joinedScore > m_estimatedScores->GetValue(sPos, ePos)) m_estimatedScores->SetValue(
|
||||
sPos, ePos, joinedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//cerr << "Square matrix:" << endl;
|
||||
//cerr << *m_estimatedScores << endl;
|
||||
//cerr << "Square matrix:" << endl;
|
||||
//cerr << *m_estimatedScores << endl;
|
||||
}
|
||||
|
||||
void Manager::OutputBest() const
|
||||
{
|
||||
stringstream out;
|
||||
const Hypothesis *bestHypo = m_search->GetBestHypothesis();
|
||||
if (bestHypo) {
|
||||
if (system.outputHypoScore) {
|
||||
out << bestHypo->GetScores().GetTotalScore() << " ";
|
||||
}
|
||||
stringstream out;
|
||||
const Hypothesis *bestHypo = m_search->GetBestHypothesis();
|
||||
if (bestHypo) {
|
||||
if (system.outputHypoScore) {
|
||||
out << bestHypo->GetScores().GetTotalScore() << " ";
|
||||
}
|
||||
|
||||
bestHypo->OutputToStream(out);
|
||||
//cerr << "BEST TRANSLATION: " << *bestHypo;
|
||||
}
|
||||
else {
|
||||
if (system.outputHypoScore) {
|
||||
out << "0 ";
|
||||
}
|
||||
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
|
||||
}
|
||||
out << "\n";
|
||||
bestHypo->OutputToStream(out);
|
||||
//cerr << "BEST TRANSLATION: " << *bestHypo;
|
||||
}
|
||||
else {
|
||||
if (system.outputHypoScore) {
|
||||
out << "0 ";
|
||||
}
|
||||
//cerr << "NO TRANSLATION " << m_input->GetTranslationId() << endl;
|
||||
}
|
||||
out << "\n";
|
||||
|
||||
system.bestCollector->Write(m_input->GetTranslationId(), out.str());
|
||||
//cerr << endl;
|
||||
system.bestCollector->Write(m_input->GetTranslationId(), out.str());
|
||||
//cerr << endl;
|
||||
}
|
||||
|
||||
void Manager::OutputNBest()
|
||||
{
|
||||
arcLists.Sort();
|
||||
arcLists.Sort();
|
||||
|
||||
set<string> distinctHypos;
|
||||
set<string> distinctHypos;
|
||||
|
||||
TrellisPaths contenders;
|
||||
//cerr << "START AddInitialTrellisPaths" << endl;
|
||||
m_search->AddInitialTrellisPaths(contenders);
|
||||
//cerr << "END AddInitialTrellisPaths" << endl;
|
||||
TrellisPaths contenders;
|
||||
//cerr << "START AddInitialTrellisPaths" << endl;
|
||||
m_search->AddInitialTrellisPaths(contenders);
|
||||
//cerr << "END AddInitialTrellisPaths" << endl;
|
||||
|
||||
long transId = m_input->GetTranslationId();
|
||||
long transId = m_input->GetTranslationId();
|
||||
|
||||
// MAIN LOOP
|
||||
stringstream out;
|
||||
size_t bestInd = 0;
|
||||
while (bestInd < system.nbestSize && !contenders.empty()) {
|
||||
//cerr << "bestInd=" << bestInd << endl;
|
||||
TrellisPath *path = contenders.Get();
|
||||
// MAIN LOOP
|
||||
stringstream out;
|
||||
size_t bestInd = 0;
|
||||
while (bestInd < system.nbestSize && !contenders.empty()) {
|
||||
//cerr << "bestInd=" << bestInd << endl;
|
||||
TrellisPath *path = contenders.Get();
|
||||
|
||||
bool ok = false;
|
||||
if (system.distinctNBest) {
|
||||
string tgtPhrase = path->ToString();
|
||||
if (distinctHypos.insert(tgtPhrase).second) {
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ok = true;
|
||||
}
|
||||
bool ok = false;
|
||||
if (system.distinctNBest) {
|
||||
string tgtPhrase = path->ToString();
|
||||
if (distinctHypos.insert(tgtPhrase).second) {
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ok = true;
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
out << transId << " |||";
|
||||
path->OutputToStream(out, system);
|
||||
out << "\n";
|
||||
}
|
||||
if (ok) {
|
||||
out << transId << " |||";
|
||||
path->OutputToStream(out, system);
|
||||
out << "\n";
|
||||
}
|
||||
|
||||
// create next paths
|
||||
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
|
||||
// create next paths
|
||||
path->CreateDeviantPaths(contenders, arcLists, GetPool(), system);
|
||||
|
||||
delete path;
|
||||
delete path;
|
||||
|
||||
++bestInd;
|
||||
}
|
||||
system.nbestCollector->Write(transId, out.str());
|
||||
++bestInd;
|
||||
}
|
||||
system.nbestCollector->Write(transId, out.str());
|
||||
|
||||
}
|
||||
|
||||
|
@ -31,41 +31,50 @@ class Hypothesis;
|
||||
class Sentence;
|
||||
class OutputCollector;
|
||||
|
||||
class Manager : public ManagerBase
|
||||
class Manager: public ManagerBase
|
||||
{
|
||||
public:
|
||||
Manager(System &sys, const TranslationTask &task, const std::string &inputStr, long translationId);
|
||||
Manager(System &sys, const TranslationTask &task, const std::string &inputStr,
|
||||
long translationId);
|
||||
|
||||
virtual ~Manager();
|
||||
virtual ~Manager();
|
||||
|
||||
Bitmaps &GetBitmaps()
|
||||
{ return *m_bitmaps; }
|
||||
Bitmaps &GetBitmaps()
|
||||
{
|
||||
return *m_bitmaps;
|
||||
}
|
||||
|
||||
const EstimatedScores &GetEstimatedScores() const
|
||||
{ return *m_estimatedScores; }
|
||||
const EstimatedScores &GetEstimatedScores() const
|
||||
{
|
||||
return *m_estimatedScores;
|
||||
}
|
||||
|
||||
const InputPaths &GetInputPaths() const
|
||||
{ return m_inputPaths; }
|
||||
const InputPaths &GetInputPaths() const
|
||||
{
|
||||
return m_inputPaths;
|
||||
}
|
||||
|
||||
const TargetPhrase &GetInitPhrase() const
|
||||
{ return *m_initPhrase; }
|
||||
const TargetPhrase &GetInitPhrase() const
|
||||
{
|
||||
return *m_initPhrase;
|
||||
}
|
||||
|
||||
void Decode();
|
||||
void Decode();
|
||||
|
||||
protected:
|
||||
|
||||
InputPaths m_inputPaths;
|
||||
Bitmaps *m_bitmaps;
|
||||
EstimatedScores *m_estimatedScores;
|
||||
TargetPhrase *m_initPhrase;
|
||||
InputPaths m_inputPaths;
|
||||
Bitmaps *m_bitmaps;
|
||||
EstimatedScores *m_estimatedScores;
|
||||
TargetPhrase *m_initPhrase;
|
||||
|
||||
Search *m_search;
|
||||
Search *m_search;
|
||||
|
||||
// must be run in same thread as Decode()
|
||||
void Init();
|
||||
void CalcFutureScore();
|
||||
void OutputBest() const;
|
||||
void OutputNBest();
|
||||
// must be run in same thread as Decode()
|
||||
void Init();
|
||||
void CalcFutureScore();
|
||||
void OutputBest() const;
|
||||
void OutputNBest();
|
||||
|
||||
};
|
||||
|
||||
|
@ -23,135 +23,131 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
SearchNormal::SearchNormal(Manager &mgr)
|
||||
:Search(mgr)
|
||||
,m_stacks(mgr)
|
||||
SearchNormal::SearchNormal(Manager &mgr) :
|
||||
Search(mgr), m_stacks(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
SearchNormal::~SearchNormal() {
|
||||
// TODO Auto-generated destructor stub
|
||||
SearchNormal::~SearchNormal()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void SearchNormal::Decode()
|
||||
{
|
||||
// init stacks
|
||||
m_stacks.Init(mgr, mgr.GetInput().GetSize() + 1);
|
||||
// init stacks
|
||||
m_stacks.Init(mgr, mgr.GetInput().GetSize() + 1);
|
||||
|
||||
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
|
||||
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap);
|
||||
initHypo->EmptyHypothesisState(mgr.GetInput());
|
||||
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
|
||||
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
|
||||
initBitmap);
|
||||
initHypo->EmptyHypothesisState(mgr.GetInput());
|
||||
|
||||
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
|
||||
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
|
||||
Decode(stackInd);
|
||||
//cerr << m_stacks << endl;
|
||||
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
|
||||
Decode(stackInd);
|
||||
//cerr << m_stacks << endl;
|
||||
|
||||
// delete stack to save mem
|
||||
if (stackInd < m_stacks.GetSize() - 1) {
|
||||
m_stacks.Delete(stackInd);
|
||||
}
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
// delete stack to save mem
|
||||
if (stackInd < m_stacks.GetSize() - 1) {
|
||||
m_stacks.Delete(stackInd);
|
||||
}
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
}
|
||||
|
||||
void SearchNormal::Decode(size_t stackInd)
|
||||
{
|
||||
Stack &stack = m_stacks[stackInd];
|
||||
if (&stack == &m_stacks.Back()) {
|
||||
// last stack. don't do anythin
|
||||
return;
|
||||
// last stack. don't do anythin
|
||||
return;
|
||||
}
|
||||
|
||||
Hypotheses &hypos = stack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
|
||||
const InputPaths &paths = mgr.GetInputPaths();
|
||||
const InputPaths &paths = mgr.GetInputPaths();
|
||||
|
||||
BOOST_FOREACH(const InputPathBase *path, paths) {
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
|
||||
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
|
||||
}
|
||||
}
|
||||
BOOST_FOREACH(const InputPathBase *path, paths){
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
|
||||
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SearchNormal::Extend(const Hypothesis &hypo, const InputPath &path)
|
||||
{
|
||||
const Bitmap &hypoBitmap = hypo.GetBitmap();
|
||||
const Range &hypoRange = hypo.GetInputPath().range;
|
||||
const Range &pathRange = path.range;
|
||||
const Bitmap &hypoBitmap = hypo.GetBitmap();
|
||||
const Range &hypoRange = hypo.GetInputPath().range;
|
||||
const Range &pathRange = path.range;
|
||||
|
||||
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
|
||||
return;
|
||||
}
|
||||
//cerr << " YES" << endl;
|
||||
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
|
||||
return;
|
||||
}
|
||||
//cerr << " YES" << endl;
|
||||
|
||||
// extend this hypo
|
||||
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
|
||||
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
|
||||
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
|
||||
// extend this hypo
|
||||
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
|
||||
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
|
||||
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
|
||||
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
const TargetPhrases **tpsAllPt = path.targetPhrases;
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = tpsAllPt[i];
|
||||
if (tps) {
|
||||
Extend(hypo, *tps, path, newBitmap, estimatedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SearchNormal::Extend(const Hypothesis &hypo,
|
||||
const TargetPhrases &tps,
|
||||
const InputPath &path,
|
||||
const Bitmap &newBitmap,
|
||||
SCORE estimatedScore)
|
||||
{
|
||||
BOOST_FOREACH(const TargetPhrase *tp, tps) {
|
||||
Extend(hypo, *tp, path, newBitmap, estimatedScore);
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
const TargetPhrases **tpsAllPt = path.targetPhrases;
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = tpsAllPt[i];
|
||||
if (tps) {
|
||||
Extend(hypo, *tps, path, newBitmap, estimatedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SearchNormal::Extend(const Hypothesis &hypo,
|
||||
const TargetPhrase &tp,
|
||||
const InputPath &path,
|
||||
const Bitmap &newBitmap,
|
||||
SCORE estimatedScore)
|
||||
void SearchNormal::Extend(const Hypothesis &hypo, const TargetPhrases &tps,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
|
||||
{
|
||||
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
|
||||
newHypo->EvaluateWhenApplied();
|
||||
BOOST_FOREACH(const TargetPhrase *tp, tps){
|
||||
Extend(hypo, *tp, path, newBitmap, estimatedScore);
|
||||
}
|
||||
}
|
||||
|
||||
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
void SearchNormal::Extend(const Hypothesis &hypo, const TargetPhrase &tp,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
|
||||
{
|
||||
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
|
||||
newHypo->EvaluateWhenApplied();
|
||||
|
||||
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
|
||||
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
|
||||
m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
|
||||
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
|
||||
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
|
||||
|
||||
}
|
||||
|
||||
const Hypothesis *SearchNormal::GetBestHypothesis() const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
const Hypotheses &sortedHypos = lastStack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
const Hypotheses &sortedHypos = lastStack.GetSortedAndPruneHypos(mgr,
|
||||
mgr.arcLists);
|
||||
|
||||
const Hypothesis *best = NULL;
|
||||
if (sortedHypos.size()) {
|
||||
best = static_cast<const Hypothesis*>(sortedHypos[0]);
|
||||
}
|
||||
return best;
|
||||
const Hypothesis *best = NULL;
|
||||
if (sortedHypos.size()) {
|
||||
best = static_cast<const Hypothesis*>(sortedHypos[0]);
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
void SearchNormal::AddInitialTrellisPaths(TrellisPaths &paths) const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, lastStack) {
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
|
||||
paths.Add(path);
|
||||
}
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, lastStack){
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
|
||||
paths.Add(path);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -23,36 +23,29 @@ class TargetPhrases;
|
||||
class TargetPhrase;
|
||||
class Stacks;
|
||||
|
||||
class SearchNormal : public Search
|
||||
class SearchNormal: public Search
|
||||
{
|
||||
public:
|
||||
SearchNormal(Manager &mgr);
|
||||
virtual ~SearchNormal();
|
||||
SearchNormal(Manager &mgr);
|
||||
virtual ~SearchNormal();
|
||||
|
||||
virtual void Decode();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
virtual void Decode();
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
|
||||
void AddInitialTrellisPaths(TrellisPaths &paths) const;
|
||||
void AddInitialTrellisPaths(TrellisPaths &paths) const;
|
||||
|
||||
protected:
|
||||
Stacks m_stacks;
|
||||
Stacks m_stacks;
|
||||
|
||||
void Decode(size_t stackInd);
|
||||
void Extend(const Hypothesis &hypo, const InputPath &path);
|
||||
void Extend(const Hypothesis &hypo,
|
||||
const TargetPhrases &tps,
|
||||
const InputPath &path,
|
||||
const Bitmap &newBitmap,
|
||||
SCORE estimatedScore);
|
||||
void Extend(const Hypothesis &hypo,
|
||||
const TargetPhrase &tp,
|
||||
const InputPath &path,
|
||||
const Bitmap &newBitmap,
|
||||
SCORE estimatedScore);
|
||||
void Decode(size_t stackInd);
|
||||
void Extend(const Hypothesis &hypo, const InputPath &path);
|
||||
void Extend(const Hypothesis &hypo, const TargetPhrases &tps,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
|
||||
void Extend(const Hypothesis &hypo, const TargetPhrase &tp,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif /* SEARCHNORMAL_H_ */
|
||||
|
@ -16,17 +16,17 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
Stack::Stack(const Manager &mgr)
|
||||
:HypothesisColl(mgr)
|
||||
Stack::Stack(const Manager &mgr) :
|
||||
HypothesisColl(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Stack::~Stack() {
|
||||
// TODO Auto-generated destructor stub
|
||||
Stack::~Stack()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -16,15 +16,14 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class Stack : public HypothesisColl
|
||||
class Stack: public HypothesisColl
|
||||
{
|
||||
public:
|
||||
Stack(const Manager &mgr);
|
||||
virtual ~Stack();
|
||||
Stack(const Manager &mgr);
|
||||
virtual ~Stack();
|
||||
|
||||
protected:
|
||||
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -14,43 +14,45 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
Stacks::Stacks(const Manager &mgr)
|
||||
:m_mgr(mgr)
|
||||
Stacks::Stacks(const Manager &mgr) :
|
||||
m_mgr(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Stacks::~Stacks() {
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
delete m_stacks[i];
|
||||
}
|
||||
Stacks::~Stacks()
|
||||
{
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
delete m_stacks[i];
|
||||
}
|
||||
}
|
||||
|
||||
void Stacks::Init(const Manager &mgr, size_t numStacks)
|
||||
{
|
||||
m_stacks.resize(numStacks);
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
m_stacks[i] = new Stack(mgr);
|
||||
}
|
||||
m_stacks.resize(numStacks);
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
m_stacks[i] = new Stack(mgr);
|
||||
}
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const Stacks &obj)
|
||||
{
|
||||
for (size_t i = 0; i < obj.GetSize(); ++i) {
|
||||
const Stack *stack = obj.m_stacks[i];
|
||||
if (stack) {
|
||||
out << stack->GetSize() << " ";
|
||||
}
|
||||
else {
|
||||
out << "N ";
|
||||
}
|
||||
const Stack *stack = obj.m_stacks[i];
|
||||
if (stack) {
|
||||
out << stack->GetSize() << " ";
|
||||
}
|
||||
else {
|
||||
out << "N ";
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists)
|
||||
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists)
|
||||
{
|
||||
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
|
||||
//cerr << "numWordsCovered=" << numWordsCovered << endl;
|
||||
@ -59,15 +61,15 @@ void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLi
|
||||
|
||||
size_t nbestSize = m_mgr.system.nbestSize;
|
||||
if (nbestSize) {
|
||||
arcLists.AddArc(added.added, hypo, added.other);
|
||||
arcLists.AddArc(added.added, hypo, added.other);
|
||||
}
|
||||
else {
|
||||
if (!added.added) {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
}
|
||||
else if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
}
|
||||
if (!added.added) {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
}
|
||||
else if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -16,33 +16,42 @@ namespace Moses2
|
||||
class Manager;
|
||||
class ArcLists;
|
||||
|
||||
class Stacks {
|
||||
friend std::ostream& operator<<(std::ostream &, const Stacks &);
|
||||
class Stacks
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const Stacks &);
|
||||
public:
|
||||
Stacks(const Manager &mgr);
|
||||
virtual ~Stacks();
|
||||
Stacks(const Manager &mgr);
|
||||
virtual ~Stacks();
|
||||
|
||||
void Init(const Manager &mgr, size_t numStacks);
|
||||
void Init(const Manager &mgr, size_t numStacks);
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_stacks.size(); }
|
||||
size_t GetSize() const
|
||||
{
|
||||
return m_stacks.size();
|
||||
}
|
||||
|
||||
const Stack &Back() const
|
||||
{ return *m_stacks.back(); }
|
||||
const Stack &Back() const
|
||||
{
|
||||
return *m_stacks.back();
|
||||
}
|
||||
|
||||
Stack &operator[](size_t ind)
|
||||
{ return *m_stacks[ind]; }
|
||||
Stack &operator[](size_t ind)
|
||||
{
|
||||
return *m_stacks[ind];
|
||||
}
|
||||
|
||||
void Delete(size_t ind) {
|
||||
delete m_stacks[ind];
|
||||
m_stacks[ind] = NULL;
|
||||
}
|
||||
void Delete(size_t ind)
|
||||
{
|
||||
delete m_stacks[ind];
|
||||
m_stacks[ind] = NULL;
|
||||
}
|
||||
|
||||
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle, ArcLists &arcLists);
|
||||
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists);
|
||||
|
||||
protected:
|
||||
const Manager &m_mgr;
|
||||
std::vector<Stack*> m_stacks;
|
||||
const Manager &m_mgr;
|
||||
std::vector<Stack*> m_stacks;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -14,97 +14,99 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
Search::Search(Manager &mgr)
|
||||
:mgr(mgr)
|
||||
Search::Search(Manager &mgr) :
|
||||
mgr(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Search::~Search() {
|
||||
// TODO Auto-generated destructor stub
|
||||
Search::~Search()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos, const Range &pathRange)
|
||||
bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
|
||||
const Range &pathRange)
|
||||
{
|
||||
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
|
||||
const size_t hypoFirstGapPos = hypoBitmap.GetFirstGapPos();
|
||||
|
||||
//cerr << "DOING " << hypoBitmap << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
|
||||
// " [" << pathRange.GetStartPos() << " " << pathRange.GetEndPos() << "]";
|
||||
//cerr << "DOING " << hypoBitmap << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
|
||||
// " [" << pathRange.GetStartPos() << " " << pathRange.GetEndPos() << "]";
|
||||
|
||||
if (hypoBitmap.Overlap(pathRange)) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
if (hypoBitmap.Overlap(pathRange)) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (mgr.system.maxDistortion >= 0) {
|
||||
// distortion limit
|
||||
int distortion = ComputeDistortionDistance(hypoRangeEndPos, pathRange.GetStartPos());
|
||||
if (distortion > mgr.system.maxDistortion) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// first question: is there a path from the closest translated word to the left
|
||||
// of the hypothesized extension to the start of the hypothesized extension?
|
||||
// long version:
|
||||
// - is there anything to our left?
|
||||
// - is it farther left than where we're starting anyway?
|
||||
// - can we get to it?
|
||||
|
||||
// closestLeft is exclusive: a value of 3 means 2 is covered, our
|
||||
// arc is currently ENDING at 3 and can start at 3 implicitly
|
||||
|
||||
// TODO is this relevant? only for lattice input?
|
||||
|
||||
// ask second question here: we already know we can get to our
|
||||
// starting point from the closest thing to the left. We now ask the
|
||||
// follow up: can we get from our end to the closest thing on the
|
||||
// right?
|
||||
//
|
||||
// long version: is anything to our right? is it farther
|
||||
// right than our (inclusive) end? can our end reach it?
|
||||
bool isLeftMostEdge = (hypoFirstGapPos == pathRange.GetStartPos());
|
||||
|
||||
size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(pathRange.GetEndPos());
|
||||
/*
|
||||
if (isWordLattice) {
|
||||
if (closestRight != endPos
|
||||
&& ((closestRight + 1) < sourceSize)
|
||||
&& !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
|
||||
continue;
|
||||
}
|
||||
if (mgr.system.maxDistortion >= 0) {
|
||||
// distortion limit
|
||||
int distortion = ComputeDistortionDistance(hypoRangeEndPos,
|
||||
pathRange.GetStartPos());
|
||||
if (distortion > mgr.system.maxDistortion) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
if (isLeftMostEdge) {
|
||||
// any length extension is okay if starting at left-most edge
|
||||
// first question: is there a path from the closest translated word to the left
|
||||
// of the hypothesized extension to the start of the hypothesized extension?
|
||||
// long version:
|
||||
// - is there anything to our left?
|
||||
// - is it farther left than where we're starting anyway?
|
||||
// - can we get to it?
|
||||
|
||||
} else { // starting somewhere other than left-most edge, use caution
|
||||
// the basic idea is this: we would like to translate a phrase
|
||||
// starting from a position further right than the left-most
|
||||
// open gap. The distortion penalty for the following phrase
|
||||
// will be computed relative to the ending position of the
|
||||
// current extension, so we ask now what its maximum value will
|
||||
// be (which will always be the value of the hypothesis starting
|
||||
// at the left-most edge). If this value is less than the
|
||||
// distortion limit, we don't allow this extension to be made.
|
||||
Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
|
||||
// closestLeft is exclusive: a value of 3 means 2 is covered, our
|
||||
// arc is currently ENDING at 3 and can start at 3 implicitly
|
||||
|
||||
if (ComputeDistortionDistance(pathRange.GetEndPos(), bestNextExtension.GetStartPos())
|
||||
> mgr.system.maxDistortion) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
// TODO is this relevant? only for lattice input?
|
||||
|
||||
// everything is fine, we're good to go
|
||||
// ask second question here: we already know we can get to our
|
||||
// starting point from the closest thing to the left. We now ask the
|
||||
// follow up: can we get from our end to the closest thing on the
|
||||
// right?
|
||||
//
|
||||
// long version: is anything to our right? is it farther
|
||||
// right than our (inclusive) end? can our end reach it?
|
||||
bool isLeftMostEdge = (hypoFirstGapPos == pathRange.GetStartPos());
|
||||
|
||||
size_t closestRight = hypoBitmap.GetEdgeToTheRightOf(pathRange.GetEndPos());
|
||||
/*
|
||||
if (isWordLattice) {
|
||||
if (closestRight != endPos
|
||||
&& ((closestRight + 1) < sourceSize)
|
||||
&& !m_source.CanIGetFromAToB(endPos + 1, closestRight + 1)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
if (isLeftMostEdge) {
|
||||
// any length extension is okay if starting at left-most edge
|
||||
|
||||
}
|
||||
else { // starting somewhere other than left-most edge, use caution
|
||||
// the basic idea is this: we would like to translate a phrase
|
||||
// starting from a position further right than the left-most
|
||||
// open gap. The distortion penalty for the following phrase
|
||||
// will be computed relative to the ending position of the
|
||||
// current extension, so we ask now what its maximum value will
|
||||
// be (which will always be the value of the hypothesis starting
|
||||
// at the left-most edge). If this value is less than the
|
||||
// distortion limit, we don't allow this extension to be made.
|
||||
Range bestNextExtension(hypoFirstGapPos, hypoFirstGapPos);
|
||||
|
||||
if (ComputeDistortionDistance(pathRange.GetEndPos(),
|
||||
bestNextExtension.GetStartPos()) > mgr.system.maxDistortion) {
|
||||
//cerr << " NO" << endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
// everything is fine, we're good to go
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -20,35 +20,38 @@ class Bitmap;
|
||||
class Range;
|
||||
class TrellisPaths;
|
||||
|
||||
class Search {
|
||||
class Search
|
||||
{
|
||||
public:
|
||||
Search(Manager &mgr);
|
||||
virtual ~Search();
|
||||
Search(Manager &mgr);
|
||||
virtual ~Search();
|
||||
|
||||
virtual void Decode() = 0;
|
||||
virtual const Hypothesis *GetBestHypothesis() const = 0;
|
||||
virtual void Decode() = 0;
|
||||
virtual const Hypothesis *GetBestHypothesis() const = 0;
|
||||
|
||||
virtual void AddInitialTrellisPaths(TrellisPaths &paths) const = 0;
|
||||
virtual void AddInitialTrellisPaths(TrellisPaths &paths) const = 0;
|
||||
|
||||
protected:
|
||||
Manager &mgr;
|
||||
//ArcLists m_arcLists;
|
||||
Manager &mgr;
|
||||
//ArcLists m_arcLists;
|
||||
|
||||
bool CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos, const Range &pathRange);
|
||||
bool CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
|
||||
const Range &pathRange);
|
||||
|
||||
inline int ComputeDistortionDistance(size_t prevEndPos, size_t currStartPos) const
|
||||
{
|
||||
int dist = 0;
|
||||
if (prevEndPos == NOT_FOUND) {
|
||||
dist = currStartPos;
|
||||
} else {
|
||||
dist = (int)prevEndPos - (int)currStartPos + 1 ;
|
||||
}
|
||||
return abs(dist);
|
||||
}
|
||||
inline int ComputeDistortionDistance(size_t prevEndPos,
|
||||
size_t currStartPos) const
|
||||
{
|
||||
int dist = 0;
|
||||
if (prevEndPos == NOT_FOUND) {
|
||||
dist = currStartPos;
|
||||
}
|
||||
else {
|
||||
dist = (int)prevEndPos - (int)currStartPos + 1;
|
||||
}
|
||||
return abs(dist);
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
@ -17,37 +17,41 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
TargetPhraseImpl *TargetPhraseImpl::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
|
||||
TargetPhraseImpl *TargetPhraseImpl::CreateFromString(MemPool &pool,
|
||||
const PhraseTable &pt, const System &system, const std::string &str)
|
||||
{
|
||||
FactorCollection &vocab = system.GetVocab();
|
||||
FactorCollection &vocab = system.GetVocab();
|
||||
|
||||
vector<string> toks = Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
TargetPhraseImpl *ret = new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, pt, system, size);
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
vector<string> toks = Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
TargetPhraseImpl *ret =
|
||||
new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, pt, system,
|
||||
size);
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
|
||||
return ret;
|
||||
return ret;
|
||||
}
|
||||
|
||||
TargetPhraseImpl::TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
|
||||
:TargetPhrase(pool, pt, system)
|
||||
,PhraseImplTemplate<Word>(pool, size)
|
||||
TargetPhraseImpl::TargetPhraseImpl(MemPool &pool, const PhraseTable &pt,
|
||||
const System &system, size_t size) :
|
||||
TargetPhrase(pool, pt, system), PhraseImplTemplate<Word>(pool, size)
|
||||
{
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
|
||||
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool,
|
||||
system.featureFunctions.GetNumScores());
|
||||
|
||||
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
|
||||
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
|
||||
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
|
||||
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
|
||||
}
|
||||
|
||||
TargetPhraseImpl::~TargetPhraseImpl() {
|
||||
// TODO Auto-generated destructor stub
|
||||
TargetPhraseImpl::~TargetPhraseImpl()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TargetPhraseImpl &obj)
|
||||
{
|
||||
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
|
||||
return out;
|
||||
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -22,30 +22,38 @@ class Manager;
|
||||
class System;
|
||||
class PhraseTable;
|
||||
|
||||
class TargetPhraseImpl : public TargetPhrase, public PhraseImplTemplate<Word>
|
||||
class TargetPhraseImpl: public TargetPhrase, public PhraseImplTemplate<Word>
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const TargetPhraseImpl &);
|
||||
friend std::ostream& operator<<(std::ostream &, const TargetPhraseImpl &);
|
||||
public:
|
||||
|
||||
static TargetPhraseImpl *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
|
||||
TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
|
||||
static TargetPhraseImpl *CreateFromString(MemPool &pool,
|
||||
const PhraseTable &pt, const System &system, const std::string &str);
|
||||
TargetPhraseImpl(MemPool &pool, const PhraseTable &pt, const System &system,
|
||||
size_t size);
|
||||
//TargetPhraseImpl(MemPool &pool, const System &system, const TargetPhraseImpl ©);
|
||||
|
||||
virtual ~TargetPhraseImpl();
|
||||
|
||||
const Word& operator[](size_t pos) const
|
||||
{ return m_words[pos]; }
|
||||
{
|
||||
return m_words[pos];
|
||||
}
|
||||
|
||||
Word& operator[](size_t pos)
|
||||
{ return m_words[pos]; }
|
||||
{
|
||||
return m_words[pos];
|
||||
}
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_size; }
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
SubPhrase GetSubPhrase(size_t start, size_t end) const
|
||||
{
|
||||
SubPhrase ret(*this, start, end);
|
||||
return ret;
|
||||
SubPhrase ret(*this, start, end);
|
||||
return ret;
|
||||
}
|
||||
|
||||
//mutable void *chartState;
|
||||
|
@ -13,34 +13,32 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const TrellisNode &node)
|
||||
{
|
||||
out << "arcList=" << node.arcList->size() << " " << node.ind;
|
||||
return out;
|
||||
out << "arcList=" << node.arcList->size() << " " << node.ind;
|
||||
return out;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
TrellisPath::TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists)
|
||||
:prevEdgeChanged(-1)
|
||||
TrellisPath::TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists) :
|
||||
prevEdgeChanged(-1)
|
||||
{
|
||||
AddNodes(hypo, arcLists);
|
||||
m_scores = &hypo->GetScores();
|
||||
AddNodes(hypo, arcLists);
|
||||
m_scores = &hypo->GetScores();
|
||||
}
|
||||
|
||||
TrellisPath::TrellisPath(const TrellisPath &origPath,
|
||||
size_t edgeIndex,
|
||||
const TrellisNode &newNode,
|
||||
const ArcLists &arcLists,
|
||||
MemPool &pool,
|
||||
const System &system)
|
||||
:prevEdgeChanged(edgeIndex)
|
||||
TrellisPath::TrellisPath(const TrellisPath &origPath, size_t edgeIndex,
|
||||
const TrellisNode &newNode, const ArcLists &arcLists, MemPool &pool,
|
||||
const System &system) :
|
||||
prevEdgeChanged(edgeIndex)
|
||||
{
|
||||
nodes.reserve(origPath.nodes.size());
|
||||
for (size_t currEdge = 0 ; currEdge < edgeIndex ; currEdge++) {
|
||||
// copy path from parent
|
||||
nodes.push_back(origPath.nodes[currEdge]);
|
||||
for (size_t currEdge = 0; currEdge < edgeIndex; currEdge++) {
|
||||
// copy path from parent
|
||||
nodes.push_back(origPath.nodes[currEdge]);
|
||||
}
|
||||
|
||||
// 1 deviation
|
||||
@ -51,9 +49,9 @@ TrellisPath::TrellisPath(const TrellisPath &origPath,
|
||||
|
||||
const Hypothesis *prevHypo = arc->GetPrevHypo();
|
||||
while (prevHypo != NULL) {
|
||||
const ArcList *arcList = arcLists.GetArcList(prevHypo);
|
||||
assert(arcList);
|
||||
TrellisNode node(*arcList, 0);
|
||||
const ArcList *arcList = arcLists.GetArcList(prevHypo);
|
||||
assert(arcList);
|
||||
TrellisNode node(*arcList, 0);
|
||||
nodes.push_back(node);
|
||||
|
||||
prevHypo = prevHypo->GetPrevHypo();
|
||||
@ -63,104 +61,96 @@ TrellisPath::TrellisPath(const TrellisPath &origPath,
|
||||
const HypothesisBase *origHypo = origNode.GetHypo();
|
||||
const HypothesisBase *newHypo = newNode.GetHypo();
|
||||
|
||||
CalcScores(origPath.GetScores(),
|
||||
origHypo->GetScores(),
|
||||
newHypo->GetScores(),
|
||||
pool,
|
||||
system);
|
||||
CalcScores(origPath.GetScores(), origHypo->GetScores(), newHypo->GetScores(),
|
||||
pool, system);
|
||||
}
|
||||
|
||||
TrellisPath::~TrellisPath() {
|
||||
// TODO Auto-generated destructor stub
|
||||
TrellisPath::~TrellisPath()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
SCORE TrellisPath::GetFutureScore() const
|
||||
{
|
||||
return m_scores->GetTotalScore();
|
||||
return m_scores->GetTotalScore();
|
||||
}
|
||||
|
||||
void TrellisPath::OutputToStream(std::ostream &out, const System &system) const
|
||||
{
|
||||
out << ToString();
|
||||
out << "||| ";
|
||||
out << ToString();
|
||||
out << "||| ";
|
||||
|
||||
GetScores().OutputBreakdownToStream(out, system);
|
||||
out << "||| ";
|
||||
GetScores().OutputBreakdownToStream(out, system);
|
||||
out << "||| ";
|
||||
|
||||
out << GetScores().GetTotalScore();
|
||||
out << GetScores().GetTotalScore();
|
||||
}
|
||||
|
||||
std::string TrellisPath::ToString() const
|
||||
{
|
||||
//cerr << "path=" << this << " " << nodes.size() << endl;
|
||||
std::stringstream out;
|
||||
for (int i = nodes.size() - 1; i >= 0; --i) {
|
||||
const TrellisNode &node = nodes[i];
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(node.GetHypo());
|
||||
//cerr << "hypo=" << hypo << " " << *hypo << endl;
|
||||
hypo->GetTargetPhrase().OutputToStream(out);
|
||||
out << " ";
|
||||
}
|
||||
return out.str();
|
||||
//cerr << "path=" << this << " " << nodes.size() << endl;
|
||||
std::stringstream out;
|
||||
for (int i = nodes.size() - 1; i >= 0; --i) {
|
||||
const TrellisNode &node = nodes[i];
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(node.GetHypo());
|
||||
//cerr << "hypo=" << hypo << " " << *hypo << endl;
|
||||
hypo->GetTargetPhrase().OutputToStream(out);
|
||||
out << " ";
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void TrellisPath::CreateDeviantPaths(TrellisPaths &paths,
|
||||
const ArcLists &arcLists,
|
||||
MemPool &pool,
|
||||
const System &system) const
|
||||
const ArcLists &arcLists, MemPool &pool, const System &system) const
|
||||
{
|
||||
const size_t sizePath = nodes.size();
|
||||
|
||||
//cerr << "prevEdgeChanged=" << prevEdgeChanged << endl;
|
||||
for (size_t currEdge = prevEdgeChanged + 1 ; currEdge < sizePath ; currEdge++) {
|
||||
TrellisNode newNode = nodes[currEdge];
|
||||
for (size_t currEdge = prevEdgeChanged + 1; currEdge < sizePath; currEdge++) {
|
||||
TrellisNode newNode = nodes[currEdge];
|
||||
assert(newNode.ind == 0);
|
||||
const ArcList &arcList = *newNode.arcList;
|
||||
const ArcList &arcList = *newNode.arcList;
|
||||
|
||||
//cerr << "arcList=" << arcList.size() << endl;
|
||||
for (size_t i = 1; i < arcList.size(); ++i) {
|
||||
//cerr << "i=" << i << endl;
|
||||
newNode.ind = i;
|
||||
for (size_t i = 1; i < arcList.size(); ++i) {
|
||||
//cerr << "i=" << i << endl;
|
||||
newNode.ind = i;
|
||||
|
||||
TrellisPath *deviantPath = new TrellisPath(*this, currEdge, newNode, arcLists, pool, system);
|
||||
//cerr << "deviantPath=" << deviantPath << endl;
|
||||
paths.Add(deviantPath);
|
||||
}
|
||||
TrellisPath *deviantPath = new TrellisPath(*this, currEdge, newNode,
|
||||
arcLists, pool, system);
|
||||
//cerr << "deviantPath=" << deviantPath << endl;
|
||||
paths.Add(deviantPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TrellisPath::CalcScores(const Scores &origScores,
|
||||
const Scores &origHypoScores,
|
||||
const Scores &newHypoScores,
|
||||
MemPool &pool,
|
||||
const System &system)
|
||||
const Scores &origHypoScores, const Scores &newHypoScores, MemPool &pool,
|
||||
const System &system)
|
||||
{
|
||||
Scores *scores = new (pool.Allocate<Scores>())
|
||||
Scores(system,
|
||||
pool,
|
||||
system.featureFunctions.GetNumScores(),
|
||||
origScores);
|
||||
scores->PlusEquals(system, newHypoScores);
|
||||
scores->MinusEquals(system, origHypoScores);
|
||||
Scores *scores = new (pool.Allocate<Scores>()) Scores(system, pool,
|
||||
system.featureFunctions.GetNumScores(), origScores);
|
||||
scores->PlusEquals(system, newHypoScores);
|
||||
scores->MinusEquals(system, origHypoScores);
|
||||
|
||||
m_scores = scores;
|
||||
m_scores = scores;
|
||||
}
|
||||
|
||||
void TrellisPath::AddNodes(const Hypothesis *hypo, const ArcLists &arcLists)
|
||||
{
|
||||
if (hypo) {
|
||||
// add this hypo
|
||||
//cerr << "hypo=" << hypo << " " << flush;
|
||||
//cerr << *hypo << endl;
|
||||
const ArcList *list = arcLists.GetArcList(hypo);
|
||||
assert(list);
|
||||
TrellisNode node(*list, 0);
|
||||
nodes.push_back(node);
|
||||
if (hypo) {
|
||||
// add this hypo
|
||||
//cerr << "hypo=" << hypo << " " << flush;
|
||||
//cerr << *hypo << endl;
|
||||
const ArcList *list = arcLists.GetArcList(hypo);
|
||||
assert(list);
|
||||
TrellisNode node(*list, 0);
|
||||
nodes.push_back(node);
|
||||
|
||||
// add prev hypos
|
||||
const Hypothesis *prev = hypo->GetPrevHypo();
|
||||
AddNodes(prev, arcLists);
|
||||
}
|
||||
// add prev hypos
|
||||
const Hypothesis *prev = hypo->GetPrevHypo();
|
||||
AddNodes(prev, arcLists);
|
||||
}
|
||||
}
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -9,7 +9,8 @@
|
||||
#include "../TypeDef.h"
|
||||
#include "../ArcLists.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class Scores;
|
||||
class MemPool;
|
||||
@ -22,62 +23,59 @@ class TrellisNode
|
||||
friend std::ostream& operator<<(std::ostream &, const TrellisNode &);
|
||||
|
||||
public:
|
||||
const ArcList *arcList;
|
||||
size_t ind;
|
||||
const ArcList *arcList;
|
||||
size_t ind;
|
||||
|
||||
TrellisNode(const ArcList &varcList, size_t vind)
|
||||
:arcList(&varcList)
|
||||
,ind(vind)
|
||||
{}
|
||||
TrellisNode(const ArcList &varcList, size_t vind) :
|
||||
arcList(&varcList), ind(vind)
|
||||
{
|
||||
}
|
||||
|
||||
const HypothesisBase *GetHypo() const
|
||||
{ return (*arcList)[ind]; }
|
||||
const HypothesisBase *GetHypo() const
|
||||
{
|
||||
return (*arcList)[ind];
|
||||
}
|
||||
};
|
||||
|
||||
class TrellisPath {
|
||||
class TrellisPath
|
||||
{
|
||||
public:
|
||||
std::vector<TrellisNode> nodes;
|
||||
int prevEdgeChanged;
|
||||
std::vector<TrellisNode> nodes;
|
||||
int prevEdgeChanged;
|
||||
|
||||
/**< the last node that was wiggled to create this path
|
||||
, or NOT_FOUND if this path is the best trans so consist of only hypos
|
||||
*/
|
||||
TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists);
|
||||
/**< the last node that was wiggled to create this path
|
||||
, or NOT_FOUND if this path is the best trans so consist of only hypos
|
||||
*/
|
||||
TrellisPath(const Hypothesis *hypo, const ArcLists &arcLists);
|
||||
|
||||
/** create path from another path, deviate at edgeIndex by using arc instead,
|
||||
* which may change other hypo back from there
|
||||
*/
|
||||
TrellisPath(const TrellisPath &origPath,
|
||||
size_t edgeIndex,
|
||||
const TrellisNode &newNode,
|
||||
const ArcLists &arcLists,
|
||||
MemPool &pool,
|
||||
const System &system);
|
||||
/** create path from another path, deviate at edgeIndex by using arc instead,
|
||||
* which may change other hypo back from there
|
||||
*/
|
||||
TrellisPath(const TrellisPath &origPath, size_t edgeIndex,
|
||||
const TrellisNode &newNode, const ArcLists &arcLists, MemPool &pool,
|
||||
const System &system);
|
||||
|
||||
virtual ~TrellisPath();
|
||||
virtual ~TrellisPath();
|
||||
|
||||
const Scores &GetScores() const
|
||||
{ return *m_scores; }
|
||||
SCORE GetFutureScore() const;
|
||||
const Scores &GetScores() const
|
||||
{
|
||||
return *m_scores;
|
||||
}
|
||||
SCORE GetFutureScore() const;
|
||||
|
||||
void OutputToStream(std::ostream &out, const System &system) const;
|
||||
std::string ToString() const;
|
||||
void OutputToStream(std::ostream &out, const System &system) const;
|
||||
std::string ToString() const;
|
||||
|
||||
//! create a set of next best paths by wiggling 1 of the node at a time.
|
||||
void CreateDeviantPaths(TrellisPaths &paths,
|
||||
const ArcLists &arcLists,
|
||||
MemPool &pool,
|
||||
const System &system) const;
|
||||
void CreateDeviantPaths(TrellisPaths &paths, const ArcLists &arcLists,
|
||||
MemPool &pool, const System &system) const;
|
||||
|
||||
protected:
|
||||
const Scores *m_scores;
|
||||
|
||||
void AddNodes(const Hypothesis *hypo, const ArcLists &arcLists);
|
||||
void CalcScores(const Scores &origScores,
|
||||
const Scores &origHypoScores,
|
||||
const Scores &newHypoScores,
|
||||
MemPool &pool,
|
||||
const System &system);
|
||||
void CalcScores(const Scores &origScores, const Scores &origHypoScores,
|
||||
const Scores &newHypoScores, MemPool &pool, const System &system);
|
||||
};
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -8,18 +8,21 @@
|
||||
#include "TrellisPaths.h"
|
||||
#include "../legacy/Util2.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
TrellisPaths::TrellisPaths() {
|
||||
// TODO Auto-generated constructor stub
|
||||
TrellisPaths::TrellisPaths()
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
TrellisPaths::~TrellisPaths() {
|
||||
while (!empty()) {
|
||||
TrellisPath *path = Get();
|
||||
delete path;
|
||||
}
|
||||
TrellisPaths::~TrellisPaths()
|
||||
{
|
||||
while (!empty()) {
|
||||
TrellisPath *path = Get();
|
||||
delete path;
|
||||
}
|
||||
}
|
||||
|
||||
void TrellisPaths::Add(TrellisPath *trellisPath)
|
||||
|
@ -10,27 +10,33 @@
|
||||
#include <queue>
|
||||
#include "TrellisPath.h"
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
struct CompareTrellisPathCollection {
|
||||
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const {
|
||||
struct CompareTrellisPathCollection
|
||||
{
|
||||
bool operator()(const TrellisPath* pathA, const TrellisPath* pathB) const
|
||||
{
|
||||
return (pathA->GetFutureScore() < pathB->GetFutureScore());
|
||||
}
|
||||
};
|
||||
|
||||
class TrellisPaths {
|
||||
class TrellisPaths
|
||||
{
|
||||
public:
|
||||
TrellisPaths();
|
||||
virtual ~TrellisPaths();
|
||||
|
||||
bool empty() const {
|
||||
bool empty() const
|
||||
{
|
||||
return m_collection.empty();
|
||||
}
|
||||
|
||||
//! add a new entry into collection
|
||||
void Add(TrellisPath *trellisPath);
|
||||
|
||||
TrellisPath *Get() {
|
||||
TrellisPath *Get()
|
||||
{
|
||||
TrellisPath *top = m_collection.top();
|
||||
|
||||
// Detach
|
||||
@ -39,7 +45,8 @@ public:
|
||||
}
|
||||
|
||||
protected:
|
||||
typedef std::priority_queue<TrellisPath*, std::vector<TrellisPath*>, CompareTrellisPathCollection> CollectionType;
|
||||
typedef std::priority_queue<TrellisPath*, std::vector<TrellisPath*>,
|
||||
CompareTrellisPathCollection> CollectionType;
|
||||
CollectionType m_collection;
|
||||
};
|
||||
|
||||
|
@ -11,6 +11,5 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
@ -5,42 +5,48 @@
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
class PhraseImpl : public Phrase, public PhraseImplTemplate<Word>
|
||||
class PhraseImpl: public Phrase, public PhraseImplTemplate<Word>
|
||||
{
|
||||
public:
|
||||
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
|
||||
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab,
|
||||
const System &system, const std::string &str)
|
||||
{
|
||||
std::vector<std::string> toks = Moses2::Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
PhraseImpl *ret;
|
||||
std::vector<std::string> toks = Moses2::Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
PhraseImpl *ret;
|
||||
|
||||
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
|
||||
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
|
||||
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
return ret;
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
return ret;
|
||||
}
|
||||
|
||||
PhraseImpl(MemPool &pool, size_t size)
|
||||
:PhraseImplTemplate<Word>(pool, size)
|
||||
{}
|
||||
PhraseImpl(MemPool &pool, size_t size) :
|
||||
PhraseImplTemplate<Word>(pool, size)
|
||||
{
|
||||
}
|
||||
|
||||
const Word& operator[](size_t pos) const
|
||||
{ return m_words[pos]; }
|
||||
{
|
||||
return m_words[pos];
|
||||
}
|
||||
|
||||
Word& operator[](size_t pos) {
|
||||
return m_words[pos];
|
||||
Word& operator[](size_t pos)
|
||||
{
|
||||
return m_words[pos];
|
||||
}
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_size; }
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
SubPhrase GetSubPhrase(size_t start, size_t size) const
|
||||
{
|
||||
SubPhrase ret(*this, start, size);
|
||||
return ret;
|
||||
SubPhrase ret(*this, start, size);
|
||||
return ret;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
@ -12,8 +12,6 @@
|
||||
#include "Phrase.h"
|
||||
#include "legacy/Util2.h"
|
||||
|
||||
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
class SubPhrase;
|
||||
@ -22,50 +20,50 @@ template<typename WORD>
|
||||
class PhraseImplTemplate
|
||||
{
|
||||
public:
|
||||
PhraseImplTemplate(MemPool &pool, size_t size)
|
||||
:m_size(size)
|
||||
PhraseImplTemplate(MemPool &pool, size_t size) :
|
||||
m_size(size)
|
||||
{
|
||||
m_words = new (pool.Allocate<WORD>(size)) WORD[size];
|
||||
|
||||
}
|
||||
|
||||
PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate ©)
|
||||
:m_size(copy.GetSize())
|
||||
PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate ©) :
|
||||
m_size(copy.GetSize())
|
||||
{
|
||||
m_words = new (pool.Allocate<WORD>(m_size)) WORD[m_size];
|
||||
for (size_t i = 0; i < m_size; ++i) {
|
||||
const WORD &word = copy[i];
|
||||
(*this)[i] = word;
|
||||
}
|
||||
for (size_t i = 0; i < m_size; ++i) {
|
||||
const WORD &word = copy[i];
|
||||
(*this)[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
virtual ~PhraseImplTemplate()
|
||||
{}
|
||||
{
|
||||
}
|
||||
|
||||
size_t GetSize() const
|
||||
{ return m_size; }
|
||||
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
|
||||
protected:
|
||||
size_t m_size;
|
||||
WORD *m_words;
|
||||
|
||||
WORD& operator[](size_t pos) {
|
||||
return m_words[pos];
|
||||
WORD& operator[](size_t pos)
|
||||
{
|
||||
return m_words[pos];
|
||||
}
|
||||
|
||||
void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks, bool addBOSEOS = false)
|
||||
void CreateFromString(FactorCollection &vocab, const System &system,
|
||||
const std::vector<std::string> &toks, bool addBOSEOS = false)
|
||||
{
|
||||
for (size_t i = 0; i < m_size; ++i) {
|
||||
WORD &word = (*this)[i];
|
||||
word.CreateFromString(vocab, system, toks[i]);
|
||||
}
|
||||
for (size_t i = 0; i < m_size; ++i) {
|
||||
WORD &word = (*this)[i];
|
||||
word.CreateFromString(vocab, system, toks[i]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
@ -7,7 +7,7 @@
|
||||
|
||||
#include "Recycler.h"
|
||||
|
||||
namespace Moses2 {
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -10,59 +10,64 @@
|
||||
#include <deque>
|
||||
#include <vector>
|
||||
|
||||
namespace Moses2 {
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
template<typename T>
|
||||
class Recycler {
|
||||
class Recycler
|
||||
{
|
||||
public:
|
||||
Recycler()
|
||||
:m_currInd(0)
|
||||
{}
|
||||
virtual ~Recycler()
|
||||
{}
|
||||
Recycler() :
|
||||
m_currInd(0)
|
||||
{
|
||||
}
|
||||
virtual ~Recycler()
|
||||
{
|
||||
}
|
||||
|
||||
T Get()
|
||||
{
|
||||
if (!m_coll.empty()) {
|
||||
T &obj = m_coll.back();
|
||||
m_coll.pop_back();
|
||||
return obj;
|
||||
}
|
||||
else if (m_currInd) {
|
||||
--m_currInd;
|
||||
T &obj = m_all[m_currInd];
|
||||
return obj;
|
||||
}
|
||||
else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
T Get()
|
||||
{
|
||||
if (!m_coll.empty()) {
|
||||
T &obj = m_coll.back();
|
||||
m_coll.pop_back();
|
||||
return obj;
|
||||
}
|
||||
else if (m_currInd) {
|
||||
--m_currInd;
|
||||
T &obj = m_all[m_currInd];
|
||||
return obj;
|
||||
}
|
||||
else {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void Clear()
|
||||
{
|
||||
m_coll.clear();
|
||||
m_currInd = m_all.size();
|
||||
}
|
||||
void Clear()
|
||||
{
|
||||
m_coll.clear();
|
||||
m_currInd = m_all.size();
|
||||
}
|
||||
|
||||
void Keep(const T& val) {
|
||||
m_all.push_back(val);
|
||||
}
|
||||
void Keep(const T& val)
|
||||
{
|
||||
m_all.push_back(val);
|
||||
}
|
||||
|
||||
void Recycle(const T& val)
|
||||
{
|
||||
m_coll.push_back(val);
|
||||
}
|
||||
void Recycle(const T& val)
|
||||
{
|
||||
m_coll.push_back(val);
|
||||
}
|
||||
|
||||
protected:
|
||||
// all objects we're looking after
|
||||
std::vector<T> m_all;
|
||||
// all objects we're looking after
|
||||
std::vector<T> m_all;
|
||||
|
||||
// pointer to the object that's just been given out.
|
||||
// to give out another obj, must decrement THEN give out
|
||||
size_t m_currInd;
|
||||
// pointer to the object that's just been given out.
|
||||
// to give out another obj, must decrement THEN give out
|
||||
size_t m_currInd;
|
||||
|
||||
// objects that have been give back to us
|
||||
std::deque<T> m_coll;
|
||||
// objects that have been give back to us
|
||||
std::deque<T> m_coll;
|
||||
};
|
||||
|
||||
} /* namespace Moses2 */
|
||||
|
@ -14,31 +14,28 @@ namespace Moses2
|
||||
namespace SCFG
|
||||
{
|
||||
|
||||
InputPath::InputPath(MemPool &pool,
|
||||
const SubPhrase &subPhrase,
|
||||
const Range &range,
|
||||
size_t numPt,
|
||||
const InputPath *prefixPath)
|
||||
:InputPathBase(pool, subPhrase, range, numPt, prefixPath)
|
||||
InputPath::InputPath(MemPool &pool, const SubPhrase &subPhrase,
|
||||
const Range &range, size_t numPt, const InputPath *prefixPath) :
|
||||
InputPathBase(pool, subPhrase, range, numPt, prefixPath)
|
||||
{
|
||||
m_activeChart = pool.Allocate<ActiveChart>(numPt);
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
ActiveChart &memAddr = m_activeChart[i];
|
||||
ActiveChart *obj =new (&memAddr) ActiveChart();
|
||||
ActiveChart &memAddr = m_activeChart[i];
|
||||
ActiveChart *obj = new (&memAddr) ActiveChart();
|
||||
}
|
||||
}
|
||||
|
||||
InputPath::~InputPath() {
|
||||
// TODO Auto-generated destructor stub
|
||||
InputPath::~InputPath()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream &out, const InputPath &obj)
|
||||
{
|
||||
out << obj.range << " " << obj.subPhrase;
|
||||
return out;
|
||||
out << obj.range << " " << obj.subPhrase;
|
||||
return out;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
@ -18,33 +18,37 @@ namespace SCFG
|
||||
class ActiveChartEntry
|
||||
{
|
||||
public:
|
||||
const void *data;
|
||||
const void *data;
|
||||
|
||||
ActiveChartEntry(const void *vdata)
|
||||
:data(vdata)
|
||||
{}
|
||||
ActiveChartEntry(const void *vdata) :
|
||||
data(vdata)
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
class ActiveChart
|
||||
{
|
||||
public:
|
||||
std::vector<ActiveChartEntry*> entries;
|
||||
std::vector<ActiveChartEntry*> entries;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
class InputPath : public InputPathBase
|
||||
class InputPath: public InputPathBase
|
||||
{
|
||||
friend std::ostream& operator<<(std::ostream &, const InputPath &);
|
||||
friend std::ostream& operator<<(std::ostream &, const InputPath &);
|
||||
public:
|
||||
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range, size_t numPt, const InputPath *prefixPath);
|
||||
virtual ~InputPath();
|
||||
InputPath(MemPool &pool, const SubPhrase &subPhrase, const Range &range,
|
||||
size_t numPt, const InputPath *prefixPath);
|
||||
virtual ~InputPath();
|
||||
|
||||
ActiveChart &GetActiveChart(size_t ptInd)
|
||||
{ return m_activeChart[ptInd]; }
|
||||
ActiveChart &GetActiveChart(size_t ptInd)
|
||||
{
|
||||
return m_activeChart[ptInd];
|
||||
}
|
||||
|
||||
protected:
|
||||
ActiveChart *m_activeChart;
|
||||
ActiveChart *m_activeChart;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -25,35 +25,36 @@ void InputPaths::Init(const Sentence &input, const ManagerBase &mgr)
|
||||
size_t size = input.GetSize();
|
||||
size_t maxLength = min(size, mgr.system.maxPhraseLength);
|
||||
|
||||
m_matrix = new (pool.Allocate< Matrix<InputPath*> >()) Matrix<InputPath*>(pool, size, maxLength);
|
||||
m_matrix = new (pool.Allocate<Matrix<InputPath*> >()) Matrix<InputPath*>(pool,
|
||||
size, maxLength);
|
||||
m_matrix->Init(NULL);
|
||||
|
||||
// create normal paths of subphrases through the sentence
|
||||
for (size_t startPos = 0; startPos < size; ++startPos) {
|
||||
const InputPath *prefixPath = NULL;
|
||||
const InputPath *prefixPath = NULL;
|
||||
|
||||
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
|
||||
size_t endPos = startPos + phaseSize - 1;
|
||||
for (size_t phaseSize = 1; phaseSize <= maxLength; ++phaseSize) {
|
||||
size_t endPos = startPos + phaseSize - 1;
|
||||
|
||||
if (endPos >= size) {
|
||||
break;
|
||||
}
|
||||
if (endPos >= size) {
|
||||
break;
|
||||
}
|
||||
|
||||
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
|
||||
Range range(startPos, endPos);
|
||||
SubPhrase subPhrase = input.GetSubPhrase(startPos, phaseSize);
|
||||
Range range(startPos, endPos);
|
||||
|
||||
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool, subPhrase, range, numPt, prefixPath);
|
||||
m_inputPaths.push_back(path);
|
||||
InputPath *path = new (pool.Allocate<InputPath>()) InputPath(pool,
|
||||
subPhrase, range, numPt, prefixPath);
|
||||
m_inputPaths.push_back(path);
|
||||
|
||||
prefixPath = path;
|
||||
prefixPath = path;
|
||||
|
||||
m_matrix->SetValue(startPos, phaseSize - 1, path);
|
||||
}
|
||||
m_matrix->SetValue(startPos, phaseSize - 1, path);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user