mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-26 19:37:58 +03:00
Code reorganization and code cleanup.
This commit is contained in:
parent
006297831c
commit
db013975b0
@ -40,175 +40,146 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "moses/FF/InputFeature.h"
|
||||
#include "util/exception.hh"
|
||||
|
||||
#include <boost/foreach.hpp>
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
/** helper for pruning */
|
||||
bool CompareTranslationOption(const TranslationOption *a, const TranslationOption *b)
|
||||
{
|
||||
return a->GetFutureScore() > b->GetFutureScore();
|
||||
}
|
||||
// bool CompareTranslationOption(const TranslationOption *a, const TranslationOption *b)
|
||||
// {
|
||||
// return a->GetFutureScore() > b->GetFutureScore();
|
||||
// }
|
||||
|
||||
/** constructor; since translation options are indexed by coverage span, the corresponding data structure is initialized here
|
||||
* This fn should be called by inherited classes
|
||||
*/
|
||||
TranslationOptionCollection::TranslationOptionCollection(
|
||||
InputType const& src, size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
|
||||
/** constructor; since translation options are indexed by coverage span, the
|
||||
* corresponding data structure is initialized here This fn should be
|
||||
* called by inherited classe */
|
||||
TranslationOptionCollection::
|
||||
TranslationOptionCollection(InputType const& src, size_t maxNoTransOptPerCoverage,
|
||||
float translationOptionThreshold)
|
||||
: m_source(src)
|
||||
,m_futureScore(src.GetSize())
|
||||
,m_maxNoTransOptPerCoverage(maxNoTransOptPerCoverage)
|
||||
,m_translationOptionThreshold(translationOptionThreshold)
|
||||
, m_futureScore(src.GetSize())
|
||||
, m_maxNoTransOptPerCoverage(maxNoTransOptPerCoverage)
|
||||
, m_translationOptionThreshold(translationOptionThreshold)
|
||||
{
|
||||
// create 2-d vector
|
||||
size_t size = src.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
|
||||
for (size_t sPos = 0 ; sPos < size ; ++sPos) {
|
||||
m_collection.push_back( vector< TranslationOptionList >() );
|
||||
|
||||
size_t maxSize = size - startPos;
|
||||
size_t maxSize = size - sPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = 0 ; endPos < maxSize ; ++endPos) {
|
||||
m_collection[startPos].push_back( TranslationOptionList() );
|
||||
for (size_t ePos = 0 ; ePos < maxSize ; ++ePos) {
|
||||
m_collection[sPos].push_back( TranslationOptionList() );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** destructor, clears out data structures */
|
||||
TranslationOptionCollection::~TranslationOptionCollection()
|
||||
TranslationOptionCollection::
|
||||
~TranslationOptionCollection()
|
||||
{
|
||||
RemoveAllInColl(m_inputPathQueue);
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::Prune()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
Prune()
|
||||
{
|
||||
// quit, if max size, threshold
|
||||
if (m_maxNoTransOptPerCoverage == 0 && m_translationOptionThreshold == -std::numeric_limits<float>::infinity())
|
||||
return;
|
||||
static float no_th = -std::numeric_limits<float>::infinity();
|
||||
|
||||
if (m_maxNoTransOptPerCoverage == 0 && m_translationOptionThreshold == no_th)
|
||||
return;
|
||||
|
||||
// bookkeeping for how many options used, pruned
|
||||
size_t total = 0;
|
||||
size_t totalPruned = 0;
|
||||
|
||||
// loop through all spans
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size; ++startPos) {
|
||||
size_t maxSize = size - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
|
||||
// consider list for a span
|
||||
TranslationOptionList &fullList = GetTranslationOptionList(startPos, endPos);
|
||||
total += fullList.size();
|
||||
|
||||
// size pruning
|
||||
if (m_maxNoTransOptPerCoverage > 0 &&
|
||||
fullList.size() > m_maxNoTransOptPerCoverage) {
|
||||
// sort in vector
|
||||
NTH_ELEMENT4(fullList.begin(), fullList.begin() + m_maxNoTransOptPerCoverage, fullList.end(), CompareTranslationOption);
|
||||
totalPruned += fullList.size() - m_maxNoTransOptPerCoverage;
|
||||
|
||||
// delete the rest
|
||||
for (size_t i = m_maxNoTransOptPerCoverage ; i < fullList.size() ; ++i) {
|
||||
delete fullList.Get(i);
|
||||
for (size_t sPos = 0 ; sPos < size; ++sPos)
|
||||
{
|
||||
BOOST_FOREACH(TranslationOptionList& fullList, m_collection[sPos])
|
||||
{
|
||||
total += fullList.size();
|
||||
totalPruned += fullList.SelectNBest(m_maxNoTransOptPerCoverage);
|
||||
totalPruned += fullList.PruneByThreshold(m_translationOptionThreshold);
|
||||
}
|
||||
fullList.resize(m_maxNoTransOptPerCoverage);
|
||||
}
|
||||
|
||||
// threshold pruning
|
||||
if (fullList.size() > 1 && m_translationOptionThreshold != -std::numeric_limits<float>::infinity()) {
|
||||
// first, find the best score
|
||||
float bestScore = -std::numeric_limits<float>::infinity();
|
||||
for (size_t i=0; i < fullList.size() ; ++i) {
|
||||
if (fullList.Get(i)->GetFutureScore() > bestScore)
|
||||
bestScore = fullList.Get(i)->GetFutureScore();
|
||||
}
|
||||
//std::cerr << "best score for span " << startPos << "-" << endPos << " is " << bestScore << "\n";
|
||||
// then, remove items that are worse than best score + threshold
|
||||
for (size_t i=0; i < fullList.size() ; ++i) {
|
||||
if (fullList.Get(i)->GetFutureScore() < bestScore + m_translationOptionThreshold) {
|
||||
//std::cerr << "\tremoving item " << i << ", score " << fullList.Get(i)->GetFutureScore() << ": " << fullList.Get(i)->GetTargetPhrase() << "\n";
|
||||
delete fullList.Get(i);
|
||||
fullList.Remove(i);
|
||||
total--;
|
||||
totalPruned++;
|
||||
i--;
|
||||
}
|
||||
//else
|
||||
//{
|
||||
// std::cerr << "\tkeeping item " << i << ", score " << fullList.Get(i)->GetFutureScore() << ": " << fullList.Get(i)->GetTargetPhrase() << "\n";
|
||||
//}
|
||||
}
|
||||
} // end of threshold pruning
|
||||
}
|
||||
} // end of loop through all spans
|
||||
|
||||
|
||||
VERBOSE(2," Total translation options: " << total << std::endl
|
||||
<< "Total translation options pruned: " << totalPruned << std::endl);
|
||||
}
|
||||
|
||||
/** Force a creation of a translation option where there are none for a particular source position.
|
||||
* ie. where a source word has not been translated, create a translation option by
|
||||
* 1. not observing the table limits on phrase/generation tables
|
||||
* 2. using the handler ProcessUnknownWord()
|
||||
* Call this function once translation option collection has been filled with translation options
|
||||
/** Force a creation of a translation option where there are none for a
|
||||
* particular source position. ie. where a source word has not been
|
||||
* translated, create a translation option by
|
||||
* 1. not observing the table limits on phrase/generation tables
|
||||
* 2. using the handler ProcessUnknownWord()
|
||||
* Call this function once translation option collection has been filled with
|
||||
* translation options
|
||||
*
|
||||
* This function calls for unknown words is complicated by the fact it must handle different input types.
|
||||
* The call stack is
|
||||
* Base::ProcessUnknownWord()
|
||||
* Inherited::ProcessUnknownWord(position)
|
||||
* Base::ProcessOneUnknownWord()
|
||||
* This function calls for unknown words is complicated by the fact it must
|
||||
* handle different input types. The call stack is
|
||||
* Base::ProcessUnknownWord()
|
||||
* Inherited::ProcessUnknownWord(position)
|
||||
* Base::ProcessOneUnknownWord()
|
||||
*
|
||||
*/
|
||||
|
||||
void TranslationOptionCollection::ProcessUnknownWord()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
ProcessUnknownWord()
|
||||
{
|
||||
const vector<DecodeGraph*>& decodeGraphList = StaticData::Instance().GetDecodeGraphs();
|
||||
const vector<DecodeGraph*>& decodeGraphList
|
||||
= StaticData::Instance().GetDecodeGraphs();
|
||||
size_t size = m_source.GetSize();
|
||||
// try to translation for coverage with no trans by expanding table limit
|
||||
for (size_t graphInd = 0 ; graphInd < decodeGraphList.size() ; graphInd++) {
|
||||
const DecodeGraph &decodeGraph = *decodeGraphList[graphInd];
|
||||
for (size_t pos = 0 ; pos < size ; ++pos) {
|
||||
TranslationOptionList &fullList = GetTranslationOptionList(pos, pos);
|
||||
size_t numTransOpt = fullList.size();
|
||||
if (numTransOpt == 0) {
|
||||
TranslationOptionList* fullList = GetTranslationOptionList(pos, pos);
|
||||
// size_t numTransOpt = fullList.size();
|
||||
if (!fullList || fullList->size() == 0) {
|
||||
CreateTranslationOptionsForRange(decodeGraph, pos, pos, false, graphInd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
bool alwaysCreateDirectTranslationOption
|
||||
= StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
// create unknown words for 1 word coverage where we don't have any trans options
|
||||
for (size_t pos = 0 ; pos < size ; ++pos) {
|
||||
TranslationOptionList &fullList = GetTranslationOptionList(pos, pos);
|
||||
if (fullList.size() == 0 || alwaysCreateDirectTranslationOption)
|
||||
TranslationOptionList* fullList = GetTranslationOptionList(pos, pos);
|
||||
if (!fullList || fullList->size() == 0 || alwaysCreateDirectTranslationOption)
|
||||
ProcessUnknownWord(pos);
|
||||
}
|
||||
}
|
||||
|
||||
/** special handling of ONE unknown words. Either add temporarily add word to translation table,
|
||||
* or drop the translation.
|
||||
* This function should be called by the ProcessOneUnknownWord() in the inherited class
|
||||
* At the moment, this unknown word handler is a bit of a hack, if copies over each factor from source
|
||||
* to target word, or uses the 'UNK' factor.
|
||||
* Ideally, this function should be in a class which can be expanded upon, for example,
|
||||
* to create a morphologically aware handler.
|
||||
*
|
||||
* \param sourceWord the unknown word
|
||||
* \param sourcePos
|
||||
* \param length length covered by this word (may be > 1 for lattice input)
|
||||
* \param inputScores a set of scores associated with unknown word (input scores from latties/CNs)
|
||||
/** special handling of ONE unknown words. Either add temporarily add word to
|
||||
* translation table, or drop the translation. This function should be
|
||||
* called by the ProcessOneUnknownWord() in the inherited class At the
|
||||
* moment, this unknown word handler is a bit of a hack, if copies over
|
||||
* each factor from source to target word, or uses the 'UNK' factor.
|
||||
* Ideally, this function should be in a class which can be expanded
|
||||
* upon, for example, to create a morphologically aware handler.
|
||||
*
|
||||
* \param sourceWord the unknown word
|
||||
* \param sourcePos
|
||||
* \param length length covered by this word (may be > 1 for lattice input)
|
||||
* \param inputScores a set of scores associated with unknown word (input scores from latties/CNs)
|
||||
*/
|
||||
void TranslationOptionCollection::ProcessOneUnknownWord(const InputPath &inputPath,
|
||||
size_t sourcePos,
|
||||
size_t length,
|
||||
const ScorePair *inputScores)
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
ProcessOneUnknownWord(const InputPath &inputPath, size_t sourcePos,
|
||||
size_t length, const ScorePair *inputScores)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const UnknownWordPenaltyProducer &unknownWordPenaltyProducer = UnknownWordPenaltyProducer::Instance();
|
||||
const UnknownWordPenaltyProducer&
|
||||
unknownWordPenaltyProducer = UnknownWordPenaltyProducer::Instance();
|
||||
float unknownScore = FloorScore(TransformScore(0));
|
||||
const Word &sourceWord = inputPath.GetPhrase().GetWord(0);
|
||||
|
||||
@ -257,14 +228,9 @@ void TranslationOptionCollection::ProcessOneUnknownWord(const InputPath &inputPa
|
||||
//create a one-to-one alignment between UNKNOWN_FACTOR and its verbatim translation
|
||||
|
||||
targetPhrase.SetAlignmentInfo("0-0");
|
||||
|
||||
} else {
|
||||
// drop source word. create blank trans opt
|
||||
|
||||
//targetPhrase.SetAlignment();
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
targetPhrase.GetScoreBreakdown().Assign(&unknownWordPenaltyProducer, unknownScore);
|
||||
|
||||
// source phrase
|
||||
@ -285,35 +251,35 @@ void TranslationOptionCollection::ProcessOneUnknownWord(const InputPath &inputPa
|
||||
* This matrix used in search.
|
||||
* Call this function once translation option collection has been filled with translation options
|
||||
*/
|
||||
void TranslationOptionCollection::CalcFutureScore()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
CalcFutureScore()
|
||||
{
|
||||
// setup the matrix (ignore lower triangle, set upper triangle to -inf
|
||||
size_t size = m_source.GetSize(); // the width of the matrix
|
||||
|
||||
for(size_t row=0; row<size; row++) {
|
||||
for(size_t row=0; row < size; row++) {
|
||||
for(size_t col=row; col<size; col++) {
|
||||
m_futureScore.SetScore(row, col, -numeric_limits<float>::infinity());
|
||||
}
|
||||
}
|
||||
|
||||
// walk all the translation options and record the cheapest option for each span
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
|
||||
size_t maxSize = m_source.GetSize() - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
|
||||
TranslationOptionList &transOptList = GetTranslationOptionList(startPos, endPos);
|
||||
|
||||
TranslationOptionList::const_iterator iterTransOpt;
|
||||
for(iterTransOpt = transOptList.begin() ; iterTransOpt != transOptList.end() ; ++iterTransOpt) {
|
||||
const TranslationOption &transOpt = **iterTransOpt;
|
||||
float score = transOpt.GetFutureScore();
|
||||
if (score > m_futureScore.GetScore(startPos, endPos))
|
||||
m_futureScore.SetScore(startPos, endPos, score);
|
||||
}
|
||||
for (size_t sPos = 0 ; sPos < size ; ++sPos)
|
||||
{
|
||||
size_t ePos = sPos;
|
||||
BOOST_FOREACH(TranslationOptionList& tol, m_collection[sPos])
|
||||
{
|
||||
TranslationOptionList::const_iterator toi;
|
||||
for(toi = tol.begin() ; toi != tol.end() ; ++toi) {
|
||||
const TranslationOption& to = **toi;
|
||||
float score = to.GetFutureScore();
|
||||
if (score > m_futureScore.GetScore(sPos, ePos))
|
||||
m_futureScore.SetScore(sPos, ePos, score);
|
||||
}
|
||||
++ePos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// now fill all the cells in the strictly upper triangle
|
||||
// there is no way to modify the diagonal now, in the case
|
||||
@ -324,42 +290,52 @@ void TranslationOptionCollection::CalcFutureScore()
|
||||
|
||||
for(size_t colstart = 1; colstart < size ; colstart++) {
|
||||
for(size_t diagshift = 0; diagshift < size-colstart ; diagshift++) {
|
||||
size_t startPos = diagshift;
|
||||
size_t endPos = colstart+diagshift;
|
||||
for(size_t joinAt = startPos; joinAt < endPos ; joinAt++) {
|
||||
float joinedScore = m_futureScore.GetScore(startPos, joinAt)
|
||||
+ m_futureScore.GetScore(joinAt+1, endPos);
|
||||
/* // uncomment to see the cell filling scheme
|
||||
TRACE_ERR( "[" <<startPos<<","<<endPos<<"] <-? ["<<startPos<<","<<joinAt<<"]+["<<joinAt+1<<","<<endPos
|
||||
<< "] (colstart: "<<colstart<<", diagshift: "<<diagshift<<")"<<endl);
|
||||
*/
|
||||
if (joinedScore > m_futureScore.GetScore(startPos, endPos))
|
||||
m_futureScore.SetScore(startPos, endPos, joinedScore);
|
||||
size_t sPos = diagshift;
|
||||
size_t ePos = colstart+diagshift;
|
||||
for(size_t joinAt = sPos; joinAt < ePos ; joinAt++) {
|
||||
float joinedScore = m_futureScore.GetScore(sPos, joinAt)
|
||||
+ m_futureScore.GetScore(joinAt+1, ePos);
|
||||
// uncomment to see the cell filling scheme
|
||||
// TRACE_ERR("[" << sPos << "," << ePos << "] <-? ["
|
||||
// << sPos << "," << joinAt << "]+["
|
||||
// << joinAt+1 << "," << ePos << "] (colstart: "
|
||||
// << colstart << ", diagshift: " << diagshift << ")"
|
||||
// << endl);
|
||||
|
||||
if (joinedScore > m_futureScore.GetScore(sPos, ePos))
|
||||
m_futureScore.SetScore(sPos, ePos, joinedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
IFVERBOSE(3)
|
||||
{
|
||||
int total = 0;
|
||||
for(size_t row = 0; row < size; row++)
|
||||
{
|
||||
size_t col = row;
|
||||
BOOST_FOREACH(TranslationOptionList& tol, m_collection[row])
|
||||
{
|
||||
// size_t maxSize = size - row;
|
||||
// size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
// maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
IFVERBOSE(3) {
|
||||
int total = 0;
|
||||
for(size_t row=0; row<size; row++) {
|
||||
size_t maxSize = size - row;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for(size_t col=row; col<row+maxSize; col++) {
|
||||
int count = GetTranslationOptionList(row, col).size();
|
||||
TRACE_ERR( "translation options spanning from "
|
||||
<< row <<" to "<< col <<" is "
|
||||
<< count <<endl);
|
||||
total += count;
|
||||
}
|
||||
// for(size_t col=row; col<row+maxSize; col++) {
|
||||
int count = tol.size();
|
||||
TRACE_ERR( "translation options spanning from "
|
||||
<< row <<" to "<< col <<" is "
|
||||
<< count <<endl);
|
||||
total += count;
|
||||
++col;
|
||||
}
|
||||
}
|
||||
TRACE_ERR( "translation options generated in total: "<< total << endl);
|
||||
|
||||
for(size_t row=0; row<size; row++)
|
||||
for(size_t col=row; col<size; col++)
|
||||
TRACE_ERR( "future cost from "<< row <<" to "<< col <<" is "
|
||||
<< m_futureScore.GetScore(row, col) <<endl);
|
||||
}
|
||||
TRACE_ERR( "translation options generated in total: "<< total << endl);
|
||||
|
||||
for(size_t row=0; row<size; row++)
|
||||
for(size_t col=row; col<size; col++)
|
||||
TRACE_ERR( "future cost from "<< row <<" to "<< col <<" is "<< m_futureScore.GetScore(row, col) <<endl);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -368,7 +344,9 @@ void TranslationOptionCollection::CalcFutureScore()
|
||||
* for a particular input sentence. This implies applying all
|
||||
* translation and generation steps. Also computes future cost matrix.
|
||||
*/
|
||||
void TranslationOptionCollection::CreateTranslationOptions()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
CreateTranslationOptions()
|
||||
{
|
||||
// loop over all substrings of the source sentence, look them up
|
||||
// in the phraseDictionary (which is the- possibly filtered-- phrase
|
||||
@ -376,158 +354,142 @@ void TranslationOptionCollection::CreateTranslationOptions()
|
||||
// for all phrases
|
||||
|
||||
// there may be multiple decoding graphs (factorizations of decoding)
|
||||
const vector <DecodeGraph*> &decodeGraphList = StaticData::Instance().GetDecodeGraphs();
|
||||
const vector <DecodeGraph*> &decodeGraphList
|
||||
= StaticData::Instance().GetDecodeGraphs();
|
||||
|
||||
// length of the sentence
|
||||
const size_t size = m_source.GetSize();
|
||||
|
||||
// loop over all decoding graphs, each generates translation options
|
||||
for (size_t graphInd = 0 ; graphInd < decodeGraphList.size() ; graphInd++) {
|
||||
if (decodeGraphList.size() > 1) {
|
||||
VERBOSE(3,"Creating translation options from decoding graph " << graphInd << endl);
|
||||
for (size_t gidx = 0 ; gidx < decodeGraphList.size() ; gidx++)
|
||||
{
|
||||
if (decodeGraphList.size() > 1)
|
||||
VERBOSE(3,"Creating translation options from decoding graph " << gidx << endl);
|
||||
|
||||
const DecodeGraph& dg = *decodeGraphList[gidx];
|
||||
size_t backoff = dg.GetBackoff();
|
||||
// iterate over spans
|
||||
for (size_t sPos = 0 ; sPos < size; sPos++)
|
||||
{
|
||||
size_t maxSize = size - sPos; // don't go over end of sentence
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t ePos = sPos ; ePos < sPos + maxSize ; ePos++)
|
||||
{
|
||||
if (gidx && backoff &&
|
||||
(ePos-sPos+1 <= backoff || // size exceeds backoff limit (HUH? UG) or ...
|
||||
m_collection[sPos][ePos-sPos].size() > 0))
|
||||
{
|
||||
VERBOSE(3,"No backoff to graph " << gidx << " for span [" << sPos << ";" << ePos << "]" << endl);
|
||||
continue;
|
||||
}
|
||||
CreateTranslationOptionsForRange(dg, sPos, ePos, true, gidx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DecodeGraph &decodeGraph = *decodeGraphList[graphInd];
|
||||
size_t backoff = decodeGraph.GetBackoff();
|
||||
// generate phrases that start at startPos ...
|
||||
for (size_t startPos = 0 ; startPos < size; startPos++) {
|
||||
size_t maxSize = size - startPos; // don't go over end of sentence
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
// ... and that end at endPos
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; endPos++) {
|
||||
if (graphInd > 0 && // only skip subsequent graphs
|
||||
backoff != 0 && // use of backoff specified
|
||||
(endPos-startPos+1 <= backoff || // size exceeds backoff limit or ...
|
||||
m_collection[startPos][endPos-startPos].size() > 0)) { // no phrases found so far
|
||||
VERBOSE(3,"No backoff to graph " << graphInd << " for span [" << startPos << ";" << endPos << "]" << endl);
|
||||
// do not create more options
|
||||
continue;
|
||||
}
|
||||
|
||||
// create translation options for that range
|
||||
CreateTranslationOptionsForRange( decodeGraph, startPos, endPos, true, graphInd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
VERBOSE(3,"Translation Option Collection\n " << *this << endl);
|
||||
|
||||
ProcessUnknownWord();
|
||||
|
||||
EvaluateWithSourceContext();
|
||||
|
||||
// Prune
|
||||
Prune();
|
||||
|
||||
Sort();
|
||||
|
||||
// future score matrix
|
||||
CalcFutureScore();
|
||||
|
||||
// Cached lex reodering costs
|
||||
CacheLexReordering();
|
||||
CalcFutureScore(); // future score matrix
|
||||
CacheLexReordering(); // Cached lex reodering costs
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
const DecodeGraph &decodeGraph
|
||||
, size_t startPos
|
||||
, size_t endPos
|
||||
, bool adhereTableLimit
|
||||
, size_t graphInd
|
||||
, InputPath &inputPath)
|
||||
|
||||
bool
|
||||
TranslationOptionCollection::
|
||||
CreateTranslationOptionsForRange
|
||||
(const DecodeGraph& dg, size_t sPos, size_t ePos,
|
||||
bool adhereTableLimit, size_t gidx, InputPath &inputPath)
|
||||
{
|
||||
if ((StaticData::Instance().GetXmlInputType() != XmlExclusive) || !HasXmlOptionsOverlappingRange(startPos,endPos)) {
|
||||
if ((StaticData::Instance().GetXmlInputType() != XmlExclusive)
|
||||
|| !HasXmlOptionsOverlappingRange(sPos,ePos))
|
||||
{
|
||||
|
||||
// partial trans opt stored in here
|
||||
PartialTranslOptColl* oldPtoc = new PartialTranslOptColl;
|
||||
size_t totalEarlyPruned = 0;
|
||||
// partial trans opt stored in here
|
||||
PartialTranslOptColl* oldPtoc = new PartialTranslOptColl;
|
||||
size_t totalEarlyPruned = 0;
|
||||
|
||||
// initial translation step
|
||||
list <const DecodeStep* >::const_iterator iterStep = decodeGraph.begin();
|
||||
const DecodeStep &decodeStep = **iterStep;
|
||||
// initial translation step
|
||||
list <const DecodeStep* >::const_iterator iterStep = dg.begin();
|
||||
const DecodeStep &dstep = **iterStep;
|
||||
|
||||
const PhraseDictionary &phraseDictionary = *decodeStep.GetPhraseDictionaryFeature();
|
||||
const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(phraseDictionary);
|
||||
const PhraseDictionary &pdict = *dstep.GetPhraseDictionaryFeature();
|
||||
const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(pdict);
|
||||
|
||||
static_cast<const DecodeStepTranslation&>(dstep).ProcessInitialTranslation
|
||||
(m_source, *oldPtoc, sPos, ePos, adhereTableLimit, inputPath, targetPhrases);
|
||||
|
||||
static_cast<const DecodeStepTranslation&>(decodeStep).ProcessInitialTranslation
|
||||
(m_source, *oldPtoc
|
||||
, startPos, endPos, adhereTableLimit
|
||||
, inputPath, targetPhrases);
|
||||
SetInputScore(inputPath, *oldPtoc);
|
||||
|
||||
SetInputScore(inputPath, *oldPtoc);
|
||||
// do rest of decode steps
|
||||
int indexStep = 0;
|
||||
|
||||
// do rest of decode steps
|
||||
int indexStep = 0;
|
||||
for (++iterStep ; iterStep != dg.end() ; ++iterStep) {
|
||||
|
||||
const DecodeStep *dstep = *iterStep;
|
||||
PartialTranslOptColl* newPtoc = new PartialTranslOptColl;
|
||||
|
||||
// go thru each intermediate trans opt just created
|
||||
const vector<TranslationOption*>& partTransOptList = oldPtoc->GetList();
|
||||
vector<TranslationOption*>::const_iterator iterPartialTranslOpt;
|
||||
for (iterPartialTranslOpt = partTransOptList.begin() ; iterPartialTranslOpt != partTransOptList.end() ; ++iterPartialTranslOpt) {
|
||||
TranslationOption &inputPartialTranslOpt = **iterPartialTranslOpt;
|
||||
|
||||
for (++iterStep ; iterStep != decodeGraph.end() ; ++iterStep) {
|
||||
|
||||
const DecodeStep *decodeStep = *iterStep;
|
||||
PartialTranslOptColl* newPtoc = new PartialTranslOptColl;
|
||||
|
||||
// go thru each intermediate trans opt just created
|
||||
const vector<TranslationOption*>& partTransOptList = oldPtoc->GetList();
|
||||
vector<TranslationOption*>::const_iterator iterPartialTranslOpt;
|
||||
for (iterPartialTranslOpt = partTransOptList.begin() ; iterPartialTranslOpt != partTransOptList.end() ; ++iterPartialTranslOpt) {
|
||||
TranslationOption &inputPartialTranslOpt = **iterPartialTranslOpt;
|
||||
|
||||
if (const DecodeStepTranslation *translateStep = dynamic_cast<const DecodeStepTranslation*>(decodeStep) ) {
|
||||
const PhraseDictionary &phraseDictionary = *translateStep->GetPhraseDictionaryFeature();
|
||||
const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(phraseDictionary);
|
||||
translateStep->Process(inputPartialTranslOpt
|
||||
, *decodeStep
|
||||
, *newPtoc
|
||||
, this
|
||||
, adhereTableLimit
|
||||
, targetPhrases);
|
||||
} else {
|
||||
const DecodeStepGeneration *genStep = dynamic_cast<const DecodeStepGeneration*>(decodeStep);
|
||||
assert(genStep);
|
||||
genStep->Process(inputPartialTranslOpt
|
||||
, *decodeStep
|
||||
, *newPtoc
|
||||
, this
|
||||
, adhereTableLimit);
|
||||
}
|
||||
}
|
||||
|
||||
// last but 1 partial trans not required anymore
|
||||
totalEarlyPruned += newPtoc->GetPrunedCount();
|
||||
delete oldPtoc;
|
||||
oldPtoc = newPtoc;
|
||||
|
||||
indexStep++;
|
||||
} // for (++iterStep
|
||||
|
||||
// add to fully formed translation option list
|
||||
PartialTranslOptColl &lastPartialTranslOptColl = *oldPtoc;
|
||||
const vector<TranslationOption*>& partTransOptList = lastPartialTranslOptColl.GetList();
|
||||
vector<TranslationOption*>::const_iterator iterColl;
|
||||
for (iterColl = partTransOptList.begin() ; iterColl != partTransOptList.end() ; ++iterColl) {
|
||||
TranslationOption *transOpt = *iterColl;
|
||||
if (StaticData::Instance().GetXmlInputType() != XmlConstraint || !ViolatesXmlOptionsConstraint(startPos,endPos,transOpt)) {
|
||||
if (const DecodeStepTranslation *tstep = dynamic_cast<const DecodeStepTranslation*>(dstep) )
|
||||
{
|
||||
const PhraseDictionary &pdict = *tstep->GetPhraseDictionaryFeature();
|
||||
const TargetPhraseCollection *targetPhrases = inputPath.GetTargetPhrases(pdict);
|
||||
tstep->Process(inputPartialTranslOpt, *dstep, *newPtoc,
|
||||
this, adhereTableLimit, targetPhrases);
|
||||
} else {
|
||||
const DecodeStepGeneration *genStep = dynamic_cast<const DecodeStepGeneration*>(dstep);
|
||||
assert(genStep);
|
||||
genStep->Process(inputPartialTranslOpt, *dstep, *newPtoc,
|
||||
this, adhereTableLimit);
|
||||
}
|
||||
}
|
||||
|
||||
// last but 1 partial trans not required anymore
|
||||
totalEarlyPruned += newPtoc->GetPrunedCount();
|
||||
delete oldPtoc;
|
||||
oldPtoc = newPtoc;
|
||||
|
||||
indexStep++;
|
||||
} // for (++iterStep
|
||||
|
||||
// add to fully formed translation option list
|
||||
PartialTranslOptColl &lastPartialTranslOptColl = *oldPtoc;
|
||||
const vector<TranslationOption*>& partTransOptList = lastPartialTranslOptColl.GetList();
|
||||
vector<TranslationOption*>::const_iterator iterColl;
|
||||
for (iterColl = partTransOptList.begin() ; iterColl != partTransOptList.end() ; ++iterColl) {
|
||||
TranslationOption *transOpt = *iterColl;
|
||||
if (StaticData::Instance().GetXmlInputType() != XmlConstraint || !ViolatesXmlOptionsConstraint(sPos,ePos,transOpt)) {
|
||||
Add(transOpt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
lastPartialTranslOptColl.DetachAll();
|
||||
totalEarlyPruned += oldPtoc->GetPrunedCount();
|
||||
delete oldPtoc;
|
||||
// TRACE_ERR( "Early translation options pruned: " << totalEarlyPruned << endl);
|
||||
} // if ((StaticData::Instance().GetXmlInputType() != XmlExclusive) || !HasXmlOptionsOverlappingRange(startPos,endPos))
|
||||
|
||||
if (graphInd == 0 && StaticData::Instance().GetXmlInputType() != XmlPassThrough && HasXmlOptionsOverlappingRange(startPos,endPos)) {
|
||||
CreateXmlOptionsForRange(startPos, endPos);
|
||||
} // if ((StaticData::Instance().GetXmlInputType() != XmlExclusive) || !HasXmlOptionsOverlappingRange(sPos,ePos))
|
||||
|
||||
if (gidx == 0 && StaticData::Instance().GetXmlInputType() != XmlPassThrough && HasXmlOptionsOverlappingRange(sPos,ePos)) {
|
||||
CreateXmlOptionsForRange(sPos, ePos);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::SetInputScore(const InputPath &inputPath, PartialTranslOptColl &oldPtoc)
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
SetInputScore(const InputPath &inputPath, PartialTranslOptColl &oldPtoc)
|
||||
{
|
||||
const ScorePair *inputScore = inputPath.GetInputScore();
|
||||
if (inputScore == NULL) {
|
||||
return;
|
||||
}
|
||||
const ScorePair* inputScore = inputPath.GetInputScore();
|
||||
if (inputScore == NULL) return;
|
||||
|
||||
const InputFeature &inputFeature = InputFeature::Instance();
|
||||
|
||||
@ -541,79 +503,73 @@ void TranslationOptionCollection::SetInputScore(const InputPath &inputPath, Part
|
||||
}
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::EvaluateWithSourceContext()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
EvaluateWithSourceContext()
|
||||
{
|
||||
const size_t size = m_source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
|
||||
size_t maxSize = m_source.GetSize() - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
|
||||
TranslationOptionList &transOptList = GetTranslationOptionList(startPos, endPos);
|
||||
|
||||
TranslationOptionList::const_iterator iterTransOpt;
|
||||
for(iterTransOpt = transOptList.begin() ; iterTransOpt != transOptList.end() ; ++iterTransOpt) {
|
||||
TranslationOption &transOpt = **iterTransOpt;
|
||||
transOpt.EvaluateWithSourceContext(m_source);
|
||||
}
|
||||
for (size_t sPos = 0 ; sPos < size ; ++sPos)
|
||||
{
|
||||
BOOST_FOREACH(TranslationOptionList& tol, m_collection[sPos])
|
||||
{
|
||||
typedef TranslationOptionList::const_iterator to_iter;
|
||||
for(to_iter i = tol.begin() ; i != tol.end() ; ++i)
|
||||
(*i)->EvaluateWithSourceContext(m_source);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::Sort()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
Sort()
|
||||
{
|
||||
static TranslationOption::Better cmp;
|
||||
size_t size = m_source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size; ++startPos) {
|
||||
size_t maxSize = size - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize; ++endPos) {
|
||||
TranslationOptionList &transOptList = GetTranslationOptionList(startPos, endPos);
|
||||
std::sort(transOptList.begin(), transOptList.end(), CompareTranslationOption);
|
||||
for (size_t sPos = 0 ; sPos < size; ++sPos)
|
||||
{
|
||||
BOOST_FOREACH(TranslationOptionList& tol, m_collection.at(sPos))
|
||||
std::sort(tol.begin(), tol.end(), cmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if this range overlaps with any XML options. This doesn't need to be an exact match, only an overlap.
|
||||
* by default, we don't support XML options. subclasses need to override this function.
|
||||
* called by CreateTranslationOptionsForRange()
|
||||
* \param startPos first position in input sentence
|
||||
* \param sPos first position in input sentence
|
||||
* \param lastPos last position in input sentence
|
||||
*/
|
||||
bool TranslationOptionCollection::HasXmlOptionsOverlappingRange(size_t, size_t) const
|
||||
{
|
||||
return false;
|
||||
//not implemented for base class
|
||||
}
|
||||
bool
|
||||
TranslationOptionCollection::
|
||||
HasXmlOptionsOverlappingRange(size_t, size_t) const
|
||||
{ return false; }
|
||||
|
||||
/** Check if an option conflicts with any constraint XML options. Okay, if XML option is substring in source and target.
|
||||
* by default, we don't support XML options. subclasses need to override this function.
|
||||
* called by CreateTranslationOptionsForRange()
|
||||
* \param startPos first position in input sentence
|
||||
* \param sPos first position in input sentence
|
||||
* \param lastPos last position in input sentence
|
||||
*/
|
||||
bool TranslationOptionCollection::ViolatesXmlOptionsConstraint(size_t, size_t, TranslationOption *) const
|
||||
{
|
||||
return false;
|
||||
//not implemented for base class
|
||||
}
|
||||
bool
|
||||
TranslationOptionCollection::
|
||||
ViolatesXmlOptionsConstraint(size_t, size_t, TranslationOption*) const
|
||||
{ return false; }
|
||||
|
||||
/** Populates the current Collection with XML options exactly covering the range specified. Default implementation does nothing.
|
||||
* called by CreateTranslationOptionsForRange()
|
||||
* \param startPos first position in input sentence
|
||||
* \param sPos first position in input sentence
|
||||
* \param lastPos last position in input sentence
|
||||
*/
|
||||
void TranslationOptionCollection::CreateXmlOptionsForRange(size_t, size_t)
|
||||
{
|
||||
//not implemented for base class
|
||||
};
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
CreateXmlOptionsForRange(size_t, size_t)
|
||||
{ }
|
||||
|
||||
|
||||
/** Add translation option to the list
|
||||
* \param translationOption translation option to be added */
|
||||
void TranslationOptionCollection::Add(TranslationOption *translationOption)
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
Add(TranslationOption *translationOption)
|
||||
{
|
||||
const WordsRange &coverage = translationOption->GetSourceWordsRange();
|
||||
|
||||
@ -632,30 +588,27 @@ TO_STRING_BODY(TranslationOptionCollection);
|
||||
std::ostream& operator<<(std::ostream& out, const TranslationOptionCollection& coll)
|
||||
{
|
||||
size_t size = coll.m_source.GetSize();
|
||||
for (size_t startPos = 0 ; startPos < size ; ++startPos) {
|
||||
size_t maxSize = size - startPos;
|
||||
for (size_t sPos = 0 ; sPos < size ; ++sPos) {
|
||||
size_t maxSize = size - sPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) {
|
||||
const TranslationOptionList& fullList = coll.GetTranslationOptionList(startPos, endPos);
|
||||
size_t sizeFull = fullList.size();
|
||||
for (size_t ePos = sPos ; ePos < sPos + maxSize ; ++ePos) {
|
||||
const TranslationOptionList* fullList
|
||||
= coll.GetTranslationOptionList(sPos, ePos);
|
||||
if (!fullList) break;
|
||||
size_t sizeFull = fullList->size();
|
||||
for (size_t i = 0; i < sizeFull; i++) {
|
||||
out << *fullList.Get(i) << std::endl;
|
||||
out << *fullList->Get(i) << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//std::vector< std::vector< TranslationOptionList > >::const_iterator i = coll.m_collection.begin();
|
||||
//size_t j = 0;
|
||||
//for (; i!=coll.m_collection.end(); ++i) {
|
||||
//out << "s[" << j++ << "].size=" << i->size() << std::endl;
|
||||
//}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::CacheLexReordering()
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
CacheLexReordering()
|
||||
{
|
||||
size_t size = m_source.GetSize();
|
||||
|
||||
@ -665,68 +618,67 @@ void TranslationOptionCollection::CacheLexReordering()
|
||||
const StatefulFeatureFunction &ff = **iter;
|
||||
if (typeid(ff) == typeid(LexicalReordering)) {
|
||||
const LexicalReordering &lexreordering = static_cast<const LexicalReordering&>(ff);
|
||||
for (size_t startPos = 0 ; startPos < size ; startPos++) {
|
||||
size_t maxSize = size - startPos;
|
||||
for (size_t sPos = 0 ; sPos < size ; sPos++) {
|
||||
size_t maxSize = size - sPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize; endPos++) {
|
||||
TranslationOptionList &transOptList = GetTranslationOptionList( startPos, endPos);
|
||||
TranslationOptionList::iterator iterTransOpt;
|
||||
for(iterTransOpt = transOptList.begin() ; iterTransOpt != transOptList.end() ; ++iterTransOpt) {
|
||||
TranslationOption &transOpt = **iterTransOpt;
|
||||
//Phrase sourcePhrase = m_source.GetSubString(WordsRange(startPos,endPos));
|
||||
const Phrase &sourcePhrase = transOpt.GetInputPath().GetPhrase();
|
||||
Scores score = lexreordering.GetProb(sourcePhrase
|
||||
, transOpt.GetTargetPhrase());
|
||||
if (!score.empty())
|
||||
transOpt.CacheLexReorderingScores(lexreordering, score);
|
||||
} // for(iterTransOpt
|
||||
} // for (size_t endPos = startPos ; endPos < startPos + maxSize; endPos++) {
|
||||
} // for (size_t startPos = 0 ; startPos < size ; startPos++) {
|
||||
BOOST_FOREACH(TranslationOptionList& tol, m_collection)
|
||||
{
|
||||
TranslationOptionList::iterator it;
|
||||
for(it = tol.begin(); it != tol.end(); ++it)
|
||||
{
|
||||
TranslationOption &o = **it;
|
||||
const Phrase &srcPhrase = o.GetInputPath().GetPhrase();
|
||||
Scores score = lexreordering.GetProb(srcPhrase, o.GetTargetPhrase());
|
||||
if (!score.empty())
|
||||
transOpt.CacheLexReorderingScores(lexreordering, score);
|
||||
} // for(iterTransOpt
|
||||
} // for (size_t ePos = sPos ; ePos < sPos + maxSize; ePos++) {
|
||||
} // for (size_t sPos = 0 ; sPos < size ; sPos++) {
|
||||
} // if (typeid(ff) == typeid(LexicalReordering)) {
|
||||
} // for (iter = ffs.begin(); iter != ffs.end(); ++iter) {
|
||||
}
|
||||
|
||||
//! list of trans opt for a particular span
|
||||
TranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos)
|
||||
TranslationOptionList*
|
||||
TranslationOptionCollection::
|
||||
GetTranslationOptionList(size_t const sPos, size_t const ePos)
|
||||
{
|
||||
size_t maxSize = endPos - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(),
|
||||
"Out of bound access: " << maxSize);
|
||||
|
||||
return m_collection[startPos][maxSize];
|
||||
}
|
||||
const TranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos) const
|
||||
{
|
||||
size_t maxSize = endPos - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = std::min(maxSize, maxSizePhrase);
|
||||
|
||||
UTIL_THROW_IF2(maxSize >= m_collection[startPos].size(),
|
||||
"Out of bound access: " << maxSize);
|
||||
return m_collection[startPos][maxSize];
|
||||
UTIL_THROW_IF2(sPos >= m_collection.size(), "Out of bound access.");
|
||||
vector<TranslationOptionList>& tol = m_collection[sPos];
|
||||
size_t idx = ePos - sPos;
|
||||
return idx < tol.size() ? &tol[idx] : NULL;
|
||||
}
|
||||
|
||||
void TranslationOptionCollection::GetTargetPhraseCollectionBatch()
|
||||
TranslationOptionList const*
|
||||
TranslationOptionCollection::
|
||||
GetTranslationOptionList(size_t sPos, size_t ePos) const
|
||||
{
|
||||
const vector <DecodeGraph*> &decodeGraphList = StaticData::Instance().GetDecodeGraphs();
|
||||
for (size_t graphInd = 0 ; graphInd < decodeGraphList.size() ; graphInd++) {
|
||||
const DecodeGraph &decodeGraph = *decodeGraphList[graphInd];
|
||||
UTIL_THROW_IF2(sPos >= m_collection.size(), "Out of bound access.");
|
||||
vector<TranslationOptionList>& tol = m_collection[sPos];
|
||||
size_t idx = ePos - sPos;
|
||||
return idx < tol.size() ? &tol[idx] : NULL;
|
||||
}
|
||||
|
||||
list <const DecodeStep* >::const_iterator iterStep;
|
||||
for (iterStep = decodeGraph.begin(); iterStep != decodeGraph.end() ; ++iterStep) {
|
||||
const DecodeStep &decodeStep = **iterStep;
|
||||
const DecodeStepTranslation *transStep = dynamic_cast<const DecodeStepTranslation *>(&decodeStep);
|
||||
if (transStep) {
|
||||
const PhraseDictionary &phraseDictionary = *transStep->GetPhraseDictionaryFeature();
|
||||
phraseDictionary.GetTargetPhraseCollectionBatch(m_inputPathQueue);
|
||||
}
|
||||
void
|
||||
TranslationOptionCollection::
|
||||
GetTargetPhraseCollectionBatch()
|
||||
{
|
||||
const vector <DecodeGraph*> &dgl = StaticData::Instance().GetDecodeGraphs();
|
||||
BOOST_FOREACH(DecodeGraph const& dg, dgl)
|
||||
{
|
||||
typedef list <const DecodeStep* >::const_iterator dsiter;
|
||||
for (dsiter i = dg.begin(); i != dg.end() ; ++i)
|
||||
{
|
||||
const DecodeStepTranslation* tstep = dynamic_cast<const DecodeStepTranslation *>(*i);
|
||||
if (tstep)
|
||||
{
|
||||
const PhraseDictionary &pdict = *tstep->GetPhraseDictionaryFeature();
|
||||
pdict.GetTargetPhraseCollectionBatch(m_inputPathQueue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
Loading…
Reference in New Issue
Block a user