2013-09-25 03:43:53 +04:00
|
|
|
// -*- c++ -*-
|
2008-06-11 14:52:57 +04:00
|
|
|
// $Id$
|
|
|
|
// vim:tabstop=2
|
|
|
|
|
2010-02-24 14:15:44 +03:00
|
|
|
#ifndef moses_PDTAimp_h
|
|
|
|
#define moses_PDTAimp_h
|
2008-06-11 14:52:57 +04:00
|
|
|
|
|
|
|
#include "StaticData.h" // needed for factor splitter
|
2012-11-27 19:08:31 +04:00
|
|
|
#include "moses/TranslationModel/PhraseDictionaryTree.h"
|
2011-07-24 03:52:34 +04:00
|
|
|
#include "UniqueObject.h"
|
|
|
|
#include "InputFileStream.h"
|
2012-11-27 19:08:31 +04:00
|
|
|
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
|
2011-07-24 03:52:34 +04:00
|
|
|
#include "Util.h"
|
2013-04-10 21:27:25 +04:00
|
|
|
#include "util/tokenize_piece.hh"
|
2013-11-19 21:23:19 +04:00
|
|
|
#include "util/exception.hh"
|
2013-06-06 21:33:18 +04:00
|
|
|
#include "moses/FF/InputFeature.h"
|
2014-01-13 18:37:05 +04:00
|
|
|
#include "util/exception.hh"
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-07-24 03:52:34 +04:00
|
|
|
inline double addLogScale(double x,double y)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
if(x>y) return addLogScale(y,x);
|
|
|
|
else return x+log(1.0+exp(y-x));
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2011-07-24 03:52:34 +04:00
|
|
|
inline double Exp(double x)
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
return exp(x);
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2012-06-29 02:29:46 +04:00
|
|
|
/** implementation of the binary phrase table for the phrase-based decoder. Used by PhraseDictionaryTreeAdaptor
|
|
|
|
*/
|
2011-02-24 16:14:42 +03:00
|
|
|
class PDTAimp
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
2011-02-24 16:14:42 +03:00
|
|
|
// only these classes are allowed to instantiate this class
|
|
|
|
friend class PhraseDictionaryTreeAdaptor;
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
protected:
|
2014-08-04 17:41:12 +04:00
|
|
|
PDTAimp(PhraseDictionaryTreeAdaptor *p);
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
public:
|
2011-02-24 16:14:42 +03:00
|
|
|
std::vector<FactorType> m_input,m_output;
|
|
|
|
PhraseDictionaryTree *m_dict;
|
2013-06-07 00:25:51 +04:00
|
|
|
const InputFeature *m_inputFeature;
|
2015-10-18 23:27:58 +03:00
|
|
|
typedef std::vector<TargetPhraseCollectionWithSourcePhrase::shared_ptr> vTPC;
|
2011-02-24 16:14:42 +03:00
|
|
|
mutable vTPC m_tgtColls;
|
|
|
|
|
2015-10-18 23:27:58 +03:00
|
|
|
typedef std::map<Phrase,TargetPhraseCollectionWithSourcePhrase::shared_ptr> MapSrc2Tgt;
|
2011-02-24 16:14:42 +03:00
|
|
|
mutable MapSrc2Tgt m_cache;
|
|
|
|
PhraseDictionaryTreeAdaptor *m_obj;
|
|
|
|
int useCache;
|
|
|
|
|
|
|
|
std::vector<vTPC> m_rangeCache;
|
|
|
|
unsigned m_numInputScores;
|
|
|
|
|
|
|
|
UniqueObjectManager<Phrase> uniqSrcPhr;
|
|
|
|
|
|
|
|
size_t totalE,distinctE;
|
|
|
|
std::vector<size_t> path1Best,pathExplored;
|
|
|
|
std::vector<double> pathCN;
|
|
|
|
|
2014-08-04 17:41:12 +04:00
|
|
|
~PDTAimp();
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
void Factors2String(Word const& w,std::string& s) const {
|
|
|
|
s=w.GetString(m_input,false);
|
|
|
|
}
|
|
|
|
|
2014-08-04 17:41:12 +04:00
|
|
|
void CleanUp();
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2015-10-18 23:27:58 +03:00
|
|
|
TargetPhraseCollectionWithSourcePhrase::shared_ptr
|
2014-08-04 17:41:12 +04:00
|
|
|
GetTargetPhraseCollection(Phrase const &src) const;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
void Create(const std::vector<FactorType> &input
|
|
|
|
, const std::vector<FactorType> &output
|
|
|
|
, const std::string &filePath
|
2014-08-04 17:41:12 +04:00
|
|
|
, const std::vector<float> &weight);
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
|
|
|
|
typedef PhraseDictionaryTree::PrefixPtr PPtr;
|
|
|
|
typedef unsigned short Position;
|
|
|
|
typedef std::pair<Position,Position> Range;
|
|
|
|
struct State {
|
|
|
|
PPtr ptr;
|
|
|
|
Range range;
|
|
|
|
std::vector<float> scores;
|
|
|
|
Phrase src;
|
|
|
|
|
2011-11-21 14:49:26 +04:00
|
|
|
State() : range(0,0),scores(0),src(ARRAY_SIZE_INCR) {}
|
2011-02-24 16:14:42 +03:00
|
|
|
State(Position b,Position e,const PPtr& v,const std::vector<float>& sv=std::vector<float>(0))
|
2011-11-21 14:49:26 +04:00
|
|
|
: ptr(v),range(b,e),scores(sv),src(ARRAY_SIZE_INCR) {}
|
2011-02-24 16:14:42 +03:00
|
|
|
State(Range const& r,const PPtr& v,const std::vector<float>& sv=std::vector<float>(0))
|
2011-11-21 14:49:26 +04:00
|
|
|
: ptr(v),range(r),scores(sv),src(ARRAY_SIZE_INCR) {}
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
Position begin() const {
|
|
|
|
return range.first;
|
|
|
|
}
|
|
|
|
Position end() const {
|
|
|
|
return range.second;
|
|
|
|
}
|
|
|
|
std::vector<float> GetScores() const {
|
|
|
|
return scores;
|
|
|
|
}
|
|
|
|
|
|
|
|
friend std::ostream& operator<<(std::ostream& out,State const& s) {
|
|
|
|
out<<" R=("<<s.begin()<<","<<s.end()<<"),";
|
|
|
|
for(std::vector<float>::const_iterator scoreIterator = s.GetScores().begin(); scoreIterator<s.GetScores().end(); scoreIterator++) {
|
|
|
|
out<<", "<<*scoreIterator;
|
|
|
|
}
|
|
|
|
out<<")";
|
|
|
|
return out;
|
|
|
|
}
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
void CreateTargetPhrase(TargetPhrase& targetPhrase,
|
2011-09-30 00:35:52 +04:00
|
|
|
StringTgtCand::Tokens const& factorStrings,
|
2015-12-12 19:23:37 +03:00
|
|
|
std::string const& factorDelimiter,
|
2013-06-07 01:03:43 +04:00
|
|
|
Scores const& transVector,
|
|
|
|
Scores const& inputVector,
|
2013-06-10 04:24:14 +04:00
|
|
|
const std::string *alignmentString,
|
2014-08-04 17:41:12 +04:00
|
|
|
Phrase const* srcPtr=0) const;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2015-10-18 23:27:58 +03:00
|
|
|
TargetPhraseCollectionWithSourcePhrase::shared_ptr PruneTargetCandidates
|
2013-08-07 17:18:12 +04:00
|
|
|
(const std::vector<TargetPhrase> & tCands,
|
|
|
|
std::vector<std::pair<float,size_t> >& costs,
|
2014-08-04 17:41:12 +04:00
|
|
|
const std::vector<Phrase> &sourcePhrases) const;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
|
|
|
|
// POD for target phrase scores
|
|
|
|
struct TScores {
|
|
|
|
float total;
|
2013-06-07 01:03:43 +04:00
|
|
|
Scores transScore, inputScores;
|
2011-02-24 16:14:42 +03:00
|
|
|
Phrase const* src;
|
|
|
|
|
|
|
|
TScores() : total(0.0),src(0) {}
|
|
|
|
};
|
|
|
|
|
2014-08-04 17:41:12 +04:00
|
|
|
void CacheSource(ConfusionNet const& src);
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
size_t GetNumInputScores() const {
|
|
|
|
return m_numInputScores;
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
};
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
}
|
2010-02-24 14:15:44 +03:00
|
|
|
#endif
|