2013-09-25 03:51:50 +04:00
|
|
|
// -*- c++ -*-
|
|
|
|
// Sampling phrase table implementation based on memory-mapped suffix arrays.
|
|
|
|
// Design and code by Ulrich Germann.
|
|
|
|
#pragma once
|
|
|
|
|
2014-03-24 17:30:22 +04:00
|
|
|
#include <time.h>
|
2013-09-25 03:51:50 +04:00
|
|
|
#include <boost/thread.hpp>
|
2014-03-24 17:30:22 +04:00
|
|
|
#include <boost/scoped_ptr.hpp>
|
2013-09-25 03:51:50 +04:00
|
|
|
|
2014-02-08 21:56:48 +04:00
|
|
|
#include "moses/TypeDef.h"
|
|
|
|
#include "moses/TranslationModel/UG/generic/sorting/VectorIndexSorter.h"
|
|
|
|
#include "moses/TranslationModel/UG/generic/sampling/Sampling.h"
|
|
|
|
#include "moses/TranslationModel/UG/generic/file_io/ug_stream.h"
|
2013-09-25 03:51:50 +04:00
|
|
|
|
2014-02-08 21:56:48 +04:00
|
|
|
#include "moses/TranslationModel/UG/mm/ug_mm_ttrack.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/ug_mm_tsa.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/tpt_tokenindex.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/ug_corpus_token.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/ug_typedefs.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/tpt_pickler.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/ug_bitext.h"
|
|
|
|
#include "moses/TranslationModel/UG/mm/ug_lexical_phrase_scorer2.h"
|
2013-09-25 03:51:50 +04:00
|
|
|
|
|
|
|
#include "moses/InputFileStream.h"
|
|
|
|
#include "moses/FactorTypeSet.h"
|
|
|
|
#include "moses/TargetPhrase.h"
|
|
|
|
#include <boost/dynamic_bitset.hpp>
|
|
|
|
#include "moses/TargetPhraseCollection.h"
|
|
|
|
#include <map>
|
|
|
|
|
2014-02-08 21:56:48 +04:00
|
|
|
#include "moses/TranslationModel/PhraseDictionary.h"
|
|
|
|
|
|
|
|
// TO DO:
|
|
|
|
// - make lexical phrase scorer take addition to the "dynamic overlay" into account
|
|
|
|
// - switch to pool of sapts, where each sapt has its own provenance feature
|
|
|
|
// RESEARCH QUESTION: is this more effective than having multiple phrase tables,
|
|
|
|
// each with its own set of features?
|
2013-09-25 03:51:50 +04:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
namespace Moses
|
|
|
|
{
|
|
|
|
using namespace bitext;
|
2014-02-21 03:25:36 +04:00
|
|
|
class Mmsapt
|
|
|
|
#ifndef NO_MOSES
|
|
|
|
: public PhraseDictionary
|
|
|
|
#endif
|
2013-09-25 03:51:50 +04:00
|
|
|
{
|
2014-03-10 15:57:40 +04:00
|
|
|
friend class Alignment;
|
2014-02-08 21:56:48 +04:00
|
|
|
public:
|
2013-09-25 03:51:50 +04:00
|
|
|
typedef L2R_Token<SimpleWordId> Token;
|
|
|
|
typedef mmBitext<Token> mmbitext;
|
2014-02-08 21:56:48 +04:00
|
|
|
typedef imBitext<Token> imbitext;
|
2014-02-21 03:25:36 +04:00
|
|
|
typedef TSA<Token> tsa;
|
2014-02-08 21:56:48 +04:00
|
|
|
private:
|
2014-02-21 03:25:36 +04:00
|
|
|
mmbitext btfix;
|
2014-02-08 21:56:48 +04:00
|
|
|
sptr<imbitext> btdyn;
|
2014-03-13 03:13:44 +04:00
|
|
|
string bname,extra_data;
|
2013-09-25 03:51:50 +04:00
|
|
|
string L1;
|
|
|
|
string L2;
|
2014-03-24 17:30:22 +04:00
|
|
|
float m_lbop_parameter;
|
|
|
|
size_t m_default_sample_size;
|
|
|
|
size_t m_workers; // number of worker threads for sampling the bitexts
|
|
|
|
char m_pfwd_denom; // denominator for computation of fwd phrase score:
|
|
|
|
// 'r' - divide by raw count
|
|
|
|
// 's' - divide by sample count
|
|
|
|
// 'g' - devide by number of "good" (i.e. coherent) samples
|
2013-09-25 03:51:50 +04:00
|
|
|
// size_t num_features;
|
|
|
|
size_t input_factor;
|
|
|
|
size_t output_factor; // we can actually return entire Tokens!
|
|
|
|
// built-in feature functions
|
2014-02-08 21:56:48 +04:00
|
|
|
PScorePfwd<Token> calc_pfwd_fix, calc_pfwd_dyn;
|
|
|
|
PScorePbwd<Token> calc_pbwd_fix, calc_pbwd_dyn;
|
2013-09-25 03:51:50 +04:00
|
|
|
PScoreLex<Token> calc_lex; // this one I'd like to see as an external ff eventually
|
|
|
|
PScorePP<Token> apply_pp; // apply phrase penalty
|
|
|
|
void init(string const& line);
|
|
|
|
mutable boost::mutex lock;
|
2014-02-08 21:56:48 +04:00
|
|
|
bool poolCounts;
|
|
|
|
vector<FactorType> ofactor;
|
|
|
|
|
2014-03-24 17:30:22 +04:00
|
|
|
|
|
|
|
public:
|
|
|
|
// typedef boost::unordered_map<uint64_t, sptr<TargetPhraseCollection> > tpcoll_cache_t;
|
|
|
|
class TargetPhraseCollectionWrapper
|
|
|
|
: public TargetPhraseCollection
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
size_t const revision; // time stamp from dynamic bitext
|
|
|
|
uint64_t const key; // phrase key
|
|
|
|
uint32_t refCount; // reference count
|
|
|
|
timespec tstamp; // last use
|
|
|
|
int idx; // position in history heap
|
|
|
|
TargetPhraseCollectionWrapper(size_t r, uint64_t const k);
|
|
|
|
~TargetPhraseCollectionWrapper();
|
|
|
|
};
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
|
|
|
TargetPhraseCollectionWrapper*
|
|
|
|
encache(TargetPhraseCollectionWrapper* const ptr) const;
|
|
|
|
|
|
|
|
void
|
|
|
|
decache(TargetPhraseCollectionWrapper* ptr) const;
|
|
|
|
|
|
|
|
typedef map<uint64_t, TargetPhraseCollectionWrapper*> tpc_cache_t;
|
|
|
|
mutable tpc_cache_t m_cache;
|
|
|
|
mutable vector<TargetPhraseCollectionWrapper*> m_history;
|
2014-02-21 03:25:36 +04:00
|
|
|
// phrase table feature weights for alignment:
|
|
|
|
vector<float> feature_weights;
|
|
|
|
|
2014-03-10 15:57:40 +04:00
|
|
|
vector<vector<id_type> > wlex21;
|
|
|
|
// word translation lexicon (without counts, get these from calc_lex.COOC)
|
|
|
|
typedef mm2dTable<id_type,id_type,uint32_t,uint32_t> mm2dtable_t;
|
|
|
|
mm2dtable_t COOCraw;
|
|
|
|
|
2014-02-08 21:56:48 +04:00
|
|
|
TargetPhrase*
|
|
|
|
createTargetPhrase
|
|
|
|
(Phrase const& src,
|
|
|
|
Bitext<Token> const& bt,
|
|
|
|
bitext::PhrasePair const& pp
|
|
|
|
) const;
|
|
|
|
|
|
|
|
void
|
|
|
|
process_pstats
|
|
|
|
(Phrase const& src,
|
|
|
|
uint64_t const pid1,
|
|
|
|
pstats const& stats,
|
|
|
|
Bitext<Token> const & bt,
|
|
|
|
TargetPhraseCollection* tpcoll
|
|
|
|
) const;
|
|
|
|
|
|
|
|
bool
|
|
|
|
pool_pstats
|
|
|
|
(Phrase const& src,
|
|
|
|
uint64_t const pid1a,
|
|
|
|
pstats * statsa,
|
|
|
|
Bitext<Token> const & bta,
|
|
|
|
uint64_t const pid1b,
|
|
|
|
pstats const* statsb,
|
|
|
|
Bitext<Token> const & btb,
|
|
|
|
TargetPhraseCollection* tpcoll
|
|
|
|
) const;
|
|
|
|
|
|
|
|
bool
|
|
|
|
combine_pstats
|
|
|
|
(Phrase const& src,
|
|
|
|
uint64_t const pid1a,
|
|
|
|
pstats * statsa,
|
|
|
|
Bitext<Token> const & bta,
|
|
|
|
uint64_t const pid1b,
|
|
|
|
pstats const* statsb,
|
|
|
|
Bitext<Token> const & btb,
|
|
|
|
TargetPhraseCollection* tpcoll
|
|
|
|
) const;
|
|
|
|
|
2014-03-13 03:13:44 +04:00
|
|
|
void
|
|
|
|
load_extra_data(string bname);
|
|
|
|
|
2014-03-24 17:30:22 +04:00
|
|
|
mutable size_t m_tpc_ctr;
|
2013-09-25 03:51:50 +04:00
|
|
|
public:
|
2014-02-22 04:25:02 +04:00
|
|
|
// Mmsapt(string const& description, string const& line);
|
2013-09-25 03:51:50 +04:00
|
|
|
Mmsapt(string const& line);
|
|
|
|
void
|
|
|
|
Load();
|
|
|
|
|
2014-02-21 03:25:36 +04:00
|
|
|
#ifndef NO_MOSES
|
2013-09-25 03:51:50 +04:00
|
|
|
TargetPhraseCollection const*
|
|
|
|
GetTargetPhraseCollectionLEGACY(const Phrase& src) const;
|
|
|
|
//! Create a sentence-specific manager for SCFG rule lookup.
|
|
|
|
ChartRuleLookupManager*
|
|
|
|
CreateRuleLookupManager(const ChartParser &, const ChartCellCollectionBase &);
|
2014-03-20 00:43:47 +04:00
|
|
|
|
|
|
|
ChartRuleLookupManager*
|
|
|
|
CreateRuleLookupManager
|
|
|
|
(const ChartParser &, const ChartCellCollectionBase &, std::size_t);
|
2014-02-21 03:25:36 +04:00
|
|
|
#endif
|
2013-09-25 03:51:50 +04:00
|
|
|
|
2014-02-08 21:56:48 +04:00
|
|
|
void add(string const& s1, string const& s2, string const& a);
|
|
|
|
|
2014-02-21 03:25:36 +04:00
|
|
|
// align two new sentences
|
|
|
|
sptr<vector<int> >
|
|
|
|
align(string const& src, string const& trg) const;
|
|
|
|
|
|
|
|
void setWeights(vector<float> const& w);
|
2014-03-20 00:43:47 +04:00
|
|
|
|
2014-03-24 17:30:22 +04:00
|
|
|
void
|
|
|
|
CleanUpAfterSentenceProcessing(const InputType& source);
|
2014-03-20 00:43:47 +04:00
|
|
|
|
2014-03-24 17:30:22 +04:00
|
|
|
void
|
|
|
|
InitializeForInput(InputType const& source);
|
|
|
|
|
|
|
|
void
|
2014-03-25 05:49:24 +04:00
|
|
|
Release(TargetPhraseCollection const* tpc) const;
|
|
|
|
|
|
|
|
bool
|
|
|
|
ProvidesPrefixCheck() const;
|
2014-03-24 17:30:22 +04:00
|
|
|
|
2014-03-25 05:49:24 +04:00
|
|
|
/// return true if prefix /phrase/ exists
|
|
|
|
bool
|
|
|
|
PrefixExists(Phrase const& phrase) const;
|
|
|
|
|
2013-09-25 03:51:50 +04:00
|
|
|
private:
|
|
|
|
};
|
|
|
|
} // end namespace
|
|
|
|
|