2008-06-11 14:52:57 +04:00
|
|
|
// $Id$
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
|
|
|
|
2010-02-24 14:15:44 +03:00
|
|
|
#ifndef moses_StaticData_h
|
|
|
|
#define moses_StaticData_h
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2010-08-10 17:12:00 +04:00
|
|
|
#include <stdexcept>
|
2010-01-28 13:39:43 +03:00
|
|
|
#include <limits>
|
2008-06-11 14:52:57 +04:00
|
|
|
#include <list>
|
|
|
|
#include <vector>
|
|
|
|
#include <map>
|
|
|
|
#include <memory>
|
2010-01-28 13:39:43 +03:00
|
|
|
#include <utility>
|
2011-08-18 01:13:21 +04:00
|
|
|
#include <fstream>
|
|
|
|
#include <string>
|
2009-08-07 20:47:54 +04:00
|
|
|
|
|
|
|
#ifdef WITH_THREADS
|
2013-05-10 15:30:01 +04:00
|
|
|
#include <boost/thread.hpp>
|
2009-08-07 20:47:54 +04:00
|
|
|
#include <boost/thread/mutex.hpp>
|
|
|
|
#endif
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
#include "TypeDef.h"
|
|
|
|
#include "FactorCollection.h"
|
|
|
|
#include "Parameter.h"
|
2011-10-13 18:27:01 +04:00
|
|
|
#include "LM/Base.h"
|
2008-06-11 14:52:57 +04:00
|
|
|
#include "SentenceStats.h"
|
|
|
|
#include "DecodeGraph.h"
|
2008-10-30 21:43:17 +03:00
|
|
|
#include "TranslationOptionList.h"
|
2010-09-17 18:25:08 +04:00
|
|
|
#include "ScoreComponentCollection.h"
|
2013-04-26 18:55:32 +04:00
|
|
|
#include "moses/TranslationModel/PhraseDictionary.h"
|
2008-10-30 21:43:17 +03:00
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
class InputType;
|
2013-02-22 23:17:57 +04:00
|
|
|
class PhraseDictionary;
|
2008-06-11 14:52:57 +04:00
|
|
|
class GenerationDictionary;
|
|
|
|
class DecodeStep;
|
2013-05-21 16:39:31 +04:00
|
|
|
class WordPenaltyProducer;
|
2008-06-11 14:52:57 +04:00
|
|
|
class UnknownWordPenaltyProducer;
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
typedef std::pair<std::string, float> UnknownLHSEntry;
|
|
|
|
typedef std::vector<UnknownLHSEntry> UnknownLHSList;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2012-06-29 02:29:46 +04:00
|
|
|
/** Contains global variables and contants.
|
|
|
|
* Only 1 object of this class should be instantiated.
|
|
|
|
* A const object of this class is accessible by any function during decoding by calling StaticData::Instance();
|
|
|
|
*/
|
2008-06-11 14:52:57 +04:00
|
|
|
class StaticData
|
|
|
|
{
|
|
|
|
private:
|
2011-02-24 16:14:42 +03:00
|
|
|
static StaticData s_instance;
|
|
|
|
protected:
|
|
|
|
|
|
|
|
std::map<long,Phrase> m_constraints;
|
2013-02-22 23:17:57 +04:00
|
|
|
std::vector<PhraseDictionary*> m_phraseDictionary;
|
2013-01-18 22:22:06 +04:00
|
|
|
std::vector<const GenerationDictionary*> m_generationDictionary;
|
2011-02-24 19:17:38 +03:00
|
|
|
Parameter *m_parameter;
|
|
|
|
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
|
2013-05-31 15:28:57 +04:00
|
|
|
mutable ScoreComponentCollection m_allWeights;
|
2012-12-19 18:37:00 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
std::vector<DecodeGraph*> m_decodeGraphs;
|
|
|
|
std::vector<size_t> m_decodeGraphBackoff;
|
|
|
|
// Initial = 0 = can be used when creating poss trans
|
|
|
|
// Other = 1 = used to calculate LM score once all steps have been processed
|
|
|
|
float
|
|
|
|
m_beamWidth,
|
|
|
|
m_earlyDiscardingThreshold,
|
|
|
|
m_translationOptionThreshold,
|
|
|
|
m_wordDeletionWeight;
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// PhraseTrans, Generation & LanguageModelScore has multiple weights.
|
|
|
|
int m_maxDistortion;
|
|
|
|
// do it differently from old pharaoh
|
|
|
|
// -ve = no limit on distortion
|
|
|
|
// 0 = no disortion (monotone in old pharaoh)
|
2011-02-24 19:17:38 +03:00
|
|
|
bool m_reorderingConstraint; //! use additional reordering constraints
|
2012-09-13 13:43:01 +04:00
|
|
|
bool m_useEarlyDistortionCost;
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t
|
2011-02-24 19:17:38 +03:00
|
|
|
m_maxHypoStackSize //! hypothesis-stack size that triggers pruning
|
|
|
|
, m_minHypoStackDiversity //! minimum number of hypothesis in stack for each source word coverage
|
2011-02-24 16:14:42 +03:00
|
|
|
, m_nBestSize
|
2011-10-04 19:46:24 +04:00
|
|
|
, m_latticeSamplesSize
|
2011-02-24 16:14:42 +03:00
|
|
|
, m_nBestFactor
|
|
|
|
, m_maxNoTransOptPerCoverage
|
|
|
|
, m_maxNoPartTransOpt
|
|
|
|
, m_maxPhraseLength
|
2012-12-06 00:21:33 +04:00
|
|
|
, m_numRealWordsInInput;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
std::string
|
|
|
|
m_constraintFileName;
|
|
|
|
|
2011-10-04 19:46:24 +04:00
|
|
|
std::string m_nBestFilePath, m_latticeSamplesFilePath;
|
2013-01-29 20:57:56 +04:00
|
|
|
bool m_labeledNBestList,m_nBestIncludesSegmentation;
|
2011-02-24 19:17:38 +03:00
|
|
|
bool m_dropUnknown; //! false = treat unknown words as unknowns, and translate them as themselves; true = drop (ignore) them
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_wordDeletionEnabled;
|
|
|
|
|
|
|
|
bool m_disableDiscarding;
|
|
|
|
bool m_printAllDerivations;
|
|
|
|
|
|
|
|
bool m_sourceStartPosMattersForRecombination;
|
|
|
|
bool m_recoverPath;
|
|
|
|
bool m_outputHypoScore;
|
|
|
|
|
2012-01-26 15:38:40 +04:00
|
|
|
ParsingAlgorithm m_parsingAlgorithm;
|
2011-02-24 16:14:42 +03:00
|
|
|
SearchAlgorithm m_searchAlgorithm;
|
|
|
|
InputTypeEnum m_inputType;
|
|
|
|
size_t m_numInputScores;
|
|
|
|
|
|
|
|
mutable size_t m_verboseLevel;
|
2012-12-19 19:38:57 +04:00
|
|
|
WordPenaltyProducer* m_wpProducer;
|
2011-02-24 16:14:42 +03:00
|
|
|
UnknownWordPenaltyProducer *m_unknownWordPenaltyProducer;
|
2012-12-19 19:38:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_reportSegmentation;
|
|
|
|
bool m_reportAllFactors;
|
|
|
|
bool m_reportAllFactorsNBest;
|
2010-05-08 19:51:59 +04:00
|
|
|
std::string m_detailedTranslationReportingFilePath;
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_onlyDistinctNBest;
|
2013-03-13 16:12:33 +04:00
|
|
|
bool m_PrintAlignmentInfo;
|
2012-11-14 23:01:25 +04:00
|
|
|
bool m_needAlignmentInfo;
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_PrintAlignmentInfoNbest;
|
|
|
|
|
|
|
|
std::string m_alignmentOutputFile;
|
|
|
|
|
|
|
|
std::string m_factorDelimiter; //! by default, |, but it can be changed
|
|
|
|
|
|
|
|
XmlInputType m_xmlInputType; //! method for handling sentence XML input
|
2011-11-16 16:38:22 +04:00
|
|
|
std::pair<std::string,std::string> m_xmlBrackets; //! strings to use as XML tags' opening and closing brackets. Default are "<" and ">"
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
bool m_mbr; //! use MBR decoder
|
2010-02-03 13:23:32 +03:00
|
|
|
bool m_useLatticeMBR; //! use MBR decoder
|
2012-05-28 10:03:45 +04:00
|
|
|
bool m_mira; // do mira training
|
2010-04-07 19:47:58 +04:00
|
|
|
bool m_useConsensusDecoding; //! Use Consensus decoding (DeNero et al 2009)
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t m_mbrSize; //! number of translation candidates considered
|
|
|
|
float m_mbrScale; //! scaling factor for computing marginal probability of candidate translation
|
2010-02-03 13:23:32 +03:00
|
|
|
size_t m_lmbrPruning; //! average number of nodes per word wanted in pruned lattice
|
2010-02-18 17:15:34 +03:00
|
|
|
std::vector<float> m_lmbrThetas; //! theta(s) for lattice mbr calculation
|
2010-02-03 14:20:20 +03:00
|
|
|
bool m_useLatticeHypSetForLatticeMBR; //! to use nbest as hypothesis set during lattice MBR
|
2010-02-03 22:46:35 +03:00
|
|
|
float m_lmbrPrecision; //! unigram precision theta - see Tromble et al 08 for more details
|
|
|
|
float m_lmbrPRatio; //! decaying factor for ngram thetas - see Tromble et al 08 for more details
|
2010-03-14 23:23:17 +03:00
|
|
|
float m_lmbrMapWeight; //! Weight given to the map solution. See Kumar et al 09 for details
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
|
2011-09-09 22:03:00 +04:00
|
|
|
bool m_lmEnableOOVFeature;
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
bool m_timeout; //! use timeout
|
|
|
|
size_t m_timeout_threshold; //! seconds after which time out is activated
|
|
|
|
|
|
|
|
bool m_useTransOptCache; //! flag indicating, if the persistent translation option cache should be used
|
|
|
|
mutable std::map<std::pair<size_t, Phrase>, std::pair<TranslationOptionList*,clock_t> > m_transOptCache; //! persistent translation option cache
|
|
|
|
size_t m_transOptCacheMaxSize; //! maximum size for persistent translation option cache
|
|
|
|
//FIXME: Single lock for cache not most efficient. However using a
|
|
|
|
//reader-writer for LRU cache is tricky - how to record last used time?
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
mutable boost::mutex m_transOptCacheMutex;
|
2009-08-07 20:47:54 +04:00
|
|
|
#endif
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_isAlwaysCreateDirectTranslationOption;
|
|
|
|
//! constructor. only the 1 static variable can be created
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_outputWordGraph; //! whether to output word graph
|
|
|
|
bool m_outputSearchGraph; //! whether to output search graph
|
|
|
|
bool m_outputSearchGraphExtended; //! ... in extended format
|
2013-02-15 22:06:54 +04:00
|
|
|
bool m_outputSearchGraphSLF; //! whether to output search graph in HTK standard lattice format (SLF)
|
2013-02-22 21:24:35 +04:00
|
|
|
bool m_outputSearchGraphHypergraph; //! whether to output search graph in hypergraph
|
2008-09-24 20:48:23 +04:00
|
|
|
#ifdef HAVE_PROTOBUF
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_outputSearchGraphPB; //! whether to output search graph as a protobuf
|
2008-09-24 20:48:23 +04:00
|
|
|
#endif
|
2011-09-16 15:58:53 +04:00
|
|
|
bool m_unprunedSearchGraph; //! do not exclude dead ends (chart decoder only)
|
2012-09-03 10:23:32 +04:00
|
|
|
bool m_includeLHSInSearchGraph; //! include LHS of rules in search graph
|
2012-09-21 11:55:37 +04:00
|
|
|
std::string m_outputUnknownsFile; //! output unknowns in this file
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t m_cubePruningPopLimit;
|
|
|
|
size_t m_cubePruningDiversity;
|
2011-06-27 19:13:15 +04:00
|
|
|
bool m_cubePruningLazyScoring;
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t m_ruleLimit;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2012-08-03 18:38:45 +04:00
|
|
|
// Whether to load compact phrase table and reordering table into memory
|
2012-08-03 14:04:39 +04:00
|
|
|
bool m_minphrMemory;
|
|
|
|
bool m_minlexrMemory;
|
2011-08-18 01:13:21 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
// Initial = 0 = can be used when creating poss trans
|
|
|
|
// Other = 1 = used to calculate LM score once all steps have been processed
|
|
|
|
Word m_inputDefaultNonTerminal, m_outputDefaultNonTerminal;
|
|
|
|
SourceLabelOverlap m_sourceLabelOverlap;
|
|
|
|
UnknownLHSList m_unknownLHS;
|
2011-08-26 06:37:52 +04:00
|
|
|
WordAlignmentSort m_wordAlignmentSort;
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2011-09-23 02:29:56 +04:00
|
|
|
int m_threadCount;
|
2011-11-13 21:14:40 +04:00
|
|
|
long m_startTranslationId;
|
2013-05-31 15:28:57 +04:00
|
|
|
|
|
|
|
// alternate weight settings
|
|
|
|
std::map< std::string, ScoreComponentCollection* > m_weightSetting;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
StaticData();
|
2008-06-11 14:52:57 +04:00
|
|
|
|
2010-10-07 02:06:49 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
void LoadChartDecodingParameters();
|
|
|
|
void LoadNonTerminals();
|
2010-04-08 21:16:10 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
//! helper fn to set bool param from ini file/command line
|
|
|
|
void SetBooleanParameter(bool *paramter, std::string parameterName, bool defaultValue);
|
2013-01-28 21:33:45 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
//! load decoding steps
|
2010-10-15 19:19:17 +04:00
|
|
|
bool LoadDecodeGraphs();
|
2011-08-19 20:09:36 +04:00
|
|
|
|
2010-10-15 19:19:17 +04:00
|
|
|
void ReduceTransOptCache() const;
|
2011-02-24 16:14:42 +03:00
|
|
|
bool m_continuePartialTranslation;
|
|
|
|
|
2012-07-31 00:07:19 +04:00
|
|
|
std::string m_binPath;
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
public:
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool IsAlwaysCreateDirectTranslationOption() const {
|
|
|
|
return m_isAlwaysCreateDirectTranslationOption;
|
|
|
|
}
|
|
|
|
//! destructor
|
|
|
|
~StaticData();
|
2011-08-19 20:09:36 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
//! return static instance for use like global variable
|
|
|
|
static const StaticData& Instance() {
|
|
|
|
return s_instance;
|
|
|
|
}
|
|
|
|
|
2011-08-19 20:09:36 +04:00
|
|
|
//! do NOT call unless you know what you're doing
|
|
|
|
static StaticData& InstanceNonConst() {
|
|
|
|
return s_instance;
|
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
/** delete current static instance and replace with another.
|
|
|
|
* Used by gui front end
|
|
|
|
*/
|
|
|
|
#ifdef WIN32
|
|
|
|
static void Reset() {
|
|
|
|
s_instance = StaticData();
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2011-02-24 19:17:38 +03:00
|
|
|
//! Load data into static instance. This function is required as LoadData() is not const
|
2012-07-31 00:07:19 +04:00
|
|
|
static bool LoadDataStatic(Parameter *parameter, const std::string &execPath);
|
2011-02-24 16:14:42 +03:00
|
|
|
|
2011-02-24 19:17:38 +03:00
|
|
|
//! Main function to load everything. Also initialize the Parameter object
|
2011-02-24 16:14:42 +03:00
|
|
|
bool LoadData(Parameter *parameter);
|
2012-04-29 08:37:48 +04:00
|
|
|
void ClearData();
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
const PARAM_VEC &GetParam(const std::string ¶mName) const {
|
|
|
|
return m_parameter->GetParam(paramName);
|
|
|
|
}
|
|
|
|
|
|
|
|
const std::vector<FactorType> &GetInputFactorOrder() const {
|
|
|
|
return m_inputFactorOrder;
|
|
|
|
}
|
|
|
|
const std::vector<FactorType> &GetOutputFactorOrder() const {
|
|
|
|
return m_outputFactorOrder;
|
|
|
|
}
|
|
|
|
|
|
|
|
inline bool GetSourceStartPosMattersForRecombination() const {
|
|
|
|
return m_sourceStartPosMattersForRecombination;
|
|
|
|
}
|
|
|
|
inline bool GetDropUnknown() const {
|
|
|
|
return m_dropUnknown;
|
|
|
|
}
|
|
|
|
inline bool GetDisableDiscarding() const {
|
2010-03-07 10:57:48 +03:00
|
|
|
return m_disableDiscarding;
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
inline size_t GetMaxNoTransOptPerCoverage() const {
|
|
|
|
return m_maxNoTransOptPerCoverage;
|
|
|
|
}
|
|
|
|
inline size_t GetMaxNoPartTransOpt() const {
|
|
|
|
return m_maxNoPartTransOpt;
|
|
|
|
}
|
|
|
|
inline const Phrase* GetConstrainingPhrase(long sentenceID) const {
|
|
|
|
std::map<long,Phrase>::const_iterator iter = m_constraints.find(sentenceID);
|
|
|
|
if (iter != m_constraints.end()) {
|
|
|
|
const Phrase& phrase = iter->second;
|
|
|
|
return &phrase;
|
|
|
|
} else {
|
|
|
|
return NULL;
|
2010-08-10 17:12:00 +04:00
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
}
|
|
|
|
inline size_t GetMaxPhraseLength() const {
|
|
|
|
return m_maxPhraseLength;
|
|
|
|
}
|
|
|
|
bool IsWordDeletionEnabled() const {
|
|
|
|
return m_wordDeletionEnabled;
|
|
|
|
}
|
|
|
|
size_t GetMaxHypoStackSize() const {
|
|
|
|
return m_maxHypoStackSize;
|
|
|
|
}
|
|
|
|
size_t GetMinHypoStackDiversity() const {
|
|
|
|
return m_minHypoStackDiversity;
|
|
|
|
}
|
|
|
|
size_t GetCubePruningPopLimit() const {
|
|
|
|
return m_cubePruningPopLimit;
|
|
|
|
}
|
|
|
|
size_t GetCubePruningDiversity() const {
|
|
|
|
return m_cubePruningDiversity;
|
|
|
|
}
|
2011-06-27 19:13:15 +04:00
|
|
|
bool GetCubePruningLazyScoring() const {
|
|
|
|
return m_cubePruningLazyScoring;
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t IsPathRecoveryEnabled() const {
|
|
|
|
return m_recoverPath;
|
|
|
|
}
|
|
|
|
int GetMaxDistortion() const {
|
|
|
|
return m_maxDistortion;
|
|
|
|
}
|
|
|
|
bool UseReorderingConstraint() const {
|
|
|
|
return m_reorderingConstraint;
|
|
|
|
}
|
|
|
|
float GetBeamWidth() const {
|
|
|
|
return m_beamWidth;
|
|
|
|
}
|
|
|
|
float GetEarlyDiscardingThreshold() const {
|
|
|
|
return m_earlyDiscardingThreshold;
|
|
|
|
}
|
|
|
|
bool UseEarlyDiscarding() const {
|
|
|
|
return m_earlyDiscardingThreshold != -std::numeric_limits<float>::infinity();
|
|
|
|
}
|
2012-09-13 13:43:01 +04:00
|
|
|
bool UseEarlyDistortionCost() const {
|
|
|
|
return m_useEarlyDistortionCost;
|
2012-06-21 19:41:05 +04:00
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
float GetTranslationOptionThreshold() const {
|
|
|
|
return m_translationOptionThreshold;
|
|
|
|
}
|
2010-10-07 02:06:49 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t GetVerboseLevel() const {
|
|
|
|
return m_verboseLevel;
|
|
|
|
}
|
|
|
|
void SetVerboseLevel(int x) const {
|
|
|
|
m_verboseLevel = x;
|
|
|
|
}
|
|
|
|
bool GetReportSegmentation() const {
|
|
|
|
return m_reportSegmentation;
|
|
|
|
}
|
|
|
|
bool GetReportAllFactors() const {
|
|
|
|
return m_reportAllFactors;
|
|
|
|
}
|
|
|
|
bool GetReportAllFactorsNBest() const {
|
|
|
|
return m_reportAllFactorsNBest;
|
|
|
|
}
|
|
|
|
bool IsDetailedTranslationReportingEnabled() const {
|
|
|
|
return !m_detailedTranslationReportingFilePath.empty();
|
|
|
|
}
|
|
|
|
const std::string &GetDetailedTranslationReportingFilePath() const {
|
|
|
|
return m_detailedTranslationReportingFilePath;
|
|
|
|
}
|
|
|
|
bool IsLabeledNBestList() const {
|
|
|
|
return m_labeledNBestList;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-08-03 14:04:39 +04:00
|
|
|
bool UseMinphrInMemory() const {
|
2013-05-29 21:16:15 +04:00
|
|
|
return m_minphrMemory;
|
2012-08-03 14:04:39 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
bool UseMinlexrInMemory() const {
|
2013-05-29 21:16:15 +04:00
|
|
|
return m_minlexrMemory;
|
2012-08-03 14:04:39 +04:00
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2012-12-06 00:21:33 +04:00
|
|
|
size_t GetNumRealWordsInInput() const {
|
|
|
|
return m_numRealWordsInInput;
|
2011-02-24 16:14:42 +03:00
|
|
|
}
|
|
|
|
const std::vector<std::string> &GetDescription() const {
|
|
|
|
return m_parameter->GetParam("description");
|
|
|
|
}
|
|
|
|
|
|
|
|
// for mert
|
|
|
|
size_t GetNBestSize() const {
|
|
|
|
return m_nBestSize;
|
|
|
|
}
|
|
|
|
const std::string &GetNBestFilePath() const {
|
|
|
|
return m_nBestFilePath;
|
|
|
|
}
|
|
|
|
bool IsNBestEnabled() const {
|
2013-02-23 00:48:40 +04:00
|
|
|
return (!m_nBestFilePath.empty()) || m_mbr || m_useLatticeMBR || m_mira || m_outputSearchGraph || m_outputSearchGraphSLF || m_outputSearchGraphHypergraph || m_useConsensusDecoding || !m_latticeSamplesFilePath.empty()
|
2008-09-24 20:48:23 +04:00
|
|
|
#ifdef HAVE_PROTOBUF
|
2011-02-24 16:14:42 +03:00
|
|
|
|| m_outputSearchGraphPB
|
2008-09-24 20:48:23 +04:00
|
|
|
#endif
|
2011-02-24 16:14:42 +03:00
|
|
|
;
|
|
|
|
}
|
2011-10-04 19:46:24 +04:00
|
|
|
size_t GetLatticeSamplesSize() const {
|
|
|
|
return m_latticeSamplesSize;
|
|
|
|
}
|
|
|
|
|
|
|
|
const std::string& GetLatticeSamplesFilePath() const {
|
|
|
|
return m_latticeSamplesFilePath;
|
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t GetNBestFactor() const {
|
|
|
|
return m_nBestFactor;
|
|
|
|
}
|
|
|
|
bool GetOutputWordGraph() const {
|
|
|
|
return m_outputWordGraph;
|
|
|
|
}
|
|
|
|
|
2013-02-22 00:03:35 +04:00
|
|
|
//! Sets the global score vector weights for a given FeatureFunction.
|
2011-02-24 16:14:42 +03:00
|
|
|
InputTypeEnum GetInputType() const {
|
|
|
|
return m_inputType;
|
|
|
|
}
|
2012-01-26 15:38:40 +04:00
|
|
|
ParsingAlgorithm GetParsingAlgorithm() const {
|
|
|
|
return m_parsingAlgorithm;
|
|
|
|
}
|
2011-02-24 16:14:42 +03:00
|
|
|
SearchAlgorithm GetSearchAlgorithm() const {
|
|
|
|
return m_searchAlgorithm;
|
|
|
|
}
|
2012-10-12 17:09:45 +04:00
|
|
|
bool IsChart() const {
|
|
|
|
return m_searchAlgorithm == ChartDecoding || m_searchAlgorithm == ChartIncremental;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
const WordPenaltyProducer *GetWordPenaltyProducer() const {
|
|
|
|
return m_wpProducer;
|
|
|
|
}
|
|
|
|
WordPenaltyProducer *GetWordPenaltyProducer() { // for mira
|
|
|
|
return m_wpProducer;
|
|
|
|
}
|
2012-12-19 19:38:57 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
const UnknownWordPenaltyProducer *GetUnknownWordPenaltyProducer() const {
|
|
|
|
return m_unknownWordPenaltyProducer;
|
|
|
|
}
|
2012-12-19 20:51:55 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
size_t GetNumInputScores() const {
|
|
|
|
return m_numInputScores;
|
|
|
|
}
|
|
|
|
|
2011-08-19 20:09:36 +04:00
|
|
|
const ScoreComponentCollection& GetAllWeights() const {
|
2011-02-24 16:14:42 +03:00
|
|
|
return m_allWeights;
|
|
|
|
}
|
|
|
|
|
2011-08-19 20:09:36 +04:00
|
|
|
void SetAllWeights(const ScoreComponentCollection& weights) {
|
2010-10-07 02:06:49 +04:00
|
|
|
m_allWeights = weights;
|
|
|
|
}
|
|
|
|
|
|
|
|
//Weight for a single-valued feature
|
2013-02-22 00:03:35 +04:00
|
|
|
float GetWeight(const FeatureFunction* sp) const {
|
2010-10-07 02:06:49 +04:00
|
|
|
return m_allWeights.GetScoreForProducer(sp);
|
|
|
|
}
|
|
|
|
|
|
|
|
//Weight for a single-valued feature
|
2013-02-22 00:03:35 +04:00
|
|
|
void SetWeight(const FeatureFunction* sp, float weight) ;
|
2010-10-07 02:06:49 +04:00
|
|
|
|
|
|
|
|
|
|
|
//Weights for feature with fixed number of values
|
2013-02-22 00:03:35 +04:00
|
|
|
std::vector<float> GetWeights(const FeatureFunction* sp) const {
|
2010-10-07 02:06:49 +04:00
|
|
|
return m_allWeights.GetScoresForProducer(sp);
|
|
|
|
}
|
|
|
|
|
2012-03-11 20:26:25 +04:00
|
|
|
float GetSparseWeight(const FName& featureName) const {
|
|
|
|
return m_allWeights.GetSparseWeight(featureName);
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
2010-10-07 02:06:49 +04:00
|
|
|
//Weights for feature with fixed number of values
|
2013-02-22 00:03:35 +04:00
|
|
|
void SetWeights(const FeatureFunction* sp, const std::vector<float>& weights);
|
2011-08-19 20:09:36 +04:00
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool GetDistinctNBest() const {
|
|
|
|
return m_onlyDistinctNBest;
|
|
|
|
}
|
|
|
|
const std::string& GetFactorDelimiter() const {
|
|
|
|
return m_factorDelimiter;
|
|
|
|
}
|
|
|
|
bool UseMBR() const {
|
|
|
|
return m_mbr;
|
|
|
|
}
|
|
|
|
bool UseLatticeMBR() const {
|
|
|
|
return m_useLatticeMBR ;
|
|
|
|
}
|
|
|
|
bool UseConsensusDecoding() const {
|
|
|
|
return m_useConsensusDecoding;
|
|
|
|
}
|
|
|
|
void SetUseLatticeMBR(bool flag) {
|
|
|
|
m_useLatticeMBR = flag;
|
|
|
|
}
|
|
|
|
size_t GetMBRSize() const {
|
|
|
|
return m_mbrSize;
|
|
|
|
}
|
|
|
|
float GetMBRScale() const {
|
|
|
|
return m_mbrScale;
|
|
|
|
}
|
|
|
|
void SetMBRScale(float scale) {
|
|
|
|
m_mbrScale = scale;
|
|
|
|
}
|
|
|
|
size_t GetLatticeMBRPruningFactor() const {
|
|
|
|
return m_lmbrPruning;
|
|
|
|
}
|
2010-02-18 17:15:34 +03:00
|
|
|
void SetLatticeMBRPruningFactor(size_t prune) {
|
2011-02-24 16:14:42 +03:00
|
|
|
m_lmbrPruning = prune;
|
|
|
|
}
|
|
|
|
const std::vector<float>& GetLatticeMBRThetas() const {
|
|
|
|
return m_lmbrThetas;
|
|
|
|
}
|
|
|
|
bool UseLatticeHypSetForLatticeMBR() const {
|
|
|
|
return m_useLatticeHypSetForLatticeMBR;
|
2010-02-18 17:15:34 +03:00
|
|
|
}
|
2010-02-03 22:46:35 +03:00
|
|
|
float GetLatticeMBRPrecision() const {
|
|
|
|
return m_lmbrPrecision;
|
|
|
|
}
|
2010-02-18 17:15:34 +03:00
|
|
|
void SetLatticeMBRPrecision(float p) {
|
2011-02-24 16:14:42 +03:00
|
|
|
m_lmbrPrecision = p;
|
2010-02-18 17:15:34 +03:00
|
|
|
}
|
2010-02-03 22:46:35 +03:00
|
|
|
float GetLatticeMBRPRatio() const {
|
|
|
|
return m_lmbrPRatio;
|
|
|
|
}
|
2010-02-18 17:15:34 +03:00
|
|
|
void SetLatticeMBRPRatio(float r) {
|
2011-02-24 16:14:42 +03:00
|
|
|
m_lmbrPRatio = r;
|
|
|
|
}
|
|
|
|
|
|
|
|
float GetLatticeMBRMapWeight() const {
|
|
|
|
return m_lmbrMapWeight;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool UseTimeout() const {
|
|
|
|
return m_timeout;
|
|
|
|
}
|
|
|
|
size_t GetTimeoutThreshold() const {
|
|
|
|
return m_timeout_threshold;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t GetLMCacheCleanupThreshold() const {
|
|
|
|
return m_lmcache_cleanup_threshold;
|
|
|
|
}
|
|
|
|
|
2011-09-09 22:03:00 +04:00
|
|
|
bool GetLMEnableOOVFeature() const {
|
|
|
|
return m_lmEnableOOVFeature;
|
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool GetOutputSearchGraph() const {
|
|
|
|
return m_outputSearchGraph;
|
|
|
|
}
|
|
|
|
void SetOutputSearchGraph(bool outputSearchGraph) {
|
|
|
|
m_outputSearchGraph = outputSearchGraph;
|
|
|
|
}
|
|
|
|
bool GetOutputSearchGraphExtended() const {
|
|
|
|
return m_outputSearchGraphExtended;
|
|
|
|
}
|
2013-02-15 22:06:54 +04:00
|
|
|
bool GetOutputSearchGraphSLF() const {
|
|
|
|
return m_outputSearchGraphSLF;
|
|
|
|
}
|
2013-02-22 21:24:35 +04:00
|
|
|
bool GetOutputSearchGraphHypergraph() const {
|
|
|
|
return m_outputSearchGraphHypergraph;
|
|
|
|
}
|
2008-09-24 20:48:23 +04:00
|
|
|
#ifdef HAVE_PROTOBUF
|
2011-02-24 16:14:42 +03:00
|
|
|
bool GetOutputSearchGraphPB() const {
|
|
|
|
return m_outputSearchGraphPB;
|
|
|
|
}
|
2008-09-24 20:48:23 +04:00
|
|
|
#endif
|
2012-09-21 11:55:37 +04:00
|
|
|
const std::string& GetOutputUnknownsFile() const {
|
|
|
|
return m_outputUnknownsFile;
|
|
|
|
}
|
|
|
|
|
2011-09-16 15:58:53 +04:00
|
|
|
bool GetUnprunedSearchGraph() const {
|
|
|
|
return m_unprunedSearchGraph;
|
|
|
|
}
|
2008-09-24 20:48:23 +04:00
|
|
|
|
2012-09-03 10:23:32 +04:00
|
|
|
bool GetIncludeLHSInSearchGraph() const {
|
|
|
|
return m_includeLHSInSearchGraph;
|
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
XmlInputType GetXmlInputType() const {
|
|
|
|
return m_xmlInputType;
|
|
|
|
}
|
|
|
|
|
2011-11-16 16:38:22 +04:00
|
|
|
std::pair<std::string,std::string> GetXmlBrackets() const {
|
|
|
|
return m_xmlBrackets;
|
|
|
|
}
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
bool GetUseTransOptCache() const {
|
|
|
|
return m_useTransOptCache;
|
|
|
|
}
|
|
|
|
|
|
|
|
void AddTransOptListToCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase, const TranslationOptionList &transOptList) const;
|
|
|
|
|
2011-05-31 13:42:27 +04:00
|
|
|
void ClearTransOptionCache() const;
|
|
|
|
|
2011-02-24 16:14:42 +03:00
|
|
|
|
|
|
|
const TranslationOptionList* FindTransOptListInCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase) const;
|
|
|
|
|
|
|
|
bool PrintAllDerivations() const {
|
|
|
|
return m_printAllDerivations;
|
|
|
|
}
|
|
|
|
|
|
|
|
const UnknownLHSList &GetUnknownLHS() const {
|
|
|
|
return m_unknownLHS;
|
|
|
|
}
|
|
|
|
|
|
|
|
const Word &GetInputDefaultNonTerminal() const {
|
|
|
|
return m_inputDefaultNonTerminal;
|
|
|
|
}
|
|
|
|
const Word &GetOutputDefaultNonTerminal() const {
|
|
|
|
return m_outputDefaultNonTerminal;
|
|
|
|
}
|
|
|
|
|
|
|
|
SourceLabelOverlap GetSourceLabelOverlap() const {
|
|
|
|
return m_sourceLabelOverlap;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool GetOutputHypoScore() const {
|
|
|
|
return m_outputHypoScore;
|
|
|
|
}
|
|
|
|
size_t GetRuleLimit() const {
|
|
|
|
return m_ruleLimit;
|
|
|
|
}
|
|
|
|
float GetRuleCountThreshold() const {
|
|
|
|
return 999999; /* TODO wtf! */
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ContinuePartialTranslation() const {
|
|
|
|
return m_continuePartialTranslation;
|
|
|
|
}
|
2011-08-19 20:09:36 +04:00
|
|
|
|
|
|
|
void ReLoadParameter();
|
2011-11-16 13:13:17 +04:00
|
|
|
void ReLoadBleuScoreFeatureParameter(float weight);
|
|
|
|
|
2011-08-19 20:09:36 +04:00
|
|
|
Parameter* GetParameter() {
|
|
|
|
return m_parameter;
|
|
|
|
}
|
2010-09-17 18:46:00 +04:00
|
|
|
|
2011-09-23 02:29:56 +04:00
|
|
|
int ThreadCount() const {
|
|
|
|
return m_threadCount;
|
|
|
|
}
|
2013-05-29 21:16:15 +04:00
|
|
|
|
|
|
|
long GetStartTranslationId() const {
|
|
|
|
return m_startTranslationId;
|
|
|
|
}
|
|
|
|
|
2012-07-31 00:07:19 +04:00
|
|
|
void SetExecPath(const std::string &path);
|
|
|
|
const std::string &GetBinDirectory() const;
|
2012-11-14 23:01:25 +04:00
|
|
|
|
|
|
|
bool NeedAlignmentInfo() const {
|
2013-05-29 21:16:15 +04:00
|
|
|
return m_needAlignmentInfo;
|
|
|
|
}
|
2012-11-14 23:01:25 +04:00
|
|
|
const std::string &GetAlignmentOutputFile() const {
|
|
|
|
return m_alignmentOutputFile;
|
|
|
|
}
|
2013-03-13 16:12:33 +04:00
|
|
|
bool PrintAlignmentInfo() const {
|
|
|
|
return m_PrintAlignmentInfo;
|
|
|
|
}
|
2012-11-14 23:01:25 +04:00
|
|
|
bool PrintAlignmentInfoInNbest() const {
|
|
|
|
return m_PrintAlignmentInfoNbest;
|
|
|
|
}
|
|
|
|
WordAlignmentSort GetWordAlignmentSort() const {
|
|
|
|
return m_wordAlignmentSort;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool NBestIncludesSegmentation() const {
|
|
|
|
return m_nBestIncludesSegmentation;
|
|
|
|
}
|
|
|
|
|
2013-05-31 15:28:57 +04:00
|
|
|
bool GetHasAlternateWeightSettings() const {
|
|
|
|
return m_weightSetting.size() > 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetWeightSetting(const std::string &settingName) const {
|
|
|
|
std::cerr << "SetWeightSetting( " << settingName << ")\n";
|
|
|
|
CHECK(GetHasAlternateWeightSettings());
|
|
|
|
std::map< std::string, ScoreComponentCollection* >::const_iterator i =
|
|
|
|
m_weightSetting.find( settingName );
|
|
|
|
// if not found, resort to default
|
|
|
|
std::cerr << "using weight setting " << settingName << std::endl;
|
|
|
|
if (i == m_weightSetting.end()) {
|
|
|
|
i = m_weightSetting.find( "default" );
|
|
|
|
std::cerr << "not found, using default weight setting instead\n";
|
|
|
|
}
|
|
|
|
m_allWeights = *(i->second);
|
|
|
|
}
|
|
|
|
|
2012-12-19 19:38:57 +04:00
|
|
|
float GetWeightWordPenalty() const;
|
|
|
|
float GetWeightUnknownWordPenalty() const;
|
2012-12-19 18:37:00 +04:00
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
const std::vector<PhraseDictionary*>& GetPhraseDictionaries() const {
|
|
|
|
return m_phraseDictionary;
|
|
|
|
}
|
|
|
|
const std::vector<const GenerationDictionary*>& GetGenerationDictionaries() const {
|
|
|
|
return m_generationDictionary;
|
|
|
|
}
|
|
|
|
const PhraseDictionary*GetTranslationScoreProducer(size_t index) const {
|
|
|
|
return GetPhraseDictionaries().at(index);
|
|
|
|
}
|
2012-12-21 18:54:43 +04:00
|
|
|
std::vector<float> GetTranslationWeights(size_t index) const {
|
|
|
|
std::vector<float> weights = GetWeights(GetTranslationScoreProducer(index));
|
|
|
|
return weights;
|
|
|
|
}
|
|
|
|
|
2013-05-29 21:16:15 +04:00
|
|
|
const std::vector<DecodeGraph*>& GetDecodeGraphs() const {
|
|
|
|
return m_decodeGraphs;
|
|
|
|
}
|
|
|
|
const std::vector<size_t>& GetDecodeGraphBackoff() const {
|
|
|
|
return m_decodeGraphBackoff;
|
|
|
|
}
|
2012-12-21 19:28:34 +04:00
|
|
|
|
2012-12-21 19:59:52 +04:00
|
|
|
//sentence (and thread) specific initialisationn and cleanup
|
2012-12-24 22:52:04 +04:00
|
|
|
void InitializeForInput(const InputType& source) const;
|
2012-12-21 19:59:52 +04:00
|
|
|
void CleanUpAfterSentenceProcessing(const InputType& source) const;
|
2013-01-17 21:15:10 +04:00
|
|
|
|
2013-01-18 21:57:26 +04:00
|
|
|
void CollectFeatureFunctions();
|
2013-02-07 00:05:00 +04:00
|
|
|
bool CheckWeights() const;
|
2013-05-31 15:28:57 +04:00
|
|
|
void ProcessAlternateWeightSettings();
|
2013-02-07 00:05:00 +04:00
|
|
|
|
2013-05-10 15:30:01 +04:00
|
|
|
|
|
|
|
void SetTemporaryMultiModelWeightsVector(std::vector<float> weights) const {
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
m_multimodelweights_tmp[boost::this_thread::get_id()] = weights;
|
|
|
|
#else
|
|
|
|
m_multimodelweights_tmp = weights;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
// multimodel
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
mutable std::map<boost::thread::id, std::vector<float> > m_multimodelweights_tmp;
|
|
|
|
#else
|
|
|
|
mutable std::vector<float> m_multimodelweights_tmp;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
const std::vector<float>* GetTemporaryMultiModelWeightsVector() const {
|
|
|
|
#ifdef WITH_THREADS
|
|
|
|
if (m_multimodelweights_tmp.find(boost::this_thread::get_id()) != m_multimodelweights_tmp.end()) {
|
|
|
|
return &m_multimodelweights_tmp.find(boost::this_thread::get_id())->second;
|
2013-05-29 21:16:15 +04:00
|
|
|
} else {
|
2013-05-10 15:30:01 +04:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
return &m_multimodelweights_tmp;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
};
|
2008-10-09 03:51:26 +04:00
|
|
|
|
|
|
|
}
|
2010-02-24 14:15:44 +03:00
|
|
|
#endif
|