mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Changes to PhraseDictionaryCompact and LexicalReorderingtableCompact:
- minphr-memory and minlexr-memory are now static members of the respective classes, not global variables; they are interpreted by class-specific option interpretation functions that are called from StaticData.cpp. - m_weight has been removed from PhraseDictionaryCompact and PhraseDecoder, as it was never used.
This commit is contained in:
parent
65f4f1f92a
commit
9196cd1461
@ -50,6 +50,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#ifdef WITH_THREADS
|
||||
#include <boost/thread.hpp>
|
||||
#endif
|
||||
#ifdef HAVE_CMPH
|
||||
#include "moses/TranslationModel/CompactPT/PhraseDictionaryCompact.h"
|
||||
#endif
|
||||
#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH
|
||||
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h"
|
||||
#endif
|
||||
|
||||
using namespace std;
|
||||
using namespace boost::algorithm;
|
||||
@ -203,12 +209,19 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
if (!ini_performance_options()) return false;
|
||||
|
||||
// Compact phrase table and reordering model
|
||||
m_parameter->SetParameter(m_minphrMemory, "minphr-memory", false );
|
||||
m_parameter->SetParameter(m_minlexrMemory, "minlexr-memory", false );
|
||||
// m_parameter->SetParameter(m_minphrMemory, "minphr-memory", false );
|
||||
// m_parameter->SetParameter(m_minlexrMemory, "minlexr-memory", false );
|
||||
|
||||
// S2T decoder
|
||||
|
||||
// FEATURE FUNCTION INITIALIZATION HAPPENS HERE ===============================
|
||||
|
||||
// set class-specific default parameters
|
||||
#if !defined WIN32 || defined __MINGW32__ || defined HAVE_CMPH
|
||||
LexicalReorderingTableCompact::SetStaticDefaultParameters(*parameter);
|
||||
PhraseDictionaryCompact::SetStaticDefaultParameters(*parameter);
|
||||
#endif
|
||||
|
||||
initialize_features();
|
||||
|
||||
if (m_parameter->GetParam("show-weights") == NULL)
|
||||
|
@ -92,26 +92,16 @@ protected:
|
||||
bool m_reorderingConstraint; //! use additional reordering constraints
|
||||
BookkeepingOptions m_bookkeeping_options;
|
||||
|
||||
size_t m_latticeSamplesSize;
|
||||
|
||||
std::string m_latticeSamplesFilePath;
|
||||
// bool m_wordDeletionEnabled;
|
||||
|
||||
bool m_printAllDerivations;
|
||||
bool m_printTranslationOptions;
|
||||
|
||||
// bool m_sourceStartPosMattersForRecombination;
|
||||
bool m_requireSortingAfterSourceContext;
|
||||
|
||||
mutable size_t m_verboseLevel;
|
||||
|
||||
std::string m_factorDelimiter; //! by default, |, but it can be changed
|
||||
|
||||
std::pair<std::string,std::string> m_xmlBrackets; //! strings to use as XML tags' opening and closing brackets. Default are "<" and ">"
|
||||
|
||||
size_t m_lmcache_cleanup_threshold; //! number of translations after which LM claenup is performed (0=never, N=after N translations; default is 1)
|
||||
|
||||
bool m_includeLHSInSearchGraph; //! include LHS of rules in search graph
|
||||
std::string m_outputUnknownsFile; //! output unknowns in this file
|
||||
|
||||
size_t m_ruleLimit;
|
||||
@ -220,16 +210,6 @@ public:
|
||||
m_verboseLevel = x;
|
||||
}
|
||||
|
||||
bool
|
||||
UseMinphrInMemory() const {
|
||||
return m_minphrMemory;
|
||||
}
|
||||
|
||||
bool
|
||||
UseMinlexrInMemory() const {
|
||||
return m_minlexrMemory;
|
||||
}
|
||||
|
||||
const ScoreComponentCollection&
|
||||
GetAllWeights() const {
|
||||
return m_allWeights;
|
||||
@ -428,10 +408,6 @@ public:
|
||||
m_treeStructure = treeStructure;
|
||||
}
|
||||
|
||||
// bool GetDefaultNonTermOnlyForEmptyRange() const {
|
||||
// return m_defaultNonTermOnlyForEmptyRange;
|
||||
// }
|
||||
|
||||
bool RequireSortingAfterSourceContext() const {
|
||||
return m_requireSortingAfterSourceContext;
|
||||
}
|
||||
|
@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
bool LexicalReorderingTableCompact::s_inMemoryByDefault = false;
|
||||
|
||||
LexicalReorderingTableCompact::
|
||||
LexicalReorderingTableCompact(const std::string& filePath,
|
||||
@ -32,7 +33,7 @@ LexicalReorderingTableCompact(const std::string& filePath,
|
||||
const std::vector<FactorType>& e_factors,
|
||||
const std::vector<FactorType>& c_factors)
|
||||
: LexicalReorderingTable(f_factors, e_factors, c_factors)
|
||||
, m_inMemory(StaticData::Instance().UseMinlexrInMemory())
|
||||
, m_inMemory(s_inMemoryByDefault)
|
||||
, m_numScoreComponent(6)
|
||||
, m_multipleScoreTrees(true)
|
||||
, m_hash(10, 16)
|
||||
@ -46,7 +47,7 @@ LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors,
|
||||
const std::vector<FactorType>& e_factors,
|
||||
const std::vector<FactorType>& c_factors)
|
||||
: LexicalReorderingTable(f_factors, e_factors, c_factors)
|
||||
, m_inMemory(StaticData::Instance().UseMinlexrInMemory())
|
||||
, m_inMemory(s_inMemoryByDefault)
|
||||
, m_numScoreComponent(6)
|
||||
, m_multipleScoreTrees(true)
|
||||
, m_hash(10, 16)
|
||||
@ -183,4 +184,12 @@ Load(std::string filePath)
|
||||
m_scoresMapped.load(pFile, true);
|
||||
}
|
||||
|
||||
void
|
||||
LexicalReorderingTableCompact::
|
||||
SetStaticDefaultParameters(Parameter const& param)
|
||||
{
|
||||
param.SetParameter(s_inMemoryByDefault, "minlexr-memory", false);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
@ -40,6 +40,7 @@ class LexicalReorderingTableCompact:
|
||||
public LexicalReorderingTable
|
||||
{
|
||||
private:
|
||||
static bool s_inMemoryByDefault;
|
||||
bool m_inMemory;
|
||||
|
||||
size_t m_numScoreComponent;
|
||||
@ -82,6 +83,10 @@ public:
|
||||
|
||||
void
|
||||
Load(std::string filePath);
|
||||
|
||||
static void
|
||||
SetStaticDefaultParameters(Parameter const& param);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -33,15 +33,15 @@ PhraseDecoder::PhraseDecoder(
|
||||
PhraseDictionaryCompact &phraseDictionary,
|
||||
const std::vector<FactorType>* input,
|
||||
const std::vector<FactorType>* output,
|
||||
size_t numScoreComponent,
|
||||
const std::vector<float>* weight
|
||||
size_t numScoreComponent
|
||||
// , const std::vector<float>* weight
|
||||
)
|
||||
: m_coding(None), m_numScoreComponent(numScoreComponent),
|
||||
m_containsAlignmentInfo(true), m_maxRank(0),
|
||||
m_symbolTree(0), m_multipleScoreTrees(false),
|
||||
m_scoreTrees(1), m_alignTree(0),
|
||||
m_phraseDictionary(phraseDictionary), m_input(input), m_output(output),
|
||||
m_weight(weight),
|
||||
// m_weight(weight),
|
||||
m_separator(" ||| ")
|
||||
{ }
|
||||
|
||||
|
@ -87,7 +87,6 @@ protected:
|
||||
|
||||
const std::vector<FactorType>* m_input;
|
||||
const std::vector<FactorType>* m_output;
|
||||
const std::vector<float>* m_weight;
|
||||
|
||||
std::string m_separator;
|
||||
|
||||
@ -121,8 +120,7 @@ public:
|
||||
PhraseDictionaryCompact &phraseDictionary,
|
||||
const std::vector<FactorType>* input,
|
||||
const std::vector<FactorType>* output,
|
||||
size_t numScoreComponent,
|
||||
const std::vector<float>* weight
|
||||
size_t numScoreComponent
|
||||
);
|
||||
|
||||
~PhraseDecoder();
|
||||
|
@ -48,11 +48,10 @@ PhraseDictionaryCompact::SentenceCache PhraseDictionaryCompact::m_sentenceCache;
|
||||
|
||||
PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line)
|
||||
:PhraseDictionary(line, true)
|
||||
,m_inMemory(true)
|
||||
,m_inMemory(true)//(s_inMemoryByDefault)
|
||||
,m_useAlignmentInfo(true)
|
||||
,m_hash(10, 16)
|
||||
,m_phraseDecoder(0)
|
||||
,m_weight(0)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
@ -64,8 +63,6 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts)
|
||||
|
||||
SetFeaturesToApply();
|
||||
|
||||
m_weight = staticData.GetWeights(this);
|
||||
|
||||
std::string tFilePath = m_filePath;
|
||||
|
||||
std::string suffix = ".minphr";
|
||||
@ -73,8 +70,8 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts)
|
||||
if (!FileExists(tFilePath))
|
||||
throw runtime_error("Error: File " + tFilePath + " does not exist.");
|
||||
|
||||
m_phraseDecoder = new PhraseDecoder(*this, &m_input, &m_output,
|
||||
m_numScoreComponents, &m_weight);
|
||||
m_phraseDecoder
|
||||
= new PhraseDecoder(*this, &m_input, &m_output, m_numScoreComponents);
|
||||
|
||||
std::FILE* pFile = std::fopen(tFilePath.c_str() , "r");
|
||||
|
||||
@ -82,7 +79,7 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts)
|
||||
//if(m_inMemory)
|
||||
// Load source phrase index into memory
|
||||
indexSize = m_hash.Load(pFile);
|
||||
// else
|
||||
// else
|
||||
// Keep source phrase index on disk
|
||||
//indexSize = m_hash.LoadIndex(pFile);
|
||||
|
||||
@ -100,16 +97,9 @@ void PhraseDictionaryCompact::Load(AllOptions::ptr const& opts)
|
||||
"Not successfully loaded");
|
||||
}
|
||||
|
||||
// now properly declared in TargetPhraseCollection.h
|
||||
// and defined in TargetPhraseCollection.cpp
|
||||
// struct CompareTargetPhrase {
|
||||
// bool operator() (const TargetPhrase &a, const TargetPhrase &b) {
|
||||
// return a.GetFutureScore() > b.GetFutureScore();
|
||||
// }
|
||||
// };
|
||||
|
||||
TargetPhraseCollection::shared_ptr
|
||||
PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const
|
||||
PhraseDictionaryCompact::
|
||||
GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const
|
||||
{
|
||||
|
||||
TargetPhraseCollection::shared_ptr ret;
|
||||
@ -145,7 +135,8 @@ PhraseDictionaryCompact::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &s
|
||||
}
|
||||
|
||||
TargetPhraseVectorPtr
|
||||
PhraseDictionaryCompact::GetTargetPhraseCollectionRaw(const Phrase &sourcePhrase) const
|
||||
PhraseDictionaryCompact::
|
||||
GetTargetPhraseCollectionRaw(const Phrase &sourcePhrase) const
|
||||
{
|
||||
|
||||
// There is no such source phrase if source phrase is longer than longest
|
||||
@ -157,40 +148,46 @@ PhraseDictionaryCompact::GetTargetPhraseCollectionRaw(const Phrase &sourcePhrase
|
||||
return m_phraseDecoder->CreateTargetPhraseCollection(sourcePhrase, true, false);
|
||||
}
|
||||
|
||||
PhraseDictionaryCompact::~PhraseDictionaryCompact()
|
||||
PhraseDictionaryCompact::
|
||||
~PhraseDictionaryCompact()
|
||||
{
|
||||
if(m_phraseDecoder)
|
||||
delete m_phraseDecoder;
|
||||
}
|
||||
|
||||
//TO_STRING_BODY(PhraseDictionaryCompact)
|
||||
|
||||
void PhraseDictionaryCompact::CacheForCleanup(TargetPhraseCollection::shared_ptr tpc)
|
||||
void
|
||||
PhraseDictionaryCompact::
|
||||
CacheForCleanup(TargetPhraseCollection::shared_ptr tpc)
|
||||
{
|
||||
if(!m_sentenceCache.get())
|
||||
m_sentenceCache.reset(new PhraseCache());
|
||||
m_sentenceCache->push_back(tpc);
|
||||
}
|
||||
|
||||
void PhraseDictionaryCompact::AddEquivPhrase(const Phrase &source,
|
||||
const TargetPhrase &targetPhrase) { }
|
||||
void
|
||||
PhraseDictionaryCompact::
|
||||
AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)
|
||||
{ }
|
||||
|
||||
void PhraseDictionaryCompact::CleanUpAfterSentenceProcessing(const InputType &source)
|
||||
void
|
||||
PhraseDictionaryCompact::
|
||||
CleanUpAfterSentenceProcessing(const InputType &source)
|
||||
{
|
||||
if(!m_sentenceCache.get())
|
||||
m_sentenceCache.reset(new PhraseCache());
|
||||
|
||||
m_phraseDecoder->PruneCache();
|
||||
// for(PhraseCache::iterator it = m_sentenceCache->begin();
|
||||
// it != m_sentenceCache->end(); it++)
|
||||
// it->reset();
|
||||
|
||||
// PhraseCache temp;
|
||||
// temp.swap(*m_sentenceCache);
|
||||
m_sentenceCache->clear();
|
||||
|
||||
ReduceCache();
|
||||
}
|
||||
|
||||
bool PhraseDictionaryCompact::s_inMemoryByDefault = false;
|
||||
void
|
||||
PhraseDictionaryCompact::
|
||||
SetStaticDefaultParameters(Parameter const& param)
|
||||
{
|
||||
param.SetParameter(s_inMemoryByDefault, "minphr-memory", true);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -48,6 +48,7 @@ class PhraseDictionaryCompact : public PhraseDictionary
|
||||
protected:
|
||||
friend class PhraseDecoder;
|
||||
|
||||
static bool s_inMemoryByDefault;
|
||||
bool m_inMemory;
|
||||
bool m_useAlignmentInfo;
|
||||
|
||||
@ -61,7 +62,6 @@ protected:
|
||||
StringVector<unsigned char, size_t, MmapAllocator> m_targetPhrasesMapped;
|
||||
StringVector<unsigned char, size_t, std::allocator> m_targetPhrasesMemory;
|
||||
|
||||
std::vector<float> m_weight;
|
||||
public:
|
||||
PhraseDictionaryCompact(const std::string &line);
|
||||
|
||||
@ -76,6 +76,7 @@ public:
|
||||
|
||||
void CacheForCleanup(TargetPhraseCollection::shared_ptr tpc);
|
||||
void CleanUpAfterSentenceProcessing(const InputType &source);
|
||||
static void SetStaticDefaultParameters(Parameter const& param);
|
||||
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const ChartParser &,
|
||||
|
Loading…
Reference in New Issue
Block a user