From b6b5642d235d39343850c51c3f12257c6dc71173 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 21 Jan 2016 11:22:55 +0000 Subject: [PATCH] create uncompressed target side --- contrib/other-builds/moses2/.cproject | 5 +++- .../other-builds/moses2/FF/FeatureFunctions.h | 2 +- .../moses2/FF/LexicalReordering.cpp | 11 +------- .../other-builds/moses2/Search/Hypothesis.cpp | 2 -- .../other-builds/moses2/Search/Manager.cpp | 8 ++---- contrib/other-builds/moses2/TargetPhrase.cpp | 9 ++++--- contrib/other-builds/moses2/TargetPhrase.h | 8 +++--- contrib/other-builds/moses2/TargetPhrases.cpp | 5 +++- contrib/other-builds/moses2/TargetPhrases.h | 4 +-- .../TranslationModel/PhraseTableMemory.cpp | 2 +- .../moses2/TranslationModel/ProbingPT.cpp | 27 +++---------------- .../TranslationModel/UnknownWordPenalty.cpp | 2 +- 12 files changed, 30 insertions(+), 55 deletions(-) diff --git a/contrib/other-builds/moses2/.cproject b/contrib/other-builds/moses2/.cproject index 8f62206dc..3fd012039 100644 --- a/contrib/other-builds/moses2/.cproject +++ b/contrib/other-builds/moses2/.cproject @@ -93,8 +93,11 @@ + + + - + diff --git a/contrib/other-builds/moses2/FF/FeatureFunctions.h b/contrib/other-builds/moses2/FF/FeatureFunctions.h index be0f5b35e..f32b79a1e 100644 --- a/contrib/other-builds/moses2/FF/FeatureFunctions.h +++ b/contrib/other-builds/moses2/FF/FeatureFunctions.h @@ -29,6 +29,7 @@ class Scores; class FeatureFunctions { public: + std::vector m_phraseTables; FeatureFunctions(System &system); virtual ~FeatureFunctions(); @@ -59,7 +60,6 @@ public: protected: std::vector m_featureFunctions; std::vector m_statefulFeatureFunctions; - std::vector m_phraseTables; std::vector m_withPhraseTableInd; System &m_system; diff --git a/contrib/other-builds/moses2/FF/LexicalReordering.cpp b/contrib/other-builds/moses2/FF/LexicalReordering.cpp index eb596b3ad..03f558fee 100644 --- a/contrib/other-builds/moses2/FF/LexicalReordering.cpp +++ b/contrib/other-builds/moses2/FF/LexicalReordering.cpp @@ -7,6 +7,7 @@ #include #include "LexicalReordering.h" +#include "../TranslationModel/PhraseTable.h" #include "../System.h" #include "../Search/Manager.h" #include "../legacy/InputFileStream.h" @@ -162,13 +163,6 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool, { if (m_propertyInd >= 0) { SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd); - /* - cerr << "scoreArr=" << scoreArr << " " << flush; - for (size_t i = 0; i < 6; ++i) { - cerr << scoreArr[i] << " "; - } - cerr << endl; - */ targetPhrase.ffData[m_PhraseTableInd] = scoreArr; } else if (m_compactModel) { @@ -231,19 +225,16 @@ void LexicalReordering::EvaluateWhenApplied(const Manager &mgr, // backwards const TargetPhrase &target = hypo.GetTargetPhrase(); - cerr << "HH D" << endl; const SCORE *values = (const SCORE *) target.ffData[m_PhraseTableInd]; if (values) { scores.PlusEquals(mgr.system, *this, values[orientation], orientation); } - cerr << "HH D2" << endl; // forwards if (prevRange->GetStartPos() != NOT_FOUND) { const TargetPhrase &prevTarget = *prevStateCast.targetPhrase; const SCORE *prevValues = (const SCORE *) prevTarget.ffData[m_PhraseTableInd]; - cerr << "HH E" << endl; if (prevValues) { scores.PlusEquals(mgr.system, *this, prevValues[orientation + 3], orientation + 3); } diff --git a/contrib/other-builds/moses2/Search/Hypothesis.cpp b/contrib/other-builds/moses2/Search/Hypothesis.cpp index c891cc86c..a7d4f4625 100644 --- a/contrib/other-builds/moses2/Search/Hypothesis.cpp +++ b/contrib/other-builds/moses2/Search/Hypothesis.cpp @@ -214,9 +214,7 @@ void Hypothesis::EvaluateWhenApplied() { const std::vector &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions(); BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) { - cerr << "BEFORE " << sfff->GetName() << endl; EvaluateWhenApplied(*sfff); - cerr << "AFTER " << sfff->GetName() << endl; } //cerr << *this << endl; } diff --git a/contrib/other-builds/moses2/Search/Manager.cpp b/contrib/other-builds/moses2/Search/Manager.cpp index 8dcc4456d..8aae19845 100644 --- a/contrib/other-builds/moses2/Search/Manager.cpp +++ b/contrib/other-builds/moses2/Search/Manager.cpp @@ -52,8 +52,8 @@ void Manager::Init() m_hypoRecycle = &system.GetHypoRecycler(); m_bitmaps = new Bitmaps(GetPool()); - m_initPhrase = new (GetPool().Allocate()) TargetPhrase(GetPool(), system, 0); - + const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0]; + m_initPhrase = new (GetPool().Allocate()) TargetPhrase(GetPool(), firstPt, system, 0); // create input phrase obj FactorCollection &vocab = system.GetVocab(); @@ -68,13 +68,9 @@ void Manager::Init() pt.Lookup(*this, m_inputPaths); } //m_inputPaths.DeleteUnusedPaths(); - std::cerr << "HH E" << std::endl; - CalcFutureScore(); - std::cerr << "HH F" << std::endl; m_bitmaps->Init(m_input->GetSize(), vector(0)); - std::cerr << "HH G" << std::endl; switch (system.searchAlgorithm) { case Normal: diff --git a/contrib/other-builds/moses2/TargetPhrase.cpp b/contrib/other-builds/moses2/TargetPhrase.cpp index f84e86dad..39a424046 100644 --- a/contrib/other-builds/moses2/TargetPhrase.cpp +++ b/contrib/other-builds/moses2/TargetPhrase.cpp @@ -17,21 +17,22 @@ using namespace std; namespace Moses2 { -TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const System &system, const std::string &str) +TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str) { FactorCollection &vocab = system.GetVocab(); vector toks = Tokenize(str); size_t size = toks.size(); - TargetPhrase *ret = new (pool.Allocate()) TargetPhrase(pool, system, size); + TargetPhrase *ret = new (pool.Allocate()) TargetPhrase(pool, pt, system, size); ret->PhraseImpl::CreateFromString(vocab, system, toks); return ret; } -TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size) +TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size) :PhraseImpl(pool, size) ,scoreProperties(NULL) +,pt(pt) { m_scores = new (pool.Allocate()) Scores(system, pool, system.featureFunctions.GetNumScores()); @@ -39,6 +40,7 @@ TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size) ffData = new (pool.Allocate(numWithPtData)) void *[numWithPtData]; } +/* TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©) :PhraseImpl(pool, copy) ,scoreProperties(NULL) @@ -50,6 +52,7 @@ TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhra size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size(); ffData = new (pool.Allocate(numWithPtData)) void *[numWithPtData]; } +*/ TargetPhrase::~TargetPhrase() { // TODO Auto-generated destructor stub diff --git a/contrib/other-builds/moses2/TargetPhrase.h b/contrib/other-builds/moses2/TargetPhrase.h index 5d9e964ce..2bc631350 100644 --- a/contrib/other-builds/moses2/TargetPhrase.h +++ b/contrib/other-builds/moses2/TargetPhrase.h @@ -17,6 +17,7 @@ namespace Moses2 class Scores; class Manager; class System; +class PhraseTable; class TargetPhrase : public PhraseImpl { @@ -24,10 +25,11 @@ class TargetPhrase : public PhraseImpl public: mutable void **ffData; SCORE *scoreProperties; + const PhraseTable &pt; - static TargetPhrase *CreateFromString(MemPool &pool, const System &system, const std::string &str); - TargetPhrase(MemPool &pool, const System &system, size_t size); - TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©); + static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str); + TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size); + //TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©); virtual ~TargetPhrase(); diff --git a/contrib/other-builds/moses2/TargetPhrases.cpp b/contrib/other-builds/moses2/TargetPhrases.cpp index 3455977e4..e58d9b318 100644 --- a/contrib/other-builds/moses2/TargetPhrases.cpp +++ b/contrib/other-builds/moses2/TargetPhrases.cpp @@ -19,6 +19,7 @@ TargetPhrases::TargetPhrases(MemPool &pool, size_t reserve) { } +/* TargetPhrases::TargetPhrases(MemPool &pool, const System &system, const TargetPhrases ©) :m_coll(pool, copy.m_coll.size()) { @@ -29,7 +30,7 @@ TargetPhrases::TargetPhrases(MemPool &pool, const System &system, const TargetPh m_coll[i] = tpClone; } } - +*/ TargetPhrases::~TargetPhrases() { // TODO Auto-generated destructor stub @@ -61,11 +62,13 @@ void TargetPhrases::SortAndPrune(size_t tableLimit) //cerr << "TargetPhrases=" << GetSize() << endl; } +/* const TargetPhrases *TargetPhrases::Clone(MemPool &pool, const System &system) const { const TargetPhrases *ret = new (pool.Allocate()) TargetPhrases(pool, system, *this); return ret; } +*/ } diff --git a/contrib/other-builds/moses2/TargetPhrases.h b/contrib/other-builds/moses2/TargetPhrases.h index 878b3a4a0..5b2d1e63b 100644 --- a/contrib/other-builds/moses2/TargetPhrases.h +++ b/contrib/other-builds/moses2/TargetPhrases.h @@ -28,7 +28,7 @@ public: } TargetPhrases(MemPool &pool, size_t reserve); - TargetPhrases(MemPool &pool, const System &system, const TargetPhrases ©); + //TargetPhrases(MemPool &pool, const System &system, const TargetPhrases ©); virtual ~TargetPhrases(); void AddTargetPhrase(const TargetPhrase &targetPhrase) @@ -44,7 +44,7 @@ public: void SortAndPrune(size_t tableLimit); - const TargetPhrases *Clone(MemPool &pool, const System &system) const; + //const TargetPhrases *Clone(MemPool &pool, const System &system) const; protected: Coll m_coll; size_t m_currInd; diff --git a/contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.cpp b/contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.cpp index 1e7496b56..381c51c65 100644 --- a/contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.cpp +++ b/contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.cpp @@ -128,7 +128,7 @@ void PhraseTableMemory::Load(System &system) PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[0]); //cerr << "created soure" << endl; - TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, system, toks[1]); + TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, *this, system, toks[1]); //cerr << "created target" << endl; target->GetScores().CreateFromString(toks[2], *this, system, true); //cerr << "created scores" << endl; diff --git a/contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp b/contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp index 4327b0419..e7323357d 100644 --- a/contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp +++ b/contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp @@ -82,7 +82,6 @@ void ProbingPT::Load(System &system) data = file.data(); size_t size = file.size(); - //std::cerr << "size=" << size << std::endl; // cache CreateCache(system); @@ -165,7 +164,6 @@ TargetPhrases *ProbingPT::CreateTargetPhrase( RecycleData &recycler) const { TargetPhrases *tps = NULL; - cerr << "sourcePhrase=" << sourcePhrase << endl; //Actual lookup std::pair query_result; // 1st=found, 2nd=target file offset @@ -175,8 +173,6 @@ TargetPhrases *ProbingPT::CreateTargetPhrase( const char *offset = data + query_result.second; uint64_t *numTP = (uint64_t*) offset; - //cerr << "query_result=" << query_result.second << " " << *numTP << endl; - tps = new (pool.Allocate()) TargetPhrases(pool, *numTP); offset += sizeof(uint64_t); @@ -190,8 +186,9 @@ TargetPhrases *ProbingPT::CreateTargetPhrase( } + tps->SortAndPrune(m_tableLimit); + system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); //cerr << *tps << endl; - } return tps; @@ -203,9 +200,7 @@ TargetPhrase *ProbingPT::CreateTargetPhrase( const char *&offset) const { TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; - //cerr << "tpInfo=" << tpInfo->numWords << endl; - - TargetPhrase *tp = new (pool.Allocate()) TargetPhrase(pool, system, tpInfo->numWords); + TargetPhrase *tp = new (pool.Allocate()) TargetPhrase(pool, *this, system, tpInfo->numWords); offset += sizeof(TargetPhraseInfo); @@ -214,26 +209,13 @@ TargetPhrase *ProbingPT::CreateTargetPhrase( size_t totalNumScores = m_engine->num_scores + m_engine->num_lex_scores; - cerr << "scores="; - for (size_t i = 0; i < totalNumScores; ++i) { - cerr << scores[i] << " "; - } - cerr << endl; - - //cerr << "HH A " << m_engine->num_scores << " " << m_engine->num_lex_scores << " " << totalNumScores << endl; if (m_engine->IsLogProb()) { // set pt score for rule tp->GetScores().PlusEquals(system, *this, scores); - //cerr << "HH B " << endl; // save scores for other FF, eg. lex RO. Just give the offset if (m_engine->num_lex_scores) { tp->scoreProperties = scores + m_engine->num_scores; - if (tp->scoreProperties == NULL) { - cerr << "BOOO!"; - abort(); - } - cerr << "HH C " << scores << " " << tp->scoreProperties << endl; } } else { @@ -256,10 +238,8 @@ TargetPhrase *ProbingPT::CreateTargetPhrase( offset += sizeof(SCORE) * totalNumScores; // words - //cerr << "HH D " << endl; for (size_t i = 0; i < tpInfo->numWords; ++i) { uint32_t *probingId = (uint32_t*) offset; - //cerr << "HH E " << *probingId << endl; const Factor *factor = GetTargetFactor(*probingId); assert(factor); @@ -324,7 +304,6 @@ void ProbingPT::CreateCache(System &system) TargetPhrases *tps = CreateTargetPhrase(pool, system, *sourcePhrase, retStruct.second, recycler); assert(tps); - //cerr << retStruct.second << " " << *sourcePhrase << endl; m_cache[retStruct.second] = tps; } diff --git a/contrib/other-builds/moses2/TranslationModel/UnknownWordPenalty.cpp b/contrib/other-builds/moses2/TranslationModel/UnknownWordPenalty.cpp index 01684e1ea..e8783a6b7 100644 --- a/contrib/other-builds/moses2/TranslationModel/UnknownWordPenalty.cpp +++ b/contrib/other-builds/moses2/TranslationModel/UnknownWordPenalty.cpp @@ -65,7 +65,7 @@ TargetPhrases *UnknownWordPenalty::Lookup(const Manager &mgr, MemPool &pool, Inp tps = new (pool.Allocate()) TargetPhrases(pool, 1); - TargetPhrase *target = new (pool.Allocate()) TargetPhrase(pool, system, 1); + TargetPhrase *target = new (pool.Allocate()) TargetPhrase(pool, *this, system, 1); Word &word = (*target)[0]; //FactorCollection &fc = system.vocab;