From 2a2514026e1e14eb9ac7ed4050f8f77c632bb6a0 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Thu, 25 Feb 2016 16:21:11 +0000 Subject: [PATCH] restart hiero --- contrib/other-builds/moses2/Jamfile | 2 - .../other-builds/moses2/SCFG/PhraseImpl.cpp | 4 +- contrib/other-builds/moses2/SCFG/PhraseImpl.h | 22 ++++++- .../other-builds/moses2/SCFG/TargetPhrase.cpp | 47 ++++++++++++-- .../other-builds/moses2/SCFG/TargetPhrase.h | 55 ++++++++++++---- .../TranslationModel/PhraseTableMemory.cpp | 63 ------------------- .../SCFG/TranslationModel/PhraseTableMemory.h | 26 -------- 7 files changed, 106 insertions(+), 113 deletions(-) delete mode 100644 contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp delete mode 100644 contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h diff --git a/contrib/other-builds/moses2/Jamfile b/contrib/other-builds/moses2/Jamfile index 48ddea832..70ebda236 100644 --- a/contrib/other-builds/moses2/Jamfile +++ b/contrib/other-builds/moses2/Jamfile @@ -95,8 +95,6 @@ alias deps : ../../..//z ../../..//boost_iostreams ../../..//boost_filesystem . SCFG/TargetPhrase.cpp SCFG/Word.cpp - SCFG/TranslationModel/PhraseTableMemory.cpp - deps ; exe moses2 : Main.cpp moses2_lib ; diff --git a/contrib/other-builds/moses2/SCFG/PhraseImpl.cpp b/contrib/other-builds/moses2/SCFG/PhraseImpl.cpp index 39e488bd1..c41046ec2 100644 --- a/contrib/other-builds/moses2/SCFG/PhraseImpl.cpp +++ b/contrib/other-builds/moses2/SCFG/PhraseImpl.cpp @@ -10,7 +10,9 @@ using namespace std; namespace Moses2 { - +namespace SCFG +{ } +} diff --git a/contrib/other-builds/moses2/SCFG/PhraseImpl.h b/contrib/other-builds/moses2/SCFG/PhraseImpl.h index a58c3d1e9..4eacde45f 100644 --- a/contrib/other-builds/moses2/SCFG/PhraseImpl.h +++ b/contrib/other-builds/moses2/SCFG/PhraseImpl.h @@ -1,12 +1,13 @@ #pragma once #include "../PhraseImplTemplate.h" +#include "../SubPhrase.h" namespace Moses2 { namespace SCFG { -class PhraseImpl : public PhraseImplTemplate +class PhraseImpl : public Phrase, public PhraseImplTemplate { public: static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str) @@ -25,9 +26,24 @@ public: :PhraseImplTemplate(pool, size) {} + const Word& operator[](size_t pos) const + { return m_words[pos]; } + + Word& operator[](size_t pos) { + return m_words[pos]; + } + + size_t GetSize() const + { return m_size; } + + SubPhrase GetSubPhrase(size_t start, size_t end) const + { + SubPhrase ret(*this, start, end); + return ret; + } + }; - +} } -} diff --git a/contrib/other-builds/moses2/SCFG/TargetPhrase.cpp b/contrib/other-builds/moses2/SCFG/TargetPhrase.cpp index 363ba37b1..104a9c73a 100644 --- a/contrib/other-builds/moses2/SCFG/TargetPhrase.cpp +++ b/contrib/other-builds/moses2/SCFG/TargetPhrase.cpp @@ -5,12 +5,12 @@ * Author: hieu */ +#include #include "TargetPhrase.h" -#include "../legacy/FactorCollection.h" -#include "../legacy/Util2.h" -#include "../System.h" #include "../Scores.h" +#include "../System.h" #include "../MemPool.h" +#include "../Search/Manager.h" using namespace std; @@ -18,14 +18,23 @@ namespace Moses2 { namespace SCFG { + TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str) { + FactorCollection &vocab = system.GetVocab(); + + vector toks = Tokenize(str); + size_t size = toks.size(); + TargetPhrase *ret = new (pool.Allocate()) TargetPhrase(pool, pt, system, size); + ret->PhraseImplTemplate::CreateFromString(vocab, system, toks); + + return ret; } TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size) -:PhraseImpl(pool, size) +:TPBase(pool, pt, system) +,PhraseImplTemplate(pool, size) ,scoreProperties(NULL) -,pt(pt) { m_scores = new (pool.Allocate()) Scores(system, pool, system.featureFunctions.GetNumScores()); @@ -33,6 +42,34 @@ TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &s ffData = new (pool.Allocate(numWithPtData)) void *[numWithPtData]; } +/* +TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©) +:PhraseImpl(pool, copy) +,scoreProperties(NULL) +{ + // scores + m_estimatedScore = copy.m_estimatedScore; + m_scores = new (pool.Allocate()) Scores(system, pool, system.featureFunctions.GetNumScores(), copy.GetScores()); + + size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size(); + ffData = new (pool.Allocate(numWithPtData)) void *[numWithPtData]; +} +*/ + +TargetPhrase::~TargetPhrase() { + // TODO Auto-generated destructor stub +} + +std::ostream& operator<<(std::ostream &out, const TargetPhrase &obj) +{ + out << (const Phrase&) obj << " SCORES:" << obj.GetScores(); + return out; +} + +SCORE *TargetPhrase::GetScoresProperty(int propertyInd) const +{ + return scoreProperties ? scoreProperties + propertyInd : NULL; +} } } diff --git a/contrib/other-builds/moses2/SCFG/TargetPhrase.h b/contrib/other-builds/moses2/SCFG/TargetPhrase.h index 4581f69a1..caed415a4 100644 --- a/contrib/other-builds/moses2/SCFG/TargetPhrase.h +++ b/contrib/other-builds/moses2/SCFG/TargetPhrase.h @@ -7,7 +7,12 @@ #pragma once -#include "PhraseImpl.h" +#include +#include "../Phrase.h" +#include "../PhraseImplTemplate.h" +#include "../MemPool.h" +#include "../Word.h" +#include "../SubPhrase.h" namespace Moses2 { @@ -19,28 +24,52 @@ class PhraseTable; namespace SCFG { -class TargetPhrase : public PhraseImpl +class TargetPhrase : public TPBase, public PhraseImplTemplate { friend std::ostream& operator<<(std::ostream &, const TargetPhrase &); public: - mutable void **ffData; - SCORE *scoreProperties; - const PhraseTable &pt; + mutable void **ffData; + SCORE *scoreProperties; - static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str); + static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str); + TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size); + //TargetPhrase(MemPool &pool, const System &system, const TargetPhrase ©); - TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size); + virtual ~TargetPhrase(); - Scores &GetScores() - { return *m_scores; } + const Word& operator[](size_t pos) const + { return m_words[pos]; } - const Scores &GetScores() const - { return *m_scores; } + Word& operator[](size_t pos) + { return m_words[pos]; } + + size_t GetSize() const + { return m_size; } + + SubPhrase GetSubPhrase(size_t start, size_t end) const + { + SubPhrase ret(*this, start, end); + return ret; + } + + SCORE *GetScoresProperty(int propertyInd) const; + + //mutable void *chartState; protected: - Scores *m_scores; - Word m_lhs; +}; +////////////////////////////////////////// +struct CompareFutureScore { + bool operator() (const TargetPhrase *a, const TargetPhrase *b) const + { + return a->GetFutureScore() > b->GetFutureScore(); + } + + bool operator() (const TargetPhrase &a, const TargetPhrase &b) const + { + return a.GetFutureScore() > b.GetFutureScore(); + } }; } diff --git a/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp b/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp deleted file mode 100644 index edb31f369..000000000 --- a/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp +++ /dev/null @@ -1,63 +0,0 @@ -/* - * PhraseTableMemory.cpp - * - * Created on: 28 Oct 2015 - * Author: hieu - */ - -#include "PhraseTableMemory.h" -#include "../../legacy/FactorCollection.h" -#include "../../legacy/InputFileStream.h" -#include "../../System.h" -#include "../../MemPool.h" -#include "../../Scores.h" -#include "../TargetPhrase.h" - -using namespace std; - -namespace Moses2 -{ -namespace SCFG -{ - -void PhraseTableMemory::Load(System &system) -{ - FactorCollection &vocab = system.GetVocab(); - - MemPool &systemPool = system.GetSystemPool(); - MemPool tmpSourcePool; - vector toks; - size_t lineNum = 0; - InputFileStream strme(m_path); - string line; - while (getline(strme, line)) { - if (++lineNum % 1000000 == 0) { - cerr << lineNum << " "; - } - toks.clear(); - TokenizeMultiCharSeparator(toks, line, "|||"); - assert(toks.size() >= 3); - //cerr << "line=" << line << endl; - - PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[0]); - //cerr << "created soure" << endl; - TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, *this, system, toks[1]); - //cerr << "created target" << endl; - target->GetScores().CreateFromString(toks[2], *this, system, true); - //cerr << "created scores" << endl; - - // properties - if (toks.size() == 7) { - //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); - //strcpy(target->properties, toks[6].c_str()); - } - - //system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, *target); - //m_root.AddRule(*source, target); - } - m_root.SortAndPrune(m_tableLimit, systemPool, system); -} - -} -} - diff --git a/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h b/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h deleted file mode 100644 index 5d698109d..000000000 --- a/contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * PhraseTableMemory.h - * - * Created on: 28 Oct 2015 - * Author: hieu - */ -#pragma once - -#include "../../TranslationModel/PhraseTableMemory.h" - -namespace Moses2 -{ -namespace SCFG -{ - -class PhraseTableMemory : public Moses2::PhraseTableMemory -{ -public: - virtual void Load(System &system); - -}; - -} -} - -