moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-05 02:22:21 +03:00
Code Issues Packages Projects Releases Wiki Activity

subphrase can make a subphrase out of any phrase class

Browse Source
This commit is contained in:
Hieu Hoang 2016-02-22 18:32:59 +00:00
parent f0563f6a21
commit b4a579a901
12 changed files with 36 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
contrib/other-builds/moses2/PhraseImpl.h
View File

@ -9,47 +9,36 @@ namespace Moses2
{ {
class SubPhrase; class SubPhrase;
class PhraseImpl : public Phrase template<typename WORD>
class PhraseImplTemplate : public Phrase
{ {
public: public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str) PhraseImplTemplate(MemPool &pool, size_t size)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret->CreateFromString(vocab, system, toks);
return ret;
}
PhraseImpl(MemPool &pool, size_t size)
:m_size(size) :m_size(size)
{ {
m_words = new (pool.Allocate<Word>(size)) Word[size]; m_words = new (pool.Allocate<WORD>(size)) WORD[size];
} }
PhraseImpl(MemPool &pool, const PhraseImpl &copy) PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate &copy)
:m_size(copy.GetSize()) :m_size(copy.GetSize())
{ {
m_words = new (pool.Allocate<Word>(m_size)) Word[m_size]; m_words = new (pool.Allocate<WORD>(m_size)) WORD[m_size];
for (size_t i = 0; i < m_size; ++i) { for (size_t i = 0; i < m_size; ++i) {
const Word &word = copy[i]; const WORD &word = copy[i];
(*this)[i] = word; (*this)[i] = word;
} }
} }
virtual ~PhraseImpl() virtual ~PhraseImplTemplate()
{} {}
const Word& operator[](size_t pos) const { const WORD& operator[](size_t pos) const {
return m_words[pos]; return m_words[pos];
} }
Word& operator[](size_t pos) { WORD& operator[](size_t pos) {
return m_words[pos]; return m_words[pos];
} }
@ -62,25 +51,37 @@ public:
return ret; return ret;
} }
void Prefetch() const
{
for (size_t i = 0; i < m_size; ++i) {
const Word &word = m_words[i];
const Factor *factor = word[0];
__builtin_prefetch(factor);
}
}
protected: protected:
size_t m_size; size_t m_size;
Word *m_words; WORD *m_words;
void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks) virtual void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks)
{ {
for (size_t i = 0; i < m_size; ++i) { for (size_t i = 0; i < m_size; ++i) {
Word &word = (*this)[i]; WORD &word = (*this)[i];
word.CreateFromString(vocab, system, toks[i]); word.CreateFromString(vocab, system, toks[i]);
} }
} }
};
class PhraseImpl : public PhraseImplTemplate<Word>
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
}
PhraseImpl(MemPool &pool, size_t size)
:PhraseImplTemplate(pool, size)
{}
}; };

30
contrib/other-builds/moses2/Search/CubePruningBitmapStack/Misc.cpp
View File

@ -154,36 +154,6 @@ void CubeEdge::CreateNext(Manager &mgr,
} }
} }
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
} }
} }

2
contrib/other-builds/moses2/Search/CubePruningBitmapStack/Misc.h
View File

@ -100,8 +100,6 @@ public:
SeenPositions &seenPositions, SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler); std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected: protected:
}; };

30
contrib/other-builds/moses2/Search/CubePruningCardinalStack/Misc.cpp
View File

@ -154,36 +154,6 @@ void CubeEdge::CreateNext(Manager &mgr,
} }
} }
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
} }
} }

2
contrib/other-builds/moses2/Search/CubePruningCardinalStack/Misc.h
View File

@ -101,8 +101,6 @@ public:
SeenPositions &seenPositions, SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler); std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected: protected:
}; };

30
contrib/other-builds/moses2/Search/CubePruningMiniStack/Misc.cpp
View File

@ -157,36 +157,6 @@ void CubeEdge::CreateNext(Manager &mgr,
} }
} }
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
} }
} }

2
contrib/other-builds/moses2/Search/CubePruningMiniStack/Misc.h
View File

@ -100,8 +100,6 @@ public:
SeenPositions &seenPositions, SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler); std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected: protected:
}; };

2
contrib/other-builds/moses2/Sentence.cpp
View File

@ -39,7 +39,7 @@ Sentence *Sentence::CreateFromString(MemPool &pool,
ret = new (pool.Allocate<Sentence>()) Sentence(translationId, pool, size); ret = new (pool.Allocate<Sentence>()) Sentence(translationId, pool, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks); ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret; return ret;

1
contrib/other-builds/moses2/SubPhrase.cpp
View File

@ -5,7 +5,6 @@
* Author: hieu * Author: hieu
*/ */
#include "SubPhrase.h" #include "SubPhrase.h"
#include "PhraseImpl.h"
using namespace std; using namespace std;

1
contrib/other-builds/moses2/SubPhrase.h
View File

@ -3,7 +3,6 @@
namespace Moses2 namespace Moses2
{ {
class PhraseImpl;
class SubPhrase : public Phrase class SubPhrase : public Phrase
{ {

2
contrib/other-builds/moses2/Syntax/TargetPhrase.cpp
View File

@ -23,7 +23,7 @@ TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &p
vector<string> toks = Tokenize(str); vector<string> toks = Tokenize(str);
size_t size = toks.size(); size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size); TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks); //ret->PhraseImpl::CreateFromString(vocab, system, toks);
return ret; return ret;
} }

2
contrib/other-builds/moses2/TargetPhrase.cpp
View File

@ -24,7 +24,7 @@ TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &p
vector<string> toks = Tokenize(str); vector<string> toks = Tokenize(str);
size_t size = toks.size(); size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size); TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks); ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret; return ret;
} }