moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-30 15:34:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

subphrase can make a subphrase out of any phrase class

Browse Source
This commit is contained in:
Hieu Hoang 2016-02-22 18:32:59 +00:00
parent f0563f6a21
commit b4a579a901
12 changed files with 36 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
contrib/other-builds/moses2/PhraseImpl.h
View File

@ -9,47 +9,36 @@ namespace Moses2
{
class SubPhrase;
class PhraseImpl : public Phrase
template<typename WORD>
class PhraseImplTemplate : public Phrase
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret->CreateFromString(vocab, system, toks);
return ret;
}
PhraseImpl(MemPool &pool, size_t size)
PhraseImplTemplate(MemPool &pool, size_t size)
:m_size(size)
{
m_words = new (pool.Allocate<Word>(size)) Word[size];
m_words = new (pool.Allocate<WORD>(size)) WORD[size];
}
PhraseImpl(MemPool &pool, const PhraseImpl &copy)
PhraseImplTemplate(MemPool &pool, const PhraseImplTemplate &copy)
:m_size(copy.GetSize())
{
m_words = new (pool.Allocate<Word>(m_size)) Word[m_size];
m_words = new (pool.Allocate<WORD>(m_size)) WORD[m_size];
for (size_t i = 0; i < m_size; ++i) {
const Word &word = copy[i];
const WORD &word = copy[i];
(*this)[i] = word;
}
}
virtual ~PhraseImpl()
virtual ~PhraseImplTemplate()
{}
const Word& operator[](size_t pos) const {
const WORD& operator[](size_t pos) const {
return m_words[pos];
}
Word& operator[](size_t pos) {
WORD& operator[](size_t pos) {
return m_words[pos];
}
@ -62,25 +51,37 @@ public:
return ret;
}
void Prefetch() const
{
for (size_t i = 0; i < m_size; ++i) {
const Word &word = m_words[i];
const Factor *factor = word[0];
__builtin_prefetch(factor);
}
}
protected:
size_t m_size;
Word *m_words;
WORD *m_words;
void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks)
virtual void CreateFromString(FactorCollection &vocab, const System &system, const std::vector<std::string> &toks)
{
for (size_t i = 0; i < m_size; ++i) {
Word &word = (*this)[i];
WORD &word = (*this)[i];
word.CreateFromString(vocab, system, toks[i]);
}
}
};
class PhraseImpl : public PhraseImplTemplate<Word>
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
{
std::vector<std::string> toks = Moses2::Tokenize(str);
size_t size = toks.size();
PhraseImpl *ret;
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
}
PhraseImpl(MemPool &pool, size_t size)
:PhraseImplTemplate(pool, size)
{}
};

30
contrib/other-builds/moses2/Search/CubePruningBitmapStack/Misc.cpp
View File

@ -154,36 +154,6 @@ void CubeEdge::CreateNext(Manager &mgr,
}
}
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
}
}

2
contrib/other-builds/moses2/Search/CubePruningBitmapStack/Misc.h
View File

@ -100,8 +100,6 @@ public:
SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected:
};

30
contrib/other-builds/moses2/Search/CubePruningCardinalStack/Misc.cpp
View File

@ -154,36 +154,6 @@ void CubeEdge::CreateNext(Manager &mgr,
}
}
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
}
}

2
contrib/other-builds/moses2/Search/CubePruningCardinalStack/Misc.h
View File

@ -101,8 +101,6 @@ public:
SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected:
};

30
contrib/other-builds/moses2/Search/CubePruningMiniStack/Misc.cpp
View File

@ -157,36 +157,6 @@ void CubeEdge::CreateNext(Manager &mgr,
}
}
void CubeEdge::Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions)
{
size_t hypoIndex = item->hypoIndex + 1;
if (hypoIndex < hypos.size() && !SetSeenPosition(hypoIndex, item->tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[item->tpIndex];
tp.Prefetch();
}
size_t tpIndex = item->tpIndex + 1;
if (tpIndex < tps.GetSize() && !SetSeenPosition(item->hypoIndex, tpIndex, seenPositions)) {
const Hypothesis *hypo = hypos[item->hypoIndex];
__builtin_prefetch(hypo);
const TargetPhrase &hypoTP = hypo->GetTargetPhrase();
hypoTP.Prefetch();
const TargetPhrase &tp = tps[tpIndex];
tp.Prefetch();
}
}
}
}

2
contrib/other-builds/moses2/Search/CubePruningMiniStack/Misc.h
View File

@ -100,8 +100,6 @@ public:
SeenPositions &seenPositions,
std::deque<QueueItem*> &queueItemRecycler);
void Prefetch(Manager &mgr, const QueueItem *item, Queue &queue, SeenPositions &seenPositions);
protected:
};

2
contrib/other-builds/moses2/Sentence.cpp
View File

@ -39,7 +39,7 @@ Sentence *Sentence::CreateFromString(MemPool &pool,
ret = new (pool.Allocate<Sentence>()) Sentence(translationId, pool, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;

1
contrib/other-builds/moses2/SubPhrase.cpp
View File

@ -5,7 +5,6 @@
* Author: hieu
*/
#include "SubPhrase.h"
#include "PhraseImpl.h"
using namespace std;

1
contrib/other-builds/moses2/SubPhrase.h
View File

@ -3,7 +3,6 @@
namespace Moses2
{
class PhraseImpl;
class SubPhrase : public Phrase
{

2
contrib/other-builds/moses2/Syntax/TargetPhrase.cpp
View File

@ -23,7 +23,7 @@ TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &p
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks);
//ret->PhraseImpl::CreateFromString(vocab, system, toks);
return ret;
}

2
contrib/other-builds/moses2/TargetPhrase.cpp
View File

@ -24,7 +24,7 @@ TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &p
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
}