moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-30 15:34:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

create uncompressed target side

Browse Source
This commit is contained in:
Hieu Hoang 2016-01-21 11:22:55 +00:00
parent 22ce897bb6
commit b6b5642d23
12 changed files with 30 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
contrib/other-builds/moses2/.cproject
View File

@ -93,8 +93,11 @@
</tool>
</toolChain>
</folderInfo>
<fileInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041.1189716504" name="Main.cpp" rcbsApplicability="disable" resourcePath="Main.cpp" toolsToInvoke="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1686613508.1641877958">
<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1686613508.1641877958" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1686613508"/>
</fileInfo>
<sourceEntries>
<entry excluding="CreateProbingPT2.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
<entry excluding="Main.cpp|CreateProbingPT2.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
</sourceEntries>
</configuration>
</storageModule>

2
contrib/other-builds/moses2/FF/FeatureFunctions.h
View File

@ -29,6 +29,7 @@ class Scores;
class FeatureFunctions {
public:
std::vector<const PhraseTable*> m_phraseTables;
FeatureFunctions(System &system);
virtual ~FeatureFunctions();
@ -59,7 +60,6 @@ public:
protected:
std::vector<const FeatureFunction*> m_featureFunctions;
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;
std::vector<const PhraseTable*> m_phraseTables;
std::vector<const FeatureFunction*> m_withPhraseTableInd;
System &m_system;

11
contrib/other-builds/moses2/FF/LexicalReordering.cpp
View File

@ -7,6 +7,7 @@
#include <boost/foreach.hpp>
#include "LexicalReordering.h"
#include "../TranslationModel/PhraseTable.h"
#include "../System.h"
#include "../Search/Manager.h"
#include "../legacy/InputFileStream.h"
@ -162,13 +163,6 @@ void LexicalReordering::EvaluateAfterTablePruning(MemPool &pool,
{
if (m_propertyInd >= 0) {
SCORE *scoreArr = targetPhrase.GetScoresProperty(m_propertyInd);
/*
cerr << "scoreArr=" << scoreArr << " " << flush;
for (size_t i = 0; i < 6; ++i) {
cerr << scoreArr[i] << " ";
}
cerr << endl;
*/
targetPhrase.ffData[m_PhraseTableInd] = scoreArr;
}
else if (m_compactModel) {
@ -231,19 +225,16 @@ void LexicalReordering::EvaluateWhenApplied(const Manager &mgr,
// backwards
const TargetPhrase &target = hypo.GetTargetPhrase();
cerr << "HH D" << endl;
const SCORE *values = (const SCORE *) target.ffData[m_PhraseTableInd];
if (values) {
scores.PlusEquals(mgr.system, *this, values[orientation], orientation);
}
cerr << "HH D2" << endl;
// forwards
if (prevRange->GetStartPos() != NOT_FOUND) {
const TargetPhrase &prevTarget = *prevStateCast.targetPhrase;
const SCORE *prevValues = (const SCORE *) prevTarget.ffData[m_PhraseTableInd];
cerr << "HH E" << endl;
if (prevValues) {
scores.PlusEquals(mgr.system, *this, prevValues[orientation + 3], orientation + 3);
}

2
contrib/other-builds/moses2/Search/Hypothesis.cpp
View File

@ -214,9 +214,7 @@ void Hypothesis::EvaluateWhenApplied()
{
const std::vector<const StatefulFeatureFunction*> &sfffs = GetManager().system.featureFunctions.GetStatefulFeatureFunctions();
BOOST_FOREACH(const StatefulFeatureFunction *sfff, sfffs) {
cerr << "BEFORE " << sfff->GetName() << endl;
EvaluateWhenApplied(*sfff);
cerr << "AFTER " << sfff->GetName() << endl;
}
//cerr << *this << endl;
}

8
contrib/other-builds/moses2/Search/Manager.cpp
View File

@ -52,8 +52,8 @@ void Manager::Init()
m_hypoRecycle = &system.GetHypoRecycler();
m_bitmaps = new Bitmaps(GetPool());
m_initPhrase = new (GetPool().Allocate<TargetPhrase>()) TargetPhrase(GetPool(), system, 0);
const PhraseTable &firstPt = *system.featureFunctions.m_phraseTables[0];
m_initPhrase = new (GetPool().Allocate<TargetPhrase>()) TargetPhrase(GetPool(), firstPt, system, 0);
// create input phrase obj
FactorCollection &vocab = system.GetVocab();
@ -68,13 +68,9 @@ void Manager::Init()
pt.Lookup(*this, m_inputPaths);
}
//m_inputPaths.DeleteUnusedPaths();
std::cerr << "HH E" << std::endl;
CalcFutureScore();
std::cerr << "HH F" << std::endl;
m_bitmaps->Init(m_input->GetSize(), vector<bool>(0));
std::cerr << "HH G" << std::endl;
switch (system.searchAlgorithm) {
case Normal:

9
contrib/other-builds/moses2/TargetPhrase.cpp
View File

@ -17,21 +17,22 @@ using namespace std;
namespace Moses2
{
TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const System &system, const std::string &str)
TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
{
FactorCollection &vocab = system.GetVocab();
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, system, size);
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImpl::CreateFromString(vocab, system, toks);
return ret;
}
TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size)
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
:PhraseImpl(pool, size)
,scoreProperties(NULL)
,pt(pt)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
@ -39,6 +40,7 @@ TargetPhrase::TargetPhrase(MemPool &pool, const System &system, size_t size)
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
/*
TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhrase &copy)
:PhraseImpl(pool, copy)
,scoreProperties(NULL)
@ -50,6 +52,7 @@ TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhra
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
*/
TargetPhrase::~TargetPhrase() {
// TODO Auto-generated destructor stub

8
contrib/other-builds/moses2/TargetPhrase.h
View File

@ -17,6 +17,7 @@ namespace Moses2
class Scores;
class Manager;
class System;
class PhraseTable;
class TargetPhrase : public PhraseImpl
{
@ -24,10 +25,11 @@ class TargetPhrase : public PhraseImpl
public:
mutable void **ffData;
SCORE *scoreProperties;
const PhraseTable &pt;
static TargetPhrase *CreateFromString(MemPool &pool, const System &system, const std::string &str);
TargetPhrase(MemPool &pool, const System &system, size_t size);
TargetPhrase(MemPool &pool, const System &system, const TargetPhrase &copy);
static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
//TargetPhrase(MemPool &pool, const System &system, const TargetPhrase &copy);
virtual ~TargetPhrase();

5
contrib/other-builds/moses2/TargetPhrases.cpp
View File

@ -19,6 +19,7 @@ TargetPhrases::TargetPhrases(MemPool &pool, size_t reserve)
{
}
/*
TargetPhrases::TargetPhrases(MemPool &pool, const System &system, const TargetPhrases &copy)
:m_coll(pool, copy.m_coll.size())
{
@ -29,7 +30,7 @@ TargetPhrases::TargetPhrases(MemPool &pool, const System &system, const TargetPh
m_coll[i] = tpClone;
}
}
*/
TargetPhrases::~TargetPhrases() {
// TODO Auto-generated destructor stub
@ -61,11 +62,13 @@ void TargetPhrases::SortAndPrune(size_t tableLimit)
//cerr << "TargetPhrases=" << GetSize() << endl;
}
/*
const TargetPhrases *TargetPhrases::Clone(MemPool &pool, const System &system) const
{
const TargetPhrases *ret = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, system, *this);
return ret;
}
*/
}

4
contrib/other-builds/moses2/TargetPhrases.h
View File

@ -28,7 +28,7 @@ public:
}
TargetPhrases(MemPool &pool, size_t reserve);
TargetPhrases(MemPool &pool, const System &system, const TargetPhrases &copy);
//TargetPhrases(MemPool &pool, const System &system, const TargetPhrases &copy);
virtual ~TargetPhrases();
void AddTargetPhrase(const TargetPhrase &targetPhrase)
@ -44,7 +44,7 @@ public:
void SortAndPrune(size_t tableLimit);
const TargetPhrases *Clone(MemPool &pool, const System &system) const;
//const TargetPhrases *Clone(MemPool &pool, const System &system) const;
protected:
Coll m_coll;
size_t m_currInd;

2
contrib/other-builds/moses2/TranslationModel/PhraseTableMemory.cpp
View File

@ -128,7 +128,7 @@ void PhraseTableMemory::Load(System &system)
PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[0]);
//cerr << "created soure" << endl;
TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, system, toks[1]);
TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, *this, system, toks[1]);
//cerr << "created target" << endl;
target->GetScores().CreateFromString(toks[2], *this, system, true);
//cerr << "created scores" << endl;

27
contrib/other-builds/moses2/TranslationModel/ProbingPT.cpp
View File

@ -82,7 +82,6 @@ void ProbingPT::Load(System &system)
data = file.data();
size_t size = file.size();
//std::cerr << "size=" << size << std::endl;
// cache
CreateCache(system);
@ -165,7 +164,6 @@ TargetPhrases *ProbingPT::CreateTargetPhrase(
RecycleData &recycler) const
{
TargetPhrases *tps = NULL;
cerr << "sourcePhrase=" << sourcePhrase << endl;
//Actual lookup
std::pair<bool, uint64_t> query_result; // 1st=found, 2nd=target file offset
@ -175,8 +173,6 @@ TargetPhrases *ProbingPT::CreateTargetPhrase(
const char *offset = data + query_result.second;
uint64_t *numTP = (uint64_t*) offset;
//cerr << "query_result=" << query_result.second << " " << *numTP << endl;
tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, *numTP);
offset += sizeof(uint64_t);
@ -190,8 +186,9 @@ TargetPhrases *ProbingPT::CreateTargetPhrase(
}
tps->SortAndPrune(m_tableLimit);
system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, sourcePhrase);
//cerr << *tps << endl;
}
return tps;
@ -203,9 +200,7 @@ TargetPhrase *ProbingPT::CreateTargetPhrase(
const char *&offset) const
{
TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset;
//cerr << "tpInfo=" << tpInfo->numWords << endl;
TargetPhrase *tp = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, system, tpInfo->numWords);
TargetPhrase *tp = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, *this, system, tpInfo->numWords);
offset += sizeof(TargetPhraseInfo);
@ -214,26 +209,13 @@ TargetPhrase *ProbingPT::CreateTargetPhrase(
size_t totalNumScores = m_engine->num_scores + m_engine->num_lex_scores;
cerr << "scores=";
for (size_t i = 0; i < totalNumScores; ++i) {
cerr << scores[i] << " ";
}
cerr << endl;
//cerr << "HH A " << m_engine->num_scores << " " << m_engine->num_lex_scores << " " << totalNumScores << endl;
if (m_engine->IsLogProb()) {
// set pt score for rule
tp->GetScores().PlusEquals(system, *this, scores);
//cerr << "HH B " << endl;
// save scores for other FF, eg. lex RO. Just give the offset
if (m_engine->num_lex_scores) {
tp->scoreProperties = scores + m_engine->num_scores;
if (tp->scoreProperties == NULL) {
cerr << "BOOO!";
abort();
}
cerr << "HH C " << scores << " " << tp->scoreProperties << endl;
}
}
else {
@ -256,10 +238,8 @@ TargetPhrase *ProbingPT::CreateTargetPhrase(
offset += sizeof(SCORE) * totalNumScores;
// words
//cerr << "HH D " << endl;
for (size_t i = 0; i < tpInfo->numWords; ++i) {
uint32_t *probingId = (uint32_t*) offset;
//cerr << "HH E " << *probingId << endl;
const Factor *factor = GetTargetFactor(*probingId);
assert(factor);
@ -324,7 +304,6 @@ void ProbingPT::CreateCache(System &system)
TargetPhrases *tps = CreateTargetPhrase(pool, system, *sourcePhrase, retStruct.second, recycler);
assert(tps);
//cerr << retStruct.second << " " << *sourcePhrase << endl;
m_cache[retStruct.second] = tps;
}

2
contrib/other-builds/moses2/TranslationModel/UnknownWordPenalty.cpp
View File

@ -65,7 +65,7 @@ TargetPhrases *UnknownWordPenalty::Lookup(const Manager &mgr, MemPool &pool, Inp
tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, 1);
TargetPhrase *target = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, system, 1);
TargetPhrase *target = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, *this, system, 1);
Word &word = (*target)[0];
//FactorCollection &fc = system.vocab;