moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-02 17:09:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

restart hiero

Browse Source
This commit is contained in:
Hieu Hoang 2016-02-25 16:21:11 +00:00
parent e464c4cf39
commit 2a2514026e
7 changed files with 106 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/other-builds/moses2/Jamfile
View File

@ -95,8 +95,6 @@ alias deps : ../../..//z ../../..//boost_iostreams ../../..//boost_filesystem .
SCFG/TargetPhrase.cpp
SCFG/Word.cpp
SCFG/TranslationModel/PhraseTableMemory.cpp
deps ;
exe moses2 : Main.cpp moses2_lib ;

4
contrib/other-builds/moses2/SCFG/PhraseImpl.cpp
View File

@ -10,7 +10,9 @@ using namespace std;
namespace Moses2
{
namespace SCFG
{
}
}

22
contrib/other-builds/moses2/SCFG/PhraseImpl.h
View File

@ -1,12 +1,13 @@
#pragma once
#include "../PhraseImplTemplate.h"
#include "../SubPhrase.h"
namespace Moses2
{
namespace SCFG
{
class PhraseImpl : public PhraseImplTemplate<Word>
class PhraseImpl : public Phrase, public PhraseImplTemplate<Word>
{
public:
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab, const System &system, const std::string &str)
@ -25,9 +26,24 @@ public:
:PhraseImplTemplate(pool, size)
{}
const Word& operator[](size_t pos) const
{ return m_words[pos]; }
Word& operator[](size_t pos) {
return m_words[pos];
}
size_t GetSize() const
{ return m_size; }
SubPhrase GetSubPhrase(size_t start, size_t end) const
{
SubPhrase ret(*this, start, end);
return ret;
}
};
}
}
}

47
contrib/other-builds/moses2/SCFG/TargetPhrase.cpp
View File

@ -5,12 +5,12 @@
* Author: hieu
*/
#include <stdlib.h>
#include "TargetPhrase.h"
#include "../legacy/FactorCollection.h"
#include "../legacy/Util2.h"
#include "../System.h"
#include "../Scores.h"
#include "../System.h"
#include "../MemPool.h"
#include "../Search/Manager.h"
using namespace std;
@ -18,14 +18,23 @@ namespace Moses2
{
namespace SCFG
{
TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
{
FactorCollection &vocab = system.GetVocab();
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
return ret;
}
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
:PhraseImpl(pool, size)
:TPBase(pool, pt, system)
,PhraseImplTemplate(pool, size)
,scoreProperties(NULL)
,pt(pt)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
@ -33,6 +42,34 @@ TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &s
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
/*
TargetPhrase::TargetPhrase(MemPool &pool, const System &system, const TargetPhrase &copy)
:PhraseImpl(pool, copy)
,scoreProperties(NULL)
{
// scores
m_estimatedScore = copy.m_estimatedScore;
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores(), copy.GetScores());
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
*/
TargetPhrase::~TargetPhrase() {
// TODO Auto-generated destructor stub
}
std::ostream& operator<<(std::ostream &out, const TargetPhrase &obj)
{
out << (const Phrase&) obj << " SCORES:" << obj.GetScores();
return out;
}
SCORE *TargetPhrase::GetScoresProperty(int propertyInd) const
{
return scoreProperties ? scoreProperties + propertyInd : NULL;
}
}
}

55
contrib/other-builds/moses2/SCFG/TargetPhrase.h
View File

@ -7,7 +7,12 @@
#pragma once
#include "PhraseImpl.h"
#include <iostream>
#include "../Phrase.h"
#include "../PhraseImplTemplate.h"
#include "../MemPool.h"
#include "../Word.h"
#include "../SubPhrase.h"
namespace Moses2
{
@ -19,28 +24,52 @@ class PhraseTable;
namespace SCFG
{
class TargetPhrase : public PhraseImpl
class TargetPhrase : public TPBase, public PhraseImplTemplate<Word>
{
friend std::ostream& operator<<(std::ostream &, const TargetPhrase &);
public:
mutable void **ffData;
SCORE *scoreProperties;
const PhraseTable &pt;
mutable void **ffData;
SCORE *scoreProperties;
static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
//TargetPhrase(MemPool &pool, const System &system, const TargetPhrase &copy);
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
virtual ~TargetPhrase();
Scores &GetScores()
{ return *m_scores; }
const Word& operator[](size_t pos) const
{ return m_words[pos]; }
const Scores &GetScores() const
{ return *m_scores; }
Word& operator[](size_t pos)
{ return m_words[pos]; }
size_t GetSize() const
{ return m_size; }
SubPhrase GetSubPhrase(size_t start, size_t end) const
{
SubPhrase ret(*this, start, end);
return ret;
}
SCORE *GetScoresProperty(int propertyInd) const;
//mutable void *chartState;
protected:
Scores *m_scores;
Word m_lhs;
};
//////////////////////////////////////////
struct CompareFutureScore {
bool operator() (const TargetPhrase *a, const TargetPhrase *b) const
{
return a->GetFutureScore() > b->GetFutureScore();
}
bool operator() (const TargetPhrase &a, const TargetPhrase &b) const
{
return a.GetFutureScore() > b.GetFutureScore();
}
};
}

63
contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp
View File

@ -1,63 +0,0 @@
/*
* PhraseTableMemory.cpp
*
* Created on: 28 Oct 2015
* Author: hieu
*/
#include "PhraseTableMemory.h"
#include "../../legacy/FactorCollection.h"
#include "../../legacy/InputFileStream.h"
#include "../../System.h"
#include "../../MemPool.h"
#include "../../Scores.h"
#include "../TargetPhrase.h"
using namespace std;
namespace Moses2
{
namespace SCFG
{
void PhraseTableMemory::Load(System &system)
{
FactorCollection &vocab = system.GetVocab();
MemPool &systemPool = system.GetSystemPool();
MemPool tmpSourcePool;
vector<string> toks;
size_t lineNum = 0;
InputFileStream strme(m_path);
string line;
while (getline(strme, line)) {
if (++lineNum % 1000000 == 0) {
cerr << lineNum << " ";
}
toks.clear();
TokenizeMultiCharSeparator(toks, line, "|||");
assert(toks.size() >= 3);
//cerr << "line=" << line << endl;
PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[0]);
//cerr << "created soure" << endl;
TargetPhrase *target = TargetPhrase::CreateFromString(systemPool, *this, system, toks[1]);
//cerr << "created target" << endl;
target->GetScores().CreateFromString(toks[2], *this, system, true);
//cerr << "created scores" << endl;
// properties
if (toks.size() == 7) {
//target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
//strcpy(target->properties, toks[6].c_str());
}
//system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, *target);
//m_root.AddRule(*source, target);
}
m_root.SortAndPrune(m_tableLimit, systemPool, system);
}
}
}

26
contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h
View File

@ -1,26 +0,0 @@
/*
* PhraseTableMemory.h
*
* Created on: 28 Oct 2015
* Author: hieu
*/
#pragma once
#include "../../TranslationModel/PhraseTableMemory.h"
namespace Moses2
{
namespace SCFG
{
class PhraseTableMemory : public Moses2::PhraseTableMemory
{
public:
virtual void Load(System &system);
};
}
}