moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2025-01-02 17:09:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

phrase-table for scfg

Browse Source
This commit is contained in:
Hieu Hoang 2016-02-22 19:32:03 +00:00
parent 8c86e1a783
commit d9f5585c4b
11 changed files with 102 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
contrib/other-builds/moses2/FF/FeatureFunction.h
View File

@ -24,6 +24,11 @@ class Scores;
class Manager;
class MemPool;
namespace SCFG
{
class TargetPhrase;
}
class FeatureFunction {
public:
@ -53,6 +58,12 @@ public:
Scores &scores,
SCORE *estimatedScore) const = 0;
virtual void
EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, const SCFG::TargetPhrase &targetPhrase,
Scores &scores,
SCORE *estimatedScore) const {}
virtual void
EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const
{}

15
contrib/other-builds/moses2/FF/FeatureFunctions.cpp
View File

@ -24,6 +24,7 @@
#include "../LM/LanguageModel.h"
//#include "../LM/LanguageModelDALM.h"
#include "../LM/KENLM.h"
#include "../SCFG/TargetPhrase.h"
#include "util/exception.hh"
using namespace std;
@ -196,5 +197,19 @@ void FeatureFunctions::EvaluateAfterTablePruning(MemPool &pool, const TargetPhra
}
void
FeatureFunctions::EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const
{
SCORE estimatedScore = 0;
BOOST_FOREACH(const FeatureFunction *ff, m_featureFunctions) {
Scores& scores = targetPhrase.GetScores();
ff->EvaluateInIsolation(pool, system, source, targetPhrase, scores, &estimatedScore);
}
//targetPhrase.SetEstimatedScore(estimatedScore);
}
}

10
contrib/other-builds/moses2/FF/FeatureFunctions.h
View File

@ -27,6 +27,11 @@ class TargetPhrase;
class TargetPhrases;
class Scores;
namespace SCFG
{
class TargetPhrase;
}
class FeatureFunctions {
public:
std::vector<const PhraseTable*> m_phraseTables;
@ -52,11 +57,14 @@ public:
const FeatureFunction *FindFeatureFunction(const std::string &name) const;
const PhraseTable *GetPhraseTablesExcludeUnknownWordPenalty(size_t ptInd);
// the pool here must be the system pool if the rule was loaded during load, or the mgr if it was loaded on demand
// the pool here must be the system pool if the rule was loaded during load, or the mgr pool if it was loaded on demand
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, TargetPhrase &targetPhrase) const;
void EvaluateAfterTablePruning(MemPool &pool, const TargetPhrases &tps, const Phrase &sourcePhrase) const;
void EvaluateInIsolation(MemPool &pool, const System &system,
const Phrase &source, SCFG::TargetPhrase &targetPhrase) const;
protected:
std::vector<const FeatureFunction*> m_featureFunctions;
std::vector<const StatefulFeatureFunction*> m_statefulFeatureFunctions;

7
contrib/other-builds/moses2/Jamfile
View File

@ -91,10 +91,11 @@ alias deps : ../../..//z ../../..//boost_iostreams ../../..//boost_filesystem .
legacy/CompactPT/MurmurHash3.cpp
legacy/CompactPT/ThrowingFwrite.cpp
Syntax/TargetPhrase.cpp
Syntax/Word.cpp
SCFG/PhraseImpl.cpp
SCFG/TargetPhrase.cpp
SCFG/Word.cpp
Syntax/TranslationModel/PhraseTableMemory.cpp
SCFG/TranslationModel/PhraseTableMemory.cpp
deps ;

38
contrib/other-builds/moses2/SCFG/TargetPhrase.cpp Normal file
View File

@ -0,0 +1,38 @@
/*
* TargetPhrase.cpp
*
* Created on: 23 Oct 2015
* Author: hieu
*/
#include "TargetPhrase.h"
#include "../legacy/FactorCollection.h"
#include "../legacy/Util2.h"
#include "../System.h"
#include "../Scores.h"
#include "../MemPool.h"
using namespace std;
namespace Moses2
{
namespace SCFG
{
TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
{
}
TargetPhrase::TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
:PhraseImpl(pool, size)
,scoreProperties(NULL)
,pt(pt)
{
m_scores = new (pool.Allocate<Scores>()) Scores(system, pool, system.featureFunctions.GetNumScores());
size_t numWithPtData = system.featureFunctions.GetWithPhraseTableInd().size();
ffData = new (pool.Allocate<void *>(numWithPtData)) void *[numWithPtData];
}
}
}

26
contrib/other-builds/moses2/Syntax/TargetPhrase.h → contrib/other-builds/moses2/SCFG/TargetPhrase.h
View File

@ -7,24 +7,38 @@
#pragma once
#include "../TargetPhrase.h"
#include "PhraseImpl.h"
namespace Moses2
{
namespace Syntax
class Scores;
class Manager;
class System;
class PhraseTable;
namespace SCFG
{
class TargetPhrase : public Moses2::TargetPhrase
class TargetPhrase : public PhraseImpl
{
friend std::ostream& operator<<(std::ostream &, const TargetPhrase &);
public:
mutable void **ffData;
SCORE *scoreProperties;
const PhraseTable &pt;
static TargetPhrase *CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str);
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size)
:Moses2::TargetPhrase(pool, pt, system, size)
{}
TargetPhrase(MemPool &pool, const PhraseTable &pt, const System &system, size_t size);
Scores &GetScores()
{ return *m_scores; }
const Scores &GetScores() const
{ return *m_scores; }
protected:
Scores *m_scores;
Word m_lhs;
};

4
contrib/other-builds/moses2/Syntax/TranslationModel/PhraseTableMemory.cpp → contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.cpp
View File

@ -17,7 +17,7 @@ using namespace std;
namespace Moses2
{
namespace Syntax
namespace SCFG
{
void PhraseTableMemory::Load(System &system)
@ -53,7 +53,7 @@ void PhraseTableMemory::Load(System &system)
}
system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, *target);
m_root.AddRule(*source, target);
//m_root.AddRule(*source, target);
}
m_root.SortAndPrune(m_tableLimit, systemPool, system);
}

2
contrib/other-builds/moses2/Syntax/TranslationModel/PhraseTableMemory.h → contrib/other-builds/moses2/SCFG/TranslationModel/PhraseTableMemory.h
View File

@ -10,7 +10,7 @@
namespace Moses2
{
namespace Syntax
namespace SCFG
{
class PhraseTableMemory : public Moses2::PhraseTableMemory

2
contrib/other-builds/moses2/Syntax/Word.cpp → contrib/other-builds/moses2/SCFG/Word.cpp
View File

@ -11,7 +11,7 @@ using namespace std;
namespace Moses2
{
namespace Syntax
namespace SCFG
{
}

2
contrib/other-builds/moses2/Syntax/Word.h → contrib/other-builds/moses2/SCFG/Word.h
View File

@ -11,7 +11,7 @@
namespace Moses2
{
namespace Syntax
namespace SCFG
{
class Word : public Moses2::Word

33
contrib/other-builds/moses2/Syntax/TargetPhrase.cpp
View File

@ -1,33 +0,0 @@
/*
* TargetPhrase.cpp
*
* Created on: 23 Oct 2015
* Author: hieu
*/
#include "TargetPhrase.h"
#include "../legacy/FactorCollection.h"
#include "../legacy/Util2.h"
#include "../System.h"
using namespace std;
namespace Moses2
{
namespace Syntax
{
TargetPhrase *TargetPhrase::CreateFromString(MemPool &pool, const PhraseTable &pt, const System &system, const std::string &str)
{
FactorCollection &vocab = system.GetVocab();
vector<string> toks = Tokenize(str);
size_t size = toks.size();
TargetPhrase *ret = new (pool.Allocate<TargetPhrase>()) TargetPhrase(pool, pt, system, size);
//ret->PhraseImpl::CreateFromString(vocab, system, toks);
return ret;
}
}
}