moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

2Merge branch 'master' of https://machinetranslation.visualstudio.com/DefaultCollection/moses-mstranslator/_git/moses-mstranslator

Browse Source
This commit is contained in:
sid jain 2020-10-06 08:46:21 +05:30
commit 5579cbd463
8 changed files with 41 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
moses2/FF/FeatureFunction.h
View File

@ -96,7 +96,7 @@ public:
const SCFG::TargetPhrases &tps, const Phrase<SCFG::Word> &sourcePhrase) const {
}
virtual void InitializeForInput(const System &system, const InputType &input) { };
virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input) { };
// clean up temporary memory, called after processing each sentence
virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const {

4
moses2/FF/FeatureFunctions.cpp
View File

@ -229,10 +229,10 @@ void FeatureFunctions::EvaluateWhenAppliedBatch(const Batch &batch) const
}
}
void FeatureFunctions::InitializeForInput(const InputType &input)
void FeatureFunctions::InitializeForInput(const ManagerBase &mgr, const InputType &input)
{
BOOST_FOREACH(FeatureFunction *ff, m_featureFunctions) {
ff->InitializeForInput(m_system, input);
ff->InitializeForInput(mgr, input);
}
}

2
moses2/FF/FeatureFunctions.h
View File

@ -87,7 +87,7 @@ public:
void EvaluateWhenAppliedBatch(const Batch &batch) const;
void InitializeForInput(const InputType &input);
void InitializeForInput(const ManagerBase &mgr, const InputType &input);
void CleanUpAfterSentenceProcessing(const InputType &input) const;
void ShowWeights(const Weights &allWeights);

2
moses2/PhraseBased/Manager.cpp
View File

@ -63,7 +63,7 @@ void Manager::Init()
//TODO: need option to choose Sentence vs SentenceWithCandidates
m_input = Moses2::SentenceWithCandidates::CreateFromString(GetPool(), vocab, system, m_inputStr);
//cerr << "Manager::Init: " << m_input->Debug(system) << endl << flush;
system.featureFunctions.InitializeForInput(*m_input);
system.featureFunctions.InitializeForInput(*this, *m_input);
m_bitmaps = new Bitmaps(GetPool());

16
moses2/PhraseBased/SentenceWithCandidates.cpp
View File

@ -38,10 +38,10 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
input_parts.push_back(copy_range<std::string>(*It));
}
cerr << "Number of subparts: " << input_parts.size() << endl;
//cerr << "Number of subparts: " << input_parts.size() << endl;
if (input_parts.size() ==2 ) {
cerr << "correct number of parts" << endl ;
//cerr << "correct number of parts" << endl ;
} else {
// TODO: how to handle wrong input format
cerr << "INCORRECT number of parts" << endl ;
@ -50,8 +50,8 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
trim(input_parts[0]);
trim(input_parts[1]);
cerr << "Input String: " << input_parts[0] << endl ;
cerr << "Phrase Table: " << input_parts[1] << endl ;
//cerr << "Input String: " << input_parts[0] << endl ;
//cerr << "Phrase Table: " << input_parts[1] << endl ;
///// Process the text part of the input
const string partstr = input_parts[0];
@ -72,7 +72,7 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n");
// ret->m_phraseTableString="constant phrase table";
// cerr << "Extracted Phrase Table String: " << ret->m_phraseTableString << endl;
cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl;
//cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl;
return ret;
}
@ -80,17 +80,17 @@ SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool,
SentenceWithCandidates::SentenceWithCandidates(MemPool &pool, size_t size)
:Sentence(pool, size)
{
cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl;
//cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl;
}
SentenceWithCandidates::~SentenceWithCandidates()
{
cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl;
//cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl;
}
std::string SentenceWithCandidates::Debug(const System &system) const
{
cerr << "SentenceWithCandidates::Debug" << endl;
return "SentenceWithCandidates::Debug";
}
} /* namespace Moses2 */

78
moses2/TranslationModel/MSPT/MSPT.cpp
View File

@ -30,19 +30,18 @@
#include "../../SCFG/Manager.h"
#include "../../PhraseBased/SentenceWithCandidates.h"
#include "../../PhraseBased/Manager.h"
using namespace std;
namespace Moses2
{
thread_local MSPT::PBNODE *MSPT::m_rootPb;
////////////////////////////////////////////////////////////////////////
MSPT::MSPT(size_t startInd, const std::string &line)
:PhraseTable(startInd, line)
,m_rootPb(NULL)
,m_rootSCFG(NULL)
{
ReadParameters();
}
@ -50,21 +49,20 @@ MSPT::MSPT(size_t startInd, const std::string &line)
MSPT::~MSPT()
{
delete m_rootPb;
delete m_rootSCFG;
}
void MSPT::CreatePTForInput(const System &system, string phraseTableString)
void MSPT::CreatePTForInput(const ManagerBase &mgr, string phraseTableString)
{
cerr << "In CreatePTForInput" << endl << flush;
//cerr << "In CreatePTForInput" << endl << flush;
const System &system = mgr.system;
FactorCollection &vocab = system.GetVocab();
MemPool &systemPool = system.GetSystemPool();
MemPool &pool = mgr.GetPool();
MemPool tmpSourcePool;
if (system.isPb) {
m_rootPb = new PBNODE();
} else {
m_rootSCFG = new SCFGNODE();
abort();
//cerr << "m_rootSCFG=" << m_rootSCFG << endl;
}
@ -86,7 +84,7 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
toks[0]);
//cerr << "created soure" << endl;
TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system,
TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, system,
toks[1]);
//cerr << "created target" << endl;
target->GetScores().CreateFromString(toks[2], *this, system, true);
@ -103,48 +101,22 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
//strcpy(target->properties, toks[6].c_str());
}
system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
system.featureFunctions.EvaluateInIsolation(pool, system, *source,
*target);
//cerr << "EvaluateInIsolation:" << *target << endl;
//cerr << "EvaluateInIsolation:" << target->Debug(system) << endl;
m_rootPb->AddRule(m_input, *source, target);
//cerr << "target=" << target->Debug(system) << endl;
} else {
SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system,
toks[0]);
//cerr << "created source:" << *source << endl;
SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this,
system, toks[1]);
//cerr << "created target " << *target << " source=" << *source << endl;
target->GetScores().CreateFromString(toks[2], *this, system, true);
//cerr << "created scores:" << *target << endl;
//vector<SCORE> scores = Tokenize<SCORE>(toks[2]);
//target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0;
target->SetAlignmentInfo(toks[3]);
// properties
if (toks.size() == 7) {
//target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1);
//strcpy(target->properties, toks[6].c_str());
}
system.featureFunctions.EvaluateInIsolation(systemPool, system, *source,
*target);
//cerr << "EvaluateInIsolation:" << *target << endl;
m_rootSCFG->AddRule(m_input, *source, target);
abort();
}
}
if (system.isPb) {
m_rootPb->SortAndPrune(m_tableLimit, systemPool, system);
m_rootPb->SortAndPrune(m_tableLimit, pool, system);
//cerr << "root=" << &m_rootPb << endl;
} else {
m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system);
//cerr << "root=" << &m_rootPb << endl;
abort();
}
/*
BOOST_FOREACH(const PtMem::Node<Word>::Children::value_type &valPair, m_rootPb.GetChildren()) {
@ -156,32 +128,28 @@ void MSPT::CreatePTForInput(const System &system, string phraseTableString)
}
void MSPT::InitializeForInput(const System &system, const InputType &input)
void MSPT::InitializeForInput(const ManagerBase &mgr, const InputType &input)
{
cerr << "InitializeForInput MSPT" << endl;
cerr << input.Debug(system) << endl << flush;
cerr << "HH1" << endl << flush;
// downcast to SentenceWithCandidates
//const SentenceWithCandidates &inputObj = static_cast<const SentenceWithCandidates&>(input);
const SentenceWithCandidates &inputObj = dynamic_cast<const SentenceWithCandidates&>(input);
cerr << "Casting done." << endl << flush;
cerr << "PhraseTableString member: " << inputObj.getPhraseTableString() << endl;
cerr << "Hardcoding sample PhraseTableString" << endl << flush;
string phraseTableString="a ||| x ||| 0.4 $$$ a ||| y ||| 0.6 $$$ b ||| y ||| 0.1 $$$ b ||| z ||| 0.9";
CreatePTForInput(system,phraseTableString);
const SentenceWithCandidates &inputObj = static_cast<const SentenceWithCandidates&>(input);
CreatePTForInput(mgr, inputObj.getPhraseTableString());
}
TargetPhrases* MSPT::Lookup(const Manager &mgr, MemPool &pool,
InputPath &inputPath) const
{
//cerr << "MSPT::Lookup inputPath:" << inputPath.Debug(mgr.system) << endl;
const SubPhrase<Moses2::Word> &phrase = inputPath.subPhrase;
TargetPhrases *tps = m_rootPb->Find(m_input, phrase);
//cerr << "MSPT::Lookup tps:" << tps->Debug(mgr.system) << endl;
//cerr << "MSPT::Lookup done" << endl;
return tps;
}
void MSPT::CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const {
delete m_rootPb;
}
void MSPT::InitActiveChart(
MemPool &pool,
const SCFG::Manager &mgr,

8
moses2/TranslationModel/MSPT/MSPT.h
View File

@ -65,11 +65,11 @@ public:
const SCFG::Stacks &stacks,
SCFG::InputPath &path) const;
virtual void InitializeForInput(const System &system, const InputType &input);
virtual void InitializeForInput(const ManagerBase &mgr, const InputType &input);
virtual void CleanUpAfterSentenceProcessing(const System &system, const InputType &input) const;
protected:
PBNODE *m_rootPb;
SCFGNODE *m_rootSCFG;
thread_local static PBNODE *m_rootPb;
void LookupGivenNode(
MemPool &pool,
@ -80,7 +80,7 @@ protected:
const Moses2::Range &subPhraseRange,
SCFG::InputPath &outPath) const;
void CreatePTForInput(const System &system, std::string phraseTableString);
void CreatePTForInput(const ManagerBase &mgr, std::string phraseTableString);
};

8
moses2/TranslationModel/PhraseTable.cpp
View File

@ -80,13 +80,7 @@ void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const
if (SatisfyBackoff(mgr, *path)) {
TargetPhrases *tpsPtr = Lookup(mgr, mgr.GetPool(), *path);
/*
cerr << "tpsPtr=" << tpsPtr << " ";
if (tps.get()) {
cerr << tps.get()->GetSize();
}
cerr << endl;
*/
//cerr << "tpsPtr=" << tpsPtr << endl;
path->AddTargetPhrases(*this, tpsPtr);
}