moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

delete batch algorithm

Browse Source
This commit is contained in:
Hieu Hoang 2016-12-05 17:05:13 +00:00
parent a0182da3c9
commit 62e2c85220
8 changed files with 3 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/moses2/Jamfile
View File

@ -114,10 +114,6 @@ alias deps : ../..//z ../..//boost_iostreams ../..//boost_filesystem ../../mose
PhraseBased/CubePruningMiniStack/Search.cpp
PhraseBased/CubePruningMiniStack/Stack.cpp
PhraseBased/Batch/Search.cpp
PhraseBased/Batch/Stack.cpp
PhraseBased/Batch/Stacks.cpp
# PhraseBased/CubePruningCardinalStack/Misc.cpp
# PhraseBased/CubePruningCardinalStack/Search.cpp
# PhraseBased/CubePruningCardinalStack/Stack.cpp

171
contrib/moses2/PhraseBased/Batch/Search.cpp
View File

@ -1,171 +0,0 @@
/*
* SearchNormal.cpp
*
* Created on: 25 Oct 2015
* Author: hieu
*/
#include "Search.h"
#include <algorithm>
#include <boost/foreach.hpp>
#include "Stack.h"
#include "../Manager.h"
#include "../TrellisPath.h"
#include "../Sentence.h"
#include "../../TrellisPaths.h"
#include "../../InputPathsBase.h"
#include "../../Phrase.h"
#include "../../System.h"
#include "../../PhraseBased/TargetPhrases.h"
using namespace std;
namespace Moses2
{
namespace NSBatch
{
Search::Search(Manager &mgr)
:Moses2::Search(mgr)
, m_stacks(mgr)
, m_batch(mgr.system.GetBatch(mgr.GetSystemPool()))
{
// TODO Auto-generated constructor stub
}
Search::~Search()
{
// TODO Auto-generated destructor stub
}
void Search::Decode()
{
// init stacks
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
m_stacks.Init(mgr, sentence.GetSize() + 1);
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
initBitmap);
initHypo->EmptyHypothesisState(mgr.GetInput());
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
Decode(stackInd);
//cerr << m_stacks << endl;
// delete stack to save mem
if (stackInd < m_stacks.GetSize() - 1) {
m_stacks.Delete(stackInd);
}
//cerr << m_stacks << endl;
}
}
void Search::Decode(size_t stackInd)
{
Stack &stack = m_stacks[stackInd];
if (&stack == &m_stacks.Back()) {
// last stack. don't do anythin
return;
}
const Hypotheses &hypos = stack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
const InputPaths &paths = mgr.GetInputPaths();
BOOST_FOREACH(const InputPathBase *path, paths){
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
}
}
// process batch
mgr.system.featureFunctions.EvaluateWhenAppliedBatch(m_batch);
for (size_t i = 0; i < m_batch.size(); ++i) {
Hypothesis *hypo = m_batch[i];
m_stacks.Add(hypo, mgr.GetHypoRecycle(), mgr.arcLists);
}
m_batch.clear();
}
void Search::Extend(const Hypothesis &hypo, const InputPath &path)
{
const Bitmap &hypoBitmap = hypo.GetBitmap();
const Range &hypoRange = hypo.GetInputPath().range;
const Range &pathRange = path.range;
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
return;
}
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
if (!reorderingConstraint.Check(hypoBitmap, pathRange.GetStartPos(), pathRange.GetEndPos())) {
return;
}
//cerr << " YES" << endl;
// extend this hypo
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
size_t numPt = mgr.system.mappings.size();
const TargetPhrases **tpsAllPt = path.targetPhrases;
for (size_t i = 0; i < numPt; ++i) {
const TargetPhrases *tps = tpsAllPt[i];
if (tps) {
Extend(hypo, *tps, path, newBitmap, estimatedScore);
}
}
}
void Search::Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
BOOST_FOREACH(const TargetPhraseImpl *tp, tps){
Extend(hypo, *tp, path, newBitmap, estimatedScore);
}
}
void Search::Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
{
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
m_batch.push_back(newHypo);
//newHypo->EvaluateWhenApplied();
//m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
}
const Hypothesis *Search::GetBestHypo() const
{
const Stack &lastStack = m_stacks.Back();
return lastStack.GetBestHypo<Hypothesis>();
}
void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
{
const Stack &lastStack = m_stacks.Back();
const Hypotheses &hypos = lastStack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos){
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
paths.Add(path);
}
}
} // namespace
}

53
contrib/moses2/PhraseBased/Batch/Search.h
View File

@ -1,53 +0,0 @@
/*
* SearchNormal.h
*
* Created on: 25 Oct 2015
* Author: hieu
*/
#pragma once
#include <vector>
#include "../../legacy/Range.h"
#include "../../legacy/Bitmap.h"
#include "../../TypeDef.h"
#include "../Search.h"
#include "Stacks.h"
namespace Moses2
{
class Hypothesis;
class InputPath;
class TargetPhrases;
class TargetPhraseImpl;
namespace NSBatch
{
class Stacks;
class Search: public Moses2::Search
{
public:
Search(Manager &mgr);
virtual ~Search();
virtual void Decode();
const Hypothesis *GetBestHypo() const;
void AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const;
protected:
Stacks m_stacks;
Batch &m_batch;
void Decode(size_t stackInd);
void Extend(const Hypothesis &hypo, const InputPath &path);
void Extend(const Hypothesis &hypo, const TargetPhrases &tps,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
void Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
};
}
}

35
contrib/moses2/PhraseBased/Batch/Stack.cpp
View File

@ -1,35 +0,0 @@
/*
* Stack.cpp
*
* Created on: 24 Oct 2015
* Author: hieu
*/
#include <boost/foreach.hpp>
#include "Stack.h"
#include "../Hypothesis.h"
#include "../Manager.h"
#include "../../Scores.h"
#include "../../HypothesisColl.h"
using namespace std;
namespace Moses2
{
namespace NSBatch
{
Stack::Stack(const Manager &mgr) :
HypothesisColl(mgr)
{
// TODO Auto-generated constructor stub
}
Stack::~Stack()
{
// TODO Auto-generated destructor stub
}
}
}

32
contrib/moses2/PhraseBased/Batch/Stack.h
View File

@ -1,32 +0,0 @@
/*
* Stack.h
*
* Created on: 24 Oct 2015
* Author: hieu
*/
#pragma once
#include <boost/unordered_set.hpp>
#include <deque>
#include "../Hypothesis.h"
#include "../../TypeDef.h"
#include "../../HypothesisColl.h"
#include "../../legacy/Util2.h"
namespace Moses2
{
namespace NSBatch
{
class Stack: public HypothesisColl
{
public:
Stack(const Manager &mgr);
virtual ~Stack();
protected:
};
}
}

67
contrib/moses2/PhraseBased/Batch/Stacks.cpp
View File

@ -1,67 +0,0 @@
/*
* Stacks.cpp
*
* Created on: 6 Nov 2015
* Author: hieu
*/
#include "Stacks.h"
#include "../Manager.h"
#include "../../System.h"
using namespace std;
namespace Moses2
{
namespace NSBatch
{
Stacks::Stacks(const Manager &mgr) :
m_mgr(mgr)
{
// TODO Auto-generated constructor stub
}
Stacks::~Stacks()
{
for (size_t i = 0; i < m_stacks.size(); ++i) {
delete m_stacks[i];
}
}
void Stacks::Init(const Manager &mgr, size_t numStacks)
{
m_stacks.resize(numStacks);
for (size_t i = 0; i < m_stacks.size(); ++i) {
m_stacks[i] = new Stack(mgr);
}
}
std::string Stacks::Debug(const System &system) const
{
stringstream out;
for (size_t i = 0; i < GetSize(); ++i) {
const Stack *stack = m_stacks[i];
if (stack) {
out << stack->GetSize() << " ";
}
else {
out << "N ";
}
}
return out.str();
}
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists)
{
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
//cerr << "numWordsCovered=" << numWordsCovered << endl;
Stack &stack = *m_stacks[numWordsCovered];
stack.Add(m_mgr.system, hypo, hypoRecycle, arcLists);
}
}
}

62
contrib/moses2/PhraseBased/Batch/Stacks.h
View File

@ -1,62 +0,0 @@
/*
* Stacks.h
*
* Created on: 6 Nov 2015
* Author: hieu
*/
#pragma once
#include <vector>
#include "Stack.h"
#include "../../Recycler.h"
namespace Moses2
{
class Manager;
class ArcLists;
namespace NSBatch
{
class Stacks
{
public:
Stacks(const Manager &mgr);
virtual ~Stacks();
void Init(const Manager &mgr, size_t numStacks);
size_t GetSize() const
{
return m_stacks.size();
}
const Stack &Back() const
{
return *m_stacks.back();
}
Stack &operator[](size_t ind)
{
return *m_stacks[ind];
}
void Delete(size_t ind)
{
delete m_stacks[ind];
m_stacks[ind] = NULL;
}
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
ArcLists &arcLists);
std::string Debug(const System &system) const;
protected:
const Manager &m_mgr;
std::vector<Stack*> m_stacks;
};
}
}

7
contrib/moses2/PhraseBased/Manager.cpp
View File

@ -16,7 +16,6 @@
#include "Normal/Search.h"
#include "CubePruningMiniStack/Search.h"
#include "Batch/Search.h"
/*
#include "CubePruningPerMiniStack/Search.h"
@ -95,7 +94,8 @@ void Manager::Init()
m_search = new NSNormal::Search(*this);
break;
case NormalBatch:
m_search = new NSBatch::Search(*this);
//m_search = new NSBatch::Search(*this);
UTIL_THROW2("Not implemented");
break;
case CubePruning:
case CubePruningMiniStack:
@ -116,8 +116,7 @@ void Manager::Init()
break;
*/
default:
cerr << "Unknown search algorithm" << endl;
abort();
UTIL_THROW2("Unknown search algorithm");
}
}