mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Merge branch 'master' of github.com:moses-smt/mosesdecoder
This commit is contained in:
commit
76a8850487
1
Jamroot
1
Jamroot
@ -183,7 +183,6 @@ requirements += [ option.get "with-mm" : : <define>MAX_NUM_FACTORS=4 ] ;
|
||||
requirements += [ option.get "unlabelled-source" : : <define>UNLABELLED_SOURCE ] ;
|
||||
|
||||
if [ option.get "with-oxlm" ] {
|
||||
external-lib boost_serialization ;
|
||||
external-lib gomp ;
|
||||
requirements += <library>boost_serialization ;
|
||||
requirements += <library>gomp ;
|
||||
|
@ -11,12 +11,12 @@
|
||||
</externalSetting>
|
||||
</externalSettings>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
@ -106,13 +106,13 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1445209421" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
|
@ -56,6 +56,9 @@ public:
|
||||
return m_arr[ind];
|
||||
}
|
||||
|
||||
T *GetArray()
|
||||
{ return m_arr; }
|
||||
|
||||
size_t hash() const
|
||||
{
|
||||
size_t seed = 0;
|
||||
|
@ -118,6 +118,8 @@ void Distortion::EvaluateWhenApplied(const ManagerBase &mgr,
|
||||
DistortionState_traditional &stateCast =
|
||||
static_cast<DistortionState_traditional&>(state);
|
||||
stateCast.Set(hypo.GetInputPath().range, hypo.GetBitmap().GetFirstGapPos());
|
||||
|
||||
//cerr << "hypo=" << hypo.Debug(mgr.system) << endl;
|
||||
}
|
||||
|
||||
SCORE Distortion::CalculateDistortionScore(const Range &prev, const Range &curr,
|
||||
|
@ -54,7 +54,7 @@ size_t BidirectionalReorderingState::hash() const
|
||||
|
||||
bool BidirectionalReorderingState::operator==(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return 0;
|
||||
if (&o == this) return true;
|
||||
|
||||
BidirectionalReorderingState const &other =
|
||||
static_cast<BidirectionalReorderingState const&>(o);
|
||||
|
@ -18,10 +18,12 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
HypothesisColl::HypothesisColl(const ManagerBase &mgr) :
|
||||
m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool())), m_sortedHypos(
|
||||
NULL)
|
||||
HypothesisColl::HypothesisColl(const ManagerBase &mgr)
|
||||
:m_coll(MemPoolAllocator<const HypothesisBase*>(mgr.GetPool()))
|
||||
,m_sortedHypos(NULL)
|
||||
{
|
||||
m_bestScore = -std::numeric_limits<float>::infinity();
|
||||
m_worstScore = std::numeric_limits<float>::infinity();
|
||||
}
|
||||
|
||||
const HypothesisBase *HypothesisColl::GetBestHypo() const
|
||||
@ -45,39 +47,82 @@ const HypothesisBase *HypothesisColl::GetBestHypo() const
|
||||
}
|
||||
|
||||
void HypothesisColl::Add(
|
||||
const System &system,
|
||||
const ManagerBase &mgr,
|
||||
HypothesisBase *hypo,
|
||||
Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists)
|
||||
{
|
||||
size_t maxStackSize = mgr.system.options.search.stack_size;
|
||||
|
||||
if (GetSize() > maxStackSize * 2) {
|
||||
//cerr << "maxStackSize=" << maxStackSize << " " << GetSize() << endl;
|
||||
PruneHypos(mgr, mgr.arcLists);
|
||||
}
|
||||
|
||||
SCORE futureScore = hypo->GetFutureScore();
|
||||
|
||||
/*
|
||||
cerr << "scores:"
|
||||
<< futureScore << " "
|
||||
<< m_bestScore << " "
|
||||
<< GetSize() << " "
|
||||
<< endl;
|
||||
*/
|
||||
if (GetSize() >= maxStackSize && futureScore < m_worstScore) {
|
||||
// beam threshold or really bad hypo that won't make the pruning cut
|
||||
// as more hypos are added, the m_worstScore stat gets out of date and isn't the optimum cut-off point
|
||||
//cerr << "Discard, really bad score:" << hypo->Debug(mgr.system) << endl;
|
||||
hypoRecycle.Recycle(hypo);
|
||||
return;
|
||||
}
|
||||
|
||||
StackAdd added = Add(hypo);
|
||||
|
||||
size_t nbestSize = system.options.nbest.nbest_size;
|
||||
size_t nbestSize = mgr.system.options.nbest.nbest_size;
|
||||
if (nbestSize) {
|
||||
arcLists.AddArc(added.added, hypo, added.other);
|
||||
}
|
||||
else {
|
||||
if (!added.added) {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
if (added.added) {
|
||||
if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
}
|
||||
}
|
||||
else if (added.other) {
|
||||
hypoRecycle.Recycle(added.other);
|
||||
else {
|
||||
hypoRecycle.Recycle(hypo);
|
||||
}
|
||||
}
|
||||
|
||||
// update beam variables
|
||||
if (added.added) {
|
||||
if (futureScore > m_bestScore) {
|
||||
m_bestScore = futureScore;
|
||||
float beamWidth = mgr.system.options.search.beam_width;
|
||||
if ( m_bestScore + beamWidth > m_worstScore ) {
|
||||
m_worstScore = m_bestScore + beamWidth;
|
||||
}
|
||||
}
|
||||
else if (GetSize() <= maxStackSize && futureScore < m_worstScore) {
|
||||
m_worstScore = futureScore;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
|
||||
{
|
||||
std::pair<_HCType::iterator, bool> addRet = m_coll.insert(hypo);
|
||||
//cerr << endl << "new=" << hypo->Debug(hypo->GetManager().system) << endl;
|
||||
|
||||
// CHECK RECOMBINATION
|
||||
if (addRet.second) {
|
||||
// equiv hypo doesn't exists
|
||||
//cerr << "Added " << hypo << endl;
|
||||
return StackAdd(true, NULL);
|
||||
}
|
||||
else {
|
||||
HypothesisBase *hypoExisting = const_cast<HypothesisBase*>(*addRet.first);
|
||||
//cerr << "hypoExisting=" << hypoExisting->Debug(hypo->GetManager().system) << endl;
|
||||
|
||||
if (hypo->GetFutureScore() > hypoExisting->GetFutureScore()) {
|
||||
// incoming hypo is better than the one we have
|
||||
const HypothesisBase * const &hypoExisting1 = *addRet.first;
|
||||
@ -85,18 +130,20 @@ StackAdd HypothesisColl::Add(const HypothesisBase *hypo)
|
||||
const_cast<const HypothesisBase *&>(hypoExisting1);
|
||||
hypoExisting2 = hypo;
|
||||
|
||||
//cerr << "Added " << hypo << " dicard existing " << hypoExisting2 << endl;
|
||||
return StackAdd(true, hypoExisting);
|
||||
}
|
||||
else {
|
||||
// already storing the best hypo. discard incoming hypo
|
||||
//cerr << "Keep existing " << hypoExisting << " dicard new " << hypo << endl;
|
||||
return StackAdd(false, hypoExisting);
|
||||
}
|
||||
}
|
||||
|
||||
assert(false);
|
||||
//assert(false);
|
||||
}
|
||||
|
||||
const Hypotheses &HypothesisColl::GetSortedAndPruneHypos(
|
||||
const Hypotheses &HypothesisColl::GetSortedAndPrunedHypos(
|
||||
const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const
|
||||
{
|
||||
@ -106,73 +153,124 @@ const Hypotheses &HypothesisColl::GetSortedAndPruneHypos(
|
||||
m_sortedHypos = new (pool.Allocate<Hypotheses>()) Hypotheses(pool,
|
||||
m_coll.size());
|
||||
|
||||
size_t ind = 0;
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, m_coll){
|
||||
(*m_sortedHypos)[ind] = hypo;
|
||||
++ind;
|
||||
}
|
||||
SortHypos(mgr, m_sortedHypos->GetArray());
|
||||
|
||||
// prune
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
|
||||
size_t maxStackSize = mgr.system.options.search.stack_size;
|
||||
if (maxStackSize && m_sortedHypos->size() > maxStackSize) {
|
||||
for (size_t i = maxStackSize; i < m_sortedHypos->size(); ++i) {
|
||||
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
|
||||
recycler.Recycle(hypo);
|
||||
|
||||
// delete from arclist
|
||||
if (mgr.system.options.nbest.nbest_size) {
|
||||
arcLists.Delete(hypo);
|
||||
}
|
||||
}
|
||||
m_sortedHypos->resize(maxStackSize);
|
||||
}
|
||||
|
||||
SortAndPruneHypos(mgr, arcLists);
|
||||
}
|
||||
|
||||
return *m_sortedHypos;
|
||||
}
|
||||
|
||||
const Hypotheses &HypothesisColl::GetSortedAndPrunedHypos() const
|
||||
void HypothesisColl::PruneHypos(const ManagerBase &mgr, ArcLists &arcLists)
|
||||
{
|
||||
UTIL_THROW_IF2(m_sortedHypos == NULL, "m_sortedHypos must be sorted beforehand");
|
||||
return *m_sortedHypos;
|
||||
size_t maxStackSize = mgr.system.options.search.stack_size;
|
||||
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
|
||||
const HypothesisBase *sortedHypos[GetSize()];
|
||||
SortHypos(mgr, sortedHypos);
|
||||
|
||||
// update worse score
|
||||
m_worstScore = sortedHypos[maxStackSize - 1]->GetFutureScore();
|
||||
|
||||
// prune
|
||||
for (size_t i = maxStackSize; i < GetSize(); ++i) {
|
||||
HypothesisBase *hypo = const_cast<HypothesisBase*>(sortedHypos[i]);
|
||||
|
||||
// delete from arclist
|
||||
if (mgr.system.options.nbest.nbest_size) {
|
||||
arcLists.Delete(hypo);
|
||||
}
|
||||
|
||||
// delete from collection
|
||||
Delete(hypo);
|
||||
|
||||
recycler.Recycle(hypo);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void HypothesisColl::SortAndPruneHypos(const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const
|
||||
void HypothesisColl::SortHypos(const ManagerBase &mgr, const HypothesisBase **sortedHypos) const
|
||||
{
|
||||
size_t stackSize = mgr.system.options.search.stack_size;
|
||||
Recycler<HypothesisBase*> &recycler = mgr.GetHypoRecycle();
|
||||
size_t maxStackSize = mgr.system.options.search.stack_size;
|
||||
//assert(maxStackSize); // can't do stack=0 - unlimited stack size. No-one ever uses that
|
||||
//assert(GetSize() > maxStackSize);
|
||||
//assert(sortedHypos.size() == GetSize());
|
||||
|
||||
/*
|
||||
/*
|
||||
cerr << "UNSORTED hypos: ";
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, m_coll) {
|
||||
cerr << hypo << "(" << hypo->GetFutureScore() << ")" << " ";
|
||||
cerr << hypo << "(" << hypo->GetFutureScore() << ")" << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
Hypotheses::iterator iterMiddle;
|
||||
iterMiddle =
|
||||
(stackSize == 0 || m_sortedHypos->size() < stackSize) ?
|
||||
m_sortedHypos->end() : m_sortedHypos->begin() + stackSize;
|
||||
*/
|
||||
size_t ind = 0;
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, m_coll){
|
||||
sortedHypos[ind] = hypo;
|
||||
++ind;
|
||||
}
|
||||
|
||||
std::partial_sort(m_sortedHypos->begin(), iterMiddle, m_sortedHypos->end(),
|
||||
HypothesisFutureScoreOrderer());
|
||||
size_t indMiddle;
|
||||
if (maxStackSize == 0) {
|
||||
indMiddle = GetSize();
|
||||
}
|
||||
else if (GetSize() > maxStackSize) {
|
||||
indMiddle = maxStackSize;
|
||||
}
|
||||
else {
|
||||
// GetSize() <= maxStackSize
|
||||
indMiddle = GetSize();
|
||||
}
|
||||
|
||||
// prune
|
||||
if (stackSize && m_sortedHypos->size() > stackSize) {
|
||||
for (size_t i = stackSize; i < m_sortedHypos->size(); ++i) {
|
||||
HypothesisBase *hypo = const_cast<HypothesisBase*>((*m_sortedHypos)[i]);
|
||||
recycler.Recycle(hypo);
|
||||
const HypothesisBase **iterMiddle = sortedHypos + indMiddle;
|
||||
|
||||
// delete from arclist
|
||||
if (mgr.system.options.nbest.nbest_size) {
|
||||
arcLists.Delete(hypo);
|
||||
}
|
||||
}
|
||||
m_sortedHypos->resize(stackSize);
|
||||
}
|
||||
std::partial_sort(
|
||||
sortedHypos,
|
||||
iterMiddle,
|
||||
sortedHypos + GetSize(),
|
||||
HypothesisFutureScoreOrderer());
|
||||
|
||||
/*
|
||||
/*
|
||||
cerr << "sorted hypos: ";
|
||||
for (size_t i = 0; i < m_sortedHypos->size(); ++i) {
|
||||
const HypothesisBase *hypo = (*m_sortedHypos)[i];
|
||||
cerr << hypo << " ";
|
||||
for (size_t i = 0; i < sortedHypos.size(); ++i) {
|
||||
const HypothesisBase *hypo = sortedHypos[i];
|
||||
cerr << hypo << " ";
|
||||
}
|
||||
cerr << endl;
|
||||
*/
|
||||
*/
|
||||
}
|
||||
|
||||
void HypothesisColl::Delete(const HypothesisBase *hypo)
|
||||
{
|
||||
//cerr << "hypo=" << hypo << " " << m_coll.size() << endl;
|
||||
|
||||
size_t erased = m_coll.erase(hypo);
|
||||
UTIL_THROW_IF2(erased != 1, "couldn't erase hypo " << hypo);
|
||||
}
|
||||
|
||||
void HypothesisColl::Clear()
|
||||
{
|
||||
m_sortedHypos = NULL;
|
||||
m_coll.clear();
|
||||
|
||||
m_bestScore = -std::numeric_limits<float>::infinity();
|
||||
m_worstScore = std::numeric_limits<float>::infinity();
|
||||
}
|
||||
|
||||
std::string HypothesisColl::Debug(const System &system) const
|
||||
|
@ -25,7 +25,7 @@ class HypothesisColl
|
||||
public:
|
||||
HypothesisColl(const ManagerBase &mgr);
|
||||
|
||||
void Add(const System &system,
|
||||
void Add(const ManagerBase &mgr,
|
||||
HypothesisBase *hypo,
|
||||
Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists);
|
||||
@ -35,12 +35,10 @@ public:
|
||||
|
||||
void Clear();
|
||||
|
||||
const Hypotheses &GetSortedAndPruneHypos(
|
||||
const Hypotheses &GetSortedAndPrunedHypos(
|
||||
const ManagerBase &mgr,
|
||||
ArcLists &arcLists) const;
|
||||
|
||||
const Hypotheses &GetSortedAndPrunedHypos() const;
|
||||
|
||||
const HypothesisBase *GetBestHypo() const;
|
||||
|
||||
template<typename T>
|
||||
@ -50,6 +48,8 @@ public:
|
||||
return hypo ? &hypo->Cast<T>() : NULL;
|
||||
}
|
||||
|
||||
void Delete(const HypothesisBase *hypo);
|
||||
|
||||
std::string Debug(const System &system) const;
|
||||
|
||||
protected:
|
||||
@ -60,8 +60,13 @@ protected:
|
||||
_HCType m_coll;
|
||||
mutable Hypotheses *m_sortedHypos;
|
||||
|
||||
SCORE m_bestScore;
|
||||
SCORE m_worstScore;
|
||||
|
||||
StackAdd Add(const HypothesisBase *hypo);
|
||||
void SortAndPruneHypos(const ManagerBase &mgr, ArcLists &arcLists) const;
|
||||
|
||||
void PruneHypos(const ManagerBase &mgr, ArcLists &arcLists);
|
||||
void SortHypos(const ManagerBase &mgr, const HypothesisBase **sortedHypos) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -114,10 +114,6 @@ alias deps : ../..//z ../..//boost_iostreams ../..//boost_filesystem ../../mose
|
||||
PhraseBased/CubePruningMiniStack/Search.cpp
|
||||
PhraseBased/CubePruningMiniStack/Stack.cpp
|
||||
|
||||
PhraseBased/Batch/Search.cpp
|
||||
PhraseBased/Batch/Stack.cpp
|
||||
PhraseBased/Batch/Stacks.cpp
|
||||
|
||||
# PhraseBased/CubePruningCardinalStack/Misc.cpp
|
||||
# PhraseBased/CubePruningCardinalStack/Search.cpp
|
||||
# PhraseBased/CubePruningCardinalStack/Stack.cpp
|
||||
|
@ -173,7 +173,6 @@ void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system,
|
||||
scores.PlusEquals(system, *this, score);
|
||||
SCORE weightedScore = Scores::CalcWeightedScore(system, *this, nonFullScore);
|
||||
estimatedScore += weightedScore;
|
||||
|
||||
}
|
||||
|
||||
void LanguageModel::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source,
|
||||
|
@ -1,171 +0,0 @@
|
||||
/*
|
||||
* SearchNormal.cpp
|
||||
*
|
||||
* Created on: 25 Oct 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
|
||||
#include "Search.h"
|
||||
#include <algorithm>
|
||||
#include <boost/foreach.hpp>
|
||||
#include "Stack.h"
|
||||
#include "../Manager.h"
|
||||
#include "../TrellisPath.h"
|
||||
#include "../Sentence.h"
|
||||
#include "../../TrellisPaths.h"
|
||||
#include "../../InputPathsBase.h"
|
||||
#include "../../Phrase.h"
|
||||
#include "../../System.h"
|
||||
#include "../../PhraseBased/TargetPhrases.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
namespace NSBatch
|
||||
{
|
||||
|
||||
Search::Search(Manager &mgr)
|
||||
:Moses2::Search(mgr)
|
||||
, m_stacks(mgr)
|
||||
, m_batch(mgr.system.GetBatch(mgr.GetSystemPool()))
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Search::~Search()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
void Search::Decode()
|
||||
{
|
||||
// init stacks
|
||||
const Sentence &sentence = static_cast<const Sentence&>(mgr.GetInput());
|
||||
m_stacks.Init(mgr, sentence.GetSize() + 1);
|
||||
|
||||
const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap();
|
||||
Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(),
|
||||
initBitmap);
|
||||
initHypo->EmptyHypothesisState(mgr.GetInput());
|
||||
|
||||
m_stacks.Add(initHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
|
||||
for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) {
|
||||
Decode(stackInd);
|
||||
//cerr << m_stacks << endl;
|
||||
|
||||
// delete stack to save mem
|
||||
if (stackInd < m_stacks.GetSize() - 1) {
|
||||
m_stacks.Delete(stackInd);
|
||||
}
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
}
|
||||
|
||||
void Search::Decode(size_t stackInd)
|
||||
{
|
||||
Stack &stack = m_stacks[stackInd];
|
||||
if (&stack == &m_stacks.Back()) {
|
||||
// last stack. don't do anythin
|
||||
return;
|
||||
}
|
||||
|
||||
const Hypotheses &hypos = stack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
|
||||
const InputPaths &paths = mgr.GetInputPaths();
|
||||
|
||||
BOOST_FOREACH(const InputPathBase *path, paths){
|
||||
BOOST_FOREACH(const HypothesisBase *hypo, hypos) {
|
||||
Extend(*static_cast<const Hypothesis*>(hypo), *static_cast<const InputPath*>(path));
|
||||
}
|
||||
}
|
||||
|
||||
// process batch
|
||||
mgr.system.featureFunctions.EvaluateWhenAppliedBatch(m_batch);
|
||||
|
||||
for (size_t i = 0; i < m_batch.size(); ++i) {
|
||||
Hypothesis *hypo = m_batch[i];
|
||||
m_stacks.Add(hypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
}
|
||||
m_batch.clear();
|
||||
}
|
||||
|
||||
void Search::Extend(const Hypothesis &hypo, const InputPath &path)
|
||||
{
|
||||
const Bitmap &hypoBitmap = hypo.GetBitmap();
|
||||
const Range &hypoRange = hypo.GetInputPath().range;
|
||||
const Range &pathRange = path.range;
|
||||
|
||||
if (!CanExtend(hypoBitmap, hypoRange.GetEndPos(), pathRange)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ReorderingConstraint &reorderingConstraint = mgr.GetInput().GetReorderingConstraint();
|
||||
if (!reorderingConstraint.Check(hypoBitmap, pathRange.GetStartPos(), pathRange.GetEndPos())) {
|
||||
return;
|
||||
}
|
||||
//cerr << " YES" << endl;
|
||||
|
||||
// extend this hypo
|
||||
const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange);
|
||||
//SCORE estimatedScore = mgr.GetEstimatedScores().CalcFutureScore2(bitmap, pathRange.GetStartPos(), pathRange.GetEndPos());
|
||||
SCORE estimatedScore = mgr.GetEstimatedScores().CalcEstimatedScore(newBitmap);
|
||||
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
const TargetPhrases **tpsAllPt = path.targetPhrases;
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
const TargetPhrases *tps = tpsAllPt[i];
|
||||
if (tps) {
|
||||
Extend(hypo, *tps, path, newBitmap, estimatedScore);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Search::Extend(const Hypothesis &hypo, const TargetPhrases &tps,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
|
||||
{
|
||||
BOOST_FOREACH(const TargetPhraseImpl *tp, tps){
|
||||
Extend(hypo, *tp, path, newBitmap, estimatedScore);
|
||||
}
|
||||
}
|
||||
|
||||
void Search::Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore)
|
||||
{
|
||||
Hypothesis *newHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr);
|
||||
newHypo->Init(mgr, hypo, path, tp, newBitmap, estimatedScore);
|
||||
|
||||
m_batch.push_back(newHypo);
|
||||
//newHypo->EvaluateWhenApplied();
|
||||
|
||||
//m_stacks.Add(newHypo, mgr.GetHypoRecycle(), mgr.arcLists);
|
||||
|
||||
//m_arcLists.AddArc(stackAdded.added, newHypo, stackAdded.other);
|
||||
//stack.Prune(mgr.GetHypoRecycle(), mgr.system.stackSize, mgr.system.stackSize * 2);
|
||||
|
||||
}
|
||||
|
||||
const Hypothesis *Search::GetBestHypo() const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
return lastStack.GetBestHypo<Hypothesis>();
|
||||
}
|
||||
|
||||
void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
const Hypotheses &hypos = lastStack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos){
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
TrellisPath *path = new TrellisPath(hypo, mgr.arcLists);
|
||||
paths.Add(path);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
}
|
||||
|
@ -1,53 +0,0 @@
|
||||
/*
|
||||
* SearchNormal.h
|
||||
*
|
||||
* Created on: 25 Oct 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include "../../legacy/Range.h"
|
||||
#include "../../legacy/Bitmap.h"
|
||||
#include "../../TypeDef.h"
|
||||
#include "../Search.h"
|
||||
#include "Stacks.h"
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
class Hypothesis;
|
||||
class InputPath;
|
||||
class TargetPhrases;
|
||||
class TargetPhraseImpl;
|
||||
|
||||
namespace NSBatch
|
||||
{
|
||||
class Stacks;
|
||||
|
||||
class Search: public Moses2::Search
|
||||
{
|
||||
public:
|
||||
Search(Manager &mgr);
|
||||
virtual ~Search();
|
||||
|
||||
virtual void Decode();
|
||||
const Hypothesis *GetBestHypo() const;
|
||||
|
||||
void AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const;
|
||||
|
||||
protected:
|
||||
Stacks m_stacks;
|
||||
|
||||
Batch &m_batch;
|
||||
|
||||
void Decode(size_t stackInd);
|
||||
void Extend(const Hypothesis &hypo, const InputPath &path);
|
||||
void Extend(const Hypothesis &hypo, const TargetPhrases &tps,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
|
||||
void Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
|
||||
const InputPath &path, const Bitmap &newBitmap, SCORE estimatedScore);
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
}
|
@ -1,35 +0,0 @@
|
||||
/*
|
||||
* Stack.cpp
|
||||
*
|
||||
* Created on: 24 Oct 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
#include <boost/foreach.hpp>
|
||||
#include "Stack.h"
|
||||
#include "../Hypothesis.h"
|
||||
#include "../Manager.h"
|
||||
#include "../../Scores.h"
|
||||
#include "../../HypothesisColl.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
namespace NSBatch
|
||||
{
|
||||
|
||||
Stack::Stack(const Manager &mgr) :
|
||||
HypothesisColl(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Stack::~Stack()
|
||||
{
|
||||
// TODO Auto-generated destructor stub
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -1,32 +0,0 @@
|
||||
/*
|
||||
* Stack.h
|
||||
*
|
||||
* Created on: 24 Oct 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <boost/unordered_set.hpp>
|
||||
#include <deque>
|
||||
#include "../Hypothesis.h"
|
||||
#include "../../TypeDef.h"
|
||||
#include "../../HypothesisColl.h"
|
||||
#include "../../legacy/Util2.h"
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
namespace NSBatch
|
||||
{
|
||||
class Stack: public HypothesisColl
|
||||
{
|
||||
public:
|
||||
Stack(const Manager &mgr);
|
||||
virtual ~Stack();
|
||||
|
||||
protected:
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
}
|
@ -1,67 +0,0 @@
|
||||
/*
|
||||
* Stacks.cpp
|
||||
*
|
||||
* Created on: 6 Nov 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
|
||||
#include "Stacks.h"
|
||||
#include "../Manager.h"
|
||||
#include "../../System.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
namespace NSBatch
|
||||
{
|
||||
|
||||
Stacks::Stacks(const Manager &mgr) :
|
||||
m_mgr(mgr)
|
||||
{
|
||||
// TODO Auto-generated constructor stub
|
||||
|
||||
}
|
||||
|
||||
Stacks::~Stacks()
|
||||
{
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
delete m_stacks[i];
|
||||
}
|
||||
}
|
||||
|
||||
void Stacks::Init(const Manager &mgr, size_t numStacks)
|
||||
{
|
||||
m_stacks.resize(numStacks);
|
||||
for (size_t i = 0; i < m_stacks.size(); ++i) {
|
||||
m_stacks[i] = new Stack(mgr);
|
||||
}
|
||||
}
|
||||
|
||||
std::string Stacks::Debug(const System &system) const
|
||||
{
|
||||
stringstream out;
|
||||
for (size_t i = 0; i < GetSize(); ++i) {
|
||||
const Stack *stack = m_stacks[i];
|
||||
if (stack) {
|
||||
out << stack->GetSize() << " ";
|
||||
}
|
||||
else {
|
||||
out << "N ";
|
||||
}
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
||||
void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists)
|
||||
{
|
||||
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
|
||||
//cerr << "numWordsCovered=" << numWordsCovered << endl;
|
||||
Stack &stack = *m_stacks[numWordsCovered];
|
||||
stack.Add(m_mgr.system, hypo, hypoRecycle, arcLists);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
@ -1,62 +0,0 @@
|
||||
/*
|
||||
* Stacks.h
|
||||
*
|
||||
* Created on: 6 Nov 2015
|
||||
* Author: hieu
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include "Stack.h"
|
||||
#include "../../Recycler.h"
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
class Manager;
|
||||
class ArcLists;
|
||||
|
||||
namespace NSBatch
|
||||
{
|
||||
|
||||
class Stacks
|
||||
{
|
||||
public:
|
||||
Stacks(const Manager &mgr);
|
||||
virtual ~Stacks();
|
||||
|
||||
void Init(const Manager &mgr, size_t numStacks);
|
||||
|
||||
size_t GetSize() const
|
||||
{
|
||||
return m_stacks.size();
|
||||
}
|
||||
|
||||
const Stack &Back() const
|
||||
{
|
||||
return *m_stacks.back();
|
||||
}
|
||||
|
||||
Stack &operator[](size_t ind)
|
||||
{
|
||||
return *m_stacks[ind];
|
||||
}
|
||||
|
||||
void Delete(size_t ind)
|
||||
{
|
||||
delete m_stacks[ind];
|
||||
m_stacks[ind] = NULL;
|
||||
}
|
||||
|
||||
void Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
ArcLists &arcLists);
|
||||
|
||||
std::string Debug(const System &system) const;
|
||||
|
||||
protected:
|
||||
const Manager &m_mgr;
|
||||
std::vector<Stack*> m_stacks;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
@ -73,10 +73,9 @@ void Search::Decode()
|
||||
//cerr << "stackInd=" << stackInd << endl;
|
||||
m_stack.Clear();
|
||||
Decode(stackInd);
|
||||
PostDecode(stackInd);
|
||||
PostDecode(stackInd);
|
||||
|
||||
//m_stack.DebugCounts();
|
||||
//cerr << m_stacks << endl;
|
||||
}
|
||||
|
||||
}
|
||||
@ -173,6 +172,9 @@ void Search::PostDecode(size_t stackInd)
|
||||
const Bitmap &hypoBitmap = *val.first.first;
|
||||
size_t firstGap = hypoBitmap.GetFirstGapPos();
|
||||
size_t hypoEndPos = val.first.second;
|
||||
|
||||
Moses2::HypothesisColl &hypos = *val.second;
|
||||
|
||||
//cerr << "key=" << hypoBitmap << " " << firstGap << " " << inputSize << endl;
|
||||
|
||||
// create edges to next hypos from existing hypos
|
||||
@ -204,7 +206,7 @@ void Search::PostDecode(size_t stackInd)
|
||||
CubeEdges &edges = *m_cubeEdges[numWords];
|
||||
|
||||
// sort hypo for a particular bitmap and hypoEndPos
|
||||
const Hypotheses &sortedHypos = val.second->GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
|
||||
|
||||
size_t numPt = mgr.system.mappings.size();
|
||||
for (size_t i = 0; i < numPt; ++i) {
|
||||
@ -229,8 +231,8 @@ void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
|
||||
{
|
||||
const Stack::Coll &coll = m_stack.GetColl();
|
||||
BOOST_FOREACH(const Stack::Coll::value_type &val, coll){
|
||||
const Moses2::HypothesisColl &hypos = *val.second;
|
||||
const Hypotheses &sortedHypos = hypos.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
Moses2::HypothesisColl &hypos = *val.second;
|
||||
const Hypotheses &sortedHypos = hypos.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
|
||||
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, sortedHypos) {
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
|
@ -47,7 +47,7 @@ void Stack::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
{
|
||||
HypoCoverage key(&hypo->GetBitmap(), hypo->GetInputPath().range.GetEndPos());
|
||||
Moses2::HypothesisColl &coll = GetMiniStack(key);
|
||||
coll.Add(m_mgr.system, hypo, hypoRecycle, arcLists);
|
||||
coll.Add(m_mgr, hypo, hypoRecycle, arcLists);
|
||||
}
|
||||
|
||||
const Hypothesis *Stack::GetBestHypo() const
|
||||
|
@ -16,7 +16,6 @@
|
||||
|
||||
#include "Normal/Search.h"
|
||||
#include "CubePruningMiniStack/Search.h"
|
||||
#include "Batch/Search.h"
|
||||
|
||||
/*
|
||||
#include "CubePruningPerMiniStack/Search.h"
|
||||
@ -95,7 +94,8 @@ void Manager::Init()
|
||||
m_search = new NSNormal::Search(*this);
|
||||
break;
|
||||
case NormalBatch:
|
||||
m_search = new NSBatch::Search(*this);
|
||||
//m_search = new NSBatch::Search(*this);
|
||||
UTIL_THROW2("Not implemented");
|
||||
break;
|
||||
case CubePruning:
|
||||
case CubePruningMiniStack:
|
||||
@ -116,8 +116,7 @@ void Manager::Init()
|
||||
break;
|
||||
*/
|
||||
default:
|
||||
cerr << "Unknown search algorithm" << endl;
|
||||
abort();
|
||||
UTIL_THROW2("Unknown search algorithm");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -60,7 +60,7 @@ void Search::Decode()
|
||||
if (stackInd < m_stacks.GetSize() - 1) {
|
||||
m_stacks.Delete(stackInd);
|
||||
}
|
||||
//cerr << m_stacks << endl;
|
||||
//cerr << m_stacks.Debug(mgr.system) << endl;
|
||||
}
|
||||
}
|
||||
|
||||
@ -73,7 +73,7 @@ void Search::Decode(size_t stackInd)
|
||||
return;
|
||||
}
|
||||
|
||||
const Hypotheses &hypos = stack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
const Hypotheses &hypos = stack.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
|
||||
//cerr << "hypos=" << hypos.size() << endl;
|
||||
|
||||
const InputPaths &paths = mgr.GetInputPaths();
|
||||
@ -140,20 +140,14 @@ void Search::Extend(const Hypothesis &hypo, const TargetPhraseImpl &tp,
|
||||
const Hypothesis *Search::GetBestHypo() const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
const Hypotheses &sortedHypos = lastStack.GetSortedAndPruneHypos(mgr,
|
||||
mgr.arcLists);
|
||||
|
||||
const Hypothesis *best = NULL;
|
||||
if (sortedHypos.size()) {
|
||||
best = static_cast<const Hypothesis*>(sortedHypos[0]);
|
||||
}
|
||||
const Hypothesis *best = lastStack.GetBestHypo<Hypothesis>();
|
||||
return best;
|
||||
}
|
||||
|
||||
void Search::AddInitialTrellisPaths(TrellisPaths<TrellisPath> &paths) const
|
||||
{
|
||||
const Stack &lastStack = m_stacks.Back();
|
||||
const Hypotheses &hypos = lastStack.GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
const Hypotheses &hypos = lastStack.GetSortedAndPrunedHypos(mgr, mgr.arcLists);
|
||||
|
||||
BOOST_FOREACH(const HypothesisBase *hypoBase, hypos){
|
||||
const Hypothesis *hypo = static_cast<const Hypothesis*>(hypoBase);
|
||||
|
@ -60,7 +60,7 @@ void Stacks::Add(Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
size_t numWordsCovered = hypo->GetBitmap().GetNumWordsCovered();
|
||||
//cerr << "numWordsCovered=" << numWordsCovered << endl;
|
||||
Stack &stack = *m_stacks[numWordsCovered];
|
||||
stack.Add(m_mgr.system, hypo, hypoRecycle, arcLists);
|
||||
stack.Add(m_mgr, hypo, hypoRecycle, arcLists);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -10,6 +10,18 @@ using namespace std;
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
PhraseImpl *PhraseImpl::CreateFromString(MemPool &pool, FactorCollection &vocab,
|
||||
const System &system, const std::string &str)
|
||||
{
|
||||
std::vector<std::string> toks = Moses2::Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
PhraseImpl *ret;
|
||||
|
||||
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
|
||||
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
return ret;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -9,17 +9,7 @@ class PhraseImpl: public PhraseImplTemplate<Word>
|
||||
{
|
||||
public:
|
||||
static PhraseImpl *CreateFromString(MemPool &pool, FactorCollection &vocab,
|
||||
const System &system, const std::string &str)
|
||||
{
|
||||
std::vector<std::string> toks = Moses2::Tokenize(str);
|
||||
size_t size = toks.size();
|
||||
PhraseImpl *ret;
|
||||
|
||||
ret = new (pool.Allocate<PhraseImpl>()) PhraseImpl(pool, size);
|
||||
|
||||
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks);
|
||||
return ret;
|
||||
}
|
||||
const System &system, const std::string &str);
|
||||
|
||||
PhraseImpl(MemPool &pool, size_t size) :
|
||||
PhraseImplTemplate<Word>(pool, size)
|
||||
|
@ -60,6 +60,7 @@ void ReorderingConstraint::FinalizeWalls()
|
||||
void ReorderingConstraint::SetWall( size_t pos, bool value )
|
||||
{
|
||||
//cerr << "SETTING reordering wall at position " << pos << std::endl;
|
||||
UTIL_THROW_IF2(pos >= m_size, "Wall over length of sentence: " << pos << " >= " << m_size);
|
||||
m_wall[pos] = value;
|
||||
m_active = true;
|
||||
}
|
||||
|
@ -39,6 +39,10 @@ bool Search::CanExtend(const Bitmap &hypoBitmap, size_t hypoRangeEndPos,
|
||||
return false;
|
||||
}
|
||||
|
||||
if (mgr.system.options.reordering.max_distortion == -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (mgr.system.options.reordering.max_distortion >= 0) {
|
||||
// distortion limit
|
||||
int distortion = ComputeDistortionDistance(hypoRangeEndPos,
|
||||
|
@ -84,7 +84,7 @@ Sentence *Sentence::CreateFromStringXML(MemPool &pool, FactorCollection &vocab,
|
||||
for(size_t i=0; i<xmlOptions.size(); i++) {
|
||||
const XMLOption *xmlOption = xmlOptions[i];
|
||||
if(strcmp(xmlOption->GetNodeName(), "wall") == 0) {
|
||||
UTIL_THROW_IF2(xmlOption->startPos >= ret->GetSize(), "wall is beyond the sentence"); // no buggy walls, please
|
||||
UTIL_THROW_IF2(xmlOption->startPos > ret->GetSize(), "wall is beyond the sentence"); // no buggy walls, please
|
||||
reorderingConstraint.SetWall(xmlOption->startPos - 1, true);
|
||||
}
|
||||
else if (strcmp(xmlOption->GetNodeName(), "zone") == 0) {
|
||||
|
@ -90,9 +90,8 @@ void Manager::Decode()
|
||||
}
|
||||
}
|
||||
|
||||
const Stack *stack;
|
||||
|
||||
/*
|
||||
const Stack *stack;
|
||||
stack = &m_stacks.GetStack(0, 5);
|
||||
cerr << "stack 0,12:" << stack->Debug(system) << endl;
|
||||
*/
|
||||
|
@ -33,7 +33,7 @@ void Stack::Add(SCFG::Hypothesis *hypo, Recycler<HypothesisBase*> &hypoRecycle,
|
||||
//cerr << "lhs=" << lhs << endl;
|
||||
|
||||
HypothesisColl &coll = GetColl(lhs);
|
||||
coll.Add(m_mgr.system, hypo, hypoRecycle, arcLists);
|
||||
coll.Add(m_mgr, hypo, hypoRecycle, arcLists);
|
||||
}
|
||||
|
||||
size_t Stack::GetSize() const
|
||||
@ -76,7 +76,7 @@ Moses2::HypothesisColl &Stack::GetColl(const SCFG::Word &nt)
|
||||
const Hypothesis *Stack::GetBestHypo() const
|
||||
{
|
||||
SCORE bestScore = -std::numeric_limits<SCORE>::infinity();
|
||||
const HypothesisBase *bestHypo;
|
||||
const HypothesisBase *bestHypo = NULL;
|
||||
BOOST_FOREACH(const Coll::value_type &val, m_coll){
|
||||
const Moses2::HypothesisColl &hypos = *val.second;
|
||||
const Moses2::HypothesisBase *hypo = hypos.GetBestHypo();
|
||||
|
@ -42,7 +42,7 @@ void KBestExtractor::OutputToStream(std::stringstream &strm)
|
||||
UTIL_THROW_IF2(lastStack.GetColl().size() != 1, "Only suppose to be 1 hypo coll in last stack");
|
||||
UTIL_THROW_IF2(lastStack.GetColl().begin()->second == NULL, "NULL hypo collection");
|
||||
|
||||
const Hypotheses &hypos = lastStack.GetColl().begin()->second->GetSortedAndPrunedHypos();
|
||||
const Hypotheses &hypos = lastStack.GetColl().begin()->second->GetSortedAndPrunedHypos(m_mgr, m_mgr.arcLists);
|
||||
UTIL_THROW_IF2(hypos.size() != 1, "Only suppose to be 1 hypo in collection");
|
||||
const HypothesisBase *hypo = hypos[0];
|
||||
|
||||
|
@ -50,7 +50,7 @@ NBest::NBest(
|
||||
}
|
||||
|
||||
stringstream strm;
|
||||
OutputToStream(mgr, strm, nbestColl);
|
||||
OutputToStream(mgr, strm);
|
||||
m_str = strm.str();
|
||||
}
|
||||
|
||||
@ -83,7 +83,7 @@ NBest::NBest(const SCFG::Manager &mgr,
|
||||
m_scores->PlusEquals(mgr.system, newScores);
|
||||
|
||||
stringstream strm;
|
||||
OutputToStream(mgr, strm, nbestColl);
|
||||
OutputToStream(mgr, strm);
|
||||
m_str = strm.str();
|
||||
}
|
||||
|
||||
@ -141,8 +141,7 @@ void NBest::CreateDeviants(
|
||||
|
||||
void NBest::OutputToStream(
|
||||
const SCFG::Manager &mgr,
|
||||
std::stringstream &strm,
|
||||
const NBestColl &nbestColl) const
|
||||
std::stringstream &strm) const
|
||||
{
|
||||
const SCFG::Hypothesis &hypo = GetHypo();
|
||||
//strm << &hypo << " ";
|
||||
|
@ -81,8 +81,7 @@ protected:
|
||||
|
||||
void OutputToStream(
|
||||
const SCFG::Manager &mgr,
|
||||
std::stringstream &strm,
|
||||
const NBestColl &nbestColl) const;
|
||||
std::stringstream &strm) const;
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
|
@ -86,7 +86,7 @@ public:
|
||||
|
||||
}
|
||||
|
||||
void OutputToStream(const System &system, const Phrase<Moses2::Word> &inputPhrase, std::ostream &out) const
|
||||
void OutputToStream(const System &system, const Phrase<WORD> &inputPhrase, std::ostream &out) const
|
||||
{
|
||||
// get placeholders
|
||||
FactorType placeholderFactor = system.options.input.placeholder_factor;
|
||||
|
@ -29,6 +29,7 @@
|
||||
#include <queue>
|
||||
#include <cstring>
|
||||
#include <cstdio>
|
||||
#include <boost/thread.hpp>
|
||||
|
||||
#include "MurmurHash3.h"
|
||||
#include "StringVector.h"
|
||||
|
@ -87,8 +87,8 @@ void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const
|
||||
|
||||
if (SatisfyBackoff(mgr, *path)) {
|
||||
TargetPhrases *tpsPtr = Lookup(mgr, mgr.GetPool(), *path);
|
||||
cerr << "tpsPtr=" << tpsPtr << " ";
|
||||
/*
|
||||
cerr << "tpsPtr=" << tpsPtr << " ";
|
||||
if (tps.get()) {
|
||||
cerr << tps.get()->GetSize();
|
||||
}
|
||||
@ -154,7 +154,7 @@ void PhraseTable::LookupNT(
|
||||
BOOST_FOREACH (const SCFG::Stack::Coll::value_type &valPair, stackColl) {
|
||||
const SCFG::Word &ntSought = valPair.first;
|
||||
const Moses2::HypothesisColl *hypos = valPair.second;
|
||||
const Moses2::Hypotheses &sortedHypos = hypos->GetSortedAndPruneHypos(mgr, mgr.arcLists);
|
||||
const Moses2::Hypotheses &sortedHypos = hypos->GetSortedAndPrunedHypos(mgr, mgr.arcLists);
|
||||
//cerr << "ntSought=" << ntSought << ntSought.isNonTerminal << endl;
|
||||
LookupGivenWord(pool, mgr, prevPath, ntSought, &sortedHypos, subPhraseRange, outPath);
|
||||
}
|
||||
|
@ -69,6 +69,7 @@ std::pair<bool, uint64_t> ProbingPT::ActiveChartEntryProbing::GetKey(const SCFG:
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
ProbingPT::ProbingPT(size_t startInd, const std::string &line)
|
||||
:PhraseTable(startInd, line)
|
||||
,load_method(util::POPULATE_OR_READ)
|
||||
{
|
||||
ReadParameters();
|
||||
}
|
||||
@ -80,7 +81,7 @@ ProbingPT::~ProbingPT()
|
||||
|
||||
void ProbingPT::Load(System &system)
|
||||
{
|
||||
m_engine = new QueryEngine(m_path.c_str());
|
||||
m_engine = new QueryEngine(m_path.c_str(), load_method);
|
||||
|
||||
m_unkId = 456456546456;
|
||||
|
||||
@ -149,6 +150,24 @@ void ProbingPT::Load(System &system)
|
||||
CreateCache(system);
|
||||
}
|
||||
|
||||
void ProbingPT::SetParameter(const std::string& key, const std::string& value)
|
||||
{
|
||||
if (key == "load") {
|
||||
if (value == "lazy") {
|
||||
load_method = util::LAZY;
|
||||
}
|
||||
else if (value == "populate") {
|
||||
load_method = util::POPULATE_OR_READ;
|
||||
}
|
||||
else {
|
||||
UTIL_THROW2("load method not supported" << value);
|
||||
}
|
||||
}
|
||||
else {
|
||||
PhraseTable::SetParameter(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
void ProbingPT::CreateAlignmentMap(System &system, const std::string path)
|
||||
{
|
||||
const std::vector< std::vector<unsigned char> > &probingAlignColl = m_engine->getAlignments();
|
||||
|
@ -15,6 +15,7 @@
|
||||
#include "../../Vector.h"
|
||||
#include "../../Phrase.h"
|
||||
#include "../../SCFG/ActiveChart.h"
|
||||
#include "util/mmap.hh"
|
||||
|
||||
namespace Moses2
|
||||
{
|
||||
@ -69,6 +70,7 @@ public:
|
||||
virtual ~ProbingPT();
|
||||
void Load(System &system);
|
||||
|
||||
virtual void SetParameter(const std::string& key, const std::string& value);
|
||||
void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const;
|
||||
|
||||
uint64_t GetUnk() const
|
||||
@ -91,6 +93,7 @@ protected:
|
||||
std::vector<uint64_t> m_sourceVocab; // factor id -> pt id
|
||||
std::vector< std::pair<bool, const Factor*> > m_targetVocab; // pt id -> factor*
|
||||
std::vector<const AlignmentInfo*> m_aligns;
|
||||
util::LoadMethod load_method;
|
||||
|
||||
uint64_t m_unkId;
|
||||
QueryEngine *m_engine;
|
||||
|
@ -7,7 +7,7 @@ using namespace std;
|
||||
namespace Moses2
|
||||
{
|
||||
|
||||
QueryEngine::QueryEngine(const char * filepath)
|
||||
QueryEngine::QueryEngine(const char * filepath, util::LoadMethod load_method)
|
||||
{
|
||||
|
||||
//Create filepaths
|
||||
@ -17,8 +17,10 @@ QueryEngine::QueryEngine(const char * filepath)
|
||||
std::string path_to_source_vocabid = basepath + "/source_vocabids";
|
||||
std::string alignPath = basepath + "/Alignments.dat";
|
||||
|
||||
if (!FileExists(path_to_config)) {
|
||||
UTIL_THROW2("Binary table doesn't exist is didn't finish binarizing: " << path_to_config);
|
||||
file_exits(basepath);
|
||||
|
||||
if (load_method == util::POPULATE_OR_READ) {
|
||||
cat_files(basepath);
|
||||
}
|
||||
|
||||
///Source phrase vocabids
|
||||
@ -139,5 +141,46 @@ void QueryEngine::read_alignments(const std::string &alignPath)
|
||||
}
|
||||
}
|
||||
|
||||
void QueryEngine::file_exits(const std::string &basePath)
|
||||
{
|
||||
if (!FileExists(basePath + "/Alignments.dat")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/Alignments.dat");
|
||||
}
|
||||
if (!FileExists(basePath + "/TargetColl.dat")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetColl.dat");
|
||||
}
|
||||
if (!FileExists(basePath + "/TargetVocab.dat")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetVocab.dat");
|
||||
}
|
||||
if (!FileExists(basePath + "/cache")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/cache");
|
||||
}
|
||||
if (!FileExists(basePath + "/config")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/config");
|
||||
}
|
||||
if (!FileExists(basePath + "/probing_hash.dat")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/probing_hash.dat");
|
||||
}
|
||||
if (!FileExists(basePath + "/source_vocabids")) {
|
||||
UTIL_THROW2("Require file does not exist in: " << basePath << "/source_vocabids");
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
if (!FileExists(path_to_config) || !FileExists(path_to_hashtable) ||
|
||||
!FileExists(path_to_source_vocabid) || !FileExists(basepath + alignPath) ||
|
||||
!FileExists(basepath + "/TargetColl.dat") || !FileExists(basepath + "/TargetVocab.dat") ||
|
||||
!FileExists(basepath + "/cache")) {
|
||||
UTIL_THROW2("A required table doesn't exist in: " << basepath);
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
void QueryEngine::cat_files(const std::string &basePath)
|
||||
{
|
||||
system((string("cat ") + basePath + "/TargetColl.dat > /dev/null").c_str());
|
||||
system((string("cat ") + basePath + "/probing_hash.dat > /dev/null").c_str());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -27,13 +27,15 @@ class QueryEngine
|
||||
bool is_reordering;
|
||||
|
||||
void read_alignments(const std::string &alignPath);
|
||||
void file_exits(const std::string &basePath);
|
||||
void cat_files(const std::string &basePath);
|
||||
|
||||
public:
|
||||
int num_scores;
|
||||
int num_lex_scores;
|
||||
bool logProb;
|
||||
|
||||
QueryEngine(const char *);
|
||||
QueryEngine(const char *, util::LoadMethod load_method);
|
||||
~QueryEngine();
|
||||
|
||||
std::pair<bool, uint64_t> query(uint64_t key);
|
||||
|
@ -36,16 +36,6 @@ void Weights::Init(const FeatureFunctions &ffs)
|
||||
m_weights.resize(totalNumScores, 1);
|
||||
}
|
||||
|
||||
std::ostream &Weights::Debug(std::ostream &out, const System &system) const
|
||||
{
|
||||
const FeatureFunctions &ffs = system.featureFunctions;
|
||||
size_t numScores = ffs.GetNumScores();
|
||||
for (size_t i = 0; i < numScores; ++i) {
|
||||
out << m_weights[i] << " ";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::vector<SCORE> Weights::GetWeights(const FeatureFunction &ff) const
|
||||
{
|
||||
std::vector<SCORE> ret(m_weights.begin() + ff.GetStartInd(), m_weights.begin() + ff.GetStartInd() + ff.GetNumScores());
|
||||
|
@ -27,8 +27,6 @@ public:
|
||||
return m_weights[ind];
|
||||
}
|
||||
|
||||
std::ostream &Debug(std::ostream &out, const System &system) const;
|
||||
|
||||
std::vector<SCORE> GetWeights(const FeatureFunction &ff) const;
|
||||
|
||||
void SetWeights(const FeatureFunctions &ffs, const std::string &ffName, const std::vector<float> &weights);
|
||||
|
@ -30,6 +30,7 @@
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <ostream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include "util/exception.hh"
|
||||
|
||||
|
@ -79,8 +79,8 @@ Parameter::Parameter()
|
||||
desc += "8=tree-to-string (SCFG-based)\n";
|
||||
desc += "9=forest-to-string";
|
||||
AddParam(search_opts, "search-algorithm", desc);
|
||||
//AddParam(search_opts, "beam-threshold", "b",
|
||||
// "threshold for threshold pruning");
|
||||
AddParam(search_opts, "beam-threshold", "b",
|
||||
"threshold for threshold pruning");
|
||||
//AddParam(search_opts, "early-discarding-threshold", "edt",
|
||||
// "threshold for constructing hypotheses based on estimate cost");
|
||||
AddParam(search_opts, "stack", "s",
|
||||
|
@ -5,16 +5,16 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.602770742" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.602770742" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.602770742" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.602770742." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1436139469" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
|
||||
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.622899770" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
|
||||
@ -78,6 +78,7 @@
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/util/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/moses/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/lm/Debug""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.2077999464" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
@ -97,16 +98,16 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.168814843" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.168814843" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.168814843" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.release.168814843." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.844577457" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
|
||||
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.1635721038" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
|
||||
|
@ -5,16 +5,16 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.2140911490" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1880937408" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
@ -40,6 +40,7 @@
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/moses/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/util/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.libs.1975921106" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="moses"/>
|
||||
@ -74,16 +75,16 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1969822952" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
|
||||
<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1371996557" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
|
||||
|
@ -1,162 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
|
||||
<storageModule moduleId="org.eclipse.cdt.core.settings">
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.772724541">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.772724541" moduleId="org.eclipse.cdt.core.settings" name="Debug">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.772724541" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.772724541." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.935075355" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
|
||||
<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.debug.401992972" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
|
||||
<builder buildPath="${workspace_loc:/CreateProbingPT2}/Debug" id="cdt.managedbuild.target.gnu.builder.exe.debug.480448121" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.archiver.base.113368746" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.95199540" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.840000212" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1317589706" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.include.paths.1471648270" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
|
||||
<listOptionValue builtIn="false" value="/opt/local/include/"/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../boost/include""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../..""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../cmph/include""/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1223888597" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1702191805" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.477082967" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.exe.debug.option.debugging.level.1500261650" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.option.dialect.std.1780261151" name="Language standard" superClass="gnu.c.compiler.option.dialect.std" value="gnu.c.compiler.dialect.default" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1966462062" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1972061651" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.46306402" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
|
||||
<option id="gnu.cpp.link.option.libs.663062619" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="moses"/>
|
||||
<listOptionValue builtIn="false" value="moses2"/>
|
||||
<listOptionValue builtIn="false" value="cmph"/>
|
||||
<listOptionValue builtIn="false" value="search"/>
|
||||
<listOptionValue builtIn="false" value="OnDiskPt"/>
|
||||
<listOptionValue builtIn="false" value="lm"/>
|
||||
<listOptionValue builtIn="false" value="util"/>
|
||||
<listOptionValue builtIn="false" value="boost_iostreams"/>
|
||||
<listOptionValue builtIn="false" value="boost_serialization"/>
|
||||
<listOptionValue builtIn="false" value="boost_system"/>
|
||||
<listOptionValue builtIn="false" value="boost_thread"/>
|
||||
<listOptionValue builtIn="false" value="boost_filesystem"/>
|
||||
<listOptionValue builtIn="false" value="boost_program_options"/>
|
||||
<listOptionValue builtIn="false" value="pthread"/>
|
||||
<listOptionValue builtIn="false" value="z"/>
|
||||
<listOptionValue builtIn="false" value="bz2"/>
|
||||
<listOptionValue builtIn="false" value="dl"/>
|
||||
<listOptionValue builtIn="false" value="rt"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.paths.1171438767" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../cmph/lib""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/moses/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../moses2/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/lm/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/OnDiskPt/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/util/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/search/Debug""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1132334401" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.assembler.exe.debug.1524531182" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.debug">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.342176648" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
<cconfiguration id="cdt.managedbuild.config.gnu.exe.release.45069368">
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.45069368" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.45069368" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.release.45069368." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.1393364417" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
|
||||
<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.release.909144420" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
|
||||
<builder buildPath="${workspace_loc:/CreateProbingPT2}/Release" id="cdt.managedbuild.target.gnu.builder.exe.release.753583150" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.exe.release"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.archiver.base.635946464" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1512170522" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release">
|
||||
<option id="gnu.cpp.compiler.exe.release.option.optimization.level.1721853258" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.exe.release.option.debugging.level.1768982295" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1955793337" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.release.250602445" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.release">
|
||||
<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.exe.release.option.optimization.level.717790117" name="Optimization Level" superClass="gnu.c.compiler.exe.release.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.exe.release.option.debugging.level.1481719615" name="Debug Level" superClass="gnu.c.compiler.exe.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.814578542" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.release.747202810" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.release"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.release.1133208340" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.release">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.306092624" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
<additionalInput kind="additionalinput" paths="$(LIBS)"/>
|
||||
</inputType>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.assembler.exe.release.2102139460" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.release">
|
||||
<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1298181016" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
|
||||
</tool>
|
||||
</toolChain>
|
||||
</folderInfo>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
|
||||
</cconfiguration>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<project id="CreateProbingPT2.cdt.managedbuild.target.gnu.exe.294510828" name="Executable" projectType="cdt.managedbuild.target.gnu.exe"/>
|
||||
</storageModule>
|
||||
<storageModule moduleId="scannerConfiguration">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.release.45069368;cdt.managedbuild.config.gnu.exe.release.45069368.;cdt.managedbuild.tool.gnu.c.compiler.exe.release.250602445;cdt.managedbuild.tool.gnu.c.compiler.input.814578542">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.debug.772724541;cdt.managedbuild.config.gnu.exe.debug.772724541.;cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.95199540;cdt.managedbuild.tool.gnu.cpp.compiler.input.1223888597">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.release.45069368;cdt.managedbuild.config.gnu.exe.release.45069368.;cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1512170522;cdt.managedbuild.tool.gnu.cpp.compiler.input.1955793337">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
</scannerConfigBuildInfo>
|
||||
<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.debug.772724541;cdt.managedbuild.config.gnu.exe.debug.772724541.;cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1702191805;cdt.managedbuild.tool.gnu.c.compiler.input.1966462062">
|
||||
<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
|
||||
</scannerConfigBuildInfo>
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
|
||||
<storageModule moduleId="refreshScope" versionNumber="2">
|
||||
<configuration configurationName="Release">
|
||||
<resource resourceType="PROJECT" workspacePath="/CreateProbingPT2"/>
|
||||
</configuration>
|
||||
<configuration configurationName="Debug">
|
||||
<resource resourceType="PROJECT" workspacePath="/CreateProbingPT2"/>
|
||||
</configuration>
|
||||
</storageModule>
|
||||
</cproject>
|
@ -1,38 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>CreateProbingPT2</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
<project>lm</project>
|
||||
<project>moses</project>
|
||||
<project>moses2</project>
|
||||
<project>util</project>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<triggers>full,incremental,</triggers>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
</natures>
|
||||
<linkedResources>
|
||||
<link>
|
||||
<name>CreateProbingPT2.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-2-PROJECT_LOC/moses2/CreateProbingPT2.cpp</locationURI>
|
||||
</link>
|
||||
</linkedResources>
|
||||
</projectDescription>
|
@ -11,12 +11,12 @@
|
||||
</externalSetting>
|
||||
</externalSettings>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
@ -72,13 +72,13 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.macosx.exe.release.701931933" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
|
@ -61,6 +61,7 @@
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/search/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/lm/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/OnDiskPt/Debug""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1093223502" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
|
@ -14,7 +14,7 @@
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1133345948" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1133345948" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.1133345948." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1405862229" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
|
||||
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.605722566" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
|
||||
@ -40,6 +40,7 @@
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/mert_lib/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/util/Debug""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.libs.585257079" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="mert_lib"/>
|
||||
@ -77,7 +78,7 @@
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.1385955159" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.1385955159" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.release.1385955159." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.887500021" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
|
||||
<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.1965146498" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
|
||||
|
@ -11,11 +11,11 @@
|
||||
</externalSetting>
|
||||
</externalSettings>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.GNU_ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
@ -26,15 +26,15 @@
|
||||
<builder buildPath="${workspace_loc:/moses}/Debug" id="cdt.managedbuild.target.gnu.builder.exe.debug.1778877633" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.archiver.base.1097285966" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
|
||||
<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1729217620" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1455257477" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.227767392" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.include.paths.876218169" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1455257477" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.227767392" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.include.paths.876218169" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../..""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../boost/include""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../cmph/include""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc}/../../xmlrpc-c/include""/>
|
||||
</option>
|
||||
<option id="gnu.cpp.compiler.option.preprocessor.def.53427549" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
|
||||
<option id="gnu.cpp.compiler.option.preprocessor.def.53427549" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
|
||||
<listOptionValue builtIn="false" value="HAVE_XMLRPC_C"/>
|
||||
<listOptionValue builtIn="false" value="PT_UG"/>
|
||||
<listOptionValue builtIn="false" value="MOSES_VERSION_ID=0"/>
|
||||
@ -45,12 +45,12 @@
|
||||
<listOptionValue builtIn="false" value="_FILE_OFFSET_BITS=64"/>
|
||||
<listOptionValue builtIn="false" value="_LARGE_FILES"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.compiler.option.dialect.std.1869920346" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
|
||||
<option id="gnu.cpp.compiler.option.dialect.std.1869920346" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" useByScannerDiscovery="true" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1023855536" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1313249282" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.146557271" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.exe.debug.option.debugging.level.1656486500" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.146557271" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
|
||||
<option id="gnu.c.compiler.exe.debug.option.debugging.level.1656486500" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.max" valueType="enumerated"/>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.570559630" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
|
||||
</tool>
|
||||
<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1471271407" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
|
||||
@ -86,12 +86,12 @@
|
||||
<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.1911984684" moduleId="org.eclipse.cdt.core.settings" name="Release">
|
||||
<externalSettings/>
|
||||
<extensions>
|
||||
<extension id="org.eclipse.cdt.core.GNU_ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
|
||||
<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
|
@ -565,6 +565,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/OutputCollector.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/OutputFileStream.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>OutputFileStream.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/OutputFileStream.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>PCNTools.cpp</name>
|
||||
<type>1</type>
|
||||
@ -2175,6 +2185,16 @@
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h</name>
|
||||
<type>1</type>
|
||||
<locationURI>PARENT-3-PROJECT_LOC/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/PhraseDictionaryMultiModel.cpp</name>
|
||||
<type>1</type>
|
||||
|
@ -15,7 +15,7 @@
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1015532240" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1015532240" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.1015532240." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1201298107" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
|
||||
<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.debug.2097807873" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
|
||||
@ -52,6 +52,7 @@
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/lm/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/util/Debug""/>
|
||||
<listOptionValue builtIn="false" value=""${workspace_loc:}/../../boost/lib64""/>
|
||||
<listOptionValue builtIn="false" value="/opt/local/lib"/>
|
||||
</option>
|
||||
<option id="gnu.cpp.link.option.libs.1087215166" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
|
||||
<listOptionValue builtIn="false" value="moses"/>
|
||||
@ -111,7 +112,7 @@
|
||||
</extensions>
|
||||
</storageModule>
|
||||
<storageModule moduleId="cdtBuildSystem" version="4.0.0">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.179761083" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.179761083" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
|
||||
<folderInfo id="cdt.managedbuild.config.gnu.exe.release.179761083." name="/" resourcePath="">
|
||||
<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.2024222442" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
|
||||
<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.release.1098252145" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
|
||||
|
@ -15,7 +15,7 @@ size_t BidirectionalReorderingState::hash() const
|
||||
|
||||
bool BidirectionalReorderingState::operator==(const FFState& o) const
|
||||
{
|
||||
if (&o == this) return 0;
|
||||
if (&o == this) return true;
|
||||
|
||||
BidirectionalReorderingState const &other
|
||||
= static_cast<BidirectionalReorderingState const&>(o);
|
||||
|
@ -42,6 +42,7 @@ namespace Moses
|
||||
{
|
||||
LanguageModelImplementation::LanguageModelImplementation(const std::string &line)
|
||||
:LanguageModel(line)
|
||||
,m_nGramOrder(NOT_FOUND)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -11,6 +11,9 @@ SkeletonLM::SkeletonLM(const std::string &line)
|
||||
{
|
||||
ReadParameters();
|
||||
|
||||
UTIL_THROW_IF2(m_nGramOrder == NOT_FOUND, "Must set order");
|
||||
UTIL_THROW_IF2(m_nGramOrder <= 1, "Ngram order must be more than 1");
|
||||
|
||||
FactorCollection &factorCollection = FactorCollection::Instance();
|
||||
|
||||
// needed by parent language model classes. Why didn't they set these themselves?
|
||||
|
@ -32,6 +32,7 @@
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <ostream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include "Util.h"
|
||||
#include "util/exception.hh"
|
||||
|
@ -22,7 +22,8 @@ void PhraseDictionaryMemoryPerSentenceOnDemand::Load(AllOptions::ptr const& opts
|
||||
}
|
||||
|
||||
|
||||
TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const {
|
||||
TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const
|
||||
{
|
||||
|
||||
Coll &coll = GetColl();
|
||||
|
||||
|
@ -36,7 +36,7 @@ namespace sapt
|
||||
#ifndef SPTR
|
||||
#define SPTR boost::shared_ptr
|
||||
#endif
|
||||
#define iptr boost::intrusive_ptr
|
||||
#define boost_iptr boost::intrusive_ptr
|
||||
#define scoptr boost::scoped_ptr
|
||||
#define rcast reinterpret_cast
|
||||
#endif
|
||||
|
@ -76,7 +76,7 @@ int main(int argc, char* argv[])
|
||||
{
|
||||
typedef vector<PhrasePair<Token> > pplist_t;
|
||||
interpret_args(argc, argv);
|
||||
iptr<mmbitext> Bptr(new mmbitext);
|
||||
boost_iptr<mmbitext> Bptr(new mmbitext);
|
||||
mmbitext& B = *Bptr;// static_cast<mmbitext*>(Bptr.get());
|
||||
B.open(bname, L1, L2);
|
||||
B.V1->setDynamic(true);
|
||||
|
@ -176,6 +176,13 @@ sub ngrams {
|
||||
return { md5(encode_utf8($sent)) => 1 };
|
||||
} else {
|
||||
my @words = split /\s+/, $sent;
|
||||
|
||||
#factors
|
||||
if ( $sent =~ m/[|]/) {
|
||||
my $use_index = 0; # default factor is the first one
|
||||
@words = map { ( split /[|]/, $_ ) [$use_index] } @words;
|
||||
}
|
||||
|
||||
my $out;
|
||||
if ($n == 1) {
|
||||
foreach my $w (@words) {
|
||||
|
@ -19,6 +19,8 @@ binmode STDERR, ":utf8";
|
||||
# version 13a
|
||||
# * modified the scoring functions to prevent division-by-zero errors when a system segment is empty
|
||||
# * affected methods: 'bleu_score' and 'bleu_score_smoothing'
|
||||
# * use \p{Line_Breaks} instead of \p{Hyphen} when stripping end-of-line hyphenation and join lines
|
||||
# * because \p{Hyphen} is deprecated since 2016-06-01, see http://www.unicode.org/reports/tr14/#Hyphen
|
||||
#
|
||||
# version 13
|
||||
# * Uses a XML parser to read data (only when extension is .xml)
|
||||
@ -948,7 +950,7 @@ sub tokenization_international
|
||||
my ($norm_text) = @_;
|
||||
|
||||
$norm_text =~ s/<skipped>//g; # strip "skipped" tags
|
||||
$norm_text =~ s/\p{Hyphen}\p{Zl}//g; # strip end-of-line hyphenation and join lines
|
||||
$norm_text =~ s/\p{Line_Break}\p{Zl}//g; # strip end-of-line hyphenation and join lines
|
||||
$norm_text =~ s/\p{Zl}/ /g; # join lines
|
||||
|
||||
# replace entities
|
||||
|
@ -88,7 +88,7 @@ sub recognize {
|
||||
$isRecognized = 0;
|
||||
}
|
||||
|
||||
if ($end == length($input) -1 || substr($input, $end, 1) eq " ") {
|
||||
if ($end == length($input) || substr($input, $end, 1) eq " ") {
|
||||
# last word, or next char is a space
|
||||
}
|
||||
else {
|
||||
|
@ -243,9 +243,9 @@ sub tokenize
|
||||
my @protected = ();
|
||||
foreach my $protected_pattern (@protected_patterns) {
|
||||
my $t = $text;
|
||||
while ($t =~ /($protected_pattern)(.*)$/) {
|
||||
push @protected, $1;
|
||||
$t = $2;
|
||||
while ($t =~ /(?<PATTERN>$protected_pattern)(?<TAIL>.*)$/) {
|
||||
push @protected, $+{PATTERN};
|
||||
$t = $+{TAIL};
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user