mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
all hypotheses have translation option. All translation options have target phrase
This commit is contained in:
parent
a83c166678
commit
24e6d6c272
@ -45,21 +45,20 @@ namespace Moses
|
||||
ObjectPool<Hypothesis> Hypothesis::s_objectPool("Hypothesis", 300000);
|
||||
#endif
|
||||
|
||||
Hypothesis::Hypothesis(Manager& manager, InputType const& source, const TargetPhrase &emptyTarget)
|
||||
Hypothesis::Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt)
|
||||
: m_prevHypo(NULL)
|
||||
, m_targetPhrase(emptyTarget)
|
||||
, m_sourceCompleted(source.GetSize(), manager.m_source.m_sourceCompleted)
|
||||
, m_sourceInput(source)
|
||||
, m_currSourceWordsRange(
|
||||
m_sourceCompleted.GetFirstGapPos()>0 ? 0 : NOT_FOUND,
|
||||
m_sourceCompleted.GetFirstGapPos()>0 ? m_sourceCompleted.GetFirstGapPos()-1 : NOT_FOUND)
|
||||
, m_currTargetWordsRange(0, emptyTarget.GetSize()-1)
|
||||
, m_currTargetWordsRange(NOT_FOUND, NOT_FOUND)
|
||||
, m_wordDeleted(false)
|
||||
, m_totalScore(0.0f)
|
||||
, m_futureScore(0.0f)
|
||||
, m_ffStates(StatefulFeatureFunction::GetStatefulFeatureFunctions().size())
|
||||
, m_arcList(NULL)
|
||||
, m_transOpt(NULL)
|
||||
, m_transOpt(initialTransOpt)
|
||||
, m_manager(manager)
|
||||
, m_id(m_manager.GetNextHypoId())
|
||||
{
|
||||
@ -78,7 +77,6 @@ Hypothesis::Hypothesis(Manager& manager, InputType const& source, const TargetPh
|
||||
*/
|
||||
Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt)
|
||||
: m_prevHypo(&prevHypo)
|
||||
, m_targetPhrase(transOpt.GetTargetPhrase())
|
||||
, m_sourceCompleted (prevHypo.m_sourceCompleted )
|
||||
, m_sourceInput (prevHypo.m_sourceInput)
|
||||
, m_currSourceWordsRange (transOpt.GetSourceWordsRange())
|
||||
@ -90,7 +88,7 @@ Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &tran
|
||||
, m_scoreBreakdown(prevHypo.GetScoreBreakdown())
|
||||
, m_ffStates(prevHypo.m_ffStates.size())
|
||||
, m_arcList(NULL)
|
||||
, m_transOpt(&transOpt)
|
||||
, m_transOpt(transOpt)
|
||||
, m_manager(prevHypo.GetManager())
|
||||
, m_id(m_manager.GetNextHypoId())
|
||||
{
|
||||
@ -213,13 +211,13 @@ Hypothesis* Hypothesis::Create(const Hypothesis &prevHypo, const TranslationOpti
|
||||
* return the subclass of Hypothesis most appropriate to the given target phrase
|
||||
*/
|
||||
|
||||
Hypothesis* Hypothesis::Create(Manager& manager, InputType const& m_source, const TargetPhrase &emptyTarget)
|
||||
Hypothesis* Hypothesis::Create(Manager& manager, InputType const& m_source, const TranslationOption &initialTransOpt)
|
||||
{
|
||||
#ifdef USE_HYPO_POOL
|
||||
Hypothesis *ptr = s_objectPool.getPtr();
|
||||
return new(ptr) Hypothesis(manager, m_source, emptyTarget);
|
||||
return new(ptr) Hypothesis(manager, m_source, initialTransOpt);
|
||||
#else
|
||||
return new Hypothesis(manager, m_source, emptyTarget);
|
||||
return new Hypothesis(manager, m_source, initialTransOpt);
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -332,7 +330,7 @@ void Hypothesis::PrintHypothesis() const
|
||||
return;
|
||||
}
|
||||
TRACE_ERR(endl << "creating hypothesis "<< m_id <<" from "<< m_prevHypo->m_id<<" ( ");
|
||||
int end = (int)(m_prevHypo->m_targetPhrase.GetSize()-1);
|
||||
int end = (int)(m_prevHypo->GetCurrTargetPhrase().GetSize()-1);
|
||||
int start = end-1;
|
||||
if ( start < 0 ) start = 0;
|
||||
if ( m_prevHypo->m_currTargetWordsRange.GetStartPos() == NOT_FOUND ) {
|
||||
@ -342,17 +340,14 @@ void Hypothesis::PrintHypothesis() const
|
||||
}
|
||||
if (end>=0) {
|
||||
WordsRange range(start, end);
|
||||
TRACE_ERR( m_prevHypo->m_targetPhrase.GetSubString(range) << " ");
|
||||
TRACE_ERR( m_prevHypo->GetCurrTargetPhrase().GetSubString(range) << " ");
|
||||
}
|
||||
TRACE_ERR( ")"<<endl);
|
||||
TRACE_ERR( "\tbase score "<< (m_prevHypo->m_totalScore - m_prevHypo->m_futureScore) <<endl);
|
||||
TRACE_ERR( "\tcovering "<<m_currSourceWordsRange.GetStartPos()<<"-"<<m_currSourceWordsRange.GetEndPos()<<": ");
|
||||
TRACE_ERR( "\tcovering "<<m_currSourceWordsRange.GetStartPos()<<"-"<<m_currSourceWordsRange.GetEndPos()
|
||||
<<": " << m_transOpt.GetSourcePhrase() << endl);
|
||||
|
||||
if (m_transOpt) {
|
||||
TRACE_ERR(m_transOpt->GetSourcePhrase());
|
||||
}
|
||||
TRACE_ERR(endl);
|
||||
TRACE_ERR( "\ttranslated as: "<<(Phrase&) m_targetPhrase<<endl); // <<" => translation cost "<<m_score[ScoreType::PhraseTrans];
|
||||
TRACE_ERR( "\ttranslated as: "<<(Phrase&) GetCurrTargetPhrase()<<endl); // <<" => translation cost "<<m_score[ScoreType::PhraseTrans];
|
||||
|
||||
if (m_wordDeleted) TRACE_ERR( "\tword deleted"<<endl);
|
||||
// TRACE_ERR( "\tdistance: "<<GetCurrSourceWordsRange().CalcDistortion(m_prevHypo->GetCurrSourceWordsRange())); // << " => distortion cost "<<(m_score[ScoreType::Distortion]*weightDistortion)<<endl;
|
||||
@ -403,6 +398,10 @@ void Hypothesis::CleanupArcList()
|
||||
}
|
||||
}
|
||||
|
||||
const TargetPhrase &Hypothesis::GetCurrTargetPhrase() const {
|
||||
return m_transOpt.GetTargetPhrase();
|
||||
}
|
||||
|
||||
void Hypothesis::GetOutputPhrase(Phrase &out) const
|
||||
{
|
||||
if (m_prevHypo != NULL) {
|
||||
@ -439,22 +438,15 @@ ostream& operator<<(ostream& out, const Hypothesis& hypo)
|
||||
|
||||
std::string Hypothesis::GetSourcePhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{
|
||||
if (!m_prevHypo) {
|
||||
return "";
|
||||
}
|
||||
if (m_transOpt) {
|
||||
return m_transOpt->GetSourcePhrase().GetStringRep(factorsToPrint);
|
||||
}
|
||||
else {
|
||||
return "";
|
||||
}
|
||||
return m_transOpt.GetSourcePhrase().GetStringRep(factorsToPrint);
|
||||
}
|
||||
|
||||
std::string Hypothesis::GetTargetPhraseStringRep(const vector<FactorType> factorsToPrint) const
|
||||
{
|
||||
if (!m_prevHypo) {
|
||||
return "";
|
||||
}
|
||||
return m_targetPhrase.GetStringRep(factorsToPrint);
|
||||
return GetCurrTargetPhrase().GetStringRep(factorsToPrint);
|
||||
}
|
||||
|
||||
std::string Hypothesis::GetSourcePhraseStringRep() const
|
||||
|
@ -69,7 +69,6 @@ protected:
|
||||
|
||||
const Hypothesis* m_prevHypo; /*! backpointer to previous hypothesis (from which this one was created) */
|
||||
// const Phrase &m_targetPhrase; /*! target phrase being created at the current decoding step */
|
||||
const TargetPhrase &m_targetPhrase; /*! target phrase being created at the current decoding step */
|
||||
WordsBitmap m_sourceCompleted; /*! keeps track of which words have been translated so far */
|
||||
//TODO: how to integrate this into confusion network framework; what if
|
||||
//it's a confusion network in the end???
|
||||
@ -83,13 +82,13 @@ protected:
|
||||
std::vector<const FFState*> m_ffStates;
|
||||
const Hypothesis *m_winningHypo;
|
||||
ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */
|
||||
const TranslationOption *m_transOpt;
|
||||
const TranslationOption &m_transOpt;
|
||||
Manager& m_manager;
|
||||
|
||||
int m_id; /*! numeric ID of this hypothesis, used for logging */
|
||||
|
||||
/*! used by initial seeding of the translation process */
|
||||
Hypothesis(Manager& manager, InputType const& source, const TargetPhrase &emptyTarget);
|
||||
Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt);
|
||||
/*! used when creating a new hypothesis using a translation option (phrase translation) */
|
||||
Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt);
|
||||
|
||||
@ -106,7 +105,7 @@ public:
|
||||
static Hypothesis* Create(Manager& manager, const WordsBitmap &initialCoverage);
|
||||
|
||||
/** return the subclass of Hypothesis most appropriate to the given target phrase */
|
||||
static Hypothesis* Create(Manager& manager, InputType const& source, const TargetPhrase &emptyTarget);
|
||||
static Hypothesis* Create(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt);
|
||||
|
||||
/** return the subclass of Hypothesis most appropriate to the given translation option */
|
||||
Hypothesis* CreateNext(const TranslationOption &transOpt, const Phrase* constraint) const;
|
||||
@ -119,10 +118,7 @@ public:
|
||||
|
||||
/** return target phrase used to create this hypothesis */
|
||||
// const Phrase &GetCurrTargetPhrase() const
|
||||
const TargetPhrase &GetCurrTargetPhrase() const {
|
||||
return m_targetPhrase;
|
||||
}
|
||||
|
||||
const TargetPhrase &GetCurrTargetPhrase() const;
|
||||
|
||||
/** return input positions covered by the translation option (phrasal translation) used to create this hypothesis */
|
||||
inline const WordsRange &GetCurrSourceWordsRange() const {
|
||||
@ -164,10 +160,10 @@ public:
|
||||
* (ie, start of sentence would be some negative number, which is
|
||||
* not allowed- USE WITH CAUTION) */
|
||||
inline const Word &GetCurrWord(size_t pos) const {
|
||||
return m_targetPhrase.GetWord(pos);
|
||||
return GetCurrTargetPhrase().GetWord(pos);
|
||||
}
|
||||
inline const Factor *GetCurrFactor(size_t pos, FactorType factorType) const {
|
||||
return m_targetPhrase.GetFactor(pos, factorType);
|
||||
return GetCurrTargetPhrase().GetFactor(pos, factorType);
|
||||
}
|
||||
/** recursive - pos is relative from start of sentence */
|
||||
inline const Word &GetWord(size_t pos) const {
|
||||
@ -260,7 +256,7 @@ public:
|
||||
}
|
||||
|
||||
const TranslationOption &GetTranslationOption() const {
|
||||
return *m_transOpt;
|
||||
return m_transOpt;
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -36,7 +36,7 @@ MockHypothesisGuard::MockHypothesisGuard(
|
||||
const string& sourceSentence,
|
||||
const vector<Alignment>& alignments,
|
||||
const vector<string>& targetSegments)
|
||||
: m_emptyTarget(),
|
||||
: m_initialTransOpt(),
|
||||
m_sentence(),
|
||||
m_wp("WordPenalty"),
|
||||
m_uwp("UnknownWordPenalty"),
|
||||
@ -54,7 +54,7 @@ MockHypothesisGuard::MockHypothesisGuard(
|
||||
|
||||
//Initial empty hypothesis
|
||||
m_manager.ResetSentenceStats(m_sentence);
|
||||
m_hypothesis = Hypothesis::Create(m_manager, m_sentence, m_emptyTarget);
|
||||
m_hypothesis = Hypothesis::Create(m_manager, m_sentence, m_initialTransOpt);
|
||||
|
||||
//create the chain
|
||||
vector<Alignment>::const_iterator ai = alignments.begin();
|
||||
|
@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "moses/FF/WordPenaltyProducer.h"
|
||||
#include "Hypothesis.h"
|
||||
#include "Manager.h"
|
||||
#include "TranslationOption.h"
|
||||
|
||||
namespace MosesTest
|
||||
{
|
||||
@ -56,7 +57,7 @@ public:
|
||||
~MockHypothesisGuard();
|
||||
|
||||
private:
|
||||
Moses::TargetPhrase m_emptyTarget;
|
||||
Moses::TranslationOption m_initialTransOpt;
|
||||
Moses::Sentence m_sentence;
|
||||
Moses::WordPenaltyProducer m_wp;
|
||||
Moses::UnknownWordPenaltyProducer m_uwp;
|
||||
|
@ -42,7 +42,7 @@ Parameter::Parameter()
|
||||
AddParam("mapping", "description of decoding steps");
|
||||
AddParam("beam-threshold", "b", "threshold for threshold pruning");
|
||||
AddParam("config", "f", "location of the configuration file");
|
||||
AddParam("continue-partial-translation", "cpt", "start from nonempty hypothesis");
|
||||
//AddParam("continue-partial-translation", "cpt", "start from nonempty hypothesis");
|
||||
AddParam("decoding-graph-backoff", "dpb", "only use subsequent decoding paths for unknown spans of given length");
|
||||
AddParam("drop-unknown", "du", "drop unknown words instead of copying them");
|
||||
AddParam("disable-discarding", "dd", "disable hypothesis discarding");
|
||||
|
@ -41,7 +41,7 @@ SearchCubePruning::SearchCubePruning(Manager& manager, const InputType &source,
|
||||
:Search(manager)
|
||||
,m_source(source)
|
||||
,m_hypoStackColl(source.GetSize() + 1)
|
||||
,m_initialTargetPhrase(source.m_initialTargetPhrase)
|
||||
,m_initialTransOpt()
|
||||
,m_start(clock())
|
||||
,m_transOptColl(transOptColl)
|
||||
{
|
||||
@ -76,7 +76,7 @@ void SearchCubePruning::ProcessSentence()
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
// initial seed hypothesis: nothing translated, no words produced
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTargetPhrase);
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt);
|
||||
|
||||
HypothesisStackCubePruning &firstStack = *static_cast<HypothesisStackCubePruning*>(m_hypoStackColl.front());
|
||||
firstStack.AddInitial(hypo);
|
||||
|
@ -20,7 +20,7 @@ protected:
|
||||
const InputType &m_source;
|
||||
std::vector < HypothesisStack* > m_hypoStackColl; /**< stacks to store hypotheses (partial translations) */
|
||||
// no of elements = no of words in source + 1
|
||||
TargetPhrase m_initialTargetPhrase; /**< used to seed 1st hypo */
|
||||
TranslationOption m_initialTransOpt; /**< used to seed 1st hypo */
|
||||
clock_t m_start; /**< used to track time spend on translation */
|
||||
const TranslationOptionCollection &m_transOptColl; /**< pre-computed list of translation options for the phrases in this sentence */
|
||||
|
||||
|
@ -16,7 +16,7 @@ SearchNormal::SearchNormal(Manager& manager, const InputType &source, const Tran
|
||||
:Search(manager)
|
||||
,m_source(source)
|
||||
,m_hypoStackColl(source.GetSize() + 1)
|
||||
,m_initialTargetPhrase(source.m_initialTargetPhrase)
|
||||
,m_initialTransOpt()
|
||||
,m_start(clock())
|
||||
,interrupted_flag(0)
|
||||
,m_transOptColl(transOptColl)
|
||||
@ -24,10 +24,6 @@ SearchNormal::SearchNormal(Manager& manager, const InputType &source, const Tran
|
||||
VERBOSE(1, "Translating: " << m_source << endl);
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
|
||||
if (m_initialTargetPhrase.GetSize() > 0) {
|
||||
VERBOSE(1, "Search extends partial output: " << m_initialTargetPhrase<<endl);
|
||||
}
|
||||
|
||||
// only if constraint decoding (having to match a specified output)
|
||||
long sentenceID = source.GetTranslationId();
|
||||
m_constraint = staticData.GetConstrainingPhrase(sentenceID);
|
||||
@ -62,7 +58,7 @@ void SearchNormal::ProcessSentence()
|
||||
clock_t t=0; // used to track time for steps
|
||||
|
||||
// initial seed hypothesis: nothing translated, no words produced
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTargetPhrase);
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt);
|
||||
m_hypoStackColl[0]->AddPrune(hypo);
|
||||
|
||||
// go through each stack
|
||||
|
@ -6,6 +6,7 @@
|
||||
#include "HypothesisStackNormal.h"
|
||||
#include "TranslationOptionCollection.h"
|
||||
#include "Timer.h"
|
||||
#include "TranslationOption.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -23,7 +24,7 @@ protected:
|
||||
const InputType &m_source;
|
||||
std::vector < HypothesisStack* > m_hypoStackColl; /**< stacks to store hypotheses (partial translations) */
|
||||
// no of elements = no of words in source + 1
|
||||
TargetPhrase m_initialTargetPhrase; /**< used to seed 1st hypo */
|
||||
TranslationOption m_initialTransOpt; /**< used to seed 1st hypo */
|
||||
clock_t m_start; /**< starting time, used for logging */
|
||||
size_t interrupted_flag; /**< flag indicating that decoder ran out of time (see switch -time-out) */
|
||||
HypothesisStackNormal* actual_hypoStack; /**actual (full expanded) stack of hypotheses*/
|
||||
|
@ -46,7 +46,7 @@ void SearchNormalBatch::ProcessSentence()
|
||||
clock_t t=0; // used to track time for steps
|
||||
|
||||
// initial seed hypothesis: nothing translated, no words produced
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTargetPhrase);
|
||||
Hypothesis *hypo = Hypothesis::Create(m_manager,m_source, m_initialTransOpt);
|
||||
m_hypoStackColl[0]->AddPrune(hypo);
|
||||
|
||||
// go through each stack
|
||||
|
@ -32,6 +32,12 @@ using namespace std;
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
TranslationOption::TranslationOption()
|
||||
:m_targetPhrase()
|
||||
,m_sourceWordsRange(NOT_FOUND, NOT_FOUND)
|
||||
{
|
||||
}
|
||||
|
||||
//TODO this should be a factory function!
|
||||
TranslationOption::TranslationOption(const WordsRange &wordsRange
|
||||
, const TargetPhrase &targetPhrase)
|
||||
|
@ -73,6 +73,8 @@ protected:
|
||||
_ScoreCacheMap m_lexReorderingScores;
|
||||
|
||||
public:
|
||||
explicit TranslationOption(); // For initial hypo that does translate anything
|
||||
|
||||
/** constructor. Used by initial translation step */
|
||||
TranslationOption(const WordsRange &wordsRange
|
||||
, const TargetPhrase &targetPhrase);
|
||||
|
Loading…
Reference in New Issue
Block a user