mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-08-17 07:20:48 +03:00
Working
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/andre_lynum@1851 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
3baf774ca5
commit
234ffc4727
@ -9,13 +9,76 @@
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>?name?</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
|
||||
<value>false</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildLocation</key>
|
||||
<value>${workspace_loc:/misc/Debug}</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
|
||||
<value>clean</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.append_environment</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.contents</key>
|
||||
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildArguments</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildCommand</key>
|
||||
<value>make</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.stopOnError</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
@ -9,13 +9,76 @@
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>?name?</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
|
||||
<value>false</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildLocation</key>
|
||||
<value>${workspace_loc:/moses-cmd/Debug}</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
|
||||
<value>clean</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.append_environment</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.contents</key>
|
||||
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildArguments</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildCommand</key>
|
||||
<value>make</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.stopOnError</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
@ -1,3 +1,4 @@
|
||||
#Tue Nov 21 19:58:08 GMT 2006
|
||||
#Tue May 20 14:06:50 CEST 2008
|
||||
eclipse.preferences.version=1
|
||||
indexer/indexerId=org.eclipse.cdt.core.nullindexer
|
||||
indexerId=org.eclipse.cdt.core.nullindexer
|
||||
|
@ -246,15 +246,18 @@ void IOStream::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, bo
|
||||
// SCORER start
|
||||
void IOStream::OutputScore(const Hypothesis *hypo) {
|
||||
Phrase *trans = StaticData::Instance().GetTranslatedPhrase();
|
||||
size_t unkTransCount = trans->GetSize() - hypo->GetTotalTargetSize();
|
||||
float penalizedScore = hypo->GetTotalScore() +
|
||||
(StaticData::Instance().GetScorerWordPenalty() * (unkTransCount + hypo->SourceWordsNotCovered()));
|
||||
VERBOSE(1, "Source: " << hypo->GetSourcePhrase() << endl);
|
||||
VERBOSE(1, "Translation: " << *trans << endl);
|
||||
VERBOSE(1, "Hypothesis: " << *hypo << endl);
|
||||
VERBOSE(1, "Hypothesis score: " << hypo->GetTotalScore() << endl);
|
||||
VERBOSE(1, "Penalized score: " << hypo->GetPenalizedScore() << endl);
|
||||
VERBOSE(1, "Penalized score: " << penalizedScore << endl);
|
||||
VERBOSE(1, "Source unknown word count: " << hypo->SourceWordsNotCovered() << endl);
|
||||
VERBOSE(1, "Translation unknown word count: " << trans->GetSize() - hypo->GetSize() << endl);
|
||||
cout << hypo->GetPenalizedScore() << " " << hypo->GetTotalScore() << " "
|
||||
<< hypo->SourceWordsNotCovered() << " " << trans->GetSize() - hypo->GetSize() << endl ;
|
||||
VERBOSE(1, "Translation unknown word count: " << unkTransCount << endl);
|
||||
cout << penalizedScore << " " << hypo->GetTotalScore() << " "
|
||||
<< hypo->SourceWordsNotCovered() << " " << unkTransCount << endl ;
|
||||
}
|
||||
// SCORER end
|
||||
|
||||
|
@ -143,6 +143,7 @@ int main(int argc, char* argv[])
|
||||
Phrase *transPhrase = new Phrase(trans->GetSubString(WordsRange(0, transSize)));
|
||||
delete staticData.GetTranslatedPhrase();
|
||||
staticData.SetTranslatedPhrase(transPhrase);
|
||||
staticData.SetTranslationCoverage(0);
|
||||
}
|
||||
|
||||
delete trans;
|
||||
|
@ -7,13 +7,76 @@
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
|
||||
<triggers>clean,full,incremental,</triggers>
|
||||
<arguments>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>?name?</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
|
||||
<value>false</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildLocation</key>
|
||||
<value>${workspace_loc:/moses/Debug}</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
|
||||
<value>clean</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.append_environment</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.contents</key>
|
||||
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildArguments</key>
|
||||
<value></value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.buildCommand</key>
|
||||
<value>make</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
|
||||
<value>all</value>
|
||||
</dictionary>
|
||||
<dictionary>
|
||||
<key>org.eclipse.cdt.make.core.stopOnError</key>
|
||||
<value>true</value>
|
||||
</dictionary>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.cdt.core.cnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
|
||||
<nature>org.eclipse.cdt.core.ccnature</nature>
|
||||
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
@ -1,3 +1,4 @@
|
||||
#Tue Nov 14 23:21:36 GMT 2006
|
||||
#Tue May 20 14:06:50 CEST 2008
|
||||
eclipse.preferences.version=1
|
||||
indexer/indexerId=org.eclipse.cdt.core.nullindexer
|
||||
indexerId=org.eclipse.cdt.core.nullindexer
|
||||
|
@ -58,7 +58,10 @@ Hypothesis::Hypothesis(InputType const& source, const TargetPhrase &emptyTarget)
|
||||
, m_arcList(NULL)
|
||||
, m_id(0)
|
||||
, m_lmstats(NULL)
|
||||
// SCORER start
|
||||
, m_penalizedScore(0)
|
||||
, m_skippedTransWords(0)
|
||||
// SCORER end
|
||||
{ // used for initial seeding of trans process
|
||||
// initialize scores
|
||||
//_hash_computed = false;
|
||||
@ -87,7 +90,10 @@ Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &tran
|
||||
, m_arcList(NULL)
|
||||
, m_id(s_HypothesesCreated++)
|
||||
, m_lmstats(NULL)
|
||||
// SCORER start
|
||||
, m_penalizedScore(0)
|
||||
, m_skippedTransWords(0)
|
||||
// SCORER end
|
||||
{
|
||||
// assert that we are not extending our hypothesis by retranslating something
|
||||
// that this hypothesis has already translated!
|
||||
@ -328,13 +334,19 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore)
|
||||
CalcDistortionScore();
|
||||
|
||||
// LANGUAGE MODEL COST
|
||||
CalcLMScore(staticData.GetAllLM());
|
||||
// SCORER start
|
||||
if (!staticData.GetScoreFlag())
|
||||
CalcLMScore(staticData.GetAllLM());
|
||||
// SCORER end
|
||||
|
||||
// WORD PENALTY
|
||||
m_scoreBreakdown.PlusEquals(staticData.GetWordPenaltyProducer(), - (float) m_currTargetWordsRange.GetNumWordsCovered());
|
||||
|
||||
// FUTURE COST
|
||||
CalcFutureScore(futureScore);
|
||||
// SCORER start
|
||||
if (!staticData.GetScoreFlag())
|
||||
CalcFutureScore(futureScore);
|
||||
// SCORER end
|
||||
|
||||
|
||||
//LEXICAL REORDERING COST
|
||||
@ -528,23 +540,27 @@ const ScoreComponentCollection &Hypothesis::GetCachedReorderingScore() const
|
||||
// within the output positions covered by the hypothesis
|
||||
bool Hypothesis::CompareHypothesisToPhrase(const Phrase *inputPhrase, size_t wordsSoFar) const
|
||||
{
|
||||
WordsRange targetRange = WordsRange(m_currTargetWordsRange.GetStartPos(),
|
||||
m_currTargetWordsRange.GetEndPos() - m_skippedTransWords);
|
||||
// Check that the hypothesis chain doesn't cover more words than is in the phrase
|
||||
if ((wordsSoFar + m_targetPhrase.GetSize()) > inputPhrase->GetSize())
|
||||
if ((wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords) > inputPhrase->GetSize())
|
||||
return false;
|
||||
// check for compatibility or if we're in the root hypothesis (better test for this ?)
|
||||
if ((m_targetPhrase.GetSize() == 0) || inputPhrase->GetSubString(m_currTargetWordsRange).IsCompatible(m_targetPhrase)) {
|
||||
if ((m_targetPhrase.GetSize() == 0) ||
|
||||
inputPhrase->GetSubString(targetRange).IsCompatible(m_targetPhrase)) {
|
||||
// should follow the arclist too ???
|
||||
if (m_prevHypo == NULL)
|
||||
return true;
|
||||
|
||||
else if ((m_prevHypo != NULL) && m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
|
||||
else if ((m_prevHypo != NULL) &&
|
||||
m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
|
||||
return true;
|
||||
|
||||
else if ((m_arcList != NULL) && m_arcList->size() > 0) {
|
||||
for (ArcList::const_iterator it = m_arcList->begin(); it != m_arcList->end(); it++) {
|
||||
Hypothesis *hypo = *it;
|
||||
|
||||
if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
|
||||
if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -577,4 +593,18 @@ size_t Hypothesis::SourceWordsNotCovered() const {
|
||||
return GetWordsBitmap().GetSize() - GetWordsBitmap().GetNumWordsCovered();
|
||||
}
|
||||
|
||||
size_t Hypothesis::GetAllSkippedWords() const {
|
||||
if (m_prevHypo == NULL)
|
||||
return m_skippedTransWords;
|
||||
else
|
||||
return m_skippedTransWords + m_prevHypo->GetAllSkippedWords();
|
||||
}
|
||||
|
||||
size_t Hypothesis::GetTotalTargetSize() const {
|
||||
if (m_prevHypo == NULL)
|
||||
return GetCurrTargetLength();
|
||||
else
|
||||
return GetCurrTargetLength() - m_skippedTransWords + m_prevHypo->GetTotalTargetSize();
|
||||
}
|
||||
|
||||
// SCORER end
|
||||
|
@ -69,7 +69,7 @@ protected:
|
||||
//it's a confusion network in the end???
|
||||
InputType const& m_sourceInput;
|
||||
WordsRange m_currSourceWordsRange; /**< source word positions of the last phrase that was used to create this hypothesis */
|
||||
WordsRange m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
|
||||
mutable WordsRange m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
|
||||
bool m_wordDeleted;
|
||||
float m_totalScore; /**< score so far */
|
||||
float m_futureScore; /**< estimated future cost to translate rest of sentence */
|
||||
@ -81,6 +81,7 @@ protected:
|
||||
|
||||
// SCORER start
|
||||
float m_penalizedScore;
|
||||
mutable size_t m_skippedTransWords;
|
||||
// SCORER end
|
||||
|
||||
int m_id; /**< numeric ID of this hypothesis, used for logging */
|
||||
@ -270,6 +271,14 @@ public:
|
||||
float GetPenalizedScore() const {
|
||||
return m_penalizedScore;
|
||||
}
|
||||
// TODO release old WordsRange instance ???
|
||||
void SetCurrTargetWordsRange(WordsRange range) const { m_currTargetWordsRange = range; }
|
||||
|
||||
size_t GetSkippedTransWords() const { return m_skippedTransWords; }
|
||||
void SetSkippedTransWords(size_t val) const { m_skippedTransWords = val; }
|
||||
void IncrSkippedTransWords() const { m_skippedTransWords++; }
|
||||
size_t GetAllSkippedWords() const;
|
||||
size_t GetTotalTargetSize() const;
|
||||
// SCORER end
|
||||
};
|
||||
|
||||
|
@ -109,7 +109,7 @@ void Manager::ProcessSentence()
|
||||
|
||||
// the stack is pruned before processing (lazy pruning):
|
||||
VERBOSE(3,"processing hypothesis from next stack");
|
||||
// VERBOSE("processing next stack at ");
|
||||
// VERBOSE("processing next stack at ");
|
||||
sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
|
||||
VERBOSE(3,std::endl);
|
||||
sourceHypoColl.CleanupArcList();
|
||||
@ -122,6 +122,7 @@ void Manager::ProcessSentence()
|
||||
}
|
||||
// some logging
|
||||
IFVERBOSE(2) { OutputHypoStackSize(); }
|
||||
|
||||
}
|
||||
|
||||
// some more logging
|
||||
@ -138,6 +139,7 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
|
||||
// since we check for reordering limits, its good to have that limit handy
|
||||
int maxDistortion = StaticData::Instance().GetMaxDistortion();
|
||||
bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;
|
||||
bool expanded = false;
|
||||
|
||||
// no limit of reordering: only check for overlap
|
||||
if (maxDistortion < 0)
|
||||
@ -148,20 +150,37 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
|
||||
|
||||
for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
|
||||
{
|
||||
size_t maxSize = sourceSize - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
||||
size_t maxSize = sourceSize - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
|
||||
{
|
||||
if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
|
||||
{
|
||||
ExpandAllHypotheses(hypothesis
|
||||
, m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos)));
|
||||
|
||||
if (ExpandAllHypotheses(hypothesis,
|
||||
m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos))))
|
||||
expanded = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!expanded) {
|
||||
WordsRange oldRange = hypothesis.GetCurrTargetWordsRange();
|
||||
size_t newEnd = oldRange.GetEndPos() + 1;
|
||||
const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
|
||||
|
||||
if (newEnd < transPhrase->GetSize() - 1) { // problem here
|
||||
// cout << "Skipping word at " << oldRange.GetEndPos() << " continuing from " << newEnd << endl;
|
||||
|
||||
hypothesis.SetCurrTargetWordsRange(WordsRange(oldRange.GetStartPos(), newEnd));
|
||||
hypothesis.IncrSkippedTransWords();
|
||||
|
||||
ProcessOneHypothesis(hypothesis);
|
||||
}
|
||||
}
|
||||
|
||||
return; // done with special case (no reordering limit)
|
||||
}
|
||||
|
||||
@ -274,13 +293,18 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
|
||||
* \param transOptList list of translation options to be applied
|
||||
*/
|
||||
|
||||
void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
|
||||
bool Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
|
||||
{
|
||||
TranslationOptionList::const_iterator iter;
|
||||
bool expanded = false;
|
||||
|
||||
for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
|
||||
{
|
||||
ExpandHypothesis(hypothesis, **iter);
|
||||
if (ExpandHypothesis(hypothesis, **iter))
|
||||
expanded = true;
|
||||
}
|
||||
|
||||
return expanded;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -290,7 +314,8 @@ void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const Translation
|
||||
* \param transOpt translation option (phrase translation)
|
||||
* that is applied to create the new hypothesis
|
||||
*/
|
||||
void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
|
||||
// SCORER start
|
||||
bool Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
|
||||
{
|
||||
// create hypothesis and calculate all its scores
|
||||
#ifdef DEBUGLATTICE
|
||||
@ -310,20 +335,24 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp
|
||||
// add to hypothesis stack
|
||||
size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
|
||||
|
||||
// SCORER
|
||||
// Only add hypothesises that agree with translation
|
||||
if (SCORER_FILTER_HYPOS && StaticData::Instance().GetScoreFlag()) {
|
||||
const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
|
||||
|
||||
if (newHypo->CompareHypothesisToPhrase(transPhrase, 0))
|
||||
if (newHypo->CompareHypothesisToPhrase(transPhrase, 0)) {
|
||||
m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
VERBOSE(3, "Not Compatible with translation." << endl);
|
||||
FREEHYPO(newHypo);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
else {
|
||||
m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
|
||||
return true;
|
||||
}
|
||||
// SCORER end
|
||||
}
|
||||
|
||||
@ -336,12 +365,13 @@ const Hypothesis *Manager::GetBestHypothesis() const
|
||||
// SCORER start
|
||||
// search all hypothesis stacks
|
||||
const Hypothesis* bestHypo = NULL;
|
||||
|
||||
int i = 0;
|
||||
|
||||
if (StaticData::Instance().GetScoreFlag()) {
|
||||
// having trouble getting reverse iterators to work
|
||||
std::vector<HypothesisStack>::const_iterator it = m_hypoStackColl.end();
|
||||
it--;
|
||||
for (;; it--) {
|
||||
for (;;) {
|
||||
const HypothesisStack &hypoColl = *it;
|
||||
|
||||
if ((hypoColl.size() > 0) && (hypoColl.size() < 327685)) {// what constant is this ?
|
||||
@ -349,12 +379,17 @@ const Hypothesis *Manager::GetBestHypothesis() const
|
||||
if (bestHypo == NULL)
|
||||
bestHypo = hypo;
|
||||
else
|
||||
if ((hypo->GetTotalScore() < 0.0) && (hypo->GetTotalScore() > bestHypo->GetTotalScore()))
|
||||
if ((hypo->GetTotalScore() < 0.0) &&
|
||||
(hypo->GetTotalScore() > bestHypo->GetTotalScore()))
|
||||
bestHypo = hypo;
|
||||
}
|
||||
|
||||
if (it == m_hypoStackColl.begin()) break;
|
||||
if ((it == m_hypoStackColl.begin()) || (bestHypo != NULL)) break;
|
||||
i++;
|
||||
it--;
|
||||
}
|
||||
|
||||
// cout << "Best hypo " << i << " from end." << endl;
|
||||
|
||||
return bestHypo;
|
||||
}
|
||||
@ -505,3 +540,4 @@ void Manager::CalcDecoderStatistics() const
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -87,8 +87,8 @@ protected:
|
||||
|
||||
// functions for creating hypotheses
|
||||
void ProcessOneHypothesis(const Hypothesis &hypothesis);
|
||||
void ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
|
||||
void ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
|
||||
bool ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
|
||||
bool ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
|
||||
|
||||
// logging
|
||||
void OutputHypoStack(int stack = -1);
|
||||
@ -105,4 +105,5 @@ public:
|
||||
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
|
||||
*/
|
||||
void CalcDecoderStatistics() const;
|
||||
|
||||
};
|
||||
|
@ -170,7 +170,8 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
// the translation options are filtered
|
||||
if (m_score) {
|
||||
m_useTransOptCache = false;
|
||||
// m_isAlwaysCreateDirectTranslationOption = true;
|
||||
m_isAlwaysCreateDirectTranslationOption = false;
|
||||
m_maxDistortion = -1;
|
||||
}
|
||||
if (m_parameter->GetParam("scorePenalty").size() == 1) {
|
||||
m_factorDelimiter = Scan<float>(m_parameter->GetParam("scorePenalty")[0]);
|
||||
|
@ -57,6 +57,7 @@ protected:
|
||||
mutable Phrase *m_translatedPhrase;
|
||||
// unaccounted word penalty
|
||||
float m_scorerWordPenalty;
|
||||
mutable int m_translationCoverage;
|
||||
// SCORER end
|
||||
std::vector<PhraseDictionary*> m_phraseDictionary;
|
||||
std::vector<GenerationDictionary*> m_generationDictionary;
|
||||
@ -387,5 +388,8 @@ public:
|
||||
// Accessor for the scorer flag
|
||||
const bool GetScoreFlag() const { return m_score; }
|
||||
float GetScorerWordPenalty() const { return m_scorerWordPenalty; }
|
||||
int GetTranslationCoverage() const { return m_translationCoverage; }
|
||||
void SetTranslationCoverage(int x) const { m_translationCoverage = x; }
|
||||
int AddTranslationCoverage(int x) const { return m_translationCoverage += x; }
|
||||
// SCORER end
|
||||
};
|
||||
|
@ -164,7 +164,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* SCORER need to disable this
|
||||
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
// create unknown words for 1 word coverage where we don't have any trans options
|
||||
for (size_t pos = 0 ; pos < size ; ++pos)
|
||||
@ -173,6 +173,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
|
||||
if (fullList.size() == 0 || alwaysCreateDirectTranslationOption)
|
||||
ProcessUnknownWord(pos);
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
/** special handling of ONE unknown words. Either add temporarily add word to translation table,
|
||||
|
Loading…
Reference in New Issue
Block a user