moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-08-17 07:20:48 +03:00
Code Issues Packages Projects Releases Wiki Activity

Working

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/branches/andre_lynum@1851 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
sinnatagg 2008-06-11 13:51:33 +00:00
parent 3baf774ca5
commit 234ffc4727
14 changed files with 313 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
misc/.project
View File

@ -9,13 +9,76 @@
<buildSpec>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
<dictionary>
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>?name?</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
<value>false</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildLocation</key>
<value>${workspace_loc:/misc/Debug}</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
<value>clean</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.append_environment</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.contents</key>
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildArguments</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildCommand</key>
<value>make</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.stopOnError</key>
<value>true</value>
</dictionary>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.cdt.core.cnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
<nature>org.eclipse.cdt.core.ccnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
</natures>
</projectDescription>

65
moses-cmd/.project
View File

@ -9,13 +9,76 @@
<buildSpec>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
<dictionary>
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>?name?</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
<value>false</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildLocation</key>
<value>${workspace_loc:/moses-cmd/Debug}</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
<value>clean</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.append_environment</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.contents</key>
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildArguments</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildCommand</key>
<value>make</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.stopOnError</key>
<value>true</value>
</dictionary>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.cdt.core.cnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
<nature>org.eclipse.cdt.core.ccnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
</natures>
</projectDescription>

3
moses-cmd/.settings/org.eclipse.cdt.core.prefs
View File

@ -1,3 +1,4 @@
#Tue Nov 21 19:58:08 GMT 2006
#Tue May 20 14:06:50 CEST 2008
eclipse.preferences.version=1
indexer/indexerId=org.eclipse.cdt.core.nullindexer
indexerId=org.eclipse.cdt.core.nullindexer

11
moses-cmd/src/IOStream.cpp
View File

@ -246,15 +246,18 @@ void IOStream::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, bo
// SCORER start
void IOStream::OutputScore(const Hypothesis *hypo) {
Phrase *trans = StaticData::Instance().GetTranslatedPhrase();
size_t unkTransCount = trans->GetSize() - hypo->GetTotalTargetSize();
float penalizedScore = hypo->GetTotalScore() +
(StaticData::Instance().GetScorerWordPenalty() * (unkTransCount + hypo->SourceWordsNotCovered()));
VERBOSE(1, "Source: " << hypo->GetSourcePhrase() << endl);
VERBOSE(1, "Translation: " << *trans << endl);
VERBOSE(1, "Hypothesis: " << *hypo << endl);
VERBOSE(1, "Hypothesis score: " << hypo->GetTotalScore() << endl);
VERBOSE(1, "Penalized score: " << hypo->GetPenalizedScore() << endl);
VERBOSE(1, "Penalized score: " << penalizedScore << endl);
VERBOSE(1, "Source unknown word count: " << hypo->SourceWordsNotCovered() << endl);
VERBOSE(1, "Translation unknown word count: " << trans->GetSize() - hypo->GetSize() << endl);
cout << hypo->GetPenalizedScore() << " " << hypo->GetTotalScore() << " "
<< hypo->SourceWordsNotCovered() << " " << trans->GetSize() - hypo->GetSize() << endl ;
VERBOSE(1, "Translation unknown word count: " << unkTransCount << endl);
cout << penalizedScore << " " << hypo->GetTotalScore() << " "
<< hypo->SourceWordsNotCovered() << " " << unkTransCount << endl ;
}
// SCORER end

1
moses-cmd/src/Main.cpp
View File

@ -143,6 +143,7 @@ int main(int argc, char* argv[])
Phrase *transPhrase = new Phrase(trans->GetSubString(WordsRange(0, transSize)));
delete staticData.GetTranslatedPhrase();
staticData.SetTranslatedPhrase(transPhrase);
staticData.SetTranslationCoverage(0);
}
delete trans;

65
moses/.project
View File

@ -7,13 +7,76 @@
<buildSpec>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
<triggers>clean,full,incremental,</triggers>
<arguments>
<dictionary>
<key>org.eclipse.cdt.make.core.fullBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>?name?</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableAutoBuild</key>
<value>false</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildLocation</key>
<value>${workspace_loc:/moses/Debug}</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableFullBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.enableCleanBuild</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.cleanBuildTarget</key>
<value>clean</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.append_environment</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.contents</key>
<value>org.eclipse.cdt.make.core.activeConfigSettings</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.useDefaultBuildCmd</key>
<value>true</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildArguments</key>
<value></value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.buildCommand</key>
<value>make</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.autoBuildTarget</key>
<value>all</value>
</dictionary>
<dictionary>
<key>org.eclipse.cdt.make.core.stopOnError</key>
<value>true</value>
</dictionary>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.cdt.core.cnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
<nature>org.eclipse.cdt.core.ccnature</nature>
<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
</natures>
</projectDescription>

3
moses/.settings/org.eclipse.cdt.core.prefs
View File

@ -1,3 +1,4 @@
#Tue Nov 14 23:21:36 GMT 2006
#Tue May 20 14:06:50 CEST 2008
eclipse.preferences.version=1
indexer/indexerId=org.eclipse.cdt.core.nullindexer
indexerId=org.eclipse.cdt.core.nullindexer

42
moses/src/Hypothesis.cpp
View File

@ -58,7 +58,10 @@ Hypothesis::Hypothesis(InputType const& source, const TargetPhrase &emptyTarget)
, m_arcList(NULL)
, m_id(0)
, m_lmstats(NULL)
// SCORER start
, m_penalizedScore(0)
, m_skippedTransWords(0)
// SCORER end
{ // used for initial seeding of trans process
// initialize scores
//_hash_computed = false;
@ -87,7 +90,10 @@ Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &tran
, m_arcList(NULL)
, m_id(s_HypothesesCreated++)
, m_lmstats(NULL)
// SCORER start
, m_penalizedScore(0)
, m_skippedTransWords(0)
// SCORER end
{
// assert that we are not extending our hypothesis by retranslating something
// that this hypothesis has already translated!
@ -328,13 +334,19 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore)
CalcDistortionScore();
// LANGUAGE MODEL COST
CalcLMScore(staticData.GetAllLM());
// SCORER start
if (!staticData.GetScoreFlag())
CalcLMScore(staticData.GetAllLM());
// SCORER end
// WORD PENALTY
m_scoreBreakdown.PlusEquals(staticData.GetWordPenaltyProducer(), - (float) m_currTargetWordsRange.GetNumWordsCovered());
// FUTURE COST
CalcFutureScore(futureScore);
// SCORER start
if (!staticData.GetScoreFlag())
CalcFutureScore(futureScore);
// SCORER end
//LEXICAL REORDERING COST
@ -528,23 +540,27 @@ const ScoreComponentCollection &Hypothesis::GetCachedReorderingScore() const
// within the output positions covered by the hypothesis
bool Hypothesis::CompareHypothesisToPhrase(const Phrase *inputPhrase, size_t wordsSoFar) const
{
WordsRange targetRange = WordsRange(m_currTargetWordsRange.GetStartPos(),
m_currTargetWordsRange.GetEndPos() - m_skippedTransWords);
// Check that the hypothesis chain doesn't cover more words than is in the phrase
if ((wordsSoFar + m_targetPhrase.GetSize()) > inputPhrase->GetSize())
if ((wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords) > inputPhrase->GetSize())
return false;
// check for compatibility or if we're in the root hypothesis (better test for this ?)
if ((m_targetPhrase.GetSize() == 0) || inputPhrase->GetSubString(m_currTargetWordsRange).IsCompatible(m_targetPhrase)) {
if ((m_targetPhrase.GetSize() == 0) ||
inputPhrase->GetSubString(targetRange).IsCompatible(m_targetPhrase)) {
// should follow the arclist too ???
if (m_prevHypo == NULL)
return true;
else if ((m_prevHypo != NULL) && m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
else if ((m_prevHypo != NULL) &&
m_prevHypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
return true;
else if ((m_arcList != NULL) && m_arcList->size() > 0) {
for (ArcList::const_iterator it = m_arcList->begin(); it != m_arcList->end(); it++) {
Hypothesis *hypo = *it;
if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize()))
if (hypo->CompareHypothesisToPhrase(inputPhrase, wordsSoFar + m_targetPhrase.GetSize() + m_skippedTransWords))
return true;
}
}
@ -577,4 +593,18 @@ size_t Hypothesis::SourceWordsNotCovered() const {
return GetWordsBitmap().GetSize() - GetWordsBitmap().GetNumWordsCovered();
}
size_t Hypothesis::GetAllSkippedWords() const {
if (m_prevHypo == NULL)
return m_skippedTransWords;
else
return m_skippedTransWords + m_prevHypo->GetAllSkippedWords();
}
size_t Hypothesis::GetTotalTargetSize() const {
if (m_prevHypo == NULL)
return GetCurrTargetLength();
else
return GetCurrTargetLength() - m_skippedTransWords + m_prevHypo->GetTotalTargetSize();
}
// SCORER end

11
moses/src/Hypothesis.h
View File

@ -69,7 +69,7 @@ protected:
//it's a confusion network in the end???
InputType const& m_sourceInput;
WordsRange m_currSourceWordsRange; /**< source word positions of the last phrase that was used to create this hypothesis */
WordsRange m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
mutable WordsRange m_currTargetWordsRange; /**< target word positions of the last phrase that was used to create this hypothesis */
bool m_wordDeleted;
float m_totalScore; /**< score so far */
float m_futureScore; /**< estimated future cost to translate rest of sentence */
@ -81,6 +81,7 @@ protected:
// SCORER start
float m_penalizedScore;
mutable size_t m_skippedTransWords;
// SCORER end
int m_id; /**< numeric ID of this hypothesis, used for logging */
@ -270,6 +271,14 @@ public:
float GetPenalizedScore() const {
return m_penalizedScore;
}
// TODO release old WordsRange instance ???
void SetCurrTargetWordsRange(WordsRange range) const { m_currTargetWordsRange = range; }
size_t GetSkippedTransWords() const { return m_skippedTransWords; }
void SetSkippedTransWords(size_t val) const { m_skippedTransWords = val; }
void IncrSkippedTransWords() const { m_skippedTransWords++; }
size_t GetAllSkippedWords() const;
size_t GetTotalTargetSize() const;
// SCORER end
};

68
moses/src/Manager.cpp
View File

@ -109,7 +109,7 @@ void Manager::ProcessSentence()
// the stack is pruned before processing (lazy pruning):
VERBOSE(3,"processing hypothesis from next stack");
// VERBOSE("processing next stack at ");
// VERBOSE("processing next stack at ");
sourceHypoColl.PruneToSize(staticData.GetMaxHypoStackSize());
VERBOSE(3,std::endl);
sourceHypoColl.CleanupArcList();
@ -122,6 +122,7 @@ void Manager::ProcessSentence()
}
// some logging
IFVERBOSE(2) { OutputHypoStackSize(); }
}
// some more logging
@ -138,6 +139,7 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
// since we check for reordering limits, its good to have that limit handy
int maxDistortion = StaticData::Instance().GetMaxDistortion();
bool isWordLattice = StaticData::Instance().GetInputType() == WordLatticeInput;
bool expanded = false;
// no limit of reordering: only check for overlap
if (maxDistortion < 0)
@ -148,20 +150,37 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
{
size_t maxSize = sourceSize - startPos;
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
size_t maxSize = sourceSize - startPos;
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
{
if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
{
ExpandAllHypotheses(hypothesis
, m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos)));
if (ExpandAllHypotheses(hypothesis,
m_transOptColl->GetTranslationOptionList(WordsRange(startPos, endPos))))
expanded = true;
}
}
}
if (!expanded) {
WordsRange oldRange = hypothesis.GetCurrTargetWordsRange();
size_t newEnd = oldRange.GetEndPos() + 1;
const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
if (newEnd < transPhrase->GetSize() - 1) { // problem here
// cout << "Skipping word at " << oldRange.GetEndPos() << " continuing from " << newEnd << endl;
hypothesis.SetCurrTargetWordsRange(WordsRange(oldRange.GetStartPos(), newEnd));
hypothesis.IncrSkippedTransWords();
ProcessOneHypothesis(hypothesis);
}
}
return; // done with special case (no reordering limit)
}
@ -274,13 +293,18 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
* \param transOptList list of translation options to be applied
*/
void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
bool Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList)
{
TranslationOptionList::const_iterator iter;
bool expanded = false;
for (iter = transOptList.begin() ; iter != transOptList.end() ; ++iter)
{
ExpandHypothesis(hypothesis, **iter);
if (ExpandHypothesis(hypothesis, **iter))
expanded = true;
}
return expanded;
}
/**
@ -290,7 +314,8 @@ void Manager::ExpandAllHypotheses(const Hypothesis &hypothesis,const Translation
* \param transOpt translation option (phrase translation)
* that is applied to create the new hypothesis
*/
void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
// SCORER start
bool Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOption &transOpt)
{
// create hypothesis and calculate all its scores
#ifdef DEBUGLATTICE
@ -310,20 +335,24 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp
// add to hypothesis stack
size_t wordsTranslated = newHypo->GetWordsBitmap().GetNumWordsCovered();
// SCORER
// Only add hypothesises that agree with translation
if (SCORER_FILTER_HYPOS && StaticData::Instance().GetScoreFlag()) {
const Phrase *transPhrase = StaticData::Instance().GetTranslatedPhrase();
if (newHypo->CompareHypothesisToPhrase(transPhrase, 0))
if (newHypo->CompareHypothesisToPhrase(transPhrase, 0)) {
m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
return true;
}
else {
VERBOSE(3, "Not Compatible with translation." << endl);
FREEHYPO(newHypo);
return false;
}
}
else
else {
m_hypoStackColl[wordsTranslated].AddPrune(newHypo);
return true;
}
// SCORER end
}
@ -336,12 +365,13 @@ const Hypothesis *Manager::GetBestHypothesis() const
// SCORER start
// search all hypothesis stacks
const Hypothesis* bestHypo = NULL;
int i = 0;
if (StaticData::Instance().GetScoreFlag()) {
// having trouble getting reverse iterators to work
std::vector<HypothesisStack>::const_iterator it = m_hypoStackColl.end();
it--;
for (;; it--) {
for (;;) {
const HypothesisStack &hypoColl = *it;
if ((hypoColl.size() > 0) && (hypoColl.size() < 327685)) {// what constant is this ?
@ -349,12 +379,17 @@ const Hypothesis *Manager::GetBestHypothesis() const
if (bestHypo == NULL)
bestHypo = hypo;
else
if ((hypo->GetTotalScore() < 0.0) && (hypo->GetTotalScore() > bestHypo->GetTotalScore()))
if ((hypo->GetTotalScore() < 0.0) &&
(hypo->GetTotalScore() > bestHypo->GetTotalScore()))
bestHypo = hypo;
}
if (it == m_hypoStackColl.begin()) break;
if ((it == m_hypoStackColl.begin()) || (bestHypo != NULL)) break;
i++;
it--;
}
// cout << "Best hypo " << i << " from end." << endl;
return bestHypo;
}
@ -505,3 +540,4 @@ void Manager::CalcDecoderStatistics() const
}
}
}

5
moses/src/Manager.h
View File

@ -87,8 +87,8 @@ protected:
// functions for creating hypotheses
void ProcessOneHypothesis(const Hypothesis &hypothesis);
void ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
void ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
bool ExpandAllHypotheses(const Hypothesis &hypothesis,const TranslationOptionList &transOptList);
bool ExpandHypothesis(const Hypothesis &hypothesis,const TranslationOption &transOpt);
// logging
void OutputHypoStack(int stack = -1);
@ -105,4 +105,5 @@ public:
* to be called after processing a sentence (which may consist of more than just calling ProcessSentence() )
*/
void CalcDecoderStatistics() const;
};

3
moses/src/StaticData.cpp
View File

@ -170,7 +170,8 @@ bool StaticData::LoadData(Parameter *parameter)
// the translation options are filtered
if (m_score) {
m_useTransOptCache = false;
// m_isAlwaysCreateDirectTranslationOption = true;
m_isAlwaysCreateDirectTranslationOption = false;
m_maxDistortion = -1;
}
if (m_parameter->GetParam("scorePenalty").size() == 1) {
m_factorDelimiter = Scan<float>(m_parameter->GetParam("scorePenalty")[0]);

4
moses/src/StaticData.h
View File

@ -57,6 +57,7 @@ protected:
mutable Phrase *m_translatedPhrase;
// unaccounted word penalty
float m_scorerWordPenalty;
mutable int m_translationCoverage;
// SCORER end
std::vector<PhraseDictionary*> m_phraseDictionary;
std::vector<GenerationDictionary*> m_generationDictionary;
@ -387,5 +388,8 @@ public:
// Accessor for the scorer flag
const bool GetScoreFlag() const { return m_score; }
float GetScorerWordPenalty() const { return m_scorerWordPenalty; }
int GetTranslationCoverage() const { return m_translationCoverage; }
void SetTranslationCoverage(int x) const { m_translationCoverage = x; }
int AddTranslationCoverage(int x) const { return m_translationCoverage += x; }
// SCORER end
};

3
moses/src/TranslationOptionCollection.cpp
View File

@ -164,7 +164,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
}
}
}
/* SCORER need to disable this
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
// create unknown words for 1 word coverage where we don't have any trans options
for (size_t pos = 0 ; pos < size ; ++pos)
@ -173,6 +173,7 @@ void TranslationOptionCollection::ProcessUnknownWord(const std::vector <DecodeGr
if (fullList.size() == 0 || alwaysCreateDirectTranslationOption)
ProcessUnknownWord(pos);
}
*/
}
/** special handling of ONE unknown words. Either add temporarily add word to translation table,