mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
gt rid of word penalty of 0.434. Adjust weight accordingly
This commit is contained in:
parent
6da2cf7663
commit
b53b93f09f
@ -135,7 +135,7 @@ void ChartManager::AddXmlChartOptions() {
|
||||
i != xmlChartOptionsList.end(); ++i) {
|
||||
ChartTranslationOptions* opt = *i;
|
||||
|
||||
Moses::Scores wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
||||
Moses::Scores wordPenaltyScore(1, -1);
|
||||
opt->GetTargetPhraseCollection().GetCollection()[0]->SetScore((FeatureFunction*)staticData.GetWordPenaltyProducer(), wordPenaltyScore);
|
||||
|
||||
const WordsRange &range = opt->GetSourceWordsRange();
|
||||
|
@ -44,7 +44,7 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
|
||||
// unknown word, add as trans opt
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = staticData.GetUnknownWordPenaltyProducer();
|
||||
vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
|
||||
vector<float> wordPenaltyScore(1, -1);
|
||||
|
||||
size_t isDigit = 0;
|
||||
if (staticData.GetDropUnknown()) {
|
||||
|
@ -752,6 +752,42 @@ void Parameter::ConvertWeightArgsGeneration(const std::string &oldWeightName, co
|
||||
m_setting.erase(oldFeatureName);
|
||||
}
|
||||
|
||||
void Parameter::ConvertWeightArgsWordPenalty()
|
||||
{
|
||||
const std::string oldWeightName = "weight-w";
|
||||
const std::string newWeightName = "WordPenalty";
|
||||
|
||||
bool isChartDecoding = true;
|
||||
if (!isParamSpecified("search-algorithm") ||
|
||||
(GetParam("search-algorithm").size() > 0
|
||||
&& (Trim(GetParam("search-algorithm")[0]) == "0"
|
||||
||Trim(GetParam("search-algorithm")[0]) == "1"
|
||||
)
|
||||
)
|
||||
) {
|
||||
isChartDecoding = false;
|
||||
}
|
||||
|
||||
PARAM_MAP::iterator iterMap;
|
||||
|
||||
iterMap = m_setting.find(oldWeightName);
|
||||
if (iterMap != m_setting.end())
|
||||
{
|
||||
const PARAM_VEC &weights = iterMap->second;
|
||||
for (size_t i = 0; i < weights.size(); ++i)
|
||||
{
|
||||
float weight = Scan<float>(weights[i]);
|
||||
if (isChartDecoding) {
|
||||
weight *= 0.434294482;
|
||||
}
|
||||
SetWeight(newWeightName, i, weight);
|
||||
}
|
||||
|
||||
m_setting.erase(iterMap);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void Parameter::ConvertWeightArgs()
|
||||
{
|
||||
// can't handle discr LM. must do it manually 'cos of bigram/n-gram split
|
||||
@ -767,7 +803,7 @@ void Parameter::ConvertWeightArgs()
|
||||
cerr << "Do not mix old and new format for specify weights";
|
||||
}
|
||||
|
||||
ConvertWeightArgsSingleWeight("weight-w", "WordPenalty");
|
||||
ConvertWeightArgsWordPenalty();
|
||||
ConvertWeightArgsLM("weight-l");
|
||||
ConvertWeightArgsSingleWeight("weight-slm", "SyntacticLM");
|
||||
ConvertWeightArgsSingleWeight("weight-u", "UnknownWordPenalty");
|
||||
|
@ -72,6 +72,7 @@ protected:
|
||||
void ConvertWeightArgsLM(const std::string &oldWeightName);
|
||||
void ConvertWeightArgsDistortion();
|
||||
void ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string &newWeightName);
|
||||
void ConvertWeightArgsWordPenalty();
|
||||
void CreateWeightsMap();
|
||||
void WeightOverwrite();
|
||||
void AddFeature(const std::string &line);
|
||||
|
@ -185,7 +185,7 @@ void Phrase::CreateFromString(FactorDirection direction
|
||||
assert(lhs->IsNonTerminal());
|
||||
}
|
||||
else {
|
||||
CHECK(lhs == NULL);
|
||||
//CHECK(lhs == NULL);
|
||||
|
||||
numWords = annotatedWordVector.size();
|
||||
}
|
||||
|
@ -234,7 +234,7 @@ void TargetPhrase::SetScoreChart(const FeatureFunction* translationScoreProducer
|
||||
|
||||
// word penalty
|
||||
size_t wordCount = GetNumTerminals();
|
||||
m_scoreBreakdown.Assign(wpProducer, - (float) wordCount * 0.434294482); // TODO log -> ln ??
|
||||
m_scoreBreakdown.Assign(wpProducer, - wordCount);
|
||||
|
||||
m_fullScore = m_scoreBreakdown.GetWeightedScore() - totalNgramScore + totalFullScore + totalOOVScore;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user