mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
rescale LM score for chart decoding
This commit is contained in:
parent
8edbfebbf1
commit
aa9689d12b
@ -292,7 +292,7 @@ void PhraseAndFeatures(const TranslationSystem &system, const search::Applied fi
|
||||
const LanguageModel &model = **lmList.begin();
|
||||
model.CalcScore(phrase, full, ignored_ngram, ignored_oov);
|
||||
// CalcScore transforms, but EvaluateChart doesn't.
|
||||
features.Assign(&model, UntransformLMScore(full));
|
||||
features.Assign(&model, full);
|
||||
}
|
||||
|
||||
} // namespace Incremental
|
||||
|
@ -249,7 +249,7 @@ FFState* LanguageModelImplementation::EvaluateChart(const ChartHypothesis& hypo,
|
||||
// score a regular word added by the rule
|
||||
else
|
||||
{
|
||||
updateChartScore( &prefixScore, &finalizedScore, UntransformLMScore(GetValueGivenState(contextFactor, *lmState).score), ++wordPos );
|
||||
updateChartScore( &prefixScore, &finalizedScore, GetValueGivenState(contextFactor, *lmState).score, ++wordPos );
|
||||
}
|
||||
}
|
||||
|
||||
@ -298,7 +298,7 @@ FFState* LanguageModelImplementation::EvaluateChart(const ChartHypothesis& hypo,
|
||||
{
|
||||
const Word &word = prevState->GetPrefix().GetWord(prefixPos);
|
||||
ShiftOrPush(contextFactor, word);
|
||||
updateChartScore( &prefixScore, &finalizedScore, UntransformLMScore(GetValueGivenState(contextFactor, *lmState).score), ++wordPos );
|
||||
updateChartScore( &prefixScore, &finalizedScore, GetValueGivenState(contextFactor, *lmState).score, ++wordPos );
|
||||
}
|
||||
|
||||
// check if we are dealing with a large sub-phrase
|
||||
|
@ -631,9 +631,21 @@ void Parameter::ConvertWeightArgsDistortion()
|
||||
|
||||
}
|
||||
|
||||
void Parameter::ConvertWeightArgsLM(const string &oldWeightName)
|
||||
void Parameter::ConvertWeightArgsLM()
|
||||
{
|
||||
string oldFeatureName = "lmodel-file";
|
||||
const string oldWeightName = "weight-l";
|
||||
const string oldFeatureName = "lmodel-file";
|
||||
|
||||
bool isChartDecoding = true;
|
||||
if (!isParamSpecified("search-algorithm") ||
|
||||
(GetParam("search-algorithm").size() > 0
|
||||
&& (Trim(GetParam("search-algorithm")[0]) == "0"
|
||||
||Trim(GetParam("search-algorithm")[0]) == "1"
|
||||
)
|
||||
)
|
||||
) {
|
||||
isChartDecoding = false;
|
||||
}
|
||||
|
||||
vector<int> oovWeights;
|
||||
if (isParamSpecified("lmodel-oov-feature")) {
|
||||
@ -682,6 +694,10 @@ void Parameter::ConvertWeightArgsLM(const string &oldWeightName)
|
||||
{
|
||||
CHECK(currOldInd < weights.size());
|
||||
weightsLM[currFF] = Scan<float>(weights[currOldInd]);
|
||||
if (isChartDecoding) {
|
||||
weightsLM[currFF] = UntransformLMScore(weightsLM[currFF]);
|
||||
}
|
||||
|
||||
++currOldInd;
|
||||
}
|
||||
|
||||
@ -803,7 +819,7 @@ void Parameter::ConvertWeightArgs()
|
||||
}
|
||||
|
||||
ConvertWeightArgsWordPenalty();
|
||||
ConvertWeightArgsLM("weight-l");
|
||||
ConvertWeightArgsLM();
|
||||
ConvertWeightArgsSingleWeight("weight-slm", "SyntacticLM");
|
||||
ConvertWeightArgsSingleWeight("weight-u", "UnknownWordPenalty");
|
||||
ConvertWeightArgsGeneration("weight-generation", "Generation");
|
||||
|
@ -69,7 +69,7 @@ protected:
|
||||
void ConvertWeightArgs();
|
||||
void ConvertWeightArgsSingleWeight(const std::string &oldWeightName, const std::string &newWeightName);
|
||||
void ConvertWeightArgsPhraseModel(const std::string &oldWeightName);
|
||||
void ConvertWeightArgsLM(const std::string &oldWeightName);
|
||||
void ConvertWeightArgsLM();
|
||||
void ConvertWeightArgsDistortion();
|
||||
void ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string &newWeightName);
|
||||
void ConvertWeightArgsWordPenalty();
|
||||
|
Loading…
Reference in New Issue
Block a user