moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-12-27 22:14:57 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor LanguageModelIRST

Browse Source
This commit is contained in:
Hieu Hoang 2013-01-17 17:58:44 +00:00
parent 75ef646918
commit a57ac78741
4 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
moses/LM/IRST.cpp
View File

@ -41,6 +41,13 @@ namespace Moses
{
LanguageModelIRST::LanguageModelIRST(const std::string &line)
{
vector<string> tokens = Tokenize(line);
FactorType factorType = Scan<FactorType>(tokens[1]);
size_t nGramOrder = Scan<size_t>(tokens[2]);
const string &filePath = tokens[3];
Load(filePath, factorType, nGramOrder);
}

4
moses/LM/Ken.cpp
View File

@ -336,8 +336,8 @@ LanguageModel *ConstructKenLM(const std::string &line)
vector<string> tokens = Tokenize(line);
FactorType factorType = Scan<FactorType>(tokens[1]);
bool lazy = Scan<bool>(tokens[2]);
const string &file = tokens[3];
bool lazy = Scan<bool>(tokens[3]);
const string &file = tokens[4];
return ConstructKenLM(file, factorType, lazy);
}

6
moses/Parameter.cpp
View File

@ -549,14 +549,16 @@ void Parameter::ConvertWeightArgsLM(const string &oldWeightName)
SetWeight(newWeightName, ind, weightsLM);
string featureLine = newWeightName + " "
+ modelToks[1] + " ";
+ modelToks[1] + " " // factor
+ modelToks[2] + " "; // order
if (lmType == LazyKen) {
featureLine += "1 ";
}
else if (lmType == Ken) {
featureLine += "0 ";
}
featureLine += modelToks[3];
featureLine += modelToks[3]; // file
AddFeature(featureLine);
} // for (size_t lmIndex = 0; lmIndex < models.size(); ++lmIndex) {

6
moses/StaticData.cpp
View File

@ -617,13 +617,13 @@ SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
SetWeights(model, weights);
}
/*
else if (feature == "IRSTLM") {
LanguageModel *model = LanguageModelIRST(line);
LanguageModelIRST *irstlm = new LanguageModelIRST(line);
LanguageModel *model = new LMRefCount(irstlm);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
SetWeights(model, weights);
}
*/
else {
UserMessage::Add("Unknown feature function");
return false;