mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 13:23:25 +03:00
refactor Syntactic LM. Can't find dependencies to do this properly. Ask Lane later on
This commit is contained in:
Hieu Hoang
parent
49345b6b56
commit
00ebc1e5ab
@ -628,7 +628,13 @@ SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
|
||||
#ifdef HAVE_SYNLM
|
||||
else if (feature == "SyntacticLanguageModel") {
|
||||
SyntacticLanguageModel *model = new SyntacticLanguageModel(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
#endif
|
||||
else {
|
||||
UserMessage::Add("Unknown feature function");
|
||||
return false;
|
||||
@ -638,13 +644,6 @@ SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord);
|
||||
|
||||
CollectFeatureFunctions();
|
||||
m_fLMsLoaded = true;
|
||||
|
||||
#ifdef HAVE_SYNLM
|
||||
if (m_parameter->GetParam("slmodel-file").size() > 0) {
|
||||
if (!LoadSyntacticLanguageModel()) return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
if (!LoadPhraseTables()) return false;
|
||||
if (!LoadDecodeGraphs()) return false;
|
||||
@ -740,61 +739,6 @@ StaticData::~StaticData()
|
||||
Phrase::FinalizeMemPool();
|
||||
}
|
||||
|
||||
#ifdef HAVE_SYNLM
|
||||
bool StaticData::LoadSyntacticLanguageModel() {
|
||||
cerr << "Loading syntactic language models..." << std::endl;
|
||||
|
||||
const vector<float> weights = Scan<float>(m_parameter->GetParam("weight-slm"));
|
||||
const vector<string> files = m_parameter->GetParam("slmodel-file");
|
||||
|
||||
const FactorType factorType = (m_parameter->GetParam("slmodel-factor").size() > 0) ?
|
||||
TransformScore(Scan<int>(m_parameter->GetParam("slmodel-factor")[0]))
|
||||
: 0;
|
||||
|
||||
const size_t beamWidth = (m_parameter->GetParam("slmodel-beam").size() > 0) ?
|
||||
TransformScore(Scan<int>(m_parameter->GetParam("slmodel-beam")[0]))
|
||||
: 500;
|
||||
|
||||
if (files.size() < 1) {
|
||||
cerr << "No syntactic language model files specified!" << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
// check if feature is used
|
||||
if (weights.size() >= 1) {
|
||||
|
||||
//cout.setf(ios::scientific,ios::floatfield);
|
||||
//cerr.setf(ios::scientific,ios::floatfield);
|
||||
|
||||
// create the feature
|
||||
m_syntacticLanguageModel = new SyntacticLanguageModel(files,weights,factorType,beamWidth);
|
||||
|
||||
/*
|
||||
/////////////////////////////////////////
|
||||
// BEGIN LANE's UNSTABLE EXPERIMENT :)
|
||||
//
|
||||
|
||||
double ppl = m_syntacticLanguageModel->perplexity();
|
||||
cerr << "Probability is " << ppl << endl;
|
||||
|
||||
|
||||
//
|
||||
// END LANE's UNSTABLE EXPERIMENT
|
||||
/////////////////////////////////////////
|
||||
*/
|
||||
|
||||
|
||||
if (m_syntacticLanguageModel==NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Doesn't load phrase tables any more. Just creates the features. */
|
||||
bool StaticData::LoadPhraseTables()
|
||||
{
|
||||
|
||||
@ -60,10 +60,6 @@ class DecodeStep;
|
||||
class UnknownWordPenaltyProducer;
|
||||
class MetaScoreProducer;
|
||||
class MetaFeatureProducer;
|
||||
|
||||
#ifdef HAVE_SYNLM
|
||||
class SyntacticLanguageModel;
|
||||
#endif
|
||||
class TranslationSystem;
|
||||
|
||||
typedef std::pair<std::string, float> UnknownLHSEntry;
|
||||
@ -87,9 +83,6 @@ protected:
|
||||
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
|
||||
LMList m_languageModel;
|
||||
ScoreComponentCollection m_allWeights;
|
||||
#ifdef HAVE_SYNLM
|
||||
SyntacticLanguageModel* m_syntacticLanguageModel;
|
||||
#endif
|
||||
|
||||
std::vector<DecodeGraph*> m_decodeGraphs;
|
||||
std::vector<size_t> m_decodeGraphBackoff;
|
||||
@ -233,10 +226,7 @@ protected:
|
||||
|
||||
//! helper fn to set bool param from ini file/command line
|
||||
void SetBooleanParameter(bool *paramter, std::string parameterName, bool defaultValue);
|
||||
#ifdef HAVE_SYNLM
|
||||
//! load syntactic language model
|
||||
bool LoadSyntacticLanguageModel();
|
||||
#endif
|
||||
|
||||
//! load not only the main phrase table but also any auxiliary tables that depend on which features are being used (e.g., word-deletion, word-insertion tables)
|
||||
bool LoadPhraseTables();
|
||||
//! load decoding steps
|
||||
|
||||
@ -10,21 +10,67 @@
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
// asnteousntaoheisnthaoesntih
|
||||
SyntacticLanguageModel::SyntacticLanguageModel(const std::vector<std::string>& filePath,
|
||||
const std::vector<float>& weights,
|
||||
const FactorType factorType,
|
||||
size_t beamWidth)
|
||||
// Initialize member variables
|
||||
SyntacticLanguageModel::SyntacticLanguageModel(const std::string &line)
|
||||
// Initialize member variables
|
||||
/*
|
||||
: m_NumScoreComponents(weights.size())
|
||||
, m_files(new SyntacticLanguageModelFiles<YModel,XModel>(filePath))
|
||||
, m_factorType(factorType)
|
||||
, m_beamWidth(beamWidth) {
|
||||
*/
|
||||
{
|
||||
/* taken from StaticData::LoadSyntacticLanguageModel()
|
||||
cerr << "Loading syntactic language models..." << std::endl;
|
||||
|
||||
// Inform Moses score manager of this feature and its weight(s)
|
||||
const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this);
|
||||
const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights);
|
||||
VERBOSE(3,"Constructed SyntacticLanguageModel" << endl);
|
||||
const vector<float> weights = Scan<float>(m_parameter->GetParam("weight-slm"));
|
||||
const vector<string> files = m_parameter->GetParam("slmodel-file");
|
||||
|
||||
const FactorType factorType = (m_parameter->GetParam("slmodel-factor").size() > 0) ?
|
||||
TransformScore(Scan<int>(m_parameter->GetParam("slmodel-factor")[0]))
|
||||
: 0;
|
||||
|
||||
const size_t beamWidth = (m_parameter->GetParam("slmodel-beam").size() > 0) ?
|
||||
TransformScore(Scan<int>(m_parameter->GetParam("slmodel-beam")[0]))
|
||||
: 500;
|
||||
|
||||
if (files.size() < 1) {
|
||||
cerr << "No syntactic language model files specified!" << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
// check if feature is used
|
||||
if (weights.size() >= 1) {
|
||||
|
||||
//cout.setf(ios::scientific,ios::floatfield);
|
||||
//cerr.setf(ios::scientific,ios::floatfield);
|
||||
|
||||
// create the feature
|
||||
m_syntacticLanguageModel = new SyntacticLanguageModel(files,weights,factorType,beamWidth);
|
||||
|
||||
|
||||
/////////////////////////////////////////
|
||||
// BEGIN LANE's UNSTABLE EXPERIMENT :)
|
||||
//
|
||||
|
||||
//double ppl = m_syntacticLanguageModel->perplexity();
|
||||
//cerr << "Probability is " << ppl << endl;
|
||||
|
||||
|
||||
//
|
||||
// END LANE's UNSTABLE EXPERIMENT
|
||||
/////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
if (m_syntacticLanguageModel==NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
*/
|
||||
}
|
||||
|
||||
SyntacticLanguageModel::~SyntacticLanguageModel() {
|
||||
|
||||
@ -17,11 +17,7 @@ namespace Moses
|
||||
class SyntacticLanguageModel : public StatefulFeatureFunction {
|
||||
|
||||
public:
|
||||
|
||||
SyntacticLanguageModel(const std::vector<std::string>& filePaths,
|
||||
const std::vector<float>& weights,
|
||||
const FactorType factorType,
|
||||
const size_t beamWidth);
|
||||
SyntacticLanguageModel(const std::string &line);
|
||||
|
||||
~SyntacticLanguageModel();
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user