moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow overriding of feature name

Browse Source
This commit is contained in:
Hieu Hoang 2013-03-06 14:04:09 +00:00
parent 3e60705ec2
commit 881787de3f
10 changed files with 65 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/other-builds/moses-chart-cmd/.cproject
View File

@ -79,6 +79,7 @@
<listOptionValue builtIn="false" value="boost_thread-mt"/>
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
<listOptionValue builtIn="false" value="cmph"/>
<listOptionValue builtIn="false" value="pthread"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.128214028" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
@ -169,4 +170,5 @@
</storageModule>
<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
</cproject>

5
contrib/other-builds/moses/.project
View File

@ -1356,6 +1356,11 @@
<type>2</type>
<locationURI>virtual:/virtual</locationURI>
</link>
<link>
<name>TranslationModel/CompactPT</name>
<type>2</type>
<locationURI>PARENT-8-PROJECT_LOC/home/s0565741/workspace/github/hh/moses/TranslationModel/CompactPT</locationURI>
</link>
<link>
<name>LM/bin/gcc-4.6</name>
<type>2</type>

10
misc/create-ini/FF.cpp
View File

@ -15,3 +15,13 @@ std::string FF::OutputFactors(const std::vector<int> &factors) const
}
return ret.str();
}
void FF::OutputWeights(std::ostream &out) const
{
out << name << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}

9
misc/create-ini/FF.h
View File

@ -25,14 +25,7 @@ public:
}
virtual void Output(std::ostream &out) const = 0;
virtual void OutputWeights(std::ostream &out) const
{
out << name << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}
virtual void OutputWeights(std::ostream &out) const;
};

10
misc/create-ini/PT.cpp
View File

@ -33,6 +33,7 @@ PT::PT(const std::string &line, int numFeatures, bool isHierarchical)
void PT::Output(std::ostream &out) const
{
out << name
<< " name=TranslationModel" << index
<< " num-features=" << numFeatures
<< " path=" << path;
@ -42,3 +43,12 @@ void PT::Output(std::ostream &out) const
out << std::endl;
}
void PT::OutputWeights(std::ostream &out) const
{
out << "TranslationModel" << index << "= ";
for (size_t i = 0; i < numFeatures; ++i) {
out << GetWeight() << " ";
}
out << std::endl;
}

2
misc/create-ini/PT.h
View File

@ -17,4 +17,6 @@ class PT : public FF
public:
PT(const std::string &line, int numFeatures, bool isHierarchical);
void OutputWeights(std::ostream &out) const;
};

11
moses/Parameter.cpp
View File

@ -324,11 +324,11 @@ bool Parameter::LoadParam(int argc, char* argv[])
return Validate() && noErrorFlag;
}
std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
std::vector<float> &Parameter::GetWeights(const std::string &name)
{
std::vector<float> &ret = m_weights[name + SPrint(ind)];
std::vector<float> &ret = m_weights[name];
cerr << "WEIGHT " << name << ind << "=";
cerr << "WEIGHT " << name << "=";
for (size_t i = 0; i < ret.size(); ++i) {
cerr << ret[i] << ",";
}
@ -336,6 +336,11 @@ std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
return ret;
}
std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
{
return GetWeights(name + SPrint(ind));
}
void Parameter::SetWeight(const std::string &name, size_t ind, float weight)
{
PARAM_VEC &newWeights = m_setting["weight"];

1
moses/Parameter.h
View File

@ -113,6 +113,7 @@ public:
OverwriteParam(GetFullName(paramShortName),values);
}
std::vector<float> &GetWeights(const std::string &name);
std::vector<float> &GetWeights(const std::string &name, size_t ind);
std::set<std::string> GetWeightNames() const;

53
moses/StaticData.cpp
View File

@ -564,99 +564,98 @@ bool StaticData::LoadData(Parameter *parameter)
if (feature == "GlobalLexicalModel") {
GlobalLexicalModel *model = new GlobalLexicalModel(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "glm") {
GlobalLexicalModelUnlimited *model = NULL; //new GlobalLexicalModelUnlimited(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "swd") {
SourceWordDeletionFeature *model = new SourceWordDeletionFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "twi") {
TargetWordInsertionFeature *model = new TargetWordInsertionFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "PhraseBoundaryFeature") {
PhraseBoundaryFeature *model = new PhraseBoundaryFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "pl") {
PhraseLengthFeature *model = new PhraseLengthFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "WordTranslationFeature") {
WordTranslationFeature *model = new WordTranslationFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "TargetBigramFeature") {
TargetBigramFeature *model = new TargetBigramFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
TargetBigramFeature *model = new TargetBigramFeature(line);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "TargetNgramFeature") {
TargetNgramFeature *model = new TargetNgramFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
TargetNgramFeature *model = new TargetNgramFeature(line);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "PhrasePairFeature") {
PhrasePairFeature *model = new PhrasePairFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
//SetWeights(model, weights);
}
else if (feature == "LexicalReordering") {
LexicalReordering *model = new LexicalReordering(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "KENLM") {
LanguageModel *model = ConstructKenLM(feature, line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "IRSTLM") {
LanguageModelIRST *model = new LanguageModelIRST(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "Generation") {
GenerationDictionary *model = new GenerationDictionary(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "BleuScoreFeature") {
BleuScoreFeature *model = new BleuScoreFeature(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "SparsePhraseDictionaryFeature") {
SparsePhraseDictionaryFeature *model = new SparsePhraseDictionaryFeature(line);
//const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
//SetWeights(model, weights);
m_sparsePhraseDictionary.push_back(model);
}
else if (feature == "Distortion") {
DistortionScoreProducer *model = new DistortionScoreProducer(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
else if (feature == "WordPenalty") {
WordPenaltyProducer *model = new WordPenaltyProducer(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_wpProducer = model;
}
else if (feature == "UnknownWordPenalty") {
UnknownWordPenaltyProducer *model = new UnknownWordPenaltyProducer(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
if (weights.size() == 0)
weights.push_back(1.0f);
SetWeights(model, weights);
@ -664,31 +663,31 @@ bool StaticData::LoadData(Parameter *parameter)
}
else if (feature == "PhraseDictionaryMemory") {
PhraseDictionaryMemory* model = new PhraseDictionaryMemory(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_phraseDictionary.push_back(model);
}
else if (feature == "PhraseDictionaryTreeAdaptor") {
PhraseDictionaryTreeAdaptor* model = new PhraseDictionaryTreeAdaptor(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_phraseDictionary.push_back(model);
}
else if (feature == "PhraseDictionaryOnDisk") {
PhraseDictionaryOnDisk* model = new PhraseDictionaryOnDisk(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_phraseDictionary.push_back(model);
}
else if (feature == "PhraseDictionarySCFG") {
PhraseDictionarySCFG* model = new PhraseDictionarySCFG(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_phraseDictionary.push_back(model);
}
else if (feature == "PhraseDictionaryCompact") {
PhraseDictionaryCompact* model = new PhraseDictionaryCompact(line);
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
m_phraseDictionary.push_back(model);
}
@ -697,7 +696,7 @@ bool StaticData::LoadData(Parameter *parameter)
#ifdef HAVE_SYNLM
else if (feature == "SyntacticLanguageModel") {
SyntacticLanguageModel *model = new SyntacticLanguageModel(line);
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
SetWeights(model, weights);
}
#endif

15
scripts/training/filter-model-given-input.pl
View File

@ -85,9 +85,6 @@ safesystem("mkdir -p $dir") or die "Can't mkdir $dir";
# get tables to be filtered (and modify config file)
my (@TABLE,@TABLE_FACTORS,@TABLE_NEW_NAME,%CONSIDER_FACTORS,%KNOWN_TTABLE,@TABLE_WEIGHTS,%TABLE_NUMBER);
my @PT_FEATURE_NAME;
my $pt_weight_index = 0;
my %new_name_used = ();
open(INI_OUT,">$dir/moses.ini") or die "Can't write $dir/moses.ini";
open(INI,$config) or die "Can't read $config";
@ -159,7 +156,6 @@ while(my $line = <INI>) {
}
$toks[0] = $phrase_table_impl;
push @PT_FEATURE_NAME, $phrase_table_impl;
print INI_OUT join_array(\@toks)."\n";
@ -215,17 +211,6 @@ while(my $line = <INI>) {
} #elsif (/LexicalReordering /) {
elsif ($line =~ /PhraseDictionaryMemory[0-9]*= /
|| $line =~ /PhraseDictionaryTreeAdaptor[0-9]*= /
|| $line =~ /PhraseDictionaryOnDisk[0-9]*= /
|| $line =~ /PhraseDictionarySCFG[0-9]*= /) {
my $newFeatureName = $PT_FEATURE_NAME[$pt_weight_index];
$line =~ s/PhraseDictionary[a-zA-Z]*/$newFeatureName/g;
print INI_OUT "$line\n";
++$pt_weight_index;
}
else {
print INI_OUT "$line\n";
}