mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Allow overriding of feature name
This commit is contained in:
parent
3e60705ec2
commit
881787de3f
@ -79,6 +79,7 @@
|
||||
<listOptionValue builtIn="false" value="boost_thread-mt"/>
|
||||
<listOptionValue builtIn="false" value="boost_filesystem-mt"/>
|
||||
<listOptionValue builtIn="false" value="cmph"/>
|
||||
<listOptionValue builtIn="false" value="pthread"/>
|
||||
</option>
|
||||
<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.128214028" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
|
||||
<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
|
||||
@ -169,4 +170,5 @@
|
||||
</storageModule>
|
||||
<storageModule moduleId="org.eclipse.cdt.make.core.buildtargets"/>
|
||||
<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
|
||||
<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
|
||||
</cproject>
|
||||
|
@ -1356,6 +1356,11 @@
|
||||
<type>2</type>
|
||||
<locationURI>virtual:/virtual</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>TranslationModel/CompactPT</name>
|
||||
<type>2</type>
|
||||
<locationURI>PARENT-8-PROJECT_LOC/home/s0565741/workspace/github/hh/moses/TranslationModel/CompactPT</locationURI>
|
||||
</link>
|
||||
<link>
|
||||
<name>LM/bin/gcc-4.6</name>
|
||||
<type>2</type>
|
||||
|
@ -15,3 +15,13 @@ std::string FF::OutputFactors(const std::vector<int> &factors) const
|
||||
}
|
||||
return ret.str();
|
||||
}
|
||||
|
||||
void FF::OutputWeights(std::ostream &out) const
|
||||
{
|
||||
out << name << index << "= ";
|
||||
for (size_t i = 0; i < numFeatures; ++i) {
|
||||
out << GetWeight() << " ";
|
||||
}
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
|
@ -25,14 +25,7 @@ public:
|
||||
}
|
||||
|
||||
virtual void Output(std::ostream &out) const = 0;
|
||||
virtual void OutputWeights(std::ostream &out) const
|
||||
{
|
||||
out << name << index << "= ";
|
||||
for (size_t i = 0; i < numFeatures; ++i) {
|
||||
out << GetWeight() << " ";
|
||||
}
|
||||
out << std::endl;
|
||||
}
|
||||
virtual void OutputWeights(std::ostream &out) const;
|
||||
|
||||
};
|
||||
|
||||
|
@ -33,6 +33,7 @@ PT::PT(const std::string &line, int numFeatures, bool isHierarchical)
|
||||
void PT::Output(std::ostream &out) const
|
||||
{
|
||||
out << name
|
||||
<< " name=TranslationModel" << index
|
||||
<< " num-features=" << numFeatures
|
||||
<< " path=" << path;
|
||||
|
||||
@ -42,3 +43,12 @@ void PT::Output(std::ostream &out) const
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
void PT::OutputWeights(std::ostream &out) const
|
||||
{
|
||||
out << "TranslationModel" << index << "= ";
|
||||
for (size_t i = 0; i < numFeatures; ++i) {
|
||||
out << GetWeight() << " ";
|
||||
}
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
|
@ -17,4 +17,6 @@ class PT : public FF
|
||||
|
||||
public:
|
||||
PT(const std::string &line, int numFeatures, bool isHierarchical);
|
||||
|
||||
void OutputWeights(std::ostream &out) const;
|
||||
};
|
||||
|
@ -324,11 +324,11 @@ bool Parameter::LoadParam(int argc, char* argv[])
|
||||
return Validate() && noErrorFlag;
|
||||
}
|
||||
|
||||
std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
|
||||
std::vector<float> &Parameter::GetWeights(const std::string &name)
|
||||
{
|
||||
std::vector<float> &ret = m_weights[name + SPrint(ind)];
|
||||
std::vector<float> &ret = m_weights[name];
|
||||
|
||||
cerr << "WEIGHT " << name << ind << "=";
|
||||
cerr << "WEIGHT " << name << "=";
|
||||
for (size_t i = 0; i < ret.size(); ++i) {
|
||||
cerr << ret[i] << ",";
|
||||
}
|
||||
@ -336,6 +336,11 @@ std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<float> &Parameter::GetWeights(const std::string &name, size_t ind)
|
||||
{
|
||||
return GetWeights(name + SPrint(ind));
|
||||
}
|
||||
|
||||
void Parameter::SetWeight(const std::string &name, size_t ind, float weight)
|
||||
{
|
||||
PARAM_VEC &newWeights = m_setting["weight"];
|
||||
|
@ -113,6 +113,7 @@ public:
|
||||
OverwriteParam(GetFullName(paramShortName),values);
|
||||
}
|
||||
|
||||
std::vector<float> &GetWeights(const std::string &name);
|
||||
std::vector<float> &GetWeights(const std::string &name, size_t ind);
|
||||
std::set<std::string> GetWeightNames() const;
|
||||
|
||||
|
@ -564,99 +564,98 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
|
||||
if (feature == "GlobalLexicalModel") {
|
||||
GlobalLexicalModel *model = new GlobalLexicalModel(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "glm") {
|
||||
GlobalLexicalModelUnlimited *model = NULL; //new GlobalLexicalModelUnlimited(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "swd") {
|
||||
SourceWordDeletionFeature *model = new SourceWordDeletionFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "twi") {
|
||||
TargetWordInsertionFeature *model = new TargetWordInsertionFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "PhraseBoundaryFeature") {
|
||||
PhraseBoundaryFeature *model = new PhraseBoundaryFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "pl") {
|
||||
PhraseLengthFeature *model = new PhraseLengthFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "WordTranslationFeature") {
|
||||
WordTranslationFeature *model = new WordTranslationFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "TargetBigramFeature") {
|
||||
TargetBigramFeature *model = new TargetBigramFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
TargetBigramFeature *model = new TargetBigramFeature(line);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "TargetNgramFeature") {
|
||||
TargetNgramFeature *model = new TargetNgramFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
TargetNgramFeature *model = new TargetNgramFeature(line);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "PhrasePairFeature") {
|
||||
PhrasePairFeature *model = new PhrasePairFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
//SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "LexicalReordering") {
|
||||
LexicalReordering *model = new LexicalReordering(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "KENLM") {
|
||||
LanguageModel *model = ConstructKenLM(feature, line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "IRSTLM") {
|
||||
LanguageModelIRST *model = new LanguageModelIRST(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "Generation") {
|
||||
GenerationDictionary *model = new GenerationDictionary(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "BleuScoreFeature") {
|
||||
BleuScoreFeature *model = new BleuScoreFeature(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "SparsePhraseDictionaryFeature") {
|
||||
SparsePhraseDictionaryFeature *model = new SparsePhraseDictionaryFeature(line);
|
||||
//const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
//SetWeights(model, weights);
|
||||
m_sparsePhraseDictionary.push_back(model);
|
||||
}
|
||||
else if (feature == "Distortion") {
|
||||
DistortionScoreProducer *model = new DistortionScoreProducer(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
else if (feature == "WordPenalty") {
|
||||
WordPenaltyProducer *model = new WordPenaltyProducer(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_wpProducer = model;
|
||||
}
|
||||
else if (feature == "UnknownWordPenalty") {
|
||||
UnknownWordPenaltyProducer *model = new UnknownWordPenaltyProducer(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
if (weights.size() == 0)
|
||||
weights.push_back(1.0f);
|
||||
SetWeights(model, weights);
|
||||
@ -664,31 +663,31 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
}
|
||||
else if (feature == "PhraseDictionaryMemory") {
|
||||
PhraseDictionaryMemory* model = new PhraseDictionaryMemory(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_phraseDictionary.push_back(model);
|
||||
}
|
||||
else if (feature == "PhraseDictionaryTreeAdaptor") {
|
||||
PhraseDictionaryTreeAdaptor* model = new PhraseDictionaryTreeAdaptor(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_phraseDictionary.push_back(model);
|
||||
}
|
||||
else if (feature == "PhraseDictionaryOnDisk") {
|
||||
PhraseDictionaryOnDisk* model = new PhraseDictionaryOnDisk(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_phraseDictionary.push_back(model);
|
||||
}
|
||||
else if (feature == "PhraseDictionarySCFG") {
|
||||
PhraseDictionarySCFG* model = new PhraseDictionarySCFG(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_phraseDictionary.push_back(model);
|
||||
}
|
||||
else if (feature == "PhraseDictionaryCompact") {
|
||||
PhraseDictionaryCompact* model = new PhraseDictionaryCompact(line);
|
||||
vector<float> weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
m_phraseDictionary.push_back(model);
|
||||
}
|
||||
@ -697,7 +696,7 @@ bool StaticData::LoadData(Parameter *parameter)
|
||||
#ifdef HAVE_SYNLM
|
||||
else if (feature == "SyntacticLanguageModel") {
|
||||
SyntacticLanguageModel *model = new SyntacticLanguageModel(line);
|
||||
const vector<float> &weights = m_parameter->GetWeights(feature, featureIndex);
|
||||
vector<float> weights = m_parameter->GetWeights(model->GetScoreProducerDescription());
|
||||
SetWeights(model, weights);
|
||||
}
|
||||
#endif
|
||||
|
@ -85,9 +85,6 @@ safesystem("mkdir -p $dir") or die "Can't mkdir $dir";
|
||||
# get tables to be filtered (and modify config file)
|
||||
my (@TABLE,@TABLE_FACTORS,@TABLE_NEW_NAME,%CONSIDER_FACTORS,%KNOWN_TTABLE,@TABLE_WEIGHTS,%TABLE_NUMBER);
|
||||
|
||||
my @PT_FEATURE_NAME;
|
||||
my $pt_weight_index = 0;
|
||||
|
||||
my %new_name_used = ();
|
||||
open(INI_OUT,">$dir/moses.ini") or die "Can't write $dir/moses.ini";
|
||||
open(INI,$config) or die "Can't read $config";
|
||||
@ -159,7 +156,6 @@ while(my $line = <INI>) {
|
||||
}
|
||||
|
||||
$toks[0] = $phrase_table_impl;
|
||||
push @PT_FEATURE_NAME, $phrase_table_impl;
|
||||
|
||||
print INI_OUT join_array(\@toks)."\n";
|
||||
|
||||
@ -215,17 +211,6 @@ while(my $line = <INI>) {
|
||||
|
||||
|
||||
} #elsif (/LexicalReordering /) {
|
||||
elsif ($line =~ /PhraseDictionaryMemory[0-9]*= /
|
||||
|| $line =~ /PhraseDictionaryTreeAdaptor[0-9]*= /
|
||||
|| $line =~ /PhraseDictionaryOnDisk[0-9]*= /
|
||||
|| $line =~ /PhraseDictionarySCFG[0-9]*= /) {
|
||||
|
||||
my $newFeatureName = $PT_FEATURE_NAME[$pt_weight_index];
|
||||
$line =~ s/PhraseDictionary[a-zA-Z]*/$newFeatureName/g;
|
||||
print INI_OUT "$line\n";
|
||||
|
||||
++$pt_weight_index;
|
||||
}
|
||||
else {
|
||||
print INI_OUT "$line\n";
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user