mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-26 13:23:25 +03:00
Revert "Merge branch 'master' of ../mosesdecoder.export.bundle"
This reverts commita106d27bf7
, reversing changes made to7c93c47548
.
This commit is contained in:
parent
a106d27bf7
commit
2b0f3f11c1
@ -65,6 +65,7 @@ StaticData::StaticData()
|
|||||||
,m_factorDelimiter("|") // default delimiter between factors
|
,m_factorDelimiter("|") // default delimiter between factors
|
||||||
,m_lmEnableOOVFeature(false)
|
,m_lmEnableOOVFeature(false)
|
||||||
,m_isAlwaysCreateDirectTranslationOption(false)
|
,m_isAlwaysCreateDirectTranslationOption(false)
|
||||||
|
,m_currentWeightSetting("default")
|
||||||
,m_treeStructure(NULL)
|
,m_treeStructure(NULL)
|
||||||
{
|
{
|
||||||
m_xmlBrackets.first="<";
|
m_xmlBrackets.first="<";
|
||||||
@ -560,7 +561,7 @@ bool StaticData::LoadData(Parameter *parameter)
|
|||||||
UserMessage::Add("Unable to load weights from " + extraWeightConfig[0]);
|
UserMessage::Add("Unable to load weights from " + extraWeightConfig[0]);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
AccessAllWeights().PlusEquals(extraWeights);
|
m_allWeights.PlusEquals(extraWeights);
|
||||||
}
|
}
|
||||||
|
|
||||||
//Load sparse features from config (overrules weight file)
|
//Load sparse features from config (overrules weight file)
|
||||||
@ -596,14 +597,14 @@ void StaticData::SetBooleanParameter( bool *parameter, string parameterName, boo
|
|||||||
|
|
||||||
void StaticData::SetWeight(const FeatureFunction* sp, float weight)
|
void StaticData::SetWeight(const FeatureFunction* sp, float weight)
|
||||||
{
|
{
|
||||||
AccessAllWeights().Resize();
|
m_allWeights.Resize();
|
||||||
AccessAllWeights().Assign(sp,weight);
|
m_allWeights.Assign(sp,weight);
|
||||||
}
|
}
|
||||||
|
|
||||||
void StaticData::SetWeights(const FeatureFunction* sp, const std::vector<float>& weights)
|
void StaticData::SetWeights(const FeatureFunction* sp, const std::vector<float>& weights)
|
||||||
{
|
{
|
||||||
AccessAllWeights().Resize();
|
m_allWeights.Resize();
|
||||||
AccessAllWeights().Assign(sp,weights);
|
m_allWeights.Assign(sp,weights);
|
||||||
}
|
}
|
||||||
|
|
||||||
void StaticData::LoadNonTerminals()
|
void StaticData::LoadNonTerminals()
|
||||||
@ -1006,7 +1007,7 @@ void StaticData::LoadSparseWeightsFromConfig() {
|
|||||||
// this indicates that it is sparse feature
|
// this indicates that it is sparse feature
|
||||||
if (featureNames.find(iter->first) == featureNames.end()) {
|
if (featureNames.find(iter->first) == featureNames.end()) {
|
||||||
UTIL_THROW_IF2(iter->second.size() != 1, "ERROR: only one weight per sparse feature allowed: " << iter->first);
|
UTIL_THROW_IF2(iter->second.size() != 1, "ERROR: only one weight per sparse feature allowed: " << iter->first);
|
||||||
AccessAllWeights().Assign(iter->first, iter->second[0]);
|
m_allWeights.Assign(iter->first, iter->second[0]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1031,7 +1032,7 @@ bool StaticData::LoadAlternateWeightSettings()
|
|||||||
}
|
}
|
||||||
|
|
||||||
// copy main weight setting as default
|
// copy main weight setting as default
|
||||||
m_weightSetting["default"] = new ScoreComponentCollection( AccessAllWeights() );
|
m_weightSetting["default"] = new ScoreComponentCollection( m_allWeights );
|
||||||
|
|
||||||
// go through specification in config file
|
// go through specification in config file
|
||||||
string currentId = "";
|
string currentId = "";
|
||||||
@ -1194,7 +1195,7 @@ void StaticData::CheckLEGACYPT()
|
|||||||
|
|
||||||
void StaticData::ResetWeights(const std::string &denseWeights, const std::string &sparseFile)
|
void StaticData::ResetWeights(const std::string &denseWeights, const std::string &sparseFile)
|
||||||
{
|
{
|
||||||
SetAllWeights(ScoreComponentCollection());
|
m_allWeights = ScoreComponentCollection();
|
||||||
|
|
||||||
// dense weights
|
// dense weights
|
||||||
string name("");
|
string name("");
|
||||||
@ -1209,7 +1210,7 @@ void StaticData::ResetWeights(const std::string &denseWeights, const std::string
|
|||||||
if (name != "") {
|
if (name != "") {
|
||||||
// save previous ff
|
// save previous ff
|
||||||
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
|
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
|
||||||
AccessAllWeights().Assign(&ff, weights);
|
m_allWeights.Assign(&ff, weights);
|
||||||
weights.clear();
|
weights.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1222,7 +1223,7 @@ void StaticData::ResetWeights(const std::string &denseWeights, const std::string
|
|||||||
}
|
}
|
||||||
|
|
||||||
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
|
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(name);
|
||||||
AccessAllWeights().Assign(&ff, weights);
|
m_allWeights.Assign(&ff, weights);
|
||||||
|
|
||||||
// sparse weights
|
// sparse weights
|
||||||
InputFileStream sparseStrme(sparseFile);
|
InputFileStream sparseStrme(sparseFile);
|
||||||
@ -1235,7 +1236,7 @@ void StaticData::ResetWeights(const std::string &denseWeights, const std::string
|
|||||||
UTIL_THROW_IF2(names.size() != 2, "Incorrect sparse weight name. Should be FFName_spareseName");
|
UTIL_THROW_IF2(names.size() != 2, "Incorrect sparse weight name. Should be FFName_spareseName");
|
||||||
|
|
||||||
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(names[0]);
|
const FeatureFunction &ff = FeatureFunction::FindFeatureFunction(names[0]);
|
||||||
AccessAllWeights().Assign(&ff, names[1], Scan<float>(toks[1]));
|
m_allWeights.Assign(&ff, names[1], Scan<float>(toks[1]));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -36,7 +36,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|||||||
#ifdef WITH_THREADS
|
#ifdef WITH_THREADS
|
||||||
#include <boost/thread.hpp>
|
#include <boost/thread.hpp>
|
||||||
#include <boost/thread/mutex.hpp>
|
#include <boost/thread/mutex.hpp>
|
||||||
#include <boost/thread/tss.hpp>
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include "Parameter.h"
|
#include "Parameter.h"
|
||||||
@ -65,46 +64,10 @@ class StaticData
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
static StaticData s_instance;
|
static StaticData s_instance;
|
||||||
|
|
||||||
ScoreComponentCollection& AccessAllWeights() const {
|
|
||||||
#ifdef WITH_THREADS
|
|
||||||
return *(m_allWeights.get());
|
|
||||||
#else
|
|
||||||
return m_allWeights;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string& AccessCurrentWeightSetting() const {
|
|
||||||
#ifdef WITH_THREADS
|
|
||||||
std::string * pointer = m_currentWeightSetting.get();
|
|
||||||
if (pointer == NULL) {
|
|
||||||
SetCurrentWeightSetting("default");
|
|
||||||
return *(m_currentWeightSetting.get());
|
|
||||||
} else {
|
|
||||||
return *pointer;
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
return m_currentWeightSetting;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
void SetCurrentWeightSetting(std::string weightSetting) const {
|
|
||||||
#ifdef WITH_THREADS
|
|
||||||
m_currentWeightSetting.reset(new std::string(weightSetting));
|
|
||||||
#else
|
|
||||||
m_currentWeightSetting = weightSetting;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
Parameter *m_parameter;
|
Parameter *m_parameter;
|
||||||
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
|
std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder;
|
||||||
|
|
||||||
#ifdef WITH_THREADS
|
|
||||||
mutable boost::thread_specific_ptr< ScoreComponentCollection > m_allWeights;
|
|
||||||
#else
|
|
||||||
mutable ScoreComponentCollection m_allWeights;
|
mutable ScoreComponentCollection m_allWeights;
|
||||||
#endif
|
|
||||||
|
|
||||||
std::vector<DecodeGraph*> m_decodeGraphs;
|
std::vector<DecodeGraph*> m_decodeGraphs;
|
||||||
|
|
||||||
@ -228,11 +191,7 @@ protected:
|
|||||||
long m_startTranslationId;
|
long m_startTranslationId;
|
||||||
|
|
||||||
// alternate weight settings
|
// alternate weight settings
|
||||||
#ifdef WITH_THREADS
|
|
||||||
mutable boost::thread_specific_ptr< std::string > m_currentWeightSetting;
|
|
||||||
#else
|
|
||||||
mutable std::string m_currentWeightSetting;
|
mutable std::string m_currentWeightSetting;
|
||||||
#endif
|
|
||||||
std::map< std::string, ScoreComponentCollection* > m_weightSetting; // core weights
|
std::map< std::string, ScoreComponentCollection* > m_weightSetting; // core weights
|
||||||
std::map< std::string, std::set< std::string > > m_weightSettingIgnoreFF; // feature function
|
std::map< std::string, std::set< std::string > > m_weightSettingIgnoreFF; // feature function
|
||||||
std::map< std::string, std::set< size_t > > m_weightSettingIgnoreDP; // decoding path
|
std::map< std::string, std::set< size_t > > m_weightSettingIgnoreDP; // decoding path
|
||||||
@ -468,20 +427,16 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
const ScoreComponentCollection& GetAllWeights() const {
|
const ScoreComponentCollection& GetAllWeights() const {
|
||||||
return AccessAllWeights();
|
return m_allWeights;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SetAllWeights(const ScoreComponentCollection& weights) {
|
void SetAllWeights(const ScoreComponentCollection& weights) {
|
||||||
#ifdef WITH_THREADS
|
|
||||||
m_allWeights.reset(new ScoreComponentCollection(weights));
|
|
||||||
#else
|
|
||||||
m_allWeights = weights;
|
m_allWeights = weights;
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//Weight for a single-valued feature
|
//Weight for a single-valued feature
|
||||||
float GetWeight(const FeatureFunction* sp) const {
|
float GetWeight(const FeatureFunction* sp) const {
|
||||||
return AccessAllWeights().GetScoreForProducer(sp);
|
return m_allWeights.GetScoreForProducer(sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
//Weight for a single-valued feature
|
//Weight for a single-valued feature
|
||||||
@ -490,11 +445,11 @@ public:
|
|||||||
|
|
||||||
//Weights for feature with fixed number of values
|
//Weights for feature with fixed number of values
|
||||||
std::vector<float> GetWeights(const FeatureFunction* sp) const {
|
std::vector<float> GetWeights(const FeatureFunction* sp) const {
|
||||||
return AccessAllWeights().GetScoresForProducer(sp);
|
return m_allWeights.GetScoresForProducer(sp);
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetSparseWeight(const FName& featureName) const {
|
float GetSparseWeight(const FName& featureName) const {
|
||||||
return AccessAllWeights().GetSparseWeight(featureName);
|
return m_allWeights.GetSparseWeight(featureName);
|
||||||
}
|
}
|
||||||
|
|
||||||
//Weights for feature with fixed number of values
|
//Weights for feature with fixed number of values
|
||||||
@ -694,7 +649,7 @@ public:
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
std::map< std::string, std::set< std::string > >::const_iterator lookupIgnoreFF
|
std::map< std::string, std::set< std::string > >::const_iterator lookupIgnoreFF
|
||||||
= m_weightSettingIgnoreFF.find( AccessCurrentWeightSetting() );
|
= m_weightSettingIgnoreFF.find( m_currentWeightSetting );
|
||||||
if (lookupIgnoreFF == m_weightSettingIgnoreFF.end()) {
|
if (lookupIgnoreFF == m_weightSettingIgnoreFF.end()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -712,7 +667,7 @@ public:
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
std::map< std::string, std::set< size_t > >::const_iterator lookupIgnoreDP
|
std::map< std::string, std::set< size_t > >::const_iterator lookupIgnoreDP
|
||||||
= m_weightSettingIgnoreDP.find( AccessCurrentWeightSetting() );
|
= m_weightSettingIgnoreDP.find( m_currentWeightSetting );
|
||||||
if (lookupIgnoreDP == m_weightSettingIgnoreDP.end()) {
|
if (lookupIgnoreDP == m_weightSettingIgnoreDP.end()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -725,7 +680,7 @@ public:
|
|||||||
void SetWeightSetting(const std::string &settingName) const {
|
void SetWeightSetting(const std::string &settingName) const {
|
||||||
|
|
||||||
// if no change in weight setting, do nothing
|
// if no change in weight setting, do nothing
|
||||||
if (AccessCurrentWeightSetting() == settingName) {
|
if (m_currentWeightSetting == settingName) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -736,7 +691,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
// find the setting
|
// find the setting
|
||||||
SetCurrentWeightSetting(settingName);
|
m_currentWeightSetting = settingName;
|
||||||
std::map< std::string, ScoreComponentCollection* >::const_iterator i =
|
std::map< std::string, ScoreComponentCollection* >::const_iterator i =
|
||||||
m_weightSetting.find( settingName );
|
m_weightSetting.find( settingName );
|
||||||
|
|
||||||
@ -747,11 +702,11 @@ public:
|
|||||||
<< " does not exist in model, using default weight setting instead";
|
<< " does not exist in model, using default weight setting instead";
|
||||||
UserMessage::Add(strme.str());
|
UserMessage::Add(strme.str());
|
||||||
i = m_weightSetting.find( "default" );
|
i = m_weightSetting.find( "default" );
|
||||||
SetCurrentWeightSetting("default");
|
m_currentWeightSetting = "default";
|
||||||
}
|
}
|
||||||
|
|
||||||
// set weights
|
// set weights
|
||||||
AccessAllWeights() = *(i->second);
|
m_allWeights = *(i->second);
|
||||||
}
|
}
|
||||||
|
|
||||||
float GetWeightWordPenalty() const;
|
float GetWeightWordPenalty() const;
|
||||||
|
Loading…
Reference in New Issue
Block a user