mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
fixed regression test failing. Number of features for generation models MUST be specified in ini file, no backward compatability hack
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@1209 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972
parent
6b4dfc4db2
commit
e247f1da6f
@ -61,7 +61,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" "$(SolutionDir)\$(ConfigurationName)\irstlm.lib" zlib.lib"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" zlib.lib"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
@ -136,7 +136,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" "$(SolutionDir)\$(ConfigurationName)\irstlm.lib" zlib.lib"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" zlib.lib"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" "$(SolutionDir)\$(ConfigurationName)\irstlm.lib" zlib.lib"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" zlib.lib"
|
||||
OutputFile="$(ProjectDir)$(ConfigurationName)\$(ProjectName).exe"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
@ -137,7 +137,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" "$(SolutionDir)\$(ConfigurationName)\irstlm.lib" zlib.lib"
|
||||
AdditionalDependencies=""$(SolutionDir)\$(ConfigurationName)\moses.lib" zlib.lib"
|
||||
OutputFile="$(ProjectDir)$(ConfigurationName)\$(ProjectName).exe"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
|
||||
@ -41,16 +41,10 @@ bool GenerationDictionary::Load(const std::vector<FactorType> &input
|
||||
, const std::vector<FactorType> &output
|
||||
, FactorCollection &factorCollection
|
||||
, const std::string &filePath
|
||||
, FactorDirection direction
|
||||
, bool forceSingleFeatureValue)
|
||||
, FactorDirection direction)
|
||||
{
|
||||
const size_t numFeatureValuesInConfig = this->GetNumScoreComponents();
|
||||
|
||||
// old hack - originally, moses assumed single generation values
|
||||
if (forceSingleFeatureValue) {
|
||||
assert(numFeatureValuesInConfig == 1);
|
||||
}
|
||||
|
||||
//factors
|
||||
m_inputFactors = FactorMask(input);
|
||||
m_outputFactors = FactorMask(output);
|
||||
@ -96,7 +90,6 @@ bool GenerationDictionary::Load(const std::vector<FactorType> &input
|
||||
}
|
||||
|
||||
size_t numFeaturesInFile = token.size() - 2;
|
||||
if (forceSingleFeatureValue) numFeaturesInFile = 1;
|
||||
if (numFeaturesInFile < numFeatureValuesInConfig) {
|
||||
stringstream strme;
|
||||
strme << filePath << ":" << lineNum << ": expected " << numFeatureValuesInConfig
|
||||
|
||||
@ -64,8 +64,7 @@ public:
|
||||
, const std::vector<FactorType> &output
|
||||
, FactorCollection &factorCollection
|
||||
, const std::string &filePath
|
||||
, FactorDirection direction
|
||||
, bool forceSingleFeatureValue);
|
||||
, FactorDirection direction);
|
||||
|
||||
size_t GetNumScoreComponents() const;
|
||||
const std::string GetScoreProducerDescription(int idx = 0) const;
|
||||
|
||||
@ -250,11 +250,11 @@ bool Parameter::Validate()
|
||||
{
|
||||
noErrorFlag = FileExists(m_setting["input-file"][0]);
|
||||
}
|
||||
|
||||
// checks on non-mandatory tables:
|
||||
if (noErrorFlag && m_setting["generation-file"].size())
|
||||
noErrorFlag = FilesExist("generation-file", 3);
|
||||
if (noErrorFlag && m_setting["distortion-file"].size())
|
||||
// generation tables
|
||||
if (noErrorFlag)
|
||||
noErrorFlag = FilesExist("generation-file", 3);
|
||||
// distortion
|
||||
if (noErrorFlag)
|
||||
noErrorFlag = FilesExist("distortion-file", 3);
|
||||
|
||||
return noErrorFlag;
|
||||
@ -278,8 +278,8 @@ bool Parameter::FilesExist(const string ¶mName, size_t tokenizeIndex,std::ve
|
||||
if (tokenizeIndex >= vec.size())
|
||||
{
|
||||
stringstream errorMsg("");
|
||||
errorMsg << "Expected " << tokenizeIndex << " tokens per"
|
||||
<< " entry in '" << paramName << "', but only found "
|
||||
errorMsg << "Expected at least " << (tokenizeIndex+1) << " tokens per emtry in '"
|
||||
<< paramName << "', but only found "
|
||||
<< vec.size();
|
||||
UserMessage::Add(errorMsg.str());
|
||||
return false;
|
||||
|
||||
@ -482,37 +482,20 @@ bool StaticData::LoadGenerationTables()
|
||||
for(size_t currDict = 0 ; currDict < generationVector.size(); currDict++)
|
||||
{
|
||||
vector<string> token = Tokenize(generationVector[currDict]);
|
||||
bool oldFormat = (token.size() == 3);
|
||||
vector<FactorType> input = Tokenize<FactorType>(token[0], ",")
|
||||
,output = Tokenize<FactorType>(token[1], ",");
|
||||
m_maxFactorIdx[1] = CalcMax(m_maxFactorIdx[1], input, output);
|
||||
string filePath;
|
||||
size_t numFeatures = 1;
|
||||
if (oldFormat)
|
||||
filePath = token[2];
|
||||
else {
|
||||
numFeatures = Scan<size_t>(token[2]);
|
||||
filePath = token[3];
|
||||
}
|
||||
size_t numFeatures;
|
||||
|
||||
numFeatures = Scan<size_t>(token[2]);
|
||||
filePath = token[3];
|
||||
|
||||
if (!FileExists(filePath) && FileExists(filePath + ".gz")) {
|
||||
filePath = filePath + ".gz";
|
||||
}
|
||||
if (!FileExists(filePath))
|
||||
{
|
||||
stringstream strme;
|
||||
strme << "Generation dictionary '"<<filePath<<"' does not exist!\n";
|
||||
UserMessage::Add(strme.str());
|
||||
return false;
|
||||
filePath += ".gz";
|
||||
}
|
||||
|
||||
TRACE_ERR( filePath << endl);
|
||||
if (oldFormat) {
|
||||
TRACE_ERR( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||||
" [WARNING] config file contains old style generation config format.\n"
|
||||
" Only the first feature value will be read. Please use the 4-format\n"
|
||||
" form (similar to the phrase table spec) to specify the # of features.\n"
|
||||
"!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");
|
||||
}
|
||||
|
||||
m_generationDictionary.push_back(new GenerationDictionary(numFeatures));
|
||||
assert(m_generationDictionary.back() && "could not create GenerationDictionary");
|
||||
@ -520,8 +503,7 @@ bool StaticData::LoadGenerationTables()
|
||||
, output
|
||||
, m_factorCollection
|
||||
, filePath
|
||||
, Output // always target, should we allow source?
|
||||
, oldFormat))
|
||||
, Output))
|
||||
{
|
||||
delete m_generationDictionary.back();
|
||||
return false;
|
||||
|
||||
@ -20,7 +20,7 @@ T 1
|
||||
|
||||
# generation models: source-factors, target-factors
|
||||
[generation-file]
|
||||
0 1 ${MODEL_PATH}/multi-factor-drop/generation.0-1.gz
|
||||
0 1 2 ${MODEL_PATH}/multi-factor-drop/generation.0-1.gz
|
||||
|
||||
# language models: 0, factors, type, file
|
||||
[lmodel-file]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user