gt rid of word penalty of 0.434. Adjust weight accordingly

moses/ChartManager.cpp

@@ -135,7 +135,7 @@ void ChartManager::AddXmlChartOptions() {
       i != xmlChartOptionsList.end(); ++i) {
     ChartTranslationOptions* opt = *i;
 
-    Moses::Scores wordPenaltyScore(1, -0.434294482); // TODO what is this number?
+    Moses::Scores wordPenaltyScore(1, -1);
     opt->GetTargetPhraseCollection().GetCollection()[0]->SetScore((FeatureFunction*)staticData.GetWordPenaltyProducer(), wordPenaltyScore);
 
     const WordsRange &range = opt->GetSourceWordsRange();

moses/ChartParser.cpp

@@ -44,7 +44,7 @@ void ChartParserUnknown::Process(const Word &sourceWord, const WordsRange &range
   // unknown word, add as trans opt
   const StaticData &staticData = StaticData::Instance();
   const UnknownWordPenaltyProducer *unknownWordPenaltyProducer = staticData.GetUnknownWordPenaltyProducer();
-  vector<float> wordPenaltyScore(1, -0.434294482); // TODO what is this number?
+  vector<float> wordPenaltyScore(1, -1);
   
   size_t isDigit = 0;
   if (staticData.GetDropUnknown()) {

moses/Parameter.cpp

@@ -752,6 +752,42 @@ void Parameter::ConvertWeightArgsGeneration(const std::string &oldWeightName, co
   m_setting.erase(oldFeatureName);
 }
 
+void Parameter::ConvertWeightArgsWordPenalty()
+{
+  const std::string oldWeightName = "weight-w";
+  const std::string newWeightName = "WordPenalty";
+
+  bool isChartDecoding = true;
+  if (!isParamSpecified("search-algorithm") ||
+	 (GetParam("search-algorithm").size() > 0
+	   && (Trim(GetParam("search-algorithm")[0]) == "0"
+		 ||Trim(GetParam("search-algorithm")[0]) == "1"
+		  )
+	 )
+   ) {
+    isChartDecoding = false;
+  }
+
+  PARAM_MAP::iterator iterMap;
+
+  iterMap = m_setting.find(oldWeightName);
+  if (iterMap != m_setting.end())
+  {
+    const PARAM_VEC &weights = iterMap->second;
+    for (size_t i = 0; i < weights.size(); ++i)
+    {
+      float weight = Scan<float>(weights[i]);
+      if (isChartDecoding) {
+        weight *= 0.434294482;
+      }
+      SetWeight(newWeightName, i, weight);
+    }
+
+    m_setting.erase(iterMap);
+  }
+
+}
+
 void Parameter::ConvertWeightArgs()
 {
   // can't handle discr LM. must do it manually 'cos of bigram/n-gram split
@@ -767,7 +803,7 @@ void Parameter::ConvertWeightArgs()
     cerr << "Do not mix old and new format for specify weights";
   }
 
-  ConvertWeightArgsSingleWeight("weight-w", "WordPenalty");
+  ConvertWeightArgsWordPenalty();
   ConvertWeightArgsLM("weight-l");
   ConvertWeightArgsSingleWeight("weight-slm", "SyntacticLM");
   ConvertWeightArgsSingleWeight("weight-u", "UnknownWordPenalty");

moses/Parameter.h

@@ -72,6 +72,7 @@ protected:
   void ConvertWeightArgsLM(const std::string &oldWeightName);
   void ConvertWeightArgsDistortion();
   void ConvertWeightArgsGeneration(const std::string &oldWeightName, const std::string &newWeightName);
+  void ConvertWeightArgsWordPenalty();
   void CreateWeightsMap();
   void WeightOverwrite();
   void AddFeature(const std::string &line);

moses/Phrase.cpp

@@ -185,7 +185,7 @@ void Phrase::CreateFromString(FactorDirection direction
     assert(lhs->IsNonTerminal());
   }
   else {
-    CHECK(lhs == NULL);
+    //CHECK(lhs == NULL);
 
     numWords = annotatedWordVector.size();
   }

moses/TargetPhrase.cpp

@@ -234,7 +234,7 @@ void TargetPhrase::SetScoreChart(const FeatureFunction* translationScoreProducer
 
   // word penalty
   size_t wordCount = GetNumTerminals();
-  m_scoreBreakdown.Assign(wpProducer, - (float) wordCount * 0.434294482); // TODO log -> ln ??
+  m_scoreBreakdown.Assign(wpProducer, - wordCount);
 
   m_fullScore = m_scoreBreakdown.GetWeightedScore() - totalNgramScore + totalFullScore + totalOOVScore;
 }