Add a show-weights option. It prints out the moses features and exits. May

load tables as a side-effect.


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3744 1f5c12ca-751b-0410-a591-d2e778427230

moses-cmd/src/Main.cpp

@@ -57,10 +57,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 using namespace std;
 using namespace Moses;
 
+static const size_t PRECISION = 3;
+
 /** Enforce rounding */
-void fix(std::ostream& stream) {
+void fix(std::ostream& stream, size_t size) {
     stream.setf(std::ios::fixed);
-    stream.precision(3);
+    stream.precision(size);
 }
 
 
@@ -145,7 +147,7 @@ class TranslationTask : public Task {
             //Word Graph
             if (m_wordGraphCollector) {
                 ostringstream out;
-                fix(out);
+                fix(out,PRECISION);
                 manager.GetWordGraph(m_lineNumber, out);
                 m_wordGraphCollector->Write(m_lineNumber, out.str());
             }
@@ -153,7 +155,7 @@ class TranslationTask : public Task {
             //Search Graph
             if (m_searchGraphCollector) {
                 ostringstream out;
-                fix(out);
+                fix(out,PRECISION);
                 manager.OutputSearchGraph(m_lineNumber, out);
                 m_searchGraphCollector->Write(m_lineNumber, out.str());
 
@@ -174,7 +176,7 @@ class TranslationTask : public Task {
             if (m_outputCollector) {
                 ostringstream out;
                 ostringstream debug;
-                fix(debug);
+                fix(debug,PRECISION);
                 
                 //All derivations - send them to debug stream
                 if (staticData.PrintAllDerivations()) {
@@ -267,7 +269,7 @@ class TranslationTask : public Task {
             //detailed translation reporting
             if (m_detailedTranslationCollector) {
 								ostringstream out;
-                fix(out);
+                fix(out,PRECISION);
                 TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
                 m_detailedTranslationCollector->Write(m_lineNumber,out.str());
             }
@@ -291,6 +293,39 @@ class TranslationTask : public Task {
 
 };
 
+static void PrintFeatureWeight(const FeatureFunction* ff) {
+  
+  size_t weightStart  = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID());
+  size_t weightEnd  = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID());
+  for (size_t i = weightStart; i < weightEnd; ++i) {
+    cout << ff->GetScoreProducerDescription() <<  " " << ff->GetScoreProducerWeightShortName() << " " 
+        << StaticData::Instance().GetAllWeights()[i] << endl;
+  }
+}
+
+
+static void ShowWeights() {
+  fix(cout,6);
+  const StaticData& staticData = StaticData::Instance();
+  const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
+  const vector<const StatelessFeatureFunction*>& slf =system.GetStatelessFeatureFunctions();
+  const vector<const StatefulFeatureFunction*>& sff = system.GetStatefulFeatureFunctions();
+  const vector<PhraseDictionaryFeature*>& pds = system.GetPhraseDictionaries();
+  const vector<GenerationDictionary*>& gds = system.GetGenerationDictionaries();
+  for (size_t i = 0; i < sff.size(); ++i) {
+    PrintFeatureWeight(sff[i]);
+  }
+  for (size_t i = 0; i < slf.size(); ++i) {
+    PrintFeatureWeight(slf[i]);
+  }
+  for (size_t i = 0; i < pds.size(); ++i) {
+    PrintFeatureWeight(pds[i]);
+  }
+  for (size_t i = 0; i < gds.size(); ++i) {
+    PrintFeatureWeight(gds[i]);
+  }
+}
+
 int main(int argc, char** argv) {
     
 #ifdef HAVE_PROTOBUF
@@ -303,8 +338,8 @@ int main(int argc, char** argv) {
         TRACE_ERR(endl);
     }
 
-    fix(cout);
-    fix(cerr);
+    fix(cout,PRECISION);
+    fix(cerr,PRECISION);
 
 
     Parameter* params = new Parameter();
@@ -334,6 +369,11 @@ int main(int argc, char** argv) {
     if (!StaticData::LoadDataStatic(params)) {
         exit(1);
     }
+    
+    if (params->isParamSpecified("show-weights")) {
+      ShowWeights();
+      exit(0);
+    }
 
     const StaticData& staticData = StaticData::Instance();
     // set up read/writing class

moses/src/Parameter.cpp

@@ -132,6 +132,7 @@ Parameter::Parameter()
 	AddParam("output-hypo-score", "Output the hypo score to stdout with the output string. For search error analysis. Default is false");
 	AddParam("unknown-lhs", "file containing target lhs of unknown words. 1 per line: LHS prob");
     AddParam("translation-systems", "specify multiple translation systems, each consisting of an id, followed by a set of models ids, eg '0 T1 R1 L0'");
+    AddParam("show-weights", "print feature weights and exit");
 }
 
 Parameter::~Parameter()

regression-testing/run-test-suite.pl

@@ -27,6 +27,8 @@ my @tests = qw (
   phrase.lexicalized-reordering-cn
   phrase.consensus-decoding-surface
   phrase.continue-partial-translation
+  phrase.show-weights.lex-reorder
+  phrase.show-weights
 );
 #  xml-markup
 

regression-testing/tests/phrase.show-weights.lex-reorder/filter-stderr.pl

@@ -0,0 +1,9 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+}

regression-testing/tests/phrase.show-weights.lex-reorder/filter-stdout.pl

@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "WEIGHT_$x=$_\n";
+}

regression-testing/tests/phrase.show-weights.lex-reorder/moses.ini

@@ -0,0 +1,66 @@
+#########################
+### MOSES CONFIG FILE ###
+#########################
+
+# input factors
+[input-factors]
+0
+
+# mapping steps
+[mapping]
+T 0
+
+# translation tables: source-factors, target-factors, number of scores, file 
+[ttable-file]
+0 0 0 5 ${MODEL_PATH}/lexicalized-reordering/phrase-table.0-0.gz
+
+# no generation models, no generation-file section
+
+# language models: type(srilm/irstlm), factors, order, file
+[lmodel-file]
+0 0 5 ${MODEL_PATH}/lexicalized-reordering/europarl.lm
+
+
+# limit on how many phrase translations e for each phrase f are loaded
+# 0 = all elements loaded
+[ttable-limit]
+20
+0
+# distortion (reordering) files
+[distortion-file]
+0-0 msd-bidirectional-fe 6 ${MODEL_PATH}/lexicalized-reordering/reordering-table.msd-bidirectional-fe.0.5.0-0.gz
+
+# distortion (reordering) weight
+[weight-d]
+0.5
+0.3
+0.3
+0.3
+0.3
+0.3
+0.3
+
+# language model weights
+[weight-l]
+0.5000
+
+
+# translation model weights
+[weight-t]
+0.2
+0.2
+0.2
+0.2
+0.2
+
+# no generation models, no weight-generation section
+
+# word penalty
+[weight-w]
+-1
+
+[distortion-limit]
+6
+
+[show-weights]
+1

regression-testing/tests/phrase.show-weights.lex-reorder/truth/results.txt

@@ -0,0 +1,15 @@
+WEIGHT_1 = Distortion d 0.500000
+WEIGHT_2 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_3 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_4 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_5 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_6 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_7 = LexicalReordering_msd-bidirectional-fe d 0.300000
+WEIGHT_8 = LM_5gram lm 0.500000
+WEIGHT_9 = WordPenalty w -1.000000
+WEIGHT_10 = PhraseModel tm 0.200000
+WEIGHT_11 = PhraseModel tm 0.200000
+WEIGHT_12 = PhraseModel tm 0.200000
+WEIGHT_13 = PhraseModel tm 0.200000
+WEIGHT_14 = PhraseModel tm 0.200000
+TOTAL_WALLTIME ~ 28

regression-testing/tests/phrase.show-weights/filter-stderr.pl

@@ -0,0 +1,9 @@
+#!/usr/bin/perl
+
+BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
+use RegTestUtils;
+
+$x=0;
+while (<>) {
+  chomp;
+}

regression-testing/tests/phrase.show-weights/filter-stdout.pl

@@ -0,0 +1,7 @@
+#!/usr/bin/perl
+$x=0;
+while (<>) {
+  chomp;
+  $x++;
+  print "WEIGHT_$x=$_\n";
+}

regression-testing/tests/phrase.show-weights/moses.ini

@@ -0,0 +1,58 @@
+# Moses configuration file
+# automatic exodus from pharaoh.ini Wed Jul 12 18:24:14 EDT 2006
+
+###########################
+### PHARAOH CONFIG FILE ###
+###########################
+
+# phrase table f, n, p(n|f)
+[ttable-file]
+1 0 0 5 ${MODEL_PATH}/basic-surface-binptable/phrase-table.gz
+
+# language model
+[lmodel-file]
+0 0 3 ${LM_PATH}/europarl.en.srilm.gz
+
+# limit on how many phrase translations e for each phrase f are loaded
+[ttable-limit]
+#ttable element load limit 0 = all elements loaded
+20
+
+# distortion (reordering) weight
+[weight-d]
+0.141806519223522
+
+# language model weight
+[weight-l]
+0.142658800199951
+
+# translation model weight (phrase translation, lexical weighting)
+[weight-t]
+0.00402447059454402
+0.0685647475075862
+0.294089113124688
+0.0328320356515851
+-0.0426081987467227
+
+# word penalty
+[weight-w]
+-0.273416114951401
+
+[distortion-limit]
+4
+
+[beam-threshold]
+0.03
+
+[input-factors]
+0
+
+[mapping]
+T 0
+
+
+[verbose]
+2
+
+[show-weights]
+1

regression-testing/tests/phrase.show-weights/truth/results.txt

@@ -0,0 +1,9 @@
+WEIGHT_1 = Distortion d 0.141807
+WEIGHT_2 = LM_3gram lm 0.142659
+WEIGHT_3 = WordPenalty w -0.273416
+WEIGHT_4 = PhraseModel tm 0.004024
+WEIGHT_5 = PhraseModel tm 0.068565
+WEIGHT_6 = PhraseModel tm 0.294089
+WEIGHT_7 = PhraseModel tm 0.032832
+WEIGHT_8 = PhraseModel tm -0.042608
+TOTAL_WALLTIME ~ 28