mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Add a show-weights option. It prints out the moses features and exits. May
load tables as a side-effect. git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3744 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
ddabdf6b1b
commit
50f0e6c07d
@ -57,10 +57,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
|
||||
static const size_t PRECISION = 3;
|
||||
|
||||
/** Enforce rounding */
|
||||
void fix(std::ostream& stream) {
|
||||
void fix(std::ostream& stream, size_t size) {
|
||||
stream.setf(std::ios::fixed);
|
||||
stream.precision(3);
|
||||
stream.precision(size);
|
||||
}
|
||||
|
||||
|
||||
@ -145,7 +147,7 @@ class TranslationTask : public Task {
|
||||
//Word Graph
|
||||
if (m_wordGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out);
|
||||
fix(out,PRECISION);
|
||||
manager.GetWordGraph(m_lineNumber, out);
|
||||
m_wordGraphCollector->Write(m_lineNumber, out.str());
|
||||
}
|
||||
@ -153,7 +155,7 @@ class TranslationTask : public Task {
|
||||
//Search Graph
|
||||
if (m_searchGraphCollector) {
|
||||
ostringstream out;
|
||||
fix(out);
|
||||
fix(out,PRECISION);
|
||||
manager.OutputSearchGraph(m_lineNumber, out);
|
||||
m_searchGraphCollector->Write(m_lineNumber, out.str());
|
||||
|
||||
@ -174,7 +176,7 @@ class TranslationTask : public Task {
|
||||
if (m_outputCollector) {
|
||||
ostringstream out;
|
||||
ostringstream debug;
|
||||
fix(debug);
|
||||
fix(debug,PRECISION);
|
||||
|
||||
//All derivations - send them to debug stream
|
||||
if (staticData.PrintAllDerivations()) {
|
||||
@ -267,7 +269,7 @@ class TranslationTask : public Task {
|
||||
//detailed translation reporting
|
||||
if (m_detailedTranslationCollector) {
|
||||
ostringstream out;
|
||||
fix(out);
|
||||
fix(out,PRECISION);
|
||||
TranslationAnalysis::PrintTranslationAnalysis(manager.GetTranslationSystem(), out, manager.GetBestHypothesis());
|
||||
m_detailedTranslationCollector->Write(m_lineNumber,out.str());
|
||||
}
|
||||
@ -291,6 +293,39 @@ class TranslationTask : public Task {
|
||||
|
||||
};
|
||||
|
||||
static void PrintFeatureWeight(const FeatureFunction* ff) {
|
||||
|
||||
size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID());
|
||||
size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID());
|
||||
for (size_t i = weightStart; i < weightEnd; ++i) {
|
||||
cout << ff->GetScoreProducerDescription() << " " << ff->GetScoreProducerWeightShortName() << " "
|
||||
<< StaticData::Instance().GetAllWeights()[i] << endl;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void ShowWeights() {
|
||||
fix(cout,6);
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT);
|
||||
const vector<const StatelessFeatureFunction*>& slf =system.GetStatelessFeatureFunctions();
|
||||
const vector<const StatefulFeatureFunction*>& sff = system.GetStatefulFeatureFunctions();
|
||||
const vector<PhraseDictionaryFeature*>& pds = system.GetPhraseDictionaries();
|
||||
const vector<GenerationDictionary*>& gds = system.GetGenerationDictionaries();
|
||||
for (size_t i = 0; i < sff.size(); ++i) {
|
||||
PrintFeatureWeight(sff[i]);
|
||||
}
|
||||
for (size_t i = 0; i < slf.size(); ++i) {
|
||||
PrintFeatureWeight(slf[i]);
|
||||
}
|
||||
for (size_t i = 0; i < pds.size(); ++i) {
|
||||
PrintFeatureWeight(pds[i]);
|
||||
}
|
||||
for (size_t i = 0; i < gds.size(); ++i) {
|
||||
PrintFeatureWeight(gds[i]);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
#ifdef HAVE_PROTOBUF
|
||||
@ -303,8 +338,8 @@ int main(int argc, char** argv) {
|
||||
TRACE_ERR(endl);
|
||||
}
|
||||
|
||||
fix(cout);
|
||||
fix(cerr);
|
||||
fix(cout,PRECISION);
|
||||
fix(cerr,PRECISION);
|
||||
|
||||
|
||||
Parameter* params = new Parameter();
|
||||
@ -334,6 +369,11 @@ int main(int argc, char** argv) {
|
||||
if (!StaticData::LoadDataStatic(params)) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (params->isParamSpecified("show-weights")) {
|
||||
ShowWeights();
|
||||
exit(0);
|
||||
}
|
||||
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
// set up read/writing class
|
||||
|
@ -132,6 +132,7 @@ Parameter::Parameter()
|
||||
AddParam("output-hypo-score", "Output the hypo score to stdout with the output string. For search error analysis. Default is false");
|
||||
AddParam("unknown-lhs", "file containing target lhs of unknown words. 1 per line: LHS prob");
|
||||
AddParam("translation-systems", "specify multiple translation systems, each consisting of an id, followed by a set of models ids, eg '0 T1 R1 L0'");
|
||||
AddParam("show-weights", "print feature weights and exit");
|
||||
}
|
||||
|
||||
Parameter::~Parameter()
|
||||
|
@ -27,6 +27,8 @@ my @tests = qw (
|
||||
phrase.lexicalized-reordering-cn
|
||||
phrase.consensus-decoding-surface
|
||||
phrase.continue-partial-translation
|
||||
phrase.show-weights.lex-reorder
|
||||
phrase.show-weights
|
||||
);
|
||||
# xml-markup
|
||||
|
||||
|
@ -0,0 +1,9 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
|
||||
use RegTestUtils;
|
||||
|
||||
$x=0;
|
||||
while (<>) {
|
||||
chomp;
|
||||
}
|
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/perl
|
||||
$x=0;
|
||||
while (<>) {
|
||||
chomp;
|
||||
$x++;
|
||||
print "WEIGHT_$x=$_\n";
|
||||
}
|
@ -0,0 +1,66 @@
|
||||
#########################
|
||||
### MOSES CONFIG FILE ###
|
||||
#########################
|
||||
|
||||
# input factors
|
||||
[input-factors]
|
||||
0
|
||||
|
||||
# mapping steps
|
||||
[mapping]
|
||||
T 0
|
||||
|
||||
# translation tables: source-factors, target-factors, number of scores, file
|
||||
[ttable-file]
|
||||
0 0 0 5 ${MODEL_PATH}/lexicalized-reordering/phrase-table.0-0.gz
|
||||
|
||||
# no generation models, no generation-file section
|
||||
|
||||
# language models: type(srilm/irstlm), factors, order, file
|
||||
[lmodel-file]
|
||||
0 0 5 ${MODEL_PATH}/lexicalized-reordering/europarl.lm
|
||||
|
||||
|
||||
# limit on how many phrase translations e for each phrase f are loaded
|
||||
# 0 = all elements loaded
|
||||
[ttable-limit]
|
||||
20
|
||||
0
|
||||
# distortion (reordering) files
|
||||
[distortion-file]
|
||||
0-0 msd-bidirectional-fe 6 ${MODEL_PATH}/lexicalized-reordering/reordering-table.msd-bidirectional-fe.0.5.0-0.gz
|
||||
|
||||
# distortion (reordering) weight
|
||||
[weight-d]
|
||||
0.5
|
||||
0.3
|
||||
0.3
|
||||
0.3
|
||||
0.3
|
||||
0.3
|
||||
0.3
|
||||
|
||||
# language model weights
|
||||
[weight-l]
|
||||
0.5000
|
||||
|
||||
|
||||
# translation model weights
|
||||
[weight-t]
|
||||
0.2
|
||||
0.2
|
||||
0.2
|
||||
0.2
|
||||
0.2
|
||||
|
||||
# no generation models, no weight-generation section
|
||||
|
||||
# word penalty
|
||||
[weight-w]
|
||||
-1
|
||||
|
||||
[distortion-limit]
|
||||
6
|
||||
|
||||
[show-weights]
|
||||
1
|
@ -0,0 +1,15 @@
|
||||
WEIGHT_1 = Distortion d 0.500000
|
||||
WEIGHT_2 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_3 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_4 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_5 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_6 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_7 = LexicalReordering_msd-bidirectional-fe d 0.300000
|
||||
WEIGHT_8 = LM_5gram lm 0.500000
|
||||
WEIGHT_9 = WordPenalty w -1.000000
|
||||
WEIGHT_10 = PhraseModel tm 0.200000
|
||||
WEIGHT_11 = PhraseModel tm 0.200000
|
||||
WEIGHT_12 = PhraseModel tm 0.200000
|
||||
WEIGHT_13 = PhraseModel tm 0.200000
|
||||
WEIGHT_14 = PhraseModel tm 0.200000
|
||||
TOTAL_WALLTIME ~ 28
|
9
regression-testing/tests/phrase.show-weights/filter-stderr.pl
Executable file
9
regression-testing/tests/phrase.show-weights/filter-stderr.pl
Executable file
@ -0,0 +1,9 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; }
|
||||
use RegTestUtils;
|
||||
|
||||
$x=0;
|
||||
while (<>) {
|
||||
chomp;
|
||||
}
|
7
regression-testing/tests/phrase.show-weights/filter-stdout.pl
Executable file
7
regression-testing/tests/phrase.show-weights/filter-stdout.pl
Executable file
@ -0,0 +1,7 @@
|
||||
#!/usr/bin/perl
|
||||
$x=0;
|
||||
while (<>) {
|
||||
chomp;
|
||||
$x++;
|
||||
print "WEIGHT_$x=$_\n";
|
||||
}
|
58
regression-testing/tests/phrase.show-weights/moses.ini
Normal file
58
regression-testing/tests/phrase.show-weights/moses.ini
Normal file
@ -0,0 +1,58 @@
|
||||
# Moses configuration file
|
||||
# automatic exodus from pharaoh.ini Wed Jul 12 18:24:14 EDT 2006
|
||||
|
||||
###########################
|
||||
### PHARAOH CONFIG FILE ###
|
||||
###########################
|
||||
|
||||
# phrase table f, n, p(n|f)
|
||||
[ttable-file]
|
||||
1 0 0 5 ${MODEL_PATH}/basic-surface-binptable/phrase-table.gz
|
||||
|
||||
# language model
|
||||
[lmodel-file]
|
||||
0 0 3 ${LM_PATH}/europarl.en.srilm.gz
|
||||
|
||||
# limit on how many phrase translations e for each phrase f are loaded
|
||||
[ttable-limit]
|
||||
#ttable element load limit 0 = all elements loaded
|
||||
20
|
||||
|
||||
# distortion (reordering) weight
|
||||
[weight-d]
|
||||
0.141806519223522
|
||||
|
||||
# language model weight
|
||||
[weight-l]
|
||||
0.142658800199951
|
||||
|
||||
# translation model weight (phrase translation, lexical weighting)
|
||||
[weight-t]
|
||||
0.00402447059454402
|
||||
0.0685647475075862
|
||||
0.294089113124688
|
||||
0.0328320356515851
|
||||
-0.0426081987467227
|
||||
|
||||
# word penalty
|
||||
[weight-w]
|
||||
-0.273416114951401
|
||||
|
||||
[distortion-limit]
|
||||
4
|
||||
|
||||
[beam-threshold]
|
||||
0.03
|
||||
|
||||
[input-factors]
|
||||
0
|
||||
|
||||
[mapping]
|
||||
T 0
|
||||
|
||||
|
||||
[verbose]
|
||||
2
|
||||
|
||||
[show-weights]
|
||||
1
|
@ -0,0 +1,9 @@
|
||||
WEIGHT_1 = Distortion d 0.141807
|
||||
WEIGHT_2 = LM_3gram lm 0.142659
|
||||
WEIGHT_3 = WordPenalty w -0.273416
|
||||
WEIGHT_4 = PhraseModel tm 0.004024
|
||||
WEIGHT_5 = PhraseModel tm 0.068565
|
||||
WEIGHT_6 = PhraseModel tm 0.294089
|
||||
WEIGHT_7 = PhraseModel tm 0.032832
|
||||
WEIGHT_8 = PhraseModel tm -0.042608
|
||||
TOTAL_WALLTIME ~ 28
|
Loading…
Reference in New Issue
Block a user