moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-11-13 10:40:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

only filter rule and reordering tables, if the tables changes, not just when weights or other settings changed

Browse Source
This commit is contained in:
phikoehn 2012-03-22 00:13:02 +00:00
parent 15c83daa21
commit 1794bccd90
3 changed files with 89 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scripts/ems/experiment.meta
View File

@ -520,12 +520,20 @@ split-reference
multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl
template: $output-splitter -model IN1.$output-extension < IN > OUT
filter
in: TRAINING:config input
in: input TRAINING:phrase-translation-table TRAINING:reordering-table
out: filtered-dir
default-name: tuning/filtered
rerun-on-change: filter-settings
pass-if: TRAINING:binarize-all
ignore-if: use-hiero
error: already exists. Please delete
apply-filter
in: TRAINING:config filtered-dir
out: filtered-config
default-name: tuning/moses.filtered.ini
rerun-on-change: filter-settings
pass-if: TRAINING:binarize-all
pass-if: TRAINING:binarize-all
ignore-if: use-hiero
template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT
tune
in: filtered-config input reference
out: weight-config
@ -616,12 +624,20 @@ split-input
pass-unless: input-splitter
template: $input-splitter -model IN1.$input-extension < IN > OUT
filter
in: TUNING:config-with-reused-weights input
in: input TRAINING:phrase-translation-table TRAINING:reordering-table
out: filtered-dir
default-name: evaluation/filtered
rerun-on-change: filter-settings report-precision-by-coverage
pass-if: TRAINING:binarize-all
ignore-if: use-hiero
error: already exists. Please delete
apply-filter
in: TUNING:config-with-reused-weights filtered-dir
out: filtered-config
default-name: evaluation/filtered.ini
pass-if: TRAINING:binarize-all
rerun-on-change: filter-settings report-precision-by-coverage
error: already exists. Please delete
ignore-if: use-hiero
template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT
decode
in: filtered-config input
out: system-output

28
scripts/ems/experiment.perl
View File

@ -2133,8 +2133,8 @@ sub define_tuningevaluation_filter {
my $dir = &check_and_get("GENERAL:working-dir");
my $tuning_flag = !defined($set);
my ($filter_config,
$config,$input) = &get_output_and_input($step_id);
my ($filter_dir,
$input,$phrase_translation_table,$reordering_table) = &get_output_and_input($step_id);
my $binarizer = &get("GENERAL:ttable-binarizer");
my $hierarchical = &get("TRAINING:hierarchical-rule-set");
@ -2148,9 +2148,7 @@ sub define_tuningevaluation_filter {
$input_filter = &get("TUNING:input-filter") if $tuning_flag;
$input_filter = $input unless $input_filter;
my $filter_dir = "$dir/tuning/filtered.$VERSION";
$filter_dir = "$dir/evaluation/filtered.$set.$VERSION" unless $tuning_flag;
# additional settings
my $settings = &backoff_and_get("EVALUATION:$set:filter-settings") unless $tuning_flag;
$settings = &get("TUNING:filter-settings") if $tuning_flag;
$settings = "" unless $settings;
@ -2160,12 +2158,22 @@ sub define_tuningevaluation_filter {
$settings .= " -Binarizer \"$binarizer\"" if $binarizer;
$settings .= " --Hierarchical" if &get("TRAINING:hierarchical-rule-set");
my $cmd = "$scripts/training/filter-model-given-input.pl";
$cmd .= " $filter_dir $config $input_filter $settings";
# create pseudo-config file
my $config = "$dir/tuning/moses.table.ini.$VERSION";
my $cmd = &get_training_setting(9);
$cmd .= &get_table_name_settings("translation-factors","phrase-translation-table",$phrase_translation_table);
$cmd .= &get_table_name_settings("reordering-factors","reordering-table",$reordering_table)
if $reordering_table;
$cmd .= "-lm 0:3:$dir "; # dummy
$cmd .= "-config $config\n";
# filter command
$cmd .= "$scripts/training/filter-model-given-input.pl";
$cmd .= " $filter_dir $config $input_filter $settings\n";
# clean-up
$cmd .= "rm $config";
# copy moses.ini into specified file location
$cmd .= "\ncp $filter_dir/moses.ini $filter_config\n";
&create_step($step_id,$cmd);
}

49
scripts/ems/support/substitute-filtered-tables.perl Executable file
View File

@ -0,0 +1,49 @@
#!/usr/bin/perl -w
# experiment.perl support script
# get filtered rule and reordering tables and place them into a configuration file
if (scalar @ARGV < 1 || ! -e $ARGV[0]) {
die("ERROR: could not find pseudo-config with filtered tables");
}
# read config sections about filtered tables
my %CONFIG;
my $status = 0;
my $section;
open(FILTERED,$ARGV[0]);
while(<FILTERED>) {
if (/^\[(.+)\]\s*$/) {
if ($1 eq "ttable-file" || $1 eq "distortion-file") {
$section = $1;
$status = 1;
print STDERR "found $section\n";
}
else {
$status = 0;
}
}
elsif ($status) {
$CONFIG{$section} .= $_;
}
}
close(FILTERED);
# pass through master config file and replace table sections
($status,$section) = (0);
while(<STDIN>) {
if (/^\[(.+)\]\s*$/) {
print $_;
if ($1 eq "ttable-file" || $1 eq "distortion-file") {
print STDERR "replacing $1\n";
print $CONFIG{$1};
$status = 1;
}
else {
$status = 0;
}
}
elsif (!$status) {
print $_;
}
}