diff --git a/mira/Main.cpp b/mira/Main.cpp index e3678ff44..666960f7e 100644 --- a/mira/Main.cpp +++ b/mira/Main.cpp @@ -388,6 +388,7 @@ int main(int argc, char** argv) { // initialise Moses // add initial Bleu weight and references to initialize Bleu feature + boost::trim(decoder_settings); decoder_settings += " -weight-bl 1 -references"; if (trainWithMultipleFolds) { decoder_settings += " "; diff --git a/mira/training-expt.perl b/mira/training-expt.perl index bcdefebb7..9a759d55b 100755 --- a/mira/training-expt.perl +++ b/mira/training-expt.perl @@ -393,26 +393,22 @@ sub createTestScriptAndSubmit { my $skip_submit = $_[8]; #file names - my $job_name = $name."_".$testtype."_".$train_iteration.$suffix; - - my $test_script = "$name-$testtype"; - my $test_script_file = $working_dir."/".$test_script.".$train_iteration".$suffix.".sh"; - my $test_out = $test_script . ".$train_iteration" . $suffix . ".out"; - my $test_err = $test_script . ".$train_iteration" . $suffix . ".err"; - my $output_file; my $output_error_file; my $bleu_file; + my $file_id = ""; if ($weight_dump_frequency == 1) { if ($train_iteration < 10) { $output_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".out"; $output_error_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".err"; $bleu_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".bleu"; + $file_id = "0".$train_iteration.$suffix; } else { $output_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".out"; $output_error_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".err"; $bleu_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".bleu"; + $file_id = $train_iteration.$suffix; } } else { @@ -420,14 +416,23 @@ sub createTestScriptAndSubmit { $output_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".out"; $output_error_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".err"; $bleu_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu"; + $file_id = "0".$epoch."_".$epoch_slice.$suffix; } else { $output_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".out"; $output_error_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".err"; $bleu_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu"; + $file_id = $epoch."_".$epoch_slice.$suffix; } } + my $job_name = $name."_".$testtype."_".$file_id; + + my $test_script = "$name-$testtype"; + my $test_script_file = "$working_dir/$test_script.$file_id.sh"; + my $test_out = "$test_script.$file_id.out"; + my $test_err = "$test_script.$file_id.err"; + if (! (open TEST, ">$test_script_file" )) { die "Unable to create test script $test_script_file\n"; } @@ -445,6 +450,7 @@ sub createTestScriptAndSubmit { my $readCoreWeights = 0; my $readExtraWeights = 0; my %extra_weights; + my $abs_weights = 0; while() { chomp; my ($name,$value) = split; @@ -455,22 +461,28 @@ sub createTestScriptAndSubmit { } else { if ($name eq "WordPenalty") { $wordpenalty_weight = $value; + $abs_weights += abs($value); $readCoreWeights += 1; } elsif ($name =~ /^PhraseModel/) { push @phrasemodel_weights,$value; + $abs_weights += abs($value); $readCoreWeights += 1; } elsif ($name =~ /^LM\:2/) { $lm2_weight = $value; + $abs_weights += abs($value); $readCoreWeights += 1; } elsif ($name =~ /^LM/) { $lm_weight = $value; + $abs_weights += abs($value); $readCoreWeights += 1; } elsif ($name eq "Distortion") { $distortion_weight = $value; + $abs_weights += abs($value); $readCoreWeights += 1; } elsif ($name =~ /^LexicalReordering/) { push @lexicalreordering_weights,$value; + $abs_weights += abs($value); $readCoreWeights += 1; } else { $extra_weights{$name} = $value; @@ -480,60 +492,65 @@ sub createTestScriptAndSubmit { } close WEIGHTS; - if (!defined $core_weight_file) { - print "Number of core weights read: ".$readCoreWeights."\n"; - } + print "Number of core weights read: ".$readCoreWeights."\n"; print "Number of extra weights read: ".$readExtraWeights."\n"; # If there is a core weight file, we have to load the core weights from that file (NOTE: this is not necessary if the core weights are also printed to the weights file) - if (defined $core_weight_file) { - @phrasemodel_weights = (); - @lexicalreordering_weights = (); - $readCoreWeights = 0; - if (! (open CORE_WEIGHTS, "$core_weight_file")) { - die "Unable to open core weights file $core_weight_file\n"; - } - print "Reading core weights from file..\n"; - while() { - chomp; - my ($name,$value) = split; - next if ($name =~ /^!Unknown/); - next if ($name =~ /^BleuScore/); - if ($name eq "DEFAULT_") { - $default_weight = $value; - } - else { - if ($name eq "WordPenalty") { - $wordpenalty_weight = $value; - $readCoreWeights += 1; - } elsif ($name =~ /^PhraseModel/) { - push @phrasemodel_weights,$value; - $readCoreWeights += 1; - } elsif ($name =~ /^LM\:2/) { - $lm2_weight = $value; - $readCoreWeights += 1; - } - elsif ($name =~ /^LM/) { - $lm_weight = $value; - $readCoreWeights += 1; - } elsif ($name eq "Distortion") { - $distortion_weight = $value; - $readCoreWeights += 1; - } elsif ($name =~ /^LexicalReordering/) { - push @lexicalreordering_weights,$value; - $readCoreWeights += 1; - } else { - # there should be no extra weights in the core weights file - print "weight not matched: $name:$value\n"; - } - } - } - close CORE_WEIGHTS; - print "Number of core weights read: ".$readCoreWeights."\n"; - } +# if (defined $core_weight_file) { +# @phrasemodel_weights = (); +# @lexicalreordering_weights = (); +# $readCoreWeights = 0; +# if (! (open CORE_WEIGHTS, "$core_weight_file")) { +# die "Unable to open core weights file $core_weight_file\n"; +# } +# print "Reading core weights from file..\n"; +# while() { +# chomp; +# my ($name,$value) = split; +# next if ($name =~ /^!Unknown/); +# next if ($name =~ /^BleuScore/); +# if ($name eq "DEFAULT_") { +# $default_weight = $value; +# } +# else { +# if ($name eq "WordPenalty") { +# $wordpenalty_weight = $value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } elsif ($name =~ /^PhraseModel/) { +# push @phrasemodel_weights,$value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } elsif ($name =~ /^LM\:2/) { +# $lm2_weight = $value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } +# elsif ($name =~ /^LM/) { +# $lm_weight = $value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } elsif ($name eq "Distortion") { +# $distortion_weight = $value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } elsif ($name =~ /^LexicalReordering/) { +# push @lexicalreordering_weights,$value; +# $abs_weights += abs($value); +# $readCoreWeights += 1; +# } else { +# # there should be no extra weights in the core weights file +# print "weight not matched: $name:$value\n"; +# } +# } +# } +# close CORE_WEIGHTS; +# print "Number of core weights read: ".$readCoreWeights."\n"; +# } - # Create new ini file - my $new_ini_file = $working_dir."/".$test_script.".".$train_iteration.$suffix.".ini"; + # Create new ini file (changing format: expt1-devtest.00_2.ini instead of expt1-devtest.3.ini) + # my $new_ini_file = $working_dir."/".$test_script.".".$train_iteration.$suffix.".ini"; + my $new_ini_file = "$working_dir/$test_script.$file_id.ini"; if (! (open NEWINI, ">$new_ini_file" )) { die "Unable to create ini file $new_ini_file\n"; } @@ -541,15 +558,16 @@ sub createTestScriptAndSubmit { die "Unable to read ini file $old_ini_file\n"; } + # write normalized weights to ini file while() { if (/weight-l/) { print NEWINI "[weight-l]\n"; - print NEWINI $lm_weight; + print NEWINI ($lm_weight/$abs_weights); print NEWINI "\n"; if (defined $lm2_weight) { readline(OLDINI); - print NEWINI $lm2_weight; + print NEWINI ($lm2_weight/$abs_weights); print NEWINI "\n"; } @@ -557,38 +575,37 @@ sub createTestScriptAndSubmit { } elsif (/weight-t/) { print NEWINI "[weight-t]\n"; foreach my $phrasemodel_weight (@phrasemodel_weights) { - print NEWINI $phrasemodel_weight; + print NEWINI ($phrasemodel_weight/$abs_weights); print NEWINI "\n"; readline(OLDINI); } } elsif (/weight-d/) { print NEWINI "[weight-d]\n"; - print NEWINI $distortion_weight; + print NEWINI ($distortion_weight/$abs_weights); print NEWINI "\n"; readline(OLDINI); foreach my $lexicalreordering_weight (@lexicalreordering_weights) { - print NEWINI $lexicalreordering_weight; + print NEWINI ($lexicalreordering_weight/$abs_weights); print NEWINI "\n"; readline(OLDINI); } } elsif (/weight-w/) { print NEWINI "[weight-w]\n"; - print NEWINI $wordpenalty_weight; + print NEWINI ($wordpenalty_weight/$abs_weights); print NEWINI "\n"; readline(OLDINI); } else { print NEWINI; } } - close NEWINI; close OLDINI; my $writtenExtraWeights = 0; - # if there are any non-core weights, write them to a weights file + # if there are any non-core weights, write them to a weights file (normalized) my $extra_weight_file = undef; if (%extra_weights) { - $extra_weight_file = "$new_weight_file.scaled"; + $extra_weight_file = "$new_weight_file.sparse.scaled"; if (! (open EXTRAWEIGHT,">$extra_weight_file")) { print "Warning: unable to create extra weights file $extra_weight_file"; next; @@ -601,7 +618,7 @@ sub createTestScriptAndSubmit { foreach my $name (sort keys %extra_weights) { next if ($name eq "core"); next if ($name eq "DEFAULT_"); - my $value = $extra_weights{$name}; + my $value = $extra_weights{$name}/$abs_weights; # write only non-zero feature weights to file if ($value) { @@ -611,6 +628,11 @@ sub createTestScriptAndSubmit { } } } + + # add specification of sparse weight file to ini + print NEWINI "\n[weight-file] \n"; + print NEWINI "$extra_weight_file \n"; + close NEWINI; print TEST "#!/bin/sh\n"; print TEST "#\$ -N $job_name\n"; @@ -626,9 +648,10 @@ sub createTestScriptAndSubmit { print TEST "export LD_LIBRARY_PATH=/exports/informatics/inf_iccs_smt/shared/boost/lib:\$LD_LIBRARY_PATH\n"; } print TEST "$test_exe $decoder_settings -i $input_file -f $new_ini_file "; - if ($extra_weight_file) { - print TEST "-weight-file $extra_weight_file "; - } +# now written to ini file +# if ($extra_weight_file) { +# print TEST "-weight-file $extra_weight_file "; +# } print TEST $extra_args; print TEST " 1> $output_file 2> $output_error_file\n"; print TEST "echo \"Decoding of ".$testtype." set finished.\"\n"; diff --git a/moses-chart-cmd/src/Main.cpp b/moses-chart-cmd/src/Main.cpp index f41bdb701..d6d603c1b 100644 --- a/moses-chart-cmd/src/Main.cpp +++ b/moses-chart-cmd/src/Main.cpp @@ -89,6 +89,7 @@ public: VERBOSE(2,"\nTRANSLATING(" << lineNumber << "): " << *m_source); + if ((*m_source).GetSize() == 0) return; ChartManager manager(*m_source, &system); manager.ProcessSentence(); diff --git a/moses-cmd/src/Main.cpp b/moses-cmd/src/Main.cpp index 0eeaa74e5..44eca6535 100644 --- a/moses-cmd/src/Main.cpp +++ b/moses-cmd/src/Main.cpp @@ -102,6 +102,7 @@ public: // execute the translation // note: this executes the search, resulting in a search graph // we still need to apply the decision rule (MAP, MBR, ...) + if ((*m_source).GetSize() == 0) return; Manager manager(*m_source,staticData.GetSearchAlgorithm(), &system); manager.ProcessSentence(); diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index c68145b15..4944286f3 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -523,11 +523,18 @@ filter in: TRAINING:config input out: filtered-config default-name: tuning/moses.filtered.ini - rerun-on-change: filter-settings - pass-if: TRAINING:binarize-all + rerun-on-change: filter-settings + pass-if: TRAINING:binarize-all ignore-if: use-hiero +filter-devtest + in: TRAINING:config input-devtest + out: filtered-config-devtest + default-name: tuning/moses.filtered.devtest.ini + rerun-on-change: filter-settings + pass-if: TRAINING:binarize-all + ignore-if: use-hiero tune - in: filtered-config input reference + in: filtered-config input reference filtered-config-devtest input-devtest reference-devtest out: weight-config ignore-if: use-hiero qsub-script: yes diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index c95db9623..1443fbe74 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -947,6 +947,9 @@ sub define_step { elsif ($DO_STEP[$i] eq 'TUNING:filter') { &define_tuningevaluation_filter(undef,$i); } + elsif ($DO_STEP[$i] eq 'TUNING:filter-devtest') { + &define_tuningevaluation_filter(undef,$i,"devtest"); + } elsif ($DO_STEP[$i] eq 'TUNING:tune') { &define_tuning_tune($i); } @@ -1543,9 +1546,10 @@ sub factorize_one_language { sub define_tuning_tune { my ($step_id) = @_; my $dir = &check_and_get("GENERAL:working-dir"); + + # the last variable only apply for mira tuning (devtest input and reference are read out later) + my ($tuned_config,$config,$input,$reference,$config_devtest) = &get_output_and_input($step_id); - my ($tuned_config, - $config,$input,$reference) = &get_output_and_input($step_id); my $tuning_script = &check_and_get("TUNING:tuning-script"); my $scripts = &check_backoff_and_get("TUNING:moses-script-dir"); my $nbest_size = &check_and_get("TUNING:nbest"); @@ -1562,23 +1566,188 @@ sub define_tuning_tune { my $tuning_settings = &backoff_and_get("TUNING:tuning-settings"); $tuning_settings = "" unless $tuning_settings; - my $cmd = "$tuning_script $input $reference $decoder $config --nbest $nbest_size --working-dir $dir/tuning/tmp.$VERSION --decoder-flags \"$decoder_settings\" --rootdir $scripts $tuning_settings --no-filter-phrase-table"; - $cmd .= " --lambdas \"$lambda\"" if $lambda; - $cmd .= " --continue" if $tune_continue; - $cmd .= " --inputtype $tune_inputtype" if $tune_inputtype; + my $use_mira = &check_and_get("TUNING:use-mira"); + my $cmd = ""; + if ($use_mira eq "true") { + my $experiment_dir = "$dir/tuning/tmp.$VERSION"; + system("mkdir -p $experiment_dir"); - my $qsub_args = &get_qsub_args("TUNING"); - $cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args); - $cmd .= " --jobs $jobs" if $CLUSTER && $jobs; + my $mira_config = "$experiment_dir/mira-config.$VERSION."; + my $mira_config_log = $mira_config."log"; + $mira_config .= "cfg"; + + write_mira_config($mira_config, $experiment_dir, $config, $config_devtest); + $cmd = "$tuning_script -config $mira_config -exec >& $mira_config_log"; + + # write script to select the best set of weights after training for the specified number of epochs --> + # cp to tuning/tmp.?/moses.ini + my $script_filename = "$experiment_dir/selectBestWeights."; + my $script_filename_log = $script_filename."log"; + $script_filename .= "perl"; + my $weight_output_file = "$experiment_dir/moses.ini"; + write_selectBestMiraWeights($experiment_dir, $script_filename, $weight_output_file); + $cmd .= "\n$script_filename >& $script_filename_log"; + } + else { + $cmd = "$tuning_script $input $reference $decoder $config --nbest $nbest_size --working-dir $dir/tuning/tmp.$VERSION --decoder-flags \"$decoder_settings\" --rootdir $scripts $tuning_settings --no-filter-phrase-table"; + $cmd .= " --lambdas \"$lambda\"" if $lambda; + $cmd .= " --continue" if $tune_continue; + $cmd .= " --inputtype $tune_inputtype" if $tune_inputtype; + + my $qsub_args = &get_qsub_args("TUNING"); + $cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args); + $cmd .= " --jobs $jobs" if $CLUSTER && $jobs; + my $tuning_dir = $tuned_config; + $tuning_dir =~ s/\/[^\/]+$//; + $cmd .= "\nmkdir -p $tuning_dir"; + } - my $tuning_dir = $tuned_config; - $tuning_dir =~ s/\/[^\/]+$//; - $cmd .= "\nmkdir -p $tuning_dir"; $cmd .= "\ncp $dir/tuning/tmp.$VERSION/moses.ini $tuned_config"; &create_step($step_id,$cmd); } +sub write_mira_config { + my ($config_filename, $expt_dir, $tune_filtered_ini, $devtest_filtered_ini) = @_; + + my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir"); + my $tuning_decoder_settings = &check_and_get("TUNING:decoder-settings"); + my $core_weights = &check_and_get("TUNING:core-weight-config"); + my $input = &check_and_get("TUNING:input"); + my $reference = &check_and_get("TUNING:reference"); + my $tuning_settings = &check_and_get("TUNING:tuning-settings"); + my @settings = split(/ /, $tuning_settings); + my $mira_tuning_settings = &check_and_get("TUNING:mira-tuning-settings"); + my $input_devtest = &check_and_get("TUNING:input-devtest"); + my $reference_devtest = &check_and_get("TUNING:reference-devtest"); + + # convert core weights into format expected by mira + my $core_file = "$expt_dir/core_weights"; + if ($core_weights) { + open(INI, $core_weights); + #print STDERR "Reading core weights from file $core_weights \n"; + open(CORE, ">$core_file"); + while() { + if (/weight-l/) { + my @lm_weights; + while () { + last if $_ eq "\n"; + push(@lm_weights, $_); + } + + print CORE "LM ".$lm_weights[0]; + for my $i (1 .. $#lm_weights) { + print CORE "LM:".($i+1)." ".$lm_weights[$i]; + } + + } elsif (/weight-t/) { + my @pm_weights; + while () { + last if $_ eq "\n"; + push(@pm_weights, $_); + } + for my $i (0 .. $#pm_weights) { + print CORE "PhraseModel_".($i+1)." ".$pm_weights[$i]; + } + } elsif (/weight-d/) { + my @d_weights; + while () { + last if $_ eq "\n"; + push(@d_weights, $_); + } + for my $i (0 .. $#d_weights) { + if ($i == 0) { + print CORE "Distortion ".$d_weights[0]; + } + else { + print CORE "LexicalReordering_wbe-msd-bidirectional-fe-allff_".($i+1)." ".$d_weights[$i]; + } + } + + } elsif (/weight-w/) { + my $w = ; + print CORE "WordPenalty ".$w; + } + } + close INI; + close CORE; + } + + # mira config file + open(CFG, ">$config_filename"); + print CFG "[general] \n"; + print CFG "name=expt \n"; + print CFG "fold=0 \n"; + print CFG "mpienv=openmpi_fillup_mark2 \n"; + print CFG "moses-home=".$moses_src_dir."\n"; + print CFG "working-dir=".$expt_dir."\n"; + print CFG "decoder-settings=".$tuning_decoder_settings."\n\n"; + + if ($core_weights) { + print CFG "[core] \n"; + print CFG "weightfile=".$core_file."\n\n"; + } + + print CFG "[train] \n"; + print CFG "trainer=\${moses-home}/dist/bin/mira \n"; + print CFG "input-file=".$input."\n"; + print CFG "reference-files=".$reference."\n"; + print CFG "moses-ini-file=".$tune_filtered_ini."\n"; + print CFG "hours=48 \n"; + foreach my $setting (@settings) { + print CFG $setting."\n"; + } + print CFG "extra-args=".$mira_tuning_settings."\n\n"; + + print CFG "[devtest] \n"; + if (&get("TRAINING:hierarchical-rule-set")) { + print CFG "moses=\${moses-home}/moses-chart-cmd/src/moses_chart \n"; + } + else { + print CFG "moses=\${moses-home}/moses-cmd/src/moses \n"; + } + # use multi-bleu to select the best set of weights + print CFG "bleu=\${moses-home}/scripts/generic/multi-bleu.perl \n"; + print CFG "input-file=".$input_devtest."\n"; + print CFG "reference-file=".$reference_devtest."\n"; + print CFG "moses-ini-file=".$devtest_filtered_ini."\n"; + print CFG "hours=12 \nextra-args= \nskip-dev=1 \nskip-devtest=0 \nskip-submit=0 \n"; + close(CFG); +} + +sub write_selectBestMiraWeights { + my ($expt_dir, $script_filename, $weight_out_file) = @_; + open(SCR, ">$script_filename"); + + print SCR "#!/usr/bin/perl -w \nuse strict; \n\n"; + print SCR "my \@devtest_bleu = glob(\"$expt_dir/*_devtest.bleu\"); \# expt_00_0_devtest.bleu \n"; + print SCR "if (scalar(\@devtest_bleu) == 0) { \n"; + print SCR "\tprint STDERR \"ERROR: no bleu files globbed, cannot find best weights.\\n\"; \n"; + print SCR "\texit(1); \n"; + print SCR "} \n\n"; + print SCR "my (\$best_weights, \$best_id); \n"; + print SCR "my \$best_bleu = -1; \n"; + print SCR "my \$best_ratio = 0; \n"; + print SCR "foreach my \$bleu_file (\@devtest_bleu) { \n"; + print SCR "\t\$bleu_file =~ /_([\\d_]+)_devtest.bleu/; \n"; + print SCR "\tmy \$id = \$1; \n"; + print SCR "\topen(BLEU, \$bleu_file); \n"; + print SCR "\tmy \$bleu = ; \n"; + print SCR "\t\$bleu =~ /BLEU = ([\\d\\.]+), .*ratio=([\\d\\.]+), /; \n"; + print SCR "\tif (\$1 > \$best_bleu || (\$1 == \$best_bleu && (abs(1-\$2) < abs(1-\$best_ratio)))) { \n"; + print SCR "\t\t\$best_bleu = \$1; \n"; + print SCR "\t\t\$best_ratio = \$2; \n"; + print SCR "\t\t# expt1-devtest.00_0.ini (incl. path to sparse weights) \n"; + print SCR "\t\t(\$best_weights) = glob(\"$expt_dir/*devtest.\$id.ini\"); \n"; + print SCR "\t} \n"; + print SCR "} \n\n"; + print SCR "print STDERR \"Best weights according to BLEU on devtest set: \$best_weights \\n\"; \n"; + print SCR "system(\"cp \$best_weights $weight_out_file\"); \n\n"; + + close(SCR); + system("chmod u+x $script_filename"); +} + sub define_training_prepare_data { my ($step_id) = @_; @@ -2128,7 +2297,7 @@ sub encode_factor_list { } sub define_tuningevaluation_filter { - my ($set,$step_id) = @_; + my ($set,$step_id, $type) = @_; my $scripts = &check_and_get("GENERAL:moses-script-dir"); my $dir = &check_and_get("GENERAL:working-dir"); my $tuning_flag = !defined($set); @@ -2148,7 +2317,14 @@ sub define_tuningevaluation_filter { $input_filter = &get("TUNING:input-filter") if $tuning_flag; $input_filter = $input unless $input_filter; - my $filter_dir = "$dir/tuning/filtered.$VERSION"; + my $filter_dir; + if ($type) { + $filter_dir = "$dir/tuning/filtered.$type.$VERSION"; + } + else { + $filter_dir = "$dir/tuning/filtered.$VERSION"; + } + $filter_dir = "$dir/evaluation/filtered.$set.$VERSION" unless $tuning_flag; my $settings = &backoff_and_get("EVALUATION:$set:filter-settings") unless $tuning_flag;